diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..84bed3ed --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "build/storage/spdk"] + path = build/storage/spdk + url = https://github.com/spdk/spdk.git diff --git a/build/scripts/run-shellcheck.sh b/build/scripts/run-shellcheck.sh index e25ac7af..586d836f 100755 --- a/build/scripts/run-shellcheck.sh +++ b/build/scripts/run-shellcheck.sh @@ -12,8 +12,14 @@ is_bash() { return 1 } +is_excluded_from_check() { + [[ $1 == */spdk/* ]] && return 0 + + return 1 +} + while IFS= read -r -d $'' file; do - if is_bash "$file"; then + if ! is_excluded_from_check "$file" && is_bash "$file" ; then shellcheck -x -W0 -s bash "$file" || continue fi done < <(find . -type f \! -path "./.git/*" -print0) diff --git a/build/storage/.dockerignore b/build/storage/.dockerignore new file mode 100644 index 00000000..0f293b79 --- /dev/null +++ b/build/storage/.dockerignore @@ -0,0 +1 @@ +*.qcow2 \ No newline at end of file diff --git a/build/storage/.gitignore b/build/storage/.gitignore index 4d83a419..fa0735c3 100644 --- a/build/storage/.gitignore +++ b/build/storage/.gitignore @@ -1,4 +1,5 @@ *.qcow2 +__pycache__ *.swp diff --git a/build/storage/Dockerfile b/build/storage/Dockerfile new file mode 100644 index 00000000..7aa31bff --- /dev/null +++ b/build/storage/Dockerfile @@ -0,0 +1,178 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE +# WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK +# +################################################################################ +# build_base +# +# This image is used to provide environment for spdk build and get it in the +# form of spdk packages +################################################################################ +FROM fedora:36 AS base + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY +ARG SPDK_VERSION + +RUN dnf install -y git +COPY spdk/ /spdk +RUN mkdir /spdk-rpm +COPY core/build_base/pre-install /install +RUN chmod +x /install +RUN /install + +################################################################################ +# spdk +# +# Contains installed SPDK from build_base rpm packages. +# Does not contain dependencies required to build SPDK +################################################################################ +FROM fedora:36 AS spdk + +LABEL maintainer=spdk.io + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +# Copy SPDK's RPMs built during pre-install step. +# This allows to reduce final image size since we won't have any dependencies +# which are only required to perform build. +RUN mkdir /spdk-rpm +COPY --from=base /spdk-rpm/*.rpm /spdk-rpm/ +COPY --from=base /spdk-rpm/fio /spdk-rpm/ +# Wrap up the image +COPY core/build_base/post-install /install +RUN chmod +x /install +RUN /install + + +################################################################################ +# spdk-app +# +# This image in addition to installed SPDK binaries contains a script run at +# container boot. +# This script runs SPDK service. +################################################################################ +FROM spdk as spdk-app + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +RUN dnf install -y socat +COPY core/spdk-app/init /init + +ENTRYPOINT ["/init"] + + +################################################################################ +# storage-target +# +# This image should be place on a dedicated machine and responsible for exposing +# ideal storage target(SPDK Malloc bdev) over NVMe/TCP +# Configuration is performed by means of SPDK Json rpc. +################################################################################ +FROM spdk-app AS storage-target + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +################################################################################ +# proxy-container +# +# This image is placed on IPU and exposing storage-target NVMe/TCP devices +# It is responsible for creation of vhost virtio-blk devices and exposing them +# to hosts(KVM or physical ones) +################################################################################ +FROM spdk as proxy-container + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +RUN dnf install -y python socat python3-pip +RUN python -m pip install grpcio grpcio-tools grpcio-reflection + +COPY core/proxy-container/hot_plug.proto /hot_plug.proto +COPY scripts/socket.sh /socket.sh +COPY core/proxy-container/hot_plug_grpc_server.py /hot_plug_grpc_server.py +COPY core/proxy-container/hot_plug_provider.py /hot_plug_provider.py +COPY core/proxy-container/hot_plug_main.py /hot_plug_main.py +COPY core/proxy-container/init /init +COPY core/proxy-container/hot-plug.sh /hot-plug.sh +COPY core/proxy-container/hot-unplug.sh /hot-unplug.sh +COPY --from=spdk-app /init /init_spdk + +# Currently proxy-container relies on spdk json rpc to execute comamnds. +# However, there is no ability to hot-plug a vhost virtio-blk to a vm +# by means of SPDK servce. +# A dedicate service is currently used to provide this hot-plug functionality +# as an addition to SPDK service. +# Later, both will be replaced by SPDK SMA later. +RUN python -m grpc_tools.protoc -I/ --python_out=. --grpc_python_out=/ \ + /hot_plug.proto + +ENTRYPOINT [ "/init" ] + + +################################################################################ +# host-target +# +# This image is responsible for running fio payload over different pci devices. +# It has to be placed into host(a vm for KVM case or physical host for IPU case) +# It uses gRPC to expose this service. +################################################################################ +FROM fedora:36 AS host-target + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +RUN dnf install -y python fio python3-pip +RUN python -m pip install grpcio grpcio-tools grpcio-reflection + +COPY core/host-target/init /init +COPY core/host-target/*.py / +COPY core/host-target/host_target.proto /host_target.proto + +RUN python -m grpc_tools.protoc -I/ --python_out=. --grpc_python_out=/ \ + /host_target.proto + +ENTRYPOINT [ "/init" ] + +################################################################################ +# ipdk-unit-tests +################################################################################ +FROM fedora:36 AS ipdk-unit-tests + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +RUN dnf install -y python fio python3-pip +RUN python -m pip install grpcio-reflection pyfakefs +COPY tests/ut/proxy-container /proxy-container/tests +COPY --from=proxy-container hot_plug_*pb2.py /proxy-container/generated/ +COPY --from=proxy-container hot_plug_*pb2_grpc.py /proxy-container/generated/ +COPY --from=proxy-container hot_plug_grpc_server.py /proxy-container/src/ +COPY --from=proxy-container hot_plug_provider.py /proxy-container/src/ + +COPY tests/ut/host-target /host-target/tests +COPY --from=host-target fio_runner.py /host-target/src/ +COPY --from=host-target pci_devices.py /host-target/src/ +COPY --from=host-target device_exerciser.py /host-target/src/ +COPY --from=host-target host_target_main.py /host-target/src/ +COPY --from=host-target host_target_grpc_server.py /host-target/src/ +COPY --from=host-target host_target_*pb2.py /host-target/generated/ +COPY --from=host-target host_target_*pb2_grpc.py /host-target/generated/ + +COPY tests/ut/run_all_unit_tests.sh / + +ENV PYTHONPATH=/proxy-container/generated:/proxy-container/src:/host-target/src:/host-target/generated + +ENTRYPOINT [ "/run_all_unit_tests.sh" ] diff --git a/build/storage/Makefile b/build/storage/Makefile index c0aaa7da..fb2c0bc2 100644 --- a/build/storage/Makefile +++ b/build/storage/Makefile @@ -2,4 +2,4 @@ #SPDX-License-Identifier: Apache-2.0 # -shellcheck: ;../networking/scripts/run-shellcheck.sh \ No newline at end of file +shellcheck: ;../scripts/run-shellcheck.sh diff --git a/build/storage/README.md b/build/storage/README.md index 24d06041..8ba3082a 100644 --- a/build/storage/README.md +++ b/build/storage/README.md @@ -1,3 +1,22 @@ -# Storage Builds +# Description +IPDK Storage solution targets the audience which has existing custom storage +protocols or implementations of standards such as NVMe-over-Fabrics and wants +to accelerate them by means of a common storage solution across vendors +and platforms. -Coming soon +# Main components +The storage solution is represented by the following containers: +- _proxy-container_ which plays the role of an IPU and exposes the block +devices to a host platform. +- _storage-target_ which is deployed on a remote storage node exposing +ideal targets(ramdrives). +- _host-target_ which is responsible for running of fio traffic through +`proxy-container` to an ideal target within `storage-target`. + +# Recipes +IPDK storage scenarios are described by recipes located in a dedicated +[recipes](recipes/README.md) directory. + +# Tests +The storage solution is covered by [unit tests](tests/ut/) and +[integration tests](tests/it/). diff --git a/build/storage/core/build_base/Dockerfile b/build/storage/core/build_base/Dockerfile deleted file mode 100644 index 1a443164..00000000 --- a/build/storage/core/build_base/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# -# NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE -# WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK -# - -FROM fedora:33 AS base - -# Generic args -ARG HTTP_PROXY -ARG HTTPS_PROXY -ARG NO_PROXY - -ENV http_proxy=$HTTP_PROXY -ENV https_proxy=$HTTPS_PROXY -ENV no_proxy=$NO_PROXY - -RUN dnf install -y git -RUN git clone https://github.com/spdk/spdk.git -RUN tar -czf spdk.tar.gz --exclude='docker/*' -C ./spdk . -COPY pre-install /install -RUN chmod +x /install -RUN /install - -# We are doing a multi-stage build here. This means that previous image, -# base, is going to end up as an intermediate one, untagged, - this -# image can be then manually removed (--force-rm doesn't work here. Go -# figure). -FROM fedora:33 AS spdk - -LABEL maintainer=spdk.io - -# Proxy configuration must be set for each build separately... -ARG HTTP_PROXY -ARG HTTPS_PROXY -ARG NO_PROXY - -ENV http_proxy=$HTTP_PROXY -ENV https_proxy=$HTTPS_PROXY -ENV no_proxy=$NO_PROXY - -# Copy SPDK's RPMs built during pre-install step. -COPY --from=base /tmp/*.rpm /tmp/ -COPY --from=base /tmp/fio /tmp/ -# Wrap up the image -COPY post-install /install -RUN chmod +x /install -RUN /install diff --git a/build/storage/core/build_base/post-install b/build/storage/core/build_base/post-install index 18252c73..e5da40d3 100644 --- a/build/storage/core/build_base/post-install +++ b/build/storage/core/build_base/post-install @@ -8,7 +8,7 @@ # set -e -dnf install -y /tmp/*.rpm +dnf install -y /spdk-rpm/*.rpm # Be nice for docker exec and link SPDK scripts|binaries under common PATH # location like /usr/bin. @@ -21,7 +21,7 @@ ln -s /usr/libexec/spdk/include/spdk /usr/include ln -s /usr/libexec/spdk/scripts/ /usr mkdir -p /usr/src/fio -mv /tmp/fio /usr/src/fio +mv /spdk-rpm/fio /usr/src/fio dnf clean all rm -f /tmp/*.rpm diff --git a/build/storage/core/build_base/pre-install b/build/storage/core/build_base/pre-install index ab69aa51..81bded24 100644 --- a/build/storage/core/build_base/pre-install +++ b/build/storage/core/build_base/pre-install @@ -7,28 +7,17 @@ # WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK # set -e -set -x -spdk_repo=$(mktemp -dt "spdk.XXXXXX") -spdk_tar=/spdk.tar.gz +spdk_repo=/spdk cleanup() { rm -f "$HOME/rpmbuild/rpm/x86_64/"*.rpm - rm -f "$spdk_tar" rm -rf "$spdk_repo" } trap 'cleanup' EXIT -if [[ ! -e $spdk_tar ]]; then - printf 'Missing %s\n' "$spdk_tar" >&2 - exit 1 -fi - -tar -C "$spdk_repo" -xf "$spdk_tar" -cd "$spdk_repo" && git checkout v21.10 && git submodule update --init && cd - - # Required for building RPM dnf install -y rpm-build @@ -48,6 +37,6 @@ DEPS="no" "$spdk_repo/rpmbuild/rpm.sh" \ --with-virtio \ --with-fio -mv "$HOME/rpmbuild/rpm/x86_64/"*.rpm /tmp -mv "/usr/src/fio/fio" /tmp +mv "$HOME/rpmbuild/rpm/x86_64/"*.rpm "/spdk-rpm/" +mv "/usr/src/fio/fio" "/spdk-rpm/" dnf clean all diff --git a/build/storage/core/host-target/device_exerciser.py b/build/storage/core/host-target/device_exerciser.py new file mode 100644 index 00000000..3de8eb14 --- /dev/null +++ b/build/storage/core/host-target/device_exerciser.py @@ -0,0 +1,28 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +import fio_runner +import pci_devices +import os + + +class DeviceExerciserError(RuntimeError): + pass + + +class DeviceExerciser: + def __init__( + self, + fio_runner=fio_runner.run_fio, + virtio_blk_detector=pci_devices.get_virtio_blk_path_by_pci_address, + ): + self.fio_runner = fio_runner + self.virtio_blk_detector = virtio_blk_detector + + def run_fio(self, pci_address, fio_args): + try: + device_path = self.virtio_blk_detector(pci_address) + fio_args_with_device = fio_args + " --filename=" + device_path + return self.fio_runner(fio_args_with_device) + except BaseException as ex: + raise DeviceExerciserError(str(ex)) diff --git a/build/storage/core/host-target/fio_runner.py b/build/storage/core/host-target/fio_runner.py new file mode 100644 index 00000000..f2bf5b14 --- /dev/null +++ b/build/storage/core/host-target/fio_runner.py @@ -0,0 +1,28 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +import subprocess + + +class FioExecutionError(RuntimeError): + pass + + +def run_fio(fio_args, subprocess_run=subprocess.run): + fio_cmd = [] + try: + fio_cmd = ["fio"] + fio_args.split() + result = subprocess_run(fio_cmd, capture_output=True, text=True) + if result.returncode != 0: + raise FioExecutionError( + "fio execution error: '" + + str(result.stdout) + + "' | '" + + str(result.stderr) + + "' " + ) + return result.stdout + except BaseException as ex: + raise FioExecutionError( + "Cannot execute cmd '" + " ".join(fio_cmd) + "' Error: " + str(ex) + ) diff --git a/build/storage/core/host-target/host_target.proto b/build/storage/core/host-target/host_target.proto new file mode 100644 index 00000000..dfb5c4ce --- /dev/null +++ b/build/storage/core/host-target/host_target.proto @@ -0,0 +1,20 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +syntax = "proto3"; + +package host_target; + +service HostTarget { + rpc RunFio (RunFioRequest) returns (RunFioReply) {} +} + +message RunFioRequest { + string pciAddress = 1; + string fioArgs = 2; +} + +message RunFioReply { + string fioOutput = 1; +} diff --git a/build/storage/core/host-target/host_target_grpc_server.py b/build/storage/core/host-target/host_target_grpc_server.py new file mode 100644 index 00000000..fb632297 --- /dev/null +++ b/build/storage/core/host-target/host_target_grpc_server.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +from concurrent import futures + +import grpc +from grpc_reflection.v1alpha import reflection +import host_target_pb2 +import host_target_pb2_grpc +from device_exerciser import DeviceExerciser +from fio_runner import run_fio +from pci_devices import get_virtio_blk_path_by_pci_address + + +class InvalidHotPlugProvider(RuntimeError): + def __init__(self, message): + super().__init__(message) + + +class HostTargetService(host_target_pb2_grpc.HostTargetServicer): + def __init__(self, fio_runner, virtio_blk_detector): + super().__init__() + self.device_exerciser = DeviceExerciser(fio_runner, virtio_blk_detector) + + def RunFio(self, request, context): + output = None + try: + output = self.device_exerciser.run_fio(request.pciAddress, request.fioArgs) + except BaseException as ex: + context.set_code(grpc.StatusCode.FAILED_PRECONDITION) + context.set_details(str(ex)) + return host_target_pb2.RunFioReply(fioOutput=output) + + +def run_grpc_server(ip_address, port, server_creator=grpc.server): + try: + server = server_creator(futures.ThreadPoolExecutor(max_workers=10)) + host_target_pb2_grpc.add_HostTargetServicer_to_server( + HostTargetService(run_fio, get_virtio_blk_path_by_pci_address), server + ) + service_names = ( + host_target_pb2.DESCRIPTOR.services_by_name["HostTarget"].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(service_names, server) + server.add_insecure_port(ip_address + ":" + str(port)) + server.start() + server.wait_for_termination() + return 0 + except KeyboardInterrupt as ex: + return 0 + except BaseException as ex: + print("Couldn't run gRPC server. Error: " + str(ex)) + return 1 diff --git a/build/storage/core/host-target/host_target_main.py b/build/storage/core/host-target/host_target_main.py new file mode 100755 index 00000000..fdb04e83 --- /dev/null +++ b/build/storage/core/host-target/host_target_main.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import logging +import argparse + +from host_target_grpc_server import run_grpc_server + + +def parse_arguments(): + parser = argparse.ArgumentParser( + description="Runs service for hot-plug/hot-detach virtio-blk devices to vms" + ) + parser.add_argument("--ip", required=True, help="ip address the server listens to") + parser.add_argument( + "--port", type=int, default=50051, help="port number the server listens to" + ) + + args = parser.parse_args() + return args + + +if __name__ == "__main__": + logging.basicConfig() + + args = parse_arguments() + + run_grpc_server(args.ip, args.port) diff --git a/build/storage/core/host-target/init b/build/storage/core/host-target/init new file mode 100755 index 00000000..b7fb8dfc --- /dev/null +++ b/build/storage/core/host-target/init @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x && export GRPC_VERBOSITY=debug + +echo "Running host target server for ${IP_ADDR}:${PORT}" +/host_target_main.py --ip "${IP_ADDR}" --port "${PORT}" diff --git a/build/storage/core/host-target/pci_devices.py b/build/storage/core/host-target/pci_devices.py new file mode 100644 index 00000000..ece5f2d5 --- /dev/null +++ b/build/storage/core/host-target/pci_devices.py @@ -0,0 +1,119 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +from lib2to3.pytree import Base +import os +import re +import glob + + +pci_validator = re.compile( + r"[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-1]{1}[0-9a-fA-F]{1}\.[0-7]{1}" +) + + +class PciAddress: + def validate_pci_address(pci_address): + return pci_validator.search(pci_address) != None + + def _parse_pci_address(pci_address): + split_pci_address = pci_address.replace(".", ":").split(":") + split_pci_address.reverse() + function = split_pci_address[0].strip() + device = split_pci_address[1].strip() + bus = split_pci_address[2].strip() + domain = split_pci_address[3].strip() + return (domain, bus, device, function) + + def __init__(self, pci_address) -> None: + if not PciAddress.validate_pci_address(pci_address): + raise InvalidPciAddress(pci_address + " is invalid") + ( + self.domain, + self.bus, + self.device, + self.function, + ) = PciAddress._parse_pci_address(pci_address) + + def get_domain_bus_prefix(self): + return self.domain + ":" + self.bus + + def get_bus_device_function_address(self): + return self.bus + ":" + self.device + "." + self.function + + def get_full_address(self): + return self.domain + ":" + self.get_bus_device_function_address() + + +def get_directories(path): + if os.path.exists(path): + unused0, dirs, unused1 = next(os.walk(path)) + return dirs + else: + return None + + +def get_all_files_by_pattern(pattern): + return glob.glob(pattern) + + +def directory_find(atom, root="."): + dirs = get_directories(root) + if dirs and atom in dirs: + return os.path.join(root, atom) + return None + + +class InvalidPciAddress(ValueError): + pass + + +class FailedPciDeviceDetection(RuntimeError): + pass + + +def get_virtio_blk_path_by_pci_address(pci_address): + addr = PciAddress(pci_address) + + domain_bus_dir = os.path.join("/sys/devices", "pci" + addr.get_domain_bus_prefix()) + pci_dev_sysfs_dir = directory_find(addr.get_full_address(), domain_bus_dir) + if pci_dev_sysfs_dir == None: + raise FailedPciDeviceDetection( + "No pci device with " + pci_address + " exists under " + domain_bus_dir + ) + block_directory_pattern = os.path.join(pci_dev_sysfs_dir, "virtio*/block") + block_device_matches = get_all_files_by_pattern(block_directory_pattern) + if len(block_device_matches) == 0: + raise FailedPciDeviceDetection( + "No devices found for pattern " + block_directory_pattern + ) + elif len(block_device_matches) > 1: + raise FailedPciDeviceDetection( + "Found more than one device for pattern" + + block_directory_pattern + + " : " + + str(block_device_matches) + ) + + devices = get_directories(block_device_matches[0]) + if not devices or len(devices) == 0: + raise FailedPciDeviceDetection( + "No device exist under " + + block_directory_pattern + + " for pci device '" + + pci_address + + "'" + ) + elif len(devices) > 1: + raise FailedPciDeviceDetection( + "Multiple devices are dected " + + str(devices) + + " for pci address '" + + pci_address + + "'" + ) + device_path = os.path.join("/dev", devices[0]) + if not os.path.exists(device_path): + raise FailedPciDeviceDetection("Device " + device_path + " does not exist") + + return device_path diff --git a/build/storage/core/proxy-container/hot-plug.sh b/build/storage/core/proxy-container/hot-plug.sh new file mode 100755 index 00000000..ea157028 --- /dev/null +++ b/build/storage/core/proxy-container/hot-plug.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x + +# shellcheck disable=SC1091 +source socket.sh + +function attach_virtio_blk() { + vm_monitor_socket=${1} + vhost_socket=${2} + id="${3}" + num_queues=2 + error_word="error" + + echo "Attaching ${vhost_socket} to vm socket ${vm_monitor_socket}" + add_chardev_cmd="chardev-add socket,id=${id},path=${vhost_socket}" + if ! send_command_over_unix_socket_and_no_word_found \ + "${vm_monitor_socket}" "${add_chardev_cmd}" 1 "${error_word}" ; then + return 1 + fi + + add_device_cmd="device_add vhost-user-blk-pci,chardev=${id},num-queues=${num_queues},id=${id}" + if ! send_command_over_unix_socket_and_no_word_found \ + "${vm_monitor_socket}" "${add_device_cmd}" 1 "${error_word}" ; then + return 1 + fi + return 0 +} + +if [[ $# != 3 ]] ; then + echo "Not all arguments are specified. " + exit 1 +fi + +attach_virtio_blk "${1}" "${2}" "${3}" +exit $? diff --git a/build/storage/core/proxy-container/hot-unplug.sh b/build/storage/core/proxy-container/hot-unplug.sh new file mode 100755 index 00000000..da33885b --- /dev/null +++ b/build/storage/core/proxy-container/hot-unplug.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x + +# shellcheck disable=SC1091 +source socket.sh + +function dettach_virtio_blk() { + vm_monitor_socket=${1} + id="${2}" + error_word="error" + + echo "Dettaching ${id} from ${vm_monitor_socket}" + device_del_cmd="device_del ${id}" + if ! send_command_over_unix_socket_and_no_word_found \ + "${vm_monitor_socket}" "${device_del_cmd}" 1 "${error_word}" ; then + return 1 + fi + + remove_chardev_cmd="chardev-remove ${id}" + if ! send_command_over_unix_socket_and_no_word_found \ + "${vm_monitor_socket}" "${remove_chardev_cmd}" 1 "${error_word}" ; then + return 1 + fi + return 0 +} + +if [[ $# != 2 ]] ; then + echo "Not all arguments are specified. " + exit 1 +fi + +dettach_virtio_blk "${1}" "${2}" +exit $? diff --git a/build/storage/core/proxy-container/hot_plug.proto b/build/storage/core/proxy-container/hot_plug.proto new file mode 100644 index 00000000..e90fe6a9 --- /dev/null +++ b/build/storage/core/proxy-container/hot_plug.proto @@ -0,0 +1,28 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +syntax = "proto3"; + +package hot_plug; + +service HotPlug { + rpc HotPlugVirtioBlk (HotPlugRequest) returns (HotPlugReply) {} + rpc HotUnplugVirtioBlk (HotUnplugRequest) returns (HotUnplugReply) {} +} + +message HotPlugRequest { + string vmId = 1; + string vhostVirtioBlkId = 2; +} + +message HotPlugReply { +} + +message HotUnplugRequest { + string vmId = 1; + string vhostVirtioBlkId = 2; +} + +message HotUnplugReply { +} \ No newline at end of file diff --git a/build/storage/core/proxy-container/hot_plug_grpc_server.py b/build/storage/core/proxy-container/hot_plug_grpc_server.py new file mode 100755 index 00000000..cc1a27e9 --- /dev/null +++ b/build/storage/core/proxy-container/hot_plug_grpc_server.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +from concurrent import futures + +import grpc +from grpc_reflection.v1alpha import reflection +import hot_plug_pb2 +import hot_plug_pb2_grpc + + +class InvalidHotPlugProvider(RuntimeError): + def __init__(self, message): + super().__init__(message) + + +class HotPlugService(hot_plug_pb2_grpc.HotPlugServicer): + def __init__(self, hot_plug_provider): + super().__init__() + if hot_plug_provider == None: + raise InvalidHotPlugProvider("HotPlugProvider cannot be None") + self.hot_plug_provider = hot_plug_provider + + def __execute_server_operation(self, request, context, operation): + try: + operation(request.vmId, request.vhostVirtioBlkId) + except BaseException as ex: + context.set_code(grpc.StatusCode.FAILED_PRECONDITION) + context.set_details(str(ex)) + return hot_plug_pb2.HotPlugReply() + + def HotPlugVirtioBlk(self, request, context): + return self.__execute_server_operation( + request, context, self.hot_plug_provider.hot_plug_vhost_virtio_blk + ) + + def HotUnplugVirtioBlk(self, request, context): + return self.__execute_server_operation( + request, context, self.hot_plug_provider.hot_unplug_vhost_virtio_blk + ) + + +def run_grpc_server(ip_address, port, hot_plug_provider): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + hot_plug_pb2_grpc.add_HotPlugServicer_to_server( + HotPlugService(hot_plug_provider), server + ) + service_names = ( + hot_plug_pb2.DESCRIPTOR.services_by_name["HotPlug"].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(service_names, server) + server.add_insecure_port(ip_address + ":" + str(port)) + server.start() + server.wait_for_termination() diff --git a/build/storage/core/proxy-container/hot_plug_main.py b/build/storage/core/proxy-container/hot_plug_main.py new file mode 100755 index 00000000..8dd26130 --- /dev/null +++ b/build/storage/core/proxy-container/hot_plug_main.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import logging +import argparse + +from hot_plug_grpc_server import run_grpc_server +from hot_plug_provider import HotPlugProvider + + +def parse_arguments(): + parser = argparse.ArgumentParser( + description="Runs service for hot-plug/hot-detach virtio-blk devices to vms" + ) + parser.add_argument("--ip", required=True, help="ip address the server listens to") + parser.add_argument( + "--port", type=int, default=50051, help="port number the server listens to" + ) + parser.add_argument( + "--shared-dir", + type=str, + required=True, + help="Directory path to shared with Host resources", + ) + parser.add_argument( + "--host-shared-dir", + type=str, + required=True, + help="Directory path to shared with container resources", + ) + + args = parser.parse_args() + return args + + +if __name__ == "__main__": + logging.basicConfig() + + args = parse_arguments() + + ip_address = args.ip + port = args.port + hot_plug_provider = HotPlugProvider(args.shared_dir, args.host_shared_dir) + run_grpc_server(ip_address, port, hot_plug_provider) diff --git a/build/storage/core/proxy-container/hot_plug_provider.py b/build/storage/core/proxy-container/hot_plug_provider.py new file mode 100755 index 00000000..62e53a19 --- /dev/null +++ b/build/storage/core/proxy-container/hot_plug_provider.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +from multiprocessing.sharedctypes import Value +import os.path +import errno +import subprocess +from pathlib import Path + + +class ActionExecutionError(RuntimeError): + def __init__(self, message): + super().__init__(message) + + +class HotPlugProvider: + def __init__(self, shared_dir_path, host_shared_dir_path): + if shared_dir_path == None: + raise ValueError("shared_dir_path value cannot be None") + if host_shared_dir_path == None: + raise ValueError("host_shared_dir_path value cannot be None") + self.shared_dir_path = shared_dir_path + self.host_shared_dir_path = host_shared_dir_path + + def _hot_plug_action(self, vm_monitor, vhost_virtio_blk, device_id): + vm_monitor_path = os.path.join(self.shared_dir_path, vm_monitor) + vhost_path = os.path.join(self.host_shared_dir_path, vhost_virtio_blk) + ret = subprocess.call( + "/hot-plug.sh " + vm_monitor_path + " " + vhost_path + " " + str(device_id), + shell=True, + ) + if ret != 0: + raise ActionExecutionError("Error at hot-plug execution") + + def _hot_unplug_action(self, vm_monitor, unused, device_id): + vm_monitor_path = os.path.join(self.shared_dir_path, vm_monitor) + ret = subprocess.call( + "/hot-unplug.sh " + vm_monitor_path + " " + str(device_id), shell=True + ) + if ret != 0: + raise ActionExecutionError("Error at hot-unplug execution") + + def _adjust_vm_monitor(vm_monitor): + return vm_monitor.strip() + + def _adjust_vhost_virtio_blk(vhost_virtio_blk): + return vhost_virtio_blk.strip() + + def __is_socket(socket_path): + return Path(socket_path).is_socket() + + def __check_socket_path(path): + if not HotPlugProvider.__is_socket(path): + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) + + def _validate_vhost_virtio_blk(self, vhost_virtio_blk): + HotPlugProvider.__check_socket_path( + os.path.join(self.shared_dir_path, vhost_virtio_blk) + ) + + def _validate_vm_monitor(self, vm_monitor): + HotPlugProvider.__check_socket_path( + os.path.join(self.shared_dir_path, vm_monitor) + ) + + def _validate_input(self, vm_monitor, vhost_virtio_blk): + self._validate_vhost_virtio_blk(vhost_virtio_blk) + self._validate_vm_monitor(vm_monitor) + + def _generate_device_id(self, vhost_virtio_blk): + return vhost_virtio_blk.strip() + + def _perform_disk_operation(self, vm_monitor, vhost_virtio_blk, disk_operation): + if vm_monitor == None: + raise ValueError("vm_monitor value cannot be None") + if vhost_virtio_blk == None: + raise ValueError("vhost_virtio_blk value cannot be None") + + vm_monitor = HotPlugProvider._adjust_vm_monitor(vm_monitor) + vhost_virtio_blk = HotPlugProvider._adjust_vhost_virtio_blk(vhost_virtio_blk) + + self._validate_input(vm_monitor, vhost_virtio_blk) + + device_id = self._generate_device_id(vhost_virtio_blk) + + try: + disk_operation(vm_monitor, vhost_virtio_blk, device_id) + except BaseException as ex: + raise ActionExecutionError(str(ex)) + + def hot_plug_vhost_virtio_blk(self, vm_monitor, vhost_virtio_blk): + self._perform_disk_operation( + vm_monitor, vhost_virtio_blk, self._hot_plug_action + ) + + def hot_unplug_vhost_virtio_blk(self, vm_monitor, vhost_virtio_blk): + self._perform_disk_operation( + vm_monitor, vhost_virtio_blk, self._hot_unplug_action + ) diff --git a/build/storage/core/proxy-container/init b/build/storage/core/proxy-container/init new file mode 100755 index 00000000..13c433f6 --- /dev/null +++ b/build/storage/core/proxy-container/init @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x && export GRPC_VERBOSITY=debug + +spdk_pid_file="/spdk_pid" +rm -f "$spdk_pid_file" + +/init_spdk "$@" & + +function wait_for_spdk_pid_file() { + spdk_pid_file="$1" + max_wait_for_pid_counter="$2" + wait_for_pid_counter=0 + while [ ! -f "$spdk_pid_file" ] ; do + if [ "$wait_for_pid_counter" -ge "$max_wait_for_pid_counter" ] ; then + return 1 + fi + wait_for_pid_counter=$(( wait_for_pid_counter + 1 )) + sleep 1 + done + return 0 +} + +function is_pid_running() { + pid="$1" + kill -0 "$pid" &> /dev/null + return $? +} + +wait_max_sec=5 +if ! wait_for_spdk_pid_file "$spdk_pid_file" "$wait_max_sec" ; then + echo "Failed to start spdk-app or socat to expose port." >&2 + exit 1 +fi + +pid=$(cat "$spdk_pid_file") +if ! is_pid_running "$pid" ; then + echo "Spdk is not running." >&2 + exit 1 +fi + +HOT_PLUG_SERVICE_IP_ADDR="${HOT_PLUG_SERVICE_IP_ADDR:-"0.0.0.0"}" +HOT_PLUG_SERVICE_PORT="${HOT_PLUG_SERVICE_PORT:-50051}" + +/hot_plug_main.py --ip "${HOT_PLUG_SERVICE_IP_ADDR}" --port "${HOT_PLUG_SERVICE_PORT}" \ + --shared-dir /ipdk-shared --host-shared-dir "${HOST_SHARED_VOLUME}" diff --git a/build/storage/core/spdk-app/Dockerfile b/build/storage/core/spdk-app/Dockerfile deleted file mode 100644 index aba6b9c6..00000000 --- a/build/storage/core/spdk-app/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# -# NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE -# WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK -# - -FROM spdk - -# Generic args -ARG HTTP_PROXY -ARG HTTPS_PROXY -ARG NO_PROXY - -ENV http_proxy=$HTTP_PROXY -ENV https_proxy=$HTTPS_PROXY -ENV no_proxy=$NO_PROXY - -COPY init /init - -ENTRYPOINT ["/init"] diff --git a/build/storage/core/spdk-app/init b/build/storage/core/spdk-app/init index 0fde2fa8..fd74b4aa 100755 --- a/build/storage/core/spdk-app/init +++ b/build/storage/core/spdk-app/init @@ -10,6 +10,8 @@ [ "$DEBUG" == 'true' ] && set -x app=spdk_tgt args=() limit_args=() +SPDK_PORT="${SPDK_PORT:-5260}" +SPDK_IP_ADDR="${SPDK_IP_ADDR:-"0.0.0.0"}" # Override default app if [[ -n $SPDK_APP ]]; then @@ -35,7 +37,21 @@ if [[ -e /config ]]; then args+=("--json" "/config") fi -# Wait a bit to make sure ip is in place -sleep 2s +function is_port_on_ip_addr_open() { + ip_addr="$1" + port="$2" + timeout 1 bash -c "cat < /dev/null > /dev/tcp/${ip_addr}/${port}" &> /dev/null + return $? +} -exec "$app" "${args[@]}" +if is_port_on_ip_addr_open "$SPDK_IP_ADDR" "$SPDK_PORT"; then + echo "Cannot run socat for ${SPDK_IP_ADDR} and port ${SPDK_PORT}. Port is already in use." >&2 + exit 1 +fi + +socat TCP-LISTEN:"${SPDK_PORT}",reuseaddr,bind="${SPDK_IP_ADDR}",fork \ + UNIX-CLIENT:/var/tmp/spdk.sock & + +spdk_pid_file="/spdk_pid" +rm -f "$spdk_pid_file" +$app "${args[@]}" -r /var/tmp/spdk.sock -f "$spdk_pid_file" diff --git a/build/storage/recipes/README.md b/build/storage/recipes/README.md new file mode 100644 index 00000000..7dcc9b4a --- /dev/null +++ b/build/storage/recipes/README.md @@ -0,0 +1,13 @@ +# Supported recipes +Currently, the following scenarios are represented: + +* [hot-plug](./hot-plug.md) - describes how to deploy containers between +different machines and demonstrates virtio-blk hot-plug to a running host. +* [fio](./fio.md) - extends `hot-plug` scenario and describes how to run +fio traffic by means of `host-target` container through a hot-plugged virtio-blk +device. + +In both cases host target platform is implied as KVM. + +The picture below demonstrates the configuration exercised in these recipes +![System configuration for recipes](./system_configuration.png "System configuration for recipes") diff --git a/build/storage/recipes/environment_setup.md b/build/storage/recipes/environment_setup.md new file mode 100644 index 00000000..b1b402bb --- /dev/null +++ b/build/storage/recipes/environment_setup.md @@ -0,0 +1,170 @@ +# Environment preparation +Currently the recipe environment should be represented by 2 separate machines, +referred as `storage-target-platform` and `proxy-container-platform`, +located in the same network. For each of them the steps from +[System setup](#system-setup) section have to be performed. +The containers running on those platforms are named `storage-target` and +`proxy-container` respectively. + +# System setup +The basic requirement for running the recipes is a modern Linux distribution +with docker support. +So far, the recipe has been successfully tested on the following systems: +- Fedora 34 +- Ubuntu 18.04 + +To run the recipe some basic system preparations are required, in particular: + +## Virtualization support +Make sure that VT-x/AMD-v support is enabled in BIOS +``` +$ lscpu | grep -i virtualization +Virtualization: VT-x +``` +and that kvm modules are loaded +``` +$ lsmod | grep -i kvm +kvm_intel 217088 0 +kvm 614400 1 kvm_intel +irqbypass 16384 1 kvm +``` + +## Tool installation +Make sure that following tools are installed on your system or install them +using the corresponding package manager. + +### wget +Installation on Fedora +``` +$ sudo dnf install wget +``` +or on Ubuntu +``` +$ sudo apt install wget +``` + +### docker +``` +$ sudo dnf install docker +``` +or +``` +$ sudo apt install docker +``` +**Note:** +Make sure that required proxy settings are configured for docker. +Please, refer to [this](https://docs.docker.com/config/daemon/systemd/#httphttps-proxy) +page. + +### libguestfs-tools +``` +$ sudo dnf install libguestfs-tools-c +``` +or +``` +$ sudo apt install libguestfs-tools +``` + +**Note:** +To run `libguestfs` tools without root privileges, you may need to workaround +the problem of +Linux kernel image not being readable by issuing: +``` +$ sudo chmod +r /boot/vmlinuz-* +``` + +## Setting security policies +Make sure that required security permissions are configured to enable VM network +connectivity or disable them temporarily. +On Fedora SELinux can be disabled by means of the following command +``` +$ sudo setenforce 0 +``` +For Ubuntu AppArmor is often used and can be disabled by issuing +``` +$ sudo systemctl stop apparmor +``` + +# Deploy containers on the machines + +1. Download repositories on both platforms: `proxy-container-platform` and +`storage-target-platform` + +2. Run docker containers providing configuration scripts: + +On `storage-target-platform` +``` +$ scripts/run_storage_target_container.sh +``` + +On `proxy-container-platform` +``` +$ SHARED_VOLUME= \ +scripts/run_proxy_container.sh +``` + +`SHARED_VOLUME` points to a directory where vhost storage device and vm monitor +will be exposed. + + +It is also possible to specify ip addresses and ports where spdk service is +exposed on by specifying `SPDK_IP_ADDR` and `SPDK_PORT` environment variables. +Those environment variables can be applied to both `storage-target` and +`proxy-container` +e.g. + +``` +SPDK_IP_ADDR="127.0.0.1" SPDK_PORT=5261 scripts/run_storage_target_container.sh +``` +or +``` +$ SPDK_IP_ADDR="127.0.0.1" SPDK_PORT=5262 \ +SHARED_VOLUME= \ +scripts/run_proxy_container.sh +``` +By default, `SPDK_IP_ADDR` is set to `0.0.0.0` and `SPDK_PORT` is set to `5260` + +3. Run the vm instance on `proxy-container-platform` platform +``` +$ SHARED_VOLUME= scripts/vm/run_vm.sh +``` + +`SHARED_VOLUME` points to a directory where vhost storage device and vm monitor +will be exposed(exactly one specified in `SHARED_VOLUME` to run +`proxy-container`). + + +Finally vm console will be opened. + + +login:password pair for the vm is `root:root`. +Run `host-target` container within the vm console +``` +$ run_host_target_container.sh & +``` + +4. Prepare environment to send commands to the storage containers. +For that purpose we need to have spdk rpc.py and grpc-cli tools available. +`test-driver` image from the [integration tests](../tests/it/README.md#introduction) +fits for this purpose well since it contains all required tools. +Let's use `test-driver` image instance to send all required commands. +In the recipes this `test-driver` will be referred as `cmd-sender` to +increase comprehension of the text. However, we should keep in mind that +`cmd-sender` is a running instance of `test-driver` image. + +To use test-driver [install docker-compose](../tests/it/README.md#docker-compose-setup) +and run the integration tests to build that container on `proxy-container-platform` +machine. +``` +$ tests/it/run.sh +``` + +Run `test-driver` on `proxy-container-platform` machine +``` +$ docker run -it --privileged --network host --entrypoint /bin/bash test-driver +``` + +Source supplementary scripts in running `test-driver` container +``` +$ source /scripts/disk_infrastructure.sh +``` diff --git a/build/storage/recipes/fio.md b/build/storage/recipes/fio.md new file mode 100644 index 00000000..53119191 --- /dev/null +++ b/build/storage/recipes/fio.md @@ -0,0 +1,28 @@ +# Fio recipe + +This recipe describes how to run fio from Host to an ideal target +with IPDK containers. + +For this recipe 2 machines required. +They are referred as `storage-target-platform` and `proxy-container-platform`. +The containers running on those platforms are named `storage-target` and +`proxy-container` respectively. + +To apply this scenario the following steps need to be applied: + +1. Perform all steps described in [environment setup](environment_setup.md) + +2. Run [hot-plug scenario](hot-plug.md) until a virtio-blk device is attached to +the vm(step 5) + +3. Run fio. Execute the following command from `cmd-sender` +``` +$ no_grpc_proxy= grpc_cli call :50051 \ +RunFio "pciAddress: '0000:00:04.0' fioArgs: '--direct=1 --rw=randrw --bs=4k --ioengine=libaio --iodepth=256 --runtime=1 --numjobs=4 --time_based --group_reporting --name=iops-test-job'" +``` + +Expected output +``` +connecting to 192.168.53.76:50051 +fioOutput: "iops-test-job: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=256\n...\nfio-3.21\nStarting 4 processes\n\niops-test-job: (groupid=0, jobs=4): err= 0: pid=18: Fri Feb 25 07:18:16 2022\n read: IOPS=18.0k, BW=74.2MiB/s (77.8MB/s)(742MiB/10008msec)\n slat (usec): min=2, max=11636, avg=102.74, stdev=479.43\n clat (usec): min=6703, max=62683, avg=26796.45, stdev=9296.56\n lat (usec): min=6707, max=62704, avg=26899.45, stdev=9327.52\n clat percentiles (usec):\n | 1.00th=[11469], 5.00th=[12125], 10.00th=[12780], 20.00th=[16909],\n | 30.00th=[22938], 40.00th=[25035], 50.00th=[26870], 60.00th=[29492],\n | 70.00th=[31851], 80.00th=[34866], 90.00th=[38536], 95.00th=[41681],\n | 99.00th=[48497], 99.50th=[51643], 99.90th=[56886], 99.95th=[58459],\n | 99.99th=[60556]\n bw ( KiB/s): min=55800, max=100312, per=99.78%, avg=75797.47, stdev=2986.36, samples=76\n iops : min=13950, max=25078, avg=18949.37, stdev=746.59, samples=76\n write: IOPS=18.0k, BW=74.1MiB/s (77.7MB/s)(742MiB/10008msec); 0 zone resets\n slat (usec): min=2, max=9856, avg=103.06, stdev=476.92\n clat (usec): min=5141, max=62700, avg=26812.92, stdev=9302.68\n lat (usec): min=6695, max=62704, avg=26916.24, stdev=9334.34\n clat percentiles (usec):\n | 1.00th=[11469], 5.00th=[12125], 10.00th=[12780], 20.00th=[16909],\n | 30.00th=[22938], 40.00th=[25035], 50.00th=[26870], 60.00th=[29492],\n | 70.00th=[31851], 80.00th=[34866], 90.00th=[38536], 95.00th=[41681],\n | 99.00th=[48497], 99.50th=[51643], 99.90th=[56361], 99.95th=[57934],\n | 99.99th=[60031]\n bw ( KiB/s): min=56728, max=100464, per=99.83%, avg=75788.21, stdev=2943.55, samples=76\n iops : min=14182, max=25116, avg=18947.05, stdev=735.89, samples=76\n lat (msec) : 10=0.06%, 20=24.99%, 50=74.23%, 100=0.72%\n cpu : usr=2.21%, sys=4.23%, ctx=41669, majf=0, minf=55\n IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%\n submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%\n complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%\n issued rwts: total=190063,189952,0,0 short=0,0,0,0 dropped=0,0,0,0\n latency : target=0, window=0, percentile=100.00%, depth=256\n\nRun status group 0 (all jobs):\n READ: bw=74.2MiB/s (77.8MB/s), 74.2MiB/s-74.2MiB/s (77.8MB/s-77.8MB/s), io=742MiB (778MB), run=10008-10008msec\n WRITE: bw=74.1MiB/s (77.7MB/s), 74.1MiB/s-74.1MiB/s (77.7MB/s-77.7MB/s), io=742MiB (778MB), run=10008-10008msec\n\nDisk stats (read/write):\n vda: ios=187480/187373, merge=0/0, ticks=1237509/1230145, in_queue=2467655, util=99.67%\n" +``` diff --git a/build/storage/recipes/hot-plug.md b/build/storage/recipes/hot-plug.md new file mode 100644 index 00000000..3fd0ff63 --- /dev/null +++ b/build/storage/recipes/hot-plug.md @@ -0,0 +1,109 @@ +# Hot-plug recipe +This recipe describes how to perform virtio-blk hot-plug to a running KVM host. +The virtio-blk device is backed up by an ideal target on `storage-target-platform` +machine exposed over NVMe/TCP. + +For this recipe 2 machines required. They are referred as +`storage-target-platform` and `proxy-container-platform`. +The containers running on those platforms are named `storage-target` and +`proxy-container` respectively. + +To apply this scenario the following steps need to be applied: + +1. Perform all steps described in [environment setup](environment_setup.md) + +2. Make sure there is no virtio-blk device attached +Run in [vm console](environment_setup.md#vm-console) +``` +$ lsblk +``` +Expected output +``` +NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT +sda 8:0 0 4G 0 disk +└─sda1 8:1 0 4G 0 part / +``` + +3. Create subsystem and expose it to `proxy-container` +Send from your `cmd-sender` +``` +$ create_subsystem_and_expose_to_another_machine \ + nqn.2016-06.io.spdk:cnode0 \ + Nvme0 +``` +or +``` +$ create_subsystem_and_expose_to_another_machine \ + nqn.2016-06.io.spdk:cnode0 \ + Nvme0 \ + +``` +Where `storage_target_SPDK_PORT` is non-default `SPDK_PORT` specified for +`run_storage_target_container.sh` +Where `proxy_container_SPDK_PORT` is non-default `SPDK_PORT` specified for +`run_proxy_container.sh` +Please see [here](environment_setup.md#non-default-port) + +4. Create ramdrive on `storage-target` and attach to the subsystem +Send from your `cmd-sender` +``` +$ create_ramdrive_and_attach_as_ns_to_subsystem \ + Malloc0 16 nqn.2016-06.io.spdk:cnode0 +``` +or +``` +$ create_ramdrive_and_attach_as_ns_to_subsystem \ + Malloc0 16 nqn.2016-06.io.spdk:cnode0 \ + +``` +Where `storage_target_SPDK_PORT` is non-default `SPDK_PORT` specified for +`run_storage_target_container.sh` +Please see [here](environment_setup.md#non-default-port) + +5. Attach exposed ramdrive to the vm +Send from your `cmd-sender` +``` +$ attach_ns_as_virtio_blk \ + VirtioBlk0 Nvme0n1 50051 \ + vm_monitor +``` +or +``` +$ attach_ns_as_virtio_blk \ + VirtioBlk0 Nvme0n1 50051 \ + vm_monitor +``` +Where `proxy_container_SPDK_PORT` is non-default `SPDK_PORT` specified for +`run_proxy_container.sh` +Please see [here](environment_setup.md#non-default-port) + +6. Check if virtio-blk is attached to the vm +Open the [vm console](environment_setup.md#vm-console) machine and run the following command +``` +$ lsblk +``` +The expected output is +``` +NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT +sda 8:0 0 4G 0 disk +└─sda1 8:1 0 4G 0 part / +vda 252:0 0 16M 0 disk +``` + +7. Perform unplug +Run the command below on `cmd-sender` to hot-unplug device +``` +dettach_virtio_blk 50051 vm_monitor VirtioBlk0 +``` + +8. Check there is no virtio-blk device +Open the [vm console](environment_setup.md#vm-console) and run the following command +``` +$ lsblk +``` +The expected output is +``` +NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT +sda 8:0 0 4G 0 disk +└─sda1 8:1 0 4G 0 part / +``` diff --git a/build/storage/recipes/system_configuration.png b/build/storage/recipes/system_configuration.png new file mode 100644 index 00000000..c366c571 Binary files /dev/null and b/build/storage/recipes/system_configuration.png differ diff --git a/build/storage/scripts/allocate_hugepages.sh b/build/storage/scripts/allocate_hugepages.sh new file mode 100755 index 00000000..eadf35f1 --- /dev/null +++ b/build/storage/scripts/allocate_hugepages.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +[ "$DEBUG" == 'true' ] && set -x +set -e + +required_number_of_pages="${REQUIRED_NUMBER_OF_2048KB_PAGES:-3072}" + +number_of_2mb_hugepages_folder=/sys/kernel/mm/hugepages/hugepages-2048kB +number_of_2mb_hugepages_provider=${number_of_2mb_hugepages_folder}/nr_hugepages + +if [ -d ${number_of_2mb_hugepages_folder} ] ; then + number_of_2mb_pages=$(cat ${number_of_2mb_hugepages_provider}) + if [ "$number_of_2mb_pages" -lt "$required_number_of_pages" ] ; then + echo "${required_number_of_pages}" | \ + sudo tee ${number_of_2mb_hugepages_provider} + fi +fi diff --git a/build/storage/scripts/build_container.sh b/build/storage/scripts/build_container.sh new file mode 100755 index 00000000..f31a93b4 --- /dev/null +++ b/build/storage/scripts/build_container.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x +set -e + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +root_dir="${script_dir}/.." +# shellcheck disable=SC1091 +source "${script_dir}"/spdk_version.sh +"${script_dir}"/prepare_to_build.sh + +DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} + +build_proxies="--build-arg HTTP_PROXY=${HTTP_PROXY} \ + --build-arg HTTPS_PROXY=${HTTPS_PROXY} \ + --build-arg NO_PROXY=${NO_PROXY}" +spdk_version_build_arg="--build-arg SPDK_VERSION=$(get_spdk_version)" + +join_by() { + local d=${1-} f=${2-} + if shift 2; then + printf %s "$f" "${@/#/$d}" + fi +} + +container_to_build="${1}" + +possible_containers=("storage-target" "proxy-container" "ipdk-unit-tests" "host-target") +if [[ " ${possible_containers[*]} " =~ ${container_to_build} ]]; then + export DOCKER_BUILDKIT="${DOCKER_BUILDKIT}" + + docker_build="docker build ${build_proxies} \ + ${spdk_version_build_arg} \ + -t ${container_to_build} --target ${container_to_build} ${root_dir}" + $docker_build +else + echo "Unknown container '${container_to_build}'" + possible_containers_as_string=$(join_by ", " "${possible_containers[@]}") + echo "Possible containers: ${possible_containers_as_string}" + exit 1 +fi + diff --git a/build/storage/scripts/disk_infrastructure.sh b/build/storage/scripts/disk_infrastructure.sh new file mode 100755 index 00000000..5da04f90 --- /dev/null +++ b/build/storage/scripts/disk_infrastructure.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +export DEFAULT_SPDK_PORT=5260 + +function get_number_of_virtio_blk() { + cmd="lsblk --output \"NAME,VENDOR,SUBSYSTEMS\"" + out=$( send_command_over_unix_socket "${1}" "${cmd}" 1 ) + number_of_virio_blk_devices=$(echo "${out}" | grep -c "block:virtio:pci") + echo "${number_of_virio_blk_devices}" +} + +function is_virtio_blk_attached() { + number_of_virio_blk_devices=$(get_number_of_virtio_blk "${1}") + + if [[ "${number_of_virio_blk_devices}" == 0 ]]; then + echo "virtio-blk is not found" + return 1 + else + echo "virtio-blk is found" + return 0 + fi +} + +function is_virtio_blk_not_attached() { + if is_virtio_blk_attached "${1}"; then + return 1 + fi + + return 0 +} + +function check_number_of_virtio_blk_devices() { + vm_serial="${1}" + expected_number_of_devices="${2}" + number_of_devices=$(get_number_of_virtio_blk "${vm_serial}") + if [[ "${number_of_devices}" != "${expected_number_of_devices}" ]]; then + echo "Required number of devices '${expected_number_of_devices}' does" + echo "not equal to actual number of devices '${number_of_devices}'" + return 1 + else + echo "Number of attached virtio-blk devices is '${number_of_devices}'" + return 0 + fi +} + +function attach_virtio_blk() { + proxy_ip="${1}" + hot_plug_service_port="${2}" + vm_monitor="${3}" + virtio_blk_socket="${4}" + no_grpc_proxy="" grpc_cli call "${proxy_ip}":"${hot_plug_service_port}" HotPlugVirtioBlk \ + "vmId: '${vm_monitor}' vhostVirtioBlkId: '${virtio_blk_socket}'" + return "$?" +} + +function dettach_virtio_blk() { + proxy_ip="${1}" + hot_plug_service_port="${2}" + vm_monitor="${3}" + virtio_blk_socket="${4}" + no_grpc_proxy="" grpc_cli call "${proxy_ip}":"${hot_plug_service_port}" HotUnplugVirtioBlk \ + "vmId: '${vm_monitor}' vhostVirtioBlkId: '${virtio_blk_socket}'" + return $? +} + +function create_and_expose_sybsystem_over_tcp() { + ip_addr="${1}" + nqn="${2}" + storage_target_port="${3:-"$DEFAULT_SPDK_PORT"}" + + rpc.py -s "${ip_addr}" -p "$storage_target_port" \ + nvmf_create_subsystem "${nqn}" \ + -s SPDK00000000000001 -a + rpc.py -s "${ip_addr}" -p "$storage_target_port" \ + nvmf_create_transport -t TCP -u 8192 + rpc.py -s "${ip_addr}" -p "$storage_target_port" \ + nvmf_subsystem_add_listener "${nqn}" -t TCP \ + -f IPv4 -a "${ip_addr}" -s 4420 +} + +function create_ramdrive_and_attach_as_ns_to_subsystem() { + ip_addr="${1}" + ramdrive_name="${2}" + number_of_512b_blocks="${3}" + nqn="${4}" + storage_target_port="${5:-"$DEFAULT_SPDK_PORT"}" + + rpc.py -s "${ip_addr}" -p "${storage_target_port}" \ + bdev_malloc_create -b "${ramdrive_name}" \ + "${number_of_512b_blocks}" 512 + rpc.py -s "${ip_addr}" -p "${storage_target_port}" \ + nvmf_subsystem_add_ns "${nqn}" "${ramdrive_name}" +} + +function attach_controller() { + ip_addr="${1}" + storage_target_ip_addr="${2}" + nqn="${3}" + controller_name="${4}" + proxy_container_port="${5:-"$DEFAULT_SPDK_PORT"}" + + rpc.py -s "${ip_addr}" -p "$proxy_container_port" \ + bdev_nvme_attach_controller -b "${controller_name}" -t TCP \ + -f ipv4 -a "${storage_target_ip_addr}" -s 4420 -n "${nqn}" +} + +function create_disk() { + ip_addr="${1}" + vhost_path="${2}" + ns="${3}" + proxy_container_port="${4:-"$DEFAULT_SPDK_PORT"}" + rpc.py -s "${ip_addr}" -p "${proxy_container_port}" \ + vhost_create_blk_controller \ + "${vhost_path}" "${ns}" +} + +function attach_ns_as_virtio_blk() { + proxy_ip="${1}" + vhost_name="${2}" + exposed_controller_ns="${3}" + hot_plug_service_port="${4}" + vm_monitor="${5}" + proxy_container_port="${6:-"$DEFAULT_SPDK_PORT"}" + + create_disk "${proxy_ip}" \ + "/ipdk-shared/${vhost_name}" "${exposed_controller_ns}" \ + "${proxy_container_port}" + + attach_virtio_blk "${proxy_ip}" "${hot_plug_service_port}" \ + "${vm_monitor}" "${vhost_name}" +} + +function create_subsystem_and_expose_to_another_machine() { + storage_target_ip="${1}" + nqn="${2}" + proxy_ip="${3}" + controller_name="${4}" + storage_target_port="${5:-"$DEFAULT_SPDK_PORT"}" + proxy_container_port="${6:-"$DEFAULT_SPDK_PORT"}" + create_and_expose_sybsystem_over_tcp "${storage_target_ip}" "${nqn}" \ + "${storage_target_port}" + attach_controller "${proxy_ip}" "${storage_target_ip}" "${nqn}" \ + "${controller_name}" "${proxy_container_port}" +} diff --git a/build/storage/scripts/prepare_to_build.sh b/build/storage/scripts/prepare_to_build.sh new file mode 100755 index 00000000..3dbcdf79 --- /dev/null +++ b/build/storage/scripts/prepare_to_build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x + +git submodule update --init --recursive --force diff --git a/build/storage/scripts/run_host_target_container.sh b/build/storage/scripts/run_host_target_container.sh new file mode 100755 index 00000000..45f13734 --- /dev/null +++ b/build/storage/scripts/run_host_target_container.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x + +declare https_proxy +declare http_proxy +declare no_proxy +IP_ADDR="${IP_ADDR:-"0.0.0.0"}" +PORT="${PORT:-50051}" + +scripts_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +function print_error_and_exit() { + echo "${1}" + exit 1 +} + +function check_all_variables_are_set() { + number_re='^[0-9]+$' + ip_addr_re='^(([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))\.){3}([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))$' + + if [ -z "${IP_ADDR}" ]; then + print_error_and_exit "IP_ADDR is not set" + elif ! [[ "${IP_ADDR}" =~ ${ip_addr_re} ]]; then + print_error_and_exit "IP_ADDR does not represent ip address" + elif [ -z "${PORT}" ]; then + print_error_and_exit "PORT is not set" + elif ! [[ "${PORT}" =~ ${number_re} ]] ; then + print_error_and_exit "PORT is not a number" + fi +} + +check_all_variables_are_set + +if [[ $(docker images --filter=reference='host-target' -q) == "" ]]; then + bash "${scripts_dir}"/build_container.sh host-target +fi + +docker_run="docker run \ + -it \ + --privileged \ + -e IP_ADDR=${IP_ADDR} \ + -e PORT=${PORT} \ + -e DEBUG=${DEBUG} \ + -e HTTPS_PROXY=${https_proxy} \ + -e HTTP_PROXY=${http_proxy} \ + -e NO_PROXY=${no_proxy} \ + --network host \ + -v /dev:/dev \ + host-target" +$docker_run + + diff --git a/build/storage/scripts/run_proxy_container.sh b/build/storage/scripts/run_proxy_container.sh new file mode 100755 index 00000000..6b4b4ba9 --- /dev/null +++ b/build/storage/scripts/run_proxy_container.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + + +[ "$DEBUG" == 'true' ] && set -x + +declare https_proxy +declare http_proxy +declare no_proxy +SHARED_VOLUME=${SHARED_VOLUME:-$(realpath .)} +SPDK_IP_ADDR="${SPDK_IP_ADDR:-"0.0.0.0"}" +SPDK_PORT="${SPDK_PORT:-5260}" +HOT_PLUG_SERVICE_IP_ADDR="${HOT_PLUG_SERVICE_IP_ADDR:-"0.0.0.0"}" +HOT_PLUG_SERVICE_PORT="${HOT_PLUG_SERVICE_PORT:-50051}" + +scripts_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +function print_error_and_exit() { + echo "${1}" + exit 1 +} + +function check_all_variables_are_set() { + number_re='^[0-9]+$' + ip_addr_re='^(([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))\.){3}([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))$' + + if [ -z "${SHARED_VOLUME}" ]; then + print_error_and_exit "SHARED_VOLUME is not defined" + elif [ ! -d "${SHARED_VOLUME}" ]; then + print_error_and_exit "SHARED_VOLUME is not a directory" + elif [ -z "${HOT_PLUG_SERVICE_IP_ADDR}" ]; then + print_error_and_exit "HOT_PLUG_SERVICE_IP_ADDR is not set" + elif ! [[ "${HOT_PLUG_SERVICE_IP_ADDR}" =~ ${ip_addr_re} ]]; then + print_error_and_exit "HOT_PLUG_SERVICE_IP_ADDR does not represent ip address" + elif [ -z "${HOT_PLUG_SERVICE_PORT}" ]; then + print_error_and_exit "PORT is not set" + elif ! [[ "${HOT_PLUG_SERVICE_PORT}" =~ ${number_re} ]] ; then + print_error_and_exit "PORT is not a number" + elif [ -z "${SPDK_PORT}" ]; then + print_error_and_exit "SPDK_PORT is not set" + elif ! [[ "${SPDK_PORT}" =~ ${number_re} ]] ; then + print_error_and_exit "SPDK_PORT is not a number" + elif [ -z "${SPDK_IP_ADDR}" ]; then + print_error_and_exit "SPDK_IP_ADDR is not set" + elif ! [[ "${SPDK_IP_ADDR}" =~ ${ip_addr_re} ]]; then + print_error_and_exit "SPDK_IP_ADDR does not represent ip address" + fi +} + +check_all_variables_are_set +SHARED_VOLUME=$(realpath "${SHARED_VOLUME}") + +bash "${scripts_dir}"/build_container.sh proxy-container +bash "${scripts_dir}"/allocate_hugepages.sh + +config_file_option= +if [[ -n "${SPDK_CONFIG_FILE}" ]]; then + SPDK_CONFIG_FILE=$(realpath "${SPDK_CONFIG_FILE}") + config_file_option="-v ${SPDK_CONFIG_FILE}:/config" +fi + +docker_run="docker run \ + -it \ + --privileged \ + -v /dev/hugepages:/dev/hugepages \ + -v ${SHARED_VOLUME}:/ipdk-shared \ + ${config_file_option} \ + -e SPDK_IP_ADDR=${SPDK_IP_ADDR} \ + -e SPDK_PORT=${SPDK_PORT} \ + -e HOT_PLUG_SERVICE_IP_ADDR=${HOT_PLUG_SERVICE_IP_ADDR} \ + -e HOT_PLUG_SERVICE_PORT=${HOT_PLUG_SERVICE_PORT} \ + -e HOST_SHARED_VOLUME=${SHARED_VOLUME} \ + -e DEBUG=${DEBUG} \ + -e HTTPS_PROXY=${https_proxy} \ + -e HTTP_PROXY=${http_proxy} \ + -e NO_PROXY=${no_proxy} \ + --network host \ + proxy-container" +$docker_run diff --git a/build/storage/scripts/run_storage_target_container.sh b/build/storage/scripts/run_storage_target_container.sh new file mode 100755 index 00000000..c6820e91 --- /dev/null +++ b/build/storage/scripts/run_storage_target_container.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x + +declare https_proxy +declare http_proxy +declare no_proxy +SPDK_CONFIG_FILE="${SPDK_CONFIG_FILE:-}" +SPDK_IP_ADDR="${SPDK_IP_ADDR:-"0.0.0.0"}" +SPDK_PORT="${SPDK_PORT:-5260}" + +scripts_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +function print_error_and_exit() { + echo "${1}" + exit 1 +} + +function check_all_variables_are_set() { + number_re='^[0-9]+$' + ip_addr_re='^(([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))\.){3}([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))$' + + if [ -z "${SPDK_IP_ADDR}" ]; then + print_error_and_exit "SPDK_IP_ADDR is not set" + elif ! [[ "${SPDK_IP_ADDR}" =~ ${ip_addr_re} ]]; then + print_error_and_exit "SPDK_IP_ADDR does not represent ip address" + elif [ -z "${SPDK_PORT}" ]; then + print_error_and_exit "SPDK_PORT is not set" + elif ! [[ "${SPDK_PORT}" =~ ${number_re} ]] ; then + print_error_and_exit "SPDK_PORT is not a number" + fi +} + +check_all_variables_are_set +SPDK_CONFIG_FILE=$(realpath "${SPDK_CONFIG_FILE}") + +bash "${scripts_dir}"/build_container.sh storage-target +bash "${scripts_dir}"/allocate_hugepages.sh + +config_file_option= +if [[ -n "${SPDK_CONFIG_FILE}" ]]; then + SPDK_CONFIG_FILE=$(realpath "${SPDK_CONFIG_FILE}") + config_file_option="-v ${SPDK_CONFIG_FILE}:/config" +fi + +docker_run="docker run \ + -it \ + --privileged \ + -v /dev/hugepages:/dev/hugepages \ + ${config_file_option} \ + -e SPDK_IP_ADDR=${SPDK_IP_ADDR} \ + -e SPDK_PORT=${SPDK_PORT} \ + -e DEBUG=${DEBUG} \ + -e HTTPS_PROXY=${https_proxy} \ + -e HTTP_PROXY=${http_proxy} \ + -e NO_PROXY=${no_proxy} \ + --network host \ + storage-target" + +$docker_run diff --git a/build/storage/scripts/socket.sh b/build/storage/scripts/socket.sh new file mode 100755 index 00000000..73be4f39 --- /dev/null +++ b/build/storage/scripts/socket.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +function send_command_over_unix_socket() { + socket="${1}" + cmd="${2}" + wait_for_secs="${3}" + echo "${cmd}" | socat -T"${wait_for_secs}" -,ignoreeof unix-connect:"${socket}" +} + +function send_command_over_unix_socket_and_no_word_found() { + out=$( send_command_over_unix_socket "${1}" "${2}" "${3}" ) + + searched_word="${4}" + result=$(echo "${out}" | grep -i -c "${searched_word}") + return "${result}" +} + +function get_output_from_unix_socket() { + socket="${1}" + wait_for_secs="${2}" + out=$( socat -T"${wait_for_secs}" -,ignoreeof unix-connect:"${socket}" ) + result=$? + echo "${out}" +} diff --git a/build/storage/scripts/spdk_version.sh b/build/storage/scripts/spdk_version.sh new file mode 100755 index 00000000..e2c00682 --- /dev/null +++ b/build/storage/scripts/spdk_version.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +function get_spdk_version() { + local script_dir root_dir spdk_dir spdk_version + script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + root_dir=${script_dir}/.. + spdk_dir="${root_dir}/spdk" + spdk_version=$(git -C "${spdk_dir}" describe --tags --abbrev=0) + echo "${spdk_version}" +} diff --git a/build/storage/scripts/vm/create_nat_for_vm.sh b/build/storage/scripts/vm/create_nat_for_vm.sh new file mode 100755 index 00000000..6b987eb9 --- /dev/null +++ b/build/storage/scripts/vm/create_nat_for_vm.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# This script allows to access vm ports from the host + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +declare net_mask_bits +declare gateway +declare dhcp_range +declare bridge +# shellcheck disable=SC1091 +source "${current_script_dir}"/nat_variables.sh + +tap_device="${1}" + +bridge_does_not_exist() { + bridge_name="${1}" + if bridge vlan show | grep "^${bridge_name}" &> /dev/null; then + return 1 + else + return 0 + fi +} + +if bridge_does_not_exist "${bridge}"; then + echo "1" | tee /proc/sys/net/ipv4/ip_forward + + ip link add "${bridge}" type bridge + ip addr add "${gateway}/${net_mask_bits}" dev "${bridge}" + ip link set "${bridge}" up + + dnsmasq \ + --interface="${bridge}" \ + --dhcp-range="${dhcp_range}" \ + --listen-address="${gateway}" \ + --pid-file="/var/run/qemu-dnsmasq-${bridge}.pid" \ + --dhcp-leasefile="/var/run/qemu-dnsmasq-${bridge}.leases" \ + --strict-order \ + --except-interface=lo \ + --bind-interfaces \ + --dhcp-no-override + +fi + +if test "${tap_device}" ; then + ip addr add 0.0.0.0 dev "${tap_device}" + ip link set "${tap_device}" up + ip link set "${tap_device}" master "${bridge}" +fi diff --git a/build/storage/scripts/vm/delete_nat_for_vm.sh b/build/storage/scripts/vm/delete_nat_for_vm.sh new file mode 100755 index 00000000..c156ad05 --- /dev/null +++ b/build/storage/scripts/vm/delete_nat_for_vm.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +declare bridge +# shellcheck disable=SC1091 +source "${current_script_dir}"/nat_variables.sh + +ip link set "${bridge}" down +ip link delete "${bridge}" type bridge + +kill -15 "$(cat "/var/run/qemu-dnsmasq-${bridge}.pid")" + diff --git a/build/storage/scripts/vm/nat_variables.sh b/build/storage/scripts/vm/nat_variables.sh new file mode 100755 index 00000000..e508afc9 --- /dev/null +++ b/build/storage/scripts/vm/nat_variables.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +export network=192.168.32.0 +export net_mask_bits=24 +export gateway=192.168.32.1 +export dhcp_range=192.168.32.2,192.168.32.254 +export bridge=br-ipdk-0 + diff --git a/build/storage/scripts/vm/prepare_vm.sh b/build/storage/scripts/vm/prepare_vm.sh new file mode 100755 index 00000000..b99f550b --- /dev/null +++ b/build/storage/scripts/vm/prepare_vm.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x +set -e + +scripts_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)/.. + +if [[ $# != 1 ]] ; then + echo "Directory to place vm file has to be specified" + exit 1 +fi + +export LIBGUESTFS_BACKEND=direct + +path_to_place_vm_file=${1} +vm_file=${path_to_place_vm_file}/vm.qcow2 + +if [ ! -f "${vm_file}" ]; then + vm_tmp_file="${path_to_place_vm_file}/vm_original.qcow2" + wget -O "${vm_tmp_file}" https://download.fedoraproject.org/pub/fedora/linux/\ +releases/33/Cloud/x86_64/images/Fedora-Cloud-Base-33-1.2.x86_64.qcow2 + virt-customize -a "${vm_tmp_file}" \ + --memsize 1260 \ + --root-password password:root \ + --install fio + + if [ "${WITHOUT_HOST_TARGET}" == "true" ]; then + echo "Skip Host-Target container installation" + else + "${scripts_dir}"/build_container.sh host-target + host_target_container="${path_to_place_vm_file}/host-target.tar" + docker save -o "${host_target_container}" host-target + run_customize=(virt-customize -a "${vm_tmp_file}") + run_customize+=(--run-command 'dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo') + run_customize+=(--run-command 'grubby --update-kernel ALL --args selinux=0') + run_customize+=(--install "dnf-plugins-core,docker-ce,docker-ce-cli,containerd.io") + run_customize+=(--copy-in "${path_to_place_vm_file}/host-target.tar:/") + run_customize+=(--copy-in "${scripts_dir}/run_host_target_container.sh:/usr/local/bin") + run_customize+=(--run-command 'systemctl enable docker.service') + run_customize+=(--run-command 'service docker start') + run_customize+=(--firstboot-command 'docker load --input /host-target.tar') + run_customize+=(--firstboot-command 'rm -f /host-target.tar') + "${run_customize[@]}" + rm -f "${host_target_container}" + fi + mv "${vm_tmp_file}" "${vm_file}" +fi diff --git a/build/storage/scripts/vm/run_vm.sh b/build/storage/scripts/vm/run_vm.sh new file mode 100755 index 00000000..aa710523 --- /dev/null +++ b/build/storage/scripts/vm/run_vm.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x + +scripts_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)/.. + +SHARED_VOLUME=${SHARED_VOLUME:-.} + +bash "${scripts_dir}"/vm/prepare_vm.sh "${SHARED_VOLUME}" +bash "${scripts_dir}"/allocate_hugepages.sh + +run_vm="sudo qemu-system-x86_64 \ + --enable-kvm \ + -cpu host \ + -m 1G -object memory-backend-file,id=mem0,size=1G,mem-path=/dev/hugepages,share=on -numa node,memdev=mem0 \ + -smp 2 \ + -drive file=${SHARED_VOLUME}/vm.qcow2,if=none,id=disk \ + -device ide-hd,drive=disk,bootindex=0 \ + -net nic -net tap,script=${scripts_dir}/vm/create_nat_for_vm.sh,\ +downscript=${scripts_dir}/vm/delete_nat_for_vm.sh \ + -monitor unix:${SHARED_VOLUME}/vm_monitor,server,nowait \ + --nographic" + +$run_vm \ No newline at end of file diff --git a/build/storage/spdk b/build/storage/spdk new file mode 160000 index 00000000..4e4f11ff --- /dev/null +++ b/build/storage/spdk @@ -0,0 +1 @@ +Subproject commit 4e4f11ff7840f02cb6d9497acd42897844a43f87 diff --git a/build/storage/tests/README.md b/build/storage/tests/README.md deleted file mode 100644 index 5d8a1c54..00000000 --- a/build/storage/tests/README.md +++ /dev/null @@ -1,132 +0,0 @@ -# Introduction -This recipe show-cases hot-plugging virtio-blk device into a VM and running -fio traffic from that VM using virtio-blk interface to a storage target -using NVMe/TCP protocol. - -The scenario is realized using docker-compose in a single host environment. -The figure below illustrates the design: - -![Figure 1 - not found](img_virtio_blk_over_nvmetcp_1.svg "Running virtio-blk traffic over NVMeTCP") - -There are three basic containers: -- _traffic-generator_ which plays the role of a "host" and runs the VM -- _proxy-container_ which plays the role of an IPU and exposes the block device -- _storage-target_ which plays the role of a remote storage node - -In addition, there is a _test-drivers_ container used to exercise the scenario by running -dedicated tests. - -# System setup -The basic requirement for running the recipe is a modern Linux distribution with docker support. -So far, the recipe has been successfully tested on the following systems: -- Fedora 33 -- Ubuntu 18.04 - -To run the recipe some basic system preparations are required, in particular: - -## Virtualization support -Make sure that VT-x/AMD-v support is enabled in BIOS -``` -$ lscpu | grep -i virtualization -Virtualization: VT-x -``` -and that kvm modules are loaded -``` -$ lsmod | grep -i kvm -kvm_intel 217088 0 -kvm 614400 1 kvm_intel -irqbypass 16384 1 kvm -``` - -## Tool installation -Make sure that following tools are installed on your system or install them -using the corresponding package manager. - -### wget -Installation on Fedora -``` -$ sudo dnf install wget -``` -or on Ubuntu -``` -$ sudo apt install wget -``` - -### docker -``` -$ sudo dnf install docker -``` -or -``` -$ sudo apt install docker -``` -**Note:** -Make sure that required proxy settings are configured for docker. -Please, refer to [this](https://docs.docker.com/config/daemon/systemd/#httphttps-proxy) page. - -### docker compose -``` -$ sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose -$ sudo chmod +x /usr/local/bin/docker-compose -$ sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose -``` - -### libguestfs-tools -``` -$ sudo dnf install libguestfs-tools-c -``` -or -``` -$ sudo apt install libguestfs-tools -``` - -**Note:** -To run `libguestfs` tools without root privileges, you may need to workaround the problem of -Linux kernel image not being readable by issuing: -``` -$ sudo chmod +r /boot/vmlinuz-* -``` - -## Setting security policies -Make sure that required security permissions are configured to enable VM network connectivity -or disable them temporarily. -On Fedora SELinux can be disabled by means of the following command -``` -$ sudo setenforce 0 -``` -For Ubuntu AppArmor is often used and can be disabled by issuing -``` -$ sudo systemctl stop apparmor -``` - -# Running the tests -In order to run all defined tests -``` -$ ./run.sh -``` -or if there is a proxy -``` -$ https_proxy= \ -http_proxy= \ -no_proxy= \ -./run.sh -``` - -To run a specific test -``` -$ ./run.sh -``` -for example -``` -$ ./run.sh hot-plug -$ # or -$ ./run.sh fio -``` -The script `run.sh` returns a non-zero value only in case of an error. - -**Note:** -The script `run.sh` will download a `Fedora 33` image into `traffic-generator` -directory and set up login-password pair as root-root if there is no `vm.qcow2` -image provided in that directory. `run.sh` will also try to allocate 2048 2MB -hugepages if not yet allocated and it will request administrative privileges -to execute this operation. diff --git a/build/storage/tests/img_virtio_blk_over_nvmetcp_1.svg b/build/storage/tests/img_virtio_blk_over_nvmetcp_1.svg deleted file mode 100644 index 14573bd7..00000000 --- a/build/storage/tests/img_virtio_blk_over_nvmetcp_1.svg +++ /dev/null @@ -1,184 +0,0 @@ - - - - - - - - - - - - - - - - VBackground-1 - - Solid - - - - - Page-1 - - Round Corner Rectangle.37 - IPDK docker-compose files - - IPDK docker-compose files - - Round Corner Rectangle.4 - traffic-generator - - traffic-generator - - Round Corner Rectangle.3 - QEMU - - QEMU - - Round Corner Rectangle - fio - - fio - - Round Corner Rectangle.5 - proxy-container - - proxy-container - - Round Corner Rectangle.6 - storage-target - - storage-target - - Round Corner Rectangle.7 - test-drivers - - test-drivers - - Sheet.8 - vhost/virtio-blk - - vhost/virtio-blk - - Sheet.9 - qemu monitor - - qemu monitor - - Sheet.10 - qemu serial - - qemu serial - - Sheet.11 - RUN ./init - - RUN ./init - - Sheet.12 - init.fio script - - init.fio script - - Sheet.13 - init.hot-plug script - - init.hot-plug script - - Sheet.14 - other tests - - other tests - - Dynamic connector - - - - Dynamic connector.16 - - - - Dynamic connector.18 - - - - Dynamic connector.19 - - - - Dynamic connector.20 - - - - Dynamic connector.21 - - - - Dynamic connector.22 - - - - Sheet.23 - NVMe/TCP - - NVMe/TCP - - Round Corner Rectangle.24 - - - - Sheet.25 - - - - Sheet.26 - containers - - containers - - Sheet.27 - shared objects/ docker volumes - - shared objects/ docker volumes - - Dynamic connector.29 - - - - Sheet.31 - init script return value - - init script return value - - diff --git a/build/storage/tests/it/README.md b/build/storage/tests/it/README.md new file mode 100644 index 00000000..5eb1a0d0 --- /dev/null +++ b/build/storage/tests/it/README.md @@ -0,0 +1,63 @@ +# Introduction +These tests covers the recipes described in [recipes](../../recipes/README.md) +directory. + +In addition to [the main IPDK images](../../README.md#main-components), +the following images are introduced for testing purposes: +- _traffic-generator_ which has a vm instance on board exposing interfaces for +access to the vm instance. +- _test-driver_ container used to exercise the scenario by running +dedicated tests(apply a specific storage configuration, check if hot-plugged +devices are visible on host, run traffic from host and etc.). + +All containers are run in the isolated docker compose environment described by +`docker-compose` files. It allows to configure the environment to run tests in +a flexible way. +The figure below illustrates the overall testing design: +![Running virtio-blk traffic over NVMeTCP](img_virtio_blk_over_nvmetcp_1.png "Running virtio-blk traffic over NVMeTCP") + +# Test environment preparation +Please make sure that steps described in +[System setup](../../recipes/environment_setup.md#system-setup) +section are performed on testing platform. + +For testing environment additionally the steps bellow should be applied: + +## docker compose setup +``` +$ sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +$ sudo chmod +x /usr/local/bin/docker-compose +$ sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose +``` + +# Running the tests +In order to run all defined tests +``` +$ ./run.sh +``` +or if there is a proxy +``` +$ https_proxy= \ +http_proxy= \ +no_proxy= \ +./run.sh +``` + +To run a specific test +``` +$ ./run.sh +``` +for example +``` +$ ./run.sh hot-plug +$ # or +$ ./run.sh fio +``` +The script `run.sh` returns a non-zero value only in case of an error. + +**Note:** +The script `run.sh` will download a `Fedora 36` image into `traffic-generator` +directory and set up login-password pair as root-root if there is no `vm.qcow2` +image provided in that directory. `run.sh` will also try to allocate 2048 2MB +hugepages if not yet allocated and it will request administrative privileges +to execute this operation. \ No newline at end of file diff --git a/build/storage/tests/docker-compose.yml b/build/storage/tests/it/docker-compose.yml similarity index 59% rename from build/storage/tests/docker-compose.yml rename to build/storage/tests/it/docker-compose.yml index 9cc06991..d48b1cad 100644 --- a/build/storage/tests/docker-compose.yml +++ b/build/storage/tests/it/docker-compose.yml @@ -5,73 +5,68 @@ # WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK # version: "3.8" -x-proxy-args: - &proxy-args - args: - - HTTP_PROXY=${HTTP_PROXY} - - HTTPS_PROXY=${HTTPS_PROXY} - - NO_PROXY=${NO_PROXY} - -x-debug-env-var: &debug-env-var DEBUG=${DEBUG} x-shared-volume: &shared-volume ipdk-shared:/ipdk-shared +x-debug-env-var: &debug-env-var DEBUG=${DEBUG} + services: - build_base: - image: spdk - build: - context: ../core/build_base - <<: *proxy-args - container_name: build_base - environment: - - *debug-env-var storage-target: image: spdk-app build: - context: ../core/spdk-app - <<: *proxy-args + context: ../.. + target: spdk-app + args: + - HTTP_PROXY=${HTTP_PROXY} + - HTTPS_PROXY=${HTTPS_PROXY} + - NO_PROXY=${NO_PROXY} + - SPDK_VERSION=${SPDK_VERSION} container_name: storage-target - depends_on: - - build_base networks: ipdk: ipv4_address: 192.168.42.2 volumes: - /dev/hugepages:/dev/hugepages - - ./storage-target.conf:/config environment: - - *debug-env-var - SPDK_ARGS=-m 0x2 + - *debug-env-var privileged: true proxy-container: - image: spdk-app + image: proxy-container build: - context: ../core/spdk-app - <<: *proxy-args + context: ../.. + target: proxy-container + args: + - HTTP_PROXY=${HTTP_PROXY} + - HTTPS_PROXY=${HTTPS_PROXY} + - NO_PROXY=${NO_PROXY} + - SPDK_VERSION=${SPDK_VERSION} container_name: proxy-container - depends_on: - - storage-target networks: ipdk: ipv4_address: 192.168.42.3 volumes: - /dev/hugepages:/dev/hugepages - - ./proxy-container.conf:/config - *shared-volume environment: - - SPDK_ARGS=-m 0x4 -S /ipdk-shared + - SPDK_ARGS=-m 0x4 - *debug-env-var + - HOST_SHARED_VOLUME=/ipdk-shared privileged: true traffic-generator: image: traffic-generator build: - context: ./traffic-generator - <<: *proxy-args + context: ../.. + target: traffic-generator + dockerfile: tests/it/traffic-generator/Dockerfile + args: + - HTTP_PROXY=${HTTP_PROXY} + - HTTPS_PROXY=${HTTPS_PROXY} + - NO_PROXY=${NO_PROXY} + - SPDK_VERSION=${SPDK_VERSION} container_name: traffic-generator depends_on: - proxy-container - networks: - ipdk: volumes: - /dev/hugepages:/dev/hugepages - *shared-volume @@ -81,14 +76,22 @@ services: test-driver: image: test-driver build: - context: ./test-drivers - <<: *proxy-args + context: ../.. + target: test-driver + dockerfile: tests/it/test-drivers/Dockerfile + args: + - HTTP_PROXY=${HTTP_PROXY} + - HTTPS_PROXY=${HTTPS_PROXY} + - NO_PROXY=${NO_PROXY} + - SPDK_VERSION=${SPDK_VERSION} container_name: test-driver depends_on: - traffic-generator volumes: - *shared-volume - - ./test-drivers/test-helpers:/test-helpers + networks: + ipdk: + ipv4_address: 192.168.42.4 environment: - *debug-env-var privileged: true diff --git a/build/storage/tests/it/img_virtio_blk_over_nvmetcp_1.png b/build/storage/tests/it/img_virtio_blk_over_nvmetcp_1.png new file mode 100644 index 00000000..88368664 Binary files /dev/null and b/build/storage/tests/it/img_virtio_blk_over_nvmetcp_1.png differ diff --git a/build/storage/tests/it/run.sh b/build/storage/tests/it/run.sh new file mode 100755 index 00000000..4fd8e4c9 --- /dev/null +++ b/build/storage/tests/it/run.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set -e +[ "$DEBUG" == 'true' ] && set -x + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +root_dir="${current_script_dir}"/../.. +scripts_dir="${root_dir}"/scripts +# shellcheck disable=SC1091 +source "${scripts_dir}"/spdk_version.sh +bash "${scripts_dir}"/prepare_to_build.sh +declare https_proxy +declare http_proxy +declare no_proxy +export HTTPS_PROXY=${https_proxy} +export HTTP_PROXY=${http_proxy} +export NO_PROXY=${no_proxy} +spdk_version=$(get_spdk_version) +export SPDK_VERSION="${spdk_version}" +export DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} +export COMPOSE_DOCKER_CLI_BUILD=${COMPOSE_DOCKER_CLI_BUILD:-1} + +function run_test() { + sudo_for_docker="sudo" + is_user_docker_group_member=$(groups | grep docker &> /dev/null ; echo $?) + if [ "${is_user_docker_group_member}" == "0" ]; then + sudo_for_docker= + elif [[ $(whoami) == "root" ]]; then + sudo_for_docker= + fi + if [ "${DEBUG}" == 'true' ]; then + export BUILDKIT_PROGRESS=plain + fi + + ${sudo_for_docker} docker-compose \ + -f "${current_script_dir}/docker-compose.yml" \ + -f "${current_script_dir}/test-drivers/docker-compose.$1.yml" \ + up \ + --build \ + --exit-code-from test-driver \ + --scale build_base=0 +} + +function provide_hugepages() { + bash "${scripts_dir}"/allocate_hugepages.sh +} + +provide_hugepages + +test_cases=(hot-plug fio) +if [[ $# != 0 ]]; then + run_test "${1}" +else + for i in "${test_cases[@]}"; do + run_test "${i}" + done +fi diff --git a/build/storage/tests/test-drivers/Dockerfile b/build/storage/tests/it/test-drivers/Dockerfile similarity index 74% rename from build/storage/tests/test-drivers/Dockerfile rename to build/storage/tests/it/test-drivers/Dockerfile index 41b0cc26..e3c24ade 100644 --- a/build/storage/tests/test-drivers/Dockerfile +++ b/build/storage/tests/it/test-drivers/Dockerfile @@ -4,8 +4,7 @@ # NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE # WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK # - -FROM spdk +FROM spdk-app AS test-driver # Generic args ARG HTTP_PROXY @@ -17,5 +16,9 @@ ENV https_proxy=$HTTPS_PROXY ENV no_proxy=$NO_PROXY RUN dnf install -y socat +RUN dnf install -y grpc-cli + +COPY tests/it/test-drivers/test-helpers /test-helpers +COPY scripts/ /scripts ENTRYPOINT ["/init"] diff --git a/build/storage/tests/test-drivers/docker-compose.fio.yml b/build/storage/tests/it/test-drivers/docker-compose.fio.yml similarity index 100% rename from build/storage/tests/test-drivers/docker-compose.fio.yml rename to build/storage/tests/it/test-drivers/docker-compose.fio.yml diff --git a/build/storage/tests/test-drivers/docker-compose.hot-plug.yml b/build/storage/tests/it/test-drivers/docker-compose.hot-plug.yml similarity index 100% rename from build/storage/tests/test-drivers/docker-compose.hot-plug.yml rename to build/storage/tests/it/test-drivers/docker-compose.hot-plug.yml diff --git a/build/storage/tests/it/test-drivers/init.fio b/build/storage/tests/it/test-drivers/init.fio new file mode 100755 index 00000000..316fb630 --- /dev/null +++ b/build/storage/tests/it/test-drivers/init.fio @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set -e +[ "$DEBUG" == 'true' ] && set -x + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +declare vm_serial +declare nqn +declare storage_target_ip +declare proxy_ip +# shellcheck disable=SC1091 +source "${current_script_dir}"/test-helpers + +[ -S "${vm_serial}" ] && rm "${vm_serial}" + +controller_name=Nvme0 +create_subsystem_and_expose_to_another_machine \ + "${storage_target_ip}" "${nqn}" "${proxy_ip}" "${controller_name}" +create_ramdrive_and_attach_as_ns_to_subsystem \ + "${storage_target_ip}" Malloc0 16 "${nqn}" +create_disk "${proxy_ip}" \ + /ipdk-shared/VirtioBlk0 "${controller_name}n1" + +wait_until_vm_is_up "${vm_serial}" +log_in_with_default_credentials "${vm_serial}" + +echo "check attachment" +is_virtio_blk_attached "${vm_serial}" + +echo "Run fio" +fio_cmd="fio --filename=/dev/vda --direct=1 --rw=randrw \ + --bs=4k --ioengine=libaio --iodepth=256 --runtime=10 \ + --numjobs=4 --time_based --group_reporting --name=iops-test-job" + +out=$(send_command_over_unix_socket "${vm_serial}" "${fio_cmd}" 11) +echo "${out}" + +echo "${out}" | grep "Disk stats (read/write)" +echo "fio has been executed successfully!" + +exit 0 diff --git a/build/storage/tests/it/test-drivers/init.hot-plug b/build/storage/tests/it/test-drivers/init.hot-plug new file mode 100755 index 00000000..ccd66abd --- /dev/null +++ b/build/storage/tests/it/test-drivers/init.hot-plug @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set -e +[ "$DEBUG" == 'true' ] && set -x + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +declare vm_serial +declare nqn +declare storage_target_ip +declare proxy_ip +# shellcheck disable=SC1091 +source "${current_script_dir}"/test-helpers + +controller_name=Nvme0 +create_subsystem_and_expose_to_another_machine \ + "${storage_target_ip}" "${nqn}" "${proxy_ip}" "${controller_name}" + +number_of_512b_blocks=16 +create_ramdrive_and_attach_as_ns_to_subsystem \ + "${storage_target_ip}" Malloc0 "${number_of_512b_blocks}" "${nqn}" +create_ramdrive_and_attach_as_ns_to_subsystem \ + "${storage_target_ip}" Malloc1 "${number_of_512b_blocks}" "${nqn}" +create_ramdrive_and_attach_as_ns_to_subsystem \ + "${storage_target_ip}" Malloc2 "${number_of_512b_blocks}" "${nqn}" + +HOT_PLUG_SERVICE_PORT="${HOT_PLUG_SERVICE_PORT:-50051}" + +wait_until_vm_is_up "${vm_serial}" +log_in_with_default_credentials "${vm_serial}" + +is_virtio_blk_not_attached "${vm_serial}" + +vm_monitor=vm_monitor_socket +virtio_blk_0_socket=VirtioBlk0 +echo "" +echo "### Attach 3 virtio-blk devices ###" + +attach_ns_as_virtio_blk \ + "${proxy_ip}" VirtioBlk0 "${controller_name}n1" \ + "${HOT_PLUG_SERVICE_PORT}" "${vm_monitor}" +check_number_of_virtio_blk_devices "${vm_serial}" 1 + +attach_ns_as_virtio_blk \ + "${proxy_ip}" VirtioBlk1 "${controller_name}n2" \ + "${HOT_PLUG_SERVICE_PORT}" "${vm_monitor}" +check_number_of_virtio_blk_devices "${vm_serial}" 2 + +attach_ns_as_virtio_blk \ + "${proxy_ip}" VirtioBlk2 "${controller_name}n3" \ + "${HOT_PLUG_SERVICE_PORT}" "${vm_monitor}" +check_number_of_virtio_blk_devices "${vm_serial}" 3 + +echo "" +echo "### Dettach 1st virtio-blk ###" +dettach_virtio_blk "${proxy_ip}" "${HOT_PLUG_SERVICE_PORT}" \ + "${vm_monitor}" "${virtio_blk_0_socket}" +check_number_of_virtio_blk_devices "${vm_serial}" 2 + +echo "" +echo "### Attach 1st virtio-blk back ###" +attach_virtio_blk "${proxy_ip}" "${HOT_PLUG_SERVICE_PORT}" \ + "${vm_monitor}" "${virtio_blk_0_socket}" +check_number_of_virtio_blk_devices "${vm_serial}" 3 + +exit 0 diff --git a/build/storage/tests/it/test-drivers/test-helpers b/build/storage/tests/it/test-drivers/test-helpers new file mode 100644 index 00000000..a7fbda19 --- /dev/null +++ b/build/storage/tests/it/test-drivers/test-helpers @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG_VM" == 'true' ] && export GRPC_VERBOSITY=DEBUG + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +root_dir=${current_script_dir}/../.. +scripts_dir=${root_dir}/scripts + +# shellcheck disable=SC1091 +source "${scripts_dir}"/socket.sh +# shellcheck disable=SC1091 +source "${scripts_dir}"/disk_infrastructure.sh + +function wait_until_vm_is_up() { + console="${1}" + + [ "$DEBUG_VM" == 'true' ] && echo "Start waiting for VM... and console ${console}" + + overall_wait_counter=0 + overall_max_wait_counter=12 + wait_period_sec=10 + + while [ ! -S "${console}" ] && \ + [ "${overall_wait_counter}" -le "${overall_max_wait_counter}" ] ; do + sec=$(( overall_wait_counter * wait_period_sec )) + [ "$DEBUG_VM" == 'true' ] && echo "Waiting for VM console: ${console}, sec ${sec} ..." + sleep "${wait_period_sec}" + overall_wait_counter=$(( overall_wait_counter + 1 )) + done + + sleep 4 + + socket_output_wait_counter=0 + while [ "$socket_output_wait_counter" -le 2 ] && \ + [ "$overall_wait_counter" -le "${overall_max_wait_counter}" ] ; do + sec=$(( overall_wait_counter * wait_period_sec )) + [ "$DEBUG_VM" == 'true' ] && echo "Waiting for VM completes booting, sec ${sec} ..." + overall_wait_counter=$(( overall_wait_counter + 1 )) + out=$( get_output_from_unix_socket "${console}" ${wait_period_sec} ) + [ "$DEBUG_VM" == 'true' ] && echo "${out}" + socket_output_wait_counter=$(( socket_output_wait_counter + 1 )) + if [[ -n "${out}" ]] ; then + socket_output_wait_counter=0 + fi + done + return 0 +} + +function log_in() { + out=$( send_command_over_unix_socket "${1}" "${2}" 3 ) + [ "$DEBUG_VM" == 'true' ] && echo "${out}" + out=$( send_command_over_unix_socket "${1}" "${3}" 3 ) + [ "$DEBUG_VM" == 'true' ] && echo "${out}" + return 0 +} + +function log_in_with_default_credentials() { + log_in "${1}" "root" "root" +} + +function hostname_to_ip() { + getent hosts "${1}" | awk '{ print $1 }' +} + +storage_target_ip=$(hostname_to_ip "storage-target") +export storage_target_ip="${storage_target_ip}" +proxy_ip=$(hostname_to_ip "proxy-container") +export proxy_ip="${proxy_ip}" +export nqn="nqn.2016-06.io.spdk:cnode1" +export shared_volume=/ipdk-shared +export vm_serial=${shared_volume}/vm_socket +export vm_monitor=${shared_volume}/vm_monitor_socket diff --git a/build/storage/tests/it/traffic-generator/Dockerfile b/build/storage/tests/it/traffic-generator/Dockerfile new file mode 100644 index 00000000..23b9ad27 --- /dev/null +++ b/build/storage/tests/it/traffic-generator/Dockerfile @@ -0,0 +1,40 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE +# WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK +# +FROM fedora:36 as traffic-generator-base + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +ENV http_proxy=$HTTP_PROXY +ENV https_proxy=$HTTPS_PROXY +ENV no_proxy=$NO_PROXY + +RUN dnf install -y wget +RUN dnf install -y libguestfs-tools-c + +COPY scripts/* /scripts/ +COPY scripts/vm/* /scripts/vm/ +RUN WITHOUT_HOST_TARGET=true /scripts/vm/prepare_vm.sh / + +FROM fedora:36 AS traffic-generator + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +ENV http_proxy=$HTTP_PROXY +ENV https_proxy=$HTTPS_PROXY +ENV no_proxy=$NO_PROXY + +RUN dnf install -y qemu-kvm +COPY tests/it/traffic-generator/init /init +RUN chmod +x /init +COPY --from=traffic-generator-base /vm.qcow2 /vm.qcow2 + + +ENTRYPOINT ["/init"] diff --git a/build/storage/tests/traffic-generator/init b/build/storage/tests/it/traffic-generator/init similarity index 52% rename from build/storage/tests/traffic-generator/init rename to build/storage/tests/it/traffic-generator/init index c262ba2c..80dd0e90 100644 --- a/build/storage/tests/traffic-generator/init +++ b/build/storage/tests/it/traffic-generator/init @@ -4,24 +4,40 @@ # SPDX-License-Identifier: Apache-2.0 # -[ "$DEBUG" == 'true' ] && set -x +[ "$DEBUG" == 'true' ] && set -x ; export DEBUG_VM=true shared_volume=/ipdk-shared +virtio_blk_socket=${shared_volume}/VirtioBlk0 + attach_default_virtio_blk=" \ --chardev socket,id=spdk_vhost_blk0,path=${shared_volume}/VirtioBlk0 \ +-chardev socket,id=spdk_vhost_blk0,path=${virtio_blk_socket} \ -device vhost-user-blk-pci,chardev=spdk_vhost_blk0,num-queues=2 " if [ "${DO_NOT_ATTACH_VIRTIO_BLK}" == "true" ]; then attach_default_virtio_blk="" +else + wait_counter=1 + + while [ ! -S ${virtio_blk_socket} ] && [ ${wait_counter} -le 10 ] ; do + echo "Wait for virtio-blk socket: ${virtio_blk_socket}" + sleep 5 + wait_counter=$(( wait_counter + 1 )) + done + if [ ! -S ${virtio_blk_socket} ] ; then + echo "ERROR virtio-blk socket is not detected: ${virtio_blk_socket}" + exit 1 + else + echo "virtio-blk socket ${virtio_blk_socket} is detected" + fi + sleep 5 fi -sleep 2 echo "Starting vm" run_qemu="qemu-kvm \ -${attach_default_virtio_blk} \ +${attach_default_virtio_blk} --enable-kvm \ -cpu host \ -m 1G \ @@ -34,4 +50,6 @@ ${attach_default_virtio_blk} \ -serial unix:${shared_volume}/vm_socket,server,nowait \ --nographic" -$run_qemu \ No newline at end of file +$run_qemu + +echo "VM stopped" diff --git a/build/storage/tests/proxy-container.conf b/build/storage/tests/proxy-container.conf deleted file mode 100644 index 0a8caa32..00000000 --- a/build/storage/tests/proxy-container.conf +++ /dev/null @@ -1,34 +0,0 @@ -{ - "subsystems": [ - { - "subsystem": "bdev", - "config": [ - { - "method": "bdev_nvme_attach_controller", - "params": { - "name": "Nvme0", - "trtype": "TCP", - "adrfam": "IPv4", - "traddr": "192.168.42.2", - "trsvcid": "4420", - "subnqn": "nqn.2016-06.io.spdk:cnode1", - "prchk_reftag": false, - "prchk_guard": false - } - } - ] - }, - { - "subsystem": "vhost", - "config": [ - { - "method": "vhost_create_blk_controller", - "params": { - "ctrlr": "VirtioBlk0", - "dev_name": "Nvme0n1" - } - } - ] - } - ] -} diff --git a/build/storage/tests/run.sh b/build/storage/tests/run.sh deleted file mode 100755 index 54993aab..00000000 --- a/build/storage/tests/run.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -set -e -[ "$DEBUG" == 'true' ] && set -x - -vm_file=./traffic-generator/vm.qcow2 -declare https_proxy -declare http_proxy -declare no_proxy -export HTTPS_PROXY=${https_proxy} -export HTTP_PROXY=${http_proxy} -export NO_PROXY=${no_proxy} - -function run_test() { - SUDO_FOR_DOCKER="sudo" - IS_USER_DOCKER_GROUP_MEMBER=$(groups | grep docker &> /dev/null ; echo $?) - if [ "${IS_USER_DOCKER_GROUP_MEMBER}" == "0" ]; then - SUDO_FOR_DOCKER= - fi - - ${SUDO_FOR_DOCKER} docker-compose \ - -f ./docker-compose.yml \ - -f "./test-drivers/docker-compose.$1.yml" \ - up \ - --build \ - --exit-code-from test-driver \ - --scale build_base=0 -} - -function provide_hugepages() { - if [ -d /sys/kernel/mm/hugepages/hugepages-2048kB ] ; then - number_of_2mb_pages=$(cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages) - required_number_of_pages=2048 - if [ "$number_of_2mb_pages" -lt "$required_number_of_pages" ] ; then - sudo echo ${required_number_of_pages} | \ - sudo tee /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages - fi - fi -} - -function provide_vm() { - if [ ! -f "$vm_file" ]; then - local vm_tmp_file="${vm_file}_orig" - wget -O ${vm_tmp_file} https://download.fedoraproject.org/pub/fedora/linux/\ -releases/33/Cloud/x86_64/images/Fedora-Cloud-Base-33-1.2.x86_64.qcow2 - virt-customize -a ${vm_tmp_file} \ - --root-password password:root \ - --uninstall cloud-init \ - --install fio - mv "${vm_tmp_file}" "${vm_file}" - fi -} - -provide_hugepages -provide_vm - -test_cases=(hot-plug fio) -if [[ $# != 0 ]]; then - run_test "${1}" -else - for i in "${test_cases[@]}"; do - run_test "${i}" - done -fi diff --git a/build/storage/tests/storage-target.conf b/build/storage/tests/storage-target.conf deleted file mode 100644 index 1916e397..00000000 --- a/build/storage/tests/storage-target.conf +++ /dev/null @@ -1,63 +0,0 @@ -{ - "subsystems": [ - { - "subsystem": "bdev", - "config": [ - { - "method": "bdev_malloc_create", - "params": { - "name": "Malloc0", - "num_blocks": 131072, - "block_size": 512 - } - } - ] - }, - { - "subsystem": "nvmf", - "config": [ - { - "method": "nvmf_create_transport", - "params": { - "trtype": "TCP", - "io_unit_size": 8192 - } - }, - { - "method": "nvmf_create_subsystem", - "params": { - "nqn": "nqn.2016-06.io.spdk:cnode1", - "allow_any_host": true, - "serial_number": "SPDK00000000000001", - "model_number": "SPDK bdev Controller", - "max_namespaces": 32, - "min_cntlid": 1, - "max_cntlid": 65519 - } - }, - { - "method": "nvmf_subsystem_add_ns", - "params": { - "nqn": "nqn.2016-06.io.spdk:cnode1", - "namespace": { - "nsid": 1, - "bdev_name": "Malloc0" - } - } - }, - { - "method": "nvmf_subsystem_add_listener", - "params": { - "nqn": "nqn.2016-06.io.spdk:cnode1", - "listen_address": { - "trtype": "TCP", - "adrfam": "IPv4", - "traddr": "192.168.42.2", - "trsvcid": "4420" - } - } - } - ] - } - ] -} diff --git a/build/storage/tests/test-drivers/init.fio b/build/storage/tests/test-drivers/init.fio deleted file mode 100755 index da555438..00000000 --- a/build/storage/tests/test-drivers/init.fio +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -set -e -[ "$DEBUG" == 'true' ] && set -x - -declare vm_serial -# shellcheck disable=SC1091 -source /test-helpers - -wait_until_vm_is_up "${vm_serial}" -log_in_with_default_credentials "${vm_serial}" - -is_virtio_blk_attached "${vm_serial}" - -echo "Run fio" -fio_cmd="fio --filename=/dev/vda --direct=1 --rw=randrw \ - --bs=4k --ioengine=libaio --iodepth=256 --runtime=10 \ - --numjobs=4 --time_based --group_reporting --name=iops-test-job" - -out=$(send_command_over_unix_socket_and_sample_output \ - "${vm_serial}" "${fio_cmd}" 11) -echo "${out}" - -echo "${out}" | grep "Disk stats (read/write)" -echo "fio has been executed successfully!" - -exit 0 diff --git a/build/storage/tests/test-drivers/init.hot-plug b/build/storage/tests/test-drivers/init.hot-plug deleted file mode 100755 index 75fc0caa..00000000 --- a/build/storage/tests/test-drivers/init.hot-plug +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -set -e -[ "$DEBUG" == 'true' ] && set -x - -declare vm_serial -declare vm_monitor -# shellcheck disable=SC1091 -source /test-helpers - -wait_until_vm_is_up "${vm_serial}" -log_in_with_default_credentials "${vm_serial}" - -is_virtio_blk_not_attached "${vm_serial}" - -attach_virtio_blk_over_monitor "${vm_monitor}" - -is_virtio_blk_attached "${vm_serial}" - -exit 0 diff --git a/build/storage/tests/test-drivers/test-helpers b/build/storage/tests/test-drivers/test-helpers deleted file mode 100644 index 24877b46..00000000 --- a/build/storage/tests/test-drivers/test-helpers +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -export shared_volume=/ipdk-shared -export vm_serial=${shared_volume}/vm_socket -export vm_monitor=${shared_volume}/vm_monitor_socket - -function send_command_over_unix_socket() { - echo "${2}" | socat - unix-connect:"${1}" -} - -function send_command_over_unix_socket_and_sample_output() { - socket="${1}" - cmd="${2}" - wait_for_secs="${3}" - out=$(echo "${cmd}" | \ - socat -T"${wait_for_secs}" -,ignoreeof unix-connect:"${socket}") - echo "${out}" -} - -function get_output_from_unix_socket() { - socket="${1}" - wait_for_secs="${2}" - out=$( socat -T"${wait_for_secs}" -,ignoreeof unix-connect:"${socket}" ) - result=$? - echo "${out}" -} - -function wait_until_vm_is_up() { - console="${1}" - - [ "$DEBUG_VM" == 'true' ] && echo "Start waiting for VM... and console ${console}" - - overall_wait_counter=0 - overall_max_wait_counter=12 - wait_period_sec=10 - - while [ ! -S "${console}" ] && \ - [ "${overall_wait_counter}" -le "${overall_max_wait_counter}" ] ; do - sec=$(( overall_wait_counter * wait_period_sec )) - [ "$DEBUG_VM" == 'true' ] && echo "Waiting for VM console: ${console}, sec ${sec} ..." - sleep "${wait_period_sec}" - overall_wait_counter=$(( overall_wait_counter + 1 )) - done - - sleep 4 - - socket_output_wait_counter=0 - while [ "$socket_output_wait_counter" -le 2 ] && \ - [ "$overall_wait_counter" -le "${overall_max_wait_counter}" ] ; do - sec=$(( overall_wait_counter * wait_period_sec )) - [ "$DEBUG_VM" == 'true' ] && echo "Waiting for VM completes booting, sec ${sec} ..." - overall_wait_counter=$(( overall_wait_counter + 1 )) - out=$( get_output_from_unix_socket "${console}" "${wait_period_sec}" ) - [ "$DEBUG_VM" == 'true' ] && echo "${out}" - socket_output_wait_counter=$(( socket_output_wait_counter + 1 )) - if [[ -n "${out}" ]] ; then - socket_output_wait_counter=0 - fi - done - return 0 -} - -function log_in() { - send_command_over_unix_socket "${1}" "${2}" - sleep 1 - send_command_over_unix_socket "${1}" "${3}" - sleep 1 -} - -function log_in_with_default_credentials() { - log_in "${1}" "root" "root" -} - -function is_virtio_blk_attached() { - cmd="lsblk --output \"NAME,VENDOR,SUBSYSTEMS\"" - out=$(send_command_over_unix_socket_and_sample_output "${1}" "${cmd}" 1) - echo "${out}" | grep "block:virtio:pci" - result=$? - echo "${out}" - if [[ ${result} == 0 ]]; then - echo "virtio-blk is found" - else - echo "virtio-blk is not found" - fi - return "${result}" -} - -function is_virtio_blk_not_attached() { - if is_virtio_blk_attached "${1}"; then - return 1 - fi - - return 0 -} - -function attach_virtio_blk_over_monitor() { - add_chardev_cmd="chardev-add socket,id=spdk_vhost_blk0,path=${shared_volume}/VirtioBlk0" - send_command_over_unix_socket "${vm_monitor}" "${add_chardev_cmd}" - sleep 1 - add_device_cmd="device_add vhost-user-blk-pci,chardev=spdk_vhost_blk0,num-queues=2" - send_command_over_unix_socket "${vm_monitor}" "${add_device_cmd}" - sleep 1 -} - diff --git a/build/storage/tests/traffic-generator/Dockerfile b/build/storage/tests/traffic-generator/Dockerfile deleted file mode 100644 index 99066b35..00000000 --- a/build/storage/tests/traffic-generator/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# -# NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE -# WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK -# - -FROM spdk - -# Generic args -ARG HTTP_PROXY -ARG HTTPS_PROXY -ARG NO_PROXY - -ENV http_proxy=$HTTP_PROXY -ENV https_proxy=$HTTPS_PROXY -ENV no_proxy=$NO_PROXY - - -COPY vm.qcow2 /vm.qcow2 -RUN dnf install -y qemu-kvm -COPY init /init -RUN chmod +x /init - -ENTRYPOINT ["/init"] diff --git a/build/storage/tests/ut/README.md b/build/storage/tests/ut/README.md new file mode 100644 index 00000000..2358e863 --- /dev/null +++ b/build/storage/tests/ut/README.md @@ -0,0 +1,22 @@ +# Description +Unit tests covers services written written for IPDK. + +# Run unit tests +The command below build all required dependencies and run the unit tests within +the dedicated container. +``` +$ ./run.sh +``` + +# Development +By means of the command below, a container will be started with test and source +files attached as volumes. +``` +$ ./run.sh dev +``` + +In order to run the unit tests during a development session run +`run_all_unit_tests.sh` within the started container. +``` +$ /run_all_unit_tests.sh +``` diff --git a/build/storage/tests/ut/host-target/test_device_exerciser.py b/build/storage/tests/ut/host-target/test_device_exerciser.py new file mode 100644 index 00000000..d96fdd59 --- /dev/null +++ b/build/storage/tests/ut/host-target/test_device_exerciser.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +from pyfakefs.fake_filesystem_unittest import TestCase +from pci_devices import get_virtio_blk_path_by_pci_address +from device_exerciser import DeviceExerciser, DeviceExerciserError + + +class DeviceExerciserTests(TestCase): + def setUp(self): + self.setUpPyfakefs() + + def tearDown(self): + pass + + def test_successful_fio_run(self): + self.fs.create_dir("/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda") + self.fs.create_file("/dev/vda") + fio_arguments = "--name=test --size=4MB" + + def fio_do_nothing(fio_args): + fio_do_nothing.was_called = True + self.assertTrue("vda" in fio_args) + self.assertTrue(fio_arguments in fio_args) + return "output" + + fio_do_nothing.was_called = False + + exerciser = DeviceExerciser( + fio_runner=fio_do_nothing, + virtio_blk_detector=get_virtio_blk_path_by_pci_address, + ) + out = exerciser.run_fio("0000:00:04.0", fio_arguments) + self.assertTrue(fio_do_nothing.was_called) + self.assertEqual(out, "output") + + def test_any_errors_at_exercising(self): + def raise_exception(unused): + raise BaseException() + + exerciser = DeviceExerciser( + fio_runner=raise_exception, virtio_blk_detector=raise_exception + ) + with self.assertRaises(DeviceExerciserError) as ex: + exerciser.run_fio("pcie_address", "fio_arguments") diff --git a/build/storage/tests/ut/host-target/test_fio_runner.py b/build/storage/tests/ut/host-target/test_fio_runner.py new file mode 100644 index 00000000..7acea69f --- /dev/null +++ b/build/storage/tests/ut/host-target/test_fio_runner.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import unittest +import os + +from fio_runner import run_fio +from fio_runner import FioExecutionError + + +class FioRunner(unittest.TestCase): + def setUp(self): + self.fio_file = "test" + self.fio_args = "--name=test --size=4MB --filename=test" + + def tearDown(self): + if os.path.exists(self.fio_file): + os.remove(self.fio_file) + + def test_successful_fio_run(self): + try: + run_fio(self.fio_args) + except FioExecutionError as ex: + self.fail(str(ex)) + + self.assertTrue(os.path.exists(self.fio_file)) + + def test_successful_fio_run_output(self): + out = run_fio(self.fio_args) + self.assertTrue(out.find("READ: bw=") != -1) + + def test_invalid_fio_arg(self): + with self.assertRaises(FioExecutionError) as ex: + run_fio("some-non-existing-arg") + + def test_pass_invalid_arg_type(self): + with self.assertRaises(FioExecutionError) as ex: + run_fio(123) + + def test_no_concat_cmds_allowed(self): + with self.assertRaises(FioExecutionError) as ex: + run_fio("ls -l && ls") + with self.assertRaises(FioExecutionError) as ex: + run_fio("ls -l || ls") + with self.assertRaises(FioExecutionError) as ex: + run_fio("ls -l ; ls") diff --git a/build/storage/tests/ut/host-target/test_host_target_grpc_server.py b/build/storage/tests/ut/host-target/test_host_target_grpc_server.py new file mode 100644 index 00000000..c0cef21f --- /dev/null +++ b/build/storage/tests/ut/host-target/test_host_target_grpc_server.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +from multiprocessing import context +from host_target_grpc_server import HostTargetService +import host_target_pb2 + +import unittest +import unittest.mock + + +def detect_virtio_blk_device(unused): + return "non_existing_device_name" + + +def successfull_fio(unused): + return "output" + + +class HostTargetServerTests(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + pass + + def test_run_fio_success(self): + server = HostTargetService(successfull_fio, detect_virtio_blk_device) + request = host_target_pb2.RunFioRequest() + request.pciAddress = "unused" + request.fioArgs = "unused" + context = unittest.mock.MagicMock() + + reply = server.RunFio(request, context) + context.set_code.was_not_called() + context.set_details.was_not_called() + self.assertTrue(reply != None) + self.assertTrue(reply.fioOutput != None) + self.assertTrue(reply.fioOutput != "") + + def test_run_fio_does_not_propagate_exception(self): + def fio_throws_exception(unused): + raise BaseException() + + server = HostTargetService(fio_throws_exception, detect_virtio_blk_device) + request = host_target_pb2.RunFioRequest() + request.pciAddress = "unused" + request.fioArgs = "unused" + context = unittest.mock.MagicMock() + + server.RunFio(request, context) + context.set_code.assert_called() + context.set_details.assert_called() + + def test_run_fio_does_not_propagate_exception(self): + def detect_virtio_blk_throws_exception(unused): + raise BaseException() + + server = HostTargetService(successfull_fio, detect_virtio_blk_throws_exception) + request = host_target_pb2.RunFioRequest() + request.pciAddress = "unused" + request.fioArgs = "unused" + context = unittest.mock.MagicMock() + + server.RunFio(request, context) + context.set_code.assert_called() + context.set_details.assert_called() diff --git a/build/storage/tests/ut/host-target/test_pci_devices.py b/build/storage/tests/ut/host-target/test_pci_devices.py new file mode 100644 index 00000000..209e1018 --- /dev/null +++ b/build/storage/tests/ut/host-target/test_pci_devices.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import os +from pci_devices import get_virtio_blk_path_by_pci_address +from pci_devices import InvalidPciAddress +from pci_devices import FailedPciDeviceDetection +from pyfakefs.fake_filesystem_unittest import TestCase + + +class PciDeviceTests(TestCase): + def setUp(self): + self.setUpPyfakefs() + + def test_get_device_by_pci_address(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + dev_path = get_virtio_blk_path_by_pci_address("0000:00:04.0") + self.assertEqual(dev_path, "/dev/vda") + + def test_get_device_by_not_conventional_pci_address(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:00:04.9") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:00:20.0") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:00:00:0") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:00.00.0") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0:00.00.0") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:0.00.0") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:00.0.0") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:00.00.01") + + def test_corner_bus_value(self): + path = "/sys/devices/pci0000:FF/0000:FF:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + dev_path = get_virtio_blk_path_by_pci_address("0000:FF:04.0") + self.assertEqual(dev_path, "/dev/vda") + + def test_corner_device_value(self): + path = "/sys/devices/pci0000:00/0000:00:1F.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + dev_path = get_virtio_blk_path_by_pci_address("0000:00:1F.0") + self.assertEqual(dev_path, "/dev/vda") + + def test_corner_function_value(self): + path = "/sys/devices/pci0000:00/0000:00:04.7/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + dev_path = get_virtio_blk_path_by_pci_address("0000:00:04.7") + self.assertEqual(dev_path, "/dev/vda") + + def test_multiple_domains(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + path = "/sys/devices/pci0001:00/0001:00:04.0/virtio1/block/vdb" + self.fs.create_dir(path) + self.fs.create_file("/dev/vdb") + + dev_path = get_virtio_blk_path_by_pci_address("0001:00:04.0") + self.assertEqual(dev_path, "/dev/vdb") + + def test_multiple_buses(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + path = "/sys/devices/pci0000:01/0000:01:04.0/virtio1/block/vdb" + self.fs.create_dir(path) + self.fs.create_file("/dev/vdb") + + dev_path = get_virtio_blk_path_by_pci_address("0000:01:04.0") + self.assertEqual(dev_path, "/dev/vdb") + + def test_multiple_devices(self): + path = "/sys/devices/pci0000:00/0000:00:00.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + path = "/sys/devices/pci0000:00/0000:00:01.0/virtio1/block/vdb" + self.fs.create_dir(path) + self.fs.create_file("/dev/vdb") + + dev_path = get_virtio_blk_path_by_pci_address("0000:00:01.0") + self.assertEqual(dev_path, "/dev/vdb") + + def test_multiple_functions(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + path = "/sys/devices/pci0000:00/0000:00:04.1/virtio1/block/vdb" + self.fs.create_dir(path) + self.fs.create_file("/dev/vdb") + + dev_path = get_virtio_blk_path_by_pci_address("0000:00:04.1") + self.assertEqual(dev_path, "/dev/vdb") + + def test_no_block_device_exists(self): + path_to_device = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block" + self.fs.create_dir(path_to_device) + with self.assertRaises(FailedPciDeviceDetection) as ex: + get_virtio_blk_path_by_pci_address("0000:00:00.0") + + def test_multiple_block_devices_in_virtio_dir(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vdb" + self.fs.create_dir(path) + self.fs.create_file("/dev/vdb") + with self.assertRaises(FailedPciDeviceDetection) as ex: + get_virtio_blk_path_by_pci_address("0000:00:04.0") + + def test_multiple_block_devices_under_different_virtio_dirs(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio1/block/vdb" + self.fs.create_dir(path) + self.fs.create_file("/dev/vdb") + with self.assertRaises(FailedPciDeviceDetection) as ex: + get_virtio_blk_path_by_pci_address("0000:00:04.0") + + def test_no_directory_under_domain_bus_directory_in_sysfs(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + non_existing_pci_address = "0000:00:01.0" + with self.assertRaises(FailedPciDeviceDetection) as ex: + get_virtio_blk_path_by_pci_address(non_existing_pci_address) + + def test_pci_does_not_exist_in_sysfs(self): + with self.assertRaises(FailedPciDeviceDetection) as ex: + get_virtio_blk_path_by_pci_address("0000:00:04.0") + + def test_device_does_not_exist_in_dev(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + + with self.assertRaises(FailedPciDeviceDetection) as ex: + get_virtio_blk_path_by_pci_address("0000:00:04.0") diff --git a/build/storage/tests/ut/host-target/test_run_grpc_server.py b/build/storage/tests/ut/host-target/test_run_grpc_server.py new file mode 100644 index 00000000..add40647 --- /dev/null +++ b/build/storage/tests/ut/host-target/test_run_grpc_server.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import unittest +import os +import unittest.mock +import grpc +from unittest.mock import patch +from host_target_grpc_server import run_grpc_server +from host_target_grpc_server import HostTargetService +import host_target_pb2_grpc + + +class RunGrpcServer(unittest.TestCase): + def test_fail_on_invalid_address_to_listen_to(self): + self.assertNotEqual(run_grpc_server("Invalid ip addr", 1010), 0) + + def test_fail_on_invalid_port_to_listen_to(self): + self.assertNotEqual(run_grpc_server("0.0.0.0", "Invalid port"), 0) + + def test_success_on_keyboard_interrupt_exception(self): + server_mock = unittest.mock.Mock() + server_mock.add_insecure_port.side_effect = KeyboardInterrupt() + + def server_creator(unused): + return server_mock + + self.assertEqual(run_grpc_server("Unused", "Unused", server_creator), 0) + self.assertTrue(server_mock.add_insecure_port.was_called) + + @patch.object(host_target_pb2_grpc, "add_HostTargetServicer_to_server") + def test_required_methods_were_called(self, add_servicer_mock): + server_mock = unittest.mock.Mock() + + def server_creator(unused): + return server_mock + + ip_addr = "ip_addr" + port = "port" + + self.assertEqual(run_grpc_server(ip_addr, port, server_creator), 0) + + self.assertTrue(server_mock.add_insecure_port.was_called) + call_args = server_mock.add_insecure_port.call_args.args + self.assertTrue(len(call_args) == 1) + self.assertTrue(ip_addr in call_args[0]) + self.assertTrue(port in call_args[0]) + self.assertTrue(server_mock.start.was_called) + + self.assertTrue(add_servicer_mock.was_called) + self.assertTrue(len(add_servicer_mock.call_args.args) == 2) + self.assertTrue( + isinstance(add_servicer_mock.call_args.args[0], HostTargetService) + ) + self.assertTrue(add_servicer_mock.call_args.args[1] == server_mock) diff --git a/build/storage/tests/ut/proxy-container/test_hot_plug_grpc_server.py b/build/storage/tests/ut/proxy-container/test_hot_plug_grpc_server.py new file mode 100644 index 00000000..5612e590 --- /dev/null +++ b/build/storage/tests/ut/proxy-container/test_hot_plug_grpc_server.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import hot_plug_pb2 +from hot_plug_provider import HotPlugProvider +from hot_plug_grpc_server import HotPlugService, InvalidHotPlugProvider +import grpc +import unittest +from unittest.mock import patch + + +class HotPlugServerTests(unittest.TestCase): + def setUp(self): + self.server = HotPlugService(HotPlugProvider("stub", "stub")) + self.context = unittest.mock.MagicMock() + + def tearDown(self): + pass + + def _test_server_does_not_propagate_exception( + self, stub_provider_operation, server_operation + ): + err_description = "exception description" + stub_provider_operation.side_effect = BaseException(err_description) + request = hot_plug_pb2.HotPlugRequest() + + reply = server_operation(request, self.context) + self.context.set_code.assert_called() + self.context.set_details.assert_called() + self.assertTrue(reply != None) + + @patch.object(HotPlugProvider, "hot_plug_vhost_virtio_blk") + def test_hot_plug_does_not_propagate_exception(self, mock_provider): + self._test_server_does_not_propagate_exception( + mock_provider, self.server.HotPlugVirtioBlk + ) + + @patch.object(HotPlugProvider, "hot_unplug_vhost_virtio_blk") + def test_hot_unplug_does_not_propagate_exception(self, mock_provider): + self._test_server_does_not_propagate_exception( + mock_provider, self.server.HotUnplugVirtioBlk + ) + + def _test_server_operation_success(self, stub_provider_operation, server_operation): + vm = "vm" + vhost = "vhost" + request = hot_plug_pb2.HotPlugRequest() + request.vmId = vm + request.vhostVirtioBlkId = vhost + + reply = server_operation(request, self.context) + + stub_provider_operation.assert_called_with(vm, vhost) + + self.context.set_code.was_not_called() + self.context.set_details.was_not_called() + self.assertTrue(reply != None) + + @patch.object(HotPlugProvider, "hot_plug_vhost_virtio_blk") + def test_hot_plug_success(self, mock_provider): + self._test_server_operation_success(mock_provider, self.server.HotPlugVirtioBlk) + + @patch.object(HotPlugProvider, "hot_unplug_vhost_virtio_blk") + def test_hot_unplug_success(self, mock_provider): + self._test_server_operation_success( + mock_provider, self.server.HotUnplugVirtioBlk + ) diff --git a/build/storage/tests/ut/proxy-container/test_hot_plug_provider.py b/build/storage/tests/ut/proxy-container/test_hot_plug_provider.py new file mode 100644 index 00000000..4ae9007e --- /dev/null +++ b/build/storage/tests/ut/proxy-container/test_hot_plug_provider.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +from multiprocessing.sharedctypes import Value +from hot_plug_provider import ActionExecutionError, HotPlugProvider +from pathlib import Path +import socket +import os +from unittest.mock import patch +import unittest + + +class SocketTestEnvironment(unittest.TestCase): + def setUp(self): + self.fake_socket_name = "fake_socket" + self.fake_socket_path = "/tmp/" + self.fake_socket_name + self.fake_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.fake_socket.bind(self.fake_socket_path) + + def tearDown(self): + self.fake_socket.close() + os.remove(self.fake_socket_path) + + +class HotPlugValidationBaseTestCases: + @patch("hot_plug_provider.subprocess.call") + class Tests(SocketTestEnvironment): + def setUp(self): + SocketTestEnvironment.setUp(self) + self.shared_dir_path = "/tmp" + self.host_shared_dir_path = "/home/user/ipdk-shared_dir" + self.non_existing_path = "/tmp/non-existing_path" + + self.provider = HotPlugProvider( + self.shared_dir_path, self.host_shared_dir_path + ) + self.disk_operation = None + + def tearDown(self): + SocketTestEnvironment.tearDown(self) + + def test_pass_none_shared_dir(self, unused): + with self.assertRaises(ValueError) as ex: + HotPlugProvider(None, self.host_shared_dir_path) + + def test_pass_none_host_shared_dir(self, unused): + with self.assertRaises(ValueError) as ex: + HotPlugProvider(self.shared_dir_path, None) + + def test_non_existing_path_to_vm_monitor(self, mock_subprocess_call): + with self.assertRaises(FileNotFoundError) as ex: + self.disk_operation( + vm_monitor=self.non_existing_path, + vhost_virtio_blk=self.fake_socket_path, + ) + mock_subprocess_call.assert_not_called() + + def test_non_existing_path_to_vhost(self, mock_subprocess_call): + with self.assertRaises(FileNotFoundError) as ex: + self.disk_operation( + vm_monitor=self.fake_socket_path, + vhost_virtio_blk=self.non_existing_path, + ) + mock_subprocess_call.assert_not_called() + + def test_pass_valid_sockets(self, mock_subprocess_call): + mock_subprocess_call.return_value = 0 + was_exception_raised = False + try: + self.disk_operation(self.fake_socket_path, self.fake_socket_path) + except: + was_exception_raised = True + + self.assertEqual(was_exception_raised, False) + mock_subprocess_call.assert_called_once() + + def test_hot_plug_call_failed(self, mock_subprocess_call): + mock_subprocess_call.return_value = 1 + + with self.assertRaises(ActionExecutionError) as ex: + self.disk_operation(self.fake_socket_path, self.fake_socket_path) + + def test_hot_plug_generates_correct_device_id(self, mock_subprocess_call): + mock_subprocess_call.return_value = 0 + + vm = self.fake_socket_name + vhost0 = " " + self.fake_socket_name + " " + vhost1 = vhost0.strip() + + self.disk_operation(vm, vhost0) + self.disk_operation(vm, vhost1) + call_args = mock_subprocess_call.call_args_list + self.assertTrue(len(call_args) == 2) + self.assertTrue(call_args[0] == call_args[1]) + + def test_pass_none_vm(self, unused): + with self.assertRaises(ValueError) as ex: + self.disk_operation(None, self.fake_socket_path) + + def test_pass_none_vhost(self, unused): + with self.assertRaises(ValueError) as ex: + self.disk_operation(self.fake_socket_path, None) + + +class HotPlugValidation(HotPlugValidationBaseTestCases.Tests): + def setUp(self): + HotPlugValidationBaseTestCases.Tests.setUp(self) + self.disk_operation = self.provider.hot_plug_vhost_virtio_blk + + def tearDown(self): + HotPlugValidationBaseTestCases.Tests.tearDown(self) + + @patch("hot_plug_provider.subprocess.call") + @patch.object(HotPlugProvider, "_generate_device_id") + def test_hot_plug_call_args(self, mock_device_id_generator, mock_subprocess_call): + device_id = "42" + mock_subprocess_call.return_value = 0 + mock_device_id_generator.return_value = device_id + + vm = self.fake_socket_name + vhost = self.fake_socket_name + + self.disk_operation(vm, vhost) + mock_subprocess_call.assert_called_once_with( + "/hot-plug.sh " + + os.path.join(self.shared_dir_path, vm) + + " " + + os.path.join(self.host_shared_dir_path, vhost) + + " " + + device_id, + shell=True, + ) + + @patch("hot_plug_provider.subprocess.call") + @patch.object(HotPlugProvider, "_generate_device_id") + def test_hot_plug_call_args_with_trailing_whitespaces( + self, mock_device_id_generator, mock_subprocess_call + ): + device_id = "42" + mock_subprocess_call.return_value = 0 + mock_device_id_generator.return_value = device_id + + vm = " " + self.fake_socket_name + " " + vhost = " " + self.fake_socket_name + " " + + self.disk_operation(vm, vhost) + + mock_subprocess_call.assert_called_once_with( + "/hot-plug.sh " + + os.path.join(self.shared_dir_path, self.fake_socket_name) + + " " + + os.path.join(self.host_shared_dir_path, self.fake_socket_name) + + " " + + device_id, + shell=True, + ) + + +class HotUnplugValidation(HotPlugValidationBaseTestCases.Tests): + def setUp(self): + HotPlugValidationBaseTestCases.Tests.setUp(self) + self.disk_operation = self.provider.hot_unplug_vhost_virtio_blk + + def tearDown(self): + HotPlugValidationBaseTestCases.Tests.tearDown(self) + + @patch("hot_plug_provider.subprocess.call") + @patch.object(HotPlugProvider, "_generate_device_id") + def test_hot_plug_call_args(self, mock_device_id_generator, mock_subprocess_call): + device_id = "42" + mock_subprocess_call.return_value = 0 + mock_device_id_generator.return_value = device_id + + vm = self.fake_socket_name + vhost = self.fake_socket_name + + self.disk_operation(vm, vhost) + mock_subprocess_call.assert_called_once_with( + "/hot-unplug.sh " + + os.path.join(self.shared_dir_path, vm) + + " " + + device_id, + shell=True, + ) + + @patch("hot_plug_provider.subprocess.call") + @patch.object(HotPlugProvider, "_generate_device_id") + def test_hot_plug_call_args_with_trailing_whitespaces( + self, mock_device_id_generator, mock_subprocess_call + ): + device_id = "42" + mock_subprocess_call.return_value = 0 + mock_device_id_generator.return_value = device_id + + vm = " " + self.fake_socket_name + " " + vhost = " " + self.fake_socket_name + " " + + self.disk_operation(vm, vhost) + + mock_subprocess_call.assert_called_once_with( + "/hot-unplug.sh " + + os.path.join(self.shared_dir_path, self.fake_socket_name) + + " " + + device_id, + shell=True, + ) diff --git a/build/storage/tests/ut/run.sh b/build/storage/tests/ut/run.sh new file mode 100755 index 00000000..d3e5bc3a --- /dev/null +++ b/build/storage/tests/ut/run.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +[ "$DEBUG" == 'true' ] && set -x +set -e + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +root_dir="${current_script_dir}/../.." +scripts_dir="${root_dir}/scripts" +proxy_src_dir="${root_dir}/core/proxy-container" +host_target_src_dir="${root_dir}/core/host-target" + +ut_container="ipdk-unit-tests" +"${scripts_dir}"/build_container.sh "${ut_container}" +args=(-e "DEBUG=${DEBUG}") +if [[ "${1}" == dev ]] ; then + args+=(--entrypoint "/bin/bash") + args+=(-v "${current_script_dir}/proxy-container:/proxy-container/tests") + args+=(-v "${proxy_src_dir}:/proxy-container/src") + args+=(-v "${current_script_dir}/host-target:/host-target/tests") + args+=(-v "${host_target_src_dir}:/host-target/src") +fi +docker run -it "${args[@]}" "${ut_container}" diff --git a/build/storage/tests/ut/run_all_unit_tests.sh b/build/storage/tests/ut/run_all_unit_tests.sh new file mode 100755 index 00000000..35056642 --- /dev/null +++ b/build/storage/tests/ut/run_all_unit_tests.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x +ls -l /host-target/tests + +result=0 +python -m unittest discover -v -s /proxy-container/tests +result=$(("$?" | "${result}")) +python -m unittest discover -v -s /host-target/tests +result=$(("$?" | "${result}")) + +exit "${result}"