From 7e5d2fbec37c34d7859198a0f8b69ff8086e2f5a Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Tue, 17 May 2022 16:20:49 +0200 Subject: [PATCH 1/4] Storage containers splitting Provide an ability to run the images as standalone containers outside the test scope. Signed-off-by: Artsiom Koltun --- .gitmodules | 3 + build/scripts/run-shellcheck.sh | 8 +- build/storage/.dockerignore | 1 + build/storage/.gitignore | 1 + build/storage/Dockerfile | 178 +++++++ build/storage/Makefile | 2 +- build/storage/README.md | 23 +- build/storage/core/build_base/Dockerfile | 49 -- build/storage/core/build_base/post-install | 4 +- build/storage/core/build_base/pre-install | 17 +- .../core/host-target/device_exerciser.py | 25 + build/storage/core/host-target/fio_runner.py | 24 + .../core/host-target/host_target.proto | 20 + .../host-target/host_target_grpc_server.py | 58 +++ .../core/host-target/host_target_main.py | 30 ++ build/storage/core/host-target/init | 10 + build/storage/core/host-target/pci_devices.py | 104 ++++ .../storage/core/proxy-container/hot-plug.sh | 40 ++ .../core/proxy-container/hot-unplug.sh | 38 ++ .../core/proxy-container/hot_plug.proto | 28 ++ .../proxy-container/hot_plug_grpc_server.py | 57 +++ .../core/proxy-container/hot_plug_main.py | 38 ++ .../core/proxy-container/hot_plug_provider.py | 99 ++++ build/storage/core/proxy-container/init | 50 ++ build/storage/core/spdk-app/Dockerfile | 21 - build/storage/core/spdk-app/init | 22 +- build/storage/recipes/README.md | 13 + build/storage/recipes/environment_setup.md | 170 +++++++ build/storage/recipes/fio.md | 28 ++ build/storage/recipes/hot-plug.md | 109 +++++ .../storage/recipes/system_configuration.svg | 425 +++++++++++++++++ build/storage/scripts/allocate_hugepages.sh | 20 + build/storage/scripts/build_container.sh | 46 ++ build/storage/scripts/disk_infrastructure.sh | 144 ++++++ build/storage/scripts/prepare_to_build.sh | 9 + .../scripts/run_host_target_container.sh | 57 +++ build/storage/scripts/run_proxy_container.sh | 82 ++++ .../scripts/run_storage_target_container.sh | 64 +++ build/storage/scripts/socket.sh | 28 ++ build/storage/scripts/spdk_version.sh | 14 + build/storage/scripts/vm/create_nat_for_vm.sh | 52 ++ build/storage/scripts/vm/delete_nat_for_vm.sh | 16 + build/storage/scripts/vm/nat_variables.sh | 12 + build/storage/scripts/vm/prepare_vm.sh | 51 ++ build/storage/scripts/vm/run_vm.sh | 28 ++ build/storage/spdk | 1 + build/storage/tests/README.md | 132 ------ .../tests/img_virtio_blk_over_nvmetcp_1.svg | 184 -------- build/storage/tests/it/README.md | 63 +++ .../storage/tests/{ => it}/docker-compose.yml | 75 +-- .../it/img_virtio_blk_over_nvmetcp_1.svg | 444 ++++++++++++++++++ build/storage/tests/it/run.sh | 61 +++ .../tests/{ => it}/test-drivers/Dockerfile | 7 +- .../test-drivers/docker-compose.fio.yml | 0 .../test-drivers/docker-compose.hot-plug.yml | 0 build/storage/tests/it/test-drivers/init.fio | 45 ++ .../tests/it/test-drivers/init.hot-plug | 69 +++ .../tests/it/test-drivers/test-helpers | 76 +++ .../tests/it/traffic-generator/Dockerfile | 40 ++ .../tests/{ => it}/traffic-generator/init | 28 +- build/storage/tests/proxy-container.conf | 34 -- build/storage/tests/run.sh | 68 --- build/storage/tests/storage-target.conf | 63 --- build/storage/tests/test-drivers/init.fio | 30 -- .../storage/tests/test-drivers/init.hot-plug | 23 - build/storage/tests/test-drivers/test-helpers | 107 ----- .../tests/traffic-generator/Dockerfile | 25 - build/storage/tests/ut/README.md | 22 + .../ut/host-target/test_device_exerciser.py | 46 ++ .../tests/ut/host-target/test_fio_runner.py | 49 ++ .../test_host_target_grpc_server.py | 70 +++ .../tests/ut/host-target/test_pci_devices.py | 155 ++++++ .../ut/host-target/test_run_grpc_server.py | 53 +++ .../test_hot_plug_grpc_server.py | 68 +++ .../proxy-container/test_hot_plug_provider.py | 198 ++++++++ build/storage/tests/ut/run.sh | 25 + build/storage/tests/ut/run_all_unit_tests.sh | 16 + 77 files changed, 3793 insertions(+), 802 deletions(-) create mode 100644 .gitmodules create mode 100644 build/storage/.dockerignore create mode 100644 build/storage/Dockerfile delete mode 100644 build/storage/core/build_base/Dockerfile create mode 100644 build/storage/core/host-target/device_exerciser.py create mode 100644 build/storage/core/host-target/fio_runner.py create mode 100644 build/storage/core/host-target/host_target.proto create mode 100644 build/storage/core/host-target/host_target_grpc_server.py create mode 100755 build/storage/core/host-target/host_target_main.py create mode 100755 build/storage/core/host-target/init create mode 100644 build/storage/core/host-target/pci_devices.py create mode 100755 build/storage/core/proxy-container/hot-plug.sh create mode 100755 build/storage/core/proxy-container/hot-unplug.sh create mode 100644 build/storage/core/proxy-container/hot_plug.proto create mode 100755 build/storage/core/proxy-container/hot_plug_grpc_server.py create mode 100755 build/storage/core/proxy-container/hot_plug_main.py create mode 100755 build/storage/core/proxy-container/hot_plug_provider.py create mode 100755 build/storage/core/proxy-container/init delete mode 100644 build/storage/core/spdk-app/Dockerfile create mode 100644 build/storage/recipes/README.md create mode 100644 build/storage/recipes/environment_setup.md create mode 100644 build/storage/recipes/fio.md create mode 100644 build/storage/recipes/hot-plug.md create mode 100644 build/storage/recipes/system_configuration.svg create mode 100755 build/storage/scripts/allocate_hugepages.sh create mode 100755 build/storage/scripts/build_container.sh create mode 100755 build/storage/scripts/disk_infrastructure.sh create mode 100755 build/storage/scripts/prepare_to_build.sh create mode 100755 build/storage/scripts/run_host_target_container.sh create mode 100755 build/storage/scripts/run_proxy_container.sh create mode 100755 build/storage/scripts/run_storage_target_container.sh create mode 100755 build/storage/scripts/socket.sh create mode 100755 build/storage/scripts/spdk_version.sh create mode 100755 build/storage/scripts/vm/create_nat_for_vm.sh create mode 100755 build/storage/scripts/vm/delete_nat_for_vm.sh create mode 100755 build/storage/scripts/vm/nat_variables.sh create mode 100755 build/storage/scripts/vm/prepare_vm.sh create mode 100755 build/storage/scripts/vm/run_vm.sh create mode 160000 build/storage/spdk delete mode 100644 build/storage/tests/README.md delete mode 100644 build/storage/tests/img_virtio_blk_over_nvmetcp_1.svg create mode 100644 build/storage/tests/it/README.md rename build/storage/tests/{ => it}/docker-compose.yml (59%) create mode 100644 build/storage/tests/it/img_virtio_blk_over_nvmetcp_1.svg create mode 100755 build/storage/tests/it/run.sh rename build/storage/tests/{ => it}/test-drivers/Dockerfile (74%) rename build/storage/tests/{ => it}/test-drivers/docker-compose.fio.yml (100%) rename build/storage/tests/{ => it}/test-drivers/docker-compose.hot-plug.yml (100%) create mode 100755 build/storage/tests/it/test-drivers/init.fio create mode 100755 build/storage/tests/it/test-drivers/init.hot-plug create mode 100644 build/storage/tests/it/test-drivers/test-helpers create mode 100644 build/storage/tests/it/traffic-generator/Dockerfile rename build/storage/tests/{ => it}/traffic-generator/init (52%) delete mode 100644 build/storage/tests/proxy-container.conf delete mode 100755 build/storage/tests/run.sh delete mode 100644 build/storage/tests/storage-target.conf delete mode 100755 build/storage/tests/test-drivers/init.fio delete mode 100755 build/storage/tests/test-drivers/init.hot-plug delete mode 100644 build/storage/tests/test-drivers/test-helpers delete mode 100644 build/storage/tests/traffic-generator/Dockerfile create mode 100644 build/storage/tests/ut/README.md create mode 100644 build/storage/tests/ut/host-target/test_device_exerciser.py create mode 100644 build/storage/tests/ut/host-target/test_fio_runner.py create mode 100644 build/storage/tests/ut/host-target/test_host_target_grpc_server.py create mode 100644 build/storage/tests/ut/host-target/test_pci_devices.py create mode 100644 build/storage/tests/ut/host-target/test_run_grpc_server.py create mode 100644 build/storage/tests/ut/proxy-container/test_hot_plug_grpc_server.py create mode 100644 build/storage/tests/ut/proxy-container/test_hot_plug_provider.py create mode 100755 build/storage/tests/ut/run.sh create mode 100755 build/storage/tests/ut/run_all_unit_tests.sh diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..84bed3ed --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "build/storage/spdk"] + path = build/storage/spdk + url = https://github.com/spdk/spdk.git diff --git a/build/scripts/run-shellcheck.sh b/build/scripts/run-shellcheck.sh index e25ac7af..586d836f 100755 --- a/build/scripts/run-shellcheck.sh +++ b/build/scripts/run-shellcheck.sh @@ -12,8 +12,14 @@ is_bash() { return 1 } +is_excluded_from_check() { + [[ $1 == */spdk/* ]] && return 0 + + return 1 +} + while IFS= read -r -d $'' file; do - if is_bash "$file"; then + if ! is_excluded_from_check "$file" && is_bash "$file" ; then shellcheck -x -W0 -s bash "$file" || continue fi done < <(find . -type f \! -path "./.git/*" -print0) diff --git a/build/storage/.dockerignore b/build/storage/.dockerignore new file mode 100644 index 00000000..0f293b79 --- /dev/null +++ b/build/storage/.dockerignore @@ -0,0 +1 @@ +*.qcow2 \ No newline at end of file diff --git a/build/storage/.gitignore b/build/storage/.gitignore index 4d83a419..fa0735c3 100644 --- a/build/storage/.gitignore +++ b/build/storage/.gitignore @@ -1,4 +1,5 @@ *.qcow2 +__pycache__ *.swp diff --git a/build/storage/Dockerfile b/build/storage/Dockerfile new file mode 100644 index 00000000..65af1a35 --- /dev/null +++ b/build/storage/Dockerfile @@ -0,0 +1,178 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE +# WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK +# +################################################################################ +# build_base +# +# This image is used to provide environment for spdk build and get it in the +# form of spdk packages +################################################################################ +FROM fedora:33 AS base + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY +ARG SPDK_VERSION + +RUN dnf install -y git +COPY spdk/ /spdk +RUN mkdir /spdk-rpm +COPY core/build_base/pre-install /install +RUN chmod +x /install +RUN /install + +################################################################################ +# spdk +# +# Contains installed SPDK from build_base rpm packages. +# Does not contain dependencies required to build SPDK +################################################################################ +FROM fedora:33 AS spdk + +LABEL maintainer=spdk.io + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +# Copy SPDK's RPMs built during pre-install step. +# This allows to reduce final image size since we won't have any dependencies +# which are only required to perform build. +RUN mkdir /spdk-rpm +COPY --from=base /spdk-rpm/*.rpm /spdk-rpm/ +COPY --from=base /spdk-rpm/fio /spdk-rpm/ +# Wrap up the image +COPY core/build_base/post-install /install +RUN chmod +x /install +RUN /install + + +################################################################################ +# spdk-app +# +# This image in addition to installed SPDK binaries contains a script run at +# container boot. +# This script runs SPDK service. +################################################################################ +FROM spdk as spdk-app + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +RUN dnf install -y socat +COPY core/spdk-app/init /init + +ENTRYPOINT ["/init"] + + +################################################################################ +# storage-target +# +# This image should be place on a dedicated machine and responsible for exposing +# ideal storage target(SPDK Malloc bdev) over NVMe/TCP +# Configuration is performed by means of SPDK Json rpc. +################################################################################ +FROM spdk-app AS storage-target + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +################################################################################ +# proxy-container +# +# This image is placed on IPU and exposing storage-target NVMe/TCP devices +# It is responsible for creation of vhost virtio-blk devices and exposing them +# to hosts(KVM or physical ones) +################################################################################ +FROM spdk as proxy-container + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +RUN dnf install -y python socat +RUN python -m pip install grpcio grpcio-tools grpcio-reflection + +COPY core/proxy-container/hot_plug.proto /hot_plug.proto +COPY scripts/socket.sh /socket.sh +COPY core/proxy-container/hot_plug_grpc_server.py /hot_plug_grpc_server.py +COPY core/proxy-container/hot_plug_provider.py /hot_plug_provider.py +COPY core/proxy-container/hot_plug_main.py /hot_plug_main.py +COPY core/proxy-container/init /init +COPY core/proxy-container/hot-plug.sh /hot-plug.sh +COPY core/proxy-container/hot-unplug.sh /hot-unplug.sh +COPY --from=spdk-app /init /init_spdk + +# Currently proxy-container relies on spdk json rpc to execute comamnds. +# However, there is no ability to hot-plug a vhost virtio-blk to a vm +# by means of SPDK servce. +# A dedicate service is currently used to provide this hot-plug functionality +# as an addition to SPDK service. +# Later, both will be replaced by SPDK SMA later. +RUN python -m grpc_tools.protoc -I/ --python_out=. --grpc_python_out=/ \ + /hot_plug.proto + +ENTRYPOINT [ "/init" ] + + +################################################################################ +# host-target +# +# This image is responsible for running fio payload over different pci devices. +# It has to be placed into host(a vm for KVM case or physical host for IPU case) +# It uses gRPC to expose this service. +################################################################################ +FROM fedora:33 AS host-target + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +RUN dnf install -y python fio +RUN python -m pip install grpcio grpcio-tools grpcio-reflection + +COPY core/host-target/init /init +COPY core/host-target/*.py / +COPY core/host-target/host_target.proto /host_target.proto + +RUN python -m grpc_tools.protoc -I/ --python_out=. --grpc_python_out=/ \ + /host_target.proto + +ENTRYPOINT [ "/init" ] + +################################################################################ +# ipdk-unit-tests +################################################################################ +FROM fedora:33 AS ipdk-unit-tests + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +RUN dnf install -y python fio +RUN python -m pip install grpcio-reflection pyfakefs +COPY tests/ut/proxy-container /proxy-container/tests +COPY --from=proxy-container hot_plug_*pb2.py /proxy-container/generated/ +COPY --from=proxy-container hot_plug_*pb2_grpc.py /proxy-container/generated/ +COPY --from=proxy-container hot_plug_grpc_server.py /proxy-container/src/ +COPY --from=proxy-container hot_plug_provider.py /proxy-container/src/ + +COPY tests/ut/host-target /host-target/tests +COPY --from=host-target fio_runner.py /host-target/src/ +COPY --from=host-target pci_devices.py /host-target/src/ +COPY --from=host-target device_exerciser.py /host-target/src/ +COPY --from=host-target host_target_main.py /host-target/src/ +COPY --from=host-target host_target_grpc_server.py /host-target/src/ +COPY --from=host-target host_target_*pb2.py /host-target/generated/ +COPY --from=host-target host_target_*pb2_grpc.py /host-target/generated/ + +COPY tests/ut/run_all_unit_tests.sh / + +ENV PYTHONPATH=/proxy-container/generated:/proxy-container/src:/host-target/src:/host-target/generated + +ENTRYPOINT [ "/run_all_unit_tests.sh" ] diff --git a/build/storage/Makefile b/build/storage/Makefile index c0aaa7da..fb2c0bc2 100644 --- a/build/storage/Makefile +++ b/build/storage/Makefile @@ -2,4 +2,4 @@ #SPDX-License-Identifier: Apache-2.0 # -shellcheck: ;../networking/scripts/run-shellcheck.sh \ No newline at end of file +shellcheck: ;../scripts/run-shellcheck.sh diff --git a/build/storage/README.md b/build/storage/README.md index 24d06041..8ba3082a 100644 --- a/build/storage/README.md +++ b/build/storage/README.md @@ -1,3 +1,22 @@ -# Storage Builds +# Description +IPDK Storage solution targets the audience which has existing custom storage +protocols or implementations of standards such as NVMe-over-Fabrics and wants +to accelerate them by means of a common storage solution across vendors +and platforms. -Coming soon +# Main components +The storage solution is represented by the following containers: +- _proxy-container_ which plays the role of an IPU and exposes the block +devices to a host platform. +- _storage-target_ which is deployed on a remote storage node exposing +ideal targets(ramdrives). +- _host-target_ which is responsible for running of fio traffic through +`proxy-container` to an ideal target within `storage-target`. + +# Recipes +IPDK storage scenarios are described by recipes located in a dedicated +[recipes](recipes/README.md) directory. + +# Tests +The storage solution is covered by [unit tests](tests/ut/) and +[integration tests](tests/it/). diff --git a/build/storage/core/build_base/Dockerfile b/build/storage/core/build_base/Dockerfile deleted file mode 100644 index 1a443164..00000000 --- a/build/storage/core/build_base/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# -# NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE -# WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK -# - -FROM fedora:33 AS base - -# Generic args -ARG HTTP_PROXY -ARG HTTPS_PROXY -ARG NO_PROXY - -ENV http_proxy=$HTTP_PROXY -ENV https_proxy=$HTTPS_PROXY -ENV no_proxy=$NO_PROXY - -RUN dnf install -y git -RUN git clone https://github.com/spdk/spdk.git -RUN tar -czf spdk.tar.gz --exclude='docker/*' -C ./spdk . -COPY pre-install /install -RUN chmod +x /install -RUN /install - -# We are doing a multi-stage build here. This means that previous image, -# base, is going to end up as an intermediate one, untagged, - this -# image can be then manually removed (--force-rm doesn't work here. Go -# figure). -FROM fedora:33 AS spdk - -LABEL maintainer=spdk.io - -# Proxy configuration must be set for each build separately... -ARG HTTP_PROXY -ARG HTTPS_PROXY -ARG NO_PROXY - -ENV http_proxy=$HTTP_PROXY -ENV https_proxy=$HTTPS_PROXY -ENV no_proxy=$NO_PROXY - -# Copy SPDK's RPMs built during pre-install step. -COPY --from=base /tmp/*.rpm /tmp/ -COPY --from=base /tmp/fio /tmp/ -# Wrap up the image -COPY post-install /install -RUN chmod +x /install -RUN /install diff --git a/build/storage/core/build_base/post-install b/build/storage/core/build_base/post-install index 18252c73..e5da40d3 100644 --- a/build/storage/core/build_base/post-install +++ b/build/storage/core/build_base/post-install @@ -8,7 +8,7 @@ # set -e -dnf install -y /tmp/*.rpm +dnf install -y /spdk-rpm/*.rpm # Be nice for docker exec and link SPDK scripts|binaries under common PATH # location like /usr/bin. @@ -21,7 +21,7 @@ ln -s /usr/libexec/spdk/include/spdk /usr/include ln -s /usr/libexec/spdk/scripts/ /usr mkdir -p /usr/src/fio -mv /tmp/fio /usr/src/fio +mv /spdk-rpm/fio /usr/src/fio dnf clean all rm -f /tmp/*.rpm diff --git a/build/storage/core/build_base/pre-install b/build/storage/core/build_base/pre-install index ab69aa51..81bded24 100644 --- a/build/storage/core/build_base/pre-install +++ b/build/storage/core/build_base/pre-install @@ -7,28 +7,17 @@ # WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK # set -e -set -x -spdk_repo=$(mktemp -dt "spdk.XXXXXX") -spdk_tar=/spdk.tar.gz +spdk_repo=/spdk cleanup() { rm -f "$HOME/rpmbuild/rpm/x86_64/"*.rpm - rm -f "$spdk_tar" rm -rf "$spdk_repo" } trap 'cleanup' EXIT -if [[ ! -e $spdk_tar ]]; then - printf 'Missing %s\n' "$spdk_tar" >&2 - exit 1 -fi - -tar -C "$spdk_repo" -xf "$spdk_tar" -cd "$spdk_repo" && git checkout v21.10 && git submodule update --init && cd - - # Required for building RPM dnf install -y rpm-build @@ -48,6 +37,6 @@ DEPS="no" "$spdk_repo/rpmbuild/rpm.sh" \ --with-virtio \ --with-fio -mv "$HOME/rpmbuild/rpm/x86_64/"*.rpm /tmp -mv "/usr/src/fio/fio" /tmp +mv "$HOME/rpmbuild/rpm/x86_64/"*.rpm "/spdk-rpm/" +mv "/usr/src/fio/fio" "/spdk-rpm/" dnf clean all diff --git a/build/storage/core/host-target/device_exerciser.py b/build/storage/core/host-target/device_exerciser.py new file mode 100644 index 00000000..0c20dd88 --- /dev/null +++ b/build/storage/core/host-target/device_exerciser.py @@ -0,0 +1,25 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +import fio_runner +import pci_devices +import os + + +class DeviceExerciserError(RuntimeError): + pass + + +class DeviceExerciser: + def __init__(self, fio_runner=fio_runner.run_fio, + virtio_blk_detector=pci_devices.get_virtio_blk_path_by_pci_address): + self.fio_runner = fio_runner + self.virtio_blk_detector = virtio_blk_detector + + def run_fio(self, pci_address, fio_args): + try: + device_path = self.virtio_blk_detector(pci_address) + fio_args_with_device = fio_args + " --filename=" + device_path + return self.fio_runner(fio_args_with_device) + except BaseException as ex: + raise DeviceExerciserError(str(ex)) diff --git a/build/storage/core/host-target/fio_runner.py b/build/storage/core/host-target/fio_runner.py new file mode 100644 index 00000000..5ec8b8fb --- /dev/null +++ b/build/storage/core/host-target/fio_runner.py @@ -0,0 +1,24 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +import subprocess + + +class FioExecutionError(RuntimeError): + pass + + +def run_fio(fio_args, subprocess_run=subprocess.run): + fio_cmd = [] + try: + fio_cmd = ["fio"] + fio_args.split() + result = subprocess_run(fio_cmd, + capture_output=True, text=True) + if result.returncode != 0: + raise FioExecutionError("fio execution error: '" + + str(result.stdout) + "' | '" + + str(result.stderr) + "' ") + return result.stdout + except BaseException as ex: + raise FioExecutionError( + "Cannot execute cmd '" + " ".join(fio_cmd) + "' Error: " + str(ex)) diff --git a/build/storage/core/host-target/host_target.proto b/build/storage/core/host-target/host_target.proto new file mode 100644 index 00000000..dfb5c4ce --- /dev/null +++ b/build/storage/core/host-target/host_target.proto @@ -0,0 +1,20 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +syntax = "proto3"; + +package host_target; + +service HostTarget { + rpc RunFio (RunFioRequest) returns (RunFioReply) {} +} + +message RunFioRequest { + string pciAddress = 1; + string fioArgs = 2; +} + +message RunFioReply { + string fioOutput = 1; +} diff --git a/build/storage/core/host-target/host_target_grpc_server.py b/build/storage/core/host-target/host_target_grpc_server.py new file mode 100644 index 00000000..137f3d51 --- /dev/null +++ b/build/storage/core/host-target/host_target_grpc_server.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +from concurrent import futures + +import grpc +from grpc_reflection.v1alpha import reflection +import host_target_pb2 +import host_target_pb2_grpc +from device_exerciser import DeviceExerciser +from fio_runner import run_fio +from pci_devices import get_virtio_blk_path_by_pci_address + + +class InvalidHotPlugProvider(RuntimeError): + def __init__(self, message): + super().__init__(message) + + +class HostTargetService(host_target_pb2_grpc.HostTargetServicer): + def __init__(self, fio_runner, virtio_blk_detector): + super().__init__() + self.device_exerciser = DeviceExerciser( + fio_runner, virtio_blk_detector) + + def RunFio(self, request, context): + output = None + try: + output = self.device_exerciser.run_fio( + request.pciAddress, request.fioArgs) + except BaseException as ex: + context.set_code(grpc.StatusCode.FAILED_PRECONDITION) + context.set_details(str(ex)) + return host_target_pb2.RunFioReply(fioOutput=output) + + +def run_grpc_server(ip_address, port, server_creator=grpc.server): + try: + server = server_creator(futures.ThreadPoolExecutor(max_workers=10)) + host_target_pb2_grpc.add_HostTargetServicer_to_server( + HostTargetService(run_fio, get_virtio_blk_path_by_pci_address), server) + service_names = ( + host_target_pb2.DESCRIPTOR.services_by_name['HostTarget'].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(service_names, server) + server.add_insecure_port(ip_address + ":" + str(port)) + server.start() + server.wait_for_termination() + return 0 + except KeyboardInterrupt as ex: + return 0 + except BaseException as ex: + print("Couldn't run gRPC server. Error: " + str(ex)) + return 1 diff --git a/build/storage/core/host-target/host_target_main.py b/build/storage/core/host-target/host_target_main.py new file mode 100755 index 00000000..34ac16f3 --- /dev/null +++ b/build/storage/core/host-target/host_target_main.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import logging +import argparse + +from host_target_grpc_server import run_grpc_server + + +def parse_arguments(): + parser = argparse.ArgumentParser( + description='Runs service for hot-plug/hot-detach virtio-blk devices to vms') + parser.add_argument('--ip', required=True, + help='ip address the server listens to') + parser.add_argument('--port', type=int, default=50051, + help='port number the server listens to') + + args = parser.parse_args() + return args + + +if __name__ == '__main__': + logging.basicConfig() + + args = parse_arguments() + + run_grpc_server(args.ip, args.port) diff --git a/build/storage/core/host-target/init b/build/storage/core/host-target/init new file mode 100755 index 00000000..b7fb8dfc --- /dev/null +++ b/build/storage/core/host-target/init @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x && export GRPC_VERBOSITY=debug + +echo "Running host target server for ${IP_ADDR}:${PORT}" +/host_target_main.py --ip "${IP_ADDR}" --port "${PORT}" diff --git a/build/storage/core/host-target/pci_devices.py b/build/storage/core/host-target/pci_devices.py new file mode 100644 index 00000000..ec417bc6 --- /dev/null +++ b/build/storage/core/host-target/pci_devices.py @@ -0,0 +1,104 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +from lib2to3.pytree import Base +import os +import re +import glob + + +pci_validator = re.compile( + r'[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-1]{1}[0-9a-fA-F]{1}\.[0-7]{1}') + + +class PciAddress: + def validate_pci_address(pci_address): + return pci_validator.search(pci_address) != None + + def _parse_pci_address(pci_address): + split_pci_address = pci_address.replace('.', ':').split(":") + split_pci_address.reverse() + function = split_pci_address[0].strip() + device = split_pci_address[1].strip() + bus = split_pci_address[2].strip() + domain = split_pci_address[3].strip() + return (domain, bus, device, function) + + def __init__(self, pci_address) -> None: + if not PciAddress.validate_pci_address(pci_address): + raise InvalidPciAddress(pci_address + " is invalid") + self.domain, self.bus, self.device, self.function = \ + PciAddress._parse_pci_address(pci_address) + + def get_domain_bus_prefix(self): + return self.domain + ":" + self.bus + + def get_bus_device_function_address(self): + return self.bus + ":" + self.device + "." + self.function + + def get_full_address(self): + return self.domain + ":" + self.get_bus_device_function_address() + + +def get_directories(path): + if os.path.exists(path): + unused0, dirs, unused1 = next(os.walk(path)) + return dirs + else: + return None + + +def get_all_files_by_pattern(pattern): + return glob.glob(pattern) + + +def directory_find(atom, root='.'): + dirs = get_directories(root) + if dirs and atom in dirs: + return os.path.join(root, atom) + return None + + +class InvalidPciAddress(ValueError): + pass + + +class FailedPciDeviceDetection(RuntimeError): + pass + + +def get_virtio_blk_path_by_pci_address(pci_address): + addr = PciAddress(pci_address) + + domain_bus_dir = os.path.join("/sys/devices", "pci" + + addr.get_domain_bus_prefix()) + pci_dev_sysfs_dir = directory_find(addr.get_full_address(), + domain_bus_dir) + if pci_dev_sysfs_dir == None: + raise FailedPciDeviceDetection( + "No pci device with " + pci_address + " exists under " + domain_bus_dir) + block_directory_pattern = os.path.join(pci_dev_sysfs_dir, "virtio*/block") + block_device_matches = get_all_files_by_pattern(block_directory_pattern) + if len(block_device_matches) == 0: + raise FailedPciDeviceDetection( + "No devices found for pattern " + block_directory_pattern) + elif len(block_device_matches) > 1: + raise FailedPciDeviceDetection( + "Found more than one device for pattern" + + block_directory_pattern + " : " + str(block_device_matches)) + + devices = get_directories(block_device_matches[0]) + if not devices or len(devices) == 0: + raise FailedPciDeviceDetection( + "No device exist under " + block_directory_pattern + + " for pci device '" + pci_address + "'") + elif len(devices) > 1: + raise FailedPciDeviceDetection( + "Multiple devices are dected " + str(devices) + + " for pci address '" + pci_address +"'") + device_path = os.path.join("/dev", devices[0]) + if not os.path.exists(device_path): + raise FailedPciDeviceDetection( + "Device " + device_path + " does not exist") + + return device_path diff --git a/build/storage/core/proxy-container/hot-plug.sh b/build/storage/core/proxy-container/hot-plug.sh new file mode 100755 index 00000000..ea157028 --- /dev/null +++ b/build/storage/core/proxy-container/hot-plug.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x + +# shellcheck disable=SC1091 +source socket.sh + +function attach_virtio_blk() { + vm_monitor_socket=${1} + vhost_socket=${2} + id="${3}" + num_queues=2 + error_word="error" + + echo "Attaching ${vhost_socket} to vm socket ${vm_monitor_socket}" + add_chardev_cmd="chardev-add socket,id=${id},path=${vhost_socket}" + if ! send_command_over_unix_socket_and_no_word_found \ + "${vm_monitor_socket}" "${add_chardev_cmd}" 1 "${error_word}" ; then + return 1 + fi + + add_device_cmd="device_add vhost-user-blk-pci,chardev=${id},num-queues=${num_queues},id=${id}" + if ! send_command_over_unix_socket_and_no_word_found \ + "${vm_monitor_socket}" "${add_device_cmd}" 1 "${error_word}" ; then + return 1 + fi + return 0 +} + +if [[ $# != 3 ]] ; then + echo "Not all arguments are specified. " + exit 1 +fi + +attach_virtio_blk "${1}" "${2}" "${3}" +exit $? diff --git a/build/storage/core/proxy-container/hot-unplug.sh b/build/storage/core/proxy-container/hot-unplug.sh new file mode 100755 index 00000000..da33885b --- /dev/null +++ b/build/storage/core/proxy-container/hot-unplug.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x + +# shellcheck disable=SC1091 +source socket.sh + +function dettach_virtio_blk() { + vm_monitor_socket=${1} + id="${2}" + error_word="error" + + echo "Dettaching ${id} from ${vm_monitor_socket}" + device_del_cmd="device_del ${id}" + if ! send_command_over_unix_socket_and_no_word_found \ + "${vm_monitor_socket}" "${device_del_cmd}" 1 "${error_word}" ; then + return 1 + fi + + remove_chardev_cmd="chardev-remove ${id}" + if ! send_command_over_unix_socket_and_no_word_found \ + "${vm_monitor_socket}" "${remove_chardev_cmd}" 1 "${error_word}" ; then + return 1 + fi + return 0 +} + +if [[ $# != 2 ]] ; then + echo "Not all arguments are specified. " + exit 1 +fi + +dettach_virtio_blk "${1}" "${2}" +exit $? diff --git a/build/storage/core/proxy-container/hot_plug.proto b/build/storage/core/proxy-container/hot_plug.proto new file mode 100644 index 00000000..e90fe6a9 --- /dev/null +++ b/build/storage/core/proxy-container/hot_plug.proto @@ -0,0 +1,28 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +syntax = "proto3"; + +package hot_plug; + +service HotPlug { + rpc HotPlugVirtioBlk (HotPlugRequest) returns (HotPlugReply) {} + rpc HotUnplugVirtioBlk (HotUnplugRequest) returns (HotUnplugReply) {} +} + +message HotPlugRequest { + string vmId = 1; + string vhostVirtioBlkId = 2; +} + +message HotPlugReply { +} + +message HotUnplugRequest { + string vmId = 1; + string vhostVirtioBlkId = 2; +} + +message HotUnplugReply { +} \ No newline at end of file diff --git a/build/storage/core/proxy-container/hot_plug_grpc_server.py b/build/storage/core/proxy-container/hot_plug_grpc_server.py new file mode 100755 index 00000000..39b336d1 --- /dev/null +++ b/build/storage/core/proxy-container/hot_plug_grpc_server.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +from concurrent import futures + +import grpc +from grpc_reflection.v1alpha import reflection +import hot_plug_pb2 +import hot_plug_pb2_grpc + + +class InvalidHotPlugProvider(RuntimeError): + def __init__(self, message): + super().__init__(message) + + +class HotPlugService(hot_plug_pb2_grpc.HotPlugServicer): + def __init__(self, hot_plug_provider): + super().__init__() + if hot_plug_provider == None: + raise InvalidHotPlugProvider("HotPlugProvider cannot be None") + self.hot_plug_provider = hot_plug_provider + + def __execute_server_operation(self, request, context, operation): + try: + operation(request.vmId, request.vhostVirtioBlkId) + except BaseException as ex: + context.set_code(grpc.StatusCode.FAILED_PRECONDITION) + context.set_details(str(ex)) + return hot_plug_pb2.HotPlugReply() + + def HotPlugVirtioBlk(self, request, context): + return self.__execute_server_operation( + request, context, + self.hot_plug_provider.hot_plug_vhost_virtio_blk) + + def HotUnplugVirtioBlk(self, request, context): + return self.__execute_server_operation( + request, context, + self.hot_plug_provider.hot_unplug_vhost_virtio_blk) + + +def run_grpc_server(ip_address, port, hot_plug_provider): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + hot_plug_pb2_grpc.add_HotPlugServicer_to_server( + HotPlugService(hot_plug_provider), server) + service_names = ( + hot_plug_pb2.DESCRIPTOR.services_by_name['HotPlug'].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(service_names, server) + server.add_insecure_port(ip_address + ":" + str(port)) + server.start() + server.wait_for_termination() diff --git a/build/storage/core/proxy-container/hot_plug_main.py b/build/storage/core/proxy-container/hot_plug_main.py new file mode 100755 index 00000000..07b214e9 --- /dev/null +++ b/build/storage/core/proxy-container/hot_plug_main.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import logging +import argparse + +from hot_plug_grpc_server import run_grpc_server +from hot_plug_provider import HotPlugProvider + + +def parse_arguments(): + parser = argparse.ArgumentParser( + description='Runs service for hot-plug/hot-detach virtio-blk devices to vms') + parser.add_argument('--ip', required=True, + help='ip address the server listens to') + parser.add_argument('--port', type=int, default=50051, + help='port number the server listens to') + parser.add_argument('--shared-dir', type=str, required=True, + help='Directory path to shared with Host resources') + parser.add_argument('--host-shared-dir', type=str, required=True, + help='Directory path to shared with container resources') + + args = parser.parse_args() + return args + + +if __name__ == '__main__': + logging.basicConfig() + + args = parse_arguments() + + ip_address = args.ip + port = args.port + hot_plug_provider = HotPlugProvider(args.shared_dir, args.host_shared_dir) + run_grpc_server(ip_address, port, hot_plug_provider) diff --git a/build/storage/core/proxy-container/hot_plug_provider.py b/build/storage/core/proxy-container/hot_plug_provider.py new file mode 100755 index 00000000..55aefa09 --- /dev/null +++ b/build/storage/core/proxy-container/hot_plug_provider.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +from multiprocessing.sharedctypes import Value +import os.path +import errno +import subprocess +from pathlib import Path + + +class ActionExecutionError(RuntimeError): + def __init__(self, message): + super().__init__(message) + + +class HotPlugProvider: + def __init__(self, shared_dir_path, host_shared_dir_path): + if shared_dir_path == None: + raise ValueError("shared_dir_path value cannot be None") + if host_shared_dir_path == None: + raise ValueError("host_shared_dir_path value cannot be None") + self.shared_dir_path = shared_dir_path + self.host_shared_dir_path = host_shared_dir_path + + def _hot_plug_action(self, vm_monitor, + vhost_virtio_blk, device_id): + vm_monitor_path = os.path.join(self.shared_dir_path, vm_monitor) + vhost_path = os.path.join(self.host_shared_dir_path, vhost_virtio_blk) + ret = subprocess.call("/hot-plug.sh " + vm_monitor_path + + " " + vhost_path + + " " + str(device_id), shell=True) + if ret != 0: + raise ActionExecutionError("Error at hot-plug execution") + + def _hot_unplug_action(self, vm_monitor, + unused, device_id): + vm_monitor_path = os.path.join(self.shared_dir_path, vm_monitor) + ret = subprocess.call("/hot-unplug.sh " + vm_monitor_path + + " " + str(device_id), shell=True) + if ret != 0: + raise ActionExecutionError("Error at hot-unplug execution") + + def _adjust_vm_monitor(vm_monitor): + return vm_monitor.strip() + + def _adjust_vhost_virtio_blk(vhost_virtio_blk): + return vhost_virtio_blk.strip() + + def __is_socket(socket_path): + return Path(socket_path).is_socket() + + def __check_socket_path(path): + if not HotPlugProvider.__is_socket(path): + raise FileNotFoundError(errno.ENOENT, os.strerror( + errno.ENOENT), path) + + def _validate_vhost_virtio_blk(self, vhost_virtio_blk): + HotPlugProvider.__check_socket_path(os.path.join( + self.shared_dir_path, vhost_virtio_blk)) + + def _validate_vm_monitor(self, vm_monitor): + HotPlugProvider.__check_socket_path(os.path.join( + self.shared_dir_path, vm_monitor)) + + def _validate_input(self, vm_monitor, vhost_virtio_blk): + self._validate_vhost_virtio_blk(vhost_virtio_blk) + self._validate_vm_monitor(vm_monitor) + + def _generate_device_id(self, vhost_virtio_blk): + return vhost_virtio_blk.strip() + + def _perform_disk_operation(self, vm_monitor, vhost_virtio_blk, disk_operation): + if vm_monitor == None: + raise ValueError("vm_monitor value cannot be None") + if vhost_virtio_blk == None: + raise ValueError("vhost_virtio_blk value cannot be None") + + vm_monitor = HotPlugProvider._adjust_vm_monitor(vm_monitor) + vhost_virtio_blk = HotPlugProvider._adjust_vhost_virtio_blk(vhost_virtio_blk) + + self._validate_input(vm_monitor, vhost_virtio_blk) + + device_id = self._generate_device_id(vhost_virtio_blk) + + try: + disk_operation(vm_monitor, vhost_virtio_blk, device_id) + except BaseException as ex: + raise ActionExecutionError(str(ex)) + + def hot_plug_vhost_virtio_blk(self, vm_monitor, vhost_virtio_blk): + self._perform_disk_operation( + vm_monitor, vhost_virtio_blk, self._hot_plug_action) + + def hot_unplug_vhost_virtio_blk(self, vm_monitor, vhost_virtio_blk): + self._perform_disk_operation( + vm_monitor, vhost_virtio_blk, self._hot_unplug_action) diff --git a/build/storage/core/proxy-container/init b/build/storage/core/proxy-container/init new file mode 100755 index 00000000..13c433f6 --- /dev/null +++ b/build/storage/core/proxy-container/init @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x && export GRPC_VERBOSITY=debug + +spdk_pid_file="/spdk_pid" +rm -f "$spdk_pid_file" + +/init_spdk "$@" & + +function wait_for_spdk_pid_file() { + spdk_pid_file="$1" + max_wait_for_pid_counter="$2" + wait_for_pid_counter=0 + while [ ! -f "$spdk_pid_file" ] ; do + if [ "$wait_for_pid_counter" -ge "$max_wait_for_pid_counter" ] ; then + return 1 + fi + wait_for_pid_counter=$(( wait_for_pid_counter + 1 )) + sleep 1 + done + return 0 +} + +function is_pid_running() { + pid="$1" + kill -0 "$pid" &> /dev/null + return $? +} + +wait_max_sec=5 +if ! wait_for_spdk_pid_file "$spdk_pid_file" "$wait_max_sec" ; then + echo "Failed to start spdk-app or socat to expose port." >&2 + exit 1 +fi + +pid=$(cat "$spdk_pid_file") +if ! is_pid_running "$pid" ; then + echo "Spdk is not running." >&2 + exit 1 +fi + +HOT_PLUG_SERVICE_IP_ADDR="${HOT_PLUG_SERVICE_IP_ADDR:-"0.0.0.0"}" +HOT_PLUG_SERVICE_PORT="${HOT_PLUG_SERVICE_PORT:-50051}" + +/hot_plug_main.py --ip "${HOT_PLUG_SERVICE_IP_ADDR}" --port "${HOT_PLUG_SERVICE_PORT}" \ + --shared-dir /ipdk-shared --host-shared-dir "${HOST_SHARED_VOLUME}" diff --git a/build/storage/core/spdk-app/Dockerfile b/build/storage/core/spdk-app/Dockerfile deleted file mode 100644 index aba6b9c6..00000000 --- a/build/storage/core/spdk-app/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# -# NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE -# WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK -# - -FROM spdk - -# Generic args -ARG HTTP_PROXY -ARG HTTPS_PROXY -ARG NO_PROXY - -ENV http_proxy=$HTTP_PROXY -ENV https_proxy=$HTTPS_PROXY -ENV no_proxy=$NO_PROXY - -COPY init /init - -ENTRYPOINT ["/init"] diff --git a/build/storage/core/spdk-app/init b/build/storage/core/spdk-app/init index 0fde2fa8..fd74b4aa 100755 --- a/build/storage/core/spdk-app/init +++ b/build/storage/core/spdk-app/init @@ -10,6 +10,8 @@ [ "$DEBUG" == 'true' ] && set -x app=spdk_tgt args=() limit_args=() +SPDK_PORT="${SPDK_PORT:-5260}" +SPDK_IP_ADDR="${SPDK_IP_ADDR:-"0.0.0.0"}" # Override default app if [[ -n $SPDK_APP ]]; then @@ -35,7 +37,21 @@ if [[ -e /config ]]; then args+=("--json" "/config") fi -# Wait a bit to make sure ip is in place -sleep 2s +function is_port_on_ip_addr_open() { + ip_addr="$1" + port="$2" + timeout 1 bash -c "cat < /dev/null > /dev/tcp/${ip_addr}/${port}" &> /dev/null + return $? +} -exec "$app" "${args[@]}" +if is_port_on_ip_addr_open "$SPDK_IP_ADDR" "$SPDK_PORT"; then + echo "Cannot run socat for ${SPDK_IP_ADDR} and port ${SPDK_PORT}. Port is already in use." >&2 + exit 1 +fi + +socat TCP-LISTEN:"${SPDK_PORT}",reuseaddr,bind="${SPDK_IP_ADDR}",fork \ + UNIX-CLIENT:/var/tmp/spdk.sock & + +spdk_pid_file="/spdk_pid" +rm -f "$spdk_pid_file" +$app "${args[@]}" -r /var/tmp/spdk.sock -f "$spdk_pid_file" diff --git a/build/storage/recipes/README.md b/build/storage/recipes/README.md new file mode 100644 index 00000000..326dab70 --- /dev/null +++ b/build/storage/recipes/README.md @@ -0,0 +1,13 @@ +# Supported recipes +Currently, the following scenarios are represented: + +* [hot-plug](./hot-plug.md) - describes how to deploy containers between +different machines and demonstrates virtio-blk hot-plug to a running host. +* [fio](./fio.md) - extends `hot-plug` scenario and describes how to run +fio traffic by means of `host-target` container through a hot-plugged virtio-blk +device. + +In both cases host target platform is implied as KVM. + +The picture below demonstrates the configuration exercised in these recipes +![System configuration for recipes](./system_configuration.svg "System configuration for recipes") diff --git a/build/storage/recipes/environment_setup.md b/build/storage/recipes/environment_setup.md new file mode 100644 index 00000000..a35bd666 --- /dev/null +++ b/build/storage/recipes/environment_setup.md @@ -0,0 +1,170 @@ +# Environment preparation +Currently the recipe environment should be represented by 2 separate machines, +referred as `storage-target-platform` and `proxy-container-platform`, +located in the same network. For each of them the steps from +[System setup](#system-setup) section have to be performed. +The containers running on those platforms are named `storage-target` and +`proxy-container` respectively. + +# System setup +The basic requirement for running the recipes is a modern Linux distribution +with docker support. +So far, the recipe has been successfully tested on the following systems: +- Fedora 33 +- Ubuntu 18.04 + +To run the recipe some basic system preparations are required, in particular: + +## Virtualization support +Make sure that VT-x/AMD-v support is enabled in BIOS +``` +$ lscpu | grep -i virtualization +Virtualization: VT-x +``` +and that kvm modules are loaded +``` +$ lsmod | grep -i kvm +kvm_intel 217088 0 +kvm 614400 1 kvm_intel +irqbypass 16384 1 kvm +``` + +## Tool installation +Make sure that following tools are installed on your system or install them +using the corresponding package manager. + +### wget +Installation on Fedora +``` +$ sudo dnf install wget +``` +or on Ubuntu +``` +$ sudo apt install wget +``` + +### docker +``` +$ sudo dnf install docker +``` +or +``` +$ sudo apt install docker +``` +**Note:** +Make sure that required proxy settings are configured for docker. +Please, refer to [this](https://docs.docker.com/config/daemon/systemd/#httphttps-proxy) +page. + +### libguestfs-tools +``` +$ sudo dnf install libguestfs-tools-c +``` +or +``` +$ sudo apt install libguestfs-tools +``` + +**Note:** +To run `libguestfs` tools without root privileges, you may need to workaround +the problem of +Linux kernel image not being readable by issuing: +``` +$ sudo chmod +r /boot/vmlinuz-* +``` + +## Setting security policies +Make sure that required security permissions are configured to enable VM network +connectivity or disable them temporarily. +On Fedora SELinux can be disabled by means of the following command +``` +$ sudo setenforce 0 +``` +For Ubuntu AppArmor is often used and can be disabled by issuing +``` +$ sudo systemctl stop apparmor +``` + +# Deploy containers on the machines + +1. Download repositories on both platforms: `proxy-container-platform` and +`storage-target-platform` + +2. Run docker containers providing configuration scripts: + +On `storage-target-platform` +``` +$ scripts/run_storage_target_container.sh +``` + +On `proxy-container-platform` +``` +$ SHARED_VOLUME= \ +scripts/run_proxy_container.sh +``` + +`SHARED_VOLUME` points to a directory where vhost storage device and vm monitor +will be exposed. + + +It is also possible to specify ip addresses and ports where spdk service is +exposed on by specifying `SPDK_IP_ADDR` and `SPDK_PORT` environment variables. +Those environment variables can be applied to both `storage-target` and +`proxy-container` +e.g. + +``` +SPDK_IP_ADDR="127.0.0.1" SPDK_PORT=5261 scripts/run_storage_target_container.sh +``` +or +``` +$ SPDK_IP_ADDR="127.0.0.1" SPDK_PORT=5262 \ +SHARED_VOLUME= \ +scripts/run_proxy_container.sh +``` +By default, `SPDK_IP_ADDR` is set to `0.0.0.0` and `SPDK_PORT` is set to `5260` + +3. Run the vm instance on `proxy-container-platform` platform +``` +$ SHARED_VOLUME= scripts/vm/run_vm.sh +``` + +`SHARED_VOLUME` points to a directory where vhost storage device and vm monitor +will be exposed(exactly one specified in `SHARED_VOLUME` to run +`proxy-container`). + + +Finally vm console will be opened. + + +login:password pair for the vm is `root:root`. +Run `host-target` container within the vm console +``` +$ run_host_target_container.sh & +``` + +4. Prepare environment to send commands to the storage containers. +For that purpose we need to have spdk rpc.py and grpc-cli tools available. +`test-driver` image from the [integration tests](../tests/it/README.md#introduction) +fits for this purpose well since it contains all required tools. +Let's use `test-driver` image instance to send all required commands. +In the recipes this `test-driver` will be referred as `cmd-sender` to +increase comprehension of the text. However, we should keep in mind that +`cmd-sender` is a running instance of `test-driver` image. + +To use test-driver [install docker-compose](../tests/it/README.md#docker-compose-setup) +and run the integration tests to build that container on `proxy-container-platform` +machine. +``` +$ tests/it/run.sh +``` + +Run `test-driver` on `proxy-container-platform` machine +``` +$ docker run -it --privileged --network host --entrypoint /bin/bash test-driver +``` + +Source supplementary scripts in running `test-driver` container +``` +$ source /scripts/disk_infrastructure.sh +``` diff --git a/build/storage/recipes/fio.md b/build/storage/recipes/fio.md new file mode 100644 index 00000000..53119191 --- /dev/null +++ b/build/storage/recipes/fio.md @@ -0,0 +1,28 @@ +# Fio recipe + +This recipe describes how to run fio from Host to an ideal target +with IPDK containers. + +For this recipe 2 machines required. +They are referred as `storage-target-platform` and `proxy-container-platform`. +The containers running on those platforms are named `storage-target` and +`proxy-container` respectively. + +To apply this scenario the following steps need to be applied: + +1. Perform all steps described in [environment setup](environment_setup.md) + +2. Run [hot-plug scenario](hot-plug.md) until a virtio-blk device is attached to +the vm(step 5) + +3. Run fio. Execute the following command from `cmd-sender` +``` +$ no_grpc_proxy= grpc_cli call :50051 \ +RunFio "pciAddress: '0000:00:04.0' fioArgs: '--direct=1 --rw=randrw --bs=4k --ioengine=libaio --iodepth=256 --runtime=1 --numjobs=4 --time_based --group_reporting --name=iops-test-job'" +``` + +Expected output +``` +connecting to 192.168.53.76:50051 +fioOutput: "iops-test-job: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=256\n...\nfio-3.21\nStarting 4 processes\n\niops-test-job: (groupid=0, jobs=4): err= 0: pid=18: Fri Feb 25 07:18:16 2022\n read: IOPS=18.0k, BW=74.2MiB/s (77.8MB/s)(742MiB/10008msec)\n slat (usec): min=2, max=11636, avg=102.74, stdev=479.43\n clat (usec): min=6703, max=62683, avg=26796.45, stdev=9296.56\n lat (usec): min=6707, max=62704, avg=26899.45, stdev=9327.52\n clat percentiles (usec):\n | 1.00th=[11469], 5.00th=[12125], 10.00th=[12780], 20.00th=[16909],\n | 30.00th=[22938], 40.00th=[25035], 50.00th=[26870], 60.00th=[29492],\n | 70.00th=[31851], 80.00th=[34866], 90.00th=[38536], 95.00th=[41681],\n | 99.00th=[48497], 99.50th=[51643], 99.90th=[56886], 99.95th=[58459],\n | 99.99th=[60556]\n bw ( KiB/s): min=55800, max=100312, per=99.78%, avg=75797.47, stdev=2986.36, samples=76\n iops : min=13950, max=25078, avg=18949.37, stdev=746.59, samples=76\n write: IOPS=18.0k, BW=74.1MiB/s (77.7MB/s)(742MiB/10008msec); 0 zone resets\n slat (usec): min=2, max=9856, avg=103.06, stdev=476.92\n clat (usec): min=5141, max=62700, avg=26812.92, stdev=9302.68\n lat (usec): min=6695, max=62704, avg=26916.24, stdev=9334.34\n clat percentiles (usec):\n | 1.00th=[11469], 5.00th=[12125], 10.00th=[12780], 20.00th=[16909],\n | 30.00th=[22938], 40.00th=[25035], 50.00th=[26870], 60.00th=[29492],\n | 70.00th=[31851], 80.00th=[34866], 90.00th=[38536], 95.00th=[41681],\n | 99.00th=[48497], 99.50th=[51643], 99.90th=[56361], 99.95th=[57934],\n | 99.99th=[60031]\n bw ( KiB/s): min=56728, max=100464, per=99.83%, avg=75788.21, stdev=2943.55, samples=76\n iops : min=14182, max=25116, avg=18947.05, stdev=735.89, samples=76\n lat (msec) : 10=0.06%, 20=24.99%, 50=74.23%, 100=0.72%\n cpu : usr=2.21%, sys=4.23%, ctx=41669, majf=0, minf=55\n IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%\n submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%\n complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%\n issued rwts: total=190063,189952,0,0 short=0,0,0,0 dropped=0,0,0,0\n latency : target=0, window=0, percentile=100.00%, depth=256\n\nRun status group 0 (all jobs):\n READ: bw=74.2MiB/s (77.8MB/s), 74.2MiB/s-74.2MiB/s (77.8MB/s-77.8MB/s), io=742MiB (778MB), run=10008-10008msec\n WRITE: bw=74.1MiB/s (77.7MB/s), 74.1MiB/s-74.1MiB/s (77.7MB/s-77.7MB/s), io=742MiB (778MB), run=10008-10008msec\n\nDisk stats (read/write):\n vda: ios=187480/187373, merge=0/0, ticks=1237509/1230145, in_queue=2467655, util=99.67%\n" +``` diff --git a/build/storage/recipes/hot-plug.md b/build/storage/recipes/hot-plug.md new file mode 100644 index 00000000..3fd0ff63 --- /dev/null +++ b/build/storage/recipes/hot-plug.md @@ -0,0 +1,109 @@ +# Hot-plug recipe +This recipe describes how to perform virtio-blk hot-plug to a running KVM host. +The virtio-blk device is backed up by an ideal target on `storage-target-platform` +machine exposed over NVMe/TCP. + +For this recipe 2 machines required. They are referred as +`storage-target-platform` and `proxy-container-platform`. +The containers running on those platforms are named `storage-target` and +`proxy-container` respectively. + +To apply this scenario the following steps need to be applied: + +1. Perform all steps described in [environment setup](environment_setup.md) + +2. Make sure there is no virtio-blk device attached +Run in [vm console](environment_setup.md#vm-console) +``` +$ lsblk +``` +Expected output +``` +NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT +sda 8:0 0 4G 0 disk +└─sda1 8:1 0 4G 0 part / +``` + +3. Create subsystem and expose it to `proxy-container` +Send from your `cmd-sender` +``` +$ create_subsystem_and_expose_to_another_machine \ + nqn.2016-06.io.spdk:cnode0 \ + Nvme0 +``` +or +``` +$ create_subsystem_and_expose_to_another_machine \ + nqn.2016-06.io.spdk:cnode0 \ + Nvme0 \ + +``` +Where `storage_target_SPDK_PORT` is non-default `SPDK_PORT` specified for +`run_storage_target_container.sh` +Where `proxy_container_SPDK_PORT` is non-default `SPDK_PORT` specified for +`run_proxy_container.sh` +Please see [here](environment_setup.md#non-default-port) + +4. Create ramdrive on `storage-target` and attach to the subsystem +Send from your `cmd-sender` +``` +$ create_ramdrive_and_attach_as_ns_to_subsystem \ + Malloc0 16 nqn.2016-06.io.spdk:cnode0 +``` +or +``` +$ create_ramdrive_and_attach_as_ns_to_subsystem \ + Malloc0 16 nqn.2016-06.io.spdk:cnode0 \ + +``` +Where `storage_target_SPDK_PORT` is non-default `SPDK_PORT` specified for +`run_storage_target_container.sh` +Please see [here](environment_setup.md#non-default-port) + +5. Attach exposed ramdrive to the vm +Send from your `cmd-sender` +``` +$ attach_ns_as_virtio_blk \ + VirtioBlk0 Nvme0n1 50051 \ + vm_monitor +``` +or +``` +$ attach_ns_as_virtio_blk \ + VirtioBlk0 Nvme0n1 50051 \ + vm_monitor +``` +Where `proxy_container_SPDK_PORT` is non-default `SPDK_PORT` specified for +`run_proxy_container.sh` +Please see [here](environment_setup.md#non-default-port) + +6. Check if virtio-blk is attached to the vm +Open the [vm console](environment_setup.md#vm-console) machine and run the following command +``` +$ lsblk +``` +The expected output is +``` +NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT +sda 8:0 0 4G 0 disk +└─sda1 8:1 0 4G 0 part / +vda 252:0 0 16M 0 disk +``` + +7. Perform unplug +Run the command below on `cmd-sender` to hot-unplug device +``` +dettach_virtio_blk 50051 vm_monitor VirtioBlk0 +``` + +8. Check there is no virtio-blk device +Open the [vm console](environment_setup.md#vm-console) and run the following command +``` +$ lsblk +``` +The expected output is +``` +NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT +sda 8:0 0 4G 0 disk +└─sda1 8:1 0 4G 0 part / +``` diff --git a/build/storage/recipes/system_configuration.svg b/build/storage/recipes/system_configuration.svg new file mode 100644 index 00000000..75aa28ae --- /dev/null +++ b/build/storage/recipes/system_configuration.svg @@ -0,0 +1,425 @@ + + + + + + + + + + + + + + Page-1 + + + + + Round Corner Rectangle.37 + Platform 1 + + + + + + + + + + + + + + + + + + + + + + Platform 1 + + Round Corner Rectangle.38 + Platform 2 + + + + + + + + + + + + + + + + + + + + + + Platform 2 + + Round Corner Rectangle.3 + QEMU + + + + + + + + + + + + + + + + + + + + + + QEMU + + Round Corner Rectangle.5 + proxy-container + + + + + + + + + + + + + + + + + + + + + + proxy-container + + Round Corner Rectangle.6 + storage-target + + + + + + + + + + + + + + + + + + + + + + storage-target + + Sheet.7 + vhost/virtio-blk + + + + vhost/virtio-blk + + Sheet.8 + qemu monitor + + + + qemu monitor + + Dynamic connector.22 + + + + Sheet.15 + NVMe/TCP + + + + NVMe/TCP + + Round Corner Rectangle.24 + + + + + + + + + + + + + + + + + + + + + + Sheet.17 + + + + Sheet.18 + containers + + + + containers + + Sheet.19 + shared objects/ docker volumes + + + + shared objects/ docker volumes + + Sheet.22 + Hot-plug service + + + + Hot-plugservice + + Sheet.23 + SPDK rpc + + + + SPDKrpc + + Sheet.24 + SPDK rpc + + + + SPDKrpc + + Dynamic connector.1009 + + + + Sheet.35 + + + + Sheet.36 + services + + + + services + + Dynamic connector.1005 + + + + Dynamic connector.40 + + + + Dynamic connector.41 + + + + Dynamic connector.42 + + + + Round Corner Rectangle.43 + host-target + + + + + + + + + + + + + + + + + + + + + + host-target + + Round Corner Rectangle + fio + + + + + + + + + + + + + + + + + + + + + + fio + + Dynamic connector.21 + + + + Sheet.44 + fio service + + + + fio service + + Dynamic connector.45 + + + + + + + + User.9 + + Sheet.58 + + + + + + + + + + Sheet.59 + + + + Sheet.60 + + + + + + + Sheet.61 + + + + + + + + Sheet.62 + user + + + + user + + diff --git a/build/storage/scripts/allocate_hugepages.sh b/build/storage/scripts/allocate_hugepages.sh new file mode 100755 index 00000000..eadf35f1 --- /dev/null +++ b/build/storage/scripts/allocate_hugepages.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +[ "$DEBUG" == 'true' ] && set -x +set -e + +required_number_of_pages="${REQUIRED_NUMBER_OF_2048KB_PAGES:-3072}" + +number_of_2mb_hugepages_folder=/sys/kernel/mm/hugepages/hugepages-2048kB +number_of_2mb_hugepages_provider=${number_of_2mb_hugepages_folder}/nr_hugepages + +if [ -d ${number_of_2mb_hugepages_folder} ] ; then + number_of_2mb_pages=$(cat ${number_of_2mb_hugepages_provider}) + if [ "$number_of_2mb_pages" -lt "$required_number_of_pages" ] ; then + echo "${required_number_of_pages}" | \ + sudo tee ${number_of_2mb_hugepages_provider} + fi +fi diff --git a/build/storage/scripts/build_container.sh b/build/storage/scripts/build_container.sh new file mode 100755 index 00000000..f31a93b4 --- /dev/null +++ b/build/storage/scripts/build_container.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x +set -e + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +root_dir="${script_dir}/.." +# shellcheck disable=SC1091 +source "${script_dir}"/spdk_version.sh +"${script_dir}"/prepare_to_build.sh + +DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} + +build_proxies="--build-arg HTTP_PROXY=${HTTP_PROXY} \ + --build-arg HTTPS_PROXY=${HTTPS_PROXY} \ + --build-arg NO_PROXY=${NO_PROXY}" +spdk_version_build_arg="--build-arg SPDK_VERSION=$(get_spdk_version)" + +join_by() { + local d=${1-} f=${2-} + if shift 2; then + printf %s "$f" "${@/#/$d}" + fi +} + +container_to_build="${1}" + +possible_containers=("storage-target" "proxy-container" "ipdk-unit-tests" "host-target") +if [[ " ${possible_containers[*]} " =~ ${container_to_build} ]]; then + export DOCKER_BUILDKIT="${DOCKER_BUILDKIT}" + + docker_build="docker build ${build_proxies} \ + ${spdk_version_build_arg} \ + -t ${container_to_build} --target ${container_to_build} ${root_dir}" + $docker_build +else + echo "Unknown container '${container_to_build}'" + possible_containers_as_string=$(join_by ", " "${possible_containers[@]}") + echo "Possible containers: ${possible_containers_as_string}" + exit 1 +fi + diff --git a/build/storage/scripts/disk_infrastructure.sh b/build/storage/scripts/disk_infrastructure.sh new file mode 100755 index 00000000..5da04f90 --- /dev/null +++ b/build/storage/scripts/disk_infrastructure.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +export DEFAULT_SPDK_PORT=5260 + +function get_number_of_virtio_blk() { + cmd="lsblk --output \"NAME,VENDOR,SUBSYSTEMS\"" + out=$( send_command_over_unix_socket "${1}" "${cmd}" 1 ) + number_of_virio_blk_devices=$(echo "${out}" | grep -c "block:virtio:pci") + echo "${number_of_virio_blk_devices}" +} + +function is_virtio_blk_attached() { + number_of_virio_blk_devices=$(get_number_of_virtio_blk "${1}") + + if [[ "${number_of_virio_blk_devices}" == 0 ]]; then + echo "virtio-blk is not found" + return 1 + else + echo "virtio-blk is found" + return 0 + fi +} + +function is_virtio_blk_not_attached() { + if is_virtio_blk_attached "${1}"; then + return 1 + fi + + return 0 +} + +function check_number_of_virtio_blk_devices() { + vm_serial="${1}" + expected_number_of_devices="${2}" + number_of_devices=$(get_number_of_virtio_blk "${vm_serial}") + if [[ "${number_of_devices}" != "${expected_number_of_devices}" ]]; then + echo "Required number of devices '${expected_number_of_devices}' does" + echo "not equal to actual number of devices '${number_of_devices}'" + return 1 + else + echo "Number of attached virtio-blk devices is '${number_of_devices}'" + return 0 + fi +} + +function attach_virtio_blk() { + proxy_ip="${1}" + hot_plug_service_port="${2}" + vm_monitor="${3}" + virtio_blk_socket="${4}" + no_grpc_proxy="" grpc_cli call "${proxy_ip}":"${hot_plug_service_port}" HotPlugVirtioBlk \ + "vmId: '${vm_monitor}' vhostVirtioBlkId: '${virtio_blk_socket}'" + return "$?" +} + +function dettach_virtio_blk() { + proxy_ip="${1}" + hot_plug_service_port="${2}" + vm_monitor="${3}" + virtio_blk_socket="${4}" + no_grpc_proxy="" grpc_cli call "${proxy_ip}":"${hot_plug_service_port}" HotUnplugVirtioBlk \ + "vmId: '${vm_monitor}' vhostVirtioBlkId: '${virtio_blk_socket}'" + return $? +} + +function create_and_expose_sybsystem_over_tcp() { + ip_addr="${1}" + nqn="${2}" + storage_target_port="${3:-"$DEFAULT_SPDK_PORT"}" + + rpc.py -s "${ip_addr}" -p "$storage_target_port" \ + nvmf_create_subsystem "${nqn}" \ + -s SPDK00000000000001 -a + rpc.py -s "${ip_addr}" -p "$storage_target_port" \ + nvmf_create_transport -t TCP -u 8192 + rpc.py -s "${ip_addr}" -p "$storage_target_port" \ + nvmf_subsystem_add_listener "${nqn}" -t TCP \ + -f IPv4 -a "${ip_addr}" -s 4420 +} + +function create_ramdrive_and_attach_as_ns_to_subsystem() { + ip_addr="${1}" + ramdrive_name="${2}" + number_of_512b_blocks="${3}" + nqn="${4}" + storage_target_port="${5:-"$DEFAULT_SPDK_PORT"}" + + rpc.py -s "${ip_addr}" -p "${storage_target_port}" \ + bdev_malloc_create -b "${ramdrive_name}" \ + "${number_of_512b_blocks}" 512 + rpc.py -s "${ip_addr}" -p "${storage_target_port}" \ + nvmf_subsystem_add_ns "${nqn}" "${ramdrive_name}" +} + +function attach_controller() { + ip_addr="${1}" + storage_target_ip_addr="${2}" + nqn="${3}" + controller_name="${4}" + proxy_container_port="${5:-"$DEFAULT_SPDK_PORT"}" + + rpc.py -s "${ip_addr}" -p "$proxy_container_port" \ + bdev_nvme_attach_controller -b "${controller_name}" -t TCP \ + -f ipv4 -a "${storage_target_ip_addr}" -s 4420 -n "${nqn}" +} + +function create_disk() { + ip_addr="${1}" + vhost_path="${2}" + ns="${3}" + proxy_container_port="${4:-"$DEFAULT_SPDK_PORT"}" + rpc.py -s "${ip_addr}" -p "${proxy_container_port}" \ + vhost_create_blk_controller \ + "${vhost_path}" "${ns}" +} + +function attach_ns_as_virtio_blk() { + proxy_ip="${1}" + vhost_name="${2}" + exposed_controller_ns="${3}" + hot_plug_service_port="${4}" + vm_monitor="${5}" + proxy_container_port="${6:-"$DEFAULT_SPDK_PORT"}" + + create_disk "${proxy_ip}" \ + "/ipdk-shared/${vhost_name}" "${exposed_controller_ns}" \ + "${proxy_container_port}" + + attach_virtio_blk "${proxy_ip}" "${hot_plug_service_port}" \ + "${vm_monitor}" "${vhost_name}" +} + +function create_subsystem_and_expose_to_another_machine() { + storage_target_ip="${1}" + nqn="${2}" + proxy_ip="${3}" + controller_name="${4}" + storage_target_port="${5:-"$DEFAULT_SPDK_PORT"}" + proxy_container_port="${6:-"$DEFAULT_SPDK_PORT"}" + create_and_expose_sybsystem_over_tcp "${storage_target_ip}" "${nqn}" \ + "${storage_target_port}" + attach_controller "${proxy_ip}" "${storage_target_ip}" "${nqn}" \ + "${controller_name}" "${proxy_container_port}" +} diff --git a/build/storage/scripts/prepare_to_build.sh b/build/storage/scripts/prepare_to_build.sh new file mode 100755 index 00000000..3dbcdf79 --- /dev/null +++ b/build/storage/scripts/prepare_to_build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x + +git submodule update --init --recursive --force diff --git a/build/storage/scripts/run_host_target_container.sh b/build/storage/scripts/run_host_target_container.sh new file mode 100755 index 00000000..45f13734 --- /dev/null +++ b/build/storage/scripts/run_host_target_container.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x + +declare https_proxy +declare http_proxy +declare no_proxy +IP_ADDR="${IP_ADDR:-"0.0.0.0"}" +PORT="${PORT:-50051}" + +scripts_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +function print_error_and_exit() { + echo "${1}" + exit 1 +} + +function check_all_variables_are_set() { + number_re='^[0-9]+$' + ip_addr_re='^(([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))\.){3}([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))$' + + if [ -z "${IP_ADDR}" ]; then + print_error_and_exit "IP_ADDR is not set" + elif ! [[ "${IP_ADDR}" =~ ${ip_addr_re} ]]; then + print_error_and_exit "IP_ADDR does not represent ip address" + elif [ -z "${PORT}" ]; then + print_error_and_exit "PORT is not set" + elif ! [[ "${PORT}" =~ ${number_re} ]] ; then + print_error_and_exit "PORT is not a number" + fi +} + +check_all_variables_are_set + +if [[ $(docker images --filter=reference='host-target' -q) == "" ]]; then + bash "${scripts_dir}"/build_container.sh host-target +fi + +docker_run="docker run \ + -it \ + --privileged \ + -e IP_ADDR=${IP_ADDR} \ + -e PORT=${PORT} \ + -e DEBUG=${DEBUG} \ + -e HTTPS_PROXY=${https_proxy} \ + -e HTTP_PROXY=${http_proxy} \ + -e NO_PROXY=${no_proxy} \ + --network host \ + -v /dev:/dev \ + host-target" +$docker_run + + diff --git a/build/storage/scripts/run_proxy_container.sh b/build/storage/scripts/run_proxy_container.sh new file mode 100755 index 00000000..6b4b4ba9 --- /dev/null +++ b/build/storage/scripts/run_proxy_container.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + + +[ "$DEBUG" == 'true' ] && set -x + +declare https_proxy +declare http_proxy +declare no_proxy +SHARED_VOLUME=${SHARED_VOLUME:-$(realpath .)} +SPDK_IP_ADDR="${SPDK_IP_ADDR:-"0.0.0.0"}" +SPDK_PORT="${SPDK_PORT:-5260}" +HOT_PLUG_SERVICE_IP_ADDR="${HOT_PLUG_SERVICE_IP_ADDR:-"0.0.0.0"}" +HOT_PLUG_SERVICE_PORT="${HOT_PLUG_SERVICE_PORT:-50051}" + +scripts_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +function print_error_and_exit() { + echo "${1}" + exit 1 +} + +function check_all_variables_are_set() { + number_re='^[0-9]+$' + ip_addr_re='^(([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))\.){3}([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))$' + + if [ -z "${SHARED_VOLUME}" ]; then + print_error_and_exit "SHARED_VOLUME is not defined" + elif [ ! -d "${SHARED_VOLUME}" ]; then + print_error_and_exit "SHARED_VOLUME is not a directory" + elif [ -z "${HOT_PLUG_SERVICE_IP_ADDR}" ]; then + print_error_and_exit "HOT_PLUG_SERVICE_IP_ADDR is not set" + elif ! [[ "${HOT_PLUG_SERVICE_IP_ADDR}" =~ ${ip_addr_re} ]]; then + print_error_and_exit "HOT_PLUG_SERVICE_IP_ADDR does not represent ip address" + elif [ -z "${HOT_PLUG_SERVICE_PORT}" ]; then + print_error_and_exit "PORT is not set" + elif ! [[ "${HOT_PLUG_SERVICE_PORT}" =~ ${number_re} ]] ; then + print_error_and_exit "PORT is not a number" + elif [ -z "${SPDK_PORT}" ]; then + print_error_and_exit "SPDK_PORT is not set" + elif ! [[ "${SPDK_PORT}" =~ ${number_re} ]] ; then + print_error_and_exit "SPDK_PORT is not a number" + elif [ -z "${SPDK_IP_ADDR}" ]; then + print_error_and_exit "SPDK_IP_ADDR is not set" + elif ! [[ "${SPDK_IP_ADDR}" =~ ${ip_addr_re} ]]; then + print_error_and_exit "SPDK_IP_ADDR does not represent ip address" + fi +} + +check_all_variables_are_set +SHARED_VOLUME=$(realpath "${SHARED_VOLUME}") + +bash "${scripts_dir}"/build_container.sh proxy-container +bash "${scripts_dir}"/allocate_hugepages.sh + +config_file_option= +if [[ -n "${SPDK_CONFIG_FILE}" ]]; then + SPDK_CONFIG_FILE=$(realpath "${SPDK_CONFIG_FILE}") + config_file_option="-v ${SPDK_CONFIG_FILE}:/config" +fi + +docker_run="docker run \ + -it \ + --privileged \ + -v /dev/hugepages:/dev/hugepages \ + -v ${SHARED_VOLUME}:/ipdk-shared \ + ${config_file_option} \ + -e SPDK_IP_ADDR=${SPDK_IP_ADDR} \ + -e SPDK_PORT=${SPDK_PORT} \ + -e HOT_PLUG_SERVICE_IP_ADDR=${HOT_PLUG_SERVICE_IP_ADDR} \ + -e HOT_PLUG_SERVICE_PORT=${HOT_PLUG_SERVICE_PORT} \ + -e HOST_SHARED_VOLUME=${SHARED_VOLUME} \ + -e DEBUG=${DEBUG} \ + -e HTTPS_PROXY=${https_proxy} \ + -e HTTP_PROXY=${http_proxy} \ + -e NO_PROXY=${no_proxy} \ + --network host \ + proxy-container" +$docker_run diff --git a/build/storage/scripts/run_storage_target_container.sh b/build/storage/scripts/run_storage_target_container.sh new file mode 100755 index 00000000..c6820e91 --- /dev/null +++ b/build/storage/scripts/run_storage_target_container.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x + +declare https_proxy +declare http_proxy +declare no_proxy +SPDK_CONFIG_FILE="${SPDK_CONFIG_FILE:-}" +SPDK_IP_ADDR="${SPDK_IP_ADDR:-"0.0.0.0"}" +SPDK_PORT="${SPDK_PORT:-5260}" + +scripts_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +function print_error_and_exit() { + echo "${1}" + exit 1 +} + +function check_all_variables_are_set() { + number_re='^[0-9]+$' + ip_addr_re='^(([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))\.){3}([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))$' + + if [ -z "${SPDK_IP_ADDR}" ]; then + print_error_and_exit "SPDK_IP_ADDR is not set" + elif ! [[ "${SPDK_IP_ADDR}" =~ ${ip_addr_re} ]]; then + print_error_and_exit "SPDK_IP_ADDR does not represent ip address" + elif [ -z "${SPDK_PORT}" ]; then + print_error_and_exit "SPDK_PORT is not set" + elif ! [[ "${SPDK_PORT}" =~ ${number_re} ]] ; then + print_error_and_exit "SPDK_PORT is not a number" + fi +} + +check_all_variables_are_set +SPDK_CONFIG_FILE=$(realpath "${SPDK_CONFIG_FILE}") + +bash "${scripts_dir}"/build_container.sh storage-target +bash "${scripts_dir}"/allocate_hugepages.sh + +config_file_option= +if [[ -n "${SPDK_CONFIG_FILE}" ]]; then + SPDK_CONFIG_FILE=$(realpath "${SPDK_CONFIG_FILE}") + config_file_option="-v ${SPDK_CONFIG_FILE}:/config" +fi + +docker_run="docker run \ + -it \ + --privileged \ + -v /dev/hugepages:/dev/hugepages \ + ${config_file_option} \ + -e SPDK_IP_ADDR=${SPDK_IP_ADDR} \ + -e SPDK_PORT=${SPDK_PORT} \ + -e DEBUG=${DEBUG} \ + -e HTTPS_PROXY=${https_proxy} \ + -e HTTP_PROXY=${http_proxy} \ + -e NO_PROXY=${no_proxy} \ + --network host \ + storage-target" + +$docker_run diff --git a/build/storage/scripts/socket.sh b/build/storage/scripts/socket.sh new file mode 100755 index 00000000..73be4f39 --- /dev/null +++ b/build/storage/scripts/socket.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +function send_command_over_unix_socket() { + socket="${1}" + cmd="${2}" + wait_for_secs="${3}" + echo "${cmd}" | socat -T"${wait_for_secs}" -,ignoreeof unix-connect:"${socket}" +} + +function send_command_over_unix_socket_and_no_word_found() { + out=$( send_command_over_unix_socket "${1}" "${2}" "${3}" ) + + searched_word="${4}" + result=$(echo "${out}" | grep -i -c "${searched_word}") + return "${result}" +} + +function get_output_from_unix_socket() { + socket="${1}" + wait_for_secs="${2}" + out=$( socat -T"${wait_for_secs}" -,ignoreeof unix-connect:"${socket}" ) + result=$? + echo "${out}" +} diff --git a/build/storage/scripts/spdk_version.sh b/build/storage/scripts/spdk_version.sh new file mode 100755 index 00000000..e2c00682 --- /dev/null +++ b/build/storage/scripts/spdk_version.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +function get_spdk_version() { + local script_dir root_dir spdk_dir spdk_version + script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + root_dir=${script_dir}/.. + spdk_dir="${root_dir}/spdk" + spdk_version=$(git -C "${spdk_dir}" describe --tags --abbrev=0) + echo "${spdk_version}" +} diff --git a/build/storage/scripts/vm/create_nat_for_vm.sh b/build/storage/scripts/vm/create_nat_for_vm.sh new file mode 100755 index 00000000..6b987eb9 --- /dev/null +++ b/build/storage/scripts/vm/create_nat_for_vm.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# This script allows to access vm ports from the host + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +declare net_mask_bits +declare gateway +declare dhcp_range +declare bridge +# shellcheck disable=SC1091 +source "${current_script_dir}"/nat_variables.sh + +tap_device="${1}" + +bridge_does_not_exist() { + bridge_name="${1}" + if bridge vlan show | grep "^${bridge_name}" &> /dev/null; then + return 1 + else + return 0 + fi +} + +if bridge_does_not_exist "${bridge}"; then + echo "1" | tee /proc/sys/net/ipv4/ip_forward + + ip link add "${bridge}" type bridge + ip addr add "${gateway}/${net_mask_bits}" dev "${bridge}" + ip link set "${bridge}" up + + dnsmasq \ + --interface="${bridge}" \ + --dhcp-range="${dhcp_range}" \ + --listen-address="${gateway}" \ + --pid-file="/var/run/qemu-dnsmasq-${bridge}.pid" \ + --dhcp-leasefile="/var/run/qemu-dnsmasq-${bridge}.leases" \ + --strict-order \ + --except-interface=lo \ + --bind-interfaces \ + --dhcp-no-override + +fi + +if test "${tap_device}" ; then + ip addr add 0.0.0.0 dev "${tap_device}" + ip link set "${tap_device}" up + ip link set "${tap_device}" master "${bridge}" +fi diff --git a/build/storage/scripts/vm/delete_nat_for_vm.sh b/build/storage/scripts/vm/delete_nat_for_vm.sh new file mode 100755 index 00000000..c156ad05 --- /dev/null +++ b/build/storage/scripts/vm/delete_nat_for_vm.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +declare bridge +# shellcheck disable=SC1091 +source "${current_script_dir}"/nat_variables.sh + +ip link set "${bridge}" down +ip link delete "${bridge}" type bridge + +kill -15 "$(cat "/var/run/qemu-dnsmasq-${bridge}.pid")" + diff --git a/build/storage/scripts/vm/nat_variables.sh b/build/storage/scripts/vm/nat_variables.sh new file mode 100755 index 00000000..e508afc9 --- /dev/null +++ b/build/storage/scripts/vm/nat_variables.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +export network=192.168.32.0 +export net_mask_bits=24 +export gateway=192.168.32.1 +export dhcp_range=192.168.32.2,192.168.32.254 +export bridge=br-ipdk-0 + diff --git a/build/storage/scripts/vm/prepare_vm.sh b/build/storage/scripts/vm/prepare_vm.sh new file mode 100755 index 00000000..b99f550b --- /dev/null +++ b/build/storage/scripts/vm/prepare_vm.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x +set -e + +scripts_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)/.. + +if [[ $# != 1 ]] ; then + echo "Directory to place vm file has to be specified" + exit 1 +fi + +export LIBGUESTFS_BACKEND=direct + +path_to_place_vm_file=${1} +vm_file=${path_to_place_vm_file}/vm.qcow2 + +if [ ! -f "${vm_file}" ]; then + vm_tmp_file="${path_to_place_vm_file}/vm_original.qcow2" + wget -O "${vm_tmp_file}" https://download.fedoraproject.org/pub/fedora/linux/\ +releases/33/Cloud/x86_64/images/Fedora-Cloud-Base-33-1.2.x86_64.qcow2 + virt-customize -a "${vm_tmp_file}" \ + --memsize 1260 \ + --root-password password:root \ + --install fio + + if [ "${WITHOUT_HOST_TARGET}" == "true" ]; then + echo "Skip Host-Target container installation" + else + "${scripts_dir}"/build_container.sh host-target + host_target_container="${path_to_place_vm_file}/host-target.tar" + docker save -o "${host_target_container}" host-target + run_customize=(virt-customize -a "${vm_tmp_file}") + run_customize+=(--run-command 'dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo') + run_customize+=(--run-command 'grubby --update-kernel ALL --args selinux=0') + run_customize+=(--install "dnf-plugins-core,docker-ce,docker-ce-cli,containerd.io") + run_customize+=(--copy-in "${path_to_place_vm_file}/host-target.tar:/") + run_customize+=(--copy-in "${scripts_dir}/run_host_target_container.sh:/usr/local/bin") + run_customize+=(--run-command 'systemctl enable docker.service') + run_customize+=(--run-command 'service docker start') + run_customize+=(--firstboot-command 'docker load --input /host-target.tar') + run_customize+=(--firstboot-command 'rm -f /host-target.tar') + "${run_customize[@]}" + rm -f "${host_target_container}" + fi + mv "${vm_tmp_file}" "${vm_file}" +fi diff --git a/build/storage/scripts/vm/run_vm.sh b/build/storage/scripts/vm/run_vm.sh new file mode 100755 index 00000000..aa710523 --- /dev/null +++ b/build/storage/scripts/vm/run_vm.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x + +scripts_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)/.. + +SHARED_VOLUME=${SHARED_VOLUME:-.} + +bash "${scripts_dir}"/vm/prepare_vm.sh "${SHARED_VOLUME}" +bash "${scripts_dir}"/allocate_hugepages.sh + +run_vm="sudo qemu-system-x86_64 \ + --enable-kvm \ + -cpu host \ + -m 1G -object memory-backend-file,id=mem0,size=1G,mem-path=/dev/hugepages,share=on -numa node,memdev=mem0 \ + -smp 2 \ + -drive file=${SHARED_VOLUME}/vm.qcow2,if=none,id=disk \ + -device ide-hd,drive=disk,bootindex=0 \ + -net nic -net tap,script=${scripts_dir}/vm/create_nat_for_vm.sh,\ +downscript=${scripts_dir}/vm/delete_nat_for_vm.sh \ + -monitor unix:${SHARED_VOLUME}/vm_monitor,server,nowait \ + --nographic" + +$run_vm \ No newline at end of file diff --git a/build/storage/spdk b/build/storage/spdk new file mode 160000 index 00000000..4e4f11ff --- /dev/null +++ b/build/storage/spdk @@ -0,0 +1 @@ +Subproject commit 4e4f11ff7840f02cb6d9497acd42897844a43f87 diff --git a/build/storage/tests/README.md b/build/storage/tests/README.md deleted file mode 100644 index 5d8a1c54..00000000 --- a/build/storage/tests/README.md +++ /dev/null @@ -1,132 +0,0 @@ -# Introduction -This recipe show-cases hot-plugging virtio-blk device into a VM and running -fio traffic from that VM using virtio-blk interface to a storage target -using NVMe/TCP protocol. - -The scenario is realized using docker-compose in a single host environment. -The figure below illustrates the design: - -![Figure 1 - not found](img_virtio_blk_over_nvmetcp_1.svg "Running virtio-blk traffic over NVMeTCP") - -There are three basic containers: -- _traffic-generator_ which plays the role of a "host" and runs the VM -- _proxy-container_ which plays the role of an IPU and exposes the block device -- _storage-target_ which plays the role of a remote storage node - -In addition, there is a _test-drivers_ container used to exercise the scenario by running -dedicated tests. - -# System setup -The basic requirement for running the recipe is a modern Linux distribution with docker support. -So far, the recipe has been successfully tested on the following systems: -- Fedora 33 -- Ubuntu 18.04 - -To run the recipe some basic system preparations are required, in particular: - -## Virtualization support -Make sure that VT-x/AMD-v support is enabled in BIOS -``` -$ lscpu | grep -i virtualization -Virtualization: VT-x -``` -and that kvm modules are loaded -``` -$ lsmod | grep -i kvm -kvm_intel 217088 0 -kvm 614400 1 kvm_intel -irqbypass 16384 1 kvm -``` - -## Tool installation -Make sure that following tools are installed on your system or install them -using the corresponding package manager. - -### wget -Installation on Fedora -``` -$ sudo dnf install wget -``` -or on Ubuntu -``` -$ sudo apt install wget -``` - -### docker -``` -$ sudo dnf install docker -``` -or -``` -$ sudo apt install docker -``` -**Note:** -Make sure that required proxy settings are configured for docker. -Please, refer to [this](https://docs.docker.com/config/daemon/systemd/#httphttps-proxy) page. - -### docker compose -``` -$ sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose -$ sudo chmod +x /usr/local/bin/docker-compose -$ sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose -``` - -### libguestfs-tools -``` -$ sudo dnf install libguestfs-tools-c -``` -or -``` -$ sudo apt install libguestfs-tools -``` - -**Note:** -To run `libguestfs` tools without root privileges, you may need to workaround the problem of -Linux kernel image not being readable by issuing: -``` -$ sudo chmod +r /boot/vmlinuz-* -``` - -## Setting security policies -Make sure that required security permissions are configured to enable VM network connectivity -or disable them temporarily. -On Fedora SELinux can be disabled by means of the following command -``` -$ sudo setenforce 0 -``` -For Ubuntu AppArmor is often used and can be disabled by issuing -``` -$ sudo systemctl stop apparmor -``` - -# Running the tests -In order to run all defined tests -``` -$ ./run.sh -``` -or if there is a proxy -``` -$ https_proxy= \ -http_proxy= \ -no_proxy= \ -./run.sh -``` - -To run a specific test -``` -$ ./run.sh -``` -for example -``` -$ ./run.sh hot-plug -$ # or -$ ./run.sh fio -``` -The script `run.sh` returns a non-zero value only in case of an error. - -**Note:** -The script `run.sh` will download a `Fedora 33` image into `traffic-generator` -directory and set up login-password pair as root-root if there is no `vm.qcow2` -image provided in that directory. `run.sh` will also try to allocate 2048 2MB -hugepages if not yet allocated and it will request administrative privileges -to execute this operation. diff --git a/build/storage/tests/img_virtio_blk_over_nvmetcp_1.svg b/build/storage/tests/img_virtio_blk_over_nvmetcp_1.svg deleted file mode 100644 index 14573bd7..00000000 --- a/build/storage/tests/img_virtio_blk_over_nvmetcp_1.svg +++ /dev/null @@ -1,184 +0,0 @@ - - - - - - - - - - - - - - - - VBackground-1 - - Solid - - - - - Page-1 - - Round Corner Rectangle.37 - IPDK docker-compose files - - IPDK docker-compose files - - Round Corner Rectangle.4 - traffic-generator - - traffic-generator - - Round Corner Rectangle.3 - QEMU - - QEMU - - Round Corner Rectangle - fio - - fio - - Round Corner Rectangle.5 - proxy-container - - proxy-container - - Round Corner Rectangle.6 - storage-target - - storage-target - - Round Corner Rectangle.7 - test-drivers - - test-drivers - - Sheet.8 - vhost/virtio-blk - - vhost/virtio-blk - - Sheet.9 - qemu monitor - - qemu monitor - - Sheet.10 - qemu serial - - qemu serial - - Sheet.11 - RUN ./init - - RUN ./init - - Sheet.12 - init.fio script - - init.fio script - - Sheet.13 - init.hot-plug script - - init.hot-plug script - - Sheet.14 - other tests - - other tests - - Dynamic connector - - - - Dynamic connector.16 - - - - Dynamic connector.18 - - - - Dynamic connector.19 - - - - Dynamic connector.20 - - - - Dynamic connector.21 - - - - Dynamic connector.22 - - - - Sheet.23 - NVMe/TCP - - NVMe/TCP - - Round Corner Rectangle.24 - - - - Sheet.25 - - - - Sheet.26 - containers - - containers - - Sheet.27 - shared objects/ docker volumes - - shared objects/ docker volumes - - Dynamic connector.29 - - - - Sheet.31 - init script return value - - init script return value - - diff --git a/build/storage/tests/it/README.md b/build/storage/tests/it/README.md new file mode 100644 index 00000000..deeb93d7 --- /dev/null +++ b/build/storage/tests/it/README.md @@ -0,0 +1,63 @@ +# Introduction +These tests covers the recipes described in [recipes](../../recipes/README.md) +directory. + +In addition to [the main IPDK images](../../README.md#main-components), +the following images are introduced for testing purposes: +- _traffic-generator_ which has a vm instance on board exposing interfaces for +access to the vm instance. +- _test-driver_ container used to exercise the scenario by running +dedicated tests(apply a specific storage configuration, check if hot-plugged +devices are visible on host, run traffic from host and etc.). + +All containers are run in the isolated docker compose environment described by +`docker-compose` files. It allows to configure the environment to run tests in +a flexible way. +The figure below illustrates the overall testing design: +![Running virtio-blk traffic over NVMeTCP](img_virtio_blk_over_nvmetcp_1.svg "Running virtio-blk traffic over NVMeTCP") + +# Test environment preparation +Please make sure that steps described in +[System setup](../../recipes/environment_setup.md#system-setup) +section are performed on testing platform. + +For testing environment additionally the steps bellow should be applied: + +## docker compose setup +``` +$ sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +$ sudo chmod +x /usr/local/bin/docker-compose +$ sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose +``` + +# Running the tests +In order to run all defined tests +``` +$ ./run.sh +``` +or if there is a proxy +``` +$ https_proxy= \ +http_proxy= \ +no_proxy= \ +./run.sh +``` + +To run a specific test +``` +$ ./run.sh +``` +for example +``` +$ ./run.sh hot-plug +$ # or +$ ./run.sh fio +``` +The script `run.sh` returns a non-zero value only in case of an error. + +**Note:** +The script `run.sh` will download a `Fedora 33` image into `traffic-generator` +directory and set up login-password pair as root-root if there is no `vm.qcow2` +image provided in that directory. `run.sh` will also try to allocate 2048 2MB +hugepages if not yet allocated and it will request administrative privileges +to execute this operation. \ No newline at end of file diff --git a/build/storage/tests/docker-compose.yml b/build/storage/tests/it/docker-compose.yml similarity index 59% rename from build/storage/tests/docker-compose.yml rename to build/storage/tests/it/docker-compose.yml index 9cc06991..d48b1cad 100644 --- a/build/storage/tests/docker-compose.yml +++ b/build/storage/tests/it/docker-compose.yml @@ -5,73 +5,68 @@ # WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK # version: "3.8" -x-proxy-args: - &proxy-args - args: - - HTTP_PROXY=${HTTP_PROXY} - - HTTPS_PROXY=${HTTPS_PROXY} - - NO_PROXY=${NO_PROXY} - -x-debug-env-var: &debug-env-var DEBUG=${DEBUG} x-shared-volume: &shared-volume ipdk-shared:/ipdk-shared +x-debug-env-var: &debug-env-var DEBUG=${DEBUG} + services: - build_base: - image: spdk - build: - context: ../core/build_base - <<: *proxy-args - container_name: build_base - environment: - - *debug-env-var storage-target: image: spdk-app build: - context: ../core/spdk-app - <<: *proxy-args + context: ../.. + target: spdk-app + args: + - HTTP_PROXY=${HTTP_PROXY} + - HTTPS_PROXY=${HTTPS_PROXY} + - NO_PROXY=${NO_PROXY} + - SPDK_VERSION=${SPDK_VERSION} container_name: storage-target - depends_on: - - build_base networks: ipdk: ipv4_address: 192.168.42.2 volumes: - /dev/hugepages:/dev/hugepages - - ./storage-target.conf:/config environment: - - *debug-env-var - SPDK_ARGS=-m 0x2 + - *debug-env-var privileged: true proxy-container: - image: spdk-app + image: proxy-container build: - context: ../core/spdk-app - <<: *proxy-args + context: ../.. + target: proxy-container + args: + - HTTP_PROXY=${HTTP_PROXY} + - HTTPS_PROXY=${HTTPS_PROXY} + - NO_PROXY=${NO_PROXY} + - SPDK_VERSION=${SPDK_VERSION} container_name: proxy-container - depends_on: - - storage-target networks: ipdk: ipv4_address: 192.168.42.3 volumes: - /dev/hugepages:/dev/hugepages - - ./proxy-container.conf:/config - *shared-volume environment: - - SPDK_ARGS=-m 0x4 -S /ipdk-shared + - SPDK_ARGS=-m 0x4 - *debug-env-var + - HOST_SHARED_VOLUME=/ipdk-shared privileged: true traffic-generator: image: traffic-generator build: - context: ./traffic-generator - <<: *proxy-args + context: ../.. + target: traffic-generator + dockerfile: tests/it/traffic-generator/Dockerfile + args: + - HTTP_PROXY=${HTTP_PROXY} + - HTTPS_PROXY=${HTTPS_PROXY} + - NO_PROXY=${NO_PROXY} + - SPDK_VERSION=${SPDK_VERSION} container_name: traffic-generator depends_on: - proxy-container - networks: - ipdk: volumes: - /dev/hugepages:/dev/hugepages - *shared-volume @@ -81,14 +76,22 @@ services: test-driver: image: test-driver build: - context: ./test-drivers - <<: *proxy-args + context: ../.. + target: test-driver + dockerfile: tests/it/test-drivers/Dockerfile + args: + - HTTP_PROXY=${HTTP_PROXY} + - HTTPS_PROXY=${HTTPS_PROXY} + - NO_PROXY=${NO_PROXY} + - SPDK_VERSION=${SPDK_VERSION} container_name: test-driver depends_on: - traffic-generator volumes: - *shared-volume - - ./test-drivers/test-helpers:/test-helpers + networks: + ipdk: + ipv4_address: 192.168.42.4 environment: - *debug-env-var privileged: true diff --git a/build/storage/tests/it/img_virtio_blk_over_nvmetcp_1.svg b/build/storage/tests/it/img_virtio_blk_over_nvmetcp_1.svg new file mode 100644 index 00000000..82ec5eff --- /dev/null +++ b/build/storage/tests/it/img_virtio_blk_over_nvmetcp_1.svg @@ -0,0 +1,444 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + VBackground-1 + + + Solid + + + + + + + + + + + + + + Page-1 + + + + Round Corner Rectangle.37 + IPDK docker-compose testing approach + + + + + + + + + + + + + + + + + + + + + + IPDK docker-compose testing approach + + Round Corner Rectangle.4 + traffic-generator + + + + + + + + + + + + + + + + + + + + + + traffic-generator + + Round Corner Rectangle.3 + QEMU + + + + + + + + + + + + + + + + + + + + + + QEMU + + Round Corner Rectangle + fio + + + + + + + + + + + + + + + + + + + + + + fio + + Round Corner Rectangle.5 + proxy-container + + + + + + + + + + + + + + + + + + + + + + proxy-container + + Round Corner Rectangle.6 + storage-target + + + + + + + + + + + + + + + + + + + + + + storage-target + + Sheet.8 + vhost/virtio-blk + + + + vhost/virtio-blk + + Sheet.9 + qemu monitor + + + + qemu monitor + + Sheet.10 + qemu serial + + + + qemu serial + + Dynamic connector.18 + + + + Dynamic connector.19 + + + + Dynamic connector.20 + + + + Dynamic connector.21 + + + + Dynamic connector.22 + + + + Sheet.23 + NVMe/TCP + + + + NVMe/TCP + + Round Corner Rectangle.24 + + + + + + + + + + + + + + + + + + + + + + Sheet.25 + + + + Sheet.26 + containers + + + + containers + + Sheet.27 + shared objects/ docker volumes + + + + shared objects/ docker volumes + + Dynamic connector.29 + + + + Sheet.31 + init script return value + + + + init script return value + + Sheet.1000 + Hot-plug service + + + + Hot-plugservice + + Sheet.1001 + SPDK rpc + + + + SPDKrpc + + Sheet.1002 + SPDK rpc + + + + SPDKrpc + + Round Corner Rectangle.7 + test-drivers + + + + + + + + + + + + + + + + + + + + + + test-drivers + + Sheet.11 + RUN ./init + + + + RUN ./init + + Sheet.12 + init.fio script + + + + init.fio script + + Sheet.13 + init.hot-plug script + + + + init.hot-plug script + + Sheet.14 + other tests + + + + other tests + + Dynamic connector.1005 + + + + Dynamic connector.1006 + + + + Dynamic connector.1007 + + + + Dynamic connector.1008 + + + + Dynamic connector.1009 + + + + Sheet.1010 + + + + Sheet.1011 + services + + + + services + + diff --git a/build/storage/tests/it/run.sh b/build/storage/tests/it/run.sh new file mode 100755 index 00000000..4fd8e4c9 --- /dev/null +++ b/build/storage/tests/it/run.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set -e +[ "$DEBUG" == 'true' ] && set -x + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +root_dir="${current_script_dir}"/../.. +scripts_dir="${root_dir}"/scripts +# shellcheck disable=SC1091 +source "${scripts_dir}"/spdk_version.sh +bash "${scripts_dir}"/prepare_to_build.sh +declare https_proxy +declare http_proxy +declare no_proxy +export HTTPS_PROXY=${https_proxy} +export HTTP_PROXY=${http_proxy} +export NO_PROXY=${no_proxy} +spdk_version=$(get_spdk_version) +export SPDK_VERSION="${spdk_version}" +export DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} +export COMPOSE_DOCKER_CLI_BUILD=${COMPOSE_DOCKER_CLI_BUILD:-1} + +function run_test() { + sudo_for_docker="sudo" + is_user_docker_group_member=$(groups | grep docker &> /dev/null ; echo $?) + if [ "${is_user_docker_group_member}" == "0" ]; then + sudo_for_docker= + elif [[ $(whoami) == "root" ]]; then + sudo_for_docker= + fi + if [ "${DEBUG}" == 'true' ]; then + export BUILDKIT_PROGRESS=plain + fi + + ${sudo_for_docker} docker-compose \ + -f "${current_script_dir}/docker-compose.yml" \ + -f "${current_script_dir}/test-drivers/docker-compose.$1.yml" \ + up \ + --build \ + --exit-code-from test-driver \ + --scale build_base=0 +} + +function provide_hugepages() { + bash "${scripts_dir}"/allocate_hugepages.sh +} + +provide_hugepages + +test_cases=(hot-plug fio) +if [[ $# != 0 ]]; then + run_test "${1}" +else + for i in "${test_cases[@]}"; do + run_test "${i}" + done +fi diff --git a/build/storage/tests/test-drivers/Dockerfile b/build/storage/tests/it/test-drivers/Dockerfile similarity index 74% rename from build/storage/tests/test-drivers/Dockerfile rename to build/storage/tests/it/test-drivers/Dockerfile index 41b0cc26..e3c24ade 100644 --- a/build/storage/tests/test-drivers/Dockerfile +++ b/build/storage/tests/it/test-drivers/Dockerfile @@ -4,8 +4,7 @@ # NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE # WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK # - -FROM spdk +FROM spdk-app AS test-driver # Generic args ARG HTTP_PROXY @@ -17,5 +16,9 @@ ENV https_proxy=$HTTPS_PROXY ENV no_proxy=$NO_PROXY RUN dnf install -y socat +RUN dnf install -y grpc-cli + +COPY tests/it/test-drivers/test-helpers /test-helpers +COPY scripts/ /scripts ENTRYPOINT ["/init"] diff --git a/build/storage/tests/test-drivers/docker-compose.fio.yml b/build/storage/tests/it/test-drivers/docker-compose.fio.yml similarity index 100% rename from build/storage/tests/test-drivers/docker-compose.fio.yml rename to build/storage/tests/it/test-drivers/docker-compose.fio.yml diff --git a/build/storage/tests/test-drivers/docker-compose.hot-plug.yml b/build/storage/tests/it/test-drivers/docker-compose.hot-plug.yml similarity index 100% rename from build/storage/tests/test-drivers/docker-compose.hot-plug.yml rename to build/storage/tests/it/test-drivers/docker-compose.hot-plug.yml diff --git a/build/storage/tests/it/test-drivers/init.fio b/build/storage/tests/it/test-drivers/init.fio new file mode 100755 index 00000000..316fb630 --- /dev/null +++ b/build/storage/tests/it/test-drivers/init.fio @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set -e +[ "$DEBUG" == 'true' ] && set -x + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +declare vm_serial +declare nqn +declare storage_target_ip +declare proxy_ip +# shellcheck disable=SC1091 +source "${current_script_dir}"/test-helpers + +[ -S "${vm_serial}" ] && rm "${vm_serial}" + +controller_name=Nvme0 +create_subsystem_and_expose_to_another_machine \ + "${storage_target_ip}" "${nqn}" "${proxy_ip}" "${controller_name}" +create_ramdrive_and_attach_as_ns_to_subsystem \ + "${storage_target_ip}" Malloc0 16 "${nqn}" +create_disk "${proxy_ip}" \ + /ipdk-shared/VirtioBlk0 "${controller_name}n1" + +wait_until_vm_is_up "${vm_serial}" +log_in_with_default_credentials "${vm_serial}" + +echo "check attachment" +is_virtio_blk_attached "${vm_serial}" + +echo "Run fio" +fio_cmd="fio --filename=/dev/vda --direct=1 --rw=randrw \ + --bs=4k --ioengine=libaio --iodepth=256 --runtime=10 \ + --numjobs=4 --time_based --group_reporting --name=iops-test-job" + +out=$(send_command_over_unix_socket "${vm_serial}" "${fio_cmd}" 11) +echo "${out}" + +echo "${out}" | grep "Disk stats (read/write)" +echo "fio has been executed successfully!" + +exit 0 diff --git a/build/storage/tests/it/test-drivers/init.hot-plug b/build/storage/tests/it/test-drivers/init.hot-plug new file mode 100755 index 00000000..ccd66abd --- /dev/null +++ b/build/storage/tests/it/test-drivers/init.hot-plug @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set -e +[ "$DEBUG" == 'true' ] && set -x + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +declare vm_serial +declare nqn +declare storage_target_ip +declare proxy_ip +# shellcheck disable=SC1091 +source "${current_script_dir}"/test-helpers + +controller_name=Nvme0 +create_subsystem_and_expose_to_another_machine \ + "${storage_target_ip}" "${nqn}" "${proxy_ip}" "${controller_name}" + +number_of_512b_blocks=16 +create_ramdrive_and_attach_as_ns_to_subsystem \ + "${storage_target_ip}" Malloc0 "${number_of_512b_blocks}" "${nqn}" +create_ramdrive_and_attach_as_ns_to_subsystem \ + "${storage_target_ip}" Malloc1 "${number_of_512b_blocks}" "${nqn}" +create_ramdrive_and_attach_as_ns_to_subsystem \ + "${storage_target_ip}" Malloc2 "${number_of_512b_blocks}" "${nqn}" + +HOT_PLUG_SERVICE_PORT="${HOT_PLUG_SERVICE_PORT:-50051}" + +wait_until_vm_is_up "${vm_serial}" +log_in_with_default_credentials "${vm_serial}" + +is_virtio_blk_not_attached "${vm_serial}" + +vm_monitor=vm_monitor_socket +virtio_blk_0_socket=VirtioBlk0 +echo "" +echo "### Attach 3 virtio-blk devices ###" + +attach_ns_as_virtio_blk \ + "${proxy_ip}" VirtioBlk0 "${controller_name}n1" \ + "${HOT_PLUG_SERVICE_PORT}" "${vm_monitor}" +check_number_of_virtio_blk_devices "${vm_serial}" 1 + +attach_ns_as_virtio_blk \ + "${proxy_ip}" VirtioBlk1 "${controller_name}n2" \ + "${HOT_PLUG_SERVICE_PORT}" "${vm_monitor}" +check_number_of_virtio_blk_devices "${vm_serial}" 2 + +attach_ns_as_virtio_blk \ + "${proxy_ip}" VirtioBlk2 "${controller_name}n3" \ + "${HOT_PLUG_SERVICE_PORT}" "${vm_monitor}" +check_number_of_virtio_blk_devices "${vm_serial}" 3 + +echo "" +echo "### Dettach 1st virtio-blk ###" +dettach_virtio_blk "${proxy_ip}" "${HOT_PLUG_SERVICE_PORT}" \ + "${vm_monitor}" "${virtio_blk_0_socket}" +check_number_of_virtio_blk_devices "${vm_serial}" 2 + +echo "" +echo "### Attach 1st virtio-blk back ###" +attach_virtio_blk "${proxy_ip}" "${HOT_PLUG_SERVICE_PORT}" \ + "${vm_monitor}" "${virtio_blk_0_socket}" +check_number_of_virtio_blk_devices "${vm_serial}" 3 + +exit 0 diff --git a/build/storage/tests/it/test-drivers/test-helpers b/build/storage/tests/it/test-drivers/test-helpers new file mode 100644 index 00000000..a7fbda19 --- /dev/null +++ b/build/storage/tests/it/test-drivers/test-helpers @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG_VM" == 'true' ] && export GRPC_VERBOSITY=DEBUG + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +root_dir=${current_script_dir}/../.. +scripts_dir=${root_dir}/scripts + +# shellcheck disable=SC1091 +source "${scripts_dir}"/socket.sh +# shellcheck disable=SC1091 +source "${scripts_dir}"/disk_infrastructure.sh + +function wait_until_vm_is_up() { + console="${1}" + + [ "$DEBUG_VM" == 'true' ] && echo "Start waiting for VM... and console ${console}" + + overall_wait_counter=0 + overall_max_wait_counter=12 + wait_period_sec=10 + + while [ ! -S "${console}" ] && \ + [ "${overall_wait_counter}" -le "${overall_max_wait_counter}" ] ; do + sec=$(( overall_wait_counter * wait_period_sec )) + [ "$DEBUG_VM" == 'true' ] && echo "Waiting for VM console: ${console}, sec ${sec} ..." + sleep "${wait_period_sec}" + overall_wait_counter=$(( overall_wait_counter + 1 )) + done + + sleep 4 + + socket_output_wait_counter=0 + while [ "$socket_output_wait_counter" -le 2 ] && \ + [ "$overall_wait_counter" -le "${overall_max_wait_counter}" ] ; do + sec=$(( overall_wait_counter * wait_period_sec )) + [ "$DEBUG_VM" == 'true' ] && echo "Waiting for VM completes booting, sec ${sec} ..." + overall_wait_counter=$(( overall_wait_counter + 1 )) + out=$( get_output_from_unix_socket "${console}" ${wait_period_sec} ) + [ "$DEBUG_VM" == 'true' ] && echo "${out}" + socket_output_wait_counter=$(( socket_output_wait_counter + 1 )) + if [[ -n "${out}" ]] ; then + socket_output_wait_counter=0 + fi + done + return 0 +} + +function log_in() { + out=$( send_command_over_unix_socket "${1}" "${2}" 3 ) + [ "$DEBUG_VM" == 'true' ] && echo "${out}" + out=$( send_command_over_unix_socket "${1}" "${3}" 3 ) + [ "$DEBUG_VM" == 'true' ] && echo "${out}" + return 0 +} + +function log_in_with_default_credentials() { + log_in "${1}" "root" "root" +} + +function hostname_to_ip() { + getent hosts "${1}" | awk '{ print $1 }' +} + +storage_target_ip=$(hostname_to_ip "storage-target") +export storage_target_ip="${storage_target_ip}" +proxy_ip=$(hostname_to_ip "proxy-container") +export proxy_ip="${proxy_ip}" +export nqn="nqn.2016-06.io.spdk:cnode1" +export shared_volume=/ipdk-shared +export vm_serial=${shared_volume}/vm_socket +export vm_monitor=${shared_volume}/vm_monitor_socket diff --git a/build/storage/tests/it/traffic-generator/Dockerfile b/build/storage/tests/it/traffic-generator/Dockerfile new file mode 100644 index 00000000..32809ab8 --- /dev/null +++ b/build/storage/tests/it/traffic-generator/Dockerfile @@ -0,0 +1,40 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE +# WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK +# +FROM fedora:33 as traffic-generator-base + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +ENV http_proxy=$HTTP_PROXY +ENV https_proxy=$HTTPS_PROXY +ENV no_proxy=$NO_PROXY + +RUN dnf install -y wget +RUN dnf install -y libguestfs-tools-c + +COPY scripts/* /scripts/ +COPY scripts/vm/* /scripts/vm/ +RUN WITHOUT_HOST_TARGET=true /scripts/vm/prepare_vm.sh / + +FROM fedora:33 AS traffic-generator + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +ENV http_proxy=$HTTP_PROXY +ENV https_proxy=$HTTPS_PROXY +ENV no_proxy=$NO_PROXY + +RUN dnf install -y qemu-kvm +COPY tests/it/traffic-generator/init /init +RUN chmod +x /init +COPY --from=traffic-generator-base /vm.qcow2 /vm.qcow2 + + +ENTRYPOINT ["/init"] diff --git a/build/storage/tests/traffic-generator/init b/build/storage/tests/it/traffic-generator/init similarity index 52% rename from build/storage/tests/traffic-generator/init rename to build/storage/tests/it/traffic-generator/init index c262ba2c..80dd0e90 100644 --- a/build/storage/tests/traffic-generator/init +++ b/build/storage/tests/it/traffic-generator/init @@ -4,24 +4,40 @@ # SPDX-License-Identifier: Apache-2.0 # -[ "$DEBUG" == 'true' ] && set -x +[ "$DEBUG" == 'true' ] && set -x ; export DEBUG_VM=true shared_volume=/ipdk-shared +virtio_blk_socket=${shared_volume}/VirtioBlk0 + attach_default_virtio_blk=" \ --chardev socket,id=spdk_vhost_blk0,path=${shared_volume}/VirtioBlk0 \ +-chardev socket,id=spdk_vhost_blk0,path=${virtio_blk_socket} \ -device vhost-user-blk-pci,chardev=spdk_vhost_blk0,num-queues=2 " if [ "${DO_NOT_ATTACH_VIRTIO_BLK}" == "true" ]; then attach_default_virtio_blk="" +else + wait_counter=1 + + while [ ! -S ${virtio_blk_socket} ] && [ ${wait_counter} -le 10 ] ; do + echo "Wait for virtio-blk socket: ${virtio_blk_socket}" + sleep 5 + wait_counter=$(( wait_counter + 1 )) + done + if [ ! -S ${virtio_blk_socket} ] ; then + echo "ERROR virtio-blk socket is not detected: ${virtio_blk_socket}" + exit 1 + else + echo "virtio-blk socket ${virtio_blk_socket} is detected" + fi + sleep 5 fi -sleep 2 echo "Starting vm" run_qemu="qemu-kvm \ -${attach_default_virtio_blk} \ +${attach_default_virtio_blk} --enable-kvm \ -cpu host \ -m 1G \ @@ -34,4 +50,6 @@ ${attach_default_virtio_blk} \ -serial unix:${shared_volume}/vm_socket,server,nowait \ --nographic" -$run_qemu \ No newline at end of file +$run_qemu + +echo "VM stopped" diff --git a/build/storage/tests/proxy-container.conf b/build/storage/tests/proxy-container.conf deleted file mode 100644 index 0a8caa32..00000000 --- a/build/storage/tests/proxy-container.conf +++ /dev/null @@ -1,34 +0,0 @@ -{ - "subsystems": [ - { - "subsystem": "bdev", - "config": [ - { - "method": "bdev_nvme_attach_controller", - "params": { - "name": "Nvme0", - "trtype": "TCP", - "adrfam": "IPv4", - "traddr": "192.168.42.2", - "trsvcid": "4420", - "subnqn": "nqn.2016-06.io.spdk:cnode1", - "prchk_reftag": false, - "prchk_guard": false - } - } - ] - }, - { - "subsystem": "vhost", - "config": [ - { - "method": "vhost_create_blk_controller", - "params": { - "ctrlr": "VirtioBlk0", - "dev_name": "Nvme0n1" - } - } - ] - } - ] -} diff --git a/build/storage/tests/run.sh b/build/storage/tests/run.sh deleted file mode 100755 index 54993aab..00000000 --- a/build/storage/tests/run.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -set -e -[ "$DEBUG" == 'true' ] && set -x - -vm_file=./traffic-generator/vm.qcow2 -declare https_proxy -declare http_proxy -declare no_proxy -export HTTPS_PROXY=${https_proxy} -export HTTP_PROXY=${http_proxy} -export NO_PROXY=${no_proxy} - -function run_test() { - SUDO_FOR_DOCKER="sudo" - IS_USER_DOCKER_GROUP_MEMBER=$(groups | grep docker &> /dev/null ; echo $?) - if [ "${IS_USER_DOCKER_GROUP_MEMBER}" == "0" ]; then - SUDO_FOR_DOCKER= - fi - - ${SUDO_FOR_DOCKER} docker-compose \ - -f ./docker-compose.yml \ - -f "./test-drivers/docker-compose.$1.yml" \ - up \ - --build \ - --exit-code-from test-driver \ - --scale build_base=0 -} - -function provide_hugepages() { - if [ -d /sys/kernel/mm/hugepages/hugepages-2048kB ] ; then - number_of_2mb_pages=$(cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages) - required_number_of_pages=2048 - if [ "$number_of_2mb_pages" -lt "$required_number_of_pages" ] ; then - sudo echo ${required_number_of_pages} | \ - sudo tee /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages - fi - fi -} - -function provide_vm() { - if [ ! -f "$vm_file" ]; then - local vm_tmp_file="${vm_file}_orig" - wget -O ${vm_tmp_file} https://download.fedoraproject.org/pub/fedora/linux/\ -releases/33/Cloud/x86_64/images/Fedora-Cloud-Base-33-1.2.x86_64.qcow2 - virt-customize -a ${vm_tmp_file} \ - --root-password password:root \ - --uninstall cloud-init \ - --install fio - mv "${vm_tmp_file}" "${vm_file}" - fi -} - -provide_hugepages -provide_vm - -test_cases=(hot-plug fio) -if [[ $# != 0 ]]; then - run_test "${1}" -else - for i in "${test_cases[@]}"; do - run_test "${i}" - done -fi diff --git a/build/storage/tests/storage-target.conf b/build/storage/tests/storage-target.conf deleted file mode 100644 index 1916e397..00000000 --- a/build/storage/tests/storage-target.conf +++ /dev/null @@ -1,63 +0,0 @@ -{ - "subsystems": [ - { - "subsystem": "bdev", - "config": [ - { - "method": "bdev_malloc_create", - "params": { - "name": "Malloc0", - "num_blocks": 131072, - "block_size": 512 - } - } - ] - }, - { - "subsystem": "nvmf", - "config": [ - { - "method": "nvmf_create_transport", - "params": { - "trtype": "TCP", - "io_unit_size": 8192 - } - }, - { - "method": "nvmf_create_subsystem", - "params": { - "nqn": "nqn.2016-06.io.spdk:cnode1", - "allow_any_host": true, - "serial_number": "SPDK00000000000001", - "model_number": "SPDK bdev Controller", - "max_namespaces": 32, - "min_cntlid": 1, - "max_cntlid": 65519 - } - }, - { - "method": "nvmf_subsystem_add_ns", - "params": { - "nqn": "nqn.2016-06.io.spdk:cnode1", - "namespace": { - "nsid": 1, - "bdev_name": "Malloc0" - } - } - }, - { - "method": "nvmf_subsystem_add_listener", - "params": { - "nqn": "nqn.2016-06.io.spdk:cnode1", - "listen_address": { - "trtype": "TCP", - "adrfam": "IPv4", - "traddr": "192.168.42.2", - "trsvcid": "4420" - } - } - } - ] - } - ] -} diff --git a/build/storage/tests/test-drivers/init.fio b/build/storage/tests/test-drivers/init.fio deleted file mode 100755 index da555438..00000000 --- a/build/storage/tests/test-drivers/init.fio +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -set -e -[ "$DEBUG" == 'true' ] && set -x - -declare vm_serial -# shellcheck disable=SC1091 -source /test-helpers - -wait_until_vm_is_up "${vm_serial}" -log_in_with_default_credentials "${vm_serial}" - -is_virtio_blk_attached "${vm_serial}" - -echo "Run fio" -fio_cmd="fio --filename=/dev/vda --direct=1 --rw=randrw \ - --bs=4k --ioengine=libaio --iodepth=256 --runtime=10 \ - --numjobs=4 --time_based --group_reporting --name=iops-test-job" - -out=$(send_command_over_unix_socket_and_sample_output \ - "${vm_serial}" "${fio_cmd}" 11) -echo "${out}" - -echo "${out}" | grep "Disk stats (read/write)" -echo "fio has been executed successfully!" - -exit 0 diff --git a/build/storage/tests/test-drivers/init.hot-plug b/build/storage/tests/test-drivers/init.hot-plug deleted file mode 100755 index 75fc0caa..00000000 --- a/build/storage/tests/test-drivers/init.hot-plug +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -set -e -[ "$DEBUG" == 'true' ] && set -x - -declare vm_serial -declare vm_monitor -# shellcheck disable=SC1091 -source /test-helpers - -wait_until_vm_is_up "${vm_serial}" -log_in_with_default_credentials "${vm_serial}" - -is_virtio_blk_not_attached "${vm_serial}" - -attach_virtio_blk_over_monitor "${vm_monitor}" - -is_virtio_blk_attached "${vm_serial}" - -exit 0 diff --git a/build/storage/tests/test-drivers/test-helpers b/build/storage/tests/test-drivers/test-helpers deleted file mode 100644 index 24877b46..00000000 --- a/build/storage/tests/test-drivers/test-helpers +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -export shared_volume=/ipdk-shared -export vm_serial=${shared_volume}/vm_socket -export vm_monitor=${shared_volume}/vm_monitor_socket - -function send_command_over_unix_socket() { - echo "${2}" | socat - unix-connect:"${1}" -} - -function send_command_over_unix_socket_and_sample_output() { - socket="${1}" - cmd="${2}" - wait_for_secs="${3}" - out=$(echo "${cmd}" | \ - socat -T"${wait_for_secs}" -,ignoreeof unix-connect:"${socket}") - echo "${out}" -} - -function get_output_from_unix_socket() { - socket="${1}" - wait_for_secs="${2}" - out=$( socat -T"${wait_for_secs}" -,ignoreeof unix-connect:"${socket}" ) - result=$? - echo "${out}" -} - -function wait_until_vm_is_up() { - console="${1}" - - [ "$DEBUG_VM" == 'true' ] && echo "Start waiting for VM... and console ${console}" - - overall_wait_counter=0 - overall_max_wait_counter=12 - wait_period_sec=10 - - while [ ! -S "${console}" ] && \ - [ "${overall_wait_counter}" -le "${overall_max_wait_counter}" ] ; do - sec=$(( overall_wait_counter * wait_period_sec )) - [ "$DEBUG_VM" == 'true' ] && echo "Waiting for VM console: ${console}, sec ${sec} ..." - sleep "${wait_period_sec}" - overall_wait_counter=$(( overall_wait_counter + 1 )) - done - - sleep 4 - - socket_output_wait_counter=0 - while [ "$socket_output_wait_counter" -le 2 ] && \ - [ "$overall_wait_counter" -le "${overall_max_wait_counter}" ] ; do - sec=$(( overall_wait_counter * wait_period_sec )) - [ "$DEBUG_VM" == 'true' ] && echo "Waiting for VM completes booting, sec ${sec} ..." - overall_wait_counter=$(( overall_wait_counter + 1 )) - out=$( get_output_from_unix_socket "${console}" "${wait_period_sec}" ) - [ "$DEBUG_VM" == 'true' ] && echo "${out}" - socket_output_wait_counter=$(( socket_output_wait_counter + 1 )) - if [[ -n "${out}" ]] ; then - socket_output_wait_counter=0 - fi - done - return 0 -} - -function log_in() { - send_command_over_unix_socket "${1}" "${2}" - sleep 1 - send_command_over_unix_socket "${1}" "${3}" - sleep 1 -} - -function log_in_with_default_credentials() { - log_in "${1}" "root" "root" -} - -function is_virtio_blk_attached() { - cmd="lsblk --output \"NAME,VENDOR,SUBSYSTEMS\"" - out=$(send_command_over_unix_socket_and_sample_output "${1}" "${cmd}" 1) - echo "${out}" | grep "block:virtio:pci" - result=$? - echo "${out}" - if [[ ${result} == 0 ]]; then - echo "virtio-blk is found" - else - echo "virtio-blk is not found" - fi - return "${result}" -} - -function is_virtio_blk_not_attached() { - if is_virtio_blk_attached "${1}"; then - return 1 - fi - - return 0 -} - -function attach_virtio_blk_over_monitor() { - add_chardev_cmd="chardev-add socket,id=spdk_vhost_blk0,path=${shared_volume}/VirtioBlk0" - send_command_over_unix_socket "${vm_monitor}" "${add_chardev_cmd}" - sleep 1 - add_device_cmd="device_add vhost-user-blk-pci,chardev=spdk_vhost_blk0,num-queues=2" - send_command_over_unix_socket "${vm_monitor}" "${add_device_cmd}" - sleep 1 -} - diff --git a/build/storage/tests/traffic-generator/Dockerfile b/build/storage/tests/traffic-generator/Dockerfile deleted file mode 100644 index 99066b35..00000000 --- a/build/storage/tests/traffic-generator/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# -# NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE -# WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK -# - -FROM spdk - -# Generic args -ARG HTTP_PROXY -ARG HTTPS_PROXY -ARG NO_PROXY - -ENV http_proxy=$HTTP_PROXY -ENV https_proxy=$HTTPS_PROXY -ENV no_proxy=$NO_PROXY - - -COPY vm.qcow2 /vm.qcow2 -RUN dnf install -y qemu-kvm -COPY init /init -RUN chmod +x /init - -ENTRYPOINT ["/init"] diff --git a/build/storage/tests/ut/README.md b/build/storage/tests/ut/README.md new file mode 100644 index 00000000..2358e863 --- /dev/null +++ b/build/storage/tests/ut/README.md @@ -0,0 +1,22 @@ +# Description +Unit tests covers services written written for IPDK. + +# Run unit tests +The command below build all required dependencies and run the unit tests within +the dedicated container. +``` +$ ./run.sh +``` + +# Development +By means of the command below, a container will be started with test and source +files attached as volumes. +``` +$ ./run.sh dev +``` + +In order to run the unit tests during a development session run +`run_all_unit_tests.sh` within the started container. +``` +$ /run_all_unit_tests.sh +``` diff --git a/build/storage/tests/ut/host-target/test_device_exerciser.py b/build/storage/tests/ut/host-target/test_device_exerciser.py new file mode 100644 index 00000000..335ffafe --- /dev/null +++ b/build/storage/tests/ut/host-target/test_device_exerciser.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +from pyfakefs.fake_filesystem_unittest import TestCase +from pci_devices import get_virtio_blk_path_by_pci_address +from device_exerciser import DeviceExerciser, DeviceExerciserError + + +class DeviceExerciserTests(TestCase): + def setUp(self): + self.setUpPyfakefs() + + def tearDown(self): + pass + + def test_successful_fio_run(self): + self.fs.create_dir( + "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda") + self.fs.create_file("/dev/vda") + fio_arguments = "--name=test --size=4MB" + + def fio_do_nothing(fio_args): + fio_do_nothing.was_called = True + self.assertTrue("vda" in fio_args) + self.assertTrue(fio_arguments in fio_args) + return "output" + fio_do_nothing.was_called = False + + exerciser = DeviceExerciser( + fio_runner=fio_do_nothing, + virtio_blk_detector=get_virtio_blk_path_by_pci_address) + out = exerciser.run_fio("0000:00:04.0", fio_arguments) + self.assertTrue(fio_do_nothing.was_called) + self.assertEqual(out, "output") + + def test_any_errors_at_exercising(self): + def raise_exception(unused): + raise BaseException() + exerciser = DeviceExerciser( + fio_runner=raise_exception, + virtio_blk_detector=raise_exception) + with self.assertRaises(DeviceExerciserError) as ex: + exerciser.run_fio("pcie_address", "fio_arguments") diff --git a/build/storage/tests/ut/host-target/test_fio_runner.py b/build/storage/tests/ut/host-target/test_fio_runner.py new file mode 100644 index 00000000..7acea69f --- /dev/null +++ b/build/storage/tests/ut/host-target/test_fio_runner.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import unittest +import os + +from fio_runner import run_fio +from fio_runner import FioExecutionError + + +class FioRunner(unittest.TestCase): + def setUp(self): + self.fio_file = "test" + self.fio_args = "--name=test --size=4MB --filename=test" + + def tearDown(self): + if os.path.exists(self.fio_file): + os.remove(self.fio_file) + + def test_successful_fio_run(self): + try: + run_fio(self.fio_args) + except FioExecutionError as ex: + self.fail(str(ex)) + + self.assertTrue(os.path.exists(self.fio_file)) + + def test_successful_fio_run_output(self): + out = run_fio(self.fio_args) + self.assertTrue(out.find("READ: bw=") != -1) + + def test_invalid_fio_arg(self): + with self.assertRaises(FioExecutionError) as ex: + run_fio("some-non-existing-arg") + + def test_pass_invalid_arg_type(self): + with self.assertRaises(FioExecutionError) as ex: + run_fio(123) + + def test_no_concat_cmds_allowed(self): + with self.assertRaises(FioExecutionError) as ex: + run_fio("ls -l && ls") + with self.assertRaises(FioExecutionError) as ex: + run_fio("ls -l || ls") + with self.assertRaises(FioExecutionError) as ex: + run_fio("ls -l ; ls") diff --git a/build/storage/tests/ut/host-target/test_host_target_grpc_server.py b/build/storage/tests/ut/host-target/test_host_target_grpc_server.py new file mode 100644 index 00000000..a8e5bb14 --- /dev/null +++ b/build/storage/tests/ut/host-target/test_host_target_grpc_server.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +from multiprocessing import context +from host_target_grpc_server import HostTargetService +import host_target_pb2 + +import unittest +import unittest.mock + + +def detect_virtio_blk_device(unused): + return "non_existing_device_name" + + +def successfull_fio(unused): + return "output" + + +class HostTargetServerTests(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + pass + + def test_run_fio_success(self): + server = HostTargetService(successfull_fio, detect_virtio_blk_device) + request = host_target_pb2.RunFioRequest() + request.pciAddress = "unused" + request.fioArgs = "unused" + context = unittest.mock.MagicMock() + + reply = server.RunFio(request, context) + context.set_code.was_not_called() + context.set_details.was_not_called() + self.assertTrue(reply != None) + self.assertTrue(reply.fioOutput != None) + self.assertTrue(reply.fioOutput != "") + + def test_run_fio_does_not_propagate_exception(self): + def fio_throws_exception(unused): + raise BaseException() + server = HostTargetService( + fio_throws_exception, detect_virtio_blk_device) + request = host_target_pb2.RunFioRequest() + request.pciAddress = "unused" + request.fioArgs = "unused" + context = unittest.mock.MagicMock() + + server.RunFio(request, context) + context.set_code.assert_called() + context.set_details.assert_called() + + def test_run_fio_does_not_propagate_exception(self): + def detect_virtio_blk_throws_exception(unused): + raise BaseException() + server = HostTargetService( + successfull_fio, detect_virtio_blk_throws_exception) + request = host_target_pb2.RunFioRequest() + request.pciAddress = "unused" + request.fioArgs = "unused" + context = unittest.mock.MagicMock() + + server.RunFio(request, context) + context.set_code.assert_called() + context.set_details.assert_called() diff --git a/build/storage/tests/ut/host-target/test_pci_devices.py b/build/storage/tests/ut/host-target/test_pci_devices.py new file mode 100644 index 00000000..209e1018 --- /dev/null +++ b/build/storage/tests/ut/host-target/test_pci_devices.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import os +from pci_devices import get_virtio_blk_path_by_pci_address +from pci_devices import InvalidPciAddress +from pci_devices import FailedPciDeviceDetection +from pyfakefs.fake_filesystem_unittest import TestCase + + +class PciDeviceTests(TestCase): + def setUp(self): + self.setUpPyfakefs() + + def test_get_device_by_pci_address(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + dev_path = get_virtio_blk_path_by_pci_address("0000:00:04.0") + self.assertEqual(dev_path, "/dev/vda") + + def test_get_device_by_not_conventional_pci_address(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:00:04.9") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:00:20.0") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:00:00:0") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:00.00.0") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0:00.00.0") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:0.00.0") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:00.0.0") + with self.assertRaises(InvalidPciAddress) as ex: + get_virtio_blk_path_by_pci_address("0000:00.00.01") + + def test_corner_bus_value(self): + path = "/sys/devices/pci0000:FF/0000:FF:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + dev_path = get_virtio_blk_path_by_pci_address("0000:FF:04.0") + self.assertEqual(dev_path, "/dev/vda") + + def test_corner_device_value(self): + path = "/sys/devices/pci0000:00/0000:00:1F.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + dev_path = get_virtio_blk_path_by_pci_address("0000:00:1F.0") + self.assertEqual(dev_path, "/dev/vda") + + def test_corner_function_value(self): + path = "/sys/devices/pci0000:00/0000:00:04.7/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + dev_path = get_virtio_blk_path_by_pci_address("0000:00:04.7") + self.assertEqual(dev_path, "/dev/vda") + + def test_multiple_domains(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + path = "/sys/devices/pci0001:00/0001:00:04.0/virtio1/block/vdb" + self.fs.create_dir(path) + self.fs.create_file("/dev/vdb") + + dev_path = get_virtio_blk_path_by_pci_address("0001:00:04.0") + self.assertEqual(dev_path, "/dev/vdb") + + def test_multiple_buses(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + path = "/sys/devices/pci0000:01/0000:01:04.0/virtio1/block/vdb" + self.fs.create_dir(path) + self.fs.create_file("/dev/vdb") + + dev_path = get_virtio_blk_path_by_pci_address("0000:01:04.0") + self.assertEqual(dev_path, "/dev/vdb") + + def test_multiple_devices(self): + path = "/sys/devices/pci0000:00/0000:00:00.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + path = "/sys/devices/pci0000:00/0000:00:01.0/virtio1/block/vdb" + self.fs.create_dir(path) + self.fs.create_file("/dev/vdb") + + dev_path = get_virtio_blk_path_by_pci_address("0000:00:01.0") + self.assertEqual(dev_path, "/dev/vdb") + + def test_multiple_functions(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + path = "/sys/devices/pci0000:00/0000:00:04.1/virtio1/block/vdb" + self.fs.create_dir(path) + self.fs.create_file("/dev/vdb") + + dev_path = get_virtio_blk_path_by_pci_address("0000:00:04.1") + self.assertEqual(dev_path, "/dev/vdb") + + def test_no_block_device_exists(self): + path_to_device = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block" + self.fs.create_dir(path_to_device) + with self.assertRaises(FailedPciDeviceDetection) as ex: + get_virtio_blk_path_by_pci_address("0000:00:00.0") + + def test_multiple_block_devices_in_virtio_dir(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vdb" + self.fs.create_dir(path) + self.fs.create_file("/dev/vdb") + with self.assertRaises(FailedPciDeviceDetection) as ex: + get_virtio_blk_path_by_pci_address("0000:00:04.0") + + def test_multiple_block_devices_under_different_virtio_dirs(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio1/block/vdb" + self.fs.create_dir(path) + self.fs.create_file("/dev/vdb") + with self.assertRaises(FailedPciDeviceDetection) as ex: + get_virtio_blk_path_by_pci_address("0000:00:04.0") + + def test_no_directory_under_domain_bus_directory_in_sysfs(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + self.fs.create_file("/dev/vda") + non_existing_pci_address = "0000:00:01.0" + with self.assertRaises(FailedPciDeviceDetection) as ex: + get_virtio_blk_path_by_pci_address(non_existing_pci_address) + + def test_pci_does_not_exist_in_sysfs(self): + with self.assertRaises(FailedPciDeviceDetection) as ex: + get_virtio_blk_path_by_pci_address("0000:00:04.0") + + def test_device_does_not_exist_in_dev(self): + path = "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda" + self.fs.create_dir(path) + + with self.assertRaises(FailedPciDeviceDetection) as ex: + get_virtio_blk_path_by_pci_address("0000:00:04.0") diff --git a/build/storage/tests/ut/host-target/test_run_grpc_server.py b/build/storage/tests/ut/host-target/test_run_grpc_server.py new file mode 100644 index 00000000..f98bee8a --- /dev/null +++ b/build/storage/tests/ut/host-target/test_run_grpc_server.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import unittest +import os +import unittest.mock +import grpc +from unittest.mock import patch +from host_target_grpc_server import run_grpc_server +from host_target_grpc_server import HostTargetService +import host_target_pb2_grpc + + +class RunGrpcServer(unittest.TestCase): + def test_fail_on_invalid_address_to_listen_to(self): + self.assertNotEqual(run_grpc_server("Invalid ip addr", 1010), 0) + + def test_fail_on_invalid_port_to_listen_to(self): + self.assertNotEqual(run_grpc_server("0.0.0.0", "Invalid port"), 0) + + def test_success_on_keyboard_interrupt_exception(self): + server_mock = unittest.mock.Mock() + server_mock.add_insecure_port.side_effect = KeyboardInterrupt() + def server_creator(unused): + return server_mock + self.assertEqual(run_grpc_server("Unused", "Unused", server_creator), 0) + self.assertTrue(server_mock.add_insecure_port.was_called) + + + @patch.object(host_target_pb2_grpc, 'add_HostTargetServicer_to_server') + def test_required_methods_were_called(self, add_servicer_mock): + server_mock = unittest.mock.Mock() + def server_creator(unused): + return server_mock + ip_addr = "ip_addr" + port = "port" + + self.assertEqual(run_grpc_server(ip_addr, port, server_creator), 0) + + self.assertTrue(server_mock.add_insecure_port.was_called) + call_args = server_mock.add_insecure_port.call_args.args + self.assertTrue(len(call_args) == 1) + self.assertTrue(ip_addr in call_args[0]) + self.assertTrue(port in call_args[0]) + self.assertTrue(server_mock.start.was_called) + + self.assertTrue(add_servicer_mock.was_called) + self.assertTrue(len(add_servicer_mock.call_args.args) == 2) + self.assertTrue(isinstance(add_servicer_mock.call_args.args[0], HostTargetService)) + self.assertTrue(add_servicer_mock.call_args.args[1] == server_mock) \ No newline at end of file diff --git a/build/storage/tests/ut/proxy-container/test_hot_plug_grpc_server.py b/build/storage/tests/ut/proxy-container/test_hot_plug_grpc_server.py new file mode 100644 index 00000000..358d40ab --- /dev/null +++ b/build/storage/tests/ut/proxy-container/test_hot_plug_grpc_server.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import hot_plug_pb2 +from hot_plug_provider import HotPlugProvider +from hot_plug_grpc_server import HotPlugService, InvalidHotPlugProvider +import grpc +import unittest +from unittest.mock import patch + + +class HotPlugServerTests(unittest.TestCase): + def setUp(self): + self.server = HotPlugService(HotPlugProvider("stub", "stub")) + self.context = unittest.mock.MagicMock() + + def tearDown(self): + pass + + def _test_server_does_not_propagate_exception(self, stub_provider_operation, + server_operation): + err_description = "exception description" + stub_provider_operation.side_effect = BaseException(err_description) + request = hot_plug_pb2.HotPlugRequest() + + reply = server_operation(request, self.context) + self.context.set_code.assert_called() + self.context.set_details.assert_called() + self.assertTrue(reply != None) + + @patch.object(HotPlugProvider, "hot_plug_vhost_virtio_blk") + def test_hot_plug_does_not_propagate_exception(self, mock_provider): + self._test_server_does_not_propagate_exception( + mock_provider, self.server.HotPlugVirtioBlk) + + @patch.object(HotPlugProvider, "hot_unplug_vhost_virtio_blk") + def test_hot_unplug_does_not_propagate_exception(self, mock_provider): + self._test_server_does_not_propagate_exception( + mock_provider, self.server.HotUnplugVirtioBlk) + + def _test_server_operation_success(self, stub_provider_operation, + server_operation): + vm = "vm" + vhost = "vhost" + request = hot_plug_pb2.HotPlugRequest() + request.vmId = vm + request.vhostVirtioBlkId = vhost + + reply = server_operation(request, self.context) + + stub_provider_operation.assert_called_with(vm, vhost) + + self.context.set_code.was_not_called() + self.context.set_details.was_not_called() + self.assertTrue(reply != None) + + @patch.object(HotPlugProvider, "hot_plug_vhost_virtio_blk") + def test_hot_plug_success(self, mock_provider): + self._test_server_operation_success( + mock_provider, self.server.HotPlugVirtioBlk) + + @patch.object(HotPlugProvider, "hot_unplug_vhost_virtio_blk") + def test_hot_unplug_success(self, mock_provider): + self._test_server_operation_success( + mock_provider, self.server.HotUnplugVirtioBlk) diff --git a/build/storage/tests/ut/proxy-container/test_hot_plug_provider.py b/build/storage/tests/ut/proxy-container/test_hot_plug_provider.py new file mode 100644 index 00000000..e9135b98 --- /dev/null +++ b/build/storage/tests/ut/proxy-container/test_hot_plug_provider.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +from multiprocessing.sharedctypes import Value +from hot_plug_provider import ActionExecutionError, HotPlugProvider +from pathlib import Path +import socket +import os +from unittest.mock import patch +import unittest + + +class SocketTestEnvironment(unittest.TestCase): + def setUp(self): + self.fake_socket_name = "fake_socket" + self.fake_socket_path = "/tmp/" + self.fake_socket_name + self.fake_socket = socket.socket( + socket.AF_UNIX, socket.SOCK_STREAM) + self.fake_socket.bind(self.fake_socket_path) + + def tearDown(self): + self.fake_socket.close() + os.remove(self.fake_socket_path) + + +class HotPlugValidationBaseTestCases: + @patch("hot_plug_provider.subprocess.call") + class Tests(SocketTestEnvironment): + def setUp(self): + SocketTestEnvironment.setUp(self) + self.shared_dir_path = "/tmp" + self.host_shared_dir_path = "/home/user/ipdk-shared_dir" + self.non_existing_path = "/tmp/non-existing_path" + + self.provider = HotPlugProvider(self.shared_dir_path, + self.host_shared_dir_path) + self.disk_operation = None + + def tearDown(self): + SocketTestEnvironment.tearDown(self) + + def test_pass_none_shared_dir(self, unused): + with self.assertRaises(ValueError) as ex: + HotPlugProvider(None, self.host_shared_dir_path) + + def test_pass_none_host_shared_dir(self, unused): + with self.assertRaises(ValueError) as ex: + HotPlugProvider(self.shared_dir_path, None) + + def test_non_existing_path_to_vm_monitor(self, mock_subprocess_call): + with self.assertRaises(FileNotFoundError) as ex: + self.disk_operation( + vm_monitor=self.non_existing_path, + vhost_virtio_blk=self.fake_socket_path) + mock_subprocess_call.assert_not_called() + + def test_non_existing_path_to_vhost(self, mock_subprocess_call): + with self.assertRaises(FileNotFoundError) as ex: + self.disk_operation( + vm_monitor=self.fake_socket_path, + vhost_virtio_blk=self.non_existing_path) + mock_subprocess_call.assert_not_called() + + def test_pass_valid_sockets(self, mock_subprocess_call): + mock_subprocess_call.return_value = 0 + was_exception_raised = False + try: + self.disk_operation( + self.fake_socket_path, self.fake_socket_path) + except: + was_exception_raised = True + + self.assertEqual(was_exception_raised, False) + mock_subprocess_call.assert_called_once() + + def test_hot_plug_call_failed(self, mock_subprocess_call): + mock_subprocess_call.return_value = 1 + + with self.assertRaises(ActionExecutionError) as ex: + self.disk_operation( + self.fake_socket_path, self.fake_socket_path) + + def test_hot_plug_generates_correct_device_id(self, + mock_subprocess_call): + mock_subprocess_call.return_value = 0 + + vm = self.fake_socket_name + vhost0 = " " + self.fake_socket_name + " " + vhost1 = vhost0.strip() + + self.disk_operation(vm, vhost0) + self.disk_operation(vm, vhost1) + call_args = mock_subprocess_call.call_args_list + self.assertTrue(len(call_args) == 2) + self.assertTrue(call_args[0] == call_args[1]) + + def test_pass_none_vm(self, unused): + with self.assertRaises(ValueError) as ex: + self.disk_operation( + None, self.fake_socket_path) + + def test_pass_none_vhost(self, unused): + with self.assertRaises(ValueError) as ex: + self.disk_operation( + self.fake_socket_path, None) + + +class HotPlugValidation(HotPlugValidationBaseTestCases.Tests): + def setUp(self): + HotPlugValidationBaseTestCases.Tests.setUp(self) + self.disk_operation = self.provider.hot_plug_vhost_virtio_blk + + def tearDown(self): + HotPlugValidationBaseTestCases.Tests.tearDown(self) + + @patch("hot_plug_provider.subprocess.call") + @patch.object(HotPlugProvider, "_generate_device_id") + def test_hot_plug_call_args(self, mock_device_id_generator, + mock_subprocess_call): + device_id = "42" + mock_subprocess_call.return_value = 0 + mock_device_id_generator.return_value = device_id + + vm = self.fake_socket_name + vhost = self.fake_socket_name + + self.disk_operation(vm, vhost) + mock_subprocess_call.assert_called_once_with( + "/hot-plug.sh " + + os.path.join(self.shared_dir_path, vm) + " " + + os.path.join(self.host_shared_dir_path, vhost) + " " + + device_id, shell=True) + + @patch("hot_plug_provider.subprocess.call") + @patch.object(HotPlugProvider, "_generate_device_id") + def test_hot_plug_call_args_with_trailing_whitespaces(self, + mock_device_id_generator, + mock_subprocess_call): + device_id = "42" + mock_subprocess_call.return_value = 0 + mock_device_id_generator.return_value = device_id + + vm = " " + self.fake_socket_name + " " + vhost = " " + self.fake_socket_name + " " + + self.disk_operation(vm, vhost) + + mock_subprocess_call.assert_called_once_with( + "/hot-plug.sh " + + os.path.join(self.shared_dir_path, self.fake_socket_name) + " " + + os.path.join(self.host_shared_dir_path, self.fake_socket_name) + " " + + device_id, shell=True) + + +class HotUnplugValidation(HotPlugValidationBaseTestCases.Tests): + def setUp(self): + HotPlugValidationBaseTestCases.Tests.setUp(self) + self.disk_operation = self.provider.hot_unplug_vhost_virtio_blk + + def tearDown(self): + HotPlugValidationBaseTestCases.Tests.tearDown(self) + + @patch("hot_plug_provider.subprocess.call") + @patch.object(HotPlugProvider, "_generate_device_id") + def test_hot_plug_call_args(self, mock_device_id_generator, + mock_subprocess_call): + device_id = "42" + mock_subprocess_call.return_value = 0 + mock_device_id_generator.return_value = device_id + + vm = self.fake_socket_name + vhost = self.fake_socket_name + + self.disk_operation(vm, vhost) + mock_subprocess_call.assert_called_once_with( + "/hot-unplug.sh " + os.path.join(self.shared_dir_path, vm) + " " + + device_id, shell=True) + + @patch("hot_plug_provider.subprocess.call") + @patch.object(HotPlugProvider, "_generate_device_id") + def test_hot_plug_call_args_with_trailing_whitespaces(self, + mock_device_id_generator, + mock_subprocess_call): + device_id = "42" + mock_subprocess_call.return_value = 0 + mock_device_id_generator.return_value = device_id + + vm = " " + self.fake_socket_name + " " + vhost = " " + self.fake_socket_name + " " + + self.disk_operation(vm, vhost) + + mock_subprocess_call.assert_called_once_with( + "/hot-unplug.sh " + + os.path.join(self.shared_dir_path, self.fake_socket_name) + " " + + device_id, shell=True) diff --git a/build/storage/tests/ut/run.sh b/build/storage/tests/ut/run.sh new file mode 100755 index 00000000..d3e5bc3a --- /dev/null +++ b/build/storage/tests/ut/run.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +[ "$DEBUG" == 'true' ] && set -x +set -e + +current_script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +root_dir="${current_script_dir}/../.." +scripts_dir="${root_dir}/scripts" +proxy_src_dir="${root_dir}/core/proxy-container" +host_target_src_dir="${root_dir}/core/host-target" + +ut_container="ipdk-unit-tests" +"${scripts_dir}"/build_container.sh "${ut_container}" +args=(-e "DEBUG=${DEBUG}") +if [[ "${1}" == dev ]] ; then + args+=(--entrypoint "/bin/bash") + args+=(-v "${current_script_dir}/proxy-container:/proxy-container/tests") + args+=(-v "${proxy_src_dir}:/proxy-container/src") + args+=(-v "${current_script_dir}/host-target:/host-target/tests") + args+=(-v "${host_target_src_dir}:/host-target/src") +fi +docker run -it "${args[@]}" "${ut_container}" diff --git a/build/storage/tests/ut/run_all_unit_tests.sh b/build/storage/tests/ut/run_all_unit_tests.sh new file mode 100755 index 00000000..35056642 --- /dev/null +++ b/build/storage/tests/ut/run_all_unit_tests.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +[ "$DEBUG" == 'true' ] && set -x +ls -l /host-target/tests + +result=0 +python -m unittest discover -v -s /proxy-container/tests +result=$(("$?" | "${result}")) +python -m unittest discover -v -s /host-target/tests +result=$(("$?" | "${result}")) + +exit "${result}" From 3b4f9ec0b5a014ee2e6643ce081d5f14b11d1a99 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Wed, 25 May 2022 11:21:49 +0200 Subject: [PATCH 2/4] Update to fedora:36 as base image Signed-off-by: Artsiom Koltun --- build/storage/Dockerfile | 14 +++++++------- build/storage/recipes/environment_setup.md | 2 +- build/storage/tests/it/README.md | 2 +- .../storage/tests/it/traffic-generator/Dockerfile | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/build/storage/Dockerfile b/build/storage/Dockerfile index 65af1a35..7aa31bff 100644 --- a/build/storage/Dockerfile +++ b/build/storage/Dockerfile @@ -10,7 +10,7 @@ # This image is used to provide environment for spdk build and get it in the # form of spdk packages ################################################################################ -FROM fedora:33 AS base +FROM fedora:36 AS base ARG HTTP_PROXY ARG HTTPS_PROXY @@ -30,7 +30,7 @@ RUN /install # Contains installed SPDK from build_base rpm packages. # Does not contain dependencies required to build SPDK ################################################################################ -FROM fedora:33 AS spdk +FROM fedora:36 AS spdk LABEL maintainer=spdk.io @@ -95,7 +95,7 @@ ARG HTTP_PROXY ARG HTTPS_PROXY ARG NO_PROXY -RUN dnf install -y python socat +RUN dnf install -y python socat python3-pip RUN python -m pip install grpcio grpcio-tools grpcio-reflection COPY core/proxy-container/hot_plug.proto /hot_plug.proto @@ -127,13 +127,13 @@ ENTRYPOINT [ "/init" ] # It has to be placed into host(a vm for KVM case or physical host for IPU case) # It uses gRPC to expose this service. ################################################################################ -FROM fedora:33 AS host-target +FROM fedora:36 AS host-target ARG HTTP_PROXY ARG HTTPS_PROXY ARG NO_PROXY -RUN dnf install -y python fio +RUN dnf install -y python fio python3-pip RUN python -m pip install grpcio grpcio-tools grpcio-reflection COPY core/host-target/init /init @@ -148,13 +148,13 @@ ENTRYPOINT [ "/init" ] ################################################################################ # ipdk-unit-tests ################################################################################ -FROM fedora:33 AS ipdk-unit-tests +FROM fedora:36 AS ipdk-unit-tests ARG HTTP_PROXY ARG HTTPS_PROXY ARG NO_PROXY -RUN dnf install -y python fio +RUN dnf install -y python fio python3-pip RUN python -m pip install grpcio-reflection pyfakefs COPY tests/ut/proxy-container /proxy-container/tests COPY --from=proxy-container hot_plug_*pb2.py /proxy-container/generated/ diff --git a/build/storage/recipes/environment_setup.md b/build/storage/recipes/environment_setup.md index a35bd666..b1b402bb 100644 --- a/build/storage/recipes/environment_setup.md +++ b/build/storage/recipes/environment_setup.md @@ -10,7 +10,7 @@ The containers running on those platforms are named `storage-target` and The basic requirement for running the recipes is a modern Linux distribution with docker support. So far, the recipe has been successfully tested on the following systems: -- Fedora 33 +- Fedora 34 - Ubuntu 18.04 To run the recipe some basic system preparations are required, in particular: diff --git a/build/storage/tests/it/README.md b/build/storage/tests/it/README.md index deeb93d7..1b428f99 100644 --- a/build/storage/tests/it/README.md +++ b/build/storage/tests/it/README.md @@ -56,7 +56,7 @@ $ ./run.sh fio The script `run.sh` returns a non-zero value only in case of an error. **Note:** -The script `run.sh` will download a `Fedora 33` image into `traffic-generator` +The script `run.sh` will download a `Fedora 36` image into `traffic-generator` directory and set up login-password pair as root-root if there is no `vm.qcow2` image provided in that directory. `run.sh` will also try to allocate 2048 2MB hugepages if not yet allocated and it will request administrative privileges diff --git a/build/storage/tests/it/traffic-generator/Dockerfile b/build/storage/tests/it/traffic-generator/Dockerfile index 32809ab8..23b9ad27 100644 --- a/build/storage/tests/it/traffic-generator/Dockerfile +++ b/build/storage/tests/it/traffic-generator/Dockerfile @@ -4,7 +4,7 @@ # NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE # WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK # -FROM fedora:33 as traffic-generator-base +FROM fedora:36 as traffic-generator-base ARG HTTP_PROXY ARG HTTPS_PROXY @@ -21,7 +21,7 @@ COPY scripts/* /scripts/ COPY scripts/vm/* /scripts/vm/ RUN WITHOUT_HOST_TARGET=true /scripts/vm/prepare_vm.sh / -FROM fedora:33 AS traffic-generator +FROM fedora:36 AS traffic-generator ARG HTTP_PROXY ARG HTTPS_PROXY From a460f1963bfb35a9cf13d6443441ced4d378acdf Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Wed, 25 May 2022 12:29:59 +0200 Subject: [PATCH 3/4] Use editable png format for diagrams. Signed-off-by: Artsiom Koltun --- build/storage/recipes/README.md | 2 +- .../storage/recipes/system_configuration.png | Bin 0 -> 27651 bytes .../storage/recipes/system_configuration.svg | 425 ----------------- build/storage/tests/it/README.md | 2 +- .../it/img_virtio_blk_over_nvmetcp_1.png | Bin 0 -> 30281 bytes .../it/img_virtio_blk_over_nvmetcp_1.svg | 444 ------------------ 6 files changed, 2 insertions(+), 871 deletions(-) create mode 100644 build/storage/recipes/system_configuration.png delete mode 100644 build/storage/recipes/system_configuration.svg create mode 100644 build/storage/tests/it/img_virtio_blk_over_nvmetcp_1.png delete mode 100644 build/storage/tests/it/img_virtio_blk_over_nvmetcp_1.svg diff --git a/build/storage/recipes/README.md b/build/storage/recipes/README.md index 326dab70..7dcc9b4a 100644 --- a/build/storage/recipes/README.md +++ b/build/storage/recipes/README.md @@ -10,4 +10,4 @@ device. In both cases host target platform is implied as KVM. The picture below demonstrates the configuration exercised in these recipes -![System configuration for recipes](./system_configuration.svg "System configuration for recipes") +![System configuration for recipes](./system_configuration.png "System configuration for recipes") diff --git a/build/storage/recipes/system_configuration.png b/build/storage/recipes/system_configuration.png new file mode 100644 index 0000000000000000000000000000000000000000..c366c57123e44baa1b7350d1752afb9b93ff12a5 GIT binary patch literal 27651 zcmeFZ2{e@N`!H@xBTGaI*(zK1u_xP%ov|}!Y}sZp#xjh3sVosG6cK5(DO)ti5^X5! zw4fA94I;^sCE>kCpYQkkS>E^eKj*)k_x+voKgZ$m%=29LbzS$hU)Me9b~a|*yM%W! zF)?vlm>WAVF|kmYn3#`4*ufpk=;;LThdIo_48~N|ci;;X6UUWs6W4HJ6b2vY&m^S= z|9vF|Rlx;^g-fX!OF^L~f&RYXq~IWM3GNev{qPvPzu)iApej(PqKbwhRKrPCMM}*` zRgLjWLrFzL1O5AXZ>)dNAArDZ06-Rr5Aq9+32S7xDLb<{J|eHe_ui{;}t)WcccUIeKHfN%%i4$DrCj z+#CCw;)p1XXe>6|!9>%N9HHZ54^u&dhZ)aA_>;o$fKA&>``6-(55l8E{D0r|^N+y$ z0!*M%Y9@f$et2&z$(sls!T-TeH60aW6;u0QCu1*1$1t)*INS`7!T=KF4M^AqZ3kGY z2Z*1KYm|0~t2xfm$tMPn@(U*up+ptGfM`{`O_(<}3~n18 zL4rD2yF>=U@#;`lzW`fwq*JJ-nYmgJ2Jc`N6%?%l2v@;tJ7FA6jH2*%SeQ?kgSL+! z+z9RmKJh`=IRyj}OcB0NUu`R^U=t%mC^o=O6Bg#|V4`Xt9*sBhMk26LPI#iX4T1m= z3&2?hn`rykSZLdW_ia$VrerI1aNp0{CS2XZQQa>{4HXik35yH~4m7p#bJUCowo@@v z!3PJcX=;#yk**P@C=*i)i)iq+kxvZL(bq{kS_A74Knws-!gbUTC_uldk(H~RrG~MJ zpJsHljv30t+K%WC(~bhXgPG%0UA1+>f-p`%B287IyurPg2yhu4MF7GCi`E9etiqvo zMtB_}-Xa|)!+zKmoQbBDhv+|4ue^tu>J%~ z6&JFrx2X>)%--Hu+gKY8by7DASBtSmkt3V|T2NcGYNQ`3FbpWJDaKAsEyM`iwI^$v z+cM}jGPSYrj&g#zYHF%DfDMJ&BP=2!@dQ6-#-4;(*sA)t5HLn?1Oku(Ggl7@vqt(y znre9mhO0PeTWPt#Y&7hR%(O{ZO8_5=^9jSLJE-F^upobPeK@LV8?K?2wMxXJ&uU7Q?ayif?C)joP3JH#lmk=!s+0@#}F~)%$PPPv8u`>%sVhKLhIv7oLHE(Mh&9D$6GTb{*D-!GM z8yV=YN(?l@n4o+~ngLoCs!=Wx_{bEoke4-c~q2!}f1ye({DK_nv$J1oM+B-l9$8H&J0n>m|? zknuj@+A&CVHCP0~BqTT-??SU6QhJ`5_qiw6|?5IT!ckovu8UP2#06V1HZ=%}c2o(6`y-uEVOEZTD(Vb_fH%TI zqcu@TTL8?MWNv8{;cJ2jfSdU_laUs#Z z5oQQ`hZtua6;<;Pb-RdY7h9Aa!U1PujEE*jIl_ISgM0$bk-^3eaEC}$OFwU7xLKql z)LRP?j_}q3kOPpR81pa-jJ8=2Cd3bA2ensmHYKQ9<1Ed5T*CY!jnLsp9Ulj*zpD*K zg^2c5vv*S0ig5HpnUNeZ+86*q1?Ov~;ph-Z@W%wHpiR*M$S`0@(BU?&j&`aRzE;{q zZwoV5hcHzYYhdDhHC23#NqAd7dmU$6muM6q!kJ)7Kog)Uu6PGuD-9bT3mb4#8y0Ly zH1qZIC!ox%{X?7`eW6Hi3$UE8d0-GK8tEN_33JB#2S=Eh`b0+Aght>(EX~x^9r0m0 z(ZB)0u*QKJYL4nSM>LKY;$TNmS9gd2M$RFBf?zN?6e)#f~?TKt^pb7Q~5iUkLe$fG>7z^7N zq(4mEmth9+fk1@SO+z%)f;7=i(XI?B#A;#i#&(VJT1=vJ4Ih@C`-;qeja{`E z+X@YHjKHI9?NQpuKt@PlV+Ve@I0Lb>Qo%>z$!Ls|m6g!Xd88nfJmGpzy(XP#ABUM z3>kI>4mJYkZyiBKlHn03f;PdA6rdA^La2snXj^JnyAm|Re34NRY8Vg-fLH$&4Qz)7 z;Qv2DF*TDD#49UIOcG2M#zsz2p0mXq_xH9Jw6%RYd`R{P1j&){sV@`t7;*Q3i+|h-Q3ORW9`$7{edboU$eKS$?k73^fP5Sddez<6H@hI%(`!D z?TniYx$vHIBstQ#WGq@t{A1Ddo$Ap$`Wp+y)6)Wc&itMdOf0{Dym{}=3ln^;z3*yy zU<=b6b{tV$Zu}C}b-T4REN^Ej=UHW$BUAE_6HJUBW5cT*FR$}et@Cl|7pj2v%*({Q{d4jIo3_2v%M|S>$6*a~N;}z0M3g7(lHv2oY$Pt@X856RkV4kqk1`KI z${yoBkq4Gn-9>_*itsR3M(?6{zU zmTdqbg(n76v{SDPi+ORG)2^qmQ77b*2OskpoByWEhM(~|&GFr*3}%i(>iH9c$5Lo$ z?NbmAHbW*h!-sel^uF=^F#)l0suD)If_rF%w|LQfzj%d6H0LpRn@4+PB{}FcW`$iWwB(< z>sznLz550drB{rpS!wJVm`s$-JM}ZuxyDbuvUHEJO8JfquIfXn^rN{if*l#l-xMAw z>s(${Z*riP&5ZSr@+?F~4MCs33}8_5B8+*5;Z@m_?p~LsW1(=}!Zd0A>|aB5XiPKq zIddYuCat!otE7irtLuoBLia5J?@hJ*T%RuH#f4NG22=oQZIl~ykRY1{em5G-+Egl%!udu59b_? zH|U<*bZ?^;_+hL(w%OTzSCS{6<8v2N*@k_J69z$_T)0^cgUxKDTb!9rGo{e|xfS}R zp*uHqlLzUjls6vISILrDlxy{!*gX~wttD$mU-Hf)9!qyVvfTzavr{#T`D7r=jRfxn zTVqb)#3Mc*)h|*~Xg;Q=PIzqZY;sU+oOJf#kLv}w!g1oJjeZ#VDgp4d^7$)v@dR+Sfdd=e6@-sWjDP~daR7> z{7H$u=gi99&r%SD>n&=`V_(TO*FIdKycg*3U$Z(J{Czw>GM^PfZ0tL>^v-2ln~r)j z(7C_4q0r#yRB)l&llpHt`@`PcUBUFs8#}#<^1Vs44m{Vd>6s=Ywyj$t{Sp-C-$vx~ zZ_R)7-%Rt_wk_%hb>>RWP%gbXz1IEbuUc?&;28UnawzRQ!|X7z75(1zTlJm$Sof&o zCAZC9T?iZBo^2a@;??%XF1`N$iT^*r=Bq1E5N;55$&-cxY^z0&IoS-awrpE|L;pWH z4hjeHDEF2 z$n3uIm&>bX+RycOCvCgWyw2-}%7-MFC})|Vg-py^kZm8B^G_cr%?zo${3*4}NP>-l zH}9~r%Z=Ijy&2*q`8$6Le4s;xM{{0N_?MjYJN5N>9cM-I*VdCc_{;9x2`DMsr|fBs z^dRr$7Cm~wGwS1xm=7m+mJ#387m%#-6eyz)V)^rD2Fmw^>iu$<8EuJHb4DT^jc_ksn#th!tTL-B9RoifEsH3+Km-Dh;j5J4b8qtR;Q1{C4uWss? zI7JxcjnxtMc_ggSVeV_~bLYeju6V}(aGrkGpq#9EUG+3Al}&Kq)QQ;NTz}tQ_e2R&}jqd@n za?+$u;)B?>tc=|Ii>yc;a-Et?aQ3m{8@5t@a3N30<4E4_@3pLbzZMAHd5W)ZB19lW zb6*|`H;2NV9n34gcjU@ncSOj@Q@6XeYvGGWTi0AoCTUr7NEqGMv{3lg_ne-Kg8kI1 zTe~YgAE2MfSC`Z2>>EFST=VbFGs3|S1Ve~vA@&-O`zG72Zz%9D(a>{9VGpgAwvch7 z9@~GVK;*$hUoj_zJG2Fm^w6JwUSKN|(_ zX6ZiFf_qWg)8y`qYkb$f+)~GN_Qyv>FV=S=6%yh+#kS+! z`eV{NVuYBPAw;(GFW4aWjkdQb|F7G`2wJr`3~?g|w2Zk~BzXZor)dc_M~LUDU4N~% z9D^`C6fxR**AkS4g?AVtagBpF1SmX+u9e>YBf1XSbAr~S6O<;*29p5%;-PH#ID;6x zW_3O~nwJqo4=-F2nDS=>sPh?`@=JVR|1AW6-7khe{O@Q3>uhja3-1WszL#iL1dgKc zFRZOTNwlAjQ!5kzib31xEjq@3}l938UY!z`MT~8Ab|p<{VS9rv5Tch;7fdi33sHUo;bJ`F}4k zl35?Vjd;DFbqT$mZ0ass)DdNApst;5xZM;aHef4mqPSb{aCxh!cp?mLYE7B$Vt&2s z*1037+;s>4&G!hR4G!@j#uHsp=2!y96$>k|akA8J2az<6zvyzR_9@s;>K@~B{g4M#fVbcSB8D{${po|tA%!-(KVJRzFD}O zulPwvVA+b1>%kB3$We=rQm-{eD&InGt&`DlE6LU_xoTBIoA(`B66bff!mR|Y1qzVS z?_0d|50`%ORcid;K0OeuptrVefVoKTA6O8zS<}=D2^Osjczew8O84t5A@fhBBi8RR z?X$0pAN(qF2?~2P6KtQK@iTqp{`>kBgHjav$!+|r4g4%;HAY|Ldv5vD_1$_+g)w8t zl=$6pYN>!cTZmOThSL6uLs7H{@?<4b9IgS;G4Xm;uI4Gf?A%Cxlbu{g`TkmbhB)3A^=nK-!|g-o~lAbyrED~^uW znzZY89#P{vt8(%V9Ee@w^SKQl3e!E)bmM7XK}7R@mb|G{p=(c}kg5`sf_Fai6QBHA z=6?NPm28VEs<`(f`*eD(E9vfB;6$Rt_cngZf=OB5-5^}h^X0=H=0&S5Nm4P|!)VTA z7hk^|S*nA=0J$pD*kgCO<)r+K7AD!nWHOSwvcW^{cFSO%+TDyi{k)6W7Zb5H3E$R! z=onnq$MOx*VkR__T|OB%l*uL!mOe^T6@XLv=pNaCe%nEkXx^CTe+7y zC{Ha)Db|e5c5R;7JM1t-tMx0NiL3d{1S=0kyLo_O#gFgCf(McZxdH(3S@nY~l!5&P zBtEp@)41jX=FdNia<59|A9%L1rq}3XRC-bC>VxkKTaZvX`94YJRP;Tw43U>nUs$`V zB9+}5kDbhy;%aI>!EwDktp7ny2vW(ez-ls} zpo)}PDWvP+y61OL6x-}i)AGhH!EqLZCo#oBfC>uT69uX4xKtoG{Q40VN6ji#+GC#h z^Gwa%ZUpXhPn@vr^Mj=&V{zwebdt}!KYW6#vLH0+Kik0p$GyV4-_uBac)PUa9te52 z&ezaf-qQvWcW`Kv`ZoHFaQrG+M$I5NvkaM=|C7}4qPbmVkBE(NQhpflNl=Y>(OZi$ z7G^M9Z>BWhvzZ&=BrPD5E7^rr@C*#c+IP|i*8J_p@OoleH`Qvb8%nFWV8k;O*2<*AM3IvOW2q@bYw_h4_s<8nP;G&V}n3$mjOVPo9zArvxEBw778o z36Kc(J^xcmdT4jv_Ic<=+9CXRIKTHa(T)7d>(dlYzZbr6TQns~Ea9pBI5t5a)4tFS z3yDlvhaJ(E`*LxuGaXQgNCt)6%6lcY=t4=xp*j-w}>Z8~K`tG&70~Ik} zKNf{rpjHkh(b$WOIdl%d2pEY!*%k?^@Lf9rm`DhZIfb4X@T4LgX4BA8qmw)P{=Rqo zM0{V@ojqSJI&SK=#f*eo<~>YbTzfAzX|eY%UU-&}l$sGZD};@FuApYOBW&jKWs{DGR|Ju%rq&HL)Ic1} z&2rt-zE_I+#LV`(P;Tc$W=6hLRKC&O>xbYXA$z1ghd?R|PA30Hq=DEygxLc;bgqsH zbkx3`9^<*y(9mizwA7Ry)o7?sp$B4Hxhl2z(X!<;y`zEwZ32{w1Y$OYTU)j3I|G{0 zpX}A2pzW$`$+qe(6WlW#N5HvJ11-XC4!m4#(ixwqaTuhLw9eFd-J$t#0|waNJiX30 z9Db@~ICmV;8s8#8)xW9}4gpsD?teI42J1Fh-!1<7bk)=gT|xs3SsXp*f?A8<|3D&{ zu{xipuTN(|YB}47S9Vnj9REt<4cYS{#E1!YSr4e2wknD51TCP4lVK(VQoydVwl7n- zT_5nD_T{eRe{*ZD*9Izg@?0k4u`2Y`qZMA37T zgR5Xo-T4oDyT^)Wk4hSQ6R$B&=ZQ|(gms}heh+b&YN4QWT$R2L>wd%U&f9n7nZa~Y z@lkRq>+WNAX-Zf`+v+&yc>n69h}mI|8h<&=M|!`1NB#`@K@8b6D~v-iKk-}RB-Cnm z%O%&qY>I|TbXVCf;4+TGaT>=dvQS_~nV{5XK!jlFI>*vzwc54MT7L|>h5wrG#3tvd z-MZssQ=(ryOvUG3d@iaTA~BMB*3IVMeRYk>gf6h_* zj(}8`y|^r?e1<;w+zdG^3iRlB>c2IU3DjSIuUAGB3wg-wgP38I1TwhFTO_S8Mr$_} zHotgb+-?MtT=Grw+Aoax)`+TMtW9wPQk;7rVzz_N41VuFY$Edu_w6AEMv!(r>>nl(;tnfaKHJ`iYH2r1?xf2 zVe^EAbSpNb^#mgjXI%|0+u}Cf3$RlV(_C7wl!0cN{YN`_(SA5q8_LrIK`@*Ud-qH| z@m(!oZuMIIN{-gKdD|^+Zf+k$Wz#p!A%wT>vH9c=eL3=8jZo^Hsf9j{UkgF+=pJxR z3Zmx%&y7RC_f~BCU(h4c-dQT}cL%n|majXXLazuS(|HT^wvgj|E#9r?DcreLy5w~} z^f9@>vH5nTo!@$*4YFZ4jWVNcNl4cGa3IYRS3R@rkz>2lH=RBNoN(AK#l)NX#R|vw zWr*=m6wj0?>@vjNJ1s=Ob^8nxBKN?`PyR=>4V_@`R0uxO^36Ybd)2XM#k19y;a1-o zbF*}+xSwlKXKlZjD_6vu#&rn2m-XZwmX&=s#-0rP#FoNY0~nCW{lHWJnfwEjflPuV zh5qgP%VbWaFM@A-y?KXED*lu z=B9E94z536Av4LT$^kMSKTlNu6fQweyuQ|5x(DqEZw4Ov;0!vDyHeqa68YpO8~golIo*;-`YHqjd>=wNW+${pZ%#KwI~l|a)E!&m zRV!Atv$_yEySmCR4E5RNjO4jmKX$awaB1CznKJNj7e$fVrRr>*l`a%@LxTFKL+uEK zF13@CpEUDx`PPd%wK9v`o6(UvZ|cH!4mi2kwa+@gp-a5sJxvOigNax^k#qs=4c2|I z%e{Os5u7Jp@$ARIoIz%()QMOveh4x2vBJHUtq-}j`WB)`!>^WXe8q;3#jp57sib~* z75?D~=ID}79xYPqR))Bwv-TitPO)fD3+h|WFXQ5Ir}Gpp zneS959)w0yvKSH0INzq*v?R@`N=`wdy1 zsJ3j7_}R3yt$yPuRSZJ5%5;l@gsgyzSf1Bwvi$XY(&gD#T2-gtO_S^m>?co@xSQOY zPR%fgT}v+bl0qxg8G0;;*4d)#JP!Mw!`U~24GJEsk)La++IXux^`tG{da`Iu+^0%! zX=*xtLK&+)5SBVgmnliO@;qTeH`wV+V_zepy6s}Y%(PD5q+#p*)uN`UUtv>MF*|AZ z?X`0APp&k>uOxZ<;j<|N+O`tZ8;^W$i$eX3M~> zEFUR$;2l2hyMw~L^=OyF;PqcMMGFLD6LHDF%5%r8-LJ2oELfRL6r?G-HN3dG)nb}vSTd6+JH1%npL5$& z!N20p@i4Amk$K^Qe!*K&&zU(Dtrlk+mfzkgj@7prS!zs(7w!$(kq$m+i8f!ksDt6FEz} z^H@eXU{6of*yt z2B+`rvu|C!6Iy=I9{2pDf=r6juF4omq)E{|--Xh?(C&WebKDYZ_?47Ewwv}142hjP^O5JOU`Sh~!X9hFniw|fmeOigJo@8$q%>lR5?gtO)J~vn# z)2dGW*0MfS;IHwkyhljXwyz0=I9q(~qE(tN;Xt50`;@cX z#}BuXW0I;_A>^X}BPgx~`x6ug;WKxU!t}8Rk6jet-JO+DUw0q0@r(iGfzPpCvv#ct z_BC8YIav8-;GHwoS+s?PosC~%9il&!I3)dX(TM$XCWLqLnXy=7@h5$>I}*;%Kd;aY z3SZNAl8W#DN;zS$*hl!W@05cYH9$0lz2Q6#c!jk-t!mmt?jYM7$?j!z^5w^Y>xFWv zV%rU3ZD+d*$Vu5`mnxHsGqc)ob8sd^ieMMqHFU8)L@pN^2ZD_e9by*p$rzvm?7 z^c{i_xyBAl7xu_dTPm2E53I_#2U+p_%C+pvwue+Uxt&;1N^pL&Yve&io@3(XWVWoo zNM&54|Gr_mLZi5lyooO!yMKyh3>kWvdQ(YOJp`OjRXwTex`TV`QH8fuqVx= z@$*CLr#A-1J&rmJ+hZ)FgLyR^%iGXC!}aCY6ZgY#8)xORYLh`dBj;p@Hdj7~%aQa5 zSDxMQuM~#UH*!cGTqjsuifx`aSiKqWsARFXc$7SWF}1E11TmZPS2&}>5bC&p!tgkY zHz)A}lfBVlP{hZm$e7@IY381cT9#_c?+8B9sFC}NB~sk<%6csgV&&wo0Ae$e?8Xsv&R5#Z;bwd+hDB@ z6QwTDkk{rpAW-Vs5m(;7=k5GbRFMDNeD3@N|K;(TkKemGgNvB{t-Zw|K5p|XmBB$b z-PXNqFMGZ_ud(3K-<*c&@i2B9ePcaDURN49T3g7zTja9bJ;Ef9uzuOORo(n|Yx5C> zDifN*T3@oPM}DE6^rVOPgc8f%4QQ=iLGhx;rb3SQFu2VQxb?5%+3%tc{~=KJ(=Cn; zGBq7~u++l)`i6YT4I6B+e6nVKS{MEyeFkfD6<#QA`DtdtZc$LN-E4G4lKNJLXOK3Y zkqr7i|17czATnuHao-bS=X!^Brsv3j{6^{@LEy+jF z#bHJ{Vc($sA>(~8PD(qnvz37;X2+kSNJhy^+7!3A@xB`4AALW=ABV!dD|wk+zcmZb z{zC7W>uCyo{lF|3z0}PPA!dcK3!&q(CE$c09)MpBwMH6^#ttI~Rf zm%n7??TV{s-jzSaO`_rQLp=*GSpM7)hY!U*fp4n zu{@&{76|`coC+b7i`^`wk@IvJxNZfBrFUyhz3JuA~ z`}F85uI2+%G#KhK1kj672dw*7d3!r2R#%11jL=V{L|D$6t3D>pR5?WGe306yKo!xd z*r$8geW(GB;u@SnUGL<5u$u~G(3q?D%b89vMc&GkzB380aY zf7eJbZl}3Zf||z-c`R4Jlw9xUw1e}Xh5{%fFNJn9ZA!#=2|?eQE}>&h#xCnm7tuxg zX5ssaBJ2It`D`vQL=Fe+*x$?yc$+3mJ>c9lZ}6l4muG9grCdYF!Mj^9V=w*}V|rUd z33I2%bn83MpWzGtk`h}|k&2844b1-)w!$fS?YU}Yzs9%L*Y2%ca%Ul2UwLzS5Mnwh z&oDY5U;9@jeY;orlXJ#L}b}F{4FX#2N z-t$I?LWr(@_Q7u$bV2?}SN&6+hqj<}Igy&eMNH3?{n!qkWB*e@SelD6fDnsYimQ?vMi+Cg$PEUR<9v*%R{){%gE!aSZGui)R4l?loH(?W_6JYRztZZP*s4o|b2k*0GX}ZStEW)bFZ#J&B`3g=O<<4Ra%c-R)h+j>b5rP$`bA;Ls3TqHzjRut${pr=N zy+;ihF7`a&$X_hWYi(w?lssKDE`;J8f6M-$-hTP|1~2oRqw ziIDSrDlB&LPWR;H!scJ*ZS%%Yx2idmea1ApSGDUOoD)}H`biH?__gHqYU^h_d_c#o z_xWwT@P&lfqO)ywAHJv$P}>3vUPR~K(yLrdvmI5DjyPJ}q4Bt`;Mhkk)~RVVd3Vlt zxR%|m$4>v+MLZ%^m2y)l&2&AbYk~hbrM;yjDGX%CY!vQIg#EqJ#2?Q)i}W@>U+d%X zEV!l-7&m7!y!tIe<5Tq|@suSU%iGuQN>Zh~VU&2l+jI#kJ<`Vzx6&7Du9rOc^zCkn z)G1owC5uuXNTuope-}Pb1#-hUdWIv-a6@g>D=tcj7fc7TN~ICY%0fdPYICL6&gm8= zg#S2asgLQuQjN155SSkPM4FCr>C8E|=t#RjBN-&>*Xi>=9>16Yt%)`G1S{^Jv38ua zp-d5kB(@YX@E}3!b?g{Nl4}E`I6yi}1(ko!C*4(L`;J@V6-iq=cXSohqz; zxQ&W#xgcCGjI101Z2-J=JHR+Pnv#g+whtk*o_2F9P*Lx>w1GCCS zyEG23=#rKQA*V(=^N$?YWCaPLO0~N4fusndG+>5iAQK%D^7iJ zJ>pw2HR7jLnqbl+cZ4T``Tk4j|^LiRfS>vW?ge_m@MGsB#gJ9A9 z6ixV(Yr;0sbgh{mDfN*J`cDmRl!Q+|_pvz9BT?&D8@^yWrE4%+XBz<>%V8CK;CxQc zL~Wv9!uhq|M$6}`=V_$o#F)_HXPKM+PfT;>zbH6Dj!>cRy2h)N_`jcN)`GiPM>*dP z9<5E`9X5IrMA5~LNaguVDSVXGY>=e3LoO~ia!|&Pm!MK{4k6<^T+j zT-o15z@ct_ZnB-Tow!4d@vl5jyHhGN^K1UHw>C~cdtZB@jyUGx=U?JJZ#>BDdlVKsf*A7O zSQ2ZX-C68hzZa_Ht(f=d;;9}hPHlMi*97R4bj|_#Pj~)BQ2RVp+NePt#km*S0&@_? z0V8Dkt~s+Y29(Dtv<^`hZc6hDq2&@%pwz~$rv()M`63Uk(?OLLyt;zVkd(`>F?MMS}8_Bs@JTqJmq)u`%i`bw4Iqi~qsX zWLj4ZeXH9;^$%j73sN)&MT*a)_QW0a2!$hy;d_t~py|Apu!L*n)WyO)-4}g6#EW^H zAg-29!1BBGO&wwK%8yCnF66Vf-hYu8Q`+ejx4ENobfWp**=)z3?VC%3YRtM7-J z3ZznVRWfCg&jelCgPvP^ni%O1MDJ`bu5wYf!fZ@<=By2ngM+VN-O%nx#9~J9hx^yx zNi99?lBo^){EW_qf_?;Uge{4)Z1b3Zs*m!n3jRiA7gkMR8_^(m zPK;O9UmFb^pM_Hp^W0vq!*42gbr>6(;hsQV^uzZpv}ezJWWDh;&=O~b?;eDv_Qx~k zsI5UmeEX=%P9kF;7bdeo1ebYQx^f(RjX_pp45d(dR(JN)BI?{M$0>q5zk zSZ+0$m#Uq0lpy5Thgi)^f@mJ&5&4{k9XSS#o8M|~Ed)6p*8R{6z5Ukn$E}wu4G&7* zPCRQC-GlX(cD5-sYS=Ak3+W=i>fsnWDp{U+fvq*#_?t(x-^`(V^{B2LZ@-)Lu4s=x zoPyj}qNZHA>2$;dqIgBXW!hcNrOz+5-qo!>frp}@3FSo_$leH#IJ)>gGqWo8OSEK} z6;j^{^ly+5q7vW5+0eD01^iRzCr&fEv9DW9tmd;kaxG7 z+L7-e`_wVv=IF=~gHPn>l;tlMX3JjooCAk^rW_!ZCHFdKea8jSB2x@Ae{(`IIS)jz z0h*HVnT*d}JLhg8J>B8g2w1yZX@nZNY_6?swzw3w{%oQBb;a99;g6(7lG(a@2<3^o zMV_wfnY<&1A+yA5!Pj)9$Fj3+zTJv3yLS{aJ$hl`?JS~DbMMP8TP+8#7EE7r!%g%` zJJS7No=E0x5qK?6-aF!4P!Q)Gp!XbDwHHKio-m*nN_gXL$#w112l2c!Z-T)*HQgyGSp5gwTS+%eTU> z&nB+h`Nilg>fQ@FK%RSXS#YCPQWe%cV`uf87IUk-=e(sPM(BtB3)!J>J4hp6Z;UA4 z7}9FJLRPb`l99U&lP}rpwe(u@6ROam(750dNT?;L$98~vhK(_(=V5SK#Mqjc_w+?V zkDcN>kRUa|m52JSghu4Y4_4QnAN<{nC>?eu;OK=YPxM1rZ2TCrdRxMs^GHwRdZ$MD zm(xghIB2X(1IY*0#AB;4n0xlhNOOCHiNdYvS{AwH>Eidl-q_Dwxx7$w>6*-0!I?n1 z=s?knivZ-hc*1x4QQcC8Lftyp)d@mDPGCq2b?1S5MTgzvu;PiUQ7@o~y6Bvo*oqVO zcr_y(6W3nde%A9=QAEsma$e$af1Q<4RYk>nxku|^Z80tHOJ&A3Yp3-~TB2jCMEYKf z3VjY@p9WQ~JdxxO`$dg?p>|ZZK>lHsWu%g+fECYqilU~O<;DS6h6eRU-^D#$l5jVc z>+xrzhGuI-`%8xL0{m@LkoUf)Rh(J z+B35xlj*?|J{RPX`;m9r4d+^&~yqj&y6_kIC`t8EyBttSH2nOH4I-Z$6qGxL9`$My8*yAU^(O z4cfut0b9q|LHJsdVYa>(oe!wdRPyGmj12zbrQ~I3TM?IIqfbBlbKrs1Ju%rwH52a~ zn_LY}#RUPY*`=@}{oQ1Z&6ogYJ?=5WTOUhrJ&i9myDw+WJT-DU&;LTC!u{B9;WcC= zXuyS4Q7*0ze0yI5ub}v!9J#k{RkQMSuUd8!9wI@XET&A;&x#8*yNR$22DTB+MD>M z!k$dFf{|Uj51TwT$+Y34z>#lC`)6IFg34*7>bD3d5Y}n(fQH_qQ;%L`^QuTvp?2hc zKe^+YU#s1@{G4Ey3Bh<}nS{*BxTuV#9bPs1Y#Z-^s(}$-hN|&PQtSIF9)C7n(Bn1Q}~JS6-u+b@XX&WV?QU`)E)>aNvlF+b_LwKU$;qMI!+lQGuM zU9yhe=hLRTQ(K7-Mwp3vT<^h*!7NkIlYGg8`y+LaoT9Od?0mAX?)ak*Q|6G>y=V>X zcPmIGcBm4i_o6pEwf+rGXzf@wjL;niOC5Z60GwZ=cGw+tsLeIt0JvF`Xb_rZ8qrQ#xo5IQjz+G!6k z0nyL>clHd3+WT9}c5t$rnZX{OqmhoMENj&h6X`w@pZ1_zZ%1m}@U zY+Xsm13LnW%|L14{XO}dbC2kXEnq&P@ZR1=GV+g+F5z@GkYZi?_Z~=4Ul{xpgo!+V zt@Ij_toc+Dgz-t76pd8qXAp2pvM$?$WMrp$Cor0(%2Z7oq2FS|E(Yp3`&ah{0zVsQ z3}nU+w-_?XC2#O0!|Px5%|@}bhr8WBd_9t1U7~Z6QQ12>yA(F{GH8evjNC*Y^E>JF z%kSX+%JiCy{qyQYS@-v{IsLzB&a%ui@qU*-)a2Ok5B8VU{KCPE@2!ADjVbdNvF^5 z-6JoK?6&1_6!$1s(s5OZtbV_*bUNz>qoiOwBnT$;CNssJlb9*$X9O;yYz#e@ zy4J=!<0r$~Iu>e%1T9sCD#*K|F+uT%& zmQZ%ywsJkruRk7RhA1X$PFrwM&K7D9FT%lDpVMfHZ`AWg&B4JGPpy%hIsqfSj>}*I zR$M>kVDF`4d41Z=8LqV_xQH)a7^T+B9uzR&{~L%L_n&|i#&BV=lh)-o=E-4C&mIRY zo-TzT^2q4fM`8ZB2Qc|yO?ay2oGj2No8t%OgnvWO;rug%mHl~d_>$JoG|;QqS}qu3 zTYgg?u6l~_Ugr@6$uOI#s1g|Fg5j=R zspcInOTVUP017$@Wm5D@61!dQCWEFz&Pl`l+1t+?x*T|x+X#0}I&1kCIK}jnl6S#Z zf@&dTv2-mHB-)Zxd)r6|@%7UeSHwY*WbitN#FeZmcPKsK0^j!LV7Jbt1@J?^G0yw` z1dL=pTKF0q!V{AU5=Rr=IgVXTcoVH;aES?wB5BLKxN!<#0&ueB`xi{QJT(C)q@OCj zBD0l!{Qh+9Z627LCj4xcCa5F=67+>6sP_D3u?YZ~4Lh}Pu)4N6gcy=q_DFF%YkRE* z6jCFUI(Zrh#%>LGCm13~N~W=6(}dq2WXuJb>iGgH{*MD;y_au;ldtB@IGs`65!X6E zS7I=9=aD#sG5>qfj*4e>;B@hl5KM$MT>UmUFn!GOb<$p{eSrZe`xJq)k2d@@I0LjT z!9sr#9SYdMM@FD9pk~Y+^oJnS{8nYua2O9sH1)!N~hDuViDW-5{0Fd<|XAGDTk~KQ` z-$1Z3rAjx_)7k}<6M-si-v1~Yxg9jy_j~?Jr5I|;CCWwuZ~wPz!M6Gu8Rw1&H?_P7V0?vv8aSCYg>$%kVlX;+r#x;^F$#2LwQc{uma|Oh zQr7=kNKPgX+9ow-dCR?spO5i;rCh9SO&zVWM_Cz&S|9n+!+be6#KGfyrjXWhi#cyJ@R=D z+A6WWrq8Jx;Ea0`B%k$j!_#a$Z7)$!+HN8G`ltYV*&m12nOG(uZE`(lt;73SxdGM`#EPHfr^!`@W z6@&J?>(Bca{c%=$`d*hCtN7Zd6(Tl2EsnEY3VpE%F(3=>yzR(Bz>#rur3~fMAGqw3 z+}Q4>YDN=qBZViQX9x~ue?E(U0ZJwr2VWjxq6|E~cu%D@?bIJe_gnnc$5>;h$w8VV z+KS`DQJ*gllZAkHR`39Oe`WRHu0CFk`zDNhl1%6r7FVn~w0tgm9s)AoY?nPRjEtZ9 zWaJyLtR7tGQF?MRhc~~^^Sw2Z8BZC^fzXVgtP%uvGHR3sL$4Z zp>v?=F34~awsCwr{_P5zG<;{z2@dhCcRq_s=0)E#2g)2ho+0~hS29>BL`bCG&w68E z+HnZXz?*=9&^~@IU#zNhPlJe?gDg!v&8=m>4pKhs#!M6m`U64^*aW-p{VN$RDRI9 z#gv;6|E(-id!ka)`s8Z@U22BS*6(qBqDz3*Pf}9qc_SvaJtiMAIuhCDO3FMbY7!qL zz#*0L{)!Nlg$16qNBNVy$@MLhOfkDzBzO%@DyL6*rGeA3TdoqPpfZdD^DO^5#VT=L z0b~>FeD=A1WX=c7;NUa(L_{rX{~u@JnNIqFutJ(}?1ji_;Cx079Q)%yIulzC5b4Vm z^Txpkq!}Q;|8c~Ynb#bID#Y3j{s*C8E@iOyQPLmhTn!~qz>hr_wq5G*2Aiq1bNBx{ zi=5zwH4`S4gmrKNmGLo^5^vX9`qata1Oi4&8~JKyywJAt`39EP0?h1lrJujq6owJ} z02%(yLmknzd8CB70jhvh=^<0^+S_D!3Hewia*yDD44V zDC0G~*w#`xXGh%I$Ns3f?ijzdEn%HxkaO=}hrn>Y;w)_2-#f%cX+O-wYz29$NWEij z+355lik|1c^kldGp6;ESwcdviWku$or@{;EDHv&M-9Snf#l64YWuZQ`{D?~E?8~Gv zO0iIX*9((2gsZA#U)&cxE0cS13K_K(NWDCR;7e#*_Pl4KBR~Dh%p2(|b46q%X6@t=ird(m!XE|bJcHaB3@UD|zu|1_ z*>9KPcK(mDt~?yd?|o; z(_4;Auwhw($h#p&~MUKs(AL5BBN%r5=NKK)p+?J2{)RkQ5m?Q3&T zOmO-E0wY!LNQ_C=n}*3w8yZ;P^(iX2_qP3_rBQFjETJVXOB}OPpSj*Q9IjwQ=h+C( zksH;{R!sLQsrfqjCL!(=B=&cn6$(fIx<*>cARG6l|J4q;;;xs>0{<}gO=TX($nLUI zEL{S&Oe^pmJWFo<+n8dFNtRSArLW~&#=o^_-(wFIu?yQ#M+ptsYRu9^K(IQV;*{w9Ar5|Sr#YDthYMb7Q`g{bScYkFx`?v> z=;C-ywxUmrD$1}0)$CGu4m!mOo~r6mq<`qQoJL&SG`*c?3Q~=Icl0CbTFMIs_r)Iv zU{O9lqrjQJQ9uw5+H5n^t>>_y1gP$tvY161vbB+J%=Myt1kYbTq4vTOt< z6l$yHsk7cFbV?jN?0$znhym|dEI^-6>p!vZRDo^pW)XL8@yAe@SNAWs#PqiwCohFS z9$opex3j94#^<=FNtN3Y1umF$^!yQ;%C6zctp^e1Whp$Bwn(F4=ohdpf2J^G@Y;o>|*Xd%!P=-rr zro-AaG!kH39|Y6VQ`=Ajv@b?J- z=wBEs%H|x)wJaHg$-}hQ{#YdGNO^7tP#ZZw|3$q2hk0UoE^#|&t8ZIU8s_+^T%}f( zj`uJYj$`F&nU; z#@Ulv)J`~jvga9W$-mOUgO^#+hs^kamwFL6qiM@qHP~wPF-M0t$FL}ZnU-o~etJ8k z*@A-QG!tj-Z^nN)3oM?OD*Ph~EGTkyzPGZXA((j1UcSw0EQi$>?fPtYij4J1o6H=V zjizIrZt>eZR=s?e;d-YQlm;w3fvjS`Wmyi;%5$7A{;0Z`zxAwf>$2v}wXBkNZ#aew zV3WPRSt9wWU)yzvhDDYVZ#W?Q2WkHA!sjjqI$#DXBuB4oCA$#s?an#0X-hAv)TBiW zmT%ytF;4H><8KD5OSlP@E1L?M&r%<5df>0ixQq2SG`&`LNm&Qb9C9VuEIOyLBz2^X zweh>#(v=3H-GeKb=CRi;@&(#8e=PG~UwH!<^rn+d^DUJ@&7?bQ8!1{6I*TpYM&)ID zoi-%-dN^4j%Skh8j2$%b%}VMmqU!TOAm=3pJR#+Yj5Nbfz$PnjgbAwC4BvPk9Tqk5 zbX%hdoFH!MS46S7nJbjRUw6SVs4PTrt25iyWhvBxoh4in=rTd6=|LXTU&no;^<=f% zP4(t@{iB$DLf|AHffBa&D8`%;^ICYPm3010>05;1f|p#uQp{GoRrpdtRMcTLj(YJ_ z&=|zw=D#36`21RD4j6oZYLRA^;SS=ij~8c=HwbgJGfOFb+-{ADnIgBhn{cp3!Wx*c ziUaqw{h(8FG@c+EQT^|rLV;0MM(fsA>e2nPktiull)uLD+4lq?vdI^vs-o5qW?2zD zo89Rb^D#;#R;AcMtT`(^?vd7@HkEr)g4f$s80DE%kB=lb4^mEk```Q-xgho2$%x$2 z#aWBpb&Ek#zfM%W4?Z5^;HXa<4D|f$%z#le*S<@;Mg^^-)9DT`83o1Y5g8S`3`K+^ z>4D?q!q>VaFlmUaT~U-_blX>)IfNdUr}Y3F2dhT>;EbFio)OJ&3Xj#(k@hhMG5}kd z1j`%BmpYCVm=KZ+HNglKu!9l06w@QKVxF;B1g6e3qJ^R~q)9gLR$pc#AZ8fFl8==4 z0Y)v!BVseSxy0&_S>hngEwq;jRH~p?qLSEClJi;!NF5+;CjODvkZA>6kkry+6YxD3 zRc~n8i#ZrEAJA%EbR3`LE*vlQy8KsT0hcClMX_YbKXO;LG=D$tN7OvYI@`m#d-lv` zEgi52a!B5DfE@D3KIe&?qWKK~Xtk03m&>6dtEQ~^ zaacEc@plG`3J-(L49_I+u3{n!^JjL)M6 zP3QjBO9s~l0F(JOk8sao$RssRKA3q`@DcjNfI83WlQL&QjRWXRO8JeOIkmV=tr~~d%)7Qk#p;$%*rU> zukR3X)m;nCcK)0C@-A$-E7s>hh*Bgj#U-1bFR~1p$E2a5=6C1;o6b8oD2uN3lM%7O z8p%>#(I3Kah44-9i?@7Ci68PZb2*T@z&_9NvUgSxt-(eeIV?NepE>>Mx&{~lJL=5b zf0(>{T>Q>>d=#HU-vYXa;cE3@NS)HiOLHfjM`#duNeV*>&AS9G6-crrvS_e$yPp(z zaAEntjW87uDoOfPJT7}8q7{{8lMn3~P6lltgXyxJ=p1kLE6*vdt#`vuORnSKnDm|b!=+JBN^31vi=Tp;__xZvu$S!;GJFu> zQ*2o>7CNST6(WhCwKyEmxXSXf4MM2uDs+Jyh<^i_VwCcaTv=S%WOryyF%Hi@tMO0s zPFxU))N07-pynnjugD?941~#)jj^AHSPEFuWqlbyHp2w^(3^foX#YBvk`lClqu@{l zX^u!#3lbc;YR|&3>|m=ppdqTGR7hKZ+kWscEi{9dc0LrV!roE~)*!ay4~l9{kZita z5m~_q-V=z3++hg3}z##Db!7#zr`sp^Rtihnz41; zB0|{7)r-R>?k9%oj=9wH&n(uc??0(2t@FxJ0}I{U-7(hU>p9CeommP6nJ_5cpY3q` zk9J-lQdU29*hy`^)m&XO+qG!ruGmanuvu;yIgE(cGY}4dqki`H#^3&W({2WGOiM(r z##CE0N5ndknLX3kTwa;DS-d(YkOHR`-r(Tor?wRhq$)11*&tQ%S>ExqwYTYSy|Oz^ z+=B1W5=?19zMcD&>2GQer~hi=-n>3FBkTH>V+gDr@V-8dyOBpn8#>*NNuI5Ri^ZIO z0}&&z8&x4W>18BNdd_~lgxArHxXScQi{Y28yXQQERbn)8M9Rw=JgiW2YF2(Nq8}DQRXdssW66#VP#)01z zac)Qb+Xy!6adDFUQN^X36E5TAl}|Ofv&Q8GevuQp5_w(wvwbmVk<-GW7_jpK9L0ht z{(PK1%u5OtDbA=XRA60GmUm;u!^_6TIULVX7L)m~TIZaW76(WTEWIn|26Cw^V98c+ zKzoTQ)*%JNI)LAOIrSy>$MGts*r%P>RQ zzc`lKnAT~6h20l@a8EU2&dzrLT?8RqP(@(q=9k9NLw z4Zr*Fwe<7)yhPL7Dilv*xc)E`!%i*l}CxfNEqIF_$J(g%s1B>_A*w|wya z&y(vgmlWNEbb{Qzy*H-ilfUNbPH?X;znZ^)A78?efF23f98le#7Y5O6w+GZnt1aEq z;RCSPt#%}1Q+^CIZFx8^h}QA0VG)++MV?n}8d@&%c4SQ@=^xYZ9W;}2 zD=u~pRNS|l5tqxJ%p|{q z+&d2ZzU|0_n2tbaS7F~MGiTFr<6ftbD$BqYJ|{T}K%i+OTQu()HQrB%==|iINdJ=0 zHPg)ckQakp#XxVpH^NqM!NHN-;A5+q&F;X}j0kIV2`BZqns<~L98|<8Tz_;zV}&qj z`NhIZb#lv-Dd1Z%D=oE7mWuCGs(mN0Jej{%mfbs)8on8ay&umxPLsxngEQ{f0>WV9 zcw7*zPSWsqx56{qOq@p@l-#X_H>1hVPg)<^R3sNi@2Q^P>q4zs1)aGcuxc6iG+jkQ z8t-@>*%I7UM2FV6qC)SZ;bS(@6k>o=IZTm0QPJJi;J;WrUChnt5Y*Hd=&-hTJ#I{v z+pL;glb~L=gkJH~)&*Yk>so~Xx07)vD&^f!cv0F1^t(~0n`pdylVDeE zz91IcS!)i&EC}jNZJd7R0rhNIgU_Kt#~Bc?W=N5CjhWVRW|EjBo8AYJS@JJb6=fR# zH9a%ipqxF!S_Gg@eld*|Kw5~$FREqt{g9l|?vnx!WKY$8Pg#L1_xq^=kGWRY9krD( z=a55Zk+N#pkYCET)a@xX`p^xaZzVA{CB+Y6ycw5 zMn?>xH^WpUxy`8JLE~jM$XEb-R{Qs`!@=KrHD6z1pc(b?!;Lk?8I{}8B4_bI&n-)gn|8v< zG;E~TOqMper_`ZA(dZrktMyT57}*{rQgN1W3ef;CBDFpWaDM>k6!u*Uk!MrUc8OaD z7;E=?0=`#Ow|4StFCG?P{iWbe^)@j8a+A()xT2R%{34HD@VN9x=BXsT(pp3aPus;I zQB$ZXq>1A9HsLU1n;wDTc|Oh&_}(a#YH))mJAX6-a+MXe0Rw2b3GZcrwA|2}88&1%zS_xfz|(%iJ7FkLe#MOYa8MZd$Y4 z`^Hw;a12)Jq|art*Yv<#r~qXIyTu$((V&vsTMcBsY!iE;KT~#qU{B18&%A$UkuJw~ z80MlU(w4kCCp@{DECcA3nLTCkGeVXcp32 z?u2-GE2!c~dE|vmN>Uk+o5I!B0`>+RjeXRomSxCA&<@@wPt?CL@wBX}P8xon`d0q0 zLqMlQr_Lc84_(Q+j?0{{~XX|EoNH1@0+9kj-A!IY#?8tN-QyKq6TNBWiV93I_eYPorlNXA~ndECs&m Ob-+N+RJTIgE#W`(X-6Oc literal 0 HcmV?d00001 diff --git a/build/storage/recipes/system_configuration.svg b/build/storage/recipes/system_configuration.svg deleted file mode 100644 index 75aa28ae..00000000 --- a/build/storage/recipes/system_configuration.svg +++ /dev/null @@ -1,425 +0,0 @@ - - - - - - - - - - - - - - Page-1 - - - - - Round Corner Rectangle.37 - Platform 1 - - - - - - - - - - - - - - - - - - - - - - Platform 1 - - Round Corner Rectangle.38 - Platform 2 - - - - - - - - - - - - - - - - - - - - - - Platform 2 - - Round Corner Rectangle.3 - QEMU - - - - - - - - - - - - - - - - - - - - - - QEMU - - Round Corner Rectangle.5 - proxy-container - - - - - - - - - - - - - - - - - - - - - - proxy-container - - Round Corner Rectangle.6 - storage-target - - - - - - - - - - - - - - - - - - - - - - storage-target - - Sheet.7 - vhost/virtio-blk - - - - vhost/virtio-blk - - Sheet.8 - qemu monitor - - - - qemu monitor - - Dynamic connector.22 - - - - Sheet.15 - NVMe/TCP - - - - NVMe/TCP - - Round Corner Rectangle.24 - - - - - - - - - - - - - - - - - - - - - - Sheet.17 - - - - Sheet.18 - containers - - - - containers - - Sheet.19 - shared objects/ docker volumes - - - - shared objects/ docker volumes - - Sheet.22 - Hot-plug service - - - - Hot-plugservice - - Sheet.23 - SPDK rpc - - - - SPDKrpc - - Sheet.24 - SPDK rpc - - - - SPDKrpc - - Dynamic connector.1009 - - - - Sheet.35 - - - - Sheet.36 - services - - - - services - - Dynamic connector.1005 - - - - Dynamic connector.40 - - - - Dynamic connector.41 - - - - Dynamic connector.42 - - - - Round Corner Rectangle.43 - host-target - - - - - - - - - - - - - - - - - - - - - - host-target - - Round Corner Rectangle - fio - - - - - - - - - - - - - - - - - - - - - - fio - - Dynamic connector.21 - - - - Sheet.44 - fio service - - - - fio service - - Dynamic connector.45 - - - - - - - - User.9 - - Sheet.58 - - - - - - - - - - Sheet.59 - - - - Sheet.60 - - - - - - - Sheet.61 - - - - - - - - Sheet.62 - user - - - - user - - diff --git a/build/storage/tests/it/README.md b/build/storage/tests/it/README.md index 1b428f99..5eb1a0d0 100644 --- a/build/storage/tests/it/README.md +++ b/build/storage/tests/it/README.md @@ -14,7 +14,7 @@ All containers are run in the isolated docker compose environment described by `docker-compose` files. It allows to configure the environment to run tests in a flexible way. The figure below illustrates the overall testing design: -![Running virtio-blk traffic over NVMeTCP](img_virtio_blk_over_nvmetcp_1.svg "Running virtio-blk traffic over NVMeTCP") +![Running virtio-blk traffic over NVMeTCP](img_virtio_blk_over_nvmetcp_1.png "Running virtio-blk traffic over NVMeTCP") # Test environment preparation Please make sure that steps described in diff --git a/build/storage/tests/it/img_virtio_blk_over_nvmetcp_1.png b/build/storage/tests/it/img_virtio_blk_over_nvmetcp_1.png new file mode 100644 index 0000000000000000000000000000000000000000..883686642685b22dbccd980a666172d8fdbdd0f2 GIT binary patch literal 30281 zcmd43cT|(x^EV2Lh*AXsm5u@yq=f{~&_XBl5(pqjNkSl@g%%VLQ9x0OAV@I?h)S2< z!A_(Fl_DUbQWX_Vq_?}H=e+0K-}n38^{)HRy=w*X@MLH2*)y|eK2sj9SXmfx?>)4a zg@uLN#8}^kg=Ghgg@rYQgB{!v?%u5d-dIUCMhKRQKGDxCEPM9`8#o3N!?Ab*hD8RV z_vcDRMakQr6f6VLmr+qMAYeR$gZ%x#CAd%Y_rhcG7_UE{sVJ$aoK%9IRDs$lE6G4~ zl_9@>DX6Hzoc@e=$6@^bnh2_(1SUA@fcNwA4RAix#)0H*rqG90FQ z4BXKT2q0h_Fdj&Ju#Ac!6r!d8fq~mck;ZoBW-=;z;F^H<#eg>>jHjXX}o255oUzIp(f0 z!S1*}J0=<6l#zZ0L_K3ql)4EKW4VdqfQ3w zLs%g!O#;=-l+ZX0WffB&3q&~7R2QN~(69_Pu?#b|g@Q4v_OL+kriZ}j;XI8@10hDL z`W`q3V94!y$Sx_U;$O1 zFcXpwCdeq1On^Dqc{#c3SsLrAlkh60VHQfdc6z?vX2#0yYJq5fLrVl91Zn4^h6CSI zC;F+QZEOtHBCPdQl{J`>4_3GJG{PE(plvW=;c6jPeo&aPjZctSIL^<}lLQ24Xcf%d zwn3;rUd_i6ujip}W{4)*gdre?`Uak6x|r}l8?>Jf!q3#(!xRkkw(<0ZsRtlb{q4}Y z78)d7^9UOcOLsL@FVE0moH|J_7#9w;1p_fwMoQLBP+#VT^sEfo$7y(19k$w>POV^yK1c4e5eGTw#iN@LltLkGyREb{xNWfkuX^=ow z!w6MtYkP2Q47IXTH6kI92A29LcLYvX8DRu|{74ZNW?|MgP%B+OQ`0~Ll8G+K))Hgr zW#XX{;7{-X76V~VHn6rup}{0*bF_*&$sgt(<`ig)H1!Jb)-a0zLk#im1S5Mhrw}iQ z88N`j*viP;5UUpIMfQ)dL0Xsud)vB`&`^kyb%+Pf1cnMH`-Gv8WTs*SfcZ%dcpvZ< zVvkm`b5J%w8I#Pc2{sUYf88Jh@PVPNhlPc&sSQkDPd(5PY77hW$0AkL!l7t&V?Ux< zC>E*?zHJd4r01l9wZx!MhFDcqa}zz7F<#Fr(89xnM8aSQ0sf{;qhfC91k}gS+%(wG z)4+xt?qm=Y<`iZT?Bj?>IT38aJv~i{WG5WVL&Lzp(b&k}M^_bL8g8v(tRCPHs_d@^ zTRTqSF<0LV}h}PLhbZ?43LOWG9uW-lceXUN_M~y>;gi~ z3}N0dJzcQZa4)>Ir(43w4CJuCylB5c&09BrWfe!hWF zDACc^SItt@NeziWGDTz^;1vM#bTs!5&~PW&la17g_RMW7n;1J-lhhpyVERUZ-tKMYHS@L#B3h|nh(2UT4F{+vCQyYG zV2%q%5X|(I4E?eCz`r^YjRJsA2NoL>WI#e92wo~0Xd|=-!GWyn@8oM~^9aHlLiF$&K$Vn{4q;3ap=#n}X>4Q( z!4s`f8X;&6w7QdpzK@=@fsdh{s#7?L3?=(}VFAknF$Pw|a9t~?r5DDS7)J2d4fcn5 z*lSo>5&ZS6)WWUEWI{lgi89^`sp4c$RzihBf>fNC=q4;kSwGa>$KJ@^5Q#CiMF(K< zhW4TMVHRj5awtiOgdqooXc!=U)IBZKd?Qpsj01gwos`UiP`*fCsD))5hCXERrlH`nq=E zuHV}BYVTxu+Tfm~_L?rPBjfp?d=!lxWMwy>xj;Uv`57+A*Sm52}*G+tbG6LrG(HyS?N%&f6$lv_bO;v z7II%ua24#Iku0o*Y?3s?zyip>?#0LpT!7;=njJ;{S?t0E>m4b$LiJY%|8-9%V~-A{ zM)XV>_rK{q%1W)V&Yk08qwAh!-}g$m3d;F39iI zZn>JUK_jgvnTmA4*YgzXm8hg&DVbj$--XXp!&*y`6=76TP?<76#}Qr0)AWtY@6&Fu z#3}1Y*04o&U|0{{z}X8aC`i)eF}Za!W)Pf|J$c5g-mQMKXLT)0zqWq$N*;7s4in+U zpq)E`w$Q*vOON*F?pyfK4KIBib!eZavHp1ayvFI?jstf4UR}}u)w+<$E?(cS{w^tN z|9o=vs};_D<^=J?rUA8N&AVhGU6_CI6!v*!rc;C0bJ8Ta`)gFys_~Fd9$j&d$~l*W z%II8!-Yjf;#@P1hedDJ3x#af~J)1kFj}K&S__8;Q8S6(cZrUUItT#*Jqm#xGywj8O z-(_}470C25ZbVBR+dnL>Ut2_0TC9}uxwh$o{8odQ#4c^Jaqf)aV#;2<(eV(+CF!$P z6nj>Mbn*0?C?`!`lK$lTqEoS&ayAKOi5K9Ap(K23E~c@!BeT@Z2= zUgQ1JCLIx^2$}Zj*dmndw?Jo0cH+G$65F}a_#d>iUDplrE+=Ge;W)KM%{V8n2rYR9eOtYCZZ&5+m6x8MugO~+m2!=m=2^Sk-D ztiHQ6vt;vjFr*`&T5}vzFr%c@T`;_6;Y1H`PR^EZY!PV8tJAGG%r*Nn_VXiH7{-Fry`>jO;N`ChR@ZCNKi@webl+3}dH47O zl?;`ZmBN|otN30FA6$5KHiBF(&uLXt7WMO(tm9VRdXc%HK*^LIIP!CN?=!ZdAP3=@Eqj!b zdLViAkLyNMCO^|omB5jj_jAZR$7v(R&Jfgs53a&g%Ms)}431G9z zxj1Ea?Ks8<*6zMtpFfndmJGG6Y1H+lY|8AmY5gu+umF>=?|fDyD%9*cgeHv`=JW6J zTi7nrNqUO5tAHw?7@~F*46$}ww%Y?r(-ho?xgw8Zlntc{pBVdat=5TA_99_EvFOKI z=P?w+#Y{GK2wqA*Z^m#FyOr1jC&5?L_3N~^vYgHVbz8zMtvW^b)|9zrs-9B;#hULqZgE!jXBQf)Z ztI-R))QN@tpZT}mzy=6~5u?@R4^_!cj+ zmDe4#cN;#qSiJ6B-|;AHvIs7vx8y(&t&7Zm1DjeMjk>3@q|)xA*_@lVZO6G~Fvl;Q zZK_YYlHmV6E#y}a@5>G@s;$<*+8QFzzB|nBz0Z}sH6L=gF_$xHC~FT%niQdSnD*SS2t0u%$C|HY~CX8-&i43euw@YnPJ-c7r9GGU{!oM|B7STRo z@9ukdS8gXZ-`*szy`nocufLBs#!}FIl`VmLzpad)P^FJKyn0QZEDo8s6unIBZO(g> zl7maoQuq?a8d=h-EM3^koAI_p!)T^%?LAsrKprw7W~ue~oZn2Xb1UpiV=K&+Gf{JS zk^a5y*N4L*25t^#-qX78khX)8lC|1RdW^(s2H-qVKl10&RvRgS!{cvKaA$(NcmKvA zDTjf7AC4`@QkK=k_ZK*Kgje(FL{$WRLO9h2NQZ2_`~2);%WBg%#0BkB!M9$R-Liz* z8rsA!5!<%4zNM#Kx>gOZ@#^+%jI1)NIK{L496_6`ftPx=FZSiNt}f0E8C-kHOQF?^ zwcvlPtp55C$#{0Yk=Kv5`J-`tYlYJ2_o~|deRRz;25IAi@3f+T&uMi1V}|46q}=68 zrMXjZ$4^CHTiWGU5wLoPf}&kcJ^mzL)3LD~MV`swAKaaZGsfR6n9s!34O|)v(%`@q4Qs9?3aPP{ zOp66^p(Cu#1~_|_In|5}$GcNzAF#a=|Ba<2cLU|E*xi&4C*4E$ezWfr1x})2D})}BhQHJ?;eEffyt}d ztzK#T27EdwF!Pz@yR9Zk!;kM1&sTnAUFgBpJ-gT~?sGrsZe22gkH#o`#~pjVhO*zL zU5knrIj>^4T^W5@YvV`Z#rs6602H|E)fxRdQN0Yp{${-OZHeVx-bGnmtw24>59=?r z+*HWa$l~&j9*I93*ta8PR$pn>n=cxsZiEbrDn2?EpmoXd4+7TN0Q~t0>D%J)S;}-M zr3~k84&rc^Q)a(D<3GXr8w)2~*J+<>`t+7km8#z&m!Ok~8RDy_d{2pp8arR(`Rl7fH(b`L$R{@f=pohGEhQIO-j8TjJ(-|@o> zK*ja^o^T`Q2EU^BZRgIF=4y~+dv7T}U0*Ik*E}dG`Eqn`sYB(x(u(G*FOtNNH0^D; z=E{fIfhH1ep&9Bs)Sn`ymryAJ4_6A*-mqGKUxVWv92l@Vb4IMre?n{WZm%LQoj^Gj z0UQyaoO%+V4nI8!kB{<=oWcsfo-Oz1>mlM+tAan<_>4D)=zhMbN1f*thZ|?QMKz^( zMg5?amX_i>lf+aBZvpU5xOJ&%KV39VsAAWjm?h>0;LBUdqCKiG*xur4vw%w1w${W= zZN+rurEXHPxaM5+)@<~H`g#_RwQ0Zp_KC|O$SF6GnTtQ$aFr{I37U9B4=3(W`{3GlOI(T6I zzNWlpYaGgYe#>Ck&A&(6^XGI~aDFzfPnL9@fxmxSwhJD|BW9WdBk~vYg!H>qE1zEp zaBA=qo6>l39Z<4Qp_%;;heQA+@1?~<+hid$dD_O=7;*K0J7EH6_4Ia5X%N8Ly#HRC zMTI6I@Ibv@4hn_3+g_6VP)VGL_8+Ui-WY{0zszG(|Cs)+2q~NmBW52RTlw*nx2>#B ze$1+7Bq+P6dO|AW#u!qH*2JwT#Pi#dpQV5aXR@%BMvJxAdBjcuyv}(3S>#J#i0*V& zQ4Uf)L($E`xt_33O)ALw7)`#b&?s5!i&t`z*y(!pc7=24lg|65G_>mH4*t>iFd&^K z&Ze1*o3rqmji9FIyM$dNlS-Xyy{#XUDCmcBX16{J!fWJfu2nkKb(I#PrnBxld^f}Q z%1qTCoYH8>?|h>BJD}|Wp+yevs<`eW1;FfI^fK)@GJZ| zz$`Im!P1Y_KOW}`$OOK3{H^=UAHzQc46(B?zZnM`Gh7j2zxw^pA*6$Ykvlu*`E6Aa z94VvRD$_qlN1tgHA79heku(5DYX3)1??3m;z&yB<1&+KNE8x&o$^HN1DC-+Zo;?PP zNm0z>+CLPz9086_+2i zRr(+e$Vjw)nyV|x1r|8q)F|{@Cs^1_fid!oy`VD$#@C3um+}756^a0w#&?y8OXoO)xZ+7 zAE7fVN|nZO@7K<)UWtJU3fkRn2${vVnVZ{50vpaz=2~l?nfanLe+Q7QYjU%lhfatqEN|1HK&1tK>9NzxScT1V zfz=L$T+*Ewe&-^F6-x*EJ$j5+%&>Dg+A=3tL-I8I~4Hh85!_GP4B?3$ll&T_aR-t*w`Bv;W<*%Sj80F z^~PPBDKT*0M+O43;FEDK_x@$5nnOH#H*hvQrh5T_={ZaSKMDPRNMMx-1?~NV3S)Kd z+aG34>69lQ+FdR1>Wo_S@i9gIvI==1CbWKx!Xph@jFQH=ePI8=62x~MH~Ps zDYM=!M@7n|XxL92WZ2VJL#}|QseoTPdTEY;Uxgn}%F)K67^0iMJ`Spe%y{?=mW0$k zPIwUYl5lGAmxbzwFCse@?R_>Dli@o49Zi{r$vGC*Ggo>@cdWd&q^UJoe1L?2+B#us zjUiKB{ryK!K$70!#LQWnQrl^by}PKS{ShO0PBKgY`M;sGn~_i&@%mlK&){jO-h)oE z@Kqj6+)BDFjW25!MocO?Cn98>*Dn0(Q=I)k{h4zM&8LM261H9KqB}CJXUu-wVRaMK z&LmRCYhz@uiH8o<#q&xWKdpspcYw*?b6EabASdB$)mV4)aSWg-VHP5cLwb^jnW8I>O$yEJ? zD6$;)A=r~{A>p)H$;tJi#uRvR-l-fR+HKbWSC`R`iZUPFI8w+-_3_oO577y-H&SpA zQR25<^s;@-!>_7(7 z+9%HVY)I1OeG5~;bc*!1%;~=TZ>F=SOq)mf+-ppS3zw4phTAnyI;*FZ%$^ib6JjqW z&)BDnXg`1#&S;EMsJ^~pARjg@lI_9aba`f&fuMMD#2K7HP@orC|0Vg-E=Z=uH3?{X zMI+WY(H1g5qHK0P6+At7u`*50_koH+bOohH%E8L-d&3suWS%jEFjaxtnGLGU)D)?c zYx4D>?HQdd>)%_Fo4ux}skMJ#C$u*knpjd&@(jAUzAOir5_wKJbH1V_DfH{xk9C=} zu(|$xM7)Ly?bN_KJW%=s!GGJ3sjPsfXKPQUXAN0Tp#|*$Dym4r@MDZs6ae{(DaM*5f75|E%peYj(^Cwd=8A0#fzCBuznuYDNs(H&Js%}L+fxFl=zFXqFj^~SO z`}oVS@q=Z`%Y*&>Is&q0d?T1+T_x~t`36xp!^G(KfTb7d^3&0)^yyB%SNRY38O~_P zD>qcc!+oD|0;k4p?jduBY>6x*1r;-tk(7H~B5RGRm`ResNL=7ui_ilG6b({~*Gv9v z2#)95`!?rv;zKq@Eaat)_h8wtdg9o#5Y32RcPyjmX^0XHk&T9q&Fd=~es2JHm3k%r zL4J6nwa1I(1^_f&zcmHUSj^m)4j5_ zqqW8Zo^l7%G?zKZGhYL$vc39Fkv_+F=O5LU_ZcD3l(y^LZI_)Y7C*R>cUC`Yw z(_ZzX%Qwy{nW1p4{y|E)b^ShmM?=NrP#tkoTanQW)Crb2~U+ggr0f z%Q05UWhbq@7>a%Ii~xM)qjpK38-ok*lal1gh5eer91CJ-&$3Uc=zify@ysj;d6j^dD>Zz(hcc)Dwr~xVQtt;5)X}Bfg&hRA&38E zi{Ia!YKr^F8N1@+8Yn)Bj-k>c{cmFQ0E455dDUTTW|sEFSzRio2H{HH|_0U56b_>Bj%=*!5+vkq&s>YLOVZd*s}DkttXg&glPjb({By|)1P_?p7YV$ zmx=zU0NLGnKA`^|-5X}b7=OLcwoF+Te5lNg$20&G;%;I}X__H_Tf-T^np}EoTI0Gk z*j*y^YIm`FbCY>kUWUCaTV1xf4>|q%Lrms=;`zD$3H;ae=c>W|n1T6=ejj(6H13)j z;rdZKMW&Vd+w`_h+ckKrfE2YlBvH8^d4u6|etYL3h`ZLH*+G-r+F{KxBNF4gfCW7P zew@tA_ln(+cw>bM<0K~uQSMGHyWZGuN*ZOl{sX{8l7AVWs0R0Qsr=4t>7x zJd3vc?xFk6kolqO-y0`+)?Ot?O<^e0Sq1R7)YQYyxoSRMJA^`Z01~bpnbH{Ln)DY7 zn<39LPJ0iI%FVsCtjoU~_TF-{V@c4dCI}|>ruindM||UhUc`^o>$R9k(=V^e6IVv) z67b_fecF>{<;P2X48z`3x)M@ZU2DG27M#hyri3eE=gqSaiCXpb=gG~V==>6*GsMSJ@txaUiktrgqx!?a(e0*O?~0 z^?Keb+ zVg06BYgPQttN2sJAwbG;;&jpN%)7?Q=>;#Pdy@Cu)VnMi z3SVXW8AIib@|%#D5RjM~3;lZU)#NOvJ7!-wr6o6xx8n%tV#6us57F^y{Xb8gtu=S=aL z$(lX7d@8v&(l5$y-MTU?DrxMc?{qjkBia=n-6Y=5gAsYPtgJ%Q_eygy;QidGxj$`o zyd^Acqsw7y6xD}dsJZ@hXrd2yV!yfDo`3GLMYlmegGCzaQPNUz_6pPSp9@AB-jwZH zw#nmc;mrTVS+UV@o-$S?>R^}RF#I_){^qi_VV$j!O)+mko%q1E9A?QZrY)%<~VoBFVc|ZUN)MumRdsumw9u0Dmw$aa=E1rnCVSydW1!!*bl|-=dS*6qiw}jn|FiD*A?=#chU)Gm-Q&5 zqqL2>a;B>`g|3vI!Jq&%{a|lj!09lp;d>B*s7-HL2d^2%}#z9C_DkQdwJ7(LXh$l>Dm1WlxbDU+AB z_n?zLXX5O!8+3W3)>La_Ee5m6^|RZ+MokpfO(LvIDvBkGzq{#k@7f)n`-@kJFIzLG zw=#;%OYR2YFEvjfzs-BBhkdZ0@J^LkT9)d*HYMF;_sOW+_w)C25_HqFl}dBXmb}4F zhK3>E-=Q*&Rl!Ru)u8idj44eUrjb;-pckdk|1_1vGkq{?c1}@rT-+rcH(rM=KGPqh zIu2dmWxqA17(cB-d-uIcE=uO!<7ts<)xx*5!BwH|sGHsJ-Kn@mOInFnd@4>sDz)fD z(0h}-p_rbU?>zn^3g3HbPGujj-5{-UW;;ERnqQlC(^}EkZbxm?_o*_nD8*~@5l)%xrEKT54^8u6}=}9!Y4`NTAx?f{Qq-fQljzdQD68olfwI_n5*ZvOS+wVF|zaGY6lR$Ma0zIH~V3?3ak_wd24=|#4ixVo9FXsHw~ zuMxv1S+hs{zgR{yI+bD@r_r19iGIB#Wio2n|5i3GO0#_JXPWDHbd{US;I+n9_gj@W zm(?FIKovpW-=)kAo@G_I;fA*?wQqqagQ#!?OU{_z(z|R!+o9u3Qf1Z1DL)h8mrW<)Q4jwz~)>S~M|EPVP zjZWyCKQbr(DFe62#z{$&A4|~|MvvaCdCq<5xkak=a^1E*S5q|FyU2`m<=GAL4dcN)H{^YewiHFDkEWxoz)_T={6U%W^D5?Qr*U z@~qp2f1u}Ki!qL7UuW0dh7aJ8nad2_#yT+ItQhk9D%Y`G%HF(lv&D4Wr{t}?z(A*Kd9Ui@I3 zpRC+&XMybF9Oj(T5LFBe`8%uhiWzV{i8I_Ab7#T#LIx4!jvzmKuL1*@+Q!U%D08n# zF+=-AStmA0n$z2U4*E6C4~JREuo@YVd{Q4BWhS3g{=sQw;iOmO{^dHCN7SFvz$0Je z{U?tCl4P+x%v{#rt5V^fypjyjZt)B7&WnTDz=I#kzwlSGKpDV0ODpPd=iJyyb zoJ{tr)9$GevsOUPeGAj?)6svT1?!{@iF7zG&2!FnOM7);sYvf==tXvKMldutiXHzJv9?;=9`qURZG*J zo5~IlAhGh9WGwl6L}=LP_H38|dk$n2In#URMCCz5X_OT&S>d5bY^-@PVoniw}|@XVL$Hohg6o zf64z}LgIQE-3Lij%*ze*e<~g5as~twoI~(##vNW&*wt)Y#`M#8Eq>f-x4>RLYop#z zC#SQLAD#%iSL@d=No$_!SNLr{KrQ4y72pW1>Z0ib5|xCwqkJupNrLhJ+tUb=wI|;9 z-tmUwTOO$o{5zxRH|9W_xA`#K%og&6usOo=&KTL&`q*Z<5 z9w*npNiFwgwu`rWwNEU$q|9&WrcOs6DcIK2-_Bdu*4zI2I$HBvw(?DZxf}kXtA>Vf zIw-XgWWR%1|1&*@YRy0GpMlG8>EmL=ZoAEPY!dgr44PXN^x*@!c#mDAp?pJSr2voTcY`F_^Jq(6zwv4SEL)K z;UR}6X1zo<=${Q`-H}kK$55adS7mwjdHdX0&M!Lsc|krh?zsP)fDiv768lCukD%vF z88ObudvDCl4%=ET=nEL~9$uZ6qo1f<9c5;lw}b~L6x%oSqek{bJKs5U{>Y!>|37cb>hz535Us$LC-6B?8w36X5>G1YS#| zmsOdmva+7s(M)gs_I#F^12_e2=}+#9n~E)_RK-Mj?UfD<(#G4TrOqx6kjkXKQ}g2w)%Ie00j{ z#3B;MsqDsh+M?+`yhn5j9e-%_EvQZsWcU8}6>MD0D4Q31bc!6iXDeE<9E<7;Pju)# zM7Id{#LP1@X1{8)cm9sAKXtb3v2HJE zNIE-j@ebjsS=s+Q|2IUy@sI{B0rlr*g!Tf_+x#Q?KZTm9IF_k{^xT|Y%46W}df5d3 zrvY!0dPKjr+j_jQT~HzL*6Iq!>Gp!xGTsMo`wmW@$Q^Bg#eLkZ-SYEuSoYTH1R>Y? z2I*AkC#BnqyzyzGDpgh4*$?XHG4o01$BC`H;)U4>K56IOBVzeeQwNKe~{!DauOlaFCg19Jn;#y;7;Q=!<5jx?b^co9m4yY^i?*Dr`2%9rsc*xa;%%eznRi><3sp z@AmB(hu-_F!!8!%w0ZHy+kKbsO=S&}hcBM;-&;CdNuT*1ouv3>utE5w_bp4}SX~i< z7L9v-pc(V-WI@NU+qu{oiyWlwtXUC6`&`^|&CkOx-WX&?K5enUZN3AQOJoJ+bQ=BZ zmneY3Mv%Ss0w6yXCm!ix;$J^_89e{i8Bty{pam>5c{3X{P5n3eJZ&iP zTn9{Dl73t!_3>i_RUlo6?u*?Rj@|FI!|jIZ97ouZ)4&qxCOXK=+V90|yVP5oQpoQn z+8>&GosaGy3BO#ud~7k{I1R47ioEFL^Kn}yW*hX>b-%1z= z>puQ_T=(96zXZ=by#Fg%fc|w2$Yl4}3DyckFNwvtk&P`EfXPtEd3Vr6;#X}k>=t@w zZ?zj2613djtfq+=jZY22h0}10hcYnZr|<&w9uhqK!yPIIiT$=jiLr1AUR<4U$6?0$ zDCAovPGRQ=gD(CmEMGnH^y1z?&yy}v zK7G%zDxVRARMq#jBE_rMR5{PiR@^qICqMPHtZU3tx(4TM##<%QFV{|^r@wB$IVYu| zkeQXC(l>oDmy8Xa5DyVBNvfudKI@Bx=iKqGm>JO)$z1v`S|I>ZHX`rMyeLIp8C`JU zX4&2z2=e=FM#d|m?eXU?57Mo_#OXwv?R^xcJs0_I=n}m9 zEq%ND5c#i-blrxFJQb=gTzI$cP|})jNkxq`pC8$o^SL5N`)0WTh5%iO({0(|V@jFc z-4361EPr`Z?Tz4R;-tUC*7f?WFB;Z;Qk6e5&r-A9wM3ehI~EI)x0X`*I2UWV74V?d zr{l7g$f#X!O-BX#j6WY4MN7a^Kw&KdG5B*JDWCR!X#oK|$Z*jy@=^z7WuoG%9Qd;h zi1bBOu+eX;jL13pL@8yKZDZGTzaBZ~SSCK;B z7NzGt^LrS^MXfngW#UXB-ihgfv{S|gv@U_pNAqto)EeCzMa@^GY4Z}~3$79FoaVQi zZaLNYeB&!p5C{-JKLaj`0gQkZhzdv-~(Xt z7yuhrBm7zl4n@HLKEk0M>;k@!fR&SC5K=rpR}@f5RX6rP$9f7gyWusz%MahS87Vz3Hr z@?KAYVA#pKzPl&%Ewz?X8u$G8su&3zJyqdhG#}Wza~oBEr(YWKfIu4XovuWU26BS{f-__NuENO3zdPhCz z>|E{mHB~9GaXD)1t*sq2n*Cc8IeB?n{GHp=ClT3Jnlaze&F|I=1VVCSp0Y;_Glp$> z4+czem@UqpdYt4;!NAT`Z0_c*ynBDh|4vQc){f1U_PUepqH(isgx*uWoTKc}-|a{W zd&u>X+8=XqkaNc%pmEE{Ozb+E9&DTLf8bXhqvun%9PZuNLt{~g;fl!hi%=RJKDYDw z^@5o!%<7N)g$`cc&oBUkd60#I@YP-c)#mGyUsE5?{ zhtrn}+ZEQ6*SC(coqN&x`ACP~`m;0(zTuw@lf79g@rLH2C-3HTT>f!eZ6Y5yBz;Ed z3@2HOLBiEH8b;-t4l72c{tZXcPE+J7GB>}lDIFtckkU7n+S)A7pInVu=8c%ghn*k0 z=({yDI5B{B`T$Free}T8_10bZSaTjgBTM{KBL_NPGi}kOdk5!%HFNw~z~JInlA`e^ zraw{4n~1&p)#T&m8fJp;#)&-J%A2$*CFIMQ?U~NCcnR$a1<* zCt~-^c~LeOyPo?{cqL99seY3yWs8J(&7iNU;7S}S-9qe8^~Cj1*ON-a2BgpoiQM80 zpj@*6DYg}MoxaF^@+5g?)tsV{)d+K`h>xCe6WQ(=P~7|c7RVI_H>9^Kn3L-<+Q9H( z6~V7-hgE3PRtQS>z9$UHv$BjbO6W($EGx^g4>!1@4bfc?+9jltdP@&R!X?Oh4*Lvo z-zWW$HsRWv8@Tq5^A)>7Jeuj5ySMz)IE!{u`Ib(R!qHxJv9I1pZH z)mPY;4(%G2O)hPrXG;xCwsIb>%sUO5O&=#%1nKMCCmi+-m!z%S*l&C85dE5>iEX7xIlx|$Qe^c0JZHkE$O=q!VC&kuu5exF_or28*d6I0(U#TJ%k-=Uz zps}K7f{&W46{%S8oV)Zve`d~)%=-aDw<;^0RT7uBSI?}y&Q!dEWOqSoyKSeeJlP-h zLv3$GTlD1>|JCwX`YaVLS9My6onyc3BYLiE&V7lM@&oM)+-?@6L&ZsvD}~$U9b2Cl zY;aZ~t9#EswtIH_5#8k{ubo&`j&biQ^yboh@#k$DW{l&w^`jrOHV)y0TLG+?sr*v$ zS2Gw$n%)ELagwr+C-ysbc-5~g0O!u>p5a#9JE8C(s;%CszN-3Tf(}V!75bsD)8am} zX$-hRQTU5Sel`_jov24MtFcS!(h+G1Pb6$^Kjw~+F_3-K!8a+KPd#9}09sb{IY8j6 z5oUj0W2`LW%nv9C!p|N20-mt)5uuX!TXuL}xUdKiDtP8$zMkQE?yJ>1_7$wHe+I2~ck;Aaozn3?xG$E#w*x1fUEQjs zGS7iJ|69QWJK4&>&g0#U5LcE)h8!;7B|9!X5kqOZ<~iK=?fpx!2J=&N z4Km11&w;Lz%OdQtmxUZ+em}4)MMJ-zz?X5;bJ;@}o**8FXf_+A0XTGi^1x2F8~nTN zf9=`9YzWk&c#fad_zIBH^*f+mnQ(aO;DpkxXIf7WNS?jLjOYpB$mw=aE`!yBF1IV{ z2#}~S`3kwlx+4w*#!|Gn2D|Sqv2dIxCtz>F;cgQD-|m(U`UDz21>CP{nQWQb2o1xdZ5@NJ&atx5e>1ml zI4L>0E@gJ*pq%oKDUCf{Le~JBBsp&F;7|w6%z*oZZYS+C)S5-9y{fuYlBC&pH#Typ zcAO(C#}UwP&PBF}pkX56fOu(8V>1-(uGm3$5!$B!9&q>pl5~2Mp9M-tkZ{SsIko9h z-bm;DXtdY=m8}1a|N3s`a~*7;=nx0rJtEgAs}B0z0#GnF;FUE_vCp!TVH^TyVstDx zm`u7suVe%w8R)^hwc{p6!6JvO4@8Q1R8?VP+3GRK})gcF+GIB`LF) zVm#IPs4TyEwsY)li41tCac}&Rmn3a;?wyZ%Y{J%+1jmA(%m+>0jmE<%fATCVLRSMwR zz2hKg$d7~L&WbqKu~BR6KlW2VYDna^GxvRgS8@pnGKYAXt>_#e#l=PDskRI-4*=b@ zAIb>rpjDL=%+<$DC-BKO?}*6-{p_INSUo7<48W@LKot*Y@fNJqysbz1c4ZfK$Bv5M z1I~<;R`h_7+Vrj|4KV+@dd8VuQ%&;HF5saVW)&QL%lH4O$F+E*tlo#72<_UB7- z{AM3j>oN2(*nMnG^{OegruUVyx*CWgZq4mw3;MmUzLzLVLRQM`Gpo*1d{Y{z)}>$` za=o0;tvwu#zXyb3Dw|a-sWm53k84^2&HUsT3VNCTJg#&hnR()_313D7l4{^@zJT*9 zw0fx%7(Xu2r!hM0zm@6W+ZU*ltaO6CdaWEN{~6Aro)+u>GLiPU@?s~m6aV6G?!ZqI1paivJYTQZePEanX6uD# z$dBPQ>;>K67Xq;J&8SglV;H!=*4R84%6;P$08LlG$rX(X;a#5_-M7XfzctA6F*t=D zWCO!I-22#t@V6qg13?+m!_QMbLrW)1V}NVxJfv=f_y$DnsKXE!Ljz8` z&e5gL(e+`~FWwfS>oh>j5{bqqS0M81KWw`u2JrPIU>$3?L>wNbL5RTvMdatq55D&oz-PTLM*K?+9Ma(yVPILC}~b|9|gS}2EA}) zKURYHDH=z?CnYtS6F>|7fF@!kC}O}0_0kRX5~*?tv{0H3dRf}MeZT-s;H5?*)L{Fc zdtE}q4h9Wbp&f@uJP-{g^aN<`)??}yz_64;gZmhITNxVMZ7A(7{k#F%yzZb_Q6b5j z!U)-Pa~)vtjnADs0c?>QSj0s~4vW)3L>PZ?&BG^-wb6kQvw$vCCPOWtX$!pvy6BIf z<)8swfMq@SNa*&_gDwCUtt`&?YX&rWki|1pq2WHDHU5Vih~A@hQB%uq*|%_b62V|< zQg)r$CIQ;qL>Lur;2@GqcG1To)BeaRK#hMk3Z32(e41ahqyP1& zh7d3(L?=`_FEFstV%re$DlCZoepb{zH7Nr>?%3A_<{@WQ|D8NRNt(pX1>KI1njln* zyXbI25+WB-{c1=^KpbN2It;Z=0D*y0^VjC?957{iQ^~j7AU_~`7JAPq9J~kf`F{Ea za0Z6d)nUTIcsikMM#%DGWx;}Z79O9g2dp!QbBg*Ch@Y^I)(7v=f%g<9qi(*@XrM!> z-5(da#w(v%q4wn_>+FB?;HgR)h#jwr9&qBzYmt)xpu?=oVIQ|Z7c;5ZdI!yLfMzh+ zapk-x6Dr{PJ_6qD(|_9E)l2rTm+1wII}NAc3IoLIl0{!NihUx=bq}F zf%46&1wC6z<`u_Tv`^(5*aeYC@H_z;g98fPK}J|PC?zP3wi}JOX_<>Hgzt*9Pwp&b zjdGmit{S$GtC`N9^trb1>_<)YJf=qgXGtYU(_n3;;YG$X%*0ZxJYsvy8 zodvu=?ZF9k=|ff?zpWFd=NVNKJw!pR1aZi(XM!$SBGiB<`(s)~z&ITS zW;6IxnS(+KEyG1%^A@g`CKYI5e#OFEn4x=80rvGU`;oRrC18e#TH@j$oR;%Uk%u<% zPY|iN>x-Q`Mg)c-laU8-+Y;>i51Ksz4ARYs1}AG^ko+Yr1@R{U-f8NI#liebgZZz} zp&@%e17@-+L72SYf<5O3hHAx7yZgN%E&8a>8*u?8iOyb~avK?|NXMJVx?|tY3X=HH zDf|PXWOqHjbC9K=KT-qJC24lXhORDusRC8Sqj#Qdhrw%j+yRNt zr|~^-2~xn+fMWrt0SJmTS@|+)z<3)p|8uFgslkvBYnI*yMP0^6V~tq!~RzB6x#%g%L}J2py3n>nC%Er?#yLc(n1h}i4_%&%=G|>A_g(N z*Bjrk;$$UQ#PeGM7xs@Ce}&otT+Kr^;9EAnzX7=^4=0!dCiNm>1%aSMv$C5gG)JevWPMgr zbVCyX5#*xc@@I+Kg|K!IaHsnQ?P+7C9OrL<+;DQX^MJK5-v5W_EJ#gjUYV%A4DNNr zswPI%<6SRI8)I@T>;&Y_+2|xBs8Rms-N8Xj&a z1M=_U zi-`Y$cwby~cV&Q%+&o$IP}%y2Gz+T3!L+gD2#F8FV=0z}a`6#R4OUaVKL!`B{*Nfh z!?O#;(w&;qN^Yy*oFko$E~W>USK;S&DUFk$#9r5S0sLD$>8XrZK^xHDTSq+ZYx{%s zveVTk#&H>#e^kK)(cEmW(%TILY94pArmwG(r`7K#L1H1o)BHn~xnCRedY;(r57l@! z5z?>lBR0->UH=tL|AXu%I67N?GXk?iXhz4KDHu5XOJ}F`DdP|P>C>n0w6_lSuIxox zwna8ynWIJ-M*DR%L%H&@O>n?R$QDiv0ErVklXPM`Tz|{D!rP&XbhK`!8kTo`Z?dH~ z$BN4+C0C3bYw@;n@#ms*sRh~@u)p#$dPQCoJ!|oB1|TN6nYW6>#XD#2xzgO}%W3!S(UI z(;BK?lX(2@Etcy-~2AAZO?Z-Wj<@*{f^S&!-s-#+wpF? z{b@f^4_mn4B3Y^poIWX$rKZQ!2i|}JuafJe5NH#!zSdef0`@lzW-vaSz##fdpETY?Cn&{*Gq^s?dR=5?2yt^LKb}?!1y8Fd3f3M81 zU+I@_{g3Sz9@Us%nf)UHilE{lAzfIX1?VS-wunV8wFBxTsxNypw!8pH^kE?$a10gH zzdWa7hG~ZE;ph*S&0yg*dy`-*^Q)VbMZ(iXQtN{g6A|K83!j+l*shQEniWFuWu(jf ziOhH3;~r}Q%48a%oG$%`wY19f!MmP~q}ViKyq(Hl=7=={Q=e>Fam%{UJ$xF~8Cp`o zIt&1HOw5AvYTiJ{XtTt;lPxaC-Dz^aFtbJ6{@mcyvPerPrRAU}y z^l8U_cQUS(Jf809(G^gT_`(IXPp|ecPibYBqDiS4xhLr4D`)ZP6t+zF>t&zQkotj& znqN0*;o@1hxA7mD-L#%BK23T-)Z!UAN3zaW)t3T9Pyv-pbIi>nx2CrM0~i}b4%b(k zIlHCSCzC1aY5J#<5lCy;`V=J>%orf4P&2R7(&$t5^e_}|*hq0a0gz*Pu1pZFFoBd5VtP`Kd&yeGz+lHW368-s23Wlu4Q++!tQcfuIG(nC5Bh7#RQza=v#6+tNZKX zE|9|zBgzd5`QzV_fcYKB|}0Vz93ENAd=I(4VvSHr6LAc-C?ZfSrC{}`~Pj4i`;=?5%r zx*H_dFU?Xfn>~Wh(_E?%rCp(WZikwAi4?a8F1U6bwLcsw*u7+<0%tpRy27^>F#084 zL5OYq zr6$QwQvoFj8QMEj@f=7Ds(Iu0FZFsOu8f@@Ip>EJYMZq%9#N0dkqwBPbf13~R-9tg zZflt*C{(8l0u=_cq#(dn5%6lM<+norhG( z{zwF~xuOwE>AB|8!{bD=qK8@x=hU5)^pf#en&FW~Tmwi~`uLjV_~obb6^lQwlQr}6|-pf^+-FYo}G`DU6*UPQ2);Jx6A0038dIm!-zh2XQcD0 zbziuG@Q}fNCRP!VZqXK_ikKP}BMdg>f0oU2$cnp>QS#X($}G5PIWu0h)^hh|EC&^i zv*XPi3KzW}(IysF+fr8R_15=Bd4LxSVUX}l%Px#z1*Y?g5qTd1Y5I)lm!?KNXbwZR zCFT{7o~RwJnNj2`pQ-AEd2*v>(wrq529Ck5oSEZH+AF<2pScHanjYyi{$s2*oe^4Z z!3#e7(jb4D?*^u%x;6{TE<^d6lP#GXwxaz-vr`67qSL&Iaek7P6*yI-e*Yxw?qE~~ zEcualMJiFetZyzZ%4fmjmzpqyTrW<6?~LiECT*L+Qw=WW?NnVTc|fyO-+~^amxvV* zr@|2pm+q&XGTy<_)BkR1?(Z5}!#@Z?Sp`y#`bbmWLlPX6TV7Pe^twGoADr8!YL*}X ziei4^#G=!rH$rxb(@ZBe<^Y)K^!93U@2cHll;K?5bWmo9^*+y--IUB1lL0mtwev?z z?j(zlV+TA%XcJIpW0l4kuS;AaN6j7*sn*W(DNx9Du$X1en$H~D>k+KwkTHGzGS6dk zxIn&}#9VnT-SKyP$o-l(@EuAo9cq8`6LWdpXVaj*`D`Z)V~bp}6LufPvbVoxZTxlu z@9>U%W-a6Qm!e@of<-dl(1}3xo4O_^EUy?GooLSyk81n0tybek^e?ni9~(+M-`GAu6(`e#cD<@UwxW& z-}(Msl+Ek&%9xxCFIf~iHvnFo#ySx~mN{lkQiQ}eMoI;o)PhPF?V2~*v36p+D)Qq9 zMNjVU`fy&5Wec7jv?EK6{h3aeezD3}0m$pHKY2zM3kY#R`|9PI7^SDBa2_oGndOq{(Kj_`{q7IUm5Vk*Y{)k z*V#{P4CcEyp-$X*!?^d-Xp~R-1-ozG4YAzVb$ERxt0yH8S>UJ=(`TN%capzf3+OHQ( zR6VrO?xh64#3hsTU}g-Dx7B}@zCXayL^ZgXGuCrI_~UIXJ6;k@YWcdm*=h~x0(t@B_W(nUg1xHV*9SLgfQ(;^` z=KhxIWO4$+%r>f`L#_?>xX3Na(U#gSR9QPu1h33-??W7;USAcquTLq*y|E9QtG*1( z##|iU7oB^)u@H_#J>J$e-ETv_HYkQC+dZ33o(93A)S=?;Tl2|)bMhch#D^x=e5R^b z=2-4|ygiXzL)k7$xm&P&gkf`#8D&U0IkV0Lk_1y+D~Izrf*@jJsD}`qT63Ck9*Z=j z;cq+-gjr=DEclfyp>uEai^VMD`3_x@MX3eNimncP0)=qBsEj)LTx#{y zP4fGWoX)N{q{j*J>b1iv%jyO|RJCiyj%LZjTnsU+3<-VqQrB3v86LHT(5tXmjd50Nq;@GXP<9u3=+ zD+1!Q_UJrAYszwEXIVNXd({NdGg;O1h1bEwc8->*yw>yTqR4~2ag=y*hqtuV66(v7Wp^4P37}N=;O(N`> zeX6)R%rQlvT1rdkuNh#8z9h7$7^Ov|BS-I!UFPz5IblT~MEztrg@N>5TlzMDb7)*w zs$wE#ywGSaZTkeiZhqrzH50TrI(~RY`8*dN6FQoCZ>zvDp3L$%Ibfx=4OuBb>`EH; z;(RyNI5s^Pw_g*x9J0yvhp-)4tMWYNSCt#P2{b}bjo`ma4UP+k9<{bOClE7em7h4L z?YhU>KbQr0n}x6oM^b{qS8#G6^?Sf%GDzM67(@s(@pb+Tj5Cm^U&z-Bgd@>Gp|!hj ztIibj_?p)5m#D>kJ3-=$uzdg^jK4or`Zz)L9Un(SaW&YUohHA_v?fld@ej}Xkb|8! zdqtL5$+6ABxHe>>X3M0x_^6N=S3Mk`=C9JSTxm9+k%L77ik20hMoDnlb4q$j_V;&N zhkq(u71SNN#nBb0;}`k6?pjm#pOM75W2B7Zu8K_{tH~w-L%2@v9VcZ3ISJ!iz+pcM zG#uT6{NQB)aH{ZI;K=ytv;E79thRfnySNEk;oJ&8B%OzUHHZPbx4RnO{vG+5sD+Bv1Peqd3=ndHw@K}4FWXD*mC-$-p+YrFbiC7#}~ z90HQP-~{vxIstwE`om&2NTJnh&IETf7cO`|MBPWyvAtH=D7LSp(q+)eJ}H-dtjJbl zO`jMQ_;7QzC(^8^H#O#IMcX%ZDk{bn=otY&%=YFyDQ4j~S@X|*k?LJy)5*<$l5cWu z7q0q}+8kGR!NZpBG?+zb*!c}sK2CaPcd}z(HD#_AvVikb#ANAw=^gozAM3{ngFnpD z!;$@{@KY6SK%3%_Jf3AKvB3m$Stm<8`%0MAcRyh$>-h{wEBH&@=DcP`EbFopeGk~? zSs&jxE+sApCcZyfvJ0P%T_Rg$38zLSK2ih5(Z|oN!1v0DyxM2Fx2{UZRH-)>*#-c@ z3r>JCo-Uy(tbG69p6kk?n7^HiwJ={R+H!5V%N(O8OEkFX}sLw5w zG2KGE(US3OHX3H&gVD>>W=0uGWPfnQ^OG|CX&1N!NgxStoe|N2$f<9#IMISo^mvtQ z`*YGcei~V6RGgb?az?cv>HVAUr264cO6a=aRVrcJ4UD4tmk)<;Q4psnKL#iXgERUi zpZi}p)LnXMxnFsF^c<-zs}El93ox0ZLHShpIGa$xx!+dy8!QJDsS;fcBD0p8P&4o0 zAZq1`!k>96X;F0=I*ty41#MKIKqY#+c@rwgj!#PYslleO@Uo;r#a21okmO5Kr4P}& z8(T>#{Mu?EgD(fVFg7Z>TaPKG@vl-57Ru*?dhZ*jvFTu9+O2*G8XNH(0Xw!w!8*uz z=mH{Bks*CXaKM_Tj%tc`v@ckXwJZ7Q9j75iQ1RraR%wca6k)j#ugQb?(RuJ}`O6@j zZiqo#MilSAQU+hISVn(+9_8RjSm5V{dI- z+|W9D8lOmyztZoIL)DYlVUu*)lH`1p|LTbsWlHn4ngnVflXs?7k9> z5)MxOY%Q<~5$!U*8!sG>)jG*dqKzA+__ICyQ+Z6gr7VO`Y1u)2V33Adz2-TpKP*N; zkfZZqI99naRHos*BSVc^o%F7Qh_Y|s{b-O>j9!uE1%eR9=lIf(s=@df&B=a;g8e~e zjHyJmtO=$wVNM%FJ;ZTY#l@GO%|=&emr|exWfUr1Kt@6SwYR6prn}!+Cx>sKJK(FH z3}6m?mAkj%$BW4RREl~ZJwfT_Za))%%7`EDxJH{5*Ac-4N;KG!4Ea$ZFc$$!is@dQ zwIjE)ENwI~<)zNWfKG`NzUHp!8sg~Yz}P;nu`6BXjE-nS(oJDz6CwC~IIK8?%!9Yy z$e05;=+nV&bdhi?j-v*&{{aM??Klj4ufk`q9r+OXjRB_hf<7P^#%MJI7|3#5H>4P7 z4M34B{Z$GPTn%V%Co8@nU0X-5!{O>GNVBum=>lB@>7#USX=v7=Y8nTlB7z3Q6tIyh zF4nFaBHPhl)yhG|#?2mQEecfewW1G)g&qMF9**PW5qi()Qv!Qbiu?rB{BcW_keYE{ zk-1a_Zb;*@)Dc8W{5}m*8V%>c$8j`|eurvcj$~--yo<`W8e;vI38XHgqQ*fbEJlX6 zhiFjZD~ClMJdWzP3Il+-MTVcUYJ|aO7jJV4-R1#RFlXwoei}k}S~YzgBepJL_mczA zf`9BWRD~!@%3%g`NFvli;~YOLRxYe3YSuzTS=58!UmFl0B0`-(mKum|_E3kxIWDF~ z*@|R8bJhO$jsY6VQlhtYFzUKGqEUBz8h)U{+1V(H0#N<1fmWHd>#T^fum|7&t67m2 zIeoIJ2(OV2MV+mj<@3dVo_pYXX&m55!=r|BzktZ?f6R_0Nh%j5#dh;T4EeZ7wcPk5 z6ktxAXT|b_IY!N@irjs%UM>8;J;vYZdYp);zd%8f9eC3zBfNedkXolVKLa7AL7lBD zQs>XpPst#DRiXL!%m^jH)eUqGMMbHPuN)xmEo0om$meX#wirK>{5G~{A!~P zz+U|K#V+>hBtG~lkm5UXiAnpf>??#>pkjD)NhlHijy(>EF^LhT`>RL{jf$G}a#ScA zibZ9>^e&`&ed7zYBzRVXt(~#)S3SW=YN+!2_f-LEL&LnO@D4^&MsvPn`#*1t9}nBb zl7SphdQ&WqHV`WY0@B1s#+GF+%s?nF*`+N-v&^ME+b~nPYy4|?oQ@w*Y5sJ#@3dvL zpJchG%_fe!Z}lEs+>kB8AS7LAUP=Mv7IbJk58#y0&^2OdNuqfBRs|qhtmD}HELZK} zHa8Gn)O4~Z(&K76MDGS>$_nGf^Iso;1rM%CkAa{EgtW-|GGBCnM9o`B9!|mirv{`N z`0cb@WrmNyc_#))GD;w59>!o$Gd!}*5sX}% z*2~~)Iu&)r#m|N+y=0mutGwwRc4g}dFy0f$uuq;(st?_i>ptDw_s%TF?nVO2pWtz* z=dS=aRoE^&wgg@_{()g>tllna@uM>?DmM+MAa0Oy{T%sywAeC_1)YdTlC}9;M!=_5QnI^eeO7%nIc~kI|GWFJBGb%9JHwd+ z-~m`>Tnq$25?cQQuKKR<*Ia@{`#IcuIlZdu4=;dZu%!M4{X9ENZfqe5J2>e;k|OT( zTX~@@5LJy%{bsEJN~l`{#T_5a+b>Wu1i9>jWUr+w#Lcv&ihk_$T|CbrBR30bTgnc@BO0ouAG)m4qmd)%sO)9_en0 zQJSjroxZx=q*$j3gwSg@BSJ%(az!~L4Br5K^s7#DTU?KXQW0pPmZD>}yJ-Egj`?2& zd{A!(RcAoR&eiGf?~mTRYEKw)O>LhBvQ$57ma1dw$eGK>+u;EVKaJtdJimpUVf*eM zq{>$Ygm7);mXE$tz5p$CV{s8T%vL>W5md1xbR`}NuylH#YUCH_>kubikgMU@tkcSiD@|ITj~y(4)F3$*9aykwHe>@X}VD4 zxH4(|qM**W!u}uB*uh7lfC2^xV|8mH;9L?c0Aj7^0RQLPRAC9gIe_8?i5pyi;7TLZ{%hR8AHLUw?x8y5A4>gw9>U+xHYm~b>6oSy*zCK1W{O4}fsiHt peQ5wZ!Q6js81%=uR2YSx^RzaVy_cjics4B66 - - - - - - - - - - - - - - - - - - - - - - - - - - VBackground-1 - - - Solid - - - - - - - - - - - - - - Page-1 - - - - Round Corner Rectangle.37 - IPDK docker-compose testing approach - - - - - - - - - - - - - - - - - - - - - - IPDK docker-compose testing approach - - Round Corner Rectangle.4 - traffic-generator - - - - - - - - - - - - - - - - - - - - - - traffic-generator - - Round Corner Rectangle.3 - QEMU - - - - - - - - - - - - - - - - - - - - - - QEMU - - Round Corner Rectangle - fio - - - - - - - - - - - - - - - - - - - - - - fio - - Round Corner Rectangle.5 - proxy-container - - - - - - - - - - - - - - - - - - - - - - proxy-container - - Round Corner Rectangle.6 - storage-target - - - - - - - - - - - - - - - - - - - - - - storage-target - - Sheet.8 - vhost/virtio-blk - - - - vhost/virtio-blk - - Sheet.9 - qemu monitor - - - - qemu monitor - - Sheet.10 - qemu serial - - - - qemu serial - - Dynamic connector.18 - - - - Dynamic connector.19 - - - - Dynamic connector.20 - - - - Dynamic connector.21 - - - - Dynamic connector.22 - - - - Sheet.23 - NVMe/TCP - - - - NVMe/TCP - - Round Corner Rectangle.24 - - - - - - - - - - - - - - - - - - - - - - Sheet.25 - - - - Sheet.26 - containers - - - - containers - - Sheet.27 - shared objects/ docker volumes - - - - shared objects/ docker volumes - - Dynamic connector.29 - - - - Sheet.31 - init script return value - - - - init script return value - - Sheet.1000 - Hot-plug service - - - - Hot-plugservice - - Sheet.1001 - SPDK rpc - - - - SPDKrpc - - Sheet.1002 - SPDK rpc - - - - SPDKrpc - - Round Corner Rectangle.7 - test-drivers - - - - - - - - - - - - - - - - - - - - - - test-drivers - - Sheet.11 - RUN ./init - - - - RUN ./init - - Sheet.12 - init.fio script - - - - init.fio script - - Sheet.13 - init.hot-plug script - - - - init.hot-plug script - - Sheet.14 - other tests - - - - other tests - - Dynamic connector.1005 - - - - Dynamic connector.1006 - - - - Dynamic connector.1007 - - - - Dynamic connector.1008 - - - - Dynamic connector.1009 - - - - Sheet.1010 - - - - Sheet.1011 - services - - - - services - - From 2edb9b62a98a6fcd9819899814eec7a86626de78 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Wed, 25 May 2022 16:59:14 +0200 Subject: [PATCH 4/4] Format python files with Black. Signed-off-by: Artsiom Koltun --- .../core/host-target/device_exerciser.py | 7 +- build/storage/core/host-target/fio_runner.py | 16 ++-- .../host-target/host_target_grpc_server.py | 11 ++- .../core/host-target/host_target_main.py | 13 +-- build/storage/core/host-target/pci_devices.py | 53 +++++++---- .../proxy-container/hot_plug_grpc_server.py | 13 +-- .../core/proxy-container/hot_plug_main.py | 29 +++--- .../core/proxy-container/hot_plug_provider.py | 37 ++++---- .../ut/host-target/test_device_exerciser.py | 12 +-- .../test_host_target_grpc_server.py | 8 +- .../ut/host-target/test_run_grpc_server.py | 13 ++- .../test_hot_plug_grpc_server.py | 20 +++-- .../proxy-container/test_hot_plug_provider.py | 88 +++++++++++-------- 13 files changed, 187 insertions(+), 133 deletions(-) diff --git a/build/storage/core/host-target/device_exerciser.py b/build/storage/core/host-target/device_exerciser.py index 0c20dd88..3de8eb14 100644 --- a/build/storage/core/host-target/device_exerciser.py +++ b/build/storage/core/host-target/device_exerciser.py @@ -11,8 +11,11 @@ class DeviceExerciserError(RuntimeError): class DeviceExerciser: - def __init__(self, fio_runner=fio_runner.run_fio, - virtio_blk_detector=pci_devices.get_virtio_blk_path_by_pci_address): + def __init__( + self, + fio_runner=fio_runner.run_fio, + virtio_blk_detector=pci_devices.get_virtio_blk_path_by_pci_address, + ): self.fio_runner = fio_runner self.virtio_blk_detector = virtio_blk_detector diff --git a/build/storage/core/host-target/fio_runner.py b/build/storage/core/host-target/fio_runner.py index 5ec8b8fb..f2bf5b14 100644 --- a/build/storage/core/host-target/fio_runner.py +++ b/build/storage/core/host-target/fio_runner.py @@ -12,13 +12,17 @@ def run_fio(fio_args, subprocess_run=subprocess.run): fio_cmd = [] try: fio_cmd = ["fio"] + fio_args.split() - result = subprocess_run(fio_cmd, - capture_output=True, text=True) + result = subprocess_run(fio_cmd, capture_output=True, text=True) if result.returncode != 0: - raise FioExecutionError("fio execution error: '" + - str(result.stdout) + "' | '" + - str(result.stderr) + "' ") + raise FioExecutionError( + "fio execution error: '" + + str(result.stdout) + + "' | '" + + str(result.stderr) + + "' " + ) return result.stdout except BaseException as ex: raise FioExecutionError( - "Cannot execute cmd '" + " ".join(fio_cmd) + "' Error: " + str(ex)) + "Cannot execute cmd '" + " ".join(fio_cmd) + "' Error: " + str(ex) + ) diff --git a/build/storage/core/host-target/host_target_grpc_server.py b/build/storage/core/host-target/host_target_grpc_server.py index 137f3d51..fb632297 100644 --- a/build/storage/core/host-target/host_target_grpc_server.py +++ b/build/storage/core/host-target/host_target_grpc_server.py @@ -23,14 +23,12 @@ def __init__(self, message): class HostTargetService(host_target_pb2_grpc.HostTargetServicer): def __init__(self, fio_runner, virtio_blk_detector): super().__init__() - self.device_exerciser = DeviceExerciser( - fio_runner, virtio_blk_detector) + self.device_exerciser = DeviceExerciser(fio_runner, virtio_blk_detector) def RunFio(self, request, context): output = None try: - output = self.device_exerciser.run_fio( - request.pciAddress, request.fioArgs) + output = self.device_exerciser.run_fio(request.pciAddress, request.fioArgs) except BaseException as ex: context.set_code(grpc.StatusCode.FAILED_PRECONDITION) context.set_details(str(ex)) @@ -41,9 +39,10 @@ def run_grpc_server(ip_address, port, server_creator=grpc.server): try: server = server_creator(futures.ThreadPoolExecutor(max_workers=10)) host_target_pb2_grpc.add_HostTargetServicer_to_server( - HostTargetService(run_fio, get_virtio_blk_path_by_pci_address), server) + HostTargetService(run_fio, get_virtio_blk_path_by_pci_address), server + ) service_names = ( - host_target_pb2.DESCRIPTOR.services_by_name['HostTarget'].full_name, + host_target_pb2.DESCRIPTOR.services_by_name["HostTarget"].full_name, reflection.SERVICE_NAME, ) reflection.enable_server_reflection(service_names, server) diff --git a/build/storage/core/host-target/host_target_main.py b/build/storage/core/host-target/host_target_main.py index 34ac16f3..fdb04e83 100755 --- a/build/storage/core/host-target/host_target_main.py +++ b/build/storage/core/host-target/host_target_main.py @@ -12,17 +12,18 @@ def parse_arguments(): parser = argparse.ArgumentParser( - description='Runs service for hot-plug/hot-detach virtio-blk devices to vms') - parser.add_argument('--ip', required=True, - help='ip address the server listens to') - parser.add_argument('--port', type=int, default=50051, - help='port number the server listens to') + description="Runs service for hot-plug/hot-detach virtio-blk devices to vms" + ) + parser.add_argument("--ip", required=True, help="ip address the server listens to") + parser.add_argument( + "--port", type=int, default=50051, help="port number the server listens to" + ) args = parser.parse_args() return args -if __name__ == '__main__': +if __name__ == "__main__": logging.basicConfig() args = parse_arguments() diff --git a/build/storage/core/host-target/pci_devices.py b/build/storage/core/host-target/pci_devices.py index ec417bc6..ece5f2d5 100644 --- a/build/storage/core/host-target/pci_devices.py +++ b/build/storage/core/host-target/pci_devices.py @@ -8,7 +8,8 @@ pci_validator = re.compile( - r'[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-1]{1}[0-9a-fA-F]{1}\.[0-7]{1}') + r"[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-1]{1}[0-9a-fA-F]{1}\.[0-7]{1}" +) class PciAddress: @@ -16,7 +17,7 @@ def validate_pci_address(pci_address): return pci_validator.search(pci_address) != None def _parse_pci_address(pci_address): - split_pci_address = pci_address.replace('.', ':').split(":") + split_pci_address = pci_address.replace(".", ":").split(":") split_pci_address.reverse() function = split_pci_address[0].strip() device = split_pci_address[1].strip() @@ -27,8 +28,12 @@ def _parse_pci_address(pci_address): def __init__(self, pci_address) -> None: if not PciAddress.validate_pci_address(pci_address): raise InvalidPciAddress(pci_address + " is invalid") - self.domain, self.bus, self.device, self.function = \ - PciAddress._parse_pci_address(pci_address) + ( + self.domain, + self.bus, + self.device, + self.function, + ) = PciAddress._parse_pci_address(pci_address) def get_domain_bus_prefix(self): return self.domain + ":" + self.bus @@ -52,7 +57,7 @@ def get_all_files_by_pattern(pattern): return glob.glob(pattern) -def directory_find(atom, root='.'): +def directory_find(atom, root="."): dirs = get_directories(root) if dirs and atom in dirs: return os.path.join(root, atom) @@ -70,35 +75,45 @@ class FailedPciDeviceDetection(RuntimeError): def get_virtio_blk_path_by_pci_address(pci_address): addr = PciAddress(pci_address) - domain_bus_dir = os.path.join("/sys/devices", "pci" + - addr.get_domain_bus_prefix()) - pci_dev_sysfs_dir = directory_find(addr.get_full_address(), - domain_bus_dir) + domain_bus_dir = os.path.join("/sys/devices", "pci" + addr.get_domain_bus_prefix()) + pci_dev_sysfs_dir = directory_find(addr.get_full_address(), domain_bus_dir) if pci_dev_sysfs_dir == None: raise FailedPciDeviceDetection( - "No pci device with " + pci_address + " exists under " + domain_bus_dir) + "No pci device with " + pci_address + " exists under " + domain_bus_dir + ) block_directory_pattern = os.path.join(pci_dev_sysfs_dir, "virtio*/block") block_device_matches = get_all_files_by_pattern(block_directory_pattern) if len(block_device_matches) == 0: raise FailedPciDeviceDetection( - "No devices found for pattern " + block_directory_pattern) + "No devices found for pattern " + block_directory_pattern + ) elif len(block_device_matches) > 1: raise FailedPciDeviceDetection( - "Found more than one device for pattern" + - block_directory_pattern + " : " + str(block_device_matches)) + "Found more than one device for pattern" + + block_directory_pattern + + " : " + + str(block_device_matches) + ) devices = get_directories(block_device_matches[0]) if not devices or len(devices) == 0: raise FailedPciDeviceDetection( - "No device exist under " + block_directory_pattern + - " for pci device '" + pci_address + "'") + "No device exist under " + + block_directory_pattern + + " for pci device '" + + pci_address + + "'" + ) elif len(devices) > 1: raise FailedPciDeviceDetection( - "Multiple devices are dected " + str(devices) + - " for pci address '" + pci_address +"'") + "Multiple devices are dected " + + str(devices) + + " for pci address '" + + pci_address + + "'" + ) device_path = os.path.join("/dev", devices[0]) if not os.path.exists(device_path): - raise FailedPciDeviceDetection( - "Device " + device_path + " does not exist") + raise FailedPciDeviceDetection("Device " + device_path + " does not exist") return device_path diff --git a/build/storage/core/proxy-container/hot_plug_grpc_server.py b/build/storage/core/proxy-container/hot_plug_grpc_server.py index 39b336d1..cc1a27e9 100755 --- a/build/storage/core/proxy-container/hot_plug_grpc_server.py +++ b/build/storage/core/proxy-container/hot_plug_grpc_server.py @@ -34,21 +34,22 @@ def __execute_server_operation(self, request, context, operation): def HotPlugVirtioBlk(self, request, context): return self.__execute_server_operation( - request, context, - self.hot_plug_provider.hot_plug_vhost_virtio_blk) + request, context, self.hot_plug_provider.hot_plug_vhost_virtio_blk + ) def HotUnplugVirtioBlk(self, request, context): return self.__execute_server_operation( - request, context, - self.hot_plug_provider.hot_unplug_vhost_virtio_blk) + request, context, self.hot_plug_provider.hot_unplug_vhost_virtio_blk + ) def run_grpc_server(ip_address, port, hot_plug_provider): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) hot_plug_pb2_grpc.add_HotPlugServicer_to_server( - HotPlugService(hot_plug_provider), server) + HotPlugService(hot_plug_provider), server + ) service_names = ( - hot_plug_pb2.DESCRIPTOR.services_by_name['HotPlug'].full_name, + hot_plug_pb2.DESCRIPTOR.services_by_name["HotPlug"].full_name, reflection.SERVICE_NAME, ) reflection.enable_server_reflection(service_names, server) diff --git a/build/storage/core/proxy-container/hot_plug_main.py b/build/storage/core/proxy-container/hot_plug_main.py index 07b214e9..8dd26130 100755 --- a/build/storage/core/proxy-container/hot_plug_main.py +++ b/build/storage/core/proxy-container/hot_plug_main.py @@ -13,21 +13,30 @@ def parse_arguments(): parser = argparse.ArgumentParser( - description='Runs service for hot-plug/hot-detach virtio-blk devices to vms') - parser.add_argument('--ip', required=True, - help='ip address the server listens to') - parser.add_argument('--port', type=int, default=50051, - help='port number the server listens to') - parser.add_argument('--shared-dir', type=str, required=True, - help='Directory path to shared with Host resources') - parser.add_argument('--host-shared-dir', type=str, required=True, - help='Directory path to shared with container resources') + description="Runs service for hot-plug/hot-detach virtio-blk devices to vms" + ) + parser.add_argument("--ip", required=True, help="ip address the server listens to") + parser.add_argument( + "--port", type=int, default=50051, help="port number the server listens to" + ) + parser.add_argument( + "--shared-dir", + type=str, + required=True, + help="Directory path to shared with Host resources", + ) + parser.add_argument( + "--host-shared-dir", + type=str, + required=True, + help="Directory path to shared with container resources", + ) args = parser.parse_args() return args -if __name__ == '__main__': +if __name__ == "__main__": logging.basicConfig() args = parse_arguments() diff --git a/build/storage/core/proxy-container/hot_plug_provider.py b/build/storage/core/proxy-container/hot_plug_provider.py index 55aefa09..62e53a19 100755 --- a/build/storage/core/proxy-container/hot_plug_provider.py +++ b/build/storage/core/proxy-container/hot_plug_provider.py @@ -25,21 +25,21 @@ def __init__(self, shared_dir_path, host_shared_dir_path): self.shared_dir_path = shared_dir_path self.host_shared_dir_path = host_shared_dir_path - def _hot_plug_action(self, vm_monitor, - vhost_virtio_blk, device_id): + def _hot_plug_action(self, vm_monitor, vhost_virtio_blk, device_id): vm_monitor_path = os.path.join(self.shared_dir_path, vm_monitor) vhost_path = os.path.join(self.host_shared_dir_path, vhost_virtio_blk) - ret = subprocess.call("/hot-plug.sh " + vm_monitor_path + - " " + vhost_path + - " " + str(device_id), shell=True) + ret = subprocess.call( + "/hot-plug.sh " + vm_monitor_path + " " + vhost_path + " " + str(device_id), + shell=True, + ) if ret != 0: raise ActionExecutionError("Error at hot-plug execution") - def _hot_unplug_action(self, vm_monitor, - unused, device_id): + def _hot_unplug_action(self, vm_monitor, unused, device_id): vm_monitor_path = os.path.join(self.shared_dir_path, vm_monitor) - ret = subprocess.call("/hot-unplug.sh " + vm_monitor_path + - " " + str(device_id), shell=True) + ret = subprocess.call( + "/hot-unplug.sh " + vm_monitor_path + " " + str(device_id), shell=True + ) if ret != 0: raise ActionExecutionError("Error at hot-unplug execution") @@ -54,16 +54,17 @@ def __is_socket(socket_path): def __check_socket_path(path): if not HotPlugProvider.__is_socket(path): - raise FileNotFoundError(errno.ENOENT, os.strerror( - errno.ENOENT), path) + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) def _validate_vhost_virtio_blk(self, vhost_virtio_blk): - HotPlugProvider.__check_socket_path(os.path.join( - self.shared_dir_path, vhost_virtio_blk)) + HotPlugProvider.__check_socket_path( + os.path.join(self.shared_dir_path, vhost_virtio_blk) + ) def _validate_vm_monitor(self, vm_monitor): - HotPlugProvider.__check_socket_path(os.path.join( - self.shared_dir_path, vm_monitor)) + HotPlugProvider.__check_socket_path( + os.path.join(self.shared_dir_path, vm_monitor) + ) def _validate_input(self, vm_monitor, vhost_virtio_blk): self._validate_vhost_virtio_blk(vhost_virtio_blk) @@ -92,8 +93,10 @@ def _perform_disk_operation(self, vm_monitor, vhost_virtio_blk, disk_operation): def hot_plug_vhost_virtio_blk(self, vm_monitor, vhost_virtio_blk): self._perform_disk_operation( - vm_monitor, vhost_virtio_blk, self._hot_plug_action) + vm_monitor, vhost_virtio_blk, self._hot_plug_action + ) def hot_unplug_vhost_virtio_blk(self, vm_monitor, vhost_virtio_blk): self._perform_disk_operation( - vm_monitor, vhost_virtio_blk, self._hot_unplug_action) + vm_monitor, vhost_virtio_blk, self._hot_unplug_action + ) diff --git a/build/storage/tests/ut/host-target/test_device_exerciser.py b/build/storage/tests/ut/host-target/test_device_exerciser.py index 335ffafe..d96fdd59 100644 --- a/build/storage/tests/ut/host-target/test_device_exerciser.py +++ b/build/storage/tests/ut/host-target/test_device_exerciser.py @@ -17,8 +17,7 @@ def tearDown(self): pass def test_successful_fio_run(self): - self.fs.create_dir( - "/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda") + self.fs.create_dir("/sys/devices/pci0000:00/0000:00:04.0/virtio0/block/vda") self.fs.create_file("/dev/vda") fio_arguments = "--name=test --size=4MB" @@ -27,11 +26,13 @@ def fio_do_nothing(fio_args): self.assertTrue("vda" in fio_args) self.assertTrue(fio_arguments in fio_args) return "output" + fio_do_nothing.was_called = False exerciser = DeviceExerciser( fio_runner=fio_do_nothing, - virtio_blk_detector=get_virtio_blk_path_by_pci_address) + virtio_blk_detector=get_virtio_blk_path_by_pci_address, + ) out = exerciser.run_fio("0000:00:04.0", fio_arguments) self.assertTrue(fio_do_nothing.was_called) self.assertEqual(out, "output") @@ -39,8 +40,9 @@ def fio_do_nothing(fio_args): def test_any_errors_at_exercising(self): def raise_exception(unused): raise BaseException() + exerciser = DeviceExerciser( - fio_runner=raise_exception, - virtio_blk_detector=raise_exception) + fio_runner=raise_exception, virtio_blk_detector=raise_exception + ) with self.assertRaises(DeviceExerciserError) as ex: exerciser.run_fio("pcie_address", "fio_arguments") diff --git a/build/storage/tests/ut/host-target/test_host_target_grpc_server.py b/build/storage/tests/ut/host-target/test_host_target_grpc_server.py index a8e5bb14..c0cef21f 100644 --- a/build/storage/tests/ut/host-target/test_host_target_grpc_server.py +++ b/build/storage/tests/ut/host-target/test_host_target_grpc_server.py @@ -44,8 +44,8 @@ def test_run_fio_success(self): def test_run_fio_does_not_propagate_exception(self): def fio_throws_exception(unused): raise BaseException() - server = HostTargetService( - fio_throws_exception, detect_virtio_blk_device) + + server = HostTargetService(fio_throws_exception, detect_virtio_blk_device) request = host_target_pb2.RunFioRequest() request.pciAddress = "unused" request.fioArgs = "unused" @@ -58,8 +58,8 @@ def fio_throws_exception(unused): def test_run_fio_does_not_propagate_exception(self): def detect_virtio_blk_throws_exception(unused): raise BaseException() - server = HostTargetService( - successfull_fio, detect_virtio_blk_throws_exception) + + server = HostTargetService(successfull_fio, detect_virtio_blk_throws_exception) request = host_target_pb2.RunFioRequest() request.pciAddress = "unused" request.fioArgs = "unused" diff --git a/build/storage/tests/ut/host-target/test_run_grpc_server.py b/build/storage/tests/ut/host-target/test_run_grpc_server.py index f98bee8a..add40647 100644 --- a/build/storage/tests/ut/host-target/test_run_grpc_server.py +++ b/build/storage/tests/ut/host-target/test_run_grpc_server.py @@ -24,17 +24,20 @@ def test_fail_on_invalid_port_to_listen_to(self): def test_success_on_keyboard_interrupt_exception(self): server_mock = unittest.mock.Mock() server_mock.add_insecure_port.side_effect = KeyboardInterrupt() + def server_creator(unused): return server_mock + self.assertEqual(run_grpc_server("Unused", "Unused", server_creator), 0) self.assertTrue(server_mock.add_insecure_port.was_called) - - @patch.object(host_target_pb2_grpc, 'add_HostTargetServicer_to_server') + @patch.object(host_target_pb2_grpc, "add_HostTargetServicer_to_server") def test_required_methods_were_called(self, add_servicer_mock): server_mock = unittest.mock.Mock() + def server_creator(unused): return server_mock + ip_addr = "ip_addr" port = "port" @@ -49,5 +52,7 @@ def server_creator(unused): self.assertTrue(add_servicer_mock.was_called) self.assertTrue(len(add_servicer_mock.call_args.args) == 2) - self.assertTrue(isinstance(add_servicer_mock.call_args.args[0], HostTargetService)) - self.assertTrue(add_servicer_mock.call_args.args[1] == server_mock) \ No newline at end of file + self.assertTrue( + isinstance(add_servicer_mock.call_args.args[0], HostTargetService) + ) + self.assertTrue(add_servicer_mock.call_args.args[1] == server_mock) diff --git a/build/storage/tests/ut/proxy-container/test_hot_plug_grpc_server.py b/build/storage/tests/ut/proxy-container/test_hot_plug_grpc_server.py index 358d40ab..5612e590 100644 --- a/build/storage/tests/ut/proxy-container/test_hot_plug_grpc_server.py +++ b/build/storage/tests/ut/proxy-container/test_hot_plug_grpc_server.py @@ -20,8 +20,9 @@ def setUp(self): def tearDown(self): pass - def _test_server_does_not_propagate_exception(self, stub_provider_operation, - server_operation): + def _test_server_does_not_propagate_exception( + self, stub_provider_operation, server_operation + ): err_description = "exception description" stub_provider_operation.side_effect = BaseException(err_description) request = hot_plug_pb2.HotPlugRequest() @@ -34,15 +35,16 @@ def _test_server_does_not_propagate_exception(self, stub_provider_operation, @patch.object(HotPlugProvider, "hot_plug_vhost_virtio_blk") def test_hot_plug_does_not_propagate_exception(self, mock_provider): self._test_server_does_not_propagate_exception( - mock_provider, self.server.HotPlugVirtioBlk) + mock_provider, self.server.HotPlugVirtioBlk + ) @patch.object(HotPlugProvider, "hot_unplug_vhost_virtio_blk") def test_hot_unplug_does_not_propagate_exception(self, mock_provider): self._test_server_does_not_propagate_exception( - mock_provider, self.server.HotUnplugVirtioBlk) + mock_provider, self.server.HotUnplugVirtioBlk + ) - def _test_server_operation_success(self, stub_provider_operation, - server_operation): + def _test_server_operation_success(self, stub_provider_operation, server_operation): vm = "vm" vhost = "vhost" request = hot_plug_pb2.HotPlugRequest() @@ -59,10 +61,10 @@ def _test_server_operation_success(self, stub_provider_operation, @patch.object(HotPlugProvider, "hot_plug_vhost_virtio_blk") def test_hot_plug_success(self, mock_provider): - self._test_server_operation_success( - mock_provider, self.server.HotPlugVirtioBlk) + self._test_server_operation_success(mock_provider, self.server.HotPlugVirtioBlk) @patch.object(HotPlugProvider, "hot_unplug_vhost_virtio_blk") def test_hot_unplug_success(self, mock_provider): self._test_server_operation_success( - mock_provider, self.server.HotUnplugVirtioBlk) + mock_provider, self.server.HotUnplugVirtioBlk + ) diff --git a/build/storage/tests/ut/proxy-container/test_hot_plug_provider.py b/build/storage/tests/ut/proxy-container/test_hot_plug_provider.py index e9135b98..4ae9007e 100644 --- a/build/storage/tests/ut/proxy-container/test_hot_plug_provider.py +++ b/build/storage/tests/ut/proxy-container/test_hot_plug_provider.py @@ -16,8 +16,7 @@ class SocketTestEnvironment(unittest.TestCase): def setUp(self): self.fake_socket_name = "fake_socket" self.fake_socket_path = "/tmp/" + self.fake_socket_name - self.fake_socket = socket.socket( - socket.AF_UNIX, socket.SOCK_STREAM) + self.fake_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.fake_socket.bind(self.fake_socket_path) def tearDown(self): @@ -34,8 +33,9 @@ def setUp(self): self.host_shared_dir_path = "/home/user/ipdk-shared_dir" self.non_existing_path = "/tmp/non-existing_path" - self.provider = HotPlugProvider(self.shared_dir_path, - self.host_shared_dir_path) + self.provider = HotPlugProvider( + self.shared_dir_path, self.host_shared_dir_path + ) self.disk_operation = None def tearDown(self): @@ -53,22 +53,23 @@ def test_non_existing_path_to_vm_monitor(self, mock_subprocess_call): with self.assertRaises(FileNotFoundError) as ex: self.disk_operation( vm_monitor=self.non_existing_path, - vhost_virtio_blk=self.fake_socket_path) + vhost_virtio_blk=self.fake_socket_path, + ) mock_subprocess_call.assert_not_called() def test_non_existing_path_to_vhost(self, mock_subprocess_call): with self.assertRaises(FileNotFoundError) as ex: self.disk_operation( vm_monitor=self.fake_socket_path, - vhost_virtio_blk=self.non_existing_path) + vhost_virtio_blk=self.non_existing_path, + ) mock_subprocess_call.assert_not_called() def test_pass_valid_sockets(self, mock_subprocess_call): mock_subprocess_call.return_value = 0 was_exception_raised = False try: - self.disk_operation( - self.fake_socket_path, self.fake_socket_path) + self.disk_operation(self.fake_socket_path, self.fake_socket_path) except: was_exception_raised = True @@ -79,11 +80,9 @@ def test_hot_plug_call_failed(self, mock_subprocess_call): mock_subprocess_call.return_value = 1 with self.assertRaises(ActionExecutionError) as ex: - self.disk_operation( - self.fake_socket_path, self.fake_socket_path) + self.disk_operation(self.fake_socket_path, self.fake_socket_path) - def test_hot_plug_generates_correct_device_id(self, - mock_subprocess_call): + def test_hot_plug_generates_correct_device_id(self, mock_subprocess_call): mock_subprocess_call.return_value = 0 vm = self.fake_socket_name @@ -98,13 +97,11 @@ def test_hot_plug_generates_correct_device_id(self, def test_pass_none_vm(self, unused): with self.assertRaises(ValueError) as ex: - self.disk_operation( - None, self.fake_socket_path) + self.disk_operation(None, self.fake_socket_path) def test_pass_none_vhost(self, unused): with self.assertRaises(ValueError) as ex: - self.disk_operation( - self.fake_socket_path, None) + self.disk_operation(self.fake_socket_path, None) class HotPlugValidation(HotPlugValidationBaseTestCases.Tests): @@ -117,8 +114,7 @@ def tearDown(self): @patch("hot_plug_provider.subprocess.call") @patch.object(HotPlugProvider, "_generate_device_id") - def test_hot_plug_call_args(self, mock_device_id_generator, - mock_subprocess_call): + def test_hot_plug_call_args(self, mock_device_id_generator, mock_subprocess_call): device_id = "42" mock_subprocess_call.return_value = 0 mock_device_id_generator.return_value = device_id @@ -128,16 +124,20 @@ def test_hot_plug_call_args(self, mock_device_id_generator, self.disk_operation(vm, vhost) mock_subprocess_call.assert_called_once_with( - "/hot-plug.sh " + - os.path.join(self.shared_dir_path, vm) + " " + - os.path.join(self.host_shared_dir_path, vhost) + " " + - device_id, shell=True) + "/hot-plug.sh " + + os.path.join(self.shared_dir_path, vm) + + " " + + os.path.join(self.host_shared_dir_path, vhost) + + " " + + device_id, + shell=True, + ) @patch("hot_plug_provider.subprocess.call") @patch.object(HotPlugProvider, "_generate_device_id") - def test_hot_plug_call_args_with_trailing_whitespaces(self, - mock_device_id_generator, - mock_subprocess_call): + def test_hot_plug_call_args_with_trailing_whitespaces( + self, mock_device_id_generator, mock_subprocess_call + ): device_id = "42" mock_subprocess_call.return_value = 0 mock_device_id_generator.return_value = device_id @@ -148,10 +148,14 @@ def test_hot_plug_call_args_with_trailing_whitespaces(self, self.disk_operation(vm, vhost) mock_subprocess_call.assert_called_once_with( - "/hot-plug.sh " + - os.path.join(self.shared_dir_path, self.fake_socket_name) + " " + - os.path.join(self.host_shared_dir_path, self.fake_socket_name) + " " + - device_id, shell=True) + "/hot-plug.sh " + + os.path.join(self.shared_dir_path, self.fake_socket_name) + + " " + + os.path.join(self.host_shared_dir_path, self.fake_socket_name) + + " " + + device_id, + shell=True, + ) class HotUnplugValidation(HotPlugValidationBaseTestCases.Tests): @@ -164,8 +168,7 @@ def tearDown(self): @patch("hot_plug_provider.subprocess.call") @patch.object(HotPlugProvider, "_generate_device_id") - def test_hot_plug_call_args(self, mock_device_id_generator, - mock_subprocess_call): + def test_hot_plug_call_args(self, mock_device_id_generator, mock_subprocess_call): device_id = "42" mock_subprocess_call.return_value = 0 mock_device_id_generator.return_value = device_id @@ -175,14 +178,18 @@ def test_hot_plug_call_args(self, mock_device_id_generator, self.disk_operation(vm, vhost) mock_subprocess_call.assert_called_once_with( - "/hot-unplug.sh " + os.path.join(self.shared_dir_path, vm) + " " + - device_id, shell=True) + "/hot-unplug.sh " + + os.path.join(self.shared_dir_path, vm) + + " " + + device_id, + shell=True, + ) @patch("hot_plug_provider.subprocess.call") @patch.object(HotPlugProvider, "_generate_device_id") - def test_hot_plug_call_args_with_trailing_whitespaces(self, - mock_device_id_generator, - mock_subprocess_call): + def test_hot_plug_call_args_with_trailing_whitespaces( + self, mock_device_id_generator, mock_subprocess_call + ): device_id = "42" mock_subprocess_call.return_value = 0 mock_device_id_generator.return_value = device_id @@ -193,6 +200,9 @@ def test_hot_plug_call_args_with_trailing_whitespaces(self, self.disk_operation(vm, vhost) mock_subprocess_call.assert_called_once_with( - "/hot-unplug.sh " + - os.path.join(self.shared_dir_path, self.fake_socket_name) + " " + - device_id, shell=True) + "/hot-unplug.sh " + + os.path.join(self.shared_dir_path, self.fake_socket_name) + + " " + + device_id, + shell=True, + )