Merge pull request #153 from intelfisz/rel-split-containers

Storage containers splitting

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "build/storage/spdk"]
+	path = build/storage/spdk
+	url = https://github.com/spdk/spdk.git

build/scripts/run-shellcheck.sh

@@ -12,8 +12,14 @@ is_bash() {
     return 1
 }
 
+is_excluded_from_check() {
+    [[ $1 == */spdk/* ]] && return 0
+
+    return 1
+}
+
 while IFS= read -r -d $'' file; do
-    if is_bash "$file"; then
+    if ! is_excluded_from_check "$file" && is_bash "$file" ; then
         shellcheck -x -W0 -s bash "$file" || continue
     fi
 done < <(find . -type f \! -path "./.git/*" -print0)

build/storage/.dockerignore

@@ -0,0 +1 @@
+*.qcow2

build/storage/.gitignore

@@ -1,4 +1,5 @@
 *.qcow2
+__pycache__
 
 *.swp
 

build/storage/Dockerfile

@@ -0,0 +1,178 @@
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+# NOTICE: THIS FILE HAS BEEN MODIFIED BY INTEL CORPORATION UNDER COMPLIANCE
+# WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK
+#
+################################################################################
+# build_base
+#
+# This image is used to provide environment for spdk build and get it in the
+# form of spdk packages
+################################################################################
+FROM fedora:36 AS base
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+ARG SPDK_VERSION
+
+RUN dnf install -y git
+COPY spdk/ /spdk
+RUN mkdir /spdk-rpm
+COPY core/build_base/pre-install /install
+RUN chmod +x /install
+RUN /install
+
+################################################################################
+# spdk
+#
+# Contains installed SPDK from build_base rpm packages.
+# Does not contain dependencies required to build SPDK
+################################################################################
+FROM fedora:36 AS spdk
+
+LABEL maintainer=spdk.io
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+
+# Copy SPDK's RPMs built during pre-install step.
+# This allows to reduce final image size since we won't have any dependencies
+# which are only required to perform build.
+RUN mkdir /spdk-rpm
+COPY --from=base /spdk-rpm/*.rpm /spdk-rpm/
+COPY --from=base /spdk-rpm/fio /spdk-rpm/
+# Wrap up the image
+COPY core/build_base/post-install /install
+RUN chmod +x /install
+RUN /install
+
+
+################################################################################
+# spdk-app
+#
+# This image in addition to installed SPDK binaries contains a script run at
+# container boot.
+# This script runs SPDK service.
+################################################################################
+FROM spdk as spdk-app
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+
+RUN dnf install -y socat
+COPY core/spdk-app/init /init
+
+ENTRYPOINT ["/init"]
+
+
+################################################################################
+# storage-target
+#
+# This image should be place on a dedicated machine and responsible for exposing
+# ideal storage target(SPDK Malloc bdev) over NVMe/TCP
+# Configuration is performed by means of SPDK Json rpc.
+################################################################################
+FROM spdk-app AS storage-target
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+
+################################################################################
+# proxy-container
+#
+# This image is placed on IPU and exposing storage-target NVMe/TCP devices
+# It is responsible for creation of vhost virtio-blk devices and exposing them
+# to hosts(KVM or physical ones)
+################################################################################
+FROM spdk as proxy-container
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+
+RUN dnf install -y python socat python3-pip
+RUN python -m pip install grpcio grpcio-tools grpcio-reflection
+
+COPY core/proxy-container/hot_plug.proto /hot_plug.proto
+COPY scripts/socket.sh /socket.sh
+COPY core/proxy-container/hot_plug_grpc_server.py /hot_plug_grpc_server.py
+COPY core/proxy-container/hot_plug_provider.py /hot_plug_provider.py
+COPY core/proxy-container/hot_plug_main.py /hot_plug_main.py
+COPY core/proxy-container/init /init
+COPY core/proxy-container/hot-plug.sh /hot-plug.sh
+COPY core/proxy-container/hot-unplug.sh /hot-unplug.sh
+COPY --from=spdk-app /init /init_spdk
+
+# Currently proxy-container relies on spdk json rpc to execute comamnds.
+# However, there is no ability to hot-plug a vhost virtio-blk to a vm
+# by means of SPDK servce.
+# A dedicate service is currently used to provide this hot-plug functionality
+# as an addition to SPDK service.
+# Later, both will be replaced by SPDK SMA later.
+RUN python -m grpc_tools.protoc -I/ --python_out=. --grpc_python_out=/ \
+    /hot_plug.proto
+
+ENTRYPOINT [ "/init" ]
+
+
+################################################################################
+# host-target
+#
+# This image is responsible for running fio payload over different pci devices.
+# It has to be placed into host(a vm for KVM case or physical host for IPU case)
+# It uses gRPC to expose this service.
+################################################################################
+FROM fedora:36 AS host-target
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+
+RUN dnf install -y python fio python3-pip
+RUN python -m pip install grpcio grpcio-tools grpcio-reflection
+
+COPY core/host-target/init /init
+COPY core/host-target/*.py /
+COPY core/host-target/host_target.proto /host_target.proto
+
+RUN python -m grpc_tools.protoc -I/ --python_out=. --grpc_python_out=/ \
+    /host_target.proto
+
+ENTRYPOINT [ "/init" ]
+
+################################################################################
+# ipdk-unit-tests
+################################################################################
+FROM fedora:36 AS ipdk-unit-tests
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+
+RUN dnf install -y python fio python3-pip
+RUN python -m pip install grpcio-reflection pyfakefs
+COPY tests/ut/proxy-container /proxy-container/tests
+COPY --from=proxy-container hot_plug_*pb2.py /proxy-container/generated/
+COPY --from=proxy-container hot_plug_*pb2_grpc.py /proxy-container/generated/
+COPY --from=proxy-container hot_plug_grpc_server.py /proxy-container/src/
+COPY --from=proxy-container hot_plug_provider.py /proxy-container/src/
+
+COPY tests/ut/host-target /host-target/tests
+COPY --from=host-target fio_runner.py /host-target/src/
+COPY --from=host-target pci_devices.py /host-target/src/
+COPY --from=host-target device_exerciser.py /host-target/src/
+COPY --from=host-target host_target_main.py /host-target/src/
+COPY --from=host-target host_target_grpc_server.py /host-target/src/
+COPY --from=host-target host_target_*pb2.py /host-target/generated/
+COPY --from=host-target host_target_*pb2_grpc.py /host-target/generated/
+
+COPY tests/ut/run_all_unit_tests.sh /
+
+ENV PYTHONPATH=/proxy-container/generated:/proxy-container/src:/host-target/src:/host-target/generated
+
+ENTRYPOINT [ "/run_all_unit_tests.sh" ]

build/storage/Makefile

@@ -2,4 +2,4 @@
 #SPDX-License-Identifier: Apache-2.0
 #
 
-shellcheck: ;../networking/scripts/run-shellcheck.sh
+shellcheck: ;../scripts/run-shellcheck.sh

build/storage/README.md

@@ -1,3 +1,22 @@
-# Storage Builds
+# Description
+IPDK Storage solution targets the audience which has existing custom storage
+protocols or implementations of standards such as NVMe-over-Fabrics and wants
+to accelerate them by means of a common storage solution across vendors
+and platforms.
 
-Coming soon
+# Main components
+The storage solution is represented by the following containers:
+- _proxy-container_ which plays the role of an IPU and exposes the block
+devices to a host platform.
+- _storage-target_ which is deployed on a remote storage node exposing
+ideal targets(ramdrives).
+- _host-target_ which is responsible for running of fio traffic through
+`proxy-container` to an ideal target within `storage-target`.
+
+# Recipes
+IPDK storage scenarios are described by recipes located in a dedicated
+[recipes](recipes/README.md) directory.
+
+# Tests
+The storage solution is covered by [unit tests](tests/ut/) and
+[integration tests](tests/it/).

build/storage/core/build_base/post-install

@@ -8,7 +8,7 @@
 #
 set -e
 
-dnf install -y /tmp/*.rpm
+dnf install -y /spdk-rpm/*.rpm
 
 # Be nice for docker exec and link SPDK scripts|binaries under common PATH
 # location like /usr/bin.
@@ -21,7 +21,7 @@ ln -s /usr/libexec/spdk/include/spdk /usr/include
 ln -s /usr/libexec/spdk/scripts/ /usr
 
 mkdir -p /usr/src/fio
-mv /tmp/fio /usr/src/fio
+mv /spdk-rpm/fio /usr/src/fio
 
 dnf clean all
 rm -f /tmp/*.rpm

build/storage/core/build_base/pre-install

@@ -7,28 +7,17 @@
 # WITH THE APACHE 2.0 LICENSE FROM THE ORIGINAL WORK
 #
 set -e
-set -x
 
-spdk_repo=$(mktemp -dt "spdk.XXXXXX")
-spdk_tar=/spdk.tar.gz
+spdk_repo=/spdk
 
 cleanup() {
 
 	rm -f "$HOME/rpmbuild/rpm/x86_64/"*.rpm
-	rm -f "$spdk_tar"
 	rm -rf "$spdk_repo"
 }
 
 trap 'cleanup' EXIT
 
-if [[ ! -e $spdk_tar ]]; then
-	printf 'Missing %s\n' "$spdk_tar" >&2
-	exit 1
-fi
-
-tar -C "$spdk_repo" -xf "$spdk_tar"
-cd "$spdk_repo" && git checkout v21.10 && git submodule update --init && cd -
-
 # Required for building RPM
 dnf install -y rpm-build
 
@@ -48,6 +37,6 @@ DEPS="no" "$spdk_repo/rpmbuild/rpm.sh" \
 	--with-virtio \
 	--with-fio
 
-mv "$HOME/rpmbuild/rpm/x86_64/"*.rpm /tmp
-mv "/usr/src/fio/fio" /tmp
+mv "$HOME/rpmbuild/rpm/x86_64/"*.rpm "/spdk-rpm/"
+mv "/usr/src/fio/fio" "/spdk-rpm/"
 dnf clean all

build/storage/core/host-target/device_exerciser.py

@@ -0,0 +1,28 @@
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+import fio_runner
+import pci_devices
+import os
+
+
+class DeviceExerciserError(RuntimeError):
+    pass
+
+
+class DeviceExerciser:
+    def __init__(
+        self,
+        fio_runner=fio_runner.run_fio,
+        virtio_blk_detector=pci_devices.get_virtio_blk_path_by_pci_address,
+    ):
+        self.fio_runner = fio_runner
+        self.virtio_blk_detector = virtio_blk_detector
+
+    def run_fio(self, pci_address, fio_args):
+        try:
+            device_path = self.virtio_blk_detector(pci_address)
+            fio_args_with_device = fio_args + " --filename=" + device_path
+            return self.fio_runner(fio_args_with_device)
+        except BaseException as ex:
+            raise DeviceExerciserError(str(ex))