From b56ec5d974e844591239598a98a898748faa686b Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Wed, 1 Mar 2023 13:31:35 +0100 Subject: [PATCH 1/4] nvme/rc: Fixup loop transport to handle several ports The 'loop' transport allows for several distinct ports if the transport address is set to a number. Signed-off-by: Hannes Reinecke --- tests/nvme/rc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/nvme/rc b/tests/nvme/rc index 210a82ae..57a55977 100644 --- a/tests/nvme/rc +++ b/tests/nvme/rc @@ -24,6 +24,8 @@ _nvme_requires() { loop) _have_driver nvme-loop _have_configfs + def_adrfam="loop" + def_traddr="1" ;; pci) _have_driver nvme @@ -331,6 +333,8 @@ _nvme_connect_subsys() { ARGS+=(-a "${traddr}" -w "${host_traddr}") elif [[ "${trtype}" != "loop" ]]; then ARGS+=(-a "${traddr}" -s "${trsvcid}") + elif [[ "${traddr}" ]]; then + ARGS+=(-a "${traddr}") fi if [[ "${hostnqn}" != "$def_hostnqn" ]]; then ARGS+=(--hostnqn="${hostnqn}") @@ -379,7 +383,7 @@ _create_nvmet_port() { echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype" echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr" echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam" - if [[ "${adrfam}" != "fc" ]]; then + if [[ "${adrfam}" != "fc" && "${adrfam}" != "loop" ]]; then echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid" fi From c65124fb1d05b3ed0ccb02ae7b218278aea91a1f Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Fri, 6 Jul 2018 15:07:38 +0200 Subject: [PATCH 2/4] nvme/rc: enable ANA support Update functions to support ANA (Asymmetric Namespace Access). Signed-off-by: Hannes Reinecke --- tests/nvme/rc | 96 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 88 insertions(+), 8 deletions(-) diff --git a/tests/nvme/rc b/tests/nvme/rc index 57a55977..06f200f6 100644 --- a/tests/nvme/rc +++ b/tests/nvme/rc @@ -231,8 +231,7 @@ _cleanup_nvmet() { for port in "${NVMET_CFS}"/ports/*; do name=$(basename "${port}") echo "WARNING: Test did not clean up port: ${name}" - rm -f "${port}"/subsystems/* - rmdir "${port}" + _remove_nvmet_port "${name}" done for subsys in "${NVMET_CFS}"/subsystems/*; do @@ -392,7 +391,63 @@ _create_nvmet_port() { _remove_nvmet_port() { local port="$1" - rmdir "${NVMET_CFS}/ports/${port}" + local port_cfs="${NVMET_CFS}/ports/${port}" + for grp in "${port_cfs}"/ana_groups/*; do + [ -d "${grp}" ] || continue + if [[ "${grp##*/}" != "1" ]]; then + rmdir "${grp}" + fi + done + rm -f "${port_cfs}"/subsystems/* + rmdir "${port_cfs}" +} + +_create_nvmet_anagroup() { + local port="$1" + local port_cfs="${NVMET_CFS}/ports/${port}" + local grpid + + for ((grpid = 1; ; grpid++)); do + if [[ ! -e "${port_cfs}/ana_groups/${grpid}" ]]; then + break + fi + done + + mkdir "${port_cfs}/ana_groups/${grpid}" + + echo "${grpid}" +} + +_set_nvmet_anagroup_state() { + local port="$1" + local grpid="$2" + local state="$3" + local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}" + + echo "${state}" > "${ana_cfs}/ana_state" +} + +_display_ana_state() { + local nvmet_subsystem="$1" + local grpid state cntlid + + for nvme in /sys/class/nvme/* ; do + [ -f "${nvme}/subsysnqn" ] || continue + subsys="$(cat "${nvme}/subsysnqn")" + if [ "${subsys}" != "${nvmet_subsystem}" ] ; then + continue + fi + cntlid="$(cat "${nvme}/cntlid")" + for c in "${nvme}"/nvme* ; do + if [ ! -d "${c}" ] ; then + echo "${cntlid}: ANA disabled" + continue + fi + grpid="$(cat "${c}/ana_grpid")" + state="$(cat "${c}/ana_state")" + echo "${cntlid}: grpid ${grpid} state ${state}" + done + done } _create_nvmet_ns() { @@ -402,26 +457,48 @@ _create_nvmet_ns() { local uuid="00000000-0000-0000-0000-000000000000" local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" local ns_path="${subsys_path}/namespaces/${nsid}" + local ana_grpid - if [[ $# -eq 4 ]]; then + if [[ $# -ge 4 && -n "$4" ]]; then uuid="$4" fi + if [[ $# -eq 5 ]]; then + ana_grpid="$5" + fi + mkdir "${ns_path}" printf "%s" "${blkdev}" > "${ns_path}/device_path" printf "%s" "${uuid}" > "${ns_path}/device_uuid" + if [ -n "${ana_grpid}" ] ; then + printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid" + fi printf 1 > "${ns_path}/enable" } _create_nvmet_subsystem() { local nvmet_subsystem="$1" - local blkdev="$2" - local uuid=$3 + local blkdev + local uuid + local ana_grpid local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}" mkdir -p "${cfs_path}" echo 1 > "${cfs_path}/attr_allow_any_host" - _create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" + if [[ $# -ge 2 ]]; then + blkdev="$2" + fi + if [[ $# -ge 3 ]]; then + uuid="$3" + fi + if [[ $# -ge 4 ]]; then + ana_grpid="$4" + fi + if [ -z "$blkdev" ]; then + return + fi + _create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" \ + "${uuid}" "${ana_grpid}" } _create_nvmet_host() { @@ -552,8 +629,11 @@ _find_nvme_dev() { [ -e "$dev" ] || continue dev="$(basename "$dev")" subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn")" - if [[ "$subsysnqn" == "$subsys" ]]; then + if [[ "${subsysnqn}" == "${subsys}" ]]; then echo "$dev" + if [[ ! -d /sys/block/${dev} ]] ; then + return + fi for ((i = 0; i < 10; i++)); do if [[ -e /sys/block/$dev/uuid && -e /sys/block/$dev/wwid ]]; then From 90a3de0ea2b1814b3aad90460d4451bdcc34b94c Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Wed, 1 Mar 2023 14:23:52 +0100 Subject: [PATCH 3/4] nvme/rc: auto-increment port transport address When creating more than one port we need to ensure that the transport address differs, otherwise we cannot connect to it. So automatically increment the transport address (for loop) or the trsvcid (for tcp) to create distinct transport addresses. Signed-off-by: Hannes Reinecke --- tests/nvme/rc | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/nvme/rc b/tests/nvme/rc index 06f200f6..7558a136 100644 --- a/tests/nvme/rc +++ b/tests/nvme/rc @@ -372,9 +372,32 @@ _create_nvmet_port() { local trsvcid="${4:-$def_trsvcid}" local port + local portcfs for ((port = 0; ; port++)); do - if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then + portcfs="${NVMET_CFS}/ports/${port}" + if [[ ! -e "${portcfs}" ]]; then break + else + tmp_trtype=$(cat "${portcfs}/addr_trtype") + tmp_traddr=$(cat "${portcfs}/addr_traddr") + tmp_trsvcid=$(cat "${portcfs}/addr_trsvcid") + if [[ "${tmp_trtype}" != "${trtype}" ]]; then + continue + fi + if [[ "${tmp_traddr}" != "${traddr}" ]]; then + continue + fi + if [[ "${trtype}" == "loop" ]]; then + traddr=$((tmp_traddr + 1)) + elif [[ "${trtype}" == "tcp" ]]; then + if [[ "${tmp_trsvcid}" != "${trsvcid}" ]]; then + continue + fi + trsvcid=$((tmp_trsvcid + 1)) + else + # target port present, return + exit 1 + fi fi done @@ -402,6 +425,24 @@ _remove_nvmet_port() { rmdir "${port_cfs}" } +_nvmet_traddr_from_port() { + local port="$1" + local port_cfs="${NVMET_CFS}/ports/${port}" + + if [ -d "${port_cfs}" ] ; then + cat "${port_cfs}/addr_traddr" + fi +} + +_nvmet_trsvcid_from_port() { + local port="$1" + local port_cfs="${NVMET_CFS}/ports/${port}" + + if [ -d "${port_cfs}" ] ; then + cat "${port_cfs}/addr_trsvcid" + fi +} + _create_nvmet_anagroup() { local port="$1" local port_cfs="${NVMET_CFS}/ports/${port}" From f8614cf4b6ca15200e36040d245982a41455f4a6 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Fri, 6 Jul 2018 15:22:33 +0200 Subject: [PATCH 4/4] nvme/047: add test for ANA state transition Create two ports on the subsystem and connect to both of them, then run a fio job in the background and switch the ANA state of these ports to check that I/O runs during state changes. Signed-off-by: Hannes Reinecke --- tests/nvme/047 | 199 +++++++++++++++++++++++++++++++++++++++++++++ tests/nvme/047.out | 28 +++++++ 2 files changed, 227 insertions(+) create mode 100755 tests/nvme/047 create mode 100644 tests/nvme/047.out diff --git a/tests/nvme/047 b/tests/nvme/047 new file mode 100755 index 00000000..0853d76f --- /dev/null +++ b/tests/nvme/047 @@ -0,0 +1,199 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0+ +# Copyright (C) 2023 Hannes Reinecke, SUSE Labs +# +# Regression test for ANA base support + +. tests/nvme/rc +. common/xfs + +DESCRIPTION="test ANA optimized/transitioning/inaccessible support" + +requires() { + _nvme_requires + _have_loop + _have_xfs + _have_fio + _require_nvme_trtype_is_fabrics +} + +switch_nvmet_anagroup() { + local port1="$1" + local port2="$2" + local mode="$3" + + echo "ANA state ${mode}" + + if [ "${mode}" = "change" ] ; then + _set_nvmet_anagroup_state "${port1}" "1" "change" + _set_nvmet_anagroup_state "${port1}" "2" "change" + _set_nvmet_anagroup_state "${port2}" "1" "change" + _set_nvmet_anagroup_state "${port2}" "2" "change" + elif [ "${mode}" = "failover" ] ; then + _set_nvmet_anagroup_state "${port1}" "1" "inaccessible" + _set_nvmet_anagroup_state "${port1}" "2" "optimized" + _set_nvmet_anagroup_state "${port2}" "1" "optimized" + _set_nvmet_anagroup_state "${port2}" "2" "inaccessible" + elif [ "${mode}" = "soft-failover" ] ; then + _set_nvmet_anagroup_state "${port1}" "1" "non-optimized" + _set_nvmet_anagroup_state "${port1}" "2" "optimized" + _set_nvmet_anagroup_state "${port2}" "1" "optimized" + _set_nvmet_anagroup_state "${port2}" "2" "non-optimized" + elif [ "${mode}" = "soft-failback" ] ; then + _set_nvmet_anagroup_state "${port1}" "1" "optimized" + _set_nvmet_anagroup_state "${port1}" "2" "non-optimized" + _set_nvmet_anagroup_state "${port2}" "1" "non-optimized" + _set_nvmet_anagroup_state "${port2}" "2" "optimized" + else + _set_nvmet_anagroup_state "${port1}" "1" "optimized" + _set_nvmet_anagroup_state "${port1}" "2" "inaccessible" + _set_nvmet_anagroup_state "${port2}" "1" "inaccessible" + _set_nvmet_anagroup_state "${port2}" "2" "optimized" + fi +} + +execute_ana_test() { + local nvmedev=$1 + local subsys_name=$2 + local port1=$3 + local port2=$4 + local mount_dir="/mnt/blktests" + local nproc=$(nproc) + + switch_nvmet_anagroup "${port1}" "${port2}" "failover" + + # Insert a delay to allow the AEN to be processed + sleep 1 + + _display_ana_state "${subsys_name}" + + _xfs_mkfs_and_mount "/dev/${nvmedev}n1" "${mount_dir}/ns1" >> "${FULL}" 2>&1 + + _xfs_mkfs_and_mount "/dev/${nvmedev}n2" "${mount_dir}/ns2" >> "${FULL}" 2>&1 + + if (( nproc > 8 )); then + nproc=8 + fi + + _run_fio_rand_io --size=48m --numjobs="${nproc}" \ + --directory="${mount_dir}/ns1" \ + --runtime=20s --time_based & + trap 'kill %1' EXIT + + _run_fio_rand_io --size=48m --numjobs="${nproc}" \ + --directory="${mount_dir}/ns2" \ + --runtime=20s --time_based & + trap 'kill %2' EXIT + + sleep 3 + + switch_nvmet_anagroup "${port1}" "${port2}" "change" + + # Insert a delay to allow the AEN to be processed + sleep 1 + + _display_ana_state "${subsys_name}" + + sleep 1 + + switch_nvmet_anagroup "${port1}" "${port2}" "failback" + + # Insert a delay to allow the AEN to be processed + sleep 1 + + _display_ana_state "${subsys_name}" + + sleep 3 + + switch_nvmet_anagroup "${port1}" "${port2}" "soft-failover" + + # Insert a delay to allow the AEN to be processed + sleep 1 + + _display_ana_state "${subsys_name}" + + sleep 3 + + switch_nvmet_anagroup "${port1}" "${port2}" "soft-failback" + + # Insert a delay to allow the AEN to be processed + sleep 1 + + _display_ana_state "${subsys_name}" + + wait %2 + wait %1 + trap - EXIT + + umount "${mount_dir}/ns2" > /dev/null + umount "${mount_dir}/ns1" > /dev/null + rmdir "${mount_dir}/ns2" + rmdir "${mount_dir}/ns1" + rmdir "${mount_dir}" +} + +test() { + local port1 + local port2 + local subsys_name="blktests-subsystem-1" + local file_path1="${TMPDIR}/img1" + local file_path2="${TMPDIR}/img2" + local loop_dev1 + local loop_dev2 + local nvmedev + + echo "Running ${TEST_NAME}" + + _setup_nvmet + + _create_nvmet_subsystem "${subsys_name}" + + port1="$(_create_nvmet_port "${nvme_trtype}")" + _create_nvmet_anagroup "${port1}" > /dev/null + + truncate -s 512M "${file_path1}" + loop_dev1="$(losetup -f --show "${file_path1}")" + + _create_nvmet_ns "${subsys_name}" "1" "${loop_dev1}" "" "1" + + port2="$(_create_nvmet_port "${nvme_trtype}")" + _create_nvmet_anagroup "${port2}" > /dev/null + + truncate -s 512M "${file_path2}" + loop_dev2="$(losetup -f --show "${file_path2}")" + + _create_nvmet_ns "${subsys_name}" "2" "${loop_dev2}" "" "2" + + _add_nvmet_subsys_to_port "${port1}" "${subsys_name}" + _add_nvmet_subsys_to_port "${port2}" "${subsys_name}" + + traddr=$(_nvmet_traddr_from_port "${port1}") + trsvcid=$(_nvmet_trsvcid_from_port "${port1}") + _nvme_connect_subsys "${nvme_trtype}" "${subsys_name}" "${traddr}" "${trsvcid}" + traddr=$(_nvmet_traddr_from_port "${port2}") + trsvcid=$(_nvmet_trsvcid_from_port "${port2}") + _nvme_connect_subsys "${nvme_trtype}" "${subsys_name}" "${traddr}" "${trsvcid}" + + udevadm settle + + nvmedev=$(_find_nvme_dev "${subsys_name}") + if [[ -n "${nvmedev}" ]]; then + execute_ana_test "${nvmedev}" "${subsys_name}" \ + "${port1}" "${port2}" + fi + + nvme disconnect -n "${subsys_name}" + + _remove_nvmet_subsystem_from_port "${port1}" "${subsys_name}" + _remove_nvmet_subsystem_from_port "${port2}" "${subsys_name}" + _remove_nvmet_ns "${subsys_name}" "2" + _remove_nvmet_subsystem "${subsys_name}" + _remove_nvmet_port "${port1}" + _remove_nvmet_port "${port2}" + losetup --detach "${loop_dev2}" + rm "${file_path2}" + losetup --detach "${loop_dev1}" + rm "${file_path1}" + + echo "Test complete" +} diff --git a/tests/nvme/047.out b/tests/nvme/047.out new file mode 100644 index 00000000..8873b2b0 --- /dev/null +++ b/tests/nvme/047.out @@ -0,0 +1,28 @@ +Running nvme/047 +ANA state failover +1: grpid 1 state inaccessible +1: grpid 2 state optimized +2: grpid 1 state optimized +2: grpid 2 state inaccessible +ANA state change +1: grpid 1 state change +1: grpid 2 state change +2: grpid 1 state change +2: grpid 2 state change +ANA state failback +1: grpid 1 state optimized +1: grpid 2 state inaccessible +2: grpid 1 state inaccessible +2: grpid 2 state optimized +ANA state soft-failover +1: grpid 1 state non-optimized +1: grpid 2 state optimized +2: grpid 1 state optimized +2: grpid 2 state non-optimized +ANA state soft-failback +1: grpid 1 state optimized +1: grpid 2 state non-optimized +2: grpid 1 state non-optimized +2: grpid 2 state optimized +NQN:blktests-subsystem-1 disconnected 2 controller(s) +Test complete