From 1cb77193c48108e0c3e19fa8ed2e80bcb84654d3 Mon Sep 17 00:00:00 2001 From: Axel Gembe Date: Thu, 12 Oct 2023 07:52:52 +0900 Subject: [PATCH] tests: Add a pre-allocation test (#284) I used this test when diagnosing #281 and think it would be a good addition. This tests file sizes when pre-allocating a file. Expected values have been confirmed on NTFS, ReFS and FAT32. This test currently does not work on a SMB share with the Samba server because it reports a fixed cluster size based on a configuration option instead of getting the correct value from the underlying file system. ksmbd does not have the same issue, it correctly gets the values from the file system. So far this is untested with Windows SMB shares. This also makes `tests.py` use some helpers from `utils.py`. Signed-off-by: Axel Gembe --- .github/workflows/windows-build-test.yml | 33 ++ contrib/windows/tests/regression.py | 128 ++++++ contrib/windows/tests/tests.py | 36 +- contrib/windows/tests/utils.py | 476 +++++++++++++++++++++++ 4 files changed, 649 insertions(+), 24 deletions(-) create mode 100644 contrib/windows/tests/regression.py create mode 100644 contrib/windows/tests/utils.py diff --git a/.github/workflows/windows-build-test.yml b/.github/workflows/windows-build-test.yml index 83ac59a9cfdc..410f2b66a948 100644 --- a/.github/workflows/windows-build-test.yml +++ b/.github/workflows/windows-build-test.yml @@ -2735,3 +2735,36 @@ jobs: + test9_regression_test: + needs: [build_windows] + timeout-minutes: 30 + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/download-artifact@v3 + with: + name: dev_build_inno + + - name: get zfsexename + id: zfsinstaller + run: | + $p = Get-ChildItem | Where-Object {$_.Name -like 'OpenZFSOnWindows-*.exe'} | Select-Object -first 1 + echo $p + $f = (Get-Item $p ).Name + echo $f + echo "filename=$f" | Out-File -FilePath $Env:GITHUB_OUTPUT -Encoding utf8 -Append + + # https://github.com/MicrosoftDocs/windows-powershell-docs/issues/266 + - name: Import root certificate + run: | + $plaintextpwd = 'password1234' + $pwd = ConvertTo-SecureString -String $plaintextpwd -Force -AsPlainText + Import-PfxCertificate -FilePath ${{github.workspace}}/contrib/windows/TestCert/test_sign_cert_pass.pfx -CertStoreLocation Cert:\LocalMachine\Root -Password $pwd + Import-PfxCertificate -FilePath ${{github.workspace}}/contrib/windows/TestCert/test_sign_cert_pass.pfx -CertStoreLocation Cert:\LocalMachine\TrustedPublisher -Password $pwd + + - name: install zfs + run: 'Start-Process -FilePath "${{github.workspace}}\${{ steps.zfsinstaller.outputs.filename }}" -Wait -ArgumentList "/NORESTART /ALLUSERS /VERYSILENT /LOG=`"${{github.workspace}}\InnoSetup-Install.log`""' + + - name: test + run: 'python.exe -u "${{github.workspace}}\contrib\windows\tests\regression.py" --path ${{github.workspace}}\' diff --git a/contrib/windows/tests/regression.py b/contrib/windows/tests/regression.py new file mode 100644 index 000000000000..c5172252ffec --- /dev/null +++ b/contrib/windows/tests/regression.py @@ -0,0 +1,128 @@ +import argparse +import logging +import os +import pathlib +import unittest + +from utils import ( + ZfsContext, + Size, + add_common_arguments, + allocated_files, + paths_to_unc, + setup_logging, + preallocate_file_object, + get_sizes_from_path, + get_sizes_from_file, + get_cluster_size_from_file, + zpool_create, +) + + +args: argparse.Namespace +ctx: ZfsContext + + +logger = setup_logging("tests.regression", logging.INFO) +tc = unittest.TestCase() + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Process command line arguments." + ) + + add_common_arguments(parser) + + return parser.parse_args() + + +def test_preallocation(test_path: pathlib.Path): + """Tests file sizes when pre-allocating a file. + Expected values have been confirmed on NTFS, ReFS and FAT32. + This test currently does not work on a SMB share with the Samba server + because it reports a fixed cluster size based on a configuration option + instead of getting the correct value from the underlying file system. ksmbd + does not have the same issue, it correctly gets the values from the file + system. So far this is untested with Windows SMB shares. + + See https://github.com/openzfsonwindows/openzfs/issues/281 + See https://bugzilla.samba.org/show_bug.cgi?id=7436 + + Args: + test_path (pathlib.Path): The path where we want to run the test + """ + + fpath = test_path / "testfile.bin" + + try: + with open(fpath, "wb") as test_file: + csize = get_cluster_size_from_file(test_file) + tc.assertGreaterEqual(csize, 512, f"Bad cluster size for {fpath}") + + fsize = get_sizes_from_file(test_file) + tc.assertEqual( + fsize, + {"AllocationSize": 0, "EndOfFile": 0}, + f"Wrong file size after creation of {fpath}", + ) + + preallocate_file_object(test_file, 512) + + fsize = get_sizes_from_file(test_file) + tc.assertEqual( + fsize, + {"AllocationSize": csize, "EndOfFile": 0}, + f"Wrong file size after preallocation of {fpath}", + ) + + test_file.write(b"\x55" * 117) + + fsize = get_sizes_from_file(test_file) + tc.assertEqual( + fsize, + {"AllocationSize": csize, "EndOfFile": 0}, + f"Wrong file size after write to preallocated file {fpath}", + ) + + fsize = get_sizes_from_path(fpath) + tc.assertEqual( + fsize, + {"AllocationSize": csize, "EndOfFile": 117}, + f"Wrong file size after close of preallocated file {fpath}", + ) + finally: + if os.path.isfile(fpath): + os.unlink(fpath) + + +def run_tests(test_path: pathlib.Path): + test_preallocation(test_path) + + +def main(): + global args + global ctx + + args = parse_arguments() + ctx = ZfsContext(args.zfspath) + + if args.no_pool: + run_tests(args.path) + else: + with ( + allocated_files([(args.path / "test.dat", 1 * Size.GIB)]) as files, + zpool_create(ctx, "test", paths_to_unc(files)) as pool + ): + logger.info( + f'Created zpool named "{pool.name}", backed by {files}, ' + f"mounted in {pool.mount_path}" + ) + + run_tests(pool.mount_path) + + logger.info("PASSED") + + +if __name__ == "__main__": + main() diff --git a/contrib/windows/tests/tests.py b/contrib/windows/tests/tests.py index 8b67c02f3347..006e2f40b692 100644 --- a/contrib/windows/tests/tests.py +++ b/contrib/windows/tests/tests.py @@ -14,6 +14,8 @@ import logging +from utils import Size, argparse_as_abspath, allocate_file + logging.basicConfig(level=logging.DEBUG) print("Printed immediately.") @@ -22,18 +24,10 @@ def parse_arguments(): parser = argparse.ArgumentParser(description='Process command line ' 'arguments.') - parser.add_argument('-path', type=dir_path, required=True) + parser.add_argument('-path', type=argparse_as_abspath, required=True) return parser.parse_args() -def dir_path(path): - if os.path.isdir(path): - return path - else: - raise argparse.ArgumentTypeError(f"readable_dir:{path} is not a valid" - "path") - - def get_DeviceId(): magic_number_process = subprocess.run( ["wmic", "diskdrive", "get", "DeviceId"], @@ -70,12 +64,6 @@ def get_DeviceId(): return e -def allocate_file(name, size): - with open(name, 'wb') as f: - f.seek(size) - f.write(b'0') - - def delete_file(name): if os.path.exists(name): os.remove(name) @@ -199,7 +187,7 @@ def main(): print("Path:", parsed_args.path) - p = PureWindowsPath(parsed_args.path) + p = parsed_args.path print("Path object:", p) @@ -208,11 +196,11 @@ def main(): if p.is_absolute(): f1 = PureWindowsPath(p, "test01.dat") - allocate_file(f1, 1024*1024*1024) + allocate_file(f1, 1 * Size.GIB) f2 = PureWindowsPath(p, "test02.dat") - allocate_file(f2, 1024*1024*1024) + allocate_file(f2, 1 * Size.GIB) f3 = PureWindowsPath(p, "test03.dat") - allocate_file(f3, 1024*1024*1024) + allocate_file(f3, 1 * Size.GIB) preTest() ret = runWithPrint(["zpool", "create", "-f", "test01", tounc(f1)]) @@ -299,14 +287,14 @@ def main(): print("Drive letters after pool create:", get_driveletters()) f = PureWindowsPath(get_driveletters()[0][1], "test01.file") - allocate_file(f, 1024) + allocate_file(f, 1 * Size.KIB) ret = runWithPrint(["zfs", "snapshot", "testsn01@friday"]) if ret.returncode != 0: print("FAIL") f = PureWindowsPath(get_driveletters()[0][1], "test02.file") - allocate_file(f, 1024) + allocate_file(f, 1 * Size.KIB) ret = runWithPrint(["zpool", "export", "-a"]) if ret.returncode != 0: @@ -327,7 +315,7 @@ def main(): # print("Drive letters after pool create:", get_driveletters()) # # f = PureWindowsPath(get_driveletters()[0][1], "test01.file") - # allocate_file(f, 1024) + # allocate_file(f, 1 * Size.KIB) # # ret = runWithPrint(["zfs", "snapshot", "testsn02@friday"]) # if ret.returncode != 0: @@ -335,7 +323,7 @@ def main(): # # # f = PureWindowsPath(get_driveletters()[0][1], "test02.file") - # allocate_file(f, 1024) + # allocate_file(f, 1 * Size.KIB) # # ret = runWithPrint(["zfs", "mount", "testsn02@friday"]) # if ret.returncode != 0: @@ -408,7 +396,7 @@ def main(): f = PureWindowsPath(get_driveletters()[0][1], "test01.file") try: - allocate_file(f, 1024) + allocate_file(f, 1 * Size.KIB) except Exception: print("FAIL") diff --git a/contrib/windows/tests/utils.py b/contrib/windows/tests/utils.py new file mode 100644 index 000000000000..f964b2d3e83f --- /dev/null +++ b/contrib/windows/tests/utils.py @@ -0,0 +1,476 @@ +import argparse +import contextlib +import ctypes +from ctypes import wintypes +import enum +import logging +import msvcrt +import os +import pathlib +import subprocess +import time +import typing + + +class Size(enum.IntEnum): + KIB = 1024 + MIB = KIB * 1024 + GIB = MIB * 1024 + + +def setup_logging(name: str, level: int) -> logging.Logger: + logger = logging.getLogger(name) + logger.setLevel(level) + + ch = logging.StreamHandler() + ch.setLevel(level) + + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + ch.setFormatter(formatter) + + logger.addHandler(ch) + + return logger + + +def get_next_drive_path() -> pathlib.PureWindowsPath: + for driveletter in list(map(chr, range(ord("D"), ord("Z") + 1))): + path = pathlib.PureWindowsPath(f"{driveletter}:\\") + if os.path.isdir(path): + continue + return path + raise RuntimeError("Unable to find free drive letter") + + +def decode_console_cp(data: bytes) -> str: + """Decodes a string of bytes using the active console codepage. + + Args: + data (bytes): String of bytes in the consoles codepage + + Returns: + str: The decoded string + """ + + console_cp = ctypes.windll.kernel32.GetConsoleOutputCP() + return data.decode("cp" + str(console_cp), errors="strict") + + +def cmd_res_to_str(res: subprocess.CompletedProcess[bytes]) -> str: + return ( + f"Stdout:\n{decode_console_cp(res.stdout)}\n" + f"Stderr:\n{decode_console_cp(res.stderr)}" + ) + + +def run_cmd( + cmd: pathlib.Path, cmd_args: typing.Iterable[str], **kwargs +) -> subprocess.CompletedProcess[bytes]: + return subprocess.run([cmd, *cmd_args], **kwargs) + + +def path_to_unc(path: pathlib.PureWindowsPath) -> pathlib.PureWindowsPath: + """Adds the extended path prefix \\\\?\\ to a path + + Args: + path (pathlib.PureWindowsPath): The path to add the prefix to + + Returns: + pathlib.PureWindowsPath: The path with the prefix added + """ + return pathlib.PureWindowsPath("\\\\?\\" + str(path)) + + +def paths_to_unc( + paths: typing.Iterable[pathlib.PureWindowsPath], +) -> typing.Iterable[pathlib.PureWindowsPath]: + """Adds the extended path prefix \\\\?\\ to multiple paths + + Args: + paths (typing.Iterable[pathlib.PureWindowsPath]): The paths to add the + prefix to + + Returns: + typing.Iterable[pathlib.PureWindowsPath]: The paths with the prefix + added + """ + for path in paths: + yield path_to_unc(path) + + +def zfs_get_cmd( + path: pathlib.PureWindowsPath, cmd: str +) -> pathlib.PureWindowsPath: + """This searches for a ZFS command in the given path. The path can either + point to a ZFS installation or to a ZFS build output directory. + + Args: + path (pathlib.PureWindowsPath): The path to search in + cmd (str): The command to search for + + Returns: + pathlib.PureWindowsPath: Path to the command + """ + + cmd_path = (path / cmd).with_suffix(".exe") + if os.path.isfile(cmd_path): + return cmd_path + + cmd_path = (path / "cmd" / cmd / cmd).with_suffix(".exe") + if os.path.isfile(cmd_path): + return cmd_path + + return None + + +def argparse_as_abspath(path: str) -> pathlib.PureWindowsPath: + """Converts the given path string to an absolute path object + + Args: + path (str): The path to convert + + Raises: + argparse.ArgumentTypeError: When the path is not a directory + + Returns: + pathlib.PureWindowsPath: The absolute path object + """ + + abspath = os.path.abspath(path) + if os.path.isdir(abspath): + return pathlib.PureWindowsPath(abspath) + else: + raise argparse.ArgumentTypeError( + f"{path} is not a valid path, does not exist" + ) + + +def argparse_as_zfs_abspath(path: str) -> pathlib.PureWindowsPath: + """Converts the given path string to an absolute path object + + Args: + path (str): The path to convert + + Raises: + argparse.ArgumentTypeError: When the path is not a directory or not a + ZFS installation or build directory + + Returns: + pathlib.PureWindowsPath: The absolute path object + """ + + abspath = argparse_as_abspath(path) + if zfs_get_cmd(abspath, "zfs"): + return abspath + else: + raise argparse.ArgumentTypeError( + f"{path} is not a valid ZFS path, it does not contain zfs.exe" + ) + + +class ZfsContext: + def __init__(self, zfspath: pathlib.Path): + self.zfspath = zfspath + + self.ZFS = zfs_get_cmd(zfspath, "zfs") + self.ZPOOL = zfs_get_cmd(zfspath, "zpool") + self.ZDB = zfs_get_cmd(zfspath, "zdb") + + +class FILE_STANDARD_INFO(ctypes.Structure): + _fields_ = [ + ("AllocationSize", wintypes.LARGE_INTEGER), + ("EndOfFile", wintypes.LARGE_INTEGER), + ("NumberOfLinks", wintypes.DWORD), + ("DeletePending", wintypes.BOOLEAN), + ("Directory", wintypes.BOOLEAN), + ] + + +class FILE_ALLOCATION_INFO(ctypes.Structure): + _fields_ = [("AllocationSize", wintypes.LARGE_INTEGER)] + + +class IO_STATUS_BLOCK(ctypes.Structure): + _fields_ = [("Status", wintypes.LONG), ("Information", wintypes.PULONG)] + + +class FILE_FS_SIZE_INFORMATION(ctypes.Structure): + _fields_ = [ + ("TotalAllocationUnits", wintypes.LARGE_INTEGER), + ("AvailableAllocationUnits", wintypes.LARGE_INTEGER), + ("SectorsPerAllocationUnit", wintypes.ULONG), + ("BytesPerSector", wintypes.ULONG), + ] + + +class FILE_INFO_BY_HANDLE_CLASS(enum.IntEnum): + FileStandardInfo = 1 + FileAllocationInfo = 5 + + +class FS_INFORMATION_CLASS(enum.IntEnum): + FileFsSizeInformation = 3 + + +def _raise_if_zero(result, func, args): + if result == 0: + raise ctypes.WinError(ctypes.get_last_error()) + + +def _raise_if_ntstatus_nonzero(result, func, args): + if result != 0: + raise ctypes.WinError(_ntdll.RtlNtStatusToDosError(result)) + + +class DLLHelper: + def __init__(self, name): + self.dll_name = name + self.dll = ctypes.WinDLL(name, use_last_error=True) + + def add_import( + self, + name, + argtypes, + restype=wintypes.BOOL, + errcheck=_raise_if_zero, + ): + fn = getattr(self.dll, name) + fn.argtypes = argtypes + fn.restype = restype + fn.errcheck = errcheck + setattr(self, name, fn) + + +_kernel32 = DLLHelper("kernel32") +_kernel32.add_import( + "GetFileInformationByHandleEx", + [wintypes.HANDLE, wintypes.DWORD, wintypes.LPVOID, wintypes.DWORD], +) +_kernel32.add_import( + "SetFileInformationByHandle", + [wintypes.HANDLE, wintypes.DWORD, wintypes.LPVOID, wintypes.DWORD], +) + + +_ntdll = DLLHelper("ntdll") +_ntdll.add_import( + "NtQueryVolumeInformationFile", + [ + wintypes.HANDLE, + IO_STATUS_BLOCK, + wintypes.LPVOID, + wintypes.ULONG, + wintypes.DWORD, + ], + restype=wintypes.LONG, + errcheck=_raise_if_ntstatus_nonzero, +) +_ntdll.add_import( + "RtlNtStatusToDosError", [wintypes.LONG], restype=wintypes.ULONG +) + + +def get_cluster_size_from_file(file: typing.IO): + iosb = IO_STATUS_BLOCK() + fsi = FILE_FS_SIZE_INFORMATION() + + _ntdll.NtQueryVolumeInformationFile( + msvcrt.get_osfhandle(file.fileno()), + iosb, + ctypes.byref(fsi), + ctypes.sizeof(fsi), + FS_INFORMATION_CLASS.FileFsSizeInformation, + ) + + return fsi.SectorsPerAllocationUnit * fsi.BytesPerSector + + +def get_sizes_from_file(file: typing.IO): + si = FILE_STANDARD_INFO() + + _kernel32.GetFileInformationByHandleEx( + msvcrt.get_osfhandle(file.fileno()), + FILE_INFO_BY_HANDLE_CLASS.FileStandardInfo, + ctypes.byref(si), + ctypes.sizeof(si), + ) + + return { + "AllocationSize": si.AllocationSize, + "EndOfFile": si.EndOfFile, + } + + +def get_sizes_from_path(path: pathlib.Path): + with open(path, "rb") as f: + return get_sizes_from_file(f) + + +def preallocate_file_object(file: typing.BinaryIO, size: int): + ai = FILE_ALLOCATION_INFO(size) + + _kernel32.SetFileInformationByHandle( + msvcrt.get_osfhandle(file.fileno()), + FILE_INFO_BY_HANDLE_CLASS.FileAllocationInfo, + ctypes.byref(ai), + ctypes.sizeof(ai), + ) + + +def allocate_file(path: pathlib.Path, size: int): + """Creates a file of the given size + + Args: + name (pathlib.Path): The path to the file + size (int): The size of the file + """ + + with open(path, "wb") as f: + f.seek(size) + f.write(b"0") + + +@contextlib.contextmanager +def allocated_files( + files: typing.Iterable[typing.Tuple[os.PathLike, int]] +) -> typing.List[os.PathLike]: + """Context manager that allocates a file and deletes it when done + + Args: + path (pathlib.Path): The path to the file + size (int): The size of the file + + Yields: + pathlib.Path: Path of the file + """ + + files = list(files) + + for f in files: + allocate_file(f[0], f[1]) + + try: + yield [f[0] for f in files] + finally: + for f in files: + os.remove(f[0]) + + +def options_to_args( + flag: str, options: typing.Dict[str, str] +) -> typing.List[str]: + return [ + f(k, v) + for k, v in options.items() + for f in (lambda _, _2: flag, lambda k, v: f"{k}={v}") + ] + + +class ZpoolInfo: + destroy = True + + def __init__(self, name, mount_path): + self.name = name + self.mount_path = mount_path + + def __str__(self) -> str: + return f"zpool {self.name} mounted to {self.mount_path}" + + +@contextlib.contextmanager +def zpool_create( + ctx: ZfsContext, + name: str, + devices: typing.Iterable[os.PathLike], + zpool_options: typing.Dict[str, str] = {}, + zfs_options: typing.Dict[str, str] = {}, +) -> pathlib.Path: + """Context manager that creates a zpool and destroys it when done + + Args: + devices (typing.Iterable[str]): List of devices or backing files to use + options (typing.Dict[str, str]): zpool options + fs_options (typing.Dict[str, str]): zfs options + size (int): The size of the zpool backing file + + Yields: + pathlib.Path: Path of the file + """ + + if "driveletter" in zfs_options: + dl = zfs_options["driveletter"] + drive_path = pathlib.Path(f"{dl}:\\") + else: + drive_path = get_next_drive_path() + zfs_options["driveletter"] = drive_path.drive.rstrip(":") + + devices = list(devices) + res = run_cmd( + ctx.ZPOOL, + [ + "create", + "-f", + *options_to_args("-o", zpool_options), + *options_to_args("-O", zfs_options), + name, + *[str(d) for d in devices], + ], + ) + if res.returncode != 0: + raise RuntimeError("Failed to create zpool") + + pool_info = ZpoolInfo(name, drive_path) + + try: + yield pool_info + finally: + # TODO: This sleep is to protect against BSOD. Remove this sleep when + # https://github.com/openzfsonwindows/openzfs/issues/282 is fixed + time.sleep(2.0) + if pool_info.destroy: + res = run_cmd(ctx.ZPOOL, ["destroy", "-f", name]) + if res.returncode != 0: + raise RuntimeError("Failed to destroy zpool") + + +def repl(): + """This starts a REPL with the callers globals and locals available + + Raises: + RuntimeError: Is raised when the callers frame is not available + """ + import code + import inspect + + frame = inspect.currentframe() + if not frame: + raise RuntimeError("No caller frame") + + code.interact(local=dict(frame.f_back.f_globals, **frame.f_back.f_locals)) + + +def add_common_arguments(parser: argparse.ArgumentParser): + parser.add_argument("--path", type=argparse_as_abspath, required=True) + + # TODO: We need to verify that the zfs path is actually usable because the + # default path is not passed to `argparse_as_zfs_abspath`. + program_files = pathlib.PureWindowsPath(os.getenv("ProgramFiles")) + default_zfs_path = program_files / "OpenZFS On Windows" + parser.add_argument( + "--zfspath", + type=argparse_as_zfs_abspath, + default=default_zfs_path, + help="Directory path of either an OpenZFS installation or build" + " directory", + ) + + parser.add_argument( + "-np", + "--no_pool", + action="store_true", + default=False, + help="Don't create a zpool, run tests in path", + )