[Testing] Setup SSH tests to cluster (internal testing only)

docker-compose.yml

@@ -0,0 +1,25 @@
+---
+version: "2.1"
+services:
+  openssh-server:
+    image: lscr.io/linuxserver/openssh-server:latest
+    container_name: openssh-server
+    hostname: openssh-server #optional
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=Etc/UTC
+ #     - PUBLIC_KEY=yourpublickey #optional
+  #    - PUBLIC_KEY_FILE=/path/to/file #optional
+   #   - PUBLIC_KEY_DIR=/path/to/directory/containing/_only_/pubkeys #optional
+    #  - PUBLIC_KEY_URL=https://github.com/username.keys #optional
+      - SUDO_ACCESS=true #optional
+      - PASSWORD_ACCESS=true #optional
+      - USER_PASSWORD=datashuttle #optional
+    #  - USER_PASSWORD_FILE=/path/to/file #optional
+      - USER_NAME=linuxserver.io #optional
+    volumes:
+      - /path/to/appdata/config:/config
+    ports:
+      - 2222:2222
+    restart: unless-stopped

tests/configs_for_ssh_tests.yaml

@@ -0,0 +1,10 @@
+'test_ssh':
+  false
+'username':
+  'joeziminski'
+'remote_host_id':
+  'localhost'
+'test_data_filesystem_path':
+  false
+'test_data_server_path':
+  false

tests/conftest.py

@@ -1,46 +1,128 @@
-"""
-Test configs, used for setting up SSH tests.
-
-Before running these tests, it is necessary to setup
-an SSH key. This can be done through datashuttle
-ssh.setup_ssh_key(project.cfg, log=False).
-
-Store this path somewhere outside of the test environment,
-and it will be copied to the project test folder before testing.
-
-FILESYSTEM_PATH and SERVER_PATH these must point
-to the same folder on the HPC, filesystem,
-as a moutned drive and server as the linux path to
-connect through SSH
-"""
-import platform
+import os
+from pathlib import Path
 from types import SimpleNamespace
 
 import pytest
 import test_utils
+import yaml
 
-test_ssh = False
-username = "jziminski"
-remote_host_id = "hpc-gw1.hpc.swc.ucl.ac.uk"
-server_path = r"/ceph/neuroinformatics/neuroinformatics/scratch/datashuttle_tests/fake_data"
+from datashuttle.utils import utils
 
 
-if platform.system() == "Windows":
-    ssh_key_path = r"C:\Users\Joe\.datashuttle\test_file_conflicts_ssh_key"
-    filesystem_path = "X:/neuroinformatics/scratch/datashuttle_tests/fake_data"
+def get_canonical_test_dir_names(initialise=False):
+    """
+    SSH tests are run by copying data from a `local` project folder to a
+    `remote` project folder through SSH. At present, the folder that is
+    SSH too must also be accessible through the local filesystem (e.g. if
+    SSH to a HPC, then the target directly must also be accessible through
+    the local filesystem e.g. through a mounted drive).
+    `test_data_filesystem_path` gives the local filesystem path to the folder,
+    `test_data_server_path` gives the path after SSH connection to the folder.
+    If SSH to `localhost`, both these paths will be the same.
 
-else:
-    ssh_key_path = "/home/joe/test_file_conflicts_ssh_key"
-    filesystem_path = "/home/joe/ceph_mount/neuroinformatics/scratch/datashuttle_tests/fake_data"
+    Returns
+    -------
+
+    save_ssh_key_project_name : Canonical project name used to store SSH key.
+                                The test-project configs are deleted during
+                                tear-down and so any pre-set SSH key pair must
+                                be stored outside of this project.
+                                `save_ssh_key_project_name` stores the project
+                                name used internally for storing the SSH key
+                                setup with `run_to_setup_ssh_test_key.py`
+                                which is copied to the test-project configs
+                                on test set up.
+
+    test_data_filesystem_path : The path to the testing folder on the
+                                filesystem where the `local` and `remote`
+                                project folders be created during tests.
+                                For SSH testing, this should be the local
+                                filesystem path to the server path i.e. a
+                                mounted drive. e.g.
+                                W:/home/user/datashuttle_tests By default
+                                (e.g. used for SSH tests to `localhost` this
+                                is /home/.datashuttle_tests)
+
+    test_data_server_path : Path to the testing folder once SSH connected to
+                            the target server. For example
+                            /home/user/datashuttle_tests. By default (e.g.
+                            used for SSH tests to `localhost` this is
+                            /home/.datashuttle_tests)
+
+    ssh_key_path : Path to the SSH key setup by `run_to_setup_ssh_test_key.py`.
+                   This is copied to the test-project during setup.
+
+    config["test_ssh"] : bool indicating whether SSH tests should be run.
+
+    config["username"] : Username for the account to connect with via SSH.
+
+    config["remote_host_id"] : Target for SSH connection e.g. HPC address,
+                               `localhost` for SSH to host machine. Note
+                               sshing to `localhost` is only available on
+                               macOS and linux.
+    """
+    # Load SSH setup configs
+    config_filepath = (
+        Path(os.path.dirname(os.path.realpath(__file__)))
+        / "configs_for_ssh_tests.yaml"
+    )
+
+    with open(config_filepath) as file:
+        config = yaml.full_load(file)
+
+    # Check pre-setup SSH key-pair is set up
+    save_ssh_key_project_name = "tests_ssh_key_holding_project"
+    ssh_key_path = (
+        utils.get_datashuttle_path(save_ssh_key_project_name)[0]
+        / f"{save_ssh_key_project_name}_ssh_key"
+    )
+
+    if not initialise and config["test_ssh"] and not ssh_key_path.is_file():
+        raise FileNotFoundError(
+            f"Must run `run_to_setup_ssh_test_key` before running "
+            f"SSH tests. {ssh_key_path}"
+        )
+
+    # Load filesystem and remote data paths
+    if not config["test_data_filesystem_path"]:
+        test_data_filesystem_path = Path.home() / ".datashuttle_tests"
+    else:
+        test_data_filesystem_path = config["test_data_filesystem_path"]
+
+    if not config["test_data_server_path"]:
+        test_data_server_path = Path.home() / ".datashuttle_tests"
+    else:
+        test_data_server_path = config["test_data_server_path"]
+
+    return (
+        save_ssh_key_project_name,
+        test_data_filesystem_path,
+        test_data_server_path,
+        ssh_key_path,
+        config["test_ssh"],
+        config["username"],
+        config["remote_host_id"],
+    )
 
 
 def pytest_configure(config):
+
+    (
+        __,
+        test_data_filesystem_path,
+        test_data_server_path,
+        ssh_key_path,
+        test_ssh,
+        username,
+        remote_host_id,
+    ) = get_canonical_test_dir_names()
+
     pytest.ssh_config = SimpleNamespace(
         TEST_SSH=test_ssh,
         SSH_KEY_PATH=ssh_key_path,
         USERNAME=username,
         REMOTE_HOST_ID=remote_host_id,
-        FILESYSTEM_PATH=filesystem_path,  # FILESYSTEM_PATH and SERVER_PATH these must point to the same folder on the HPC, filesystem
-        SERVER_PATH=server_path,  # as a mounted drive and server as the linux path to connect through SSH
+        FILESYSTEM_PATH=str(test_data_filesystem_path),
+        SERVER_PATH=str(test_data_server_path),
     )
     test_utils.set_datashuttle_loggers(disable=True)

tests/setup_ssh_test_keys.py

@@ -0,0 +1,38 @@
+"""
+This script will setup an SSH key pair between the SSH target
+specified in `setup_configs_for_ssh_tests.yaml`. This will require
+one-time password entry.
+
+The SSH key pair is saved to a holding datashuttle project
+specified in `conftest.py`. During setup of an SSH test project,
+this SSH key pair is copied from the holding datashuttle
+project to the SSH test-project. This means that when the test
+project is deleted during tear-down, the SSH key is still available
+for future testing
+"""
+import conftest
+
+from datashuttle.datashuttle import DataShuttle
+
+(
+    save_ssh_key_project_name,
+    test_data_filesystem_path,
+    test_data_server_path,
+    ssh_key_path,
+    __,
+    username,
+    remote_host_id,
+) = conftest.get_canonical_test_dir_names(initialise=True)
+
+project = DataShuttle(save_ssh_key_project_name)
+
+project.make_config_file(
+    test_data_filesystem_path / "local",
+    test_data_server_path / "remote",
+    "ssh",
+    remote_host_id=remote_host_id,
+    remote_host_username=username,
+    use_behav=True,
+)
+
+project.setup_ssh_connection_to_remote_server()

tests/ssh_test_utils.py → tests/tests_integration/ssh_tests/ssh_test_utils.py

@@ -1,11 +1,12 @@
 import builtins
 import copy
+import shutil
 
 from datashuttle.utils import rclone, ssh
 
 
 def setup_project_for_ssh(
-    project, remote_path, remote_host_id, remote_host_username
+    project, remote_path, ssh_config, setup_ssh_connection=True
 ):
     """
     Setup the project configs to use SSH connection
@@ -15,16 +16,21 @@ def setup_project_for_ssh(
         "remote_path",
         remote_path,
     )
-    project.update_config("remote_host_id", remote_host_id)
-    project.update_config("remote_host_username", remote_host_username)
+    project.update_config("remote_host_id", ssh_config.REMOTE_HOST_ID)
+    project.update_config("remote_host_username", ssh_config.USERNAME)
     project.update_config("connection_method", "ssh")
 
-    rclone.setup_remote_as_rclone_target(
-        "ssh",
-        project.cfg,
-        project.cfg.get_rclone_config_name("ssh"),
-        project.cfg.ssh_key_path,
-    )
+    if setup_ssh_connection:
+        setup_hostkeys(project)
+
+        shutil.copy(ssh_config.SSH_KEY_PATH, project.cfg.ssh_key_path)
+
+        rclone.setup_remote_as_rclone_target(
+            "ssh",
+            project.cfg,
+            project.cfg.get_rclone_config_name("ssh"),
+            project.cfg.ssh_key_path,
+        )
 
 
 def setup_mock_input(input_):