From 49025853fc49e7b159d214f3b6fe610892c7e735 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Fri, 1 Nov 2024 15:39:43 +0100 Subject: [PATCH 1/7] Add method Client.restoreData --- doc/src/client.rst | 2 ++ src/icat/client.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/doc/src/client.rst b/doc/src/client.rst index 051edac..31aa808 100644 --- a/doc/src/client.rst +++ b/doc/src/client.rst @@ -154,4 +154,6 @@ manages the interaction with an ICAT service as a client. .. automethod:: deleteData + .. automethod:: restoreData + .. _ICAT SOAP Manual: https://repo.icatproject.org/site/icat/server/4.10.0/soap.html diff --git a/src/icat/client.py b/src/icat/client.py index 8c0cf2c..8377515 100644 --- a/src/icat/client.py +++ b/src/icat/client.py @@ -1000,5 +1000,40 @@ def deleteData(self, objs): objs = DataSelection(objs) self.ids.delete(objs) + def restoreData(self, objs): + """Request IDS to restore data. + + Check the status of the data, request a restore if needed and + wait for the restore to complete. + + :param objs: either a dict having some of the keys + `investigationIds`, `datasetIds`, and `datafileIds` + with a list of object ids as value respectively, or a list + of entity objects, or a data selection. + :type objs: :class:`dict`, :class:`list` of + :class:`icat.entity.Entity`, or + :class:`icat.ids.DataSelection` + + .. versionadded:: 1.6.0 + """ + if not self.ids: + raise RuntimeError("no IDS.") + if not isinstance(objs, DataSelection): + objs = DataSelection(objs) + while True: + self.autoRefresh() + status = self.ids.getStatus(objs) + if status == "ONLINE": + break + elif status == "RESTORING": + pass + elif status == "ARCHIVED": + self.ids.restore(objs) + else: + # Should never happen + raise IDSResponseError("unexpected response from " + "IDS getStatus() call: %s" % status) + time.sleep(30) + atexit.register(Client.cleanupall) From 9e4dd032f499428225712b38d41ab8b7358520ac Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Sat, 23 Nov 2024 22:12:31 -0500 Subject: [PATCH 2/7] Log ids.getStatus() calls in test_06_ids.py --- tests/test_06_ids.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/test_06_ids.py b/tests/test_06_ids.py index d9c42ed..8f9a01e 100644 --- a/tests/test_06_ids.py +++ b/tests/test_06_ids.py @@ -3,22 +3,34 @@ import datetime import filecmp +import logging import time import zipfile import pytest import icat +import icat.client import icat.config -from icat.ids import DataSelection +from icat.ids import IDSClient, DataSelection from icat.query import Query from conftest import DummyDatafile, UtcTimezone from conftest import getConfig, tmpSessionId, tmpClient +logger = logging.getLogger(__name__) + + +class LoggingIDSClient(IDSClient): + """Modified version of IDSClient that logs some calls. + """ + def getStatus(self, selection): + status = super().getStatus(selection) + logger.debug("getStatus(%s): %s", selection, status) + return status @pytest.fixture(scope="module") -def client(setupicat): - client, conf = getConfig(ids="mandatory") +def cleanup(setupicat): + client, conf = getConfig(confSection="root", ids="mandatory") client.login(conf.auth, conf.credentials) - yield client + yield query = "SELECT df FROM Datafile df WHERE df.location IS NOT NULL" while True: try: @@ -28,6 +40,13 @@ def client(setupicat): else: break +@pytest.fixture(scope="function") +def client(monkeypatch, cleanup): + monkeypatch.setattr(icat.client, "IDSClient", LoggingIDSClient) + client, conf = getConfig(ids="mandatory") + client.login(conf.auth, conf.credentials) + yield client + # ============================ testdata ============================ From 2faeadc398e4b58242b83a3ce336dccb3655aa46 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Sat, 23 Nov 2024 23:42:22 -0400 Subject: [PATCH 3/7] Add pro forma tests for Client.restoreData() --- tests/test_06_ids.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_06_ids.py b/tests/test_06_ids.py index 8f9a01e..ef1afc6 100644 --- a/tests/test_06_ids.py +++ b/tests/test_06_ids.py @@ -411,6 +411,30 @@ def test_restore(client, case): # outcome of the restore() call. print("Status of dataset %s is now %s" % (case['dsname'], status)) +@pytest.mark.parametrize(("case"), markeddatasets) +def test_restoreData(client, case): + """Test the high level call restoreData(). + + This is essentially a no-op as the dataset in question will + already be ONLINE. It only tests that the call does not throw an + error. + """ + dataset = getDataset(client, case) + client.restoreData([dataset]) + status = client.ids.getStatus(DataSelection([dataset])) + assert status == "ONLINE" + +@pytest.mark.parametrize(("case"), markeddatasets) +def test_restoreDataSelection(client, case): + """Test the high level call restoreData(). + + Same as last test, but now pass a DataSelection as argument. + """ + selection = DataSelection([getDataset(client, case)]) + client.restoreData(selection) + status = client.ids.getStatus(selection) + assert status == "ONLINE" + @pytest.mark.parametrize(("case"), markeddatasets) def test_reset(client, case): """Call reset() on a dataset. From 90b97afd775da4120db3ee07ffc24a4f73b03485 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Sun, 24 Nov 2024 13:01:00 +0100 Subject: [PATCH 4/7] Amend 9e4dd03: log with stacklevel=2 --- tests/test_06_ids.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_06_ids.py b/tests/test_06_ids.py index ef1afc6..bbf95c1 100644 --- a/tests/test_06_ids.py +++ b/tests/test_06_ids.py @@ -23,7 +23,7 @@ class LoggingIDSClient(IDSClient): """ def getStatus(self, selection): status = super().getStatus(selection) - logger.debug("getStatus(%s): %s", selection, status) + logger.debug("getStatus(%s): %s", selection, status, stacklevel=2) return status @pytest.fixture(scope="module") From 20af5b833182ef91f605f84bd4bee45581bef00c Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Sun, 24 Nov 2024 13:43:28 +0100 Subject: [PATCH 5/7] Add a 'slow' test marker and a command line option '--no-skip-slow': slow tests will be skipped by default. --- tests/conftest.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 104901d..9c5a8cf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,9 +48,28 @@ logging.getLogger('suds.client').setLevel(logging.CRITICAL) logging.getLogger('suds').setLevel(logging.ERROR) +_skip_slow = True testdir = Path(__file__).resolve().parent testdatadir = testdir / "data" +def pytest_addoption(parser): + parser.addoption("--no-skip-slow", action="store_true", default=False, + help="do not skip slow tests.") + +def pytest_configure(config): + global _skip_slow + _skip_slow = not config.getoption("--no-skip-slow") + config.addinivalue_line("markers", "slow: mark a test as slow, " + "the test will be skipped unless --no-skip-slow " + "is set on the command line") + +def pytest_runtest_setup(item): + """Skip slow tests by default. + """ + marker = item.get_closest_marker("slow") + if marker is not None and _skip_slow: + pytest.skip("skip slow test") + def _skip(reason): if Version(pytest.__version__) >= '3.3.0': pytest.skip(reason, allow_module_level=True) From 4cb28c5ed97b2da0998b93d24d471d1e0bd561db Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Sun, 24 Nov 2024 18:28:57 +0100 Subject: [PATCH 6/7] Add a real test for Client.restoreData() --- tests/test_06_ids.py | 69 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 8 deletions(-) diff --git a/tests/test_06_ids.py b/tests/test_06_ids.py index bbf95c1..3a4fc3d 100644 --- a/tests/test_06_ids.py +++ b/tests/test_06_ids.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) +GiB = 1073741824 class LoggingIDSClient(IDSClient): """Modified version of IDSClient that logs some calls. @@ -26,12 +27,7 @@ def getStatus(self, selection): logger.debug("getStatus(%s): %s", selection, status, stacklevel=2) return status -@pytest.fixture(scope="module") -def cleanup(setupicat): - client, conf = getConfig(confSection="root", ids="mandatory") - client.login(conf.auth, conf.credentials) - yield - query = "SELECT df FROM Datafile df WHERE df.location IS NOT NULL" +def _delete_datafiles(client, query): while True: try: client.deleteData(client.search(query)) @@ -40,6 +36,14 @@ def cleanup(setupicat): else: break +@pytest.fixture(scope="module") +def cleanup(setupicat): + client, conf = getConfig(confSection="root", ids="mandatory") + client.login(conf.auth, conf.credentials) + yield + query = "SELECT df FROM Datafile df WHERE df.location IS NOT NULL" + _delete_datafiles(client, query) + @pytest.fixture(scope="function") def client(monkeypatch, cleanup): monkeypatch.setattr(icat.client, "IDSClient", LoggingIDSClient) @@ -47,6 +51,30 @@ def client(monkeypatch, cleanup): client.login(conf.auth, conf.credentials) yield client +@pytest.fixture(scope="function") +def dataset(client, cleanup_objs): + """A dataset to be used in the test. + + The dataset will be eventually be deleted after the test. + """ + inv = client.assertedSearch(Query(client, "Investigation", conditions={ + "name": "= '10100601-ST'", + }))[0] + dstype = client.assertedSearch(Query(client, "DatasetType", conditions={ + "name": "= 'raw'", + }))[0] + dataset = client.new("Dataset", + name="e208343", complete=False, + investigation=inv, type=dstype) + dataset.create() + cleanup_objs.append(dataset) + yield dataset + query = Query(client, "Datafile", conditions={ + "dataset.id": "= %d" % dataset.id, + "location": "IS NOT NULL", + }) + _delete_datafiles(client, query) + # ============================ testdata ============================ @@ -412,7 +440,7 @@ def test_restore(client, case): print("Status of dataset %s is now %s" % (case['dsname'], status)) @pytest.mark.parametrize(("case"), markeddatasets) -def test_restoreData(client, case): +def test_restoreDataCall(client, case): """Test the high level call restoreData(). This is essentially a no-op as the dataset in question will @@ -425,7 +453,7 @@ def test_restoreData(client, case): assert status == "ONLINE" @pytest.mark.parametrize(("case"), markeddatasets) -def test_restoreDataSelection(client, case): +def test_restoreDataCallSelection(client, case): """Test the high level call restoreData(). Same as last test, but now pass a DataSelection as argument. @@ -435,6 +463,31 @@ def test_restoreDataSelection(client, case): status = client.ids.getStatus(selection) assert status == "ONLINE" +@pytest.mark.slow +def test_restoreData(tmpdirsec, client, dataset): + """Test restoring data with the high level call restoreData(). + + This test archives a dataset and calls restoreData() to restore it + again. The size of the dataset is large enough so that restoring + takes some time, so that we actually can observe the call to wait + until the restoring is finished. As a result, the test is rather + slow. It is marked as such and thus disabled by default. + """ + if not client.ids.isTwoLevel(): + pytest.skip("This IDS does not use two levels of data storage") + f = DummyDatafile(tmpdirsec, "e208343.nxs", GiB) + query = Query(client, "DatafileFormat", conditions={ + "name": "= 'NeXus'", + }) + datafileformat = client.assertedSearch(query)[0] + datafile = client.new("Datafile", name=f.fname.name, + dataset=dataset, datafileFormat=datafileformat) + client.putData(f.fname, datafile) + client.ids.archive(DataSelection([dataset])) + client.restoreData([dataset]) + status = client.ids.getStatus(DataSelection([dataset])) + assert status == "ONLINE" + @pytest.mark.parametrize(("case"), markeddatasets) def test_reset(client, case): """Call reset() on a dataset. From 9912d0c53e10d5d70a762a296a772f78ea44996c Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Wed, 27 Nov 2024 10:44:40 +0100 Subject: [PATCH 7/7] Update changelog --- CHANGES.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 5e18f09..8fa640e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,20 @@ Changelog ========= +.. _changes-1_6_0: + +1.6.0 (not yet released) +~~~~~~~~~~~~~~~~~~~~~~~~ + +New features +------------ + ++ `166`_: Add new custom IDS method + :meth:`icat.client.Client.restoreData`. + +.. _166: https://github.com/icatproject/python-icat/pull/166 + + .. _changes-1_5_1: 1.5.1 (2024-10-25)