From 93f8140a6928751b942e8545e776a406e74b00a6 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 26 Oct 2023 11:21:14 +0200 Subject: [PATCH] Fix zenodo test, drop explicit datatype mapping This isn't supported by the zenodo api anymore --- docs/commands/shed_init.rst | 2 +- docs/commands/training_fill_data_library.rst | 2 - docs/commands/training_init.rst | 2 - planemo/options.py | 12 +----- planemo/training/__init__.py | 2 - planemo/training/tutorial.py | 18 +-------- tests/data/training_datatypes.yaml | 3 -- tests/test_cmd_training_fill_data_library.py | 39 +------------------- tests/test_cmd_training_init.py | 14 ------- tests/test_training.py | 2 - tests/test_training_tutorial.py | 31 ++++------------ 11 files changed, 11 insertions(+), 116 deletions(-) delete mode 100644 tests/data/training_datatypes.yaml diff --git a/docs/commands/shed_init.rst b/docs/commands/shed_init.rst index 1ba24ade3..3ee7f01bc 100644 --- a/docs/commands/shed_init.rst +++ b/docs/commands/shed_init.rst @@ -29,7 +29,7 @@ to manage repositories in a Galaxy Tool Shed. --remote_repository_url TEXT Specify repository remote_repository_url for .shed.yml. --homepage_url TEXT Specify repository homepage_url for .shed.yml. - --category [Assembly|ChIP-seq|Climate Analysis|Combinatorial Selections|Computational chemistry|Constructive Solid Geometry|Convert Formats|Data Export|Data Managers|Data Source|Ecology|Entomology|Epigenetics|Fasta Manipulation|Fastq Manipulation|Flow Cytometry Analysis|Genome annotation|Genome editing|Genome-Wide Association Study|Genomic Interval Operations|GIS|Graphics|Imaging|InteractiveTools|Machine Learning|Metabolomics|Metagenomics|Micro-array Analysis|Molecular Dynamics|Nanopore|Next Gen Mappers|NLP|Ontology Manipulation|Phylogenetics|Proteomics|RNA|SAM|Sequence Analysis|Statistics|Structural Materials Analysis|Synthetic Biology|Systems Biology|Text Manipulation|Tool Dependency Packages|Tool Generators|Transcriptomics|Variant Analysis|Visualization|Web Services] + --category [Assembly|Astronomy|ChIP-seq|Climate Analysis|CLIP-seq|Combinatorial Selections|Computational chemistry|Constructive Solid Geometry|Convert Formats|Data Export|Data Managers|Data Source|Ecology|Entomology|Epigenetics|Fasta Manipulation|Fastq Manipulation|Flow Cytometry Analysis|Genome annotation|Genome editing|Genome-Wide Association Study|Genomic Interval Operations|GIS|Graphics|Imaging|InteractiveTools|Machine Learning|Materials science|Metabolomics|Metagenomics|Micro-array Analysis|Molecular Dynamics|Nanopore|Next Gen Mappers|NLP|Ontology Manipulation|Phylogenetics|Proteomics|RNA|SAM|Sequence Analysis|Statistics|Structural Materials Analysis|Synthetic Biology|Systems Biology|Text Manipulation|Tool Dependency Packages|Tool Generators|Transcriptomics|Variant Analysis|Visualization|Web Services] Specify repository category for .shed.yml (may specify multiple). --owner TEXT Tool Shed repository owner (username). diff --git a/docs/commands/training_fill_data_library.rst b/docs/commands/training_fill_data_library.rst index f9d141429..998ae7697 100644 --- a/docs/commands/training_fill_data_library.rst +++ b/docs/commands/training_fill_data_library.rst @@ -21,7 +21,5 @@ Build training template from workflow. --tutorial_name TEXT Name (directory name) of the tutorial to modify [required] --zenodo_link TEXT Zenodo URL with the input data - --datatypes PATH YAML file with the correspondance between Zenodo - extension and Galaxy datatypes --help Show this message and exit. diff --git a/docs/commands/training_init.rst b/docs/commands/training_init.rst index 8d1b6edde..181d71460 100644 --- a/docs/commands/training_init.rst +++ b/docs/commands/training_init.rst @@ -34,8 +34,6 @@ Build training template from workflow. workflow --workflow_id TEXT ID of the workflow on the Galaxy instance --zenodo_link TEXT Zenodo URL with the input data - --datatypes PATH YAML file with the correspondance between - Zenodo extension and Galaxy datatypes --galaxy_root DIRECTORY Root of development galaxy directory to execute command with. --galaxy_python_version [3|3.7|3.8|3.9|3.10|3.11] diff --git a/planemo/options.py b/planemo/options.py index eaaaf9cb2..180e18615 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -1283,15 +1283,6 @@ def training_tutorial_name_req_option(): return planemo_option("--tutorial_name", required=True, help="Name (directory name) of the tutorial to modify") -def training_datatype_option(): - return planemo_option( - "--datatypes", - type=click.Path(file_okay=True, resolve_path=True), - help="YAML file with the correspondance between Zenodo extension and Galaxy datatypes", - default="shared/datatypes.yaml", - ) - - def training_zenodo_option(): return planemo_option("--zenodo_link", help="Zenodo URL with the input data") @@ -1322,7 +1313,7 @@ def training_tutorial_option(): def training_init_options(): - return _compose(training_topic_option(), training_tutorial_option(), training_datatype_option()) + return _compose(training_topic_option(), training_tutorial_option()) def training_fill_data_library_options(): @@ -1330,7 +1321,6 @@ def training_fill_data_library_options(): training_topic_name_option(), training_tutorial_name_req_option(), training_zenodo_option(), - training_datatype_option(), ) diff --git a/planemo/training/__init__.py b/planemo/training/__init__.py index 558358e6c..9c4a73c2c 100644 --- a/planemo/training/__init__.py +++ b/planemo/training/__init__.py @@ -49,8 +49,6 @@ def check_topic_init_tuto(self): # initiate the tutorial self.tuto = Tutorial(training=self, topic=self.topic) self.tuto.init_from_existing_tutorial(self.kwds["tutorial_name"]) - if "datatypes" in self.kwds: - self.tuto.datatype_fp = self.kwds["datatypes"] if "workflow" in self.kwds: self.tuto.init_wf_fp = self.kwds["workflow"] if "workflow_id" in self.kwds: diff --git a/planemo/training/tutorial.py b/planemo/training/tutorial.py index 94781b0c3..f2a2448a9 100644 --- a/planemo/training/tutorial.py +++ b/planemo/training/tutorial.py @@ -279,7 +279,6 @@ def __init__(self, training, topic, name="new_tuto", title="The new tutorial", z self.init_wf_id = None self.hands_on = True self.slides = False - self.datatype_fp = "" self.set_dir_name() self.init_data_lib() self.body = templates.render( @@ -309,7 +308,6 @@ def init_from_kwds(self, kwds): self.init_wf_id = kwds["workflow_id"] self.hands_on = kwds["hands_on"] self.slides = kwds["slides"] - self.datatype_fp = kwds["datatypes"] self.set_dir_name() self.init_data_lib() @@ -443,9 +441,7 @@ def get_files_from_zenodo(self): files = [] for f in req_res["files"]: - file_dict = {"url": "", "src": "url", "ext": "", "info": self.zenodo_link} - if "type" in f: - file_dict["ext"] = get_galaxy_datatype(f["type"], self.datatype_fp) + file_dict = {"url": "", "src": "url", "ext": "auto", "info": self.zenodo_link} if "links" not in f and "self" not in f["links"]: raise ValueError("No link for file %s" % f) file_dict["url"] = f["links"]["self"] @@ -546,18 +542,6 @@ def create_tutorial(self, ctx): handle.write("---\nlayout: faq-page\n---\n") -def get_galaxy_datatype(z_ext, datatype_fp): - """Get the Galaxy datatype corresponding to a Zenodo file type.""" - g_datatype = "" - datatypes = load_yaml(datatype_fp) - if z_ext in datatypes: - g_datatype = datatypes[z_ext] - if g_datatype == "": - g_datatype = "# Please add a Galaxy datatype or update the shared/datatypes.yaml file" - info(f"Get Galaxy datatypes: {z_ext} --> {g_datatype}") - return g_datatype - - def get_zenodo_record(zenodo_link): """Get the content of a Zenodo record.""" # get the record in the Zenodo link diff --git a/tests/data/training_datatypes.yaml b/tests/data/training_datatypes.yaml deleted file mode 100644 index ed27c75e3..000000000 --- a/tests/data/training_datatypes.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -csv: csv -test: strange_datatype \ No newline at end of file diff --git a/tests/test_cmd_training_fill_data_library.py b/tests/test_cmd_training_fill_data_library.py index 309742854..0af07a427 100644 --- a/tests/test_cmd_training_fill_data_library.py +++ b/tests/test_cmd_training_fill_data_library.py @@ -1,12 +1,6 @@ """Tests for the ``training_fill_data_library`` command.""" -import os -import shutil - from .test_cmd_training_generate_from_wf import create_tutorial_dir -from .test_utils import ( - CliTestCase, - TEST_DATA_DIR, -) +from .test_utils import CliTestCase class CmdTrainingFillDataLibraryTestCase(CliTestCase): @@ -29,20 +23,6 @@ def test_training_fill_data_library_command_tutorial_topic(self): with self._isolate(): topic_n = "test" tuto_n = "test" - datatype = os.path.join(TEST_DATA_DIR, "training_datatypes.yaml") - # not working - create_tutorial_dir(topic_n, tuto_n) - training_fill_data_library_command = [ - "training_fill_data_library", - "--topic_name", - topic_n, - "--tutorial_name", - tuto_n, - "--datatypes", - datatype, - ] - shutil.rmtree("topics") - self._check_exit_code(training_fill_data_library_command, exit_code=1) # working create_tutorial_dir(topic_n, tuto_n) training_fill_data_library_command = [ @@ -51,8 +31,6 @@ def test_training_fill_data_library_command_tutorial_topic(self): topic_n, "--tutorial_name", tuto_n, - "--datatypes", - datatype, ] self._check_exit_code(training_fill_data_library_command, exit_code=0) @@ -62,19 +40,6 @@ def test_training_fill_data_library_command_tutorial_zenodo(self): topic_n = "test" tuto_n = "test" create_tutorial_dir(topic_n, tuto_n) - datatype = os.path.join(TEST_DATA_DIR, "training_datatypes.yaml") - # not working test - training_fill_data_library_command = [ - "training_fill_data_library", - "--topic_name", - topic_n, - "--tutorial_name", - tuto_n, - "--zenodo_link", - "https://zenodo.org/record/1321885", - ] - self._check_exit_code(training_fill_data_library_command, exit_code=1) - # working training_fill_data_library_command = [ "training_fill_data_library", "--topic_name", @@ -83,7 +48,5 @@ def test_training_fill_data_library_command_tutorial_zenodo(self): tuto_n, "--zenodo_link", "https://zenodo.org/record/1321885", - "--datatypes", - datatype, ] self._check_exit_code(training_fill_data_library_command, exit_code=0) diff --git a/tests/test_cmd_training_init.py b/tests/test_cmd_training_init.py index 4048daae8..75b866252 100644 --- a/tests/test_cmd_training_init.py +++ b/tests/test_cmd_training_init.py @@ -79,18 +79,6 @@ def test_training_init_command_tutorial(self): def test_training_init_command_tutorial_zenodo(self): """Test training_init command to create new tutorial with zenodo.""" with self._isolate(): - datatype = os.path.join(TEST_DATA_DIR, "training_datatypes.yaml") - # not working test - training_init_command = [ - "training_init", - "--topic_name", - "test", - "--tutorial_name", - "test", - "--zenodo_link", - "https://zenodo.org/record/1321885", - ] - self._check_exit_code(training_init_command, exit_code=1) # working training_init_command = [ "training_init", @@ -100,8 +88,6 @@ def test_training_init_command_tutorial_zenodo(self): "test", "--zenodo_link", "https://zenodo.org/record/1321885", - "--datatypes", - datatype, ] self._check_exit_code(training_init_command, exit_code=0) diff --git a/tests/test_training.py b/tests/test_training.py index 129560ea6..bb526fcfc 100644 --- a/tests/test_training.py +++ b/tests/test_training.py @@ -13,7 +13,6 @@ TEST_DATA_DIR, ) -datatype_fp = os.path.join(TEST_DATA_DIR, "training_datatypes.yaml") tuto_fp = os.path.join(TEST_DATA_DIR, "training_tutorial.md") tuto_wo_zenodo_fp = os.path.join(TEST_DATA_DIR, "training_tutorial_wo_zenodo.md") zenodo_link = "https://zenodo.org/record/1321885" @@ -40,7 +39,6 @@ "workflow": None, "workflow_id": None, "zenodo_link": None, - "datatypes": os.path.join(TEST_DATA_DIR, "training_datatypes.yaml"), "templates": None, # planemo configuation "conda_auto_init": True, diff --git a/tests/test_training_tutorial.py b/tests/test_training_tutorial.py index fdd1fbfca..e0b26b086 100644 --- a/tests/test_training_tutorial.py +++ b/tests/test_training_tutorial.py @@ -9,7 +9,6 @@ from planemo.training import Training from planemo.training.topic import Topic from planemo.training.tutorial import ( - get_galaxy_datatype, get_hands_on_boxes_from_local_galaxy, get_hands_on_boxes_from_running_galaxy, get_wf_inputs, @@ -21,7 +20,6 @@ from .test_training import ( create_existing_tutorial, CTX, - datatype_fp, KWDS, RUNNABLE, tuto_fp, @@ -36,20 +34,13 @@ training = Training(KWDS) -def test_get_galaxy_datatype() -> None: - """Test :func:`planemo.training.tutorial.get_galaxy_datatype`.""" - assert get_galaxy_datatype("csv", datatype_fp) == "csv" - assert get_galaxy_datatype("test", datatype_fp) == "strange_datatype" - assert "# Please add" in get_galaxy_datatype("unknown", datatype_fp) - - def test_get_zenodo_record() -> None: """Test :func:`planemo.training.tutorial.get_zenodo_record`.""" z_record, req_res = get_zenodo_record(zenodo_link) - file_link_prefix = "https://zenodo.org/api/files/51a1b5db-ff05-4cda-83d4-3b46682f921f" + file_link_prefix = f"https://zenodo.org/api/records/{z_record}" assert z_record == "1321885" assert "files" in req_res - assert req_res["files"][0]["type"] in ["rdata", "csv"] + assert req_res["files"][0]["filename"].endswith(".RData") assert file_link_prefix in req_res["files"][0]["links"]["self"] # check with wrong zenodo link z_record, req_res = get_zenodo_record("https://zenodo.org/api/records/zenodooo") @@ -59,10 +50,10 @@ def test_get_zenodo_record() -> None: # using DOI z_link = "https://doi.org/10.5281/zenodo.1321885" z_record, req_res = get_zenodo_record(z_link) - file_link_prefix = "https://zenodo.org/api/files/51a1b5db-ff05-4cda-83d4-3b46682f921f" + file_link_prefix = f"https://zenodo.org/api/records/{z_record}" assert z_record == "1321885" assert "files" in req_res - assert req_res["files"][0]["type"] in ["rdata", "csv"] + assert req_res["files"][0]["filename"].endswith(".RData") assert file_link_prefix in req_res["files"][0]["links"]["self"] @@ -135,7 +126,6 @@ def test_tutorial_init() -> None: assert not tuto.slides assert tuto.init_wf_id is None assert tuto.init_wf_fp is None - assert tuto.datatype_fp == "" assert "new_tuto" in tuto.dir assert "## Sub-step with **My Tool**" in tuto.body assert tuto.data_lib @@ -157,7 +147,6 @@ def test_tutorial_init_from_kwds() -> None: "workflow": WF_FP, "workflow_id": "id", "zenodo_link": None, - "datatypes": datatype_fp, } tuto = Tutorial(training=training, topic=topic) tuto.init_from_kwds(kwds) @@ -169,7 +158,6 @@ def test_tutorial_init_from_kwds() -> None: assert tuto.slides assert tuto.init_wf_id == "id" assert tuto.init_wf_fp == WF_FP - assert tuto.datatype_fp == datatype_fp assert "my_tuto" in tuto.dir @@ -289,25 +277,21 @@ def test_tutorial_export_workflow_file() -> None: def test_tutorial_get_files_from_zenodo() -> None: """Test :func:`planemo.training.tutorial.tutorial.get_files_from_zenodo`.""" tuto = Tutorial(training=training, topic=topic, zenodo_link=zenodo_link) - tuto.datatype_fp = datatype_fp files, z_record = tuto.get_files_from_zenodo() assert z_record == "1321885" # test links - file_link_prefix = "https://zenodo.org/api/files/51a1b5db-ff05-4cda-83d4-3b46682f921f" - assert file_link_prefix in tuto.zenodo_file_links[0] + assert z_record in tuto.zenodo_file_links[0] # test files dict - assert file_link_prefix in files[0]["url"] + assert z_record in files[0]["url"] assert files[0]["src"] == "url" assert files[0]["info"] == zenodo_link - assert "# Please add" in files[0]["ext"] - assert files[1]["ext"] == "csv" + assert files[1]["url"].endswith(".csv") def test_tutorial_prepare_data_library_from_zenodo() -> None: """Test :func:`planemo.training.tutorial.tutorial.prepare_data_library_from_zenodo`.""" # without zenodo link tuto = Tutorial(training=training, topic=topic) - tuto.datatype_fp = datatype_fp os.makedirs(tuto.wf_dir) tuto.prepare_data_library_from_zenodo() assert os.path.exists(tuto.data_lib_fp) @@ -383,7 +367,6 @@ def test_tutorial_create_tutorial() -> None: "workflow": WF_FP, "workflow_id": None, "zenodo_link": zenodo_link, - "datatypes": datatype_fp, } ) tuto.create_tutorial(CTX)