From 908151863c8ef5e42ef7f4628b4dd21fe686b954 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 18 Nov 2024 12:08:15 -0500 Subject: [PATCH 01/17] Typing of tool request expansion stuff. --- .../dataset_collections/subcollections.py | 20 ++++++++++++++-- lib/galaxy/tools/parameters/meta.py | 24 +++++++++++++++---- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/model/dataset_collections/subcollections.py b/lib/galaxy/model/dataset_collections/subcollections.py index af6c2a397326..fc799b1711ad 100644 --- a/lib/galaxy/model/dataset_collections/subcollections.py +++ b/lib/galaxy/model/dataset_collections/subcollections.py @@ -1,12 +1,28 @@ +from typing import ( + List, + TYPE_CHECKING, +) + from galaxy import exceptions +if TYPE_CHECKING: + from galaxy.model import ( + DatasetCollection, + DatasetCollectionElement, + HistoryDatasetCollectionAssociation, + ) + -def split_dataset_collection_instance(dataset_collection_instance, collection_type): +def split_dataset_collection_instance( + dataset_collection_instance: "HistoryDatasetCollectionAssociation", collection_type: str +) -> List["DatasetCollectionElement"]: """Split up collection into collection.""" return _split_dataset_collection(dataset_collection_instance.collection, collection_type) -def _split_dataset_collection(dataset_collection, collection_type): +def _split_dataset_collection( + dataset_collection: "DatasetCollection", collection_type: str +) -> List["DatasetCollectionElement"]: this_collection_type = dataset_collection.collection_type if not this_collection_type.endswith(collection_type) or this_collection_type == collection_type: raise exceptions.MessageException("Cannot split collection in desired fashion.") diff --git a/lib/galaxy/tools/parameters/meta.py b/lib/galaxy/tools/parameters/meta.py index b74df54fa269..5865a6698110 100644 --- a/lib/galaxy/tools/parameters/meta.py +++ b/lib/galaxy/tools/parameters/meta.py @@ -8,13 +8,20 @@ List, Optional, Tuple, + Union, ) from galaxy import ( exceptions, util, ) -from galaxy.model import HistoryDatasetCollectionAssociation +from galaxy.model import ( + DatasetCollectionElement, + DatasetInstance, + HistoryDatasetAssociation, + HistoryDatasetCollectionAssociation, + LibraryDatasetDatasetAssociation, +) from galaxy.model.dataset_collections import ( matching, subcollections, @@ -327,7 +334,12 @@ def visitor(input, value, prefix, prefixed_name, prefixed_label, error, **kwargs return (single_inputs_nested, matched_multi_inputs, multiplied_multi_inputs) -def __expand_collection_parameter(trans, input_key, incoming_val, collections_to_match, linked=False): +CollectionExpansionListT = Union[List[DatasetCollectionElement], List[DatasetInstance]] + + +def __expand_collection_parameter( + trans, input_key: str, incoming_val, collections_to_match: "matching.CollectionsToMatch", linked=False +) -> CollectionExpansionListT: # If subcollectin multirun of data_collection param - value will # be "hdca_id|subcollection_type" else it will just be hdca_id if "|" in incoming_val: @@ -348,10 +360,12 @@ def __expand_collection_parameter(trans, input_key, incoming_val, collections_to raise exceptions.ToolInputsNotReadyException("An input collection is not populated.") collections_to_match.add(input_key, hdc, subcollection_type=subcollection_type, linked=linked) if subcollection_type is not None: - subcollection_elements = subcollections.split_dataset_collection_instance(hdc, subcollection_type) + subcollection_elements: List[DatasetCollectionElement] = subcollections.split_dataset_collection_instance( + hdc, subcollection_type + ) return subcollection_elements else: - hdas = [] + hdas: List[DatasetInstance] = [] for element in hdc.collection.dataset_elements: hda = element.dataset_instance hda.element_identifier = element.element_identifier @@ -359,7 +373,7 @@ def __expand_collection_parameter(trans, input_key, incoming_val, collections_to return hdas -def __collection_multirun_parameter(value): +def __collection_multirun_parameter(value: Dict[str, Any]) -> bool: is_batch = value.get("batch", False) if not is_batch: return False From 2dc07324ace42d09247fcd108d18a79e30974033 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 14 Nov 2024 10:42:57 -0500 Subject: [PATCH 02/17] Fix unit test names. --- test/unit/tool_util/test_parameter_convert.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/unit/tool_util/test_parameter_convert.py b/test/unit/tool_util/test_parameter_convert.py index 7f5deff41e85..38efa9105aed 100644 --- a/test/unit/tool_util/test_parameter_convert.py +++ b/test/unit/tool_util/test_parameter_convert.py @@ -33,7 +33,7 @@ } -def test_encode_data(): +def test_decode_data(): tool_source = tool_source_for("parameters/gx_data") bundle = input_models_for_tool_source(tool_source) request_state = RequestToolState({"parameter": {"src": "hda", "id": EXAMPLE_ID_1_ENCODED}}) @@ -53,7 +53,7 @@ def test_encode_collection(): assert decoded_state.input_state["parameter"]["id"] == EXAMPLE_ID_1 -def test_encode_repeat(): +def test_decode_repeat(): tool_source = tool_source_for("parameters/gx_repeat_data") bundle = input_models_for_tool_source(tool_source) request_state = RequestToolState({"parameter": [{"data_parameter": {"src": "hda", "id": EXAMPLE_ID_1_ENCODED}}]}) @@ -63,7 +63,7 @@ def test_encode_repeat(): assert decoded_state.input_state["parameter"][0]["data_parameter"]["id"] == EXAMPLE_ID_1 -def test_encode_section(): +def test_decode_section(): tool_source = tool_source_for("parameters/gx_section_data") bundle = input_models_for_tool_source(tool_source) request_state = RequestToolState({"parameter": {"data_parameter": {"src": "hda", "id": EXAMPLE_ID_1_ENCODED}}}) @@ -73,7 +73,7 @@ def test_encode_section(): assert decoded_state.input_state["parameter"]["data_parameter"]["id"] == EXAMPLE_ID_1 -def test_encode_conditional(): +def test_decode_conditional(): tool_source = tool_source_for("identifier_in_conditional") bundle = input_models_for_tool_source(tool_source) request_state = RequestToolState( From 5a8ec8d4a83452d936e445fd530294872efdc330 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 18 Nov 2024 15:49:11 -0500 Subject: [PATCH 03/17] Migration for tool request implicit collections. --- ...d7bf6ac02_tool_request_implicit_outputs.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 lib/galaxy/model/migrations/alembic/versions_gxy/1d1d7bf6ac02_tool_request_implicit_outputs.py diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/1d1d7bf6ac02_tool_request_implicit_outputs.py b/lib/galaxy/model/migrations/alembic/versions_gxy/1d1d7bf6ac02_tool_request_implicit_outputs.py new file mode 100644 index 000000000000..ace78cc898c8 --- /dev/null +++ b/lib/galaxy/model/migrations/alembic/versions_gxy/1d1d7bf6ac02_tool_request_implicit_outputs.py @@ -0,0 +1,59 @@ +"""Track tool request implicit output collections. + +Revision ID: 1d1d7bf6ac02 +Revises: 75348cfb3715 +Create Date: 2024-11-18 15:39:42.900327 + +""" +from sqlalchemy import ( + Column, + Integer, + String, +) + +from galaxy.model.migrations.util import ( + create_foreign_key, + create_table, + drop_table, + transaction, +) + + +# revision identifiers, used by Alembic. +revision = '1d1d7bf6ac02' +down_revision = '75348cfb3715' +branch_labels = None +depends_on = None + +association_table_name = "ToolRequestImplicitCollectionAssociation" + + +def upgrade(): + with transaction(): + create_table( + association_table_name, + Column("id", Integer, primary_key=True), + Column("tool_request_id", Integer, index=True), + Column("dataset_collection_id", Integer, index=True), + Column("output_name", String(255), nullable=False), + ) + + create_foreign_key( + "fk_trica_tri", + association_table_name, + "tool_request", + ["tool_request_id"], + ["id"], + ) + + create_foreign_key( + "fk_trica_dci", + association_table_name, + "history_dataset_collection_association", + ["dataset_collection_id"], + ["id"], + ) + + +def downgrade(): + drop_table(association_table_name) From 5c4f4f32e77da54f9920f4d704ca49e8d4c249b1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 18 Nov 2024 12:09:41 -0500 Subject: [PATCH 04/17] Tool Request API... --- .github/workflows/framework_tools.yaml | 3 +- lib/galaxy/app.py | 9 +- lib/galaxy/celery/tasks.py | 19 +- lib/galaxy/managers/jobs.py | 134 ++++++++++- lib/galaxy/model/__init__.py | 24 ++ lib/galaxy/schema/jobs.py | 13 + lib/galaxy/schema/schema.py | 1 + lib/galaxy/schema/tasks.py | 13 + lib/galaxy/tool_util/parameters/convert.py | 28 ++- lib/galaxy/tool_util/parameters/models.py | 1 + lib/galaxy/tool_util/verify/_types.py | 11 +- lib/galaxy/tool_util/verify/interactor.py | 222 +++++++++++++++--- lib/galaxy/tool_util/verify/parse.py | 43 +++- lib/galaxy/tools/__init__.py | 155 +++++++++++- lib/galaxy/tools/_types.py | 6 + lib/galaxy/tools/execute.py | 118 +++++++++- lib/galaxy/tools/parameters/__init__.py | 184 +++++++++++++++ lib/galaxy/tools/parameters/basic.py | 3 +- lib/galaxy/tools/parameters/meta.py | 103 +++++++- lib/galaxy/webapps/galaxy/api/histories.py | 12 + lib/galaxy/webapps/galaxy/api/jobs.py | 25 +- lib/galaxy/webapps/galaxy/api/tools.py | 85 ++++++- lib/galaxy/webapps/galaxy/services/base.py | 20 +- .../webapps/galaxy/services/histories.py | 9 + lib/galaxy/webapps/galaxy/services/jobs.py | 134 ++++++++++- lib/galaxy/webapps/galaxy/services/tools.py | 79 ++++--- lib/galaxy_test/api/conftest.py | 2 +- lib/galaxy_test/api/test_tool_execute.py | 116 ++++++--- lib/galaxy_test/api/test_tool_execution.py | 135 +++++++++++ lib/galaxy_test/base/populators.py | 112 +++++++-- test/functional/test_toolbox_pytest.py | 10 +- test/unit/tool_util/test_parameter_convert.py | 26 +- 32 files changed, 1671 insertions(+), 184 deletions(-) create mode 100644 lib/galaxy_test/api/test_tool_execution.py diff --git a/.github/workflows/framework_tools.yaml b/.github/workflows/framework_tools.yaml index a55dfa316488..3ae88c7a7eda 100644 --- a/.github/workflows/framework_tools.yaml +++ b/.github/workflows/framework_tools.yaml @@ -26,6 +26,7 @@ jobs: strategy: matrix: python-version: ['3.8'] + use-legacy-api: ['if_needed', 'always'] services: postgres: image: postgres:13 @@ -63,7 +64,7 @@ jobs: path: 'galaxy root/.venv' key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-framework-tools - name: Run tests - run: ./run_tests.sh --coverage --framework-tools + run: GALAXY_TEST_USE_LEGACY_TOOL_API="${{ matrix.use-legacy-api }}" ./run_tests.sh --coverage --framework-tools working-directory: 'galaxy root' - uses: codecov/codecov-action@v3 with: diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 2076ef2599e8..e4baca865f02 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -674,6 +674,10 @@ def __init__(self, configure_logging=True, use_converters=True, use_display_appl self._register_singleton(Registry, self.datatypes_registry) galaxy.model.set_datatypes_registry(self.datatypes_registry) self.configure_sentry_client() + # Load dbkey / genome build manager + self._configure_genome_builds(data_table_name="__dbkeys__", load_old_style=True) + # Tool Data Tables + self._configure_tool_data_tables(from_shed_config=False) self._configure_tool_shed_registry() self._register_singleton(tool_shed_registry.Registry, self.tool_shed_registry) @@ -752,11 +756,6 @@ def __init__(self, **kwargs) -> None: ) self.api_keys_manager = self._register_singleton(ApiKeyManager) - # Tool Data Tables - self._configure_tool_data_tables(from_shed_config=False) - # Load dbkey / genome build manager - self._configure_genome_builds(data_table_name="__dbkeys__", load_old_style=True) - # Genomes self.genomes = self._register_singleton(Genomes) # Data providers registry. diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index 35b133fd5845..1dbc71408b6b 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -31,6 +31,7 @@ DatasetManager, ) from galaxy.managers.hdas import HDAManager +from galaxy.managers.jobs import JobSubmitter from galaxy.managers.lddas import LDDAManager from galaxy.managers.markdown_util import generate_branded_pdf from galaxy.managers.model_stores import ModelStoreManager @@ -57,6 +58,7 @@ MaterializeDatasetInstanceTaskRequest, PrepareDatasetCollectionDownload, PurgeDatasetsTaskRequest, + QueueJobs, SetupHistoryExportJob, WriteHistoryContentTo, WriteHistoryTo, @@ -78,8 +80,10 @@ def setup_data_table_manager(app): @lru_cache -def cached_create_tool_from_representation(app: MinimalManagerApp, raw_tool_source: str): - return create_tool_from_representation(app=app, raw_tool_source=raw_tool_source, tool_source_class="XmlToolSource") +def cached_create_tool_from_representation(app: MinimalManagerApp, raw_tool_source: str, tool_dir: str = ""): + return create_tool_from_representation( + app=app, raw_tool_source=raw_tool_source, tool_dir=tool_dir, tool_source_class="XmlToolSource" + ) @galaxy_task(action="recalculate a user's disk usage") @@ -336,6 +340,17 @@ def fetch_data( return abort_when_job_stops(_fetch_data, session=sa_session, job_id=job_id, setup_return=setup_return) +@galaxy_task(action="queuing up submitted jobs") +def queue_jobs(request: QueueJobs, app: MinimalManagerApp, job_submitter: JobSubmitter): + tool = cached_create_tool_from_representation( + app, request.tool_source.raw_tool_source, tool_dir=request.tool_source.tool_dir + ) + job_submitter.queue_jobs( + tool, + request, + ) + + @galaxy_task(ignore_result=True, action="setting up export history job") def export_history( model_store_manager: ModelStoreManager, diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index 1ef47ff34f46..bde5bfcdf735 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -1,5 +1,6 @@ import json import logging +from dataclasses import dataclass from datetime import ( date, datetime, @@ -11,6 +12,7 @@ Dict, List, Optional, + Tuple, Union, ) @@ -50,14 +52,20 @@ ProvidesUserContext, ) from galaxy.managers.datasets import DatasetManager -from galaxy.managers.hdas import HDAManager +from galaxy.managers.hdas import ( + dereference_input, + HDAManager, +) +from galaxy.managers.histories import HistoryManager from galaxy.managers.lddas import LDDAManager +from galaxy.managers.users import UserManager from galaxy.model import ( ImplicitCollectionJobs, ImplicitCollectionJobsJobAssociation, Job, JobMetricNumeric, JobParameter, + ToolRequest, User, Workflow, WorkflowInvocation, @@ -75,8 +83,23 @@ JobIndexQueryPayload, JobIndexSortByEnum, ) +from galaxy.schema.tasks import ( + MaterializeDatasetInstanceTaskRequest, + QueueJobs, +) from galaxy.security.idencoding import IdEncodingHelper -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import ( + MinimalManagerApp, + StructuredApp, +) +from galaxy.tool_util.parameters import ( + DataRequestInternalHda, + DataRequestUri, + dereference, + RequestInternalDereferencedToolState, + RequestInternalToolState, +) +from galaxy.tools import Tool from galaxy.tools._types import ( ToolStateDumpedToJsonInternalT, ToolStateJobInstancePopulatedT, @@ -92,6 +115,7 @@ parse_filters_structured, RawTextTerm, ) +from galaxy.work.context import WorkRequestContext log = logging.getLogger(__name__) @@ -144,6 +168,8 @@ def index_query(self, trans: ProvidesUserContext, payload: JobIndexQueryPayload) workflow_id = payload.workflow_id invocation_id = payload.invocation_id implicit_collection_jobs_id = payload.implicit_collection_jobs_id + tool_request_id = payload.tool_request_id + search = payload.search order_by = payload.order_by @@ -160,6 +186,7 @@ def build_and_apply_filters(stmt, objects, filter_func): def add_workflow_jobs(): wfi_step = select(WorkflowInvocationStep) + if workflow_id is not None: wfi_step = ( wfi_step.join(WorkflowInvocation).join(Workflow).where(Workflow.stored_workflow_id == workflow_id) @@ -174,6 +201,7 @@ def add_workflow_jobs(): ImplicitCollectionJobsJobAssociation.implicit_collection_jobs_id == wfi_step_sq.c.implicit_collection_jobs_id, ) + # Ensure the result is models, not tuples sq = stmt1.union(stmt2).subquery() # SQLite won't recognize Job.foo as a valid column for the ORDER BY clause due to the UNION clause, so we'll use the subquery `columns` collection (`sq.c`). @@ -251,6 +279,9 @@ def add_search_criteria(stmt): if history_id is not None: stmt = stmt.where(Job.history_id == history_id) + if tool_request_id is not None: + stmt = stmt.filter(model.Job.tool_request_id == tool_request_id) + order_by_columns = Job if workflow_id or invocation_id: stmt, order_by_columns = add_workflow_jobs() @@ -1250,3 +1281,102 @@ def get_jobs_to_check_at_startup(session: galaxy_scoped_session, track_jobs_in_d def get_job(session, *where_clauses): stmt = select(Job).where(*where_clauses).limit(1) return session.scalars(stmt).first() + + +@dataclass +class DereferencedDatasetPair: + hda: model.HistoryDatasetAssociation + request: DataRequestUri + + +class JobSubmitter: + def __init__( + self, + history_manager: HistoryManager, + user_manager: UserManager, + hda_manager: HDAManager, + app: MinimalManagerApp, + ): + self.history_manager = history_manager + self.user_manager = user_manager + self.hda_manager = hda_manager + self.app = app + + def materialize_request_for( + self, trans: WorkRequestContext, hda: model.HistoryDatasetAssociation + ) -> MaterializeDatasetInstanceTaskRequest: + return MaterializeDatasetInstanceTaskRequest( + user=trans.async_request_user, + history_id=trans.history.id, + source="hda", + content=hda.id, + ) + + def dereference( + self, trans: WorkRequestContext, tool: Tool, request: QueueJobs, tool_request: ToolRequest + ) -> Tuple[RequestInternalDereferencedToolState, List[DereferencedDatasetPair]]: + new_hdas: List[DereferencedDatasetPair] = [] + + def dereference_callback(data_request: DataRequestUri) -> DataRequestInternalHda: + # a deferred dataset corresponding to request + hda = dereference_input(trans, data_request) + new_hdas.append(DereferencedDatasetPair(hda, data_request)) + return DataRequestInternalHda(id=hda.id) + + tool_state = RequestInternalToolState(tool_request.request) + return dereference(tool_state, tool, dereference_callback), new_hdas + + def queue_jobs(self, tool: Tool, request: QueueJobs) -> None: + tool_request: ToolRequest = self._tool_request(request.tool_request_id) + sa_session = self.app.model.context + try: + request_context = self._context(tool_request, request) + target_history = request_context.history + use_cached_jobs = request.use_cached_jobs + rerun_remap_job_id = request.rerun_remap_job_id + tool_state: RequestInternalDereferencedToolState + new_hdas: List[DereferencedDatasetPair] + tool_state, new_hdas = self.dereference(request_context, tool, request, tool_request) + to_materialize_list: List[DereferencedDatasetPair] = [p for p in new_hdas if not p.request.deferred] + for to_materialize in to_materialize_list: + materialize_request = self.materialize_request_for(request_context, to_materialize.hda) + # API dataset materialization is immutable and produces new datasets + # here we just created the datasets - lets just materialize them in place + # and avoid extra and confusing input copies + self.hda_manager.materialize(materialize_request, in_place=True) + tool.handle_input_async( + request_context, + tool_request, + tool_state, + history=target_history, + use_cached_job=use_cached_jobs, + rerun_remap_job_id=rerun_remap_job_id, + ) + tool_request.state = ToolRequest.states.SUBMITTED + sa_session.add(tool_request) + with transaction(sa_session): + sa_session.commit() + except Exception as e: + log.exception("Problem here....") + tool_request.state = ToolRequest.states.FAILED + tool_request.state_message = str(e) + sa_session.add(tool_request) + with transaction(sa_session): + sa_session.commit() + + def _context(self, tool_request: ToolRequest, request: QueueJobs) -> WorkRequestContext: + user = self.user_manager.by_id(request.user.user_id) + target_history = tool_request.history + trans = WorkRequestContext( + self.app, + user, + history=target_history, + ) + return trans + + def _tool_request(self, tool_request_id: int) -> ToolRequest: + sa_session = self.app.model.context + tool_request: ToolRequest = cast(ToolRequest, sa_session.query(ToolRequest).get(tool_request_id)) + if tool_request is None: + raise Exception(f"Problem fetching request with ID {tool_request_id}") + return tool_request diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index bd5d59384efd..4193652c44e4 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1356,6 +1356,29 @@ class ToolRequest(Base, Dictifiable, RepresentById): tool_source: Mapped["ToolSource"] = relationship() history: Mapped[Optional["History"]] = relationship(back_populates="tool_requests") + implicit_collections: Mapped[List["ToolRequestImplicitCollectionAssociation"]] = relationship(back_populates="tool_request") + + +class ToolRequestImplicitCollectionAssociation(Base, Dictifiable, RepresentById): + __tablename__ = "tool_request_implicit_collection_association" + + id: Mapped[int] = mapped_column(primary_key=True) + tool_request_id: Mapped[int] = mapped_column( + ForeignKey("tool_request.id", name="fk_trica_tri"), index=True + ) + dataset_collection_id: Mapped[int] = mapped_column( + ForeignKey("history_dataset_collection_association.id", name="fk_trica_dci"), index=True + ) + output_name: Mapped[str] = mapped_column(String(255)) + + tool_request: Mapped["ToolRequest"] = relationship( + back_populates="implicit_collections" + ) + dataset_collection: Mapped["HistoryDatasetCollectionAssociation"] = relationship( + back_populates="tool_request_association", uselist=False + ) + + dict_collection_visible_keys = ["id", "tool_request_id", "dataset_collection_id", "output_name"] class DynamicTool(Base, Dictifiable, RepresentById): @@ -7046,6 +7069,7 @@ class HistoryDatasetCollectionAssociation( back_populates="dataset_collection", ) creating_job_associations: Mapped[List["JobToOutputDatasetCollectionAssociation"]] = relationship(viewonly=True) + tool_request_association: Mapped[Optional["ToolRequestImplicitCollectionAssociation"]] = relationship(back_populates="dataset_collection") dict_dbkeysandextensions_visible_keys = ["dbkeys", "extensions"] editable_keys = ("name", "deleted", "visible") diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py index fbca9e281a2d..339f86ae22bf 100644 --- a/lib/galaxy/schema/jobs.py +++ b/lib/galaxy/schema/jobs.py @@ -82,6 +82,19 @@ class JobOutputAssociation(JobAssociation): ) +class JobOutputCollectionAssociation(Model): + name: str = Field( + default=..., + title="name", + description="Name of the job parameter.", + ) + dataset_collection_instance: EncodedDataItemSourceId = Field( + default=..., + title="dataset_collection_instance", + description="Reference to the associated item.", + ) + + class ReportJobErrorPayload(Model): dataset_id: DecodedDatabaseIdField = Field( default=..., diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 3febee546896..3185d9903f49 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -1531,6 +1531,7 @@ class JobIndexQueryPayload(Model): workflow_id: Optional[DecodedDatabaseIdField] = None invocation_id: Optional[DecodedDatabaseIdField] = None implicit_collection_jobs_id: Optional[DecodedDatabaseIdField] = None + tool_request_id: Optional[DecodedDatabaseIdField] = None order_by: JobIndexSortByEnum = JobIndexSortByEnum.update_time search: Optional[str] = None limit: int = 500 diff --git a/lib/galaxy/schema/tasks.py b/lib/galaxy/schema/tasks.py index 313ec9f1dad0..31255d968268 100644 --- a/lib/galaxy/schema/tasks.py +++ b/lib/galaxy/schema/tasks.py @@ -119,3 +119,16 @@ class ComputeDatasetHashTaskRequest(Model): class PurgeDatasetsTaskRequest(Model): dataset_ids: List[int] + + +class ToolSource(Model): + raw_tool_source: str + tool_dir: str + + +class QueueJobs(Model): + tool_source: ToolSource + tool_request_id: int # links to request ("incoming") and history + user: RequestUser # TODO: test anonymous users through this submission path + use_cached_jobs: bool + rerun_remap_job_id: Optional[int] # link to a job to rerun & remap diff --git a/lib/galaxy/tool_util/parameters/convert.py b/lib/galaxy/tool_util/parameters/convert.py index 63dbb9ab58b9..981d2439390d 100644 --- a/lib/galaxy/tool_util/parameters/convert.py +++ b/lib/galaxy/tool_util/parameters/convert.py @@ -357,18 +357,27 @@ def encode_src_dict(src_dict: dict): else: return src_dict + def encode_element(element: dict): + if element.get("__class__") == "Batch": + encoded = element.copy() + values = encoded.pop("values") + encoded["values"] = list(map(encode_src_dict, values)) + return encoded + else: + return encode_src_dict(element) + def encode_callback(parameter: ToolParameterT, value: Any): if parameter.parameter_type == "gx_data": data_parameter = cast(DataParameterModel, parameter) if data_parameter.multiple: assert isinstance(value, list), str(value) - return list(map(encode_src_dict, value)) + return list(map(encode_element, value)) else: assert isinstance(value, dict), str(value) - return encode_src_dict(value) + return encode_element(value) elif parameter.parameter_type == "gx_data_collection": assert isinstance(value, dict), str(value) - return encode_src_dict(value) + return encode_element(value) else: return VISITOR_NO_REPLACEMENT @@ -385,6 +394,15 @@ def decode_src_dict(src_dict: dict): else: return src_dict + def decode_element(element: dict): + if element.get("__class__") == "Batch": + decoded = element.copy() + values = decoded.pop("values") + decoded["values"] = list(map(decode_src_dict, values)) + return decoded + else: + return decode_src_dict(element) + def decode_callback(parameter: ToolParameterT, value: Any): if parameter.parameter_type == "gx_data": if value is None: @@ -392,10 +410,10 @@ def decode_callback(parameter: ToolParameterT, value: Any): data_parameter = cast(DataParameterModel, parameter) if data_parameter.multiple: assert isinstance(value, list), str(value) - return list(map(decode_src_dict, value)) + return list(map(decode_element, value)) else: assert isinstance(value, dict), str(value) - return decode_src_dict(value) + return decode_element(value) elif parameter.parameter_type == "gx_data_collection": if value is None: return VISITOR_NO_REPLACEMENT diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py index d0e7e6bbcb8f..0fc92d244dbb 100644 --- a/lib/galaxy/tool_util/parameters/models.py +++ b/lib/galaxy/tool_util/parameters/models.py @@ -118,6 +118,7 @@ def allow_batching(job_template: DynamicModelInformation, batch_type: Optional[T class BatchRequest(StrictModel): meta_class: Literal["Batch"] = Field(..., alias="__class__") values: List[batch_type] # type: ignore[valid-type] + linked: Optional[bool] = None # maybe True instead? request_type = union_type([job_py_type, BatchRequest]) diff --git a/lib/galaxy/tool_util/verify/_types.py b/lib/galaxy/tool_util/verify/_types.py index e5aa85f1ddb7..c532dab9aa69 100644 --- a/lib/galaxy/tool_util/verify/_types.py +++ b/lib/galaxy/tool_util/verify/_types.py @@ -19,10 +19,15 @@ ToolSourceTestOutputs, ) -# inputs that have been processed with parse.py and expanded out +# legacy inputs for working with POST /api/tools +# + inputs that have been processed with parse.py and expanded out ExpandedToolInputs = Dict[str, Any] -# ExpandedToolInputs where any model objects have been json-ified with to_dict() +# + ExpandedToolInputs where any model objects have been json-ified with to_dict() ExpandedToolInputsJsonified = Dict[str, Any] + +# modern inputs for working with POST /api/jobs* +RawTestToolRequest = Dict[str, Any] + ExtraFileInfoDictT = Dict[str, Any] RequiredFileTuple = Tuple[str, ExtraFileInfoDictT] RequiredFilesT = List[RequiredFileTuple] @@ -36,6 +41,8 @@ class ToolTestDescriptionDict(TypedDict): name: str test_index: int inputs: ExpandedToolInputsJsonified + request: NotRequired[Optional[Dict[str, Any]]] + request_schema: NotRequired[Optional[Dict[str, Any]]] outputs: ToolSourceTestOutputs output_collections: List[TestSourceTestOutputColllection] stdout: Optional[AssertionList] diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py index ca186d823ebb..9da6f1f93d76 100644 --- a/lib/galaxy/tool_util/verify/interactor.py +++ b/lib/galaxy/tool_util/verify/interactor.py @@ -35,8 +35,18 @@ ) from galaxy import util +from galaxy.tool_util.parameters import ( + DataCollectionRequest, + DataRequestHda, + encode_test, + input_models_from_json, + TestCaseToolState, + ToolParameterBundle, +) from galaxy.tool_util.parser.interface import ( AssertionList, + JsonTestCollectionDefDict, + JsonTestDatasetDefDict, TestCollectionDef, TestCollectionOutputDef, TestSourceTestOutputColllection, @@ -53,6 +63,7 @@ from ._types import ( ExpandedToolInputs, ExpandedToolInputsJsonified, + RawTestToolRequest, RequiredDataTablesT, RequiredFilesT, RequiredLocFileT, @@ -63,6 +74,9 @@ log = getLogger(__name__) +UseLegacyApiT = Literal["always", "never", "if_needed"] +DEFAULT_USE_LEGACY_API: UseLegacyApiT = "always" + # Off by default because it can pound the database pretty heavily # and result in sqlite errors on larger tests or larger numbers of # tests. @@ -102,6 +116,8 @@ def __getitem__(self, item): class ValidToolTestDict(TypedDict): inputs: ExpandedToolInputs + request: NotRequired[Optional[RawTestToolRequest]] + request_schema: NotRequired[Optional[Dict[str, Any]]] outputs: ToolSourceTestOutputs output_collections: List[TestSourceTestOutputColllection] stdout: NotRequired[AssertionList] @@ -148,7 +164,7 @@ def stage_data_in_history( # Upload any needed files upload_waits = [] - assert tool_id + assert tool_id, "Tool id not set" if UPLOAD_ASYNC: for test_data in all_test_data: @@ -236,6 +252,15 @@ def get_tests_summary(self): assert response.status_code == 200, f"Non 200 response from tool tests available API. [{response.content}]" return response.json() + def get_tool_inputs(self, tool_id: str, tool_version: Optional[str] = None) -> ToolParameterBundle: + url = f"tools/{tool_id}/inputs" + params = {"tool_version": tool_version} if tool_version else None + response = self._get(url, data=params) + assert response.status_code == 200, f"Non 200 response from tool inputs API. [{response.content}]" + raw_inputs_array = response.json() + tool_parameter_bundle = input_models_from_json(raw_inputs_array) + return tool_parameter_bundle + def get_tool_tests(self, tool_id: str, tool_version: Optional[str] = None) -> List[ToolTestDescriptionDict]: url = f"tools/{tool_id}/test_data" params = {"tool_version": tool_version} if tool_version else None @@ -366,9 +391,27 @@ def wait_for_content(): def wait_for_job(self, job_id: str, history_id: Optional[str] = None, maxseconds=DEFAULT_TOOL_TEST_WAIT) -> None: self.wait_for(lambda: self.__job_ready(job_id, history_id), maxseconds=maxseconds) + def wait_on_tool_request(self, tool_request_id: str): + def state(): + state_response = self._get(f"tool_requests/{tool_request_id}/state") + state_response.raise_for_status() + return state_response.json() + + def is_ready(): + is_complete = state() in ["submitted", "failed"] + return True if is_complete else None + + self.wait_for(is_ready, "waiting for tool request to submit") + return state() == "submitted" + + def get_tool_request(self, tool_request_id: str): + response_raw = self._get(f"tool_requests/{tool_request_id}") + response_raw.raise_for_status() + return response_raw.json() + def wait_for(self, func: Callable, what: str = "tool test run", **kwd) -> None: walltime_exceeded = int(kwd.get("maxseconds", DEFAULT_TOOL_TEST_WAIT)) - wait_on(func, what, walltime_exceeded) + return wait_on(func, what, walltime_exceeded) def get_job_stdio(self, job_id: str) -> Dict[str, Any]: return self.__get_job_stdio(job_id).json() @@ -564,8 +607,9 @@ def stage_data_async( files["files_0|file_data"] = file_content name = os.path.basename(name) tool_input["files_0|NAME"] = name + # upload1 will always be the legacy API... submit_response_object = self.__submit_tool( - history_id, "upload1", tool_input, extra_data={"type": "upload_dataset"}, files=files + history_id, "upload1", tool_input, extra_data={"type": "upload_dataset"}, files=files, use_legacy_api=True ) submit_response = ensure_tool_run_response_okay(submit_response_object, f"upload dataset {name}") assert ( @@ -590,39 +634,71 @@ def _ensure_valid_location_in(self, test_data: dict) -> Optional[str]: return location def run_tool( - self, testdef: "ToolTestDescription", history_id: str, resource_parameters: Optional[Dict[str, Any]] = None + self, + testdef: "ToolTestDescription", + history_id: str, + resource_parameters: Optional[Dict[str, Any]] = None, + use_legacy_api: UseLegacyApiT = DEFAULT_USE_LEGACY_API, ) -> RunToolResponse: # We need to handle the case where we've uploaded a valid compressed file since the upload # tool will have uncompressed it on the fly. resource_parameters = resource_parameters or {} - inputs_tree = testdef.inputs.copy() - for key, value in inputs_tree.items(): - values = [value] if not isinstance(value, list) else value - new_values = [] - for value in values: - if isinstance(value, TestCollectionDef): - hdca_id = self._create_collection(history_id, value) - new_values = [dict(src="hdca", id=hdca_id)] - elif value in self.uploads: - new_values.append(self.uploads[value]) - else: - new_values.append(value) - inputs_tree[key] = new_values + request = testdef.request + request_schema = testdef.request_schema + submit_with_legacy_api = use_legacy_api == "always" or (use_legacy_api == "if_needed" and request is None) + if submit_with_legacy_api: + inputs_tree = testdef.inputs.copy() + for key, value in inputs_tree.items(): + values = [value] if not isinstance(value, list) else value + new_values = [] + for value in values: + if isinstance(value, TestCollectionDef): + hdca_id = self._create_collection(history_id, value) + new_values = [dict(src="hdca", id=hdca_id)] + elif value in self.uploads: + new_values.append(self.uploads[value]) + else: + new_values.append(value) + inputs_tree[key] = new_values + + # HACK: Flatten single-value lists. Required when using expand_grouping + for key, value in inputs_tree.items(): + if isinstance(value, list) and len(value) == 1: + inputs_tree[key] = value[0] + else: + assert request is not None, "Request not set" + assert request_schema is not None, "Request schema not set" + parameters = request_schema["parameters"] + + def adapt_datasets(test_input: JsonTestDatasetDefDict) -> DataRequestHda: + # if path is not set it might be a composite file with a path, + # e.g. composite_shapefile + test_input_path = test_input.get("path", "") + return DataRequestHda(**self.uploads[test_input_path]) + + def adapt_collections(test_input: JsonTestCollectionDefDict) -> DataCollectionRequest: + test_collection_def = TestCollectionDef.from_dict(test_input) + hdca_id = self._create_collection(history_id, test_collection_def) + return DataCollectionRequest(src="hdca", id=hdca_id) + + test_case_state = TestCaseToolState(input_state=request) + inputs_tree = encode_test( + test_case_state, input_models_from_json(parameters), adapt_datasets, adapt_collections + ).input_state if resource_parameters: inputs_tree["__job_resource|__job_resource__select"] = "yes" for key, value in resource_parameters.items(): inputs_tree[f"__job_resource|{key}"] = value - # HACK: Flatten single-value lists. Required when using expand_grouping - for key, value in inputs_tree.items(): - if isinstance(value, list) and len(value) == 1: - inputs_tree[key] = value[0] - submit_response = None for _ in range(DEFAULT_TOOL_TEST_WAIT): submit_response = self.__submit_tool( - history_id, tool_id=testdef.tool_id, tool_input=inputs_tree, tool_version=testdef.tool_version + history_id, + tool_id=testdef.tool_id, + tool_input=inputs_tree, + tool_version=testdef.tool_version, + use_legacy_api=submit_with_legacy_api, ) if _are_tool_inputs_not_ready(submit_response): print("Tool inputs not ready yet") @@ -631,12 +707,38 @@ def run_tool( else: break submit_response_object = ensure_tool_run_response_okay(submit_response, "execute tool", inputs_tree) + if not submit_with_legacy_api: + tool_request_id = submit_response_object["tool_request_id"] + successful = self.wait_on_tool_request(tool_request_id) + if not successful: + request = self.get_tool_request(tool_request_id) or {} + raise RunToolException( + f"Tool request failure - state {request.get('state')}, message: {request.get('state_message')}", + inputs_tree, + ) + jobs = self.jobs_for_tool_request(tool_request_id) + outputs = OutputsDict() + output_collections = {} + if len(jobs) != 1: + raise Exception(f"Found incorrect number of jobs for tool request - was expecting a single job {jobs}") + assert len(jobs) == 1, jobs + job_id = jobs[0]["id"] + job_outputs = self.job_outputs(job_id) + for job_output in job_outputs: + if "dataset" in job_output: + outputs[job_output["name"]] = job_output["dataset"] + else: + output_collections[job_output["name"]] = job_output["dataset_collection_instance"] + else: + outputs = self.__dictify_outputs(submit_response_object) + output_collections = self.__dictify_output_collections(submit_response_object) + jobs = submit_response_object["jobs"] try: return RunToolResponse( inputs=inputs_tree, - outputs=self.__dictify_outputs(submit_response_object), - output_collections=self.__dictify_output_collections(submit_response_object), - jobs=submit_response_object["jobs"], + outputs=outputs, + output_collections=output_collections, + jobs=jobs, ) except KeyError: message = ( @@ -774,14 +876,24 @@ def format_for_summary(self, blob, empty_message, prefix="| "): contents = "\n".join(f"{prefix}{line.strip()}" for line in io.StringIO(blob).readlines() if line.rstrip("\n\r")) return contents or f"{prefix}*{empty_message}*" - def _dataset_provenance(self, history_id, id): + def _dataset_provenance(self, history_id: str, id: str): provenance = self._get(f"histories/{history_id}/contents/{id}/provenance").json() return provenance - def _dataset_info(self, history_id, id): + def _dataset_info(self, history_id: str, id: str): dataset_json = self._get(f"histories/{history_id}/contents/{id}").json() return dataset_json + def jobs_for_tool_request(self, tool_request_id: str) -> List[Dict[str, Any]]: + job_list_response = self._get("jobs", data={"tool_request_id": tool_request_id}) + job_list_response.raise_for_status() + return job_list_response.json() + + def job_outputs(self, job_id: str) -> List[Dict[str, Any]]: + outputs = self._get(f"jobs/{job_id}/outputs") + outputs.raise_for_status() + return outputs.json() + def __contents(self, history_id): history_contents_response = self._get(f"histories/{history_id}/contents") history_contents_response.raise_for_status() @@ -798,12 +910,33 @@ def _state_ready(self, job_id: str, error_msg: str): ) return None - def __submit_tool(self, history_id, tool_id, tool_input, extra_data=None, files=None, tool_version=None): + def __submit_tool( + self, + history_id, + tool_id, + tool_input, + extra_data=None, + files=None, + tool_version=None, + use_legacy_api: bool = True, + ): extra_data = extra_data or {} - data = dict( - history_id=history_id, tool_id=tool_id, inputs=dumps(tool_input), tool_version=tool_version, **extra_data - ) - return self._post("tools", files=files, data=data) + if use_legacy_api: + data = dict( + history_id=history_id, + tool_id=tool_id, + inputs=dumps(tool_input), + tool_version=tool_version, + **extra_data, + ) + return self._post("tools", files=files, data=data) + else: + assert files is None + data = dict( + history_id=history_id, tool_id=tool_id, inputs=tool_input, tool_version=tool_version, **extra_data + ) + submit_tool_request_response = self._post("jobs", data=data, json=True) + return submit_tool_request_response def ensure_user_with_email(self, email, password=None): admin_key = self.master_api_key @@ -1315,6 +1448,7 @@ def verify_tool( register_job_data: Optional[JobDataCallbackT] = None, test_index: int = 0, tool_version: Optional[str] = None, + use_legacy_api: UseLegacyApiT = DEFAULT_USE_LEGACY_API, quiet: bool = False, test_history: Optional[str] = None, no_history_cleanup: bool = False, @@ -1331,11 +1465,7 @@ def verify_tool( if client_test_config is None: client_test_config = NullClientTestConfig() tool_test_dicts = _tool_test_dicts or galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) - tool_test_dict = tool_test_dicts[test_index] - if "test_index" not in tool_test_dict: - tool_test_dict["test_index"] = test_index - if "tool_id" not in tool_test_dict: - tool_test_dict["tool_id"] = tool_id + tool_test_dict: ToolTestDescriptionDict = tool_test_dicts[test_index] if tool_version is None and "tool_version" in tool_test_dict: tool_version = tool_test_dict.get("tool_version") @@ -1400,7 +1530,9 @@ def verify_tool( input_staging_exception = e raise try: - tool_response = galaxy_interactor.run_tool(testdef, test_history, resource_parameters=resource_parameters) + tool_response = galaxy_interactor.run_tool( + testdef, test_history, resource_parameters=resource_parameters, use_legacy_api=use_legacy_api + ) data_list, jobs, tool_inputs = tool_response.outputs, tool_response.jobs, tool_response.inputs data_collection_list = tool_response.output_collections except RunToolException as e: @@ -1685,6 +1817,8 @@ def adapt_tool_source_dict(processed_dict: ToolTestDict) -> ToolTestDescriptionD expect_test_failure: bool = DEFAULT_EXPECT_TEST_FAILURE inputs: ExpandedToolInputsJsonified = {} maxseconds: Optional[int] = None + request: Optional[Dict[str, Any]] = None + request_schema: Optional[Dict[str, Any]] = None if not error_in_test_definition: processed_test_dict = cast(ValidToolTestDict, processed_dict) @@ -1710,6 +1844,8 @@ def adapt_tool_source_dict(processed_dict: ToolTestDict) -> ToolTestDescriptionD expect_failure = processed_test_dict.get("expect_failure", DEFAULT_EXPECT_FAILURE) expect_test_failure = processed_test_dict.get("expect_test_failure", DEFAULT_EXPECT_TEST_FAILURE) inputs = processed_test_dict.get("inputs", {}) + request = processed_test_dict.get("request", None) + request_schema = processed_test_dict.get("request_schema", None) else: invalid_test_dict = cast(InvalidToolTestDict, processed_dict) maxseconds = DEFAULT_TOOL_TEST_WAIT @@ -1737,6 +1873,8 @@ def adapt_tool_source_dict(processed_dict: ToolTestDict) -> ToolTestDescriptionD expect_failure=expect_failure, expect_test_failure=expect_test_failure, inputs=inputs, + request=request, + request_schema=request_schema, ) @@ -1799,6 +1937,8 @@ class ToolTestDescription: expect_test_failure: bool exception: Optional[str] inputs: ExpandedToolInputs + request: Optional[Dict[str, Any]] + request_schema: Optional[Dict[str, Any]] outputs: ToolSourceTestOutputs output_collections: List[TestCollectionOutputDef] maxseconds: Optional[int] @@ -1827,6 +1967,8 @@ def __init__(self, json_dict: ToolTestDescriptionDict): self.expect_failure = json_dict.get("expect_failure", DEFAULT_EXPECT_FAILURE) self.expect_test_failure = json_dict.get("expect_test_failure", DEFAULT_EXPECT_TEST_FAILURE) self.inputs = expanded_inputs_from_json(json_dict.get("inputs", {})) + self.request = json_dict.get("request", None) + self.request_schema = json_dict.get("request_schema", None) self.tool_id = json_dict["tool_id"] self.tool_version = json_dict.get("tool_version") self.maxseconds = _get_maxseconds(json_dict) @@ -1859,6 +2001,8 @@ def to_dict(self) -> ToolTestDescriptionDict: "required_files": self.required_files, "required_data_tables": self.required_data_tables, "required_loc_files": self.required_loc_files, + "request": self.request, + "request_schema": self.request_schema, "error": self.error, "exception": self.exception, "maxseconds": self.maxseconds, diff --git a/lib/galaxy/tool_util/verify/parse.py b/lib/galaxy/tool_util/verify/parse.py index 8dd51f525944..110dc9a93628 100644 --- a/lib/galaxy/tool_util/verify/parse.py +++ b/lib/galaxy/tool_util/verify/parse.py @@ -1,8 +1,10 @@ import logging import os import traceback +from dataclasses import dataclass from typing import ( Any, + Dict, Iterable, List, Optional, @@ -15,6 +17,8 @@ from galaxy.tool_util.parameters import ( input_models_for_tool_source, test_case_state as case_state, + TestCaseToolState, + ToolParameterBundleModel, ) from galaxy.tool_util.parser.interface import ( InputSource, @@ -65,15 +69,18 @@ def parse_tool_test_descriptions( profile = tool_source.parse_profile() for i, raw_test_dict in enumerate(raw_tests_dict.get("tests", [])): validation_exception: Optional[Exception] = None - if validate_on_load: + request_and_schema: Optional[TestRequestAndSchema] = None + try: tool_parameter_bundle = input_models_for_tool_source(tool_source) - try: - case_state(raw_test_dict, tool_parameter_bundle.parameters, profile, validate=True) - except Exception as e: - # TOOD: restrict types of validation exceptions a bit probably? - validation_exception = e + validated_test_case = case_state(raw_test_dict, tool_parameter_bundle.parameters, profile, validate=True) + request_and_schema = TestRequestAndSchema( + validated_test_case.tool_state, + tool_parameter_bundle, + ) + except Exception as e: + validation_exception = e - if validation_exception: + if validation_exception and validate_on_load: tool_id, tool_version = _tool_id_and_version(tool_source, tool_guid) test = ToolTestDescription.from_tool_source_dict( InvalidToolTestDict( @@ -89,13 +96,23 @@ def parse_tool_test_descriptions( ) ) else: - test = _description_from_tool_source(tool_source, raw_test_dict, i, tool_guid) + test = _description_from_tool_source(tool_source, raw_test_dict, i, tool_guid, request_and_schema) tests.append(test) return tests +@dataclass +class TestRequestAndSchema: + request: TestCaseToolState + request_schema: ToolParameterBundleModel + + def _description_from_tool_source( - tool_source: ToolSource, raw_test_dict: ToolSourceTest, test_index: int, tool_guid: Optional[str] + tool_source: ToolSource, + raw_test_dict: ToolSourceTest, + test_index: int, + tool_guid: Optional[str], + request_and_schema: Optional[TestRequestAndSchema], ) -> ToolTestDescription: required_files: RequiredFilesT = [] required_data_tables: RequiredDataTablesT = [] @@ -108,6 +125,12 @@ def _description_from_tool_source( if maxseconds is not None: maxseconds = int(maxseconds) + request: Optional[Dict[str, Any]] = None + request_schema: Optional[Dict[str, Any]] = None + if request_and_schema: + request = request_and_schema.request.input_state + request_schema = request_and_schema.request_schema.dict() + tool_id, tool_version = _tool_id_and_version(tool_source, tool_guid) processed_test_dict: Union[ValidToolTestDict, InvalidToolTestDict] try: @@ -122,6 +145,8 @@ def _description_from_tool_source( processed_test_dict = ValidToolTestDict( { "inputs": processed_inputs, + "request": request, + "request_schema": request_schema, "outputs": raw_test_dict["outputs"], "output_collections": raw_test_dict["output_collections"], "num_outputs": num_outputs, diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 2e352e7c0f39..48d779c38e81 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -49,6 +49,7 @@ from galaxy.model import ( Job, StoredWorkflow, + ToolRequest, ) from galaxy.model.base import transaction from galaxy.model.dataset_collections.matching import MatchingCollections @@ -71,6 +72,13 @@ expand_ontology_data, ) from galaxy.tool_util.output_checker import DETECTED_JOB_STATE +from galaxy.tool_util.parameters import ( + fill_static_defaults, + input_models_for_pages, + JobInternalToolState, + RequestInternalDereferencedToolState, + ToolParameterBundle, +) from galaxy.tool_util.parser import ( get_tool_source, get_tool_source_from_representation, @@ -118,7 +126,7 @@ from galaxy.tools.evaluation import global_tool_errors from galaxy.tools.execution_helpers import ToolExecutionCache from galaxy.tools.imp_exp import JobImportHistoryArchiveWrapper -from galaxy.tools.parameters import ( +from galaxy.tools.parameters import ( # fill_dynamic_defaults, check_param, params_from_strings, params_to_incoming, @@ -126,6 +134,7 @@ params_to_json_internal, params_to_strings, populate_state, + populate_state_async, visit_input_values, ) from galaxy.tools.parameters.basic import ( @@ -152,7 +161,10 @@ UploadDataset, ) from galaxy.tools.parameters.input_translation import ToolInputTranslator -from galaxy.tools.parameters.meta import expand_meta_parameters +from galaxy.tools.parameters.meta import ( + expand_meta_parameters, + expand_meta_parameters_async, +) from galaxy.tools.parameters.populate_model import populate_model from galaxy.tools.parameters.workflow_utils import workflow_building_modes from galaxy.tools.parameters.wrapped_json import json_wrap @@ -200,6 +212,7 @@ ToolStateDumpedToJsonT, ToolStateJobInstancePopulatedT, ToolStateJobInstanceT, + ToolStateJobInstanceExpansionT, ) from .execute import ( DatasetCollectionElementsSliceT, @@ -208,7 +221,8 @@ DEFAULT_RERUN_REMAP_JOB_ID, DEFAULT_SET_OUTPUT_HID, DEFAULT_USE_CACHED_JOB, - execute as execute_job, + execute as execute_sync, + execute_async, ExecutionSlice, JobCallbackT, MappingParameters, @@ -762,7 +776,7 @@ class _Options(Bunch): refresh: str -class Tool(UsesDictVisibleKeys): +class Tool(UsesDictVisibleKeys, ToolParameterBundle): """ Represents a computational tool that can be executed through Galaxy. """ @@ -1431,6 +1445,11 @@ def parse_inputs(self, tool_source: ToolSource): self.inputs: Dict[str, Union[Group, ToolParameter]] = {} pages = tool_source.parse_input_pages() enctypes: Set[str] = set() + try: + parameters = input_models_for_pages(pages, self.profile) + self.parameters = parameters + except Exception: + pass if pages.inputs_defined: if hasattr(pages, "input_elem"): input_elem = pages.input_elem @@ -1823,6 +1842,64 @@ def visit_inputs(self, values, callback): if self.check_values: visit_input_values(self.inputs, values, callback) + def expand_incoming_async( + self, + request_context: WorkRequestContext, + tool_request_internal_state: RequestInternalDereferencedToolState, + rerun_remap_job_id: Optional[int], + ) -> Tuple[ + List[ToolStateJobInstancePopulatedT], + List[ToolStateJobInstancePopulatedT], + Optional[MatchingCollections], + List[JobInternalToolState], + ]: + """The tool request API+tasks version of expand_incoming. + + This is responsible for breaking the map over job requests into individual jobs for execution. + """ + if self.input_translator: + raise exceptions.RequestParameterInvalidException( + "Failure executing tool request with id '%s' (cannot validate inputs from this type of data source tool - please POST to /api/tools).", + self.id, + ) + + set_dataset_matcher_factory(request_context, self) + + expanded_incomings: List[ToolStateJobInstanceExpansionT] + job_tool_states: List[ToolStateJobInstanceT] + collection_info: Optional[MatchingCollections] + expanded_incomings, job_tool_states, collection_info = expand_meta_parameters_async( + request_context.app, self, tool_request_internal_state + ) + + self._ensure_expansion_is_valid(job_tool_states, rerun_remap_job_id) + + # Process incoming data + validation_timer = self.app.execution_timer_factory.get_timer( + "internals.galaxy.tools.validation", + "Validated and populated state for tool request", + ) + all_errors = [] + all_params: List[ToolStateJobInstancePopulatedT] = [] + internal_states: List[JobInternalToolState] = [] + for expanded_incoming, job_tool_state in zip(expanded_incomings, job_tool_states): + log.info(f"expanded_incoming before fill static defaults: {expanded_incoming}") + expanded_incoming = fill_static_defaults(expanded_incoming, self, self.profile) + job_tool_state = fill_static_defaults(job_tool_state, self, self.profile) + log.info(f"expanded_incoming before populate: {expanded_incoming}") + params, errors = self._populate_async(request_context, expanded_incoming) + log.info(f"expanded_incoming after: {expanded_incoming}") + internal_tool_state = JobInternalToolState(job_tool_state) + internal_tool_state.validate(self) + + internal_states.append(internal_tool_state) + all_errors.append(errors) + all_params.append(params) + unset_dataset_matcher_factory(request_context) + + log.info(validation_timer) + return all_params, all_errors, collection_info, internal_states + def expand_incoming( self, request_context: WorkRequestContext, incoming: ToolRequestT, input_format: InputFormatT = "legacy" ) -> Tuple[ @@ -1836,7 +1913,7 @@ def expand_incoming( # Fixed set of input parameters may correspond to any number of jobs. # Expand these out to individual parameters for given jobs (tool executions). - expanded_incomings: List[ToolStateJobInstanceT] + expanded_incomings: List[ToolStateJobInstanceExpansionT] collection_info: Optional[MatchingCollections] expanded_incomings, collection_info = expand_meta_parameters( request_context, self, incoming, input_format=input_format @@ -1862,7 +1939,9 @@ def expand_incoming( return all_params, all_errors, rerun_remap_job_id, collection_info def _ensure_expansion_is_valid( - self, expanded_incomings: List[ToolStateJobInstanceT], rerun_remap_job_id: Optional[int] + self, + expanded_incomings: Union[List[JobInternalToolState], List[ToolStateJobInstanceT]], + rerun_remap_job_id: Optional[int], ) -> None: """If the request corresponds to multiple jobs but this doesn't work with request configuration - raise an error. @@ -1911,6 +1990,33 @@ def _populate( self._handle_validate_input_hook(request_context, params, errors) return params, errors + def _populate_async( + self, request_context, expanded_incoming: ToolStateJobInstanceT + ) -> Tuple[ToolStateJobInstancePopulatedT, ParameterValidationErrorsT]: + """Validate expanded parameters for a job to replace references with model objects. + + So convert a ToolStateJobInstanceT to a ToolStateJobInstancePopulatedT. + """ + params: ToolStateJobInstancePopulatedT = {} + errors: ParameterValidationErrorsT = {} + if self.input_translator: + self.input_translator.translate(expanded_incoming) + if not self.check_values: + # If `self.check_values` is false we don't do any checking or + # processing on input This is used to pass raw values + # through to/from external sites. + params = cast(ToolStateJobInstancePopulatedT, expanded_incoming) + else: + populate_state_async( + request_context, + self.inputs, + expanded_incoming, + params, + errors, + ) + self._handle_validate_input_hook(request_context, params, errors) + return params, errors + def _handle_validate_input_hook( self, request_context, params: ToolStateJobInstancePopulatedT, errors: ParameterValidationErrorsT ): @@ -1944,6 +2050,39 @@ def completed_jobs( completed_jobs[i] = None return completed_jobs + def handle_input_async( + self, + request_context: WorkRequestContext, + tool_request: ToolRequest, + tool_state: RequestInternalDereferencedToolState, + history: Optional[model.History] = None, + use_cached_job: bool = DEFAULT_USE_CACHED_JOB, + preferred_object_store_id: Optional[str] = DEFAULT_PREFERRED_OBJECT_STORE_ID, + rerun_remap_job_id: Optional[int] = None, + input_format: str = "legacy", + ): + """The tool request API+tasks version of handle_input.""" + all_params, all_errors, collection_info, job_tool_states = self.expand_incoming_async( + request_context, tool_state, rerun_remap_job_id + ) + self.handle_incoming_errors(all_errors) + + mapping_params = MappingParameters(tool_request.request, all_params, tool_state, job_tool_states) + completed_jobs: Dict[int, Optional[model.Job]] = self.completed_jobs( + request_context, use_cached_job, all_params + ) + execute_async( + request_context, + self, + mapping_params, + request_context.history, + tool_request, + completed_jobs, + rerun_remap_job_id=rerun_remap_job_id, + preferred_object_store_id=preferred_object_store_id, + collection_info=collection_info, + ) + def handle_input( self, trans, @@ -1969,9 +2108,9 @@ def handle_input( # If there were errors, we stay on the same page and display them self.handle_incoming_errors(all_errors) - mapping_params = MappingParameters(incoming, all_params) + mapping_params = MappingParameters(incoming, all_params, None, None) completed_jobs: Dict[int, Optional[model.Job]] = self.completed_jobs(trans, use_cached_job, all_params) - execution_tracker = execute_job( + execution_tracker = execute_sync( trans, self, mapping_params, diff --git a/lib/galaxy/tools/_types.py b/lib/galaxy/tools/_types.py index 635a86cf459d..f55566fd443f 100644 --- a/lib/galaxy/tools/_types.py +++ b/lib/galaxy/tools/_types.py @@ -8,6 +8,8 @@ +================================+============+=================================+============+===========+ | ToolRequestT | request | src dicts of encoded ids | nope | | | ToolStateJobInstanceT | a job | src dicts of encoded ids | nope | | +| ToolStateJobInstanceExpansionT | a job | a mix I think, things that were | nope | | +| | | expanded are objects | nope | | | ToolStateJobInstancePopulatedT | a job | model objs loaded from db | check_param | | | ToolStateDumpedToJsonT | a job | src dicts of encoded ids | " | | | | | (normalized into values attr) | " | | @@ -35,6 +37,10 @@ # been "checked" (check_param has not been called). ToolStateJobInstanceT = Dict[str, Any] +# After meta.expand_incoming stuff I think expanded parameters are in model object form but the other stuff is likely +# still encoded IDs? None of this is verified though. +ToolStateJobInstanceExpansionT = Dict[str, Any] + # Input dictionary for an individual job where objects are their model objects and parameters have been # "checked" (check_param has been called). ToolStateJobInstancePopulatedT = Dict[str, Any] diff --git a/lib/galaxy/tools/execute.py b/lib/galaxy/tools/execute.py index 31369c948052..b651cf7e36c8 100644 --- a/lib/galaxy/tools/execute.py +++ b/lib/galaxy/tools/execute.py @@ -24,12 +24,17 @@ from galaxy import model from galaxy.exceptions import ToolInputsNotOKException +from galaxy.model import ToolRequest from galaxy.model.base import transaction from galaxy.model.dataset_collections.matching import MatchingCollections from galaxy.model.dataset_collections.structure import ( get_structure, tool_output_to_structure, ) +from galaxy.tool_util.parameters.state import ( + JobInternalToolState, + RequestInternalDereferencedToolState, +) from galaxy.tool_util.parser import ToolOutputCollectionPart from galaxy.tools.execution_helpers import ( filter_output, @@ -69,8 +74,58 @@ def __init__(self, execution_tracker: "ExecutionTracker"): class MappingParameters(NamedTuple): + # the raw request - might correspond to multiple jobs param_template: ToolRequestT + # parameters corresponding to individual job param_combinations: List[ToolStateJobInstancePopulatedT] + # schema driven parameters + # model validated tool request - might correspond to multiple jobs + validated_param_template: Optional[RequestInternalDereferencedToolState] = None + # validated job parameters for individual jobs + validated_param_combinations: Optional[List[JobInternalToolState]] = None + + def ensure_validated(self): + assert self.validated_param_template is not None + assert self.validated_param_combinations is not None + + +def execute_async( + trans, + tool: "Tool", + mapping_params: MappingParameters, + history: model.History, + tool_request: ToolRequest, + completed_jobs: Optional[CompletedJobsT] = None, + rerun_remap_job_id: Optional[int] = None, + preferred_object_store_id: Optional[str] = None, + collection_info: Optional[MatchingCollections] = None, + workflow_invocation_uuid: Optional[str] = None, + invocation_step: Optional[model.WorkflowInvocationStep] = None, + max_num_jobs: Optional[int] = None, + job_callback: Optional[Callable] = None, + workflow_resource_parameters: Optional[Dict[str, Any]] = None, + validate_outputs: bool = False, +) -> "ExecutionTracker": + """The tool request/async version of execute.""" + completed_jobs = completed_jobs or {} + mapping_params.ensure_validated() + return _execute( + trans, + tool, + mapping_params, + history, + tool_request, + rerun_remap_job_id, + preferred_object_store_id, + collection_info, + workflow_invocation_uuid, + invocation_step, + max_num_jobs, + job_callback, + completed_jobs, + workflow_resource_parameters, + validate_outputs, + ) def execute( @@ -88,12 +143,48 @@ def execute( completed_jobs: Optional[CompletedJobsT] = None, workflow_resource_parameters: Optional[WorkflowResourceParametersT] = None, validate_outputs: bool = False, -): +) -> "ExecutionTracker": """ Execute a tool and return object containing summary (output data, number of failures, etc...). """ completed_jobs = completed_jobs or {} + return _execute( + trans, + tool, + mapping_params, + history, + None, + rerun_remap_job_id, + preferred_object_store_id, + collection_info, + workflow_invocation_uuid, + invocation_step, + max_num_jobs, + job_callback, + completed_jobs, + workflow_resource_parameters, + validate_outputs, + ) + + +def _execute( + trans, + tool: "Tool", + mapping_params: MappingParameters, + history: model.History, + tool_request: Optional[ToolRequest], + rerun_remap_job_id: Optional[int], + preferred_object_store_id: Optional[str], + collection_info: Optional[MatchingCollections], + workflow_invocation_uuid: Optional[str], + invocation_step: Optional[model.WorkflowInvocationStep], + max_num_jobs: Optional[int], + job_callback: Optional[Callable], + completed_jobs: Dict[int, Optional[model.Job]], + workflow_resource_parameters: Optional[Dict[str, Any]], + validate_outputs: bool, +) -> "ExecutionTracker": if max_num_jobs is not None: assert invocation_step is not None if rerun_remap_job_id: @@ -118,8 +209,9 @@ def execute_single_job(execution_slice: "ExecutionSlice", completed_job: Optiona "internals.galaxy.tools.execute.job_single", SINGLE_EXECUTION_SUCCESS_MESSAGE ) params = execution_slice.param_combination - if "__data_manager_mode" in mapping_params.param_template: - params["__data_manager_mode"] = mapping_params.param_template["__data_manager_mode"] + request_state = mapping_params.param_template + if "__data_manager_mode" in request_state: + params["__data_manager_mode"] = request_state["__data_manager_mode"] if workflow_invocation_uuid: params["__workflow_invocation_uuid__"] = workflow_invocation_uuid elif "__workflow_invocation_uuid__" in params: @@ -148,6 +240,8 @@ def execute_single_job(execution_slice: "ExecutionSlice", completed_job: Optiona skip=skip, ) if job: + if tool_request: + job.tool_request = tool_request log.debug(job_timer.to_str(tool_id=tool.id, job_id=job.id)) execution_tracker.record_success(execution_slice, job, result) # associate dataset instances with the job that creates them @@ -175,7 +269,7 @@ def execute_single_job(execution_slice: "ExecutionSlice", completed_job: Optiona except ToolInputsNotOKException as e: execution_tracker.record_error(e) - execution_tracker.ensure_implicit_collections_populated(history, mapping_params.param_template) + execution_tracker.ensure_implicit_collections_populated(history, mapping_params.param_template, tool_request) job_count = len(execution_tracker.param_combinations) jobs_executed = 0 @@ -188,7 +282,11 @@ def execute_single_job(execution_slice: "ExecutionSlice", completed_job: Optiona has_remaining_jobs = True break else: - skip = execution_slice.param_combination.pop("__when_value__", None) is False + slice_params = execution_slice.param_combination + if isinstance(slice_params, JobInternalToolState): + slice_params = slice_params.input_state + + skip = slice_params.pop("__when_value__", None) is False execute_single_job(execution_slice, completed_jobs[i], skip=skip) history = execution_slice.history or history jobs_executed += 1 @@ -426,15 +524,15 @@ def _mapped_output_structure(self, trans, tool_output): mapped_output_structure = mapping_structure.multiply(output_structure) return mapped_output_structure - def ensure_implicit_collections_populated(self, history, params): + def ensure_implicit_collections_populated(self, history, params, tool_request: Optional[ToolRequest]): if not self.collection_info: return history = history or self.tool.get_default_history_by_trans(self.trans) # params = param_combinations[0] if param_combinations else mapping_params.param_template - self.precreate_output_collections(history, params) + self.precreate_output_collections(history, params, tool_request) - def precreate_output_collections(self, history, params): + def precreate_output_collections(self, history, params, tool_request: Optional[ToolRequest]): # params is just one sample tool param execution with parallelized # collection replaced with a specific dataset. Need to replace this # with the collection and wrap everything up so can evaluate output @@ -683,13 +781,13 @@ def new_collection_execution_slices(self): yield ExecutionSlice(job_index, param_combination, dataset_collection_elements) - def ensure_implicit_collections_populated(self, history, params): + def ensure_implicit_collections_populated(self, history, params, tool_request: Optional[ToolRequest]): if not self.collection_info: return history = history or self.tool.get_default_history_by_trans(self.trans) if self.invocation_step.is_new: - self.precreate_output_collections(history, params) + self.precreate_output_collections(history, params, tool_request) for output_name, implicit_collection in self.implicit_collections.items(): self.invocation_step.add_output(output_name, implicit_collection) else: diff --git a/lib/galaxy/tools/parameters/__init__.py b/lib/galaxy/tools/parameters/__init__.py index ee3a8709817d..c61fd49ba5f1 100644 --- a/lib/galaxy/tools/parameters/__init__.py +++ b/lib/galaxy/tools/parameters/__init__.py @@ -12,6 +12,10 @@ from boltons.iterutils import remap +from galaxy.model import ( + HistoryDatasetAssociation, + HistoryDatasetCollectionAssociation, +) from galaxy.util import unicodify from galaxy.util.expressions import ExpressionContext from galaxy.util.json import safe_loads @@ -20,6 +24,7 @@ DataToolParameter, ParameterValueError, SelectToolParameter, + TextToolParameter, ToolParameter, ) from .grouping import ( @@ -656,6 +661,185 @@ def _populate_state_legacy( state[input.name] = value +def populate_state_async( + request_context, + inputs: ToolInputsT, + incoming: ToolStateJobInstanceT, + state: ToolStateJobInstancePopulatedT, + errors: ParameterValidationErrorsT, + context=None, +): + context = ExpressionContext(state, context) + for input in inputs.values(): + initial_value = input.get_initial_value(request_context, context) + input_name = input.name + state[input_name] = initial_value + group_state = state[input_name] + if input.type == "repeat": + repeat_input = cast(Repeat, input) + if ( + len(incoming[repeat_input.name]) > repeat_input.max + or len(incoming[repeat_input.name]) < repeat_input.min + ): + errors[repeat_input.name] = "The number of repeat elements is outside the range specified by the tool." + else: + del group_state[:] + for rep in incoming[repeat_input.name]: + new_state: ToolStateJobInstancePopulatedT = {} + group_state.append(new_state) + repeat_errors: ParameterValidationErrorsT = {} + populate_state_async( + request_context, + repeat_input.inputs, + rep, + new_state, + repeat_errors, + context=context, + ) + if repeat_errors: + errors[repeat_input.name] = repeat_errors + + elif input.type == "conditional": + conditional_input = cast(Conditional, input) + test_param = cast(ToolParameter, conditional_input.test_param) + test_param_value = incoming.get(conditional_input.name, {}).get(test_param.name) + value, error = check_param(request_context, test_param, test_param_value, context) + if error: + errors[test_param.name] = error + else: + try: + current_case = conditional_input.get_current_case(value) + group_state = state[conditional_input.name] = {} + cast_errors: ParameterValidationErrorsT = {} + populate_state_async( + request_context, + conditional_input.cases[current_case].inputs, + cast(ToolStateJobInstanceT, incoming.get(conditional_input.name)), + group_state, + cast_errors, + context=context, + ) + if cast_errors: + errors[conditional_input.name] = cast_errors + group_state["__current_case__"] = current_case + except Exception: + errors[test_param.name] = "The selected case is unavailable/invalid." + group_state[test_param.name] = value + + elif input.type == "section": + section_input = cast(Section, input) + section_errors: ParameterValidationErrorsT = {} + populate_state_async( + request_context, + section_input.inputs, + cast(ToolStateJobInstanceT, incoming.get(section_input.name)), + group_state, + section_errors, + context=context, + ) + if section_errors: + errors[section_input.name] = section_errors + + elif input.type == "upload_dataset": + raise NotImplementedError + + else: + param_value = _get_incoming_value(incoming, input.name, state.get(input.name)) + value, error = check_param(request_context, input, param_value, context, simple_errors=False) + if error: + errors[input.name] = error + state[input.name] = value + + def to_internal_single(value): + if isinstance(value, HistoryDatasetCollectionAssociation): + return {"src": "hdca", "id": value.id} + elif isinstance(value, HistoryDatasetAssociation): + return {"src": "hda", "id": value.id} + else: + # tests and such to confirm we need DCE, LDDA, etc... + return value + + def to_internal(value): + if isinstance(value, list): + return [to_internal_single(v) for v in value] + else: + return to_internal_single(value) + + if input_name not in incoming: + if input.type == "data_column": + if isinstance(value, str): + incoming[input_name] = int(value) + elif isinstance(value, list): + incoming[input_name] = [int(v) for v in value] + else: + incoming[input_name] = value + elif input.type == "text": + text_input = cast(TextToolParameter, input) + # see behavior of tools in test_tools.py::test_null_to_text_tools + # these parameters act as empty string in this context + if value is None and not text_input.optional: + incoming[input_name] = "" + else: + incoming[input_name] = value + else: + incoming[input_name] = to_internal(value) + + +def fill_dynamic_defaults( + request_context, + inputs: ToolInputsT, + incoming: ToolStateJobInstanceT, + context=None, +): + """ + Expands incoming parameters with default values. + """ + if context is None: + context = flat_to_nested_state(incoming) + for input in inputs.values(): + if input.type == "repeat": + repeat_input = cast(Repeat, input) + for rep in incoming[repeat_input.name]: + fill_dynamic_defaults( + request_context, + repeat_input.inputs, + rep, + context=context, + ) + + elif input.type == "conditional": + conditional_input = cast(Conditional, input) + test_param = cast(ToolParameter, conditional_input.test_param) + test_param_value = incoming.get(conditional_input.name, {}).get(test_param.name) + try: + current_case = conditional_input.get_current_case(test_param_value) + fill_dynamic_defaults( + request_context, + conditional_input.cases[current_case].inputs, + cast(ToolStateJobInstanceT, incoming.get(conditional_input.name)), + context=context, + ) + except Exception: + raise Exception("The selected case is unavailable/invalid.") + + elif input.type == "section": + section_input = cast(Section, input) + fill_dynamic_defaults( + request_context, + section_input.inputs, + cast(ToolStateJobInstanceT, incoming.get(section_input.name)), + context=context, + ) + + elif input.type == "upload_dataset": + raise NotImplementedError + + else: + if input.name not in incoming: + param_value = input.get_initial_value(request_context, context) + incoming[input.name] = param_value + + def _get_incoming_value(incoming, key, default): """ Fetch value from incoming dict directly or check special nginx upload diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 6b35473959e4..bbd0927c3e60 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -13,6 +13,7 @@ from collections.abc import MutableMapping from typing import ( Any, + cast, Dict, List, Optional, @@ -2484,7 +2485,7 @@ def from_json(self, value, trans, other_values=None): rval = value elif isinstance(value, MutableMapping) and "src" in value and "id" in value: if value["src"] == "hdca": - rval = session.get(HistoryDatasetCollectionAssociation, trans.security.decode_id(value["id"])) + rval = cast(HistoryDatasetCollectionAssociation, src_id_to_item(sa_session=trans.sa_session, value=value, security=trans.security)) elif isinstance(value, list): if len(value) > 0: value = value[0] diff --git a/lib/galaxy/tools/parameters/meta.py b/lib/galaxy/tools/parameters/meta.py index 5865a6698110..fbe061458b89 100644 --- a/lib/galaxy/tools/parameters/meta.py +++ b/lib/galaxy/tools/parameters/meta.py @@ -26,6 +26,7 @@ matching, subcollections, ) +from galaxy.tool_util.parameters import RequestInternalDereferencedToolState from galaxy.util.permutations import ( build_combos, input_classification, @@ -41,6 +42,7 @@ InputFormatT, ToolRequestT, ToolStateJobInstanceT, + ToolStateDumpedToJsonInternalT, ) log = logging.getLogger(__name__) @@ -334,6 +336,79 @@ def visitor(input, value, prefix, prefixed_name, prefixed_label, error, **kwargs return (single_inputs_nested, matched_multi_inputs, multiplied_multi_inputs) +ExpandedAsyncT = Tuple[List[ToolStateJobInstanceT], List[ToolStateDumpedToJsonInternalT], Optional[matching.MatchingCollections]] + + +def expand_meta_parameters_async( + app, tool, incoming: RequestInternalDereferencedToolState +) -> ExpandedAsyncT: + # TODO: Tool State 2.0 Follow Up: rework this to only test permutation at actual input value roots. + + collections_to_match = matching.CollectionsToMatch() + + def classifier_from_value(value, input_key): + if isinstance(value, dict) and "values" in value: + # Explicit meta wrapper for inputs... + is_batch = value.get("__class__", "Batch") == "Batch" + is_linked = value.get("linked", True) + if is_batch and is_linked: + classification = input_classification.MATCHED + elif is_batch: + classification = input_classification.MULTIPLIED + else: + classification = input_classification.SINGLE + if __collection_multirun_parameter(value): + log.info("IN HERE WITH A COLLECTION MULTIRUN PARAMETER") + collection_value = value["values"][0] + values = __expand_collection_parameter_async( + app, input_key, collection_value, collections_to_match, linked=is_linked + ) + else: + log.info("NOT IN HERE WITH A COLLECTION MULTIRUN PARAMETER") + values = value["values"] + else: + classification = input_classification.SINGLE + values = value + return classification, values + + # is there a way to make Pydantic ensure reordering isn't needed - model and serialize out the parameters maybe? + reordered_incoming = reorder_parameters(tool, incoming.input_state, incoming.input_state, True) + incoming_template = reordered_incoming + + single_inputs, matched_multi_inputs, multiplied_multi_inputs = split_inputs_nested( + tool.inputs, incoming_template, classifier_from_value + ) + expanded_incomings = build_combos(single_inputs, matched_multi_inputs, multiplied_multi_inputs, nested=True) + # those all have sa model objects from expansion to be used within for additional logic (maybe?) + # but we want to record just src and IDS in the job state object - so undo that + expanded_job_states = build_combos(to_decoded_json(single_inputs), to_decoded_json(matched_multi_inputs), to_decoded_json(multiplied_multi_inputs), nested=True) + if collections_to_match.has_collections(): + collection_info = app.dataset_collection_manager.match_collections(collections_to_match) + else: + collection_info = None + return expanded_incomings, expanded_job_states, collection_info + + +def to_decoded_json(has_objects): + if isinstance(has_objects, dict): + decoded_json = {} + for key, value in has_objects.items(): + decoded_json[key] = to_decoded_json(value) + return decoded_json + elif isinstance(has_objects, list): + return [to_decoded_json(o) for o in has_objects] + elif isinstance(has_objects, DatasetCollectionElement): + return {"src": "dce", "id": has_objects.id} + elif isinstance(has_objects, HistoryDatasetAssociation): + return {"src": "hda", "id": has_objects.id} + elif isinstance(has_objects, HistoryDatasetCollectionAssociation): + return {"src": "hdca", "id": has_objects.id} + elif isinstance(has_objects, LibraryDatasetDatasetAssociation): + return {"src": "ldda", "id": has_objects.id} + else: + return has_objects + + CollectionExpansionListT = Union[List[DatasetCollectionElement], List[DatasetInstance]] @@ -373,8 +448,34 @@ def __expand_collection_parameter( return hdas +def __expand_collection_parameter_async(app, input_key, incoming_val, collections_to_match, linked=False): + # If subcollection multirun of data_collection param - value will + # be "hdca_id|subcollection_type" else it will just be hdca_id + try: + src = incoming_val["src"] + if src != "hdca": + raise exceptions.ToolMetaParameterException(f"Invalid dataset collection source type {src}") + hdc_id = incoming_val["id"] + subcollection_type = incoming_val.get("map_over_type", None) + except TypeError: + hdc_id = incoming_val + subcollection_type = None + hdc = app.model.context.get(HistoryDatasetCollectionAssociation, hdc_id) + collections_to_match.add(input_key, hdc, subcollection_type=subcollection_type, linked=linked) + if subcollection_type is not None: + subcollection_elements = subcollections.split_dataset_collection_instance(hdc, subcollection_type) + return subcollection_elements + else: + hdas = [] + for element in hdc.collection.dataset_elements: + hda = element.dataset_instance + hda.element_identifier = element.element_identifier + hdas.append(hda) + return hdas + + def __collection_multirun_parameter(value: Dict[str, Any]) -> bool: - is_batch = value.get("batch", False) + is_batch = value.get("batch", False) or value.get("__class__", None) == "Batch" if not is_batch: return False diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 57b18c1f1c17..e1cbdee66d7e 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -61,6 +61,7 @@ ShareWithPayload, SharingStatus, StoreExportPayload, + ToolRequestModel, UpdateHistoryPayload, WriteStoreToPayload, ) @@ -374,6 +375,17 @@ def citations( ) -> List[Any]: return self.service.citations(trans, history_id) + @router.get( + "/api/histories/{history_id}/tool_requests", + summary="Return all the tool requests for the tools submitted to this history.", + ) + def tool_requests( + self, + history_id: HistoryIDPathParam, + trans: ProvidesHistoryContext = DependsOnTrans, + ) -> List[ToolRequestModel]: + return self.service.tool_requests(trans, history_id) + @router.post( "/api/histories", summary="Creates a new history.", diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 4c0c27169459..6e225ab17ba8 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -45,6 +45,7 @@ JobInputAssociation, JobInputSummary, JobOutputAssociation, + JobOutputCollectionAssociation, ReportJobErrorPayload, SearchJobsPayload, ShowFullJobResponse, @@ -68,11 +69,14 @@ ) from galaxy.webapps.galaxy.api.common import query_parameter_as_list from galaxy.webapps.galaxy.services.jobs import ( + JobCreateResponse, JobIndexPayload, JobIndexViewEnum, + JobRequest, JobsService, ) from galaxy.work.context import proxy_work_context_for_history +from .tools import validate_not_protected log = logging.getLogger(__name__) @@ -156,6 +160,12 @@ description="Limit listing of jobs to those that match the specified implicit collection job ID. If none, jobs from any implicit collection execution (or from no implicit collection execution) may be returned.", ) +ToolRequestIdQueryParam: Optional[DecodedDatabaseIdField] = Query( + default=None, + title="Tool Request ID", + description="Limit listing of jobs to those that were created from the supplied tool request ID. If none, jobs from any tool request (or from no workflows) may be returned.", +) + SortByQueryParam: JobIndexSortByEnum = Query( default=JobIndexSortByEnum.update_time, title="Sort By", @@ -208,6 +218,13 @@ class FastAPIJobs: service: JobsService = depends(JobsService) + @router.post("/api/jobs") + def create( + self, trans: ProvidesHistoryContext = DependsOnTrans, job_request: JobRequest = Body(...) + ) -> JobCreateResponse: + validate_not_protected(job_request.tool_id) + return self.service.create(trans, job_request) + @router.get("/api/jobs") def index( self, @@ -224,6 +241,7 @@ def index( workflow_id: Optional[DecodedDatabaseIdField] = WorkflowIdQueryParam, invocation_id: Optional[DecodedDatabaseIdField] = InvocationIdQueryParam, implicit_collection_jobs_id: Optional[DecodedDatabaseIdField] = ImplicitCollectionJobsIdQueryParam, + tool_request_id: Optional[DecodedDatabaseIdField] = ToolRequestIdQueryParam, order_by: JobIndexSortByEnum = SortByQueryParam, search: Optional[str] = SearchQueryParam, limit: int = LimitQueryParam, @@ -242,6 +260,7 @@ def index( workflow_id=workflow_id, invocation_id=invocation_id, implicit_collection_jobs_id=implicit_collection_jobs_id, + tool_request_id=tool_request_id, order_by=order_by, search=search, limit=limit, @@ -362,12 +381,14 @@ def outputs( self, job_id: JobIdPathParam, trans: ProvidesUserContext = DependsOnTrans, - ) -> List[JobOutputAssociation]: + ) -> List[Union[JobOutputAssociation, JobOutputCollectionAssociation]]: job = self.service.get_job(trans=trans, job_id=job_id) associations = self.service.dictify_associations(trans, job.output_datasets, job.output_library_datasets) - output_associations = [] + output_associations: List[Union[JobOutputAssociation, JobOutputCollectionAssociation]] = [] for association in associations: output_associations.append(JobOutputAssociation(name=association.name, dataset=association.dataset)) + + output_associations.extend(self.service.dictify_output_collection_associations(trans, job)) return output_associations @router.get( diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index dc27f7c3408e..6914f9ab7bf3 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -12,6 +12,8 @@ from fastapi import ( Body, Depends, + Path, + Query, Request, UploadFile, ) @@ -27,10 +29,14 @@ from galaxy.managers.context import ProvidesHistoryContext from galaxy.managers.hdas import HDAManager from galaxy.managers.histories import HistoryManager +from galaxy.model import ToolRequest from galaxy.schema.fetch_data import ( FetchDataFormPayload, FetchDataPayload, ) +from galaxy.schema.fields import DecodedDatabaseIdField +from galaxy.schema.schema import ToolRequestModel +from galaxy.tool_util.parameters import ToolParameterT from galaxy.tool_util.verify import ToolTestDescriptionDict from galaxy.tools.evaluation import global_tool_errors from galaxy.util.zipstream import ZipstreamWrapper @@ -42,7 +48,11 @@ ) from galaxy.webapps.base.controller import UsesVisualizationMixin from galaxy.webapps.base.webapp import GalaxyWebTransaction -from galaxy.webapps.galaxy.services.tools import ToolsService +from galaxy.webapps.galaxy.services.base import tool_request_to_model +from galaxy.webapps.galaxy.services.tools import ( + ToolRunReference, + ToolsService, +) from . import ( APIContentTypeRoute, as_form, @@ -73,6 +83,13 @@ class JsonApiRoute(APIContentTypeRoute): FetchDataForm = as_form(FetchDataFormPayload) +ToolIDPathParam: str = Path( + ..., + title="Tool ID", + description="The tool ID for the lineage stored in Galaxy's toolbox.", +) +ToolVersionQueryParam: Optional[str] = Query(default=None, title="Tool Version", description="") + async def get_files(request: Request, files: Optional[List[UploadFile]] = None): # FastAPI's UploadFile is a very light wrapper around starlette's UploadFile @@ -106,6 +123,57 @@ def fetch_form( ): return self.service.create_fetch(trans, payload, files) + @router.get( + "/api/tool_requests/{id}", + summary="Get tool request state.", + ) + def get_tool_request( + self, + id: DecodedDatabaseIdField, + trans: ProvidesHistoryContext = DependsOnTrans, + ) -> ToolRequestModel: + tool_request = self._get_tool_request_or_raise_not_found(trans, id) + return tool_request_to_model(tool_request) + + @router.get( + "/api/tool_requests/{id}/state", + summary="Get tool request state.", + ) + def tool_request_state( + self, + id: DecodedDatabaseIdField, + trans: ProvidesHistoryContext = DependsOnTrans, + ) -> str: + tool_request = self._get_tool_request_or_raise_not_found(trans, id) + state = tool_request.state + if not state: + raise exceptions.InconsistentDatabase() + return cast(str, state) + + def _get_tool_request_or_raise_not_found( + self, trans: ProvidesHistoryContext, id: DecodedDatabaseIdField + ) -> ToolRequest: + tool_request: Optional[ToolRequest] = cast( + Optional[ToolRequest], trans.app.model.context.query(ToolRequest).get(id) + ) + if tool_request is None: + raise exceptions.ObjectNotFound() + assert tool_request + return tool_request + + @router.get( + "/api/tools/{tool_id}/inputs", + summary="Get tool inputs.", + ) + def tool_inputs( + self, + tool_id: str = ToolIDPathParam, + tool_version: Optional[str] = ToolVersionQueryParam, + trans: ProvidesHistoryContext = DependsOnTrans, + ) -> List[ToolParameterT]: + tool_run_ref = ToolRunReference(tool_id=tool_id, tool_version=tool_version, tool_uuid=None) + return self.service.inputs(trans, tool_run_ref) + class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin): """ @@ -586,16 +654,17 @@ def create(self, trans: GalaxyWebTransaction, payload, **kwd): :type input_format: str """ tool_id = payload.get("tool_id") - tool_uuid = payload.get("tool_uuid") - if tool_id in PROTECTED_TOOLS: - raise exceptions.RequestParameterInvalidException( - f"Cannot execute tool [{tool_id}] directly, must use alternative endpoint." - ) - if tool_id is None and tool_uuid is None: - raise exceptions.RequestParameterInvalidException("Must specify a valid tool_id to use this endpoint.") + validate_not_protected(tool_id) return self.service._create(trans, payload, **kwd) +def validate_not_protected(tool_id: Optional[str]): + if tool_id in PROTECTED_TOOLS: + raise exceptions.RequestParameterInvalidException( + f"Cannot execute tool [{tool_id}] directly, must use alternative endpoint." + ) + + def _kwd_or_payload(kwd: Dict[str, Any]) -> Dict[str, Any]: if "payload" in kwd: kwd = cast(Dict[str, Any], kwd.get("payload")) diff --git a/lib/galaxy/webapps/galaxy/services/base.py b/lib/galaxy/webapps/galaxy/services/base.py index dcf91e80f2f2..423df8f4b96d 100644 --- a/lib/galaxy/webapps/galaxy/services/base.py +++ b/lib/galaxy/webapps/galaxy/services/base.py @@ -23,13 +23,19 @@ ) from galaxy.managers.context import ProvidesUserContext from galaxy.managers.model_stores import create_objects_from_store -from galaxy.model import User +from galaxy.model import ( + ToolRequest, + User, +) from galaxy.model.store import ( get_export_store_factory, ModelExportStore, ) from galaxy.schema.fields import EncodedDatabaseIdField -from galaxy.schema.schema import AsyncTaskResultSummary +from galaxy.schema.schema import ( + AsyncTaskResultSummary, + ToolRequestModel, +) from galaxy.security.idencoding import IdEncodingHelper from galaxy.short_term_storage import ( ShortTermStorageAllocator, @@ -193,3 +199,13 @@ def async_task_summary(async_result: AsyncResult) -> AsyncTaskResultSummary: name=name, queue=queue, ) + + +def tool_request_to_model(tool_request: ToolRequest) -> ToolRequestModel: + as_dict = { + "id": tool_request.id, + "request": tool_request.request, + "state": tool_request.state, + "state_message": tool_request.state_message, + } + return ToolRequestModel.model_validate(as_dict) diff --git a/lib/galaxy/webapps/galaxy/services/histories.py b/lib/galaxy/webapps/galaxy/services/histories.py index 32e8a9fa8a1c..764ce5a748a1 100644 --- a/lib/galaxy/webapps/galaxy/services/histories.py +++ b/lib/galaxy/webapps/galaxy/services/histories.py @@ -70,6 +70,7 @@ ShareHistoryWithStatus, ShareWithPayload, StoreExportPayload, + ToolRequestModel, WriteStoreToPayload, ) from galaxy.schema.tasks import ( @@ -87,6 +88,7 @@ model_store_storage_target, ServesExportStores, ServiceBase, + tool_request_to_model, ) from galaxy.webapps.galaxy.services.notifications import NotificationService from galaxy.webapps.galaxy.services.sharable import ShareableService @@ -533,6 +535,13 @@ def published( ] return rval + def tool_requests( + self, trans: ProvidesHistoryContext, history_id: DecodedDatabaseIdField + ) -> List[ToolRequestModel]: + history = self.manager.get_accessible(history_id, trans.user, current_history=trans.history) + tool_requests = history.tool_requests + return [tool_request_to_model(tr) for tr in tool_requests] + def citations(self, trans: ProvidesHistoryContext, history_id: DecodedDatabaseIdField): """ Return all the citations for the tools used to produce the datasets in diff --git a/lib/galaxy/webapps/galaxy/services/jobs.py b/lib/galaxy/webapps/galaxy/services/jobs.py index c90b50068cc2..191a5e291cf5 100644 --- a/lib/galaxy/webapps/galaxy/services/jobs.py +++ b/lib/galaxy/webapps/galaxy/services/jobs.py @@ -1,3 +1,4 @@ +import logging from enum import Enum from typing import ( Any, @@ -6,24 +7,83 @@ Optional, ) +from pydantic import ( + BaseModel, + Field, +) + from galaxy import ( exceptions, model, ) +from galaxy.celery.tasks import queue_jobs from galaxy.managers import hdas from galaxy.managers.base import security_check -from galaxy.managers.context import ProvidesUserContext +from galaxy.managers.context import ( + ProvidesHistoryContext, + ProvidesUserContext, +) +from galaxy.managers.histories import HistoryManager from galaxy.managers.jobs import ( JobManager, JobSearch, view_show_job, ) -from galaxy.model import Job -from galaxy.schema.fields import DecodedDatabaseIdField -from galaxy.schema.jobs import JobAssociation -from galaxy.schema.schema import JobIndexQueryPayload +from galaxy.model import ( + Job, + ToolRequest, + ToolSource as ToolSourceModel, +) +from galaxy.model.base import transaction +from galaxy.schema.fields import ( + DecodedDatabaseIdField, + EncodedDatabaseIdField, +) +from galaxy.schema.jobs import ( + JobAssociation, + JobOutputCollectionAssociation, +) +from galaxy.schema.schema import ( + AsyncTaskResultSummary, + JobIndexQueryPayload, +) +from galaxy.schema.tasks import ( + QueueJobs, + ToolSource, +) from galaxy.security.idencoding import IdEncodingHelper -from galaxy.webapps.galaxy.services.base import ServiceBase +from galaxy.tool_util.parameters import ( + decode, + RequestToolState, +) +from galaxy.webapps.galaxy.services.base import ( + async_task_summary, + ServiceBase, +) +from .tools import ( + ToolRunReference, + validate_tool_for_running, +) + +log = logging.getLogger(__name__) + + +class JobRequest(BaseModel): + tool_id: Optional[str] = Field(default=None, title="tool_id", description="TODO") + tool_uuid: Optional[str] = Field(default=None, title="tool_uuid", description="TODO") + tool_version: Optional[str] = Field(default=None, title="tool_version", description="TODO") + history_id: Optional[DecodedDatabaseIdField] = Field(default=None, title="history_id", description="TODO") + inputs: Optional[Dict[str, Any]] = Field(default_factory=lambda: {}, title="Inputs", description="TODO") + use_cached_jobs: Optional[bool] = Field(default=None, title="use_cached_jobs") + rerun_remap_job_id: Optional[DecodedDatabaseIdField] = Field( + default=None, title="rerun_remap_job_id", description="TODO" + ) + send_email_notification: bool = Field(default=False, title="Send Email Notification", description="TODO") + + +class JobCreateResponse(BaseModel): + tool_request_id: EncodedDatabaseIdField + task_result: AsyncTaskResultSummary class JobIndexViewEnum(str, Enum): @@ -39,6 +99,7 @@ class JobsService(ServiceBase): job_manager: JobManager job_search: JobSearch hda_manager: hdas.HDAManager + history_manager: HistoryManager def __init__( self, @@ -46,11 +107,13 @@ def __init__( job_manager: JobManager, job_search: JobSearch, hda_manager: hdas.HDAManager, + history_manager: HistoryManager, ): super().__init__(security=security) self.job_manager = job_manager self.job_search = job_search self.hda_manager = hda_manager + self.history_manager = history_manager def show( self, @@ -149,3 +212,62 @@ def __dictify_association(self, trans, job_dataset_association) -> JobAssociatio else: dataset_dict = {"src": "ldda", "id": dataset.id} return JobAssociation(name=job_dataset_association.name, dataset=dataset_dict) + + def dictify_output_collection_associations(self, trans, job: model.Job) -> List[JobOutputCollectionAssociation]: + output_associations: List[JobOutputCollectionAssociation] = [] + for job_output_collection_association in job.output_dataset_collection_instances: + ref_dict = {"src": "hdca", "id": job_output_collection_association.dataset_collection_id} + output_associations.append( + JobOutputCollectionAssociation( + name=job_output_collection_association.name, + dataset_collection_instance=ref_dict, + ) + ) + return output_associations + + def create(self, trans: ProvidesHistoryContext, job_request: JobRequest) -> JobCreateResponse: + tool_run_reference = ToolRunReference(job_request.tool_id, job_request.tool_uuid, job_request.tool_version) + tool = validate_tool_for_running(trans, tool_run_reference) + history_id = job_request.history_id + target_history = None + if history_id is not None: + target_history = self.history_manager.get_owned(history_id, trans.user, current_history=trans.history) + inputs = job_request.inputs + request_state = RequestToolState(inputs or {}) + request_state.validate(tool) + request_internal_state = decode(request_state, tool, trans.security.decode_id) + tool_request = ToolRequest() + # TODO: hash and such... + tool_source_model = ToolSourceModel( + source=[p.model_dump() for p in tool.parameters], + hash="TODO", + ) + tool_request.request = request_internal_state.input_state + tool_request.tool_source = tool_source_model + tool_request.state = ToolRequest.states.NEW + tool_request.history = target_history + sa_session = trans.sa_session + sa_session.add(tool_source_model) + sa_session.add(tool_request) + with transaction(sa_session): + sa_session.commit() + tool_request_id = tool_request.id + tool_source = ToolSource( + raw_tool_source=tool.tool_source.to_string(), + tool_dir=tool.tool_dir, + ) + task_request = QueueJobs( + user=trans.async_request_user, + history_id=target_history and target_history.id, + tool_source=tool_source, + tool_request_id=tool_request_id, + use_cached_jobs=job_request.use_cached_jobs or False, + rerun_remap_job_id=job_request.rerun_remap_job_id, + ) + result = queue_jobs.delay(request=task_request) + return JobCreateResponse( + **{ + "tool_request_id": tool_request_id, + "task_result": async_task_summary(result), + } + ) diff --git a/lib/galaxy/webapps/galaxy/services/tools.py b/lib/galaxy/webapps/galaxy/services/tools.py index 6897965d112f..bd97238ef67e 100644 --- a/lib/galaxy/webapps/galaxy/services/tools.py +++ b/lib/galaxy/webapps/galaxy/services/tools.py @@ -4,8 +4,10 @@ from json import dumps from typing import ( Any, + cast, Dict, List, + NamedTuple, Optional, Union, ) @@ -34,7 +36,9 @@ FilesPayload, ) from galaxy.security.idencoding import IdEncodingHelper +from galaxy.tool_util.parameters import ToolParameterT from galaxy.tools import Tool +from galaxy.tools._types import InputFormatT from galaxy.tools.search import ToolBoxSearch from galaxy.webapps.galaxy.services._fetch_util import validate_and_normalize_targets from galaxy.webapps.galaxy.services.base import ServiceBase @@ -42,6 +46,39 @@ log = logging.getLogger(__name__) +class ToolRunReference(NamedTuple): + tool_id: Optional[str] + tool_uuid: Optional[str] + tool_version: Optional[str] + + +def get_tool(trans: ProvidesHistoryContext, tool_ref: ToolRunReference) -> Tool: + get_kwds = dict( + tool_id=tool_ref.tool_id, + tool_uuid=tool_ref.tool_uuid, + tool_version=tool_ref.tool_version, + ) + + tool = trans.app.toolbox.get_tool(**get_kwds) + if not tool: + log.debug(f"Not found tool with kwds [{tool_ref}]") + raise exceptions.ToolMissingException("Tool not found.") + return tool + + +def validate_tool_for_running(trans: ProvidesHistoryContext, tool_ref: ToolRunReference) -> Tool: + if trans.user_is_bootstrap_admin: + raise exceptions.RealUserRequiredException("Only real users can execute tools or run jobs.") + + if tool_ref.tool_id is None and tool_ref.tool_uuid is None: + raise exceptions.RequestParameterMissingException("Must specify a valid tool_id to use this endpoint.") + + tool = get_tool(trans, tool_ref) + if not tool.allow_user_access(trans.user): + raise exceptions.ItemAccessibilityException("Tool not accessible.") + return tool + + class ToolsService(ServiceBase): def __init__( self, @@ -55,6 +92,14 @@ def __init__( self.toolbox_search = toolbox_search self.history_manager = history_manager + def inputs( + self, + trans: ProvidesHistoryContext, + tool_ref: ToolRunReference, + ) -> List[ToolParameterT]: + tool = get_tool(trans, tool_ref) + return tool.parameters + def create_fetch( self, trans: ProvidesHistoryContext, @@ -100,37 +145,14 @@ def create_fetch( return self._create(trans, create_payload) def _create(self, trans: ProvidesHistoryContext, payload, **kwd): - if trans.user_is_bootstrap_admin: - raise exceptions.RealUserRequiredException("Only real users can execute tools or run jobs.") action = payload.get("action") if action == "rerun": raise Exception("'rerun' action has been deprecated") - # Get tool. - tool_version = payload.get("tool_version") - tool_id = payload.get("tool_id") - tool_uuid = payload.get("tool_uuid") - get_kwds = dict( - tool_id=tool_id, - tool_uuid=tool_uuid, - tool_version=tool_version, + tool_run_reference = ToolRunReference( + payload.get("tool_id"), payload.get("tool_uuid"), payload.get("tool_version") ) - if tool_id is None and tool_uuid is None: - raise exceptions.RequestParameterMissingException("Must specify either a tool_id or a tool_uuid.") - - tool = trans.app.toolbox.get_tool(**get_kwds) - if not tool: - log.debug(f"Not found tool with kwds [{get_kwds}]") - raise exceptions.ToolMissingException("Tool not found.") - if not tool.allow_user_access(trans.user): - raise exceptions.ItemAccessibilityException("Tool not accessible.") - if self.config.user_activation_on: - if not trans.user: - log.warning("Anonymous user attempts to execute tool, but account activation is turned on.") - elif not trans.user.active: - log.warning( - f'User "{trans.user.email}" attempts to execute tool, but account activation is turned on and user account is not active.' - ) + tool = validate_tool_for_running(trans, tool_run_reference) # Set running history from payload parameters. # History not set correctly as part of this API call for @@ -166,7 +188,10 @@ def _create(self, trans: ProvidesHistoryContext, payload, **kwd): inputs.get("use_cached_job", "false") ) preferred_object_store_id = payload.get("preferred_object_store_id") - input_format = str(payload.get("input_format", "legacy")) + input_format_raw = str(payload.get("input_format", "legacy")) + if input_format_raw not in ["legacy", "21.01"]: + raise exceptions.RequestParameterInvalidException(f"invalid input format {input_format_raw}") + input_format = cast(InputFormatT, input_format_raw) if "data_manager_mode" in payload: incoming["__data_manager_mode"] = payload["data_manager_mode"] vars = tool.handle_input( diff --git a/lib/galaxy_test/api/conftest.py b/lib/galaxy_test/api/conftest.py index 74d8958b9158..bb21408aa00a 100644 --- a/lib/galaxy_test/api/conftest.py +++ b/lib/galaxy_test/api/conftest.py @@ -148,7 +148,7 @@ def required_tool(dataset_populator: DatasetPopulator, history_id: str, required return tool -@pytest.fixture(params=["legacy", "21.01"]) +@pytest.fixture(params=["legacy", "21.01", "request"]) def tool_input_format(request) -> Iterator[DescribeToolInputs]: yield DescribeToolInputs(request.param) diff --git a/lib/galaxy_test/api/test_tool_execute.py b/lib/galaxy_test/api/test_tool_execute.py index 95bf43e27921..a5d4ffd1f78a 100644 --- a/lib/galaxy_test/api/test_tool_execute.py +++ b/lib/galaxy_test/api/test_tool_execute.py @@ -23,15 +23,33 @@ @requires_tool_id("multi_data_param") -def test_multidata_param(target_history: TargetHistory, required_tool: RequiredTool): +def test_multidata_param( + target_history: TargetHistory, required_tool: RequiredTool, tool_input_format: DescribeToolInputs +): hda1 = target_history.with_dataset("1\t2\t3").src_dict hda2 = target_history.with_dataset("4\t5\t6").src_dict - execution = required_tool.execute.with_inputs( - { - "f1": {"batch": False, "values": [hda1, hda2]}, - "f2": {"batch": False, "values": [hda2, hda1]}, - } + inputs = ( + tool_input_format.when.flat( + { + "f1": {"batch": False, "values": [hda1, hda2]}, + "f2": {"batch": False, "values": [hda2, hda1]}, + } + ) + .when.nested( + { + "f1": {"batch": False, "values": [hda1, hda2]}, + "f2": {"batch": False, "values": [hda2, hda1]}, + "advanced": {"full": "no"}, # this shouldn't be needed is it outside branch? + } + ) + .when.request( + { + "f1": [hda1, hda2], + "f2": [hda2, hda1], + } + ) ) + execution = required_tool.execute.with_inputs(inputs) execution.assert_has_job(0).with_output("out1").with_contents("1\t2\t3\n4\t5\t6\n") execution.assert_has_job(0).with_output("out2").with_contents("4\t5\t6\n1\t2\t3\n") @@ -249,20 +267,33 @@ def test_multi_run_in_repeat( multi_run_in_repeat_datasets: MultiRunInRepeatFixtures, tool_input_format: DescribeToolInputs, ): - inputs = tool_input_format.when.flat( - { - "input1": {"batch": False, "values": [multi_run_in_repeat_datasets.common_dataset]}, - "queries_0|input2": {"batch": True, "values": multi_run_in_repeat_datasets.repeat_datasets}, - } - ).when.nested( - { - "input1": {"batch": False, "values": [multi_run_in_repeat_datasets.common_dataset]}, - "queries": [ - { - "input2": {"batch": True, "values": multi_run_in_repeat_datasets.repeat_datasets}, - } - ], - } + inputs = ( + tool_input_format.when.flat( + { + "input1": {"batch": False, "values": [multi_run_in_repeat_datasets.common_dataset]}, + "queries_0|input2": {"batch": True, "values": multi_run_in_repeat_datasets.repeat_datasets}, + } + ) + .when.nested( + { + "input1": {"batch": False, "values": [multi_run_in_repeat_datasets.common_dataset]}, + "queries": [ + { + "input2": {"batch": True, "values": multi_run_in_repeat_datasets.repeat_datasets}, + } + ], + } + ) + .when.request( + { + "input1": multi_run_in_repeat_datasets.common_dataset, + "queries": [ + { + "input2": {"__class__": "Batch", "values": multi_run_in_repeat_datasets.repeat_datasets}, + } + ], + } + ) ) execute = required_tool.execute.with_inputs(inputs) _check_multi_run_in_repeat(execute) @@ -275,20 +306,33 @@ def test_multi_run_in_repeat_mismatch( tool_input_format: DescribeToolInputs, ): """Same test as above but without the batch wrapper around the common dataset shared between multirun.""" - inputs = tool_input_format.when.flat( - { - "input1": multi_run_in_repeat_datasets.common_dataset, - "queries_0|input2": {"batch": True, "values": multi_run_in_repeat_datasets.repeat_datasets}, - } - ).when.nested( - { - "input1": multi_run_in_repeat_datasets.common_dataset, - "queries": [ - { - "input2": {"batch": True, "values": multi_run_in_repeat_datasets.repeat_datasets}, - } - ], - } + inputs = ( + tool_input_format.when.flat( + { + "input1": multi_run_in_repeat_datasets.common_dataset, + "queries_0|input2": {"batch": True, "values": multi_run_in_repeat_datasets.repeat_datasets}, + } + ) + .when.nested( + { + "input1": multi_run_in_repeat_datasets.common_dataset, + "queries": [ + { + "input2": {"batch": True, "values": multi_run_in_repeat_datasets.repeat_datasets}, + } + ], + } + ) + .when.request( + { + "input1": multi_run_in_repeat_datasets.common_dataset, + "queries": [ + { + "input2": {"__class__": "Batch", "values": multi_run_in_repeat_datasets.repeat_datasets}, + } + ], + } + ) ) execute = required_tool.execute.with_inputs(inputs) _check_multi_run_in_repeat(execute) @@ -372,7 +416,9 @@ def test_map_over_collection( target_history: TargetHistory, required_tool: RequiredTool, tool_input_format: DescribeToolInputs ): hdca = target_history.with_pair(["123", "456"]) - inputs = tool_input_format.when.any({"input1": {"batch": True, "values": [hdca.src_dict]}}) + legacy = {"input1": {"batch": True, "values": [hdca.src_dict]}} + request = {"input1": {"__class__": "Batch", "values": [hdca.src_dict]}} + inputs = tool_input_format.when.flat(legacy).when.nested(legacy).when.request(request) execute = required_tool.execute.with_inputs(inputs) execute.assert_has_n_jobs(2).assert_creates_n_implicit_collections(1) output_collection = execute.assert_creates_implicit_collection(0) diff --git a/lib/galaxy_test/api/test_tool_execution.py b/lib/galaxy_test/api/test_tool_execution.py new file mode 100644 index 000000000000..226542cf5da7 --- /dev/null +++ b/lib/galaxy_test/api/test_tool_execution.py @@ -0,0 +1,135 @@ +""" +""" + +from typing import ( + Any, + Dict, +) + +import requests + +from galaxy_test.base.api_asserts import assert_status_code_is_ok +from galaxy_test.base.populators import ( + DatasetPopulator, + skip_without_tool, +) +from ._framework import ApiTestCase + + +class TestToolExecution(ApiTestCase): + dataset_populator: DatasetPopulator + + def setUp(self): + super().setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + + @skip_without_tool("gx_int") + def test_validation(self): + with self.dataset_populator.test_history() as history_id: + self._assert_request_validates("gx_int", history_id, {"parameter": 5}) + self._assert_request_invalid("gx_int", history_id, {"parameter": None}) + self._assert_request_invalid("gx_int", history_id, {"parameter": "5"}) + + @skip_without_tool("gx_int") + def test_execution(self): + with self.dataset_populator.test_history() as history_id: + response = self._run("gx_int", history_id, {"parameter": 5}) + assert_status_code_is_ok(response) + response_json = response.json() + tool_request_id = response_json.get("tool_request_id") + task_result = response_json["task_result"] + history_tool_requests = self.dataset_populator.get_history_tool_requests(history_id) + assert tool_request_id in [tr["id"] for tr in history_tool_requests] + self.dataset_populator.wait_on_task_object(task_result) + state = self.dataset_populator.wait_on_tool_request(tool_request_id) + assert state + jobs = self.galaxy_interactor.jobs_for_tool_request(tool_request_id) + self.dataset_populator.wait_for_jobs(jobs, assert_ok=True) + + @skip_without_tool("gx_data") + def test_execution_with_src_urls(self): + with self.dataset_populator.test_history() as history_id: + response = self._run( + "gx_data", + history_id, + { + "parameter": { + "src": "url", + "url": "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", + "ext": "txt", + } + }, + ) + assert_status_code_is_ok(response) + response_json = response.json() + tool_request_id = response_json.get("tool_request_id") + task_result = response_json["task_result"] + self.dataset_populator.wait_on_task_object(task_result) + state = self.dataset_populator.wait_on_tool_request(tool_request_id) + assert state, str(self.dataset_populator.get_tool_request(tool_request_id)) + jobs = self.galaxy_interactor.jobs_for_tool_request(tool_request_id) + self.dataset_populator.wait_for_jobs(jobs, assert_ok=True) + if len(jobs) != 1: + raise Exception(f"Found incorrect number of jobs for tool request - was expecting a single job {jobs}") + assert len(jobs) == 1, jobs + job_id = jobs[0]["id"] + job_outputs = self.galaxy_interactor.job_outputs(job_id) + assert len(job_outputs) == 1 + job_output = job_outputs[0] + assert job_output["name"] == "output" + content = self.dataset_populator.get_history_dataset_content(history_id, dataset=job_output["dataset"]) + assert content == "Hello World!" + + # verify input was not left deferred and materialized before the job started + input_dataset_details = self.dataset_populator.get_history_dataset_details(history_id, hid=1) + assert input_dataset_details["state"] == "ok", input_dataset_details + + @skip_without_tool("gx_data") + def test_execution_with_deferred_src_urls(self): + with self.dataset_populator.test_history() as history_id: + response = self._run( + "gx_data", + history_id, + { + "parameter": { + "src": "url", + "url": "https://raw.githubusercontent.com/galaxyproject/planemo/7be1bf5b3971a43eaa73f483125bfb8cabf1c440/tests/data/hello.txt", + "ext": "txt", + "deferred": True, + } + }, + ) + assert_status_code_is_ok(response) + response_json = response.json() + tool_request_id = response_json.get("tool_request_id") + task_result = response_json["task_result"] + self.dataset_populator.wait_on_task_object(task_result) + state = self.dataset_populator.wait_on_tool_request(tool_request_id) + assert state, str(self.dataset_populator.get_tool_request(tool_request_id)) + jobs = self.galaxy_interactor.jobs_for_tool_request(tool_request_id) + self.dataset_populator.wait_for_jobs(jobs, assert_ok=True) + if len(jobs) != 1: + raise Exception(f"Found incorrect number of jobs for tool request - was expecting a single job {jobs}") + assert len(jobs) == 1, jobs + job_id = jobs[0]["id"] + job_outputs = self.galaxy_interactor.job_outputs(job_id) + assert len(job_outputs) == 1 + job_output = job_outputs[0] + assert job_output["name"] == "output" + content = self.dataset_populator.get_history_dataset_content(history_id, dataset=job_output["dataset"]) + assert content == "Hello World!" + + # verify input was left deferred and infer must have been materialized just for the job + input_dataset_details = self.dataset_populator.get_history_dataset_details(history_id, hid=1) + assert input_dataset_details["state"] == "deferred", input_dataset_details + + def _assert_request_validates(self, tool_id: str, history_id: str, inputs: Dict[str, Any]): + response = self._run(tool_id, history_id, inputs) + assert response.status_code == 200 + + def _assert_request_invalid(self, tool_id: str, history_id: str, inputs: Dict[str, Any]): + response = self._run(tool_id, history_id, inputs) + assert response.status_code == 400 + + def _run(self, tool_id: str, history_id: str, inputs: Dict[str, Any]) -> requests.Response: + return self.dataset_populator.tool_request_raw(tool_id, inputs, history_id) diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 3e47c70c18f8..f186a194a8c9 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -144,6 +144,7 @@ DEFAULT_TIMEOUT = 60 # Secs to wait for state to turn ok SKIP_FLAKEY_TESTS_ON_ERROR = os.environ.get("GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR", None) +INPUT_FORMAT_T = Literal["legacy", "21.01", "request"] PRIVATE_ROLE_TYPE = "private" @@ -1013,6 +1014,15 @@ def run_tool_raw(self, tool_id: Optional[str], inputs: dict, history_id: str, ** payload = self.run_tool_payload(tool_id, inputs, history_id, **kwds) return self.tools_post(payload) + def tool_request_raw(self, tool_id: str, inputs: Dict[str, Any], history_id: str) -> Response: + payload = { + "tool_id": tool_id, + "history_id": history_id, + "inputs": inputs, + } + response = self._post("jobs", data=payload, json=True) + return response + def run_tool(self, tool_id: str, inputs: dict, history_id: str, **kwds): tool_response = self.run_tool_raw(tool_id, inputs, history_id, **kwds) api_asserts.assert_status_code_is(tool_response, 200) @@ -1528,8 +1538,38 @@ def is_ready(): wait_on(is_ready, "waiting for download to become ready") assert is_ready() + def wait_on_tool_request(self, tool_request_id: str): + # should this to defer to interactor's copy of this method? + + def state(): + state_response = self._get(f"tool_requests/{tool_request_id}/state") + state_response.raise_for_status() + return state_response.json() + + def is_ready(): + is_complete = state() in ["submitted", "failed"] + return True if is_complete else None + + wait_on(is_ready, "waiting for tool request to submit") + return state() == "submitted" + + def get_tool_request(self, tool_request_id: str) -> Dict[str, Any]: + response = self._get(f"tool_requests/{tool_request_id}") + api_asserts.assert_status_code_is_ok(response) + return response.json() + + def get_history_tool_requests(self, history_id: str) -> List[Dict[str, Any]]: + response = self._get(f"histories/{history_id}/tool_requests") + api_asserts.assert_status_code_is_ok(response) + return response.json() + def wait_on_task(self, async_task_response: Response): - task_id = async_task_response.json()["id"] + response_json = async_task_response.json() + self.wait_on_task_object(response_json) + + def wait_on_task_object(self, async_task_json: Dict[str, Any]): + assert "id" in async_task_json, f"Task response {async_task_json} does not contain expected 'id' field." + task_id = async_task_json["id"] return self.wait_on_task_id(task_id) def wait_on_task_id(self, task_id: str): @@ -3680,10 +3720,10 @@ def execute(self) -> "DescribeToolExecution": class DescribeToolInputs: - _input_format: str = "legacy" + _input_format: INPUT_FORMAT_T = "legacy" _inputs: Optional[Dict[str, Any]] - def __init__(self, input_format: str): + def __init__(self, input_format: INPUT_FORMAT_T): self._input_format = input_format self._inputs = None @@ -3697,7 +3737,12 @@ def flat(self, inputs: Dict[str, Any]) -> Self: return self def nested(self, inputs: Dict[str, Any]) -> Self: - if self._input_format == "21.01": + if self._input_format in ["21.01", "request"]: + self._inputs = inputs + return self + + def request(self, inputs: Dict[str, Any]) -> Self: + if self._input_format in ["request"]: self._inputs = inputs return self @@ -3710,7 +3755,8 @@ def when(self) -> Self: class DescribeToolExecution: _history_id: Optional[str] = None _execute_response: Optional[Response] = None - _input_format: Optional[str] = None + _tool_request_id: Optional[str] = None # if input_format == "request" request ID + _input_format: Optional[INPUT_FORMAT_T] = None _inputs: Dict[str, Any] def __init__(self, dataset_populator: BaseDatasetPopulator, tool_id: str): @@ -3739,14 +3785,29 @@ def with_nested_inputs(self, inputs: Dict[str, Any]) -> Self: self._input_format = "21.01" return self + def with_request(self, inputs: Dict[str, Any]) -> Self: + self._inputs = inputs + self._input_format = "request" + return self + def _execute(self): kwds = {} if self._input_format is not None: kwds["input_format"] = self._input_format history_id = self._ensure_history_id - self._execute_response = self._dataset_populator.run_tool_raw( - self._tool_id, self._inputs, history_id, assert_ok=False, **kwds - ) + if self._input_format == "request": + execute_response = self._dataset_populator.tool_request_raw( + self._tool_id, self._inputs, history_id + ) + api_asserts.assert_status_code_is_ok(execute_response) + response_json = execute_response.json() + tool_request_id = response_json.get("tool_request_id") + self._dataset_populator.wait_on_tool_request(tool_request_id) + self._execute_response = execute_response + else: + self._execute_response = self._dataset_populator.run_tool_raw( + self._tool_id, self._inputs, history_id, assert_ok=False, **kwds + ) @property def _ensure_history_id(self) -> str: @@ -3764,13 +3825,32 @@ def _assert_executed_ok(self) -> Dict[str, Any]: execute_response = self._execute_response assert execute_response is not None api_asserts.assert_status_code_is_ok(execute_response) + if self._input_format == "request": + response_json = execute_response.json() + tool_request_id = response_json.get("tool_request_id") + task_result = response_json["task_result"] + self._dataset_populator.wait_on_task_object(task_result) + self._tool_request_id = tool_request_id + return execute_response.json() + @property + def _jobs(self) -> List[Dict[str, Any]]: + if self._input_format == "request": + tool_request_id = self._tool_request_id + assert tool_request_id, "request not exected" + jobs = self._dataset_populator.galaxy_interactor.jobs_for_tool_request(tool_request_id) + else: + response = self._assert_executed_ok() + jobs = response["jobs"] + return jobs + def assert_has_n_jobs(self, n: int) -> Self: - response = self._assert_executed_ok() - jobs = response["jobs"] - if len(jobs) != n: - raise AssertionError(f"Expected tool execution to produce {n} jobs but it produced {len(jobs)}") + self._assert_executed_ok() + jobs = self._jobs + num_jobs = len(jobs) + if num_jobs != n: + raise AssertionError(f"Expected tool execution to produce {n} jobs but it produced {num_jobs}") return self def assert_creates_n_implicit_collections(self, n: int) -> Self: @@ -3792,8 +3872,8 @@ def assert_has_single_job(self) -> DescribeJob: return self.assert_has_n_jobs(1).assert_has_job(0) def assert_has_job(self, job_index: int = 0) -> DescribeJob: - response = self._assert_executed_ok() - job = response["jobs"][job_index] + self._assert_executed_ok() + job = self._jobs[job_index] history_id = self._ensure_history_id return DescribeJob(self._dataset_populator, history_id, job["id"]) @@ -3805,8 +3885,8 @@ def that_fails(self) -> DescribeFailure: if execute_response.status_code != 200: return DescribeFailure(execute_response) else: - response = self._assert_executed_ok() - jobs = response["jobs"] + self._assert_executed_ok() + jobs = self._jobs for job in jobs: final_state = self._dataset_populator.wait_for_job(job["id"]) assert final_state == "error" diff --git a/test/functional/test_toolbox_pytest.py b/test/functional/test_toolbox_pytest.py index 896e3609913e..cd6314a9fc85 100644 --- a/test/functional/test_toolbox_pytest.py +++ b/test/functional/test_toolbox_pytest.py @@ -1,11 +1,16 @@ import os from typing import ( + cast, List, NamedTuple, ) import pytest +from galaxy.tool_util.verify.interactor import ( + DEFAULT_USE_LEGACY_API, + UseLegacyApiT, +) from galaxy_test.api._framework import ApiTestCase from galaxy_test.driver.driver_util import GalaxyTestDriver @@ -61,4 +66,7 @@ class TestFrameworkTools(ApiTestCase): @pytest.mark.parametrize("testcase", cases(), ids=idfn) def test_tool(self, testcase: ToolTest): - self._test_driver.run_tool_test(testcase.tool_id, testcase.test_index, tool_version=testcase.tool_version) + use_legacy_api = cast(UseLegacyApiT, os.environ.get("GALAXY_TEST_USE_LEGACY_TOOL_API", DEFAULT_USE_LEGACY_API)) + self._test_driver.run_tool_test( + testcase.tool_id, testcase.test_index, tool_version=testcase.tool_version, use_legacy_api=use_legacy_api + ) diff --git a/test/unit/tool_util/test_parameter_convert.py b/test/unit/tool_util/test_parameter_convert.py index 38efa9105aed..473b5d4b92b3 100644 --- a/test/unit/tool_util/test_parameter_convert.py +++ b/test/unit/tool_util/test_parameter_convert.py @@ -43,7 +43,17 @@ def test_decode_data(): assert decoded_state.input_state["parameter"]["id"] == EXAMPLE_ID_1 -def test_encode_collection(): +def test_decode_data_batch(): + tool_source = tool_source_for("parameters/gx_data") + bundle = input_models_for_tool_source(tool_source) + request_state = RequestToolState({"parameter": {"__class__": "Batch", "values": [{"src": "hda", "id": EXAMPLE_ID_1_ENCODED}]}}) + request_state.validate(bundle) + decoded_state = decode(request_state, bundle, _fake_decode) + assert decoded_state.input_state["parameter"]["values"][0]["src"] == "hda" + assert decoded_state.input_state["parameter"]["values"][0]["id"] == EXAMPLE_ID_1 + + +def test_decode_collection(): tool_source = tool_source_for("parameters/gx_data_collection") bundle = input_models_for_tool_source(tool_source) request_state = RequestToolState({"parameter": {"src": "hdca", "id": EXAMPLE_ID_1_ENCODED}}) @@ -119,6 +129,20 @@ def test_landing_encode_data(): assert encoded_state.input_state["parameter"]["id"] == EXAMPLE_ID_1_ENCODED +def test_landing_encode_data_batch(): + tool_source = tool_source_for("parameters/gx_data") + bundle = input_models_for_tool_source(tool_source) + request_state = LandingRequestToolState({"parameter": {"__class__": "Batch", "values": [{"src": "hda", "id": EXAMPLE_ID_1_ENCODED}]}}) + request_state.validate(bundle) + decoded_state = landing_decode(request_state, bundle, _fake_decode) + assert decoded_state.input_state["parameter"]["values"][0]["src"] == "hda" + assert decoded_state.input_state["parameter"]["values"][0]["id"] == EXAMPLE_ID_1 + + encoded_state = landing_encode(decoded_state, bundle, _fake_encode) + assert encoded_state.input_state["parameter"]["values"][0]["src"] == "hda" + assert encoded_state.input_state["parameter"]["values"][0]["id"] == EXAMPLE_ID_1_ENCODED + + def test_dereference(): tool_source = tool_source_for("parameters/gx_data") bundle = input_models_for_tool_source(tool_source) From 62005d4f32d3264d5f92e0ec27aed593061c7ce5 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 7 Oct 2024 11:11:24 -0400 Subject: [PATCH 05/17] Refactoring that lets the tool request API work. --- lib/galaxy/tools/parameters/basic.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index bbd0927c3e60..cab76cf4a81d 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -2485,7 +2485,10 @@ def from_json(self, value, trans, other_values=None): rval = value elif isinstance(value, MutableMapping) and "src" in value and "id" in value: if value["src"] == "hdca": - rval = cast(HistoryDatasetCollectionAssociation, src_id_to_item(sa_session=trans.sa_session, value=value, security=trans.security)) + rval = cast( + HistoryDatasetCollectionAssociation, + src_id_to_item(sa_session=trans.sa_session, value=value, security=trans.security), + ) elif isinstance(value, list): if len(value) > 0: value = value[0] From 9e7219ad34f82f9173c160b30cafde991ec09835 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 18 Sep 2024 15:46:08 -0400 Subject: [PATCH 06/17] Blah.... --- lib/galaxy/tools/parameters/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index cab76cf4a81d..6aed9c85344f 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1087,7 +1087,7 @@ def _select_from_json(self, value, trans, other_values=None, require_legal_value ) if is_runtime_value(value): return None - if value in legal_values: + if value in legal_values or str(value) in legal_values: return value elif value in fallback_values: return fallback_values[value] From 91e2afe34514ba5a57b6489f199f50e2c65e1d69 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 19 Nov 2024 09:50:16 -0500 Subject: [PATCH 07/17] Rebuild schema for tool request APIs... --- client/src/api/schema/schema.ts | 1660 ++++++++++++++++++++++++++++++- 1 file changed, 1628 insertions(+), 32 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 1e9ab944ad2e..bb1e56becc26 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -2502,6 +2502,23 @@ export interface paths { patch?: never; trace?: never; }; + "/api/histories/{history_id}/tool_requests": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Return all the tool requests for the tools submitted to this history. */ + get: operations["tool_requests_api_histories__history_id__tool_requests_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/histories/{history_id}/unpublish": { parameters: { query?: never; @@ -2814,7 +2831,8 @@ export interface paths { /** Index */ get: operations["index_api_jobs_get"]; put?: never; - post?: never; + /** Create */ + post: operations["create_api_jobs_post"]; delete?: never; options?: never; head?: never; @@ -4381,6 +4399,40 @@ export interface paths { patch?: never; trace?: never; }; + "/api/tool_requests/{id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get tool request state. */ + get: operations["get_tool_request_api_tool_requests__id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/tool_requests/{id}/state": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get tool request state. */ + get: operations["tool_request_state_api_tool_requests__id__state_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/tool_shed_repositories": { parameters: { query?: never; @@ -4449,6 +4501,23 @@ export interface paths { patch?: never; trace?: never; }; + "/api/tools/{tool_id}/inputs": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get tool inputs. */ + get: operations["tool_inputs_api_tools__tool_id__inputs_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/tours": { parameters: { query?: never; @@ -6276,6 +6345,39 @@ export interface components { ) | ("cloud" | "quota" | "no_quota" | "restricted" | "user_defined"); }; + /** BaseUrlParameterModel */ + BaseUrlParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_baseurl + * @constant + * @enum {string} + */ + parameter_type: "gx_baseurl"; + }; /** BasicRoleModel */ BasicRoleModel: { /** @@ -6391,6 +6493,48 @@ export interface components { /** Targets */ targets: unknown; }; + /** BooleanParameterModel */ + BooleanParameterModel: { + /** Argument */ + argument?: string | null; + /** Falsevalue */ + falsevalue?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_boolean + * @constant + * @enum {string} + */ + parameter_type: "gx_boolean"; + /** Truevalue */ + truevalue?: string | null; + /** + * Value + * @default false + */ + value: boolean | null; + }; /** BroadcastNotificationContent */ BroadcastNotificationContent: { /** @@ -6705,6 +6849,41 @@ export interface components { * @enum {string} */ ColletionSourceType: "hda" | "ldda" | "hdca" | "new_collection"; + /** ColorParameterModel */ + ColorParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_color + * @constant + * @enum {string} + */ + parameter_type: "gx_color"; + /** Value */ + value?: string | null; + }; /** CompositeDataElement */ CompositeDataElement: { /** Md5 */ @@ -6854,6 +7033,82 @@ export interface components { */ source: string | null; }; + /** ConditionalParameterModel */ + ConditionalParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_conditional + * @constant + * @enum {string} + */ + parameter_type: "gx_conditional"; + /** Test Parameter */ + test_parameter: + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["SelectParameterModel"]; + /** Whens */ + whens: components["schemas"]["ConditionalWhen"][]; + }; + /** ConditionalWhen */ + ConditionalWhen: { + /** Discriminator */ + discriminator: boolean | string; + /** Is Default When */ + is_default_when: boolean; + /** Parameters */ + parameters: ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + | components["schemas"]["TextParameterModel"] + | components["schemas"]["IntegerParameterModel"] + | components["schemas"]["FloatParameterModel"] + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["HiddenParameterModel"] + | components["schemas"]["SelectParameterModel"] + | components["schemas"]["DataParameterModel"] + | components["schemas"]["DataCollectionParameterModel"] + | components["schemas"]["DataColumnParameterModel"] + | components["schemas"]["DirectoryUriParameterModel"] + | components["schemas"]["RulesParameterModel"] + | components["schemas"]["DrillDownParameterModel"] + | components["schemas"]["GroupTagParameterModel"] + | components["schemas"]["BaseUrlParameterModel"] + | components["schemas"]["GenomeBuildParameterModel"] + | components["schemas"]["ColorParameterModel"] + | components["schemas"]["ConditionalParameterModel"] + | components["schemas"]["RepeatParameterModel"] + | components["schemas"]["SectionParameterModel"] + )[]; + }; /** ConnectAction */ ConnectAction: { /** @@ -7769,6 +8024,155 @@ export interface components { */ username_and_slug?: string | null; }; + /** CwlBooleanParameterModel */ + CwlBooleanParameterModel: { + /** Name */ + name: string; + /** + * Parameter Type + * @default cwl_boolean + * @constant + * @enum {string} + */ + parameter_type: "cwl_boolean"; + }; + /** CwlDirectoryParameterModel */ + CwlDirectoryParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default cwl_directory + * @constant + * @enum {string} + */ + parameter_type: "cwl_directory"; + }; + /** CwlFileParameterModel */ + CwlFileParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default cwl_file + * @constant + * @enum {string} + */ + parameter_type: "cwl_file"; + }; + /** CwlFloatParameterModel */ + CwlFloatParameterModel: { + /** Name */ + name: string; + /** + * Parameter Type + * @default cwl_float + * @constant + * @enum {string} + */ + parameter_type: "cwl_float"; + }; + /** CwlIntegerParameterModel */ + CwlIntegerParameterModel: { + /** Name */ + name: string; + /** + * Parameter Type + * @default cwl_integer + * @constant + * @enum {string} + */ + parameter_type: "cwl_integer"; + }; + /** CwlNullParameterModel */ + CwlNullParameterModel: { + /** Name */ + name: string; + /** + * Parameter Type + * @default cwl_null + * @constant + * @enum {string} + */ + parameter_type: "cwl_null"; + }; + /** CwlStringParameterModel */ + CwlStringParameterModel: { + /** Name */ + name: string; + /** + * Parameter Type + * @default cwl_string + * @constant + * @enum {string} + */ + parameter_type: "cwl_string"; + }; + /** CwlUnionParameterModel */ + CwlUnionParameterModel: { + /** Name */ + name: string; + /** + * Parameter Type + * @default cwl_union + * @constant + * @enum {string} + */ + parameter_type: "cwl_union"; + /** Parameters */ + parameters: ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + )[]; + }; /** * DCESummary * @description Dataset Collection Element summary information. @@ -7858,14 +8262,93 @@ export interface components { */ populated?: boolean; }; - /** DataElementsFromTarget */ - DataElementsFromTarget: { + /** DataCollectionParameterModel */ + DataCollectionParameterModel: { + /** Argument */ + argument?: string | null; + /** Collection Type */ + collection_type?: string | null; /** - * Auto Decompress - * @description Decompress compressed data before sniffing? + * Extensions + * @default [ + * "data" + * ] + */ + extensions: string[]; + /** Help */ + help?: string | null; + /** + * Hidden * @default false */ - auto_decompress: boolean; + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_data_collection + * @constant + * @enum {string} + */ + parameter_type: "gx_data_collection"; + /** Value */ + value: Record | null; + }; + /** DataColumnParameterModel */ + DataColumnParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Multiple */ + multiple: boolean; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_data_column + * @constant + * @enum {string} + */ + parameter_type: "gx_data_column"; + }; + /** DataElementsFromTarget */ + DataElementsFromTarget: { + /** + * Auto Decompress + * @description Decompress compressed data before sniffing? + * @default false + */ + auto_decompress: boolean; /** Destination */ destination: | components["schemas"]["HdaDestination"] @@ -7914,6 +8397,55 @@ export interface components { * @enum {string} */ DataItemSourceType: "hda" | "ldda" | "hdca" | "dce" | "dc"; + /** DataParameterModel */ + DataParameterModel: { + /** Argument */ + argument?: string | null; + /** + * Extensions + * @default [ + * "data" + * ] + */ + extensions: string[]; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Max */ + max?: number | null; + /** Min */ + min?: number | null; + /** + * Multiple + * @default false + */ + multiple: boolean; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_data + * @constant + * @enum {string} + */ + parameter_type: "gx_data"; + }; /** DatasetAssociationRoles */ DatasetAssociationRoles: { /** @@ -8544,6 +9076,49 @@ export interface components { */ username: string; }; + /** DirectoryUriParameterModel */ + DirectoryUriParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_directory_uri + * @constant + * @enum {string} + */ + parameter_type: "gx_directory_uri"; + /** + * Validators + * @default [] + */ + validators: ( + | components["schemas"]["LengthParameterValidatorModel"] + | components["schemas"]["RegexParameterValidatorModel"] + | components["schemas"]["ExpressionParameterValidatorModel"] + | components["schemas"]["EmptyFieldParameterValidatorModel"] + )[]; + }; /** DisconnectAction */ DisconnectAction: { /** @@ -8587,6 +9162,59 @@ export interface components { /** Version */ version: string; }; + /** DrillDownOptionsDict */ + DrillDownOptionsDict: { + /** Name */ + name: string | null; + /** Options */ + options: components["schemas"]["DrillDownOptionsDict"][]; + /** Selected */ + selected: boolean; + /** Value */ + value: string; + }; + /** DrillDownParameterModel */ + DrillDownParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Hierarchy + * @enum {string} + */ + hierarchy: "recurse" | "exact"; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Multiple */ + multiple: boolean; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** Options */ + options?: components["schemas"]["DrillDownOptionsDict"][] | null; + /** + * Parameter Type + * @default gx_drill_down + * @constant + * @enum {string} + */ + parameter_type: "gx_drill_down"; + }; /** DrsObject */ DrsObject: { /** @@ -8679,6 +9307,28 @@ export interface components { * @enum {string} */ ElementsFromType: "archive" | "bagit" | "bagit_archive" | "directory"; + /** EmptyFieldParameterValidatorModel */ + EmptyFieldParameterValidatorModel: { + /** + * Implicit + * @default false + */ + implicit: boolean; + /** Message */ + message?: string | null; + /** + * Negate + * @default false + */ + negate: boolean; + /** + * Type + * @default empty_field + * @constant + * @enum {string} + */ + type: "empty_field"; + }; /** EncodedDataItemSourceId */ EncodedDataItemSourceId: { /** @@ -8994,6 +9644,35 @@ export interface components { }; /** ExportTaskListResponse */ ExportTaskListResponse: components["schemas"]["ObjectExportTaskResponse"][]; + /** + * ExpressionParameterValidatorModel + * @description Check if a one line python expression given expression evaluates to True. + * + * The expression is given is the content of the validator tag. + */ + ExpressionParameterValidatorModel: { + /** Expression */ + expression: string; + /** + * Implicit + * @default false + */ + implicit: boolean; + /** Message */ + message?: string | null; + /** + * Negate + * @default false + */ + negate: boolean; + /** + * Type + * @default expression + * @constant + * @enum {string} + */ + type: "expression"; + }; /** ExtraFileEntry */ ExtraFileEntry: { /** @description The class of this entry, either File or Directory. */ @@ -9360,6 +10039,50 @@ export interface components { /** Step */ step: components["schemas"]["StepReferenceByOrderIndex"] | components["schemas"]["StepReferenceByLabel"]; }; + /** FloatParameterModel */ + FloatParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Max */ + max?: number | null; + /** Min */ + min?: number | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_float + * @constant + * @enum {string} + */ + parameter_type: "gx_float"; + /** + * Validators + * @default [] + */ + validators: components["schemas"]["InRangeParameterValidatorModel"][]; + /** Value */ + value?: number | null; + }; /** FolderLibraryFolderItem */ FolderLibraryFolderItem: { /** Can Manage */ @@ -9489,6 +10212,41 @@ export interface components { /** Tags */ tags?: string[] | null; }; + /** GenomeBuildParameterModel */ + GenomeBuildParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Multiple */ + multiple: boolean; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_genomebuild + * @constant + * @enum {string} + */ + parameter_type: "gx_genomebuild"; + }; /** * GroupCreatePayload * @description Payload schema for creating a group. @@ -9599,6 +10357,41 @@ export interface components { */ url: string; }; + /** GroupTagParameterModel */ + GroupTagParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Multiple */ + multiple: boolean; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_group_tag + * @constant + * @enum {string} + */ + parameter_type: "gx_group_tag"; + }; /** GroupUpdatePayload */ GroupUpdatePayload: { /** name of the group */ @@ -11198,6 +11991,51 @@ export interface components { HelpForumUser: { [key: string]: unknown; }; + /** HiddenParameterModel */ + HiddenParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_hidden + * @constant + * @enum {string} + */ + parameter_type: "gx_hidden"; + /** + * Validators + * @default [] + */ + validators: ( + | components["schemas"]["LengthParameterValidatorModel"] + | components["schemas"]["RegexParameterValidatorModel"] + | components["schemas"]["ExpressionParameterValidatorModel"] + | components["schemas"]["EmptyFieldParameterValidatorModel"] + )[]; + /** Value */ + value: string | null; + }; /** * HistoryActiveContentCounts * @description Contains the number of active, deleted or hidden items in a History. @@ -11605,10 +12443,46 @@ export interface components { */ src: "uri"; /** - * uri - * @description URI to fetch tool data bundle from (file:// URIs are fine because this is an admin-only operation) + * uri + * @description URI to fetch tool data bundle from (file:// URIs are fine because this is an admin-only operation) + */ + uri: string; + }; + /** InRangeParameterValidatorModel */ + InRangeParameterValidatorModel: { + /** + * Exclude Max + * @default false + */ + exclude_max: boolean; + /** + * Exclude Min + * @default false + */ + exclude_min: boolean; + /** + * Implicit + * @default false + */ + implicit: boolean; + /** Max */ + max?: number | null; + /** Message */ + message?: string | null; + /** Min */ + min?: number | null; + /** + * Negate + * @default false + */ + negate: boolean; + /** + * Type + * @default in_range + * @constant + * @enum {string} */ - uri: string; + type: "in_range"; }; /** InputDataCollectionStep */ InputDataCollectionStep: { @@ -11853,6 +12727,47 @@ export interface components { /** Uninstalled */ uninstalled: boolean; }; + /** IntegerParameterModel */ + IntegerParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Max */ + max?: number | null; + /** Min */ + min?: number | null; + /** Name */ + name: string; + /** Optional */ + optional: boolean; + /** + * Parameter Type + * @default gx_integer + * @constant + * @enum {string} + */ + parameter_type: "gx_integer"; + /** + * Validators + * @default [] + */ + validators: components["schemas"]["InRangeParameterValidatorModel"][]; + /** Value */ + value?: number | null; + }; /** InvocationCancellationHistoryDeletedResponse */ InvocationCancellationHistoryDeletedResponse: { /** @@ -12778,6 +13693,15 @@ export interface components { */ stdout?: string | null; }; + /** JobCreateResponse */ + JobCreateResponse: { + task_result: components["schemas"]["AsyncTaskResultSummary"]; + /** + * Tool Request Id + * @example 0123456789ABCDEF + */ + tool_request_id: string; + }; /** JobDestinationParams */ JobDestinationParams: { /** @@ -13056,6 +13980,19 @@ export interface components { */ name: string; }; + /** JobOutputCollectionAssociation */ + JobOutputCollectionAssociation: { + /** + * dataset_collection_instance + * @description Reference to the associated item. + */ + dataset_collection_instance: components["schemas"]["EncodedDataItemSourceId"]; + /** + * name + * @description Name of the job parameter. + */ + name: string; + }; /** JobParameter */ JobParameter: { /** @@ -13084,6 +14021,47 @@ export interface components { | string | null; }; + /** JobRequest */ + JobRequest: { + /** + * history_id + * @description TODO + */ + history_id?: string | null; + /** + * Inputs + * @description TODO + */ + inputs?: Record | null; + /** + * rerun_remap_job_id + * @description TODO + */ + rerun_remap_job_id?: string | null; + /** + * Send Email Notification + * @description TODO + * @default false + */ + send_email_notification: boolean; + /** + * tool_id + * @description TODO + */ + tool_id?: string | null; + /** + * tool_uuid + * @description TODO + */ + tool_uuid?: string | null; + /** + * tool_version + * @description TODO + */ + tool_version?: string | null; + /** use_cached_jobs */ + use_cached_jobs?: boolean | null; + }; /** * JobSourceType * @description Available types of job sources (model classes) that produce dataset collections. @@ -13216,6 +14194,15 @@ export interface components { */ user_email?: string | null; }; + /** LabelValue */ + LabelValue: { + /** Label */ + label: string; + /** Selected */ + selected: boolean; + /** Value */ + value: string; + }; /** * LabelValuePair * @description Generic Label/Value pair model. @@ -13264,6 +14251,32 @@ export interface components { */ LIBRARY_MODIFY_in: string[] | string | null; }; + /** LengthParameterValidatorModel */ + LengthParameterValidatorModel: { + /** + * Implicit + * @default false + */ + implicit: boolean; + /** Max */ + max?: number | null; + /** Message */ + message?: string | null; + /** Min */ + min?: number | null; + /** + * Negate + * @default false + */ + negate: boolean; + /** + * Type + * @default length + * @constant + * @enum {string} + */ + type: "length"; + }; /** LibraryAvailablePermissions */ LibraryAvailablePermissions: { /** @@ -14400,6 +15413,28 @@ export interface components { */ slug: string; }; + /** NoOptionsParameterValidatorModel */ + NoOptionsParameterValidatorModel: { + /** + * Implicit + * @default false + */ + implicit: boolean; + /** Message */ + message?: string | null; + /** + * Negate + * @default false + */ + negate: boolean; + /** + * Type + * @default no_options + * @constant + * @enum {string} + */ + type: "no_options"; + }; /** * NotificationBroadcastUpdateRequest * @description A notification update request specific for broadcasting. @@ -15537,6 +16572,37 @@ export interface components { /** Workflow */ workflow: string; }; + /** + * RegexParameterValidatorModel + * @description Check if a regular expression **matches** the value, i.e. appears + * at the beginning of the value. To enforce a match of the complete value use + * ``$`` at the end of the expression. The expression is given is the content + * of the validator tag. Note that for ``selects`` each option is checked + * separately. + */ + RegexParameterValidatorModel: { + /** Expression */ + expression: string; + /** + * Implicit + * @default false + */ + implicit: boolean; + /** Message */ + message?: string | null; + /** + * Negate + * @default false + */ + negate: boolean; + /** + * Type + * @default regex + * @constant + * @enum {string} + */ + type: "regex"; + }; /** ReloadFeedback */ ReloadFeedback: { /** Failed */ @@ -15628,6 +16694,73 @@ export interface components { */ action_type: "remove_unlabeled_workflow_outputs"; }; + /** RepeatParameterModel */ + RepeatParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Max */ + max?: number | null; + /** Min */ + min?: number | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_repeat + * @constant + * @enum {string} + */ + parameter_type: "gx_repeat"; + /** Parameters */ + parameters: ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + | components["schemas"]["TextParameterModel"] + | components["schemas"]["IntegerParameterModel"] + | components["schemas"]["FloatParameterModel"] + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["HiddenParameterModel"] + | components["schemas"]["SelectParameterModel"] + | components["schemas"]["DataParameterModel"] + | components["schemas"]["DataCollectionParameterModel"] + | components["schemas"]["DataColumnParameterModel"] + | components["schemas"]["DirectoryUriParameterModel"] + | components["schemas"]["RulesParameterModel"] + | components["schemas"]["DrillDownParameterModel"] + | components["schemas"]["GroupTagParameterModel"] + | components["schemas"]["BaseUrlParameterModel"] + | components["schemas"]["GenomeBuildParameterModel"] + | components["schemas"]["ColorParameterModel"] + | components["schemas"]["ConditionalParameterModel"] + | components["schemas"]["RepeatParameterModel"] + | components["schemas"]["SectionParameterModel"] + )[]; + }; /** Report */ Report: { /** Markdown */ @@ -15735,6 +16868,39 @@ export interface components { RootModel_Dict_str__int__: { [key: string]: number; }; + /** RulesParameterModel */ + RulesParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_rules + * @constant + * @enum {string} + */ + parameter_type: "gx_rules"; + }; /** SearchJobsPayload */ SearchJobsPayload: { /** @@ -15755,6 +16921,108 @@ export interface components { } & { [key: string]: unknown; }; + /** SectionParameterModel */ + SectionParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_section + * @constant + * @enum {string} + */ + parameter_type: "gx_section"; + /** Parameters */ + parameters: ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + | components["schemas"]["TextParameterModel"] + | components["schemas"]["IntegerParameterModel"] + | components["schemas"]["FloatParameterModel"] + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["HiddenParameterModel"] + | components["schemas"]["SelectParameterModel"] + | components["schemas"]["DataParameterModel"] + | components["schemas"]["DataCollectionParameterModel"] + | components["schemas"]["DataColumnParameterModel"] + | components["schemas"]["DirectoryUriParameterModel"] + | components["schemas"]["RulesParameterModel"] + | components["schemas"]["DrillDownParameterModel"] + | components["schemas"]["GroupTagParameterModel"] + | components["schemas"]["BaseUrlParameterModel"] + | components["schemas"]["GenomeBuildParameterModel"] + | components["schemas"]["ColorParameterModel"] + | components["schemas"]["ConditionalParameterModel"] + | components["schemas"]["RepeatParameterModel"] + | components["schemas"]["SectionParameterModel"] + )[]; + }; + /** SelectParameterModel */ + SelectParameterModel: { + /** Argument */ + argument?: string | null; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Multiple */ + multiple: boolean; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** Options */ + options?: components["schemas"]["LabelValue"][] | null; + /** + * Parameter Type + * @default gx_select + * @constant + * @enum {string} + */ + parameter_type: "gx_select"; + /** Validators */ + validators: components["schemas"]["NoOptionsParameterValidatorModel"][]; + }; /** ServerDirElement */ ServerDirElement: { /** Md5 */ @@ -16742,31 +18010,86 @@ export interface components { /** Name */ name: string; /** - * Type + * Type + * @constant + * @enum {string} + */ + type: "string"; + }; + /** TestUpdateInstancePayload */ + TestUpdateInstancePayload: { + /** Variables */ + variables?: { + [key: string]: string | boolean | number; + } | null; + }; + /** TestUpgradeInstancePayload */ + TestUpgradeInstancePayload: { + /** Secrets */ + secrets: { + [key: string]: string; + }; + /** Template Version */ + template_version: number; + /** Variables */ + variables: { + [key: string]: string | boolean | number; + }; + }; + /** TextParameterModel */ + TextParameterModel: { + /** + * Area + * @default false + */ + area: boolean; + /** Argument */ + argument?: string | null; + /** + * Default Options + * @default [] + */ + default_options: components["schemas"]["LabelValue"][]; + /** Help */ + help?: string | null; + /** + * Hidden + * @default false + */ + hidden: boolean; + /** + * Is Dynamic + * @default false + */ + is_dynamic: boolean; + /** Label */ + label?: string | null; + /** Name */ + name: string; + /** + * Optional + * @default false + */ + optional: boolean; + /** + * Parameter Type + * @default gx_text * @constant * @enum {string} */ - type: "string"; - }; - /** TestUpdateInstancePayload */ - TestUpdateInstancePayload: { - /** Variables */ - variables?: { - [key: string]: string | boolean | number; - } | null; - }; - /** TestUpgradeInstancePayload */ - TestUpgradeInstancePayload: { - /** Secrets */ - secrets: { - [key: string]: string; - }; - /** Template Version */ - template_version: number; - /** Variables */ - variables: { - [key: string]: string | boolean | number; - }; + parameter_type: "gx_text"; + /** + * Validators + * @default [] + */ + validators: ( + | components["schemas"]["LengthParameterValidatorModel"] + | components["schemas"]["RegexParameterValidatorModel"] + | components["schemas"]["ExpressionParameterValidatorModel"] + | components["schemas"]["EmptyFieldParameterValidatorModel"] + )[]; + /** Value */ + value?: string | null; }; /** ToolDataDetails */ ToolDataDetails: { @@ -16848,6 +18171,25 @@ export interface components { */ values: string; }; + /** ToolRequestModel */ + ToolRequestModel: { + /** + * ID + * @description Encoded ID of the role + * @example 0123456789ABCDEF + */ + id: string; + /** Request */ + request: Record; + state: components["schemas"]["ToolRequestState"]; + /** State Message */ + state_message: string | null; + }; + /** + * ToolRequestState + * @enum {string} + */ + ToolRequestState: "new" | "submitted" | "failed"; /** ToolStep */ ToolStep: { /** @@ -26892,6 +28234,50 @@ export interface operations { }; }; }; + tool_requests_api_histories__history_id__tool_requests_get: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + /** @description The encoded database identifier of the History. */ + history_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ToolRequestModel"][]; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; unpublish_api_histories__history_id__unpublish_put: { parameters: { query?: never; @@ -27809,6 +29195,8 @@ export interface operations { invocation_id?: string | null; /** @description Limit listing of jobs to those that match the specified implicit collection job ID. If none, jobs from any implicit collection execution (or from no implicit collection execution) may be returned. */ implicit_collection_jobs_id?: string | null; + /** @description Limit listing of jobs to those that were created from the supplied tool request ID. If none, jobs from any tool request (or from no workflows) may be returned. */ + tool_request_id?: string | null; /** @description Sort results by specified field. */ order_by?: components["schemas"]["JobIndexSortByEnum"]; /** @description A mix of free text and GitHub-style tags used to filter the index operation. @@ -27901,6 +29289,51 @@ export interface operations { }; }; }; + create_api_jobs_post: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["JobRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["JobCreateResponse"]; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; search_jobs_api_jobs_search_post: { parameters: { query?: never; @@ -28390,7 +29823,10 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["JobOutputAssociation"][]; + "application/json": ( + | components["schemas"]["JobOutputAssociation"] + | components["schemas"]["JobOutputCollectionAssociation"] + )[]; }; }; /** @description Request Error */ @@ -32738,6 +34174,92 @@ export interface operations { }; }; }; + get_tool_request_api_tool_requests__id__get: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ToolRequestModel"]; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; + tool_request_state_api_tool_requests__id__state_get: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": string; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; index_api_tool_shed_repositories_get: { parameters: { query?: { @@ -32916,6 +34438,80 @@ export interface operations { }; }; }; + tool_inputs_api_tools__tool_id__inputs_get: { + parameters: { + query?: { + tool_version?: string | null; + }; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + /** @description The tool ID for the lineage stored in Galaxy's toolbox. */ + tool_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + | components["schemas"]["TextParameterModel"] + | components["schemas"]["IntegerParameterModel"] + | components["schemas"]["FloatParameterModel"] + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["HiddenParameterModel"] + | components["schemas"]["SelectParameterModel"] + | components["schemas"]["DataParameterModel"] + | components["schemas"]["DataCollectionParameterModel"] + | components["schemas"]["DataColumnParameterModel"] + | components["schemas"]["DirectoryUriParameterModel"] + | components["schemas"]["RulesParameterModel"] + | components["schemas"]["DrillDownParameterModel"] + | components["schemas"]["GroupTagParameterModel"] + | components["schemas"]["BaseUrlParameterModel"] + | components["schemas"]["GenomeBuildParameterModel"] + | components["schemas"]["ColorParameterModel"] + | components["schemas"]["ConditionalParameterModel"] + | components["schemas"]["RepeatParameterModel"] + | components["schemas"]["SectionParameterModel"] + )[]; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; index_api_tours_get: { parameters: { query?: never; From b914b1aed720ba20cd25f72803668c3dbd05af54 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 4 Nov 2024 13:22:55 -0500 Subject: [PATCH 08/17] Tool Landing API... --- lib/galaxy/managers/landing.py | 30 +++++++++++-- lib/galaxy/managers/tools.py | 27 ++++++++++++ lib/galaxy/webapps/galaxy/api/tools.py | 47 ++++++++++++++++++--- lib/galaxy/webapps/galaxy/services/jobs.py | 6 +-- lib/galaxy/webapps/galaxy/services/tools.py | 29 +++---------- lib/galaxy_test/api/test_landing.py | 31 ++++++++++++++ lib/galaxy_test/base/populators.py | 10 +++-- test/unit/app/managers/test_landing.py | 31 +++++++++++++- 8 files changed, 170 insertions(+), 41 deletions(-) diff --git a/lib/galaxy/managers/landing.py b/lib/galaxy/managers/landing.py index 5012d2280134..b5bfd49a7359 100644 --- a/lib/galaxy/managers/landing.py +++ b/lib/galaxy/managers/landing.py @@ -30,9 +30,20 @@ WorkflowLandingRequest, ) from galaxy.security.idencoding import IdEncodingHelper -from galaxy.structured_app import StructuredApp +from galaxy.structured_app import ( + MinimalManagerApp, + StructuredApp, +) +from galaxy.tool_util.parameters import ( + landing_decode, + LandingRequestToolState, +) from galaxy.util import safe_str_cmp from .context import ProvidesUserContext +from .tools import ( + get_tool_from_toolbox, + ToolRunReference, +) LandingRequestModel = Union[ToolLandingRequestModel, WorkflowLandingRequestModel] @@ -44,16 +55,27 @@ def __init__( sa_session: galaxy_scoped_session, security: IdEncodingHelper, workflow_contents_manager: WorkflowContentsManager, + app: MinimalManagerApp, ): self.sa_session = sa_session self.security = security self.workflow_contents_manager = workflow_contents_manager + self.app = app def create_tool_landing_request(self, payload: CreateToolLandingRequestPayload, user_id=None) -> ToolLandingRequest: + tool_id = payload.tool_id + tool_version = payload.tool_version + request_state = payload.request_state + + ref = ToolRunReference(tool_id=tool_id, tool_version=tool_version, tool_uuid=None) + tool = get_tool_from_toolbox(self.app.toolbox, ref) + landing_request_state = LandingRequestToolState(request_state or {}) + internal_landing_request_state = landing_decode(landing_request_state, tool, self.security.decode_id) + model = ToolLandingRequestModel() - model.tool_id = payload.tool_id - model.tool_version = payload.tool_version - model.request_state = payload.request_state + model.tool_id = tool_id + model.tool_version = tool_version + model.request_state = internal_landing_request_state.input_state model.uuid = uuid4() model.client_secret = payload.client_secret model.public = payload.public diff --git a/lib/galaxy/managers/tools.py b/lib/galaxy/managers/tools.py index c6dbe471dc84..fa4c42eadefd 100644 --- a/lib/galaxy/managers/tools.py +++ b/lib/galaxy/managers/tools.py @@ -1,5 +1,6 @@ import logging from typing import ( + NamedTuple, Optional, TYPE_CHECKING, Union, @@ -16,8 +17,10 @@ model, ) from galaxy.exceptions import DuplicatedIdentifierException +from galaxy.managers.context import ProvidesUserContext from galaxy.model import DynamicTool from galaxy.tool_util.cwl import tool_proxy +from galaxy.tools import Tool from .base import ( ModelManager, raise_filter_err, @@ -30,6 +33,30 @@ from galaxy.managers.base import OrmFilterParsersType +class ToolRunReference(NamedTuple): + tool_id: Optional[str] + tool_uuid: Optional[str] + tool_version: Optional[str] + + +def get_tool_from_trans(trans: ProvidesUserContext, tool_ref: ToolRunReference) -> Tool: + return get_tool_from_toolbox(trans.app.toolbox, tool_ref) + + +def get_tool_from_toolbox(toolbox, tool_ref: ToolRunReference) -> Tool: + get_kwds = dict( + tool_id=tool_ref.tool_id, + tool_uuid=tool_ref.tool_uuid, + tool_version=tool_ref.tool_version, + ) + + tool = toolbox.get_tool(**get_kwds) + if not tool: + log.debug(f"Not found tool with kwds [{tool_ref}]") + raise exceptions.ToolMissingException("Tool not found.") + return tool + + class DynamicToolManager(ModelManager): """Manages dynamic tools stored in Galaxy's database.""" diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 6914f9ab7bf3..dc0c2ecaeb28 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -17,6 +17,7 @@ Request, UploadFile, ) +from pydantic import UUID4 from starlette.datastructures import UploadFile as StarletteUploadFile from galaxy import ( @@ -26,16 +27,26 @@ ) from galaxy.datatypes.data import get_params_and_input_name from galaxy.managers.collections import DatasetCollectionManager -from galaxy.managers.context import ProvidesHistoryContext +from galaxy.managers.context import ( + ProvidesHistoryContext, + ProvidesUserContext, +) from galaxy.managers.hdas import HDAManager from galaxy.managers.histories import HistoryManager +from galaxy.managers.landing import LandingRequestManager +from galaxy.managers.tools import ToolRunReference from galaxy.model import ToolRequest from galaxy.schema.fetch_data import ( FetchDataFormPayload, FetchDataPayload, ) from galaxy.schema.fields import DecodedDatabaseIdField -from galaxy.schema.schema import ToolRequestModel +from galaxy.schema.schema import ( + ClaimLandingPayload, + CreateToolLandingRequestPayload, + ToolLandingRequest, + ToolRequestModel, +) from galaxy.tool_util.parameters import ToolParameterT from galaxy.tool_util.verify import ToolTestDescriptionDict from galaxy.tools.evaluation import global_tool_errors @@ -49,16 +60,14 @@ from galaxy.webapps.base.controller import UsesVisualizationMixin from galaxy.webapps.base.webapp import GalaxyWebTransaction from galaxy.webapps.galaxy.services.base import tool_request_to_model -from galaxy.webapps.galaxy.services.tools import ( - ToolRunReference, - ToolsService, -) +from galaxy.webapps.galaxy.services.tools import ToolsService from . import ( APIContentTypeRoute, as_form, BaseGalaxyAPIController, depends, DependsOnTrans, + LandingUuidPathParam, Router, ) @@ -105,6 +114,7 @@ async def get_files(request: Request, files: Optional[List[UploadFile]] = None): @router.cbv class FetchTools: service: ToolsService = depends(ToolsService) + landing_manager: LandingRequestManager = depends(LandingRequestManager) @router.post("/api/tools/fetch", summary="Upload files to Galaxy", route_class_override=JsonApiRoute) def fetch_json(self, payload: FetchDataPayload = Body(...), trans: ProvidesHistoryContext = DependsOnTrans): @@ -161,6 +171,31 @@ def _get_tool_request_or_raise_not_found( assert tool_request return tool_request + @router.post("/api/tool_landings", public=True) + def create_landing( + self, + trans: ProvidesUserContext = DependsOnTrans, + tool_landing_request: CreateToolLandingRequestPayload = Body(...), + ) -> ToolLandingRequest: + return self.landing_manager.create_tool_landing_request(tool_landing_request) + + @router.post("/api/tool_landings/{uuid}/claim") + def claim_landing( + self, + trans: ProvidesUserContext = DependsOnTrans, + uuid: UUID4 = LandingUuidPathParam, + payload: Optional[ClaimLandingPayload] = Body(...), + ) -> ToolLandingRequest: + return self.landing_manager.claim_tool_landing_request(trans, uuid, payload) + + @router.get("/api/tool_landings/{uuid}") + def get_landing( + self, + trans: ProvidesUserContext = DependsOnTrans, + uuid: UUID4 = LandingUuidPathParam, + ) -> ToolLandingRequest: + return self.landing_manager.get_tool_landing_request(trans, uuid) + @router.get( "/api/tools/{tool_id}/inputs", summary="Get tool inputs.", diff --git a/lib/galaxy/webapps/galaxy/services/jobs.py b/lib/galaxy/webapps/galaxy/services/jobs.py index 191a5e291cf5..610e1c468d78 100644 --- a/lib/galaxy/webapps/galaxy/services/jobs.py +++ b/lib/galaxy/webapps/galaxy/services/jobs.py @@ -29,6 +29,7 @@ JobSearch, view_show_job, ) +from galaxy.managers.tools import ToolRunReference from galaxy.model import ( Job, ToolRequest, @@ -60,10 +61,7 @@ async_task_summary, ServiceBase, ) -from .tools import ( - ToolRunReference, - validate_tool_for_running, -) +from .tools import validate_tool_for_running log = logging.getLogger(__name__) diff --git a/lib/galaxy/webapps/galaxy/services/tools.py b/lib/galaxy/webapps/galaxy/services/tools.py index bd97238ef67e..256031433304 100644 --- a/lib/galaxy/webapps/galaxy/services/tools.py +++ b/lib/galaxy/webapps/galaxy/services/tools.py @@ -7,7 +7,6 @@ cast, Dict, List, - NamedTuple, Optional, Union, ) @@ -25,6 +24,10 @@ ProvidesUserContext, ) from galaxy.managers.histories import HistoryManager +from galaxy.managers.tools import ( + get_tool_from_trans, + ToolRunReference, +) from galaxy.model import ( LibraryDatasetDatasetAssociation, PostJobAction, @@ -46,26 +49,6 @@ log = logging.getLogger(__name__) -class ToolRunReference(NamedTuple): - tool_id: Optional[str] - tool_uuid: Optional[str] - tool_version: Optional[str] - - -def get_tool(trans: ProvidesHistoryContext, tool_ref: ToolRunReference) -> Tool: - get_kwds = dict( - tool_id=tool_ref.tool_id, - tool_uuid=tool_ref.tool_uuid, - tool_version=tool_ref.tool_version, - ) - - tool = trans.app.toolbox.get_tool(**get_kwds) - if not tool: - log.debug(f"Not found tool with kwds [{tool_ref}]") - raise exceptions.ToolMissingException("Tool not found.") - return tool - - def validate_tool_for_running(trans: ProvidesHistoryContext, tool_ref: ToolRunReference) -> Tool: if trans.user_is_bootstrap_admin: raise exceptions.RealUserRequiredException("Only real users can execute tools or run jobs.") @@ -73,7 +56,7 @@ def validate_tool_for_running(trans: ProvidesHistoryContext, tool_ref: ToolRunRe if tool_ref.tool_id is None and tool_ref.tool_uuid is None: raise exceptions.RequestParameterMissingException("Must specify a valid tool_id to use this endpoint.") - tool = get_tool(trans, tool_ref) + tool = get_tool_from_trans(trans, tool_ref) if not tool.allow_user_access(trans.user): raise exceptions.ItemAccessibilityException("Tool not accessible.") return tool @@ -97,7 +80,7 @@ def inputs( trans: ProvidesHistoryContext, tool_ref: ToolRunReference, ) -> List[ToolParameterT]: - tool = get_tool(trans, tool_ref) + tool = get_tool_from_trans(trans, tool_ref) return tool.parameters def create_fetch( diff --git a/lib/galaxy_test/api/test_landing.py b/lib/galaxy_test/api/test_landing.py index 53182fadae2a..a8f99acb1e01 100644 --- a/lib/galaxy_test/api/test_landing.py +++ b/lib/galaxy_test/api/test_landing.py @@ -5,9 +5,14 @@ ) from galaxy.schema.schema import ( + CreateToolLandingRequestPayload, CreateWorkflowLandingRequestPayload, WorkflowLandingRequest, ) +from galaxy_test.base.api_asserts import ( + assert_error_code_is, + assert_status_code_is, +) from galaxy_test.base.populators import ( DatasetPopulator, skip_without_tool, @@ -25,6 +30,32 @@ def setUp(self): self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.workflow_populator = WorkflowPopulator(self.galaxy_interactor) + @skip_without_tool("cat") + def test_tool_landing(self): + request = CreateToolLandingRequestPayload( + tool_id="create_2", + tool_version=None, + request_state={"sleep_time": 0}, + ) + response = self.dataset_populator.create_tool_landing(request) + assert response.tool_id == "create_2" + assert response.state == "unclaimed" + response = self.dataset_populator.claim_tool_landing(response.uuid) + assert response.tool_id == "create_2" + assert response.state == "claimed" + + @skip_without_tool("gx_int") + def test_tool_landing_invalid(self): + request = CreateToolLandingRequestPayload( + tool_id="gx_int", + tool_version=None, + request_state={"parameter": "foobar"}, + ) + response = self.dataset_populator.create_tool_landing_raw(request) + assert_status_code_is(response, 400) + assert_error_code_is(response, 400008) + assert "Input should be a valid integer" in response.text + @skip_without_tool("cat1") def test_create_public_workflow_landing_authenticated_user(self): request = _get_simple_landing_payload(self.workflow_populator, public=True) diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index f186a194a8c9..1a1068c64020 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -795,13 +795,17 @@ def _wait_for_purge(): return self._get(dataset_url) def create_tool_landing(self, payload: CreateToolLandingRequestPayload) -> ToolLandingRequest: - create_url = "tool_landings" - json = payload.model_dump(mode="json") - create_response = self._post(create_url, json, json=True, anon=True) + create_response = self.create_tool_landing_raw(payload) api_asserts.assert_status_code_is(create_response, 200) create_response.raise_for_status() return ToolLandingRequest.model_validate(create_response.json()) + def create_tool_landing_raw(self, payload: CreateToolLandingRequestPayload) -> Response: + create_url = "tool_landings" + json = payload.model_dump(mode="json") + create_response = self._post(create_url, json, json=True, anon=True) + return create_response + def create_workflow_landing(self, payload: CreateWorkflowLandingRequestPayload) -> WorkflowLandingRequest: create_url = "workflow_landings" json = payload.model_dump(mode="json") diff --git a/test/unit/app/managers/test_landing.py b/test/unit/app/managers/test_landing.py index f2ccb059b4bf..dbfdf73014fc 100644 --- a/test/unit/app/managers/test_landing.py +++ b/test/unit/app/managers/test_landing.py @@ -1,3 +1,7 @@ +from typing import ( + cast, + List, +) from uuid import uuid4 from galaxy.config import GalaxyAppConfiguration @@ -22,6 +26,11 @@ ToolLandingRequest, WorkflowLandingRequest, ) +from galaxy.structured_app import MinimalManagerApp +from galaxy.tool_util.parameters import ( + DataParameterModel, + ToolParameterT, +) from galaxy.workflow.trs_proxy import TrsProxy from .base import BaseTestCase @@ -37,13 +46,33 @@ CLIENT_SECRET = "mycoolsecret" +class MockApp: + + @property + def toolbox(self): + return MockToolbox() + + +class MockToolbox: + + def get_tool(self, tool_id, tool_uuid, tool_version): + return MockTool() + + +class MockTool: + + @property + def parameters(self) -> List[ToolParameterT]: + return [DataParameterModel(name="input1")] + + class TestLanding(BaseTestCase): def setUp(self): super().setUp() self.workflow_contents_manager = WorkflowContentsManager(self.app, self.app.trs_proxy) self.landing_manager = LandingRequestManager( - self.trans.sa_session, self.app.security, self.workflow_contents_manager + self.trans.sa_session, self.app.security, self.workflow_contents_manager, cast(MinimalManagerApp, MockApp()), ) self.trans.app.trs_proxy = TrsProxy(GalaxyAppConfiguration(override_tempdir=False)) From 01d081cffaef6ebc4592afdb58379e2db330f027 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 7 Oct 2024 12:19:54 -0400 Subject: [PATCH 09/17] Regenerate schema --- client/src/api/schema/schema.ts | 215 ++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index bb1e56becc26..4cb35a7068ee 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -4399,6 +4399,57 @@ export interface paths { patch?: never; trace?: never; }; + "/api/tool_landings": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create Landing */ + post: operations["create_landing_api_tool_landings_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/tool_landings/{uuid}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Landing */ + get: operations["get_landing_api_tool_landings__uuid__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/tool_landings/{uuid}/claim": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Claim Landing */ + post: operations["claim_landing_api_tool_landings__uuid__claim_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/tool_requests/{id}": { parameters: { query?: never; @@ -7551,6 +7602,17 @@ export interface components { */ url: string; }; + /** CreateToolLandingRequestPayload */ + CreateToolLandingRequestPayload: { + /** Client Secret */ + client_secret?: string | null; + /** Request State */ + request_state?: Record | null; + /** Tool Id */ + tool_id: string; + /** Tool Version */ + tool_version?: string | null; + }; /** * CreateType * @enum {string} @@ -18171,6 +18233,22 @@ export interface components { */ values: string; }; + /** ToolLandingRequest */ + ToolLandingRequest: { + /** Request State */ + request_state?: Record | null; + state: components["schemas"]["LandingRequestState"]; + /** Tool Id */ + tool_id: string; + /** Tool Version */ + tool_version?: string | null; + /** + * UUID + * Format: uuid4 + * @description Universal unique identifier for this dataset. + */ + uuid: string; + }; /** ToolRequestModel */ ToolRequestModel: { /** @@ -34174,6 +34252,143 @@ export interface operations { }; }; }; + create_landing_api_tool_landings_post: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateToolLandingRequestPayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ToolLandingRequest"]; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; + get_landing_api_tool_landings__uuid__get: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + /** @description The UUID used to identify a persisted landing request. */ + uuid: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ToolLandingRequest"]; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; + claim_landing_api_tool_landings__uuid__claim_post: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + /** @description The UUID used to identify a persisted landing request. */ + uuid: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ClaimLandingPayload"] | null; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ToolLandingRequest"]; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; get_tool_request_api_tool_requests__id__get: { parameters: { query?: never; From 1b2e5cc995a1186ff3da29d52a901deb955d2299 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 30 Sep 2024 15:35:00 -0400 Subject: [PATCH 10/17] More json schema API options... --- lib/galaxy/tool_util/parameters/__init__.py | 2 + lib/galaxy/webapps/galaxy/api/__init__.py | 13 +++++ lib/galaxy/webapps/galaxy/api/tools.py | 64 ++++++++++++++++++++- lib/galaxy_test/api/test_tools.py | 27 +++++++++ lib/tool_shed/webapp/api2/tools.py | 43 +++++++++++--- 5 files changed, 139 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/tool_util/parameters/__init__.py b/lib/galaxy/tool_util/parameters/__init__.py index 22dd7e6053aa..3435f81d25e1 100644 --- a/lib/galaxy/tool_util/parameters/__init__.py +++ b/lib/galaxy/tool_util/parameters/__init__.py @@ -66,6 +66,7 @@ ValidationFunctionT, ) from .state import ( + HasToolParameters, JobInternalToolState, LandingRequestInternalToolState, LandingRequestToolState, @@ -140,6 +141,7 @@ "ToolState", "TestCaseToolState", "ToolParameterT", + "HasToolParameters", "to_json_schema_string", "test_case_state", "validate_test_cases_for_tool_source", diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index 95ec6cf4069a..8d2db1544981 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -81,6 +81,11 @@ from galaxy.schema.fields import DecodedDatabaseIdField from galaxy.security.idencoding import IdEncodingHelper from galaxy.structured_app import StructuredApp +from galaxy.tool_util.parameters import ( + HasToolParameters, + to_json_schema_string, + ToolState, +) from galaxy.web.framework.decorators import require_admin_message from galaxy.webapps.base.controller import BaseAPIController from galaxy.webapps.galaxy.api.cbv import cbv @@ -611,6 +616,14 @@ async def _as_form(**data): return cls +def json_schema_response_for_tool_state_model( + state_type: Type[ToolState], has_parameters: HasToolParameters +) -> Response: + pydantic_model = state_type.parameter_model_for(has_parameters) + json_str = to_json_schema_string(pydantic_model) + return Response(content=json_str, media_type="application/json") + + async def try_get_request_body_as_json(request: Request) -> Optional[Any]: """Returns the request body as a JSON object if the content type is JSON.""" if "application/json" in request.headers.get("content-type", ""): diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index dc0c2ecaeb28..d2591bf7335a 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -15,6 +15,7 @@ Path, Query, Request, + Response, UploadFile, ) from pydantic import UUID4 @@ -47,7 +48,12 @@ ToolLandingRequest, ToolRequestModel, ) -from galaxy.tool_util.parameters import ToolParameterT +from galaxy.tool_util.parameters import ( + LandingRequestToolState, + RequestToolState, + TestCaseToolState, + ToolParameterT, +) from galaxy.tool_util.verify import ToolTestDescriptionDict from galaxy.tools.evaluation import global_tool_errors from galaxy.util.zipstream import ZipstreamWrapper @@ -67,6 +73,7 @@ BaseGalaxyAPIController, depends, DependsOnTrans, + json_schema_response_for_tool_state_model, LandingUuidPathParam, Router, ) @@ -209,6 +216,61 @@ def tool_inputs( tool_run_ref = ToolRunReference(tool_id=tool_id, tool_version=tool_version, tool_uuid=None) return self.service.inputs(trans, tool_run_ref) + @router.get( + "/api/tools/{tool_id}/parameter_request_schema", + operation_id="tools__parameter_request_schema", + summary="Return a JSON schema description of the tool's inputs for the tool request API that will be added to Galaxy at some point", + description="The tool request schema includes validation of map/reduce concepts that can be consumed by the tool execution API and not just the request for a single execution.", + ) + def tool_state_request( + self, + tool_id: str = ToolIDPathParam, + tool_version: Optional[str] = ToolVersionQueryParam, + trans: ProvidesHistoryContext = DependsOnTrans, + ) -> Response: + tool_run_ref = ToolRunReference(tool_id=tool_id, tool_version=tool_version, tool_uuid=None) + inputs = self.service.inputs(trans, tool_run_ref) + return json_schema_response_for_tool_state_model( + RequestToolState, + inputs, + ) + + @router.get( + "/api/tools/{tool_id}/parameter_landing_request_schema", + operation_id="tools__parameter_landing_request_schema", + summary="Return a JSON schema description of the tool's inputs for the tool landing request API.", + ) + def tool_state_landing_request( + self, + tool_id: str = ToolIDPathParam, + tool_version: Optional[str] = ToolVersionQueryParam, + trans: ProvidesHistoryContext = DependsOnTrans, + ) -> Response: + tool_run_ref = ToolRunReference(tool_id=tool_id, tool_version=tool_version, tool_uuid=None) + inputs = self.service.inputs(trans, tool_run_ref) + return json_schema_response_for_tool_state_model( + LandingRequestToolState, + inputs, + ) + + @router.get( + "/api/tools/{tool_id}/parameter_test_case_xml_schema", + operation_id="tools__parameter_test_case_xml_schema", + summary="Return a JSON schema description of the tool's inputs for test case construction.", + ) + def tool_state_test_case_xml( + self, + tool_id: str = ToolIDPathParam, + tool_version: Optional[str] = ToolVersionQueryParam, + trans: ProvidesHistoryContext = DependsOnTrans, + ) -> Response: + tool_run_ref = ToolRunReference(tool_id=tool_id, tool_version=tool_version, tool_uuid=None) + inputs = self.service.inputs(trans, tool_run_ref) + return json_schema_response_for_tool_state_model( + TestCaseToolState, + inputs, + ) + class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin): """ diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index 41d982d246c9..ff2d9a6fe7a0 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -13,6 +13,8 @@ from uuid import uuid4 import pytest +from jsonschema import validate +from jsonschema.exceptions import ValidationError from requests import ( get, put, @@ -247,6 +249,31 @@ def test_legacy_biotools_xref_injection(self): assert xref["reftype"] == "bio.tools" assert xref["value"] == "bwa" + @skip_without_tool("gx_int") + def test_tool_schemas(self): + tool_id = "gx_int" + + def get_jsonschema(state_type: str): + schema_url = self._api_url(f"tools/{tool_id}/parameter_{state_type}_schema") + schema_response = get(schema_url) + schema_response.raise_for_status() + return schema_response.json() + + request_schema = get_jsonschema("request") + validate(instance={"parameter": 5}, schema=request_schema) + with pytest.raises(ValidationError): + validate(instance={"parameter": "Foobar"}, schema=request_schema) + + test_case_schema = get_jsonschema("test_case_xml") + validate(instance={"parameter": 5}, schema=test_case_schema) + with pytest.raises(ValidationError): + validate(instance={"parameter": "Foobar"}, schema=test_case_schema) + + landing_schema = get_jsonschema("landing_request") + validate(instance={"parameter": 5}, schema=landing_schema) + with pytest.raises(ValidationError): + validate(instance={"parameter": "Foobar"}, schema=landing_schema) + @skip_without_tool("test_data_source") @skip_if_github_down def test_data_source_ok_request(self): diff --git a/lib/tool_shed/webapp/api2/tools.py b/lib/tool_shed/webapp/api2/tools.py index be5d04da9aeb..b8709060eda9 100644 --- a/lib/tool_shed/webapp/api2/tools.py +++ b/lib/tool_shed/webapp/api2/tools.py @@ -9,9 +9,11 @@ from galaxy.tool_util.models import ParsedTool from galaxy.tool_util.parameters import ( + LandingRequestToolState, RequestToolState, - to_json_schema_string, + TestCaseToolState, ) +from galaxy.webapps.galaxy.api import json_schema_response_for_tool_state_model from tool_shed.context import SessionRequestContext from tool_shed.managers.tools import ( parsed_tool_model_cached_for, @@ -57,11 +59,6 @@ ) -def json_schema_response(pydantic_model) -> Response: - json_str = to_json_schema_string(pydantic_model) - return Response(content=json_str, media_type="application/json") - - @router.cbv class FastAPITools: app: ToolShedApp = depends(ToolShedApp) @@ -158,15 +155,43 @@ def show_tool( @router.get( "/api/tools/{tool_id}/versions/{tool_version}/parameter_request_schema", - operation_id="tools__parameter_request_model", + operation_id="tools__parameter_request_schema", summary="Return a JSON schema description of the tool's inputs for the tool request API that will be added to Galaxy at some point", description="The tool request schema includes validation of map/reduce concepts that can be consumed by the tool execution API and not just the request for a single execution.", ) - def tool_state( + def tool_state_request( + self, + trans: SessionRequestContext = DependsOnTrans, + tool_id: str = TOOL_ID_PATH_PARAM, + tool_version: str = TOOL_VERSION_PATH_PARAM, + ) -> Response: + parsed_tool = parsed_tool_model_cached_for(trans, tool_id, tool_version) + return json_schema_response_for_tool_state_model(RequestToolState, parsed_tool.inputs) + + @router.get( + "/api/tools/{tool_id}/versions/{tool_version}/parameter_landing_request_schema", + operation_id="tools__parameter_landing_request_schema", + summary="Return a JSON schema description of the tool's inputs for the tool landing request API.", + ) + def tool_state_landing_request( + self, + trans: SessionRequestContext = DependsOnTrans, + tool_id: str = TOOL_ID_PATH_PARAM, + tool_version: str = TOOL_VERSION_PATH_PARAM, + ) -> Response: + parsed_tool = parsed_tool_model_cached_for(trans, tool_id, tool_version) + return json_schema_response_for_tool_state_model(LandingRequestToolState, parsed_tool.inputs) + + @router.get( + "/api/tools/{tool_id}/versions/{tool_version}/parameter_test_case_xml_schema", + operation_id="tools__parameter_test_case_xml_schema", + summary="Return a JSON schema description of the tool's inputs for test case construction.", + ) + def tool_state_test_case_xml( self, trans: SessionRequestContext = DependsOnTrans, tool_id: str = TOOL_ID_PATH_PARAM, tool_version: str = TOOL_VERSION_PATH_PARAM, ) -> Response: parsed_tool = parsed_tool_model_cached_for(trans, tool_id, tool_version) - return json_schema_response(RequestToolState.parameter_model_for(parsed_tool.inputs)) + return json_schema_response_for_tool_state_model(TestCaseToolState, parsed_tool.inputs) From 0f5fb1c296e4921f0337e17ec32430e077aa52f5 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 30 Sep 2024 18:55:09 -0400 Subject: [PATCH 11/17] Update API schema for more tool state schema APIs... --- client/src/api/schema/schema.ts | 192 ++++++++++++++++++ .../webapp/frontend/src/schema/schema.ts | 124 ++++++++++- 2 files changed, 314 insertions(+), 2 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 4cb35a7068ee..39e3a778258c 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -4569,6 +4569,60 @@ export interface paths { patch?: never; trace?: never; }; + "/api/tools/{tool_id}/parameter_landing_request_schema": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Return a JSON schema description of the tool's inputs for the tool landing request API. */ + get: operations["tools__parameter_landing_request_schema"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/tools/{tool_id}/parameter_request_schema": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Return a JSON schema description of the tool's inputs for the tool request API that will be added to Galaxy at some point + * @description The tool request schema includes validation of map/reduce concepts that can be consumed by the tool execution API and not just the request for a single execution. + */ + get: operations["tools__parameter_request_schema"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/tools/{tool_id}/parameter_test_case_xml_schema": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Return a JSON schema description of the tool's inputs for test case construction. */ + get: operations["tools__parameter_test_case_xml_schema"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/tours": { parameters: { query?: never; @@ -34727,6 +34781,144 @@ export interface operations { }; }; }; + tools__parameter_landing_request_schema: { + parameters: { + query?: { + tool_version?: string | null; + }; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + /** @description The tool ID for the lineage stored in Galaxy's toolbox. */ + tool_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; + tools__parameter_request_schema: { + parameters: { + query?: { + tool_version?: string | null; + }; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + /** @description The tool ID for the lineage stored in Galaxy's toolbox. */ + tool_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; + tools__parameter_test_case_xml_schema: { + parameters: { + query?: { + tool_version?: string | null; + }; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + /** @description The tool ID for the lineage stored in Galaxy's toolbox. */ + tool_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; index_api_tours_get: { parameters: { query?: never; diff --git a/lib/tool_shed/webapp/frontend/src/schema/schema.ts b/lib/tool_shed/webapp/frontend/src/schema/schema.ts index 14aab45a3c36..d52683b36513 100644 --- a/lib/tool_shed/webapp/frontend/src/schema/schema.ts +++ b/lib/tool_shed/webapp/frontend/src/schema/schema.ts @@ -532,6 +532,23 @@ export interface paths { patch?: never trace?: never } + "/api/tools/{tool_id}/versions/{tool_version}/parameter_landing_request_schema": { + parameters: { + query?: never + header?: never + path?: never + cookie?: never + } + /** Return a JSON schema description of the tool's inputs for the tool landing request API. */ + get: operations["tools__parameter_landing_request_schema"] + put?: never + post?: never + delete?: never + options?: never + head?: never + patch?: never + trace?: never + } "/api/tools/{tool_id}/versions/{tool_version}/parameter_request_schema": { parameters: { query?: never @@ -543,7 +560,24 @@ export interface paths { * Return a JSON schema description of the tool's inputs for the tool request API that will be added to Galaxy at some point * @description The tool request schema includes validation of map/reduce concepts that can be consumed by the tool execution API and not just the request for a single execution. */ - get: operations["tools__parameter_request_model"] + get: operations["tools__parameter_request_schema"] + put?: never + post?: never + delete?: never + options?: never + head?: never + patch?: never + trace?: never + } + "/api/tools/{tool_id}/versions/{tool_version}/parameter_test_case_xml_schema": { + parameters: { + query?: never + header?: never + path?: never + cookie?: never + } + /** Return a JSON schema description of the tool's inputs for test case construction. */ + get: operations["tools__parameter_test_case_xml_schema"] put?: never post?: never delete?: never @@ -4422,7 +4456,93 @@ export interface operations { } } } - tools__parameter_request_model: { + tools__parameter_landing_request_schema: { + parameters: { + query?: never + header?: never + path: { + /** @description See also https://ga4gh.github.io/tool-registry-service-schemas/DataModel/#trs-tool-and-trs-tool-version-ids */ + tool_id: string + /** @description The full version string defined on the Galaxy tool wrapper. */ + tool_version: string + } + cookie?: never + } + requestBody?: never + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown + } + content: { + "application/json": unknown + } + } + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown + } + content: { + "application/json": components["schemas"]["MessageExceptionModel"] + } + } + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown + } + content: { + "application/json": components["schemas"]["MessageExceptionModel"] + } + } + } + } + tools__parameter_request_schema: { + parameters: { + query?: never + header?: never + path: { + /** @description See also https://ga4gh.github.io/tool-registry-service-schemas/DataModel/#trs-tool-and-trs-tool-version-ids */ + tool_id: string + /** @description The full version string defined on the Galaxy tool wrapper. */ + tool_version: string + } + cookie?: never + } + requestBody?: never + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown + } + content: { + "application/json": unknown + } + } + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown + } + content: { + "application/json": components["schemas"]["MessageExceptionModel"] + } + } + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown + } + content: { + "application/json": components["schemas"]["MessageExceptionModel"] + } + } + } + } + tools__parameter_test_case_xml_schema: { parameters: { query?: never header?: never From c4eb000d8785686579ea0196930bd9ff6540ad9e Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 27 Nov 2024 10:49:21 -0500 Subject: [PATCH 12/17] Latest prototype of a minimal workflow UI. --- .../components/ActivityBar/ActivityBar.vue | 11 +++ .../Workflow/List/WorkflowActionsExtend.vue | 2 + .../components/Workflow/List/WorkflowCard.vue | 5 +- .../components/Workflow/List/WorkflowList.vue | 22 +++++- .../Workflow/List/WorkflowListActions.vue | 9 +++ .../src/entry/analysis/modules/Analysis.vue | 10 ++- client/src/entry/analysis/modules/Home.vue | 11 +++ .../analysis/modules/WorkflowLanding.vue | 16 +++++ client/src/stores/activitySetup.ts | 69 +++++++++++++++---- client/src/stores/activityStore.ts | 65 +++++++++++++++-- lib/galaxy/config/schemas/config_schema.yml | 27 ++++++++ lib/galaxy/managers/configuration.py | 2 + 12 files changed, 224 insertions(+), 25 deletions(-) create mode 100644 client/src/entry/analysis/modules/WorkflowLanding.vue diff --git a/client/src/components/ActivityBar/ActivityBar.vue b/client/src/components/ActivityBar/ActivityBar.vue index 5371d14f84fe..22dba51a22a6 100644 --- a/client/src/components/ActivityBar/ActivityBar.vue +++ b/client/src/components/ActivityBar/ActivityBar.vue @@ -193,6 +193,17 @@ defineExpose({ isActiveSideBar, setActiveSideBar, }); + +const syncActivities = () => { + activityStore.sync(); + if (config.value && ["workflow_centric", "workflow_runner"].indexOf(config.value.client_mode) >= 0) { + userStore.untoggleToolbarIfNeeded(); + } +}; + +watch(() => hashedUserId.value, syncActivities); + +watch(isConfigLoaded, syncActivities); - + @@ -13,11 +18,14 @@ import WorkflowRun from "components/Workflow/Run/WorkflowRun"; import decodeUriComponent from "decode-uri-component"; import CenterFrame from "entry/analysis/modules/CenterFrame"; +import WorkflowLanding from "./WorkflowLanding.vue"; + export default { components: { CenterFrame, ToolForm, WorkflowRun, + WorkflowLanding, }, props: { config: { @@ -30,6 +38,9 @@ export default { }, }, computed: { + isWorkflowCentric() { + return ["workflow_centric", "workflow_runner"].indexOf(this.config.client_mode) >= 0; + }, isController() { return this.query.m_c && this.query.m_a; }, diff --git a/client/src/entry/analysis/modules/WorkflowLanding.vue b/client/src/entry/analysis/modules/WorkflowLanding.vue new file mode 100644 index 000000000000..f733c1d1993c --- /dev/null +++ b/client/src/entry/analysis/modules/WorkflowLanding.vue @@ -0,0 +1,16 @@ + + + diff --git a/client/src/stores/activitySetup.ts b/client/src/stores/activitySetup.ts index af494b4b0409..5b3b548f49f6 100644 --- a/client/src/stores/activitySetup.ts +++ b/client/src/stores/activitySetup.ts @@ -17,22 +17,42 @@ import { faWrench, } from "@fortawesome/free-solid-svg-icons"; -import { type Activity } from "@/stores/activityStore"; +import { type Activity, type ClientMode, type RawActivity } from "@/stores/activityStore"; import { type EventData } from "@/stores/eventStore"; -export const defaultActivities = [ +function isWorkflowCentric(clientMode: ClientMode): boolean { + return ["workflow_centric", "workflow_runner"].indexOf(clientMode) >= 0; +} + +function unlessWorkflowCentric(clientMode: ClientMode): boolean { + if (isWorkflowCentric(clientMode)) { + return false; + } else { + return true; + } +} + +function ifWorkflowCentric(clientMode: ClientMode): boolean { + if (isWorkflowCentric(clientMode)) { + return true; + } else { + return false; + } +} + +export const ActivitiesRaw: RawActivity[] = [ { anonymous: false, description: "Displays currently running interactive tools (ITs), if these are enabled by the administrator.", icon: faLaptop, id: "interactivetools", mutable: false, - optional: false, + optional: ifWorkflowCentric, panel: false, title: "Interactive Tools", tooltip: "Show active interactive tools", to: "/interactivetool_entry_points/list", - visible: true, + visible: unlessWorkflowCentric, }, { anonymous: true, @@ -40,12 +60,12 @@ export const defaultActivities = [ icon: faUpload, id: "upload", mutable: false, - optional: false, + optional: ifWorkflowCentric, panel: false, title: "Upload", to: null, tooltip: "Download from URL or upload files from disk", - visible: true, + visible: unlessWorkflowCentric, }, { anonymous: true, @@ -53,12 +73,12 @@ export const defaultActivities = [ icon: faWrench, id: "tools", mutable: false, - optional: false, + optional: ifWorkflowCentric, panel: true, title: "Tools", - to: null, + to: "/tools", tooltip: "Search and run tools", - visible: true, + visible: unlessWorkflowCentric, }, { anonymous: true, @@ -97,7 +117,7 @@ export const defaultActivities = [ title: "Visualization", to: null, tooltip: "Visualize datasets", - visible: true, + visible: unlessWorkflowCentric, }, { anonymous: true, @@ -110,7 +130,7 @@ export const defaultActivities = [ title: "Histories", tooltip: "Show all histories", to: "/histories/list", - visible: true, + visible: unlessWorkflowCentric, }, { anonymous: false, @@ -123,7 +143,7 @@ export const defaultActivities = [ title: "History Multiview", tooltip: "Select histories to show in History Multiview", to: "/histories/view_multiple", - visible: true, + visible: unlessWorkflowCentric, }, { anonymous: false, @@ -136,7 +156,7 @@ export const defaultActivities = [ title: "Datasets", tooltip: "Show all datasets", to: "/datasets/list", - visible: true, + visible: unlessWorkflowCentric, }, { anonymous: true, @@ -149,7 +169,7 @@ export const defaultActivities = [ title: "Pages", tooltip: "Show all pages", to: "/pages/list", - visible: true, + visible: unlessWorkflowCentric, }, { anonymous: false, @@ -162,10 +182,29 @@ export const defaultActivities = [ title: "Libraries", tooltip: "Access data libraries", to: "/libraries", - visible: true, + visible: unlessWorkflowCentric, }, ] as const; +function resolveActivity(activity: RawActivity, clientMode: ClientMode): Activity { + let optional = activity.optional; + let visible = activity.visible; + if (typeof optional === "function") { + optional = optional(clientMode); + } + if (typeof visible === "function") { + visible = visible(clientMode); + } + return { ...activity, optional, visible }; +} + +export function getActivities(clientMode: ClientMode) { + const resolve = (activity: RawActivity) => { + return resolveActivity(activity, clientMode); + }; + return ActivitiesRaw.map(resolve); +} + export function convertDropData(data: EventData): Activity | null { if (data.history_content_type === "dataset") { return { diff --git a/client/src/stores/activityStore.ts b/client/src/stores/activityStore.ts index 466ee226bf00..f068e26bc960 100644 --- a/client/src/stores/activityStore.ts +++ b/client/src/stores/activityStore.ts @@ -6,13 +6,14 @@ import { useDebounceFn, watchImmediate } from "@vueuse/core"; import { computed, type Ref, ref, set } from "vue"; import { useHashedUserId } from "@/composables/hashedUserId"; +import { useConfig } from "@/composables/config"; import { useUserLocalStorage } from "@/composables/userLocalStorage"; import { ensureDefined } from "@/utils/assertions"; -import { defaultActivities } from "./activitySetup"; import { defineScopedStore } from "./scopedStore"; export type ActivityVariant = "primary" | "danger" | "disabled"; +import { getActivities } from "./activitySetup"; export interface Activity { // determine wether an anonymous user can access this activity @@ -52,12 +53,42 @@ function defaultActivityMeta(): ActivityMeta { }; } +export type ClientMode = "full" | "workflow_centric" | "workflow_runner"; + +// config materializes a RawActivity into an Activity +export interface RawActivity { + // determine wether an anonymous user can access this activity + anonymous: boolean; + // description of the activity + description: string; + // unique identifier + id: string; + // icon to be displayed in activity bar + icon: string; + // indicate if this activity can be modified and/or deleted + mutable: boolean; + // indicate wether this activity can be disabled by the user + optional: boolean | ((mode: ClientMode) => boolean); + // specifiy wether this activity utilizes the side panel + panel: boolean; + // title to be displayed in the activity bar + title: string; + // route to be executed upon selecting the activity + to: string | null; + // tooltip to be displayed when hovering above the icon + tooltip: string; + // indicate wether the activity should be visible by default + visible: boolean | ((mode: ClientMode) => boolean); +} + export const useActivityStore = defineScopedStore("activityStore", (scope) => { const activities: Ref> = useUserLocalStorage(`activity-store-activities-${scope}`, []); const activityMeta: Ref> = ref({}); const { hashedUserId } = useHashedUserId(); + const defaultActivities = ref([]); + const customDefaultActivities = ref(null); const currentDefaultActivities = computed(() => customDefaultActivities.value ?? defaultActivities); @@ -67,6 +98,12 @@ export const useActivityStore = defineScopedStore("activityStore", (scope) => { toggledSideBar.value = toggledSideBar.value === currentOpen ? "" : currentOpen; } + function untoggleToolbarIfNeeded() { + if (toggledSideBar.value == "tools") { + toggledSideBar.value = ""; + } + } + function overrideDefaultActivities(activities: Activity[]) { customDefaultActivities.value = activities; sync(); @@ -82,6 +119,12 @@ export const useActivityStore = defineScopedStore("activityStore", (scope) => { */ function restore() { activities.value = currentDefaultActivities.value.slice(); + const { config, isConfigLoaded } = useConfig(); + if (!isConfigLoaded.value) { + return; + } + + activities.value = getActivities(config.value.client_mode); } /** @@ -97,6 +140,18 @@ export const useActivityStore = defineScopedStore("activityStore", (scope) => { activitiesMap[a.id] = a; }); + const { config, isConfigLoaded } = useConfig(); + if (!isConfigLoaded.value) { + return; + } + + defaultActivities.value = getActivities(config.value.client_mode); + + const activityDefs = getActivities(config.value.client_mode); + activityDefs.forEach((a) => { + activitiesMap[a.id] = a; + }); + // create an updated array of activities const newActivities: Array = []; const foundActivity = new Set(); @@ -121,7 +176,7 @@ export const useActivityStore = defineScopedStore("activityStore", (scope) => { }); // add new built-in activities - currentDefaultActivities.value.forEach((a) => { + activityDefs.forEach((a) => { if (!foundActivity.has(a.id)) { newActivities.push({ ...a }); } @@ -134,7 +189,7 @@ export const useActivityStore = defineScopedStore("activityStore", (scope) => { // if toggled side-bar does not exist, choose the first option if (toggledSideBar.value !== "") { - const allSideBars = activities.value.flatMap((activity) => { + const allSideBars = activities.value.flatMap((activity: Activity) => { if (activity.panel) { return [activity.id]; } else { @@ -149,6 +204,7 @@ export const useActivityStore = defineScopedStore("activityStore", (scope) => { toggledSideBar.value = firstSideBar; } } + }, 10); function getAll() { @@ -199,6 +255,7 @@ export const useActivityStore = defineScopedStore("activityStore", (scope) => { return { toggledSideBar, toggleSideBar, + untoggleToolbarIfNeeded, activities, activityMeta, metaForId, @@ -213,4 +270,4 @@ export const useActivityStore = defineScopedStore("activityStore", (scope) => { overrideDefaultActivities, resetDefaultActivities, }; -}); +}); \ No newline at end of file diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index d1e55b6a6eac..4c1d03021704 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -3135,6 +3135,33 @@ mapping: When false, the most recently added compatible item in the history will be used for each "Set at Runtime" input, independent of others in the workflow. + client_mode: + type: str + default: 'full' + enum: ['full', 'workflow_centric', 'workflow_runner'] + required: false + per_host: true + desc: | + This will change the modality and focus on the client UI. The traditional + full Galaxy with default activity bar is the default of 'full'. + 'workflow_centric' & 'workflow_runner' yield client applications + that are geared to center a collection of workflows in Galaxy and attempts + to hide the concept of histories from users. The 'workflow_centric' view + still allows the user to manage & edit a collection of their own workflows. + 'workflow_runner' is a mode that disables workflow management to even further + simplify the UI - this may be appropriate for instances that really just want + enable particular workflows as-is. + + simplified_workflow_landing_initial_filter_text: + type: str + required: false + per_host: true + desc: | + If the Galaxy client is in 'workflow_centric' or 'workflow_runner' "client mode", + this controls the initial filtering of the workflow search textbox. This can + be used to foreground workflows in the published workflow list by tar (e.g. 'tag:XXX') + or username (e.g. 'username:XXXX'). + simplified_workflow_run_ui: type: str default: 'prefer' diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py index d0ccfa8d53c2..8a6d30e8454d 100644 --- a/lib/galaxy/managers/configuration.py +++ b/lib/galaxy/managers/configuration.py @@ -164,6 +164,8 @@ def _config_is_truthy(item, key, **context): "enable_unique_workflow_defaults": _use_config, "enable_beta_markdown_export": _use_config, "enable_beacon_integration": _use_config, + "client_mode": _use_config, + "simplified_workflow_landing_initial_filter_text": _use_config, "simplified_workflow_run_ui": _use_config, "simplified_workflow_run_ui_target_history": _use_config, "simplified_workflow_run_ui_job_cache": _use_config, From abc5a411a2590261de7f0b300850c250fd9eca38 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 27 Aug 2024 12:23:22 -0400 Subject: [PATCH 13/17] gxformat2 abstraction layer... --- lib/galaxy/managers/workflows.py | 41 +++++++---------------------- lib/galaxy/workflow/format2.py | 36 +++++++++++++++++++++++++ test/unit/workflows/test_convert.py | 35 ++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 31 deletions(-) create mode 100644 lib/galaxy/workflow/format2.py create mode 100644 test/unit/workflows/test_convert.py diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index 20354c029198..6e0ae257e46f 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -14,13 +14,6 @@ ) import sqlalchemy -import yaml -from gxformat2 import ( - from_galaxy_native, - ImporterGalaxyInterface, - ImportOptions, - python_to_workflow, -) from gxformat2.abstract import from_dict from gxformat2.cytoscape import to_cytoscape from gxformat2.yaml import ordered_dump @@ -117,6 +110,10 @@ RawTextTerm, ) from galaxy.work.context import WorkRequestContext +from galaxy.workflow.format2 import ( + convert_from_format2, + convert_to_format2, +) from galaxy.workflow.modules import ( module_factory, ToolModule, @@ -616,10 +613,7 @@ def read_workflow_from_path(self, app, user, path, allow_in_directory=None) -> m workflow_class, as_dict, object_id = artifact_class(trans, as_dict, allow_in_directory=allow_in_directory) assert workflow_class == "GalaxyWorkflow" # Format 2 Galaxy workflow. - galaxy_interface = Format2ConverterGalaxyInterface() - import_options = ImportOptions() - import_options.deduplicate_subworkflows = True - as_dict = python_to_workflow(as_dict, galaxy_interface, workflow_directory=None, import_options=import_options) + as_dict = convert_from_format2(as_dict, None) raw_description = RawWorkflowDescription(as_dict, path) created_workflow = self.build_workflow_from_raw_description(trans, raw_description, WorkflowCreateOptions()) return created_workflow.workflow @@ -646,15 +640,7 @@ def normalize_workflow_format(self, trans, as_dict): workflow_class, as_dict, object_id = artifact_class(trans, as_dict) if workflow_class == "GalaxyWorkflow" or "yaml_content" in as_dict: # Format 2 Galaxy workflow. - galaxy_interface = Format2ConverterGalaxyInterface() - import_options = ImportOptions() - import_options.deduplicate_subworkflows = True - try: - as_dict = python_to_workflow( - as_dict, galaxy_interface, workflow_directory=workflow_directory, import_options=import_options - ) - except yaml.scanner.ScannerError as e: - raise exceptions.MalformedContents(str(e)) + as_dict = convert_from_format2(as_dict, workflow_directory) return RawWorkflowDescription(as_dict, workflow_path) @@ -918,8 +904,8 @@ def workflow_to_dict(self, trans, stored, style="export", version=None, history= fields like 'url' and 'url' and actual unencoded step ids instead of 'order_index'. """ - def to_format_2(wf_dict, **kwds): - return from_galaxy_native(wf_dict, None, **kwds) + def to_format_2(wf_dict, json_wrapper: bool): + return convert_to_format2(wf_dict, json_wrapper=json_wrapper) if version == "": version = None @@ -940,7 +926,7 @@ def to_format_2(wf_dict, **kwds): wf_dict = self._workflow_to_dict_preview(trans, workflow=workflow) elif style == "format2": wf_dict = self._workflow_to_dict_export(trans, stored, workflow=workflow) - wf_dict = to_format_2(wf_dict) + wf_dict = to_format_2(wf_dict, json_wrapper=False) elif style == "format2_wrapped_yaml": wf_dict = self._workflow_to_dict_export(trans, stored, workflow=workflow) wf_dict = to_format_2(wf_dict, json_wrapper=True) @@ -990,7 +976,7 @@ def store_workflow_to_path(self, workflow_path, stored_workflow, workflow, **kwd ordered_dump(abstract_dict, f) else: wf_dict = self._workflow_to_dict_export(trans, stored_workflow, workflow=workflow) - wf_dict = from_galaxy_native(wf_dict, None, json_wrapper=True) + wf_dict = convert_to_format2(wf_dict, json_wrapper=True) f.write(wf_dict["yaml_content"]) def _workflow_to_dict_run(self, trans, stored, workflow, history=None): @@ -2190,13 +2176,6 @@ def __init__(self, as_dict, workflow_path=None): self.workflow_path = workflow_path -class Format2ConverterGalaxyInterface(ImporterGalaxyInterface): - def import_workflow(self, workflow, **kwds): - raise NotImplementedError( - "Direct format 2 import of nested workflows is not yet implemented, use bioblend client." - ) - - def _get_stored_workflow(session, workflow_uuid, workflow_id, by_stored_id): stmt = select(StoredWorkflow) if workflow_uuid is not None: diff --git a/lib/galaxy/workflow/format2.py b/lib/galaxy/workflow/format2.py new file mode 100644 index 000000000000..a9ddca8073d0 --- /dev/null +++ b/lib/galaxy/workflow/format2.py @@ -0,0 +1,36 @@ +from typing import Optional + +import yaml +from gxformat2 import ( + from_galaxy_native, + ImporterGalaxyInterface, + ImportOptions, + python_to_workflow, +) + +from galaxy.exceptions import MalformedContents + + +def convert_to_format2(as_dict, json_wrapper: bool): + return from_galaxy_native(as_dict, None, json_wrapper=json_wrapper) + + +def convert_from_format2(as_dict, workflow_directory: Optional[str]): + # Format 2 Galaxy workflow. + galaxy_interface = Format2ConverterGalaxyInterface() + import_options = ImportOptions() + import_options.deduplicate_subworkflows = True + try: + as_dict = python_to_workflow( + as_dict, galaxy_interface, workflow_directory=workflow_directory, import_options=import_options + ) + except yaml.scanner.ScannerError as e: + raise MalformedContents(str(e)) + return as_dict + + +class Format2ConverterGalaxyInterface(ImporterGalaxyInterface): + def import_workflow(self, workflow, **kwds): + raise NotImplementedError( + "Direct format 2 import of nested workflows is not yet implemented, use bioblend client." + ) diff --git a/test/unit/workflows/test_convert.py b/test/unit/workflows/test_convert.py new file mode 100644 index 000000000000..4567e9afd275 --- /dev/null +++ b/test/unit/workflows/test_convert.py @@ -0,0 +1,35 @@ +from galaxy.workflow.format2 import ( + convert_from_format2, + convert_to_format2, +) + +WORKFLOW_WITH_OUTPUTS = """ +class: GalaxyWorkflow +inputs: + input1: data +outputs: + wf_output_1: + outputSource: first_cat/out_file1 +steps: + first_cat: + tool_id: cat1 + in: + input1: input1 + queries_0|input2: input1 +""" + + +def test_convert_from_simple(): + as_dict = {"yaml_content": WORKFLOW_WITH_OUTPUTS} + ga_format = convert_from_format2(as_dict, None) + assert ga_format["a_galaxy_workflow"] == "true" + assert ga_format["format-version"] == "0.1" + + +def test_convert_to_simple(): + as_dict = {"yaml_content": WORKFLOW_WITH_OUTPUTS} + ga_format = convert_from_format2(as_dict, None) + + as_dict = convert_to_format2(ga_format, True) + assert "yaml_content" in as_dict + assert "yaml_content" not in as_dict From 25eec9e6f058517b38cacd9d320c0eb65286caf1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 10 Jul 2024 20:21:56 -0400 Subject: [PATCH 14/17] Named type for parsed tool versions. --- lib/galaxy/tool_util/version_util.py | 10 ++++++++++ lib/galaxy/tools/__init__.py | 10 ++++------ 2 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 lib/galaxy/tool_util/version_util.py diff --git a/lib/galaxy/tool_util/version_util.py b/lib/galaxy/tool_util/version_util.py new file mode 100644 index 000000000000..644d58297e4f --- /dev/null +++ b/lib/galaxy/tool_util/version_util.py @@ -0,0 +1,10 @@ +from typing import Union + +from packaging.version import Version + +from .version import LegacyVersion + +AnyVersionT = Union[LegacyVersion, Version] + + +__all__ = ["AnyVersionT"] diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 48d779c38e81..39119f265f51 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -110,10 +110,8 @@ from galaxy.tool_util.verify.interactor import ToolTestDescription from galaxy.tool_util.verify.parse import parse_tool_test_descriptions from galaxy.tool_util.verify.test_data import TestDataNotFoundError -from galaxy.tool_util.version import ( - LegacyVersion, - parse_version, -) +from galaxy.tool_util.version import parse_version +from galaxy.tool_util.version_util import AnyVersionT from galaxy.tools import expressions from galaxy.tools.actions import ( DefaultToolAction, @@ -343,8 +341,8 @@ class safe_update(NamedTuple): - min_version: Union[LegacyVersion, Version] - current_version: Union[LegacyVersion, Version] + min_version: AnyVersionT + current_version: AnyVersionT # Tool updates that did not change parameters in a way that requires rebuilding workflows From 7768cef87190d619922c56aebfd1daf502059ad5 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 11 Jul 2024 14:14:16 -0400 Subject: [PATCH 15/17] Workflow tool state validation plumbing. --- .../dev/tool_state_state_classes.plantuml.svg | 44 ++++ .../dev/tool_state_state_classes.plantuml.txt | 22 ++ lib/galaxy/tool_util/parameters/__init__.py | 2 + lib/galaxy/tool_util/parameters/models.py | 12 +- .../tool_util/workflow_state/__init__.py | 23 ++ lib/galaxy/tool_util/workflow_state/_types.py | 28 +++ .../tool_util/workflow_state/convert.py | 131 +++++++++++ .../tool_util/workflow_state/validation.py | 21 ++ .../workflow_state/validation_format2.py | 142 ++++++++++++ .../workflow_state/validation_native.py | 211 ++++++++++++++++++ lib/galaxy/workflow/gx_validator.py | 64 ++++++ packages/tool_util/setup.cfg | 1 + .../test_workflow_state_helpers.py | 23 ++ .../invalid/extra_attribute.gxwf.yml | 15 ++ .../workflows/invalid/missing_link.gxwf.yml | 11 + .../invalid/wrong_link_name.gxwf.yml | 13 ++ .../test_workflow_state_conversion.py | 16 ++ .../workflows/test_workflow_validation.py | 80 +++++++ .../test_workflow_validation_helpers.py | 13 ++ .../unit/workflows/valid/simple_data.gxwf.yml | 13 ++ test/unit/workflows/valid/simple_int.gxwf.yml | 13 ++ 21 files changed, 896 insertions(+), 2 deletions(-) create mode 100644 lib/galaxy/tool_util/workflow_state/__init__.py create mode 100644 lib/galaxy/tool_util/workflow_state/_types.py create mode 100644 lib/galaxy/tool_util/workflow_state/convert.py create mode 100644 lib/galaxy/tool_util/workflow_state/validation.py create mode 100644 lib/galaxy/tool_util/workflow_state/validation_format2.py create mode 100644 lib/galaxy/tool_util/workflow_state/validation_native.py create mode 100644 lib/galaxy/workflow/gx_validator.py create mode 100644 test/unit/tool_util/workflow_state/test_workflow_state_helpers.py create mode 100644 test/unit/workflows/invalid/extra_attribute.gxwf.yml create mode 100644 test/unit/workflows/invalid/missing_link.gxwf.yml create mode 100644 test/unit/workflows/invalid/wrong_link_name.gxwf.yml create mode 100644 test/unit/workflows/test_workflow_state_conversion.py create mode 100644 test/unit/workflows/test_workflow_validation.py create mode 100644 test/unit/workflows/test_workflow_validation_helpers.py create mode 100644 test/unit/workflows/valid/simple_data.gxwf.yml create mode 100644 test/unit/workflows/valid/simple_int.gxwf.yml diff --git a/doc/source/dev/tool_state_state_classes.plantuml.svg b/doc/source/dev/tool_state_state_classes.plantuml.svg index 28c7da1c9092..5874d39c6c1d 100644 --- a/doc/source/dev/tool_state_state_classes.plantuml.svg +++ b/doc/source/dev/tool_state_state_classes.plantuml.svg @@ -50,10 +50,31 @@ state_representation = "job_internal" } note bottom: Object references of the form \n{src: "hda", id: }.\n Mapping constructs expanded out.\n (Defaults are inserted?) +class TestCaseToolState { +state_representation = "test_case" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] +} +note bottom: Object references of the form file name and URIs.\n Mapping constructs not allowed.\n + +class WorkflowStepToolState { +state_representation = "workflow_step" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] +} +note bottom: Nearly everything optional except conditional discriminators.\n + +class WorkflowStepLinkedToolState { +state_representation = "workflow_step_linked" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] +} +note bottom: Expect pre-process ``in`` dictionaries and bring in representation\n of links and defaults and validate them in model.\n + ToolState <|- - RequestToolState ToolState <|- - RequestInternalToolState ToolState <|- - RequestInternalDereferencedToolState ToolState <|- - JobInternalToolState +ToolState <|- - TestCaseToolState +ToolState <|- - WorkflowStepToolState +ToolState <|- - WorkflowStepLinkedToolState RequestToolState - RequestInternalToolState : decode > @@ -61,6 +82,7 @@ RequestInternalToolState - RequestInternalDereferencedToolState : dereference > RequestInternalDereferencedToolState o- - JobInternalToolState : expand > +WorkflowStepToolState o- - WorkflowStepLinkedToolState : preprocess_links_and_defaults > } @enduml @@ -150,10 +172,31 @@ state_representation = "job_internal" } note bottom: Object references of the form \n{src: "hda", id: }.\n Mapping constructs expanded out.\n (Defaults are inserted?) +class TestCaseToolState { +state_representation = "test_case" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] +} +note bottom: Object references of the form file name and URIs.\n Mapping constructs not allowed.\n + +class WorkflowStepToolState { +state_representation = "workflow_step" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] +} +note bottom: Nearly everything optional except conditional discriminators.\n + +class WorkflowStepLinkedToolState { +state_representation = "workflow_step_linked" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] +} +note bottom: Expect pre-process ``in`` dictionaries and bring in representation\n of links and defaults and validate them in model.\n + ToolState <|- - RequestToolState ToolState <|- - RequestInternalToolState ToolState <|- - RequestInternalDereferencedToolState ToolState <|- - JobInternalToolState +ToolState <|- - TestCaseToolState +ToolState <|- - WorkflowStepToolState +ToolState <|- - WorkflowStepLinkedToolState RequestToolState - RequestInternalToolState : decode > @@ -161,6 +204,7 @@ RequestInternalToolState - RequestInternalDereferencedToolState : dereference > RequestInternalDereferencedToolState o- - JobInternalToolState : expand > +WorkflowStepToolState o- - WorkflowStepLinkedToolState : preprocess_links_and_defaults > } @enduml diff --git a/doc/source/dev/tool_state_state_classes.plantuml.txt b/doc/source/dev/tool_state_state_classes.plantuml.txt index 0c2c82951eb3..d37366e96f11 100644 --- a/doc/source/dev/tool_state_state_classes.plantuml.txt +++ b/doc/source/dev/tool_state_state_classes.plantuml.txt @@ -35,10 +35,31 @@ state_representation = "job_internal" } note bottom: Object references of the form \n{src: "hda", id: }.\n Mapping constructs expanded out.\n (Defaults are inserted?) +class TestCaseToolState { +state_representation = "test_case" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] +} +note bottom: Object references of the form file name and URIs.\n Mapping constructs not allowed.\n + +class WorkflowStepToolState { +state_representation = "workflow_step" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] +} +note bottom: Nearly everything optional except conditional discriminators.\n + +class WorkflowStepLinkedToolState { +state_representation = "workflow_step_linked" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] +} +note bottom: Expect pre-process ``in`` dictionaries and bring in representation\n of links and defaults and validate them in model.\n + ToolState <|-- RequestToolState ToolState <|-- RequestInternalToolState ToolState <|-- RequestInternalDereferencedToolState ToolState <|-- JobInternalToolState +ToolState <|-- TestCaseToolState +ToolState <|-- WorkflowStepToolState +ToolState <|-- WorkflowStepLinkedToolState RequestToolState - RequestInternalToolState : decode > @@ -46,5 +67,6 @@ RequestInternalToolState - RequestInternalDereferencedToolState : dereference > RequestInternalDereferencedToolState o-- JobInternalToolState : expand > +WorkflowStepToolState o-- WorkflowStepLinkedToolState : preprocess_links_and_defaults > } @enduml \ No newline at end of file diff --git a/lib/galaxy/tool_util/parameters/__init__.py b/lib/galaxy/tool_util/parameters/__init__.py index 3435f81d25e1..55789159464e 100644 --- a/lib/galaxy/tool_util/parameters/__init__.py +++ b/lib/galaxy/tool_util/parameters/__init__.py @@ -43,6 +43,7 @@ FloatParameterModel, HiddenParameterModel, IntegerParameterModel, + is_optional, LabelValue, RawStateDict, RepeatParameterModel, @@ -127,6 +128,7 @@ "RepeatParameterModel", "RawStateDict", "ValidationFunctionT", + "is_optional", "validate_against_model", "validate_internal_job", "validate_internal_landing_request", diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py index 0fc92d244dbb..2f4ec69a6bd3 100644 --- a/lib/galaxy/tool_util/parameters/models.py +++ b/lib/galaxy/tool_util/parameters/models.py @@ -59,7 +59,7 @@ from ._types import ( cast_as_type, expand_annotation, - is_optional, + is_optional as is_python_type_optional, list_type, optional, optional_if_needed, @@ -153,7 +153,7 @@ def dynamic_model_information_from_py_type( if requires_value is None: requires_value = param_model.request_requires_value initialize = ... if requires_value else None - py_type_is_optional = is_optional(py_type) + py_type_is_optional = is_python_type_optional(py_type) validators = validators or {} if not py_type_is_optional and not requires_value: validators["not_null"] = field_validator(name)(Validators.validate_not_none) @@ -1459,6 +1459,14 @@ class ToolParameterModel(RootModel): CwlUnionParameterModel.model_rebuild() +def is_optional(tool_parameter: ToolParameterT): + if isinstance(tool_parameter, BaseGalaxyToolParameterModelDefinition): + return tool_parameter.optional + else: + # refine CWL logic in CWL branch... + return False + + class ToolParameterBundle(Protocol): """An object having a dictionary of input models (i.e. a 'Tool')""" diff --git a/lib/galaxy/tool_util/workflow_state/__init__.py b/lib/galaxy/tool_util/workflow_state/__init__.py new file mode 100644 index 000000000000..9ae61481fe1c --- /dev/null +++ b/lib/galaxy/tool_util/workflow_state/__init__.py @@ -0,0 +1,23 @@ +"""Abstractions for reasoning about tool state within Galaxy workflows. + +Like everything else in galaxy-tool-util, this package should be independent of +Galaxy's runtime. It is meant to provide utilities for reasonsing about tool state +(largely building on the abstractions in galaxy.tool_util.parameters) within the +context of workflows. +""" + +from ._types import GetToolInfo +from .convert import ( + ConversionValidationFailure, + convert_state_to_format2, + Format2State, +) +from .validation import validate_workflow + +__all__ = ( + "ConversionValidationFailure", + "convert_state_to_format2", + "GetToolInfo", + "Format2State", + "validate_workflow", +) diff --git a/lib/galaxy/tool_util/workflow_state/_types.py b/lib/galaxy/tool_util/workflow_state/_types.py new file mode 100644 index 000000000000..cf91508624b5 --- /dev/null +++ b/lib/galaxy/tool_util/workflow_state/_types.py @@ -0,0 +1,28 @@ +from typing import ( + Any, + Dict, + Optional, + Union, +) + +from typing_extensions import ( + Literal, + Protocol, +) + +from galaxy.tool_util.models import ParsedTool + +NativeWorkflowDict = Dict[str, Any] +Format2WorkflowDict = Dict[str, Any] +AnyWorkflowDict = Union[NativeWorkflowDict, Format2WorkflowDict] +WorkflowFormat = Literal["gxformat2", "native"] +NativeStepDict = Dict[str, Any] +Format2StepDict = Dict[str, Any] +NativeToolStateDict = Dict[str, Any] +Format2StateDict = Dict[str, Any] + + +class GetToolInfo(Protocol): + """An interface for fetching tool information for steps in a workflow.""" + + def get_tool_info(self, tool_id: str, tool_version: Optional[str]) -> ParsedTool: ... diff --git a/lib/galaxy/tool_util/workflow_state/convert.py b/lib/galaxy/tool_util/workflow_state/convert.py new file mode 100644 index 000000000000..37f0c8702bb8 --- /dev/null +++ b/lib/galaxy/tool_util/workflow_state/convert.py @@ -0,0 +1,131 @@ +from typing import ( + Dict, + List, + Optional, +) + +from pydantic import ( + BaseModel, + Field, +) + +from galaxy.tool_util.models import ParsedTool +from galaxy.tool_util.parameters import ToolParameterT +from ._types import ( + Format2StateDict, + GetToolInfo, + NativeStepDict, +) +from .validation_format2 import validate_step_against +from .validation_native import ( + get_parsed_tool_for_native_step, + native_tool_state, + validate_native_step_against, +) + +Format2InputsDictT = Dict[str, str] + + +class Format2State(BaseModel): + state: Format2StateDict + inputs: Format2InputsDictT = Field(alias="in") + + +class ConversionValidationFailure(Exception): + pass + + +def convert_state_to_format2(native_step_dict: NativeStepDict, get_tool_info: GetToolInfo) -> Format2State: + parsed_tool = get_parsed_tool_for_native_step(native_step_dict, get_tool_info) + return convert_state_to_format2_using(native_step_dict, parsed_tool) + + +def convert_state_to_format2_using(native_step_dict: NativeStepDict, parsed_tool: Optional[ParsedTool]) -> Format2State: + """Create a "clean" gxformat2 workflow tool state from a native workflow step. + + gxformat2 does not know about tool specifications so it cannot reason about the native + tool state attribute and just copies it as is. This native state can be pretty ugly. The purpose + of this function is to build a cleaned up state to replace the gxformat2 copied native tool_state + with that is more readable and has stronger typing by using the tool's inputs to guide + the conversion (the parsed_tool parameter). + + This method validates both the native tool state and the resulting gxformat2 tool state + so that we can be more confident the conversion doesn't corrupt the workflow. If no meta + model to validate against is supplied or if either validation fails this method throws + ConversionValidationFailure to signal the caller to just use the native tool state as is + instead of trying to convert it to a cleaner gxformat2 tool state - under the assumption + it is better to have an "ugly" workflow than a corrupted one during conversion. + """ + if parsed_tool is None: + raise ConversionValidationFailure("Could not resolve tool inputs") + try: + validate_native_step_against(native_step_dict, parsed_tool) + except Exception: + raise ConversionValidationFailure( + "Failed to validate native step - not going to convert a tool state that isn't understood" + ) + result = _convert_valid_state_to_format2(native_step_dict, parsed_tool) + print(result.dict()) + try: + validate_step_against(result.dict(), parsed_tool) + except Exception: + raise ConversionValidationFailure( + "Failed to validate resulting cleaned step - not going to convert to an unvalidated tool state" + ) + return result + + +def _convert_valid_state_to_format2(native_step_dict: NativeStepDict, parsed_tool: ParsedTool) -> Format2State: + format2_state: Format2StateDict = {} + format2_in: Format2InputsDictT = {} + + root_tool_state = native_tool_state(native_step_dict) + tool_inputs = parsed_tool.inputs + _convert_state_level(native_step_dict, tool_inputs, root_tool_state, format2_state, format2_in) + return Format2State( + **{ + "state": format2_state, + "in": format2_in, + } + ) + + +def _convert_state_level( + step: NativeStepDict, + tool_inputs: List[ToolParameterT], + native_state: dict, + format2_state_at_level: dict, + format2_in: Format2InputsDictT, + prefix: Optional[str] = None, +) -> None: + prefix = prefix or "" + assert prefix is not None + for tool_input in tool_inputs: + _convert_state_at_level(step, tool_input, native_state, format2_state_at_level, format2_in, prefix) + + +def _convert_state_at_level( + step: NativeStepDict, + tool_input: ToolParameterT, + native_state_at_level: dict, + format2_state_at_level: dict, + format2_in: Format2InputsDictT, + prefix: str, +) -> None: + parameter_type = tool_input.parameter_type + parameter_name = tool_input.name + value = native_state_at_level.get(parameter_name, None) + state_path = parameter_name if prefix is None else f"{prefix}|{parameter_name}" + if parameter_type == "gx_integer": + # check for runtime input + format2_value = int(value) + format2_state_at_level[parameter_name] = format2_value + elif parameter_type == "gx_data": + input_connections = step.get("input_connections", {}) + print(state_path) + print(input_connections) + if state_path in input_connections: + format2_in[state_path] = "placeholder" + else: + pass + # raise NotImplementedError(f"Unhandled parameter type {parameter_type}") diff --git a/lib/galaxy/tool_util/workflow_state/validation.py b/lib/galaxy/tool_util/workflow_state/validation.py new file mode 100644 index 000000000000..56a225fbb8f8 --- /dev/null +++ b/lib/galaxy/tool_util/workflow_state/validation.py @@ -0,0 +1,21 @@ +from ._types import ( + AnyWorkflowDict, + GetToolInfo, + WorkflowFormat, +) +from .validation_format2 import validate_workflow_format2 +from .validation_native import validate_workflow_native + + +def validate_workflow(workflow_dict: AnyWorkflowDict, get_tool_info: GetToolInfo): + if _format(workflow_dict) == "gxformat2": + validate_workflow_format2(workflow_dict, get_tool_info) + else: + validate_workflow_native(workflow_dict, get_tool_info) + + +def _format(workflow_dict: AnyWorkflowDict) -> WorkflowFormat: + if workflow_dict.get("a_galaxy_workflow") == "true": + return "native" + else: + return "gxformat2" diff --git a/lib/galaxy/tool_util/workflow_state/validation_format2.py b/lib/galaxy/tool_util/workflow_state/validation_format2.py new file mode 100644 index 000000000000..160576c3abe7 --- /dev/null +++ b/lib/galaxy/tool_util/workflow_state/validation_format2.py @@ -0,0 +1,142 @@ +from typing import ( + cast, + Optional, +) + +from gxformat2.model import ( + get_native_step_type, + pop_connect_from_step_dict, + setup_connected_values, + steps_as_list, +) + +from galaxy.tool_util.models import ParsedTool +from galaxy.tool_util.parameters import ( + ConditionalParameterModel, + ConditionalWhen, + flat_state_path, + keys_starting_with, + repeat_inputs_to_array, + RepeatParameterModel, + ToolParameterT, + validate_explicit_conditional_test_value, + WorkflowStepLinkedToolState, + WorkflowStepToolState, +) +from ._types import ( + Format2StepDict, + Format2WorkflowDict, + GetToolInfo, +) + + +def validate_workflow_format2(workflow_dict: Format2WorkflowDict, get_tool_info: GetToolInfo): + steps = steps_as_list(workflow_dict) + for step in steps: + validate_step_format2(step, get_tool_info) + + +def validate_step_format2(step_dict: Format2StepDict, get_tool_info: GetToolInfo): + step_type = get_native_step_type(step_dict) + if step_type != "tool": + return + tool_id = cast(str, step_dict.get("tool_id")) + tool_version: Optional[str] = cast(Optional[str], step_dict.get("tool_version")) + parsed_tool = get_tool_info.get_tool_info(tool_id, tool_version) + if parsed_tool is not None: + validate_step_against(step_dict, parsed_tool) + + +def validate_step_against(step_dict: Format2StepDict, parsed_tool: ParsedTool): + source_tool_state_model = WorkflowStepToolState.parameter_model_for(parsed_tool.inputs) + linked_tool_state_model = WorkflowStepLinkedToolState.parameter_model_for(parsed_tool.inputs) + contains_format2_state = "state" in step_dict + contains_native_state = "tool_state" in step_dict + if contains_format2_state: + assert source_tool_state_model + source_tool_state_model.model_validate(step_dict["state"]) + if not contains_native_state: + if not contains_format2_state: + step_dict["state"] = {} + # setup links and then validate against model... + linked_step = merge_inputs(step_dict, parsed_tool) + linked_tool_state_model.model_validate(linked_step["state"]) + + +def merge_inputs(step_dict: Format2StepDict, parsed_tool: ParsedTool) -> Format2StepDict: + connect = pop_connect_from_step_dict(step_dict) + step_dict = setup_connected_values(step_dict, connect) + tool_inputs = parsed_tool.inputs + + state_at_level = step_dict["state"] + + for tool_input in tool_inputs: + _merge_into_state(connect, tool_input, state_at_level) + + for key in connect: + raise Exception(f"Failed to find parameter definition matching workflow linked key {key}") + return step_dict + + +def _merge_into_state( + connect, tool_input: ToolParameterT, state: dict, prefix: Optional[str] = None, branch_connect=None +): + if branch_connect is None: + branch_connect = connect + + name = tool_input.name + parameter_type = tool_input.parameter_type + state_path = flat_state_path(name, prefix) + if parameter_type == "gx_conditional": + conditional_state = state.get(name, {}) + if name not in state: + state[name] = conditional_state + + conditional = cast(ConditionalParameterModel, tool_input) + when: ConditionalWhen = _select_which_when(conditional, conditional_state) + test_parameter = conditional.test_parameter + conditional_connect = keys_starting_with(branch_connect, state_path) + _merge_into_state( + connect, test_parameter, conditional_state, prefix=state_path, branch_connect=conditional_connect + ) + for when_parameter in when.parameters: + _merge_into_state( + connect, when_parameter, conditional_state, prefix=state_path, branch_connect=conditional_connect + ) + elif parameter_type == "gx_repeat": + repeat_state_array = state.get(name, []) + repeat = cast(RepeatParameterModel, tool_input) + repeat_instance_connects = repeat_inputs_to_array(state_path, connect) + for i, repeat_instance_connect in enumerate(repeat_instance_connects): + while len(repeat_state_array) <= i: + repeat_state_array.append({}) + + repeat_instance_prefix = f"{state_path}_{i}" + for repeat_parameter in repeat.parameters: + _merge_into_state( + connect, + repeat_parameter, + repeat_state_array[i], + prefix=repeat_instance_prefix, + branch_connect=repeat_instance_connect, + ) + if repeat_state_array and name not in state: + state[name] = repeat_state_array + else: + if state_path in branch_connect: + state[name] = {"__class__": "ConnectedValue"} + del connect[state_path] + + +def _select_which_when(conditional: ConditionalParameterModel, state: dict) -> ConditionalWhen: + test_parameter = conditional.test_parameter + test_parameter_name = test_parameter.name + explicit_test_value = state.get(test_parameter_name) + test_value = validate_explicit_conditional_test_value(test_parameter_name, explicit_test_value) + for when in conditional.whens: + if test_value is None and when.is_default_when: + return when + elif test_value == when.discriminator: + return when + else: + raise Exception(f"Invalid conditional test value ({explicit_test_value}) for parameter ({test_parameter_name})") diff --git a/lib/galaxy/tool_util/workflow_state/validation_native.py b/lib/galaxy/tool_util/workflow_state/validation_native.py new file mode 100644 index 000000000000..ae08d15b4226 --- /dev/null +++ b/lib/galaxy/tool_util/workflow_state/validation_native.py @@ -0,0 +1,211 @@ +import json +from typing import ( + cast, + List, + Optional, +) + +from galaxy.tool_util.models import ParsedTool +from galaxy.tool_util.parameters import ( + ConditionalParameterModel, + ConditionalWhen, + flat_state_path, + is_optional, + repeat_inputs_to_array, + RepeatParameterModel, + SelectParameterModel, + ToolParameterT, + validate_explicit_conditional_test_value, +) +from ._types import ( + GetToolInfo, + NativeStepDict, + NativeToolStateDict, + NativeWorkflowDict, +) + + +def validate_native_step_against(step: NativeStepDict, parsed_tool: ParsedTool): + tool_state_jsonified = step.get("tool_state") + assert tool_state_jsonified + tool_state = json.loads(tool_state_jsonified) + tool_inputs = parsed_tool.inputs + + # merge input connections into ConnectedValues if there aren't already there... + _merge_inputs_into_state_dict(step, tool_inputs, tool_state) + + allowed_extra_keys = ["__page__", "__rerun_remap_job_id__"] + _validate_native_state_level(step, tool_inputs, tool_state, allowed_extra_keys=allowed_extra_keys) + + +def _validate_native_state_level( + step: NativeStepDict, tool_inputs: List[ToolParameterT], state_at_level: dict, allowed_extra_keys=None +): + if allowed_extra_keys is None: + allowed_extra_keys = [] + + keys_processed = set() + for tool_input in tool_inputs: + parameter_name = tool_input.name + keys_processed.add(parameter_name) + _validate_native_state_at_level(step, tool_input, state_at_level) + + for key in state_at_level.keys(): + if key not in keys_processed and key not in allowed_extra_keys: + raise Exception(f"Unknown key found {key}, failing state validation") + + +def _validate_native_state_at_level( + step: NativeStepDict, tool_input: ToolParameterT, state_at_level: dict, prefix: Optional[str] = None +): + parameter_type = tool_input.parameter_type + parameter_name = tool_input.name + value = state_at_level.get(parameter_name, None) + # state_path = parameter_name if prefix is None else f"{prefix}|{parameter_name}" + if parameter_type == "gx_integer": + try: + int(value) + except ValueError: + raise Exception(f"Invalid integer data found {value}") + elif parameter_type in ["gx_data", "gx_data_collection"]: + if isinstance(value, dict): + assert "__class__" in value + assert value["__class__"] in ["RuntimeValue", "ConnectedValue"] + else: + assert value in [None, "null"] + connections = native_connections_for(step, tool_input, prefix) + optional = is_optional(tool_input) + if not optional and not connections: + raise Exception( + "Disconnected non-optional input found, not attempting to validate non-practice workflow" + ) + + elif parameter_type == "gx_select": + select = cast(SelectParameterModel, tool_input) + options = select.options + if options is not None: + valid_values = [o.value for o in options] + if value not in valid_values: + raise Exception(f"Invalid select option found {value}") + elif parameter_type == "gx_conditional": + conditional_state = state_at_level.get(parameter_name, None) + conditional = cast(ConditionalParameterModel, tool_input) + when: ConditionalWhen = _select_which_when_native(conditional, conditional_state) + test_parameter = conditional.test_parameter + test_parameter_name = test_parameter.name + _validate_native_state_at_level(step, test_parameter, conditional_state) + _validate_native_state_level( + step, when.parameters, conditional_state, allowed_extra_keys=["__current_case__", test_parameter_name] + ) + else: + raise NotImplementedError(f"Unhandled parameter type ({parameter_type})") + + +def _select_which_when_native(conditional: ConditionalParameterModel, conditional_state: dict) -> ConditionalWhen: + test_parameter = conditional.test_parameter + test_parameter_name = test_parameter.name + explicit_test_value = conditional_state.get(test_parameter_name) + test_value = validate_explicit_conditional_test_value(test_parameter_name, explicit_test_value) + target_when = None + for when in conditional.whens: + # deal with native string -> bool issues in here... + if test_value is None and when.is_default_when: + target_when = when + elif test_value == when.discriminator: + target_when = when + + recorded_case = conditional_state.get("__current_case__") + if recorded_case is not None: + if not isinstance(recorded_case, int): + raise Exception(f"Unknown type of value for __current_case__ encountered {recorded_case}") + if recorded_case < 0 or recorded_case >= len(conditional.whens): + raise Exception(f"Unknown index value for __current_case__ encountered {recorded_case}") + recorded_when = conditional.whens[recorded_case] + + if target_when is None: + raise Exception("is this okay? I need more tests") + if target_when and recorded_when and target_when != recorded_when: + raise Exception( + f"Problem parsing out tool state - inferred conflicting tool states for parameter {test_parameter_name}" + ) + return target_when + + +def _merge_inputs_into_state_dict( + step_dict: NativeStepDict, tool_inputs: List[ToolParameterT], state_at_level: dict, prefix: Optional[str] = None +): + for tool_input in tool_inputs: + _merge_into_state(step_dict, tool_input, state_at_level, prefix=prefix) + + +def _merge_into_state(step_dict: NativeStepDict, tool_input: ToolParameterT, state: dict, prefix: Optional[str] = None): + name = tool_input.name + parameter_type = tool_input.parameter_type + state_path = flat_state_path(name, prefix) + if parameter_type == "gx_conditional": + conditional_state = state.get(name, {}) + if name not in state: + state[name] = conditional_state + + conditional = cast(ConditionalParameterModel, tool_input) + when: ConditionalWhen = _select_which_when_native(conditional, conditional_state) + test_parameter = conditional.test_parameter + _merge_into_state(step_dict, test_parameter, conditional_state, prefix=state_path) + for when_parameter in when.parameters: + _merge_into_state(step_dict, when_parameter, conditional_state, prefix=state_path) + elif parameter_type == "gx_repeat": + repeat_state_array = state.get(name, []) + repeat = cast(RepeatParameterModel, tool_input) + repeat_instance_connects = repeat_inputs_to_array(state_path, step_dict) + for i, _ in enumerate(repeat_instance_connects): + while len(repeat_state_array) <= i: + repeat_state_array.append({}) + + repeat_instance_prefix = f"{state_path}_{i}" + for repeat_parameter in repeat.parameters: + _merge_into_state( + step_dict, + repeat_parameter, + repeat_state_array[i], + prefix=repeat_instance_prefix, + ) + if repeat_state_array and name not in state: + state[name] = repeat_state_array + else: + input_connections = step_dict.get("input_connections", {}) + if state_path in input_connections and state.get(name) is None: + state[name] = {"__class__": "ConnectedValue"} + + +def validate_step_native(step: NativeStepDict, get_tool_info: GetToolInfo): + parsed_tool = get_parsed_tool_for_native_step(step, get_tool_info) + if parsed_tool is not None: + validate_native_step_against(step, parsed_tool) + + +def get_parsed_tool_for_native_step(step: NativeStepDict, get_tool_info: GetToolInfo) -> Optional[ParsedTool]: + tool_id = cast(str, step.get("tool_id")) + if not tool_id: + return None + tool_version: Optional[str] = cast(Optional[str], step.get("tool_version")) + parsed_tool = get_tool_info.get_tool_info(tool_id, tool_version) + return parsed_tool + + +def validate_workflow_native(workflow_dict: NativeWorkflowDict, get_tool_info: GetToolInfo): + for step_def in workflow_dict["steps"].values(): + validate_step_native(step_def, get_tool_info) + + +def native_tool_state(step: NativeStepDict) -> NativeToolStateDict: + tool_state_jsonified = step.get("tool_state") + assert tool_state_jsonified + tool_state = json.loads(tool_state_jsonified) + return tool_state + + +def native_connections_for(step: NativeStepDict, parameter: ToolParameterT, prefix: Optional[str]): + parameter_name = parameter.name + state_path = parameter_name if prefix is None else f"{prefix}|{parameter_name}" + step.get("input_connections", {}) + return step.get(state_path) diff --git a/lib/galaxy/workflow/gx_validator.py b/lib/galaxy/workflow/gx_validator.py new file mode 100644 index 000000000000..3d3a99912c2a --- /dev/null +++ b/lib/galaxy/workflow/gx_validator.py @@ -0,0 +1,64 @@ +""""A validator for Galaxy workflows that is hooked up to Galaxy internals. + +The interface is designed to be usable from the tool shed for external tooling, +but for internal tooling - Galaxy should have its own tool available. +""" + +from typing import ( + Dict, + Optional, +) + +from galaxy.tool_util.models import ( + parse_tool, + ParsedTool, +) +from galaxy.tool_util.version import parse_version +from galaxy.tool_util.version_util import AnyVersionT +from galaxy.tool_util.workflow_state import ( + GetToolInfo, + validate_workflow as validate_workflow_generic, +) +from galaxy.tools.stock import stock_tool_sources + + +class GalaxyGetToolInfo(GetToolInfo): + stock_tools_by_version: Dict[str, Dict[AnyVersionT, ParsedTool]] + stock_tools_latest_version: Dict[str, AnyVersionT] + + def __init__(self): + # todo take in a toolbox in the future... + stock_tools: Dict[str, Dict[AnyVersionT, ParsedTool]] = {} + stock_tools_latest_version: Dict[str, AnyVersionT] = {} + for stock_tool in stock_tool_sources(): + id = stock_tool.parse_id() + version = stock_tool.parse_version() + version_object = None + if version is not None: + version_object = parse_version(version) + if id not in stock_tools: + stock_tools[id] = {} + if version_object is not None: + stock_tools_latest_version[id] = version_object + try: + stock_tools[id][version_object] = parse_tool(stock_tool) + except Exception: + pass + if version_object and version_object > stock_tools_latest_version[id]: + stock_tools_latest_version[id] = version_object + self.stock_tools = stock_tools + self.stock_tools_latest_version = stock_tools_latest_version + + def get_tool_info(self, tool_id: str, tool_version: Optional[str]) -> ParsedTool: + if tool_version is not None: + return self.stock_tools[tool_id][parse_version(tool_version)] + else: + latest_verison = self.stock_tools_latest_version[tool_id] + return self.stock_tools[tool_id][latest_verison] + + +GET_TOOL_INFO = GalaxyGetToolInfo() + + +def validate_workflow(as_dict): + return validate_workflow_generic(as_dict, get_tool_info=GET_TOOL_INFO) diff --git a/packages/tool_util/setup.cfg b/packages/tool_util/setup.cfg index c0cdb4935917..67e181a960ab 100644 --- a/packages/tool_util/setup.cfg +++ b/packages/tool_util/setup.cfg @@ -34,6 +34,7 @@ version = 24.2.dev0 include_package_data = True install_requires = galaxy-util[image_util]>=22.1 + gxformat2 conda-package-streaming lxml!=4.2.2 MarkupSafe diff --git a/test/unit/tool_util/workflow_state/test_workflow_state_helpers.py b/test/unit/tool_util/workflow_state/test_workflow_state_helpers.py new file mode 100644 index 000000000000..e8544e22fbb2 --- /dev/null +++ b/test/unit/tool_util/workflow_state/test_workflow_state_helpers.py @@ -0,0 +1,23 @@ +from galaxy.tool_util.parameters import repeat_inputs_to_array + + +def test_repeat_inputs_to_array(): + rval = repeat_inputs_to_array( + "repeatfoo", + { + "moo": "cow", + }, + ) + assert not rval + test_state: dict = { + "moo": "cow", + "repeatfoo_0|moocow": ["moo"], + "repeatfoo_2|moocow": ["cow"], + } + rval = repeat_inputs_to_array("repeatfoo", test_state) + assert len(rval) == 3 + assert "repeatfoo_0|moocow" in rval[0] + assert "repeatfoo_0|moocow" not in rval[1] + assert "repeatfoo_0|moocow" not in rval[2] + assert "repeatfoo_2|moocow" not in rval[1] + assert "repeatfoo_2|moocow" in rval[2] diff --git a/test/unit/workflows/invalid/extra_attribute.gxwf.yml b/test/unit/workflows/invalid/extra_attribute.gxwf.yml new file mode 100644 index 000000000000..6ae50799394c --- /dev/null +++ b/test/unit/workflows/invalid/extra_attribute.gxwf.yml @@ -0,0 +1,15 @@ +class: GalaxyWorkflow +inputs: + input: + type: int +outputs: + output: + outputSource: the_step/output +steps: + the_step: + tool_id: gx_int + tool_version: "1.0.0" + state: + parameter2: 6 + in: + parameter: input diff --git a/test/unit/workflows/invalid/missing_link.gxwf.yml b/test/unit/workflows/invalid/missing_link.gxwf.yml new file mode 100644 index 000000000000..526b40f6f502 --- /dev/null +++ b/test/unit/workflows/invalid/missing_link.gxwf.yml @@ -0,0 +1,11 @@ +class: GalaxyWorkflow +inputs: + input: + type: data +outputs: + output: + outputSource: the_step/output +steps: + the_step: + tool_id: gx_data + tool_version: "1.0.0" diff --git a/test/unit/workflows/invalid/wrong_link_name.gxwf.yml b/test/unit/workflows/invalid/wrong_link_name.gxwf.yml new file mode 100644 index 000000000000..f0e0e8d12004 --- /dev/null +++ b/test/unit/workflows/invalid/wrong_link_name.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input: + type: int +outputs: + output: + outputSource: the_step/output +steps: + the_step: + tool_id: gx_int + tool_version: "1.0.0" + in: + parameterx: input diff --git a/test/unit/workflows/test_workflow_state_conversion.py b/test/unit/workflows/test_workflow_state_conversion.py new file mode 100644 index 000000000000..e73b73b6aa55 --- /dev/null +++ b/test/unit/workflows/test_workflow_state_conversion.py @@ -0,0 +1,16 @@ +from galaxy.tool_util.workflow_state.convert import ( + convert_state_to_format2, + Format2State, +) +from galaxy.workflow.gx_validator import GET_TOOL_INFO +from .test_workflow_validation import base_package_workflow_as_dict + + +def convert_state(native_step_dict: dict) -> Format2State: + return convert_state_to_format2(native_step_dict, GET_TOOL_INFO) + + +def test_simple_convert(): + workflow_dict = base_package_workflow_as_dict("test_workflow_1.ga") + cat_step = workflow_dict["steps"]["2"] + convert_state(cat_step) diff --git a/test/unit/workflows/test_workflow_validation.py b/test/unit/workflows/test_workflow_validation.py new file mode 100644 index 000000000000..33c93ede6df5 --- /dev/null +++ b/test/unit/workflows/test_workflow_validation.py @@ -0,0 +1,80 @@ +import os +from typing import Optional + +from gxformat2.yaml import ordered_load + +from galaxy.util import galaxy_directory +from galaxy.workflow.gx_validator import validate_workflow + +TEST_WORKFLOW_DIRECTORY = os.path.join(galaxy_directory(), "lib", "galaxy_test", "workflow") +TEST_BASE_DATA_DIRECTORY = os.path.join(galaxy_directory(), "lib", "galaxy_test", "base", "data") +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) + + +def test_validate_simple_functional_test_case_workflow(): + validate_workflow(framework_test_workflow_as_dict("multiple_versions")) + validate_workflow(framework_test_workflow_as_dict("zip_collection")) + validate_workflow(framework_test_workflow_as_dict("empty_collection_sort")) + validate_workflow(framework_test_workflow_as_dict("flatten_collection")) + validate_workflow(framework_test_workflow_as_dict("flatten_collection_over_execution")) + + +def test_validate_native_workflows(): + validate_workflow(base_package_workflow_as_dict("test_workflow_two_random_lines.ga")) + # disconnected input... + # validate_workflow(base_package_workflow_as_dict("test_workflow_topoambigouity.ga")) + # double nested JSON? + # validate_workflow(base_package_workflow_as_dict("test_Workflow_map_reduce_pause.ga")) + # handle subworkflows... + # validate_workflow(base_package_workflow_as_dict("test_subworkflow_with_integer_input.ga")) + # handle gx_text.... + # validate_workflow(base_package_workflow_as_dict("test_workflow_batch.ga")) + + +def test_validate_unit_test_workflows(): + validate_workflow(unit_test_workflow_as_dict("valid/simple_int")) + validate_workflow(unit_test_workflow_as_dict("valid/simple_data")) + + +def test_invalidate_with_extra_attribute(): + e = _assert_validation_failure("invalid/extra_attribute") + assert "parameter2" in str(e) + + +def test_invalidate_with_wrong_link_name(): + e = _assert_validation_failure("invalid/wrong_link_name") + assert "parameterx" in str(e) + + +def test_invalidate_with_missing_link(): + e = _assert_validation_failure("invalid/missing_link") + assert "parameter" in str(e) + assert "type=missing" in str(e) + + +def _assert_validation_failure(workflow_name: str) -> Exception: + as_dict = unit_test_workflow_as_dict(workflow_name) + exc: Optional[Exception] = None + try: + validate_workflow(as_dict) + except Exception as e: + exc = e + assert exc, f"Target workflow ({workflow_name}) did not failure validation as expected." + return exc + + +def base_package_workflow_as_dict(file_name: str) -> dict: + return _load(os.path.join(TEST_BASE_DATA_DIRECTORY, file_name)) + + +def unit_test_workflow_as_dict(workflow_name: str) -> dict: + return _load(os.path.join(SCRIPT_DIRECTORY, f"{workflow_name}.gxwf.yml")) + + +def framework_test_workflow_as_dict(workflow_name: str) -> dict: + return _load(os.path.join(TEST_WORKFLOW_DIRECTORY, f"{workflow_name}.gxwf.yml")) + + +def _load(path: str) -> dict: + with open(path) as f: + return ordered_load(f) diff --git a/test/unit/workflows/test_workflow_validation_helpers.py b/test/unit/workflows/test_workflow_validation_helpers.py new file mode 100644 index 000000000000..af74e2b32a5f --- /dev/null +++ b/test/unit/workflows/test_workflow_validation_helpers.py @@ -0,0 +1,13 @@ +from galaxy.workflow.gx_validator import GET_TOOL_INFO + + +def test_get_tool(): + parsed_tool = GET_TOOL_INFO.get_tool_info("cat1", "1.0.0") + assert parsed_tool + assert parsed_tool.id == "cat1" + assert parsed_tool.version == "1.0.0" + + parsed_tool = GET_TOOL_INFO.get_tool_info("cat1", None) + assert parsed_tool + assert parsed_tool.id == "cat1" + assert parsed_tool.version == "1.0.0" diff --git a/test/unit/workflows/valid/simple_data.gxwf.yml b/test/unit/workflows/valid/simple_data.gxwf.yml new file mode 100644 index 000000000000..44f0a90f3dd9 --- /dev/null +++ b/test/unit/workflows/valid/simple_data.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input: + type: data +outputs: + output: + outputSource: the_step/output +steps: + the_step: + tool_id: gx_data + tool_version: "1.0.0" + in: + parameter: input diff --git a/test/unit/workflows/valid/simple_int.gxwf.yml b/test/unit/workflows/valid/simple_int.gxwf.yml new file mode 100644 index 000000000000..d7c53f78d0a6 --- /dev/null +++ b/test/unit/workflows/valid/simple_int.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input: + type: int +outputs: + output: + outputSource: the_step/output +steps: + the_step: + tool_id: gx_int + tool_version: "1.0.0" + in: + parameter: input From a2655b8be04456211134753cc0b9a370e997cc22 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 28 Aug 2024 10:37:09 -0400 Subject: [PATCH 16/17] [WIP] Use tool request API from the tool form. --- client/src/components/Tool/ToolForm.vue | 235 ++++--- client/src/components/Tool/ToolSuccess.vue | 5 + .../components/Tool/ToolSuccessMessage.vue | 36 +- client/src/components/Tool/parameterModels.ts | 305 +++++++++ .../src/components/Tool/parameter_models.yml | 610 ++++++++++++++++++ .../Tool/parameter_specification.yml | 1 + client/src/components/Tool/services.js | 21 + client/src/components/Tool/structured.test.ts | 79 +++ client/src/components/Tool/structured.ts | 397 ++++++++++++ client/src/stores/jobStore.ts | 1 + 10 files changed, 1606 insertions(+), 84 deletions(-) create mode 100644 client/src/components/Tool/parameterModels.ts create mode 100644 client/src/components/Tool/parameter_models.yml create mode 120000 client/src/components/Tool/parameter_specification.yml create mode 100644 client/src/components/Tool/structured.test.ts create mode 100644 client/src/components/Tool/structured.ts diff --git a/client/src/components/Tool/ToolForm.vue b/client/src/components/Tool/ToolForm.vue index 0268d65bad8e..827a95a22645 100644 --- a/client/src/components/Tool/ToolForm.vue +++ b/client/src/components/Tool/ToolForm.vue @@ -89,7 +89,7 @@