From 92b2a308ce70371ffd8d8cdab9128479bb32c0b3 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 10 Dec 2024 13:21:35 -0500 Subject: [PATCH 01/10] Extract and typescript-ify datatype selection in wfeditor. --- .../Workflow/Editor/Forms/FormDatatype.vue | 58 +++++++++++++++++++ .../Workflow/Editor/Forms/FormOutput.vue | 29 ++-------- 2 files changed, 63 insertions(+), 24 deletions(-) create mode 100644 client/src/components/Workflow/Editor/Forms/FormDatatype.vue diff --git a/client/src/components/Workflow/Editor/Forms/FormDatatype.vue b/client/src/components/Workflow/Editor/Forms/FormDatatype.vue new file mode 100644 index 000000000000..466a9fca75e3 --- /dev/null +++ b/client/src/components/Workflow/Editor/Forms/FormDatatype.vue @@ -0,0 +1,58 @@ + + + diff --git a/client/src/components/Workflow/Editor/Forms/FormOutput.vue b/client/src/components/Workflow/Editor/Forms/FormOutput.vue index a0e41c37eb40..ac6b22944a60 100644 --- a/client/src/components/Workflow/Editor/Forms/FormOutput.vue +++ b/client/src/components/Workflow/Editor/Forms/FormOutput.vue @@ -9,14 +9,13 @@ title="Rename dataset" type="text" @input="onInput" /> - + @onChange="onDatatype" /> import FormCard from "@/components/Form/FormCard"; import FormElement from "@/components/Form/FormElement"; +import FormDatatype from "@/components/Workflow/Editor/Forms/FormDatatype"; import FormOutputLabel from "@/components/Workflow/Editor/Forms/FormOutputLabel"; const actions = [ @@ -101,6 +101,7 @@ export default { FormCard, FormElement, FormOutputLabel, + FormDatatype, }, props: { outputName: { @@ -148,26 +149,6 @@ export default { }); return index; }, - datatypeExtensions() { - const extensions = []; - for (const key in this.datatypes) { - extensions.push({ 0: this.datatypes[key], 1: this.datatypes[key] }); - } - extensions.sort((a, b) => (a.label > b.label ? 1 : a.label < b.label ? -1 : 0)); - extensions.unshift({ - 0: "Sequences", - 1: "Sequences", - }); - extensions.unshift({ - 0: "Roadmaps", - 1: "Roadmaps", - }); - extensions.unshift({ - 0: "Leave unchanged", - 1: "", - }); - return extensions; - }, renameHelp() { /* TODO: FormElement should provide a slot for custom help templating instead. */ const helpLink = `here`; From e857c629592d8a0d20d916be849e83d384b8610a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 10 Dec 2024 15:19:46 -0500 Subject: [PATCH 02/10] Build dataset collection input definition on the client. --- .../Form/Elements/FormSelection.vue | 2 +- .../Editor/Forms/FormCollectionType.vue | 54 +++++++++ .../Workflow/Editor/Forms/FormDatatype.vue | 36 ++++-- .../Workflow/Editor/Forms/FormDefault.vue | 10 +- .../Editor/Forms/FormInputCollection.vue | 104 ++++++++++++++++++ .../Editor/composables/useStepProps.ts | 2 + .../Editor/composables/useToolState.ts | 40 +++++++ .../modules/collectionTypeDescription.ts | 10 ++ lib/galaxy/webapps/galaxy/api/workflows.py | 19 +++- lib/galaxy/workflow/modules.py | 30 +---- test/unit/workflows/test_modules.py | 11 -- 11 files changed, 260 insertions(+), 58 deletions(-) create mode 100644 client/src/components/Workflow/Editor/Forms/FormCollectionType.vue create mode 100644 client/src/components/Workflow/Editor/Forms/FormInputCollection.vue create mode 100644 client/src/components/Workflow/Editor/composables/useToolState.ts diff --git a/client/src/components/Form/Elements/FormSelection.vue b/client/src/components/Form/Elements/FormSelection.vue index 15176a48f063..12d681f4a517 100644 --- a/client/src/components/Form/Elements/FormSelection.vue +++ b/client/src/components/Form/Elements/FormSelection.vue @@ -100,7 +100,7 @@ watch( ); const showSelectPreference = computed( - () => props.multiple && props.display !== "checkboxes" && props.display !== "radio" + () => props.multiple && props.display !== "checkboxes" && props.display !== "radio" && props.display !== "simple" ); const displayMany = computed(() => showSelectPreference.value && useMany.value); diff --git a/client/src/components/Workflow/Editor/Forms/FormCollectionType.vue b/client/src/components/Workflow/Editor/Forms/FormCollectionType.vue new file mode 100644 index 000000000000..a93cb5ded6f1 --- /dev/null +++ b/client/src/components/Workflow/Editor/Forms/FormCollectionType.vue @@ -0,0 +1,54 @@ + + + diff --git a/client/src/components/Workflow/Editor/Forms/FormDatatype.vue b/client/src/components/Workflow/Editor/Forms/FormDatatype.vue index 466a9fca75e3..cc50d02a612f 100644 --- a/client/src/components/Workflow/Editor/Forms/FormDatatype.vue +++ b/client/src/components/Workflow/Editor/Forms/FormDatatype.vue @@ -1,5 +1,5 @@ diff --git a/client/src/components/Workflow/Editor/Forms/FormDefault.vue b/client/src/components/Workflow/Editor/Forms/FormDefault.vue index 3a0f82c0a9d6..a455ba3a0c5b 100644 --- a/client/src/components/Workflow/Editor/Forms/FormDefault.vue +++ b/client/src/components/Workflow/Editor/Forms/FormDefault.vue @@ -43,8 +43,15 @@ v-if="isSubworkflow" :step="step" @onUpdateStep="(id, step) => emit('onUpdateStep', id, step)" /> + + +import { computed, toRef } from "vue"; + +import type { DatatypesMapperModel } from "@/components/Datatypes/model"; +import type { Step } from "@/stores/workflowStepStore"; + +import { useToolState } from "../composables/useToolState"; + +import FormElement from "@/components/Form/FormElement.vue"; +import FormCollectionType from "@/components/Workflow/Editor/Forms/FormCollectionType.vue"; +import FormDatatype from "@/components/Workflow/Editor/Forms/FormDatatype.vue"; + +interface ToolState { + collection_type: string | null; + optional: boolean; + format: string | null; + tag: string | null; +} + +const props = defineProps<{ + step: Step; + datatypes: DatatypesMapperModel["datatypes"]; +}>(); + +const stepRef = toRef(props, "step"); +const { toolState } = useToolState(stepRef); + +function cleanToolState(): ToolState { + if (toolState.value) { + return { ...toolState.value } as unknown as ToolState; + } else { + return { + collection_type: null, + optional: false, + tag: null, + format: null, + }; + } +} + +const emit = defineEmits(["onChange"]); + +function onDatatype(newDatatype: string[]) { + const state = cleanToolState(); + state.format = newDatatype.join(","); + emit("onChange", state); +} + +function onTags(newTags: string | null) { + const state = cleanToolState(); + state.tag = newTags; + emit("onChange", state); +} + +function onOptional(newOptional: boolean) { + const state = cleanToolState(); + state.optional = newOptional; + emit("onChange", state); +} + +function onCollectionType(newCollectionType: string | null) { + const state = cleanToolState(); + state.collection_type = newCollectionType; + emit("onChange", state); +} + +const formatsAsList = computed(() => { + const formatStr = toolState.value?.format as string | string[] | null; + if (formatStr && typeof formatStr === "string") { + return formatStr.split(/\s*,\s*/); + } else if (formatStr) { + return formatStr; + } else { + return []; + } +}); + +// Terrible Hack: The parent component (./FormDefault.vue) ignores the first update, so +// I am sending a dummy update here. Ideally, the parent FormDefault would not expect this. +emit("onChange", cleanToolState()); + + + diff --git a/client/src/components/Workflow/Editor/composables/useStepProps.ts b/client/src/components/Workflow/Editor/composables/useStepProps.ts index 95e4787bdfd9..fe63f09b3930 100644 --- a/client/src/components/Workflow/Editor/composables/useStepProps.ts +++ b/client/src/components/Workflow/Editor/composables/useStepProps.ts @@ -12,6 +12,7 @@ export function useStepProps(step: Ref) { inputs: stepInputs, outputs: stepOutputs, post_job_actions: postJobActions, + tool_state: toolState, } = toRefs(step); const label = computed(() => step.value.label ?? undefined); @@ -29,5 +30,6 @@ export function useStepProps(step: Ref) { stepOutputs, configForm, postJobActions, + toolState, }; } diff --git a/client/src/components/Workflow/Editor/composables/useToolState.ts b/client/src/components/Workflow/Editor/composables/useToolState.ts new file mode 100644 index 000000000000..6feb13589de9 --- /dev/null +++ b/client/src/components/Workflow/Editor/composables/useToolState.ts @@ -0,0 +1,40 @@ +import { toRefs } from "@vueuse/core"; +import { computed, type Ref } from "vue"; + +import { type Step } from "@/stores/workflowStepStore"; + +export function useToolState(step: Ref) { + const { tool_state: rawToolStateRef } = toRefs(step); + + const toolState = computed(() => { + const rawToolState: Record = rawToolStateRef.value; + const parsedToolState: Record = {}; + + // This is less than ideal in a couple ways. The fact the JSON response + // has encoded JSON is gross and it would be great for module types that + // do not use the tool form to just return a simple JSON blob without + // the extra encoded. As a step two if each of these module types could + // also define a schema so we could use typed entities shared between the + // client and server that would be ideal. + for (const key in rawToolState) { + if (Object.prototype.hasOwnProperty.call(rawToolState, key)) { + const value = rawToolState[key]; + if (typeof value === "string") { + try { + const parsedValue = JSON.parse(value); + parsedToolState[key] = parsedValue; + } catch (error) { + parsedToolState[key] = rawToolState[key]; + } + } else { + parsedToolState[key] = rawToolState[key]; + } + } + } + return parsedToolState; + }); + + return { + toolState, + }; +} diff --git a/client/src/components/Workflow/Editor/modules/collectionTypeDescription.ts b/client/src/components/Workflow/Editor/modules/collectionTypeDescription.ts index 792a842b0df9..cc48c2a69117 100644 --- a/client/src/components/Workflow/Editor/modules/collectionTypeDescription.ts +++ b/client/src/components/Workflow/Editor/modules/collectionTypeDescription.ts @@ -128,3 +128,13 @@ export class CollectionTypeDescription implements CollectionTypeDescriptor { return str.indexOf(suffix, str.length - suffix.length) !== -1; } } + +const collectionTypeRegex = /^(list|paired)(:(list|paired))*$/; + +export function isValidCollectionTypeStr(collectionType: string | undefined) { + if (collectionType) { + return collectionTypeRegex.test(collectionType); + } else { + return true; + } +} diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index b52d495d40e6..e82dc623497a 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -528,15 +528,22 @@ def build_module(self, trans: GalaxyWebTransaction, payload=None): # payload is tool state if payload is None: payload = {} + module_type = payload.get("type", "tool") inputs = payload.get("inputs", {}) trans.workflow_building_mode = workflow_building_modes.ENABLED - module = module_factory.from_dict(trans, payload, from_tool_form=True) - + from_tool_form = True if module_type != "data_collection_input" else False + if not from_tool_form and "tool_state" not in payload and "inputs" in payload: + # tool state not sent, use the manually constructed inputs + payload["tool_state"] = payload["inputs"] + module = module_factory.from_dict(trans, payload, from_tool_form=from_tool_form) module_state: Dict[str, Any] = {} errors: ParameterValidationErrorsT = {} - populate_state(trans, module.get_inputs(), inputs, module_state, errors=errors, check=True) - module.recover_state(module_state, from_tool_form=True) - module.check_and_update_state() + if from_tool_form: + populate_state(trans, module.get_inputs(), inputs, module_state, errors=errors, check=True) + module.recover_state(module_state, from_tool_form=True) + module.check_and_update_state() + else: + module_state = module.get_export_state() step_dict = { "name": module.get_name(), "tool_state": module_state, @@ -546,7 +553,7 @@ def build_module(self, trans: GalaxyWebTransaction, payload=None): "config_form": module.get_config_form(), "errors": errors or None, } - if payload["type"] == "tool": + if module_type == "tool": step_dict["tool_version"] = module.get_version() return step_dict diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 835b9ae2a710..3d0f267ff3bb 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -1114,34 +1114,8 @@ class InputDataCollectionModule(InputModule): collection_type = default_collection_type def get_inputs(self): - parameter_def = self._parse_state_into_dict() - collection_type = parameter_def["collection_type"] - tag = parameter_def["tag"] - optional = parameter_def["optional"] - collection_type_source = dict( - name="collection_type", label="Collection type", type="text", value=collection_type - ) - collection_type_source["options"] = [ - {"value": "list", "label": "List of Datasets"}, - {"value": "paired", "label": "Dataset Pair"}, - {"value": "list:paired", "label": "List of Dataset Pairs"}, - ] - input_collection_type = TextToolParameter(None, collection_type_source) - tag_source = dict( - name="tag", - label="Tag filter", - type="text", - optional="true", - value=tag, - help="Tags to automatically filter inputs", - ) - input_tag = TextToolParameter(None, tag_source) - inputs = {} - inputs["collection_type"] = input_collection_type - inputs["optional"] = optional_param(optional) - inputs["format"] = format_param(self.trans, parameter_def.get("format")) - inputs["tag"] = input_tag - return inputs + # migrated to frontend + return {} def get_runtime_inputs(self, step, connections: Optional[Iterable[WorkflowStepConnection]] = None): parameter_def = self._parse_state_into_dict() diff --git a/test/unit/workflows/test_modules.py b/test/unit/workflows/test_modules.py index 8052c6b11950..fa1e0177701f 100644 --- a/test/unit/workflows/test_modules.py +++ b/test/unit/workflows/test_modules.py @@ -125,17 +125,6 @@ def test_data_collection_input_connections(): assert output["collection_type"] == "list:paired" -def test_data_collection_input_config_form(): - module = __from_step( - type="data_collection_input", - tool_inputs={ - "collection_type": "list:paired", - }, - ) - result = module.get_config_form() - assert result["inputs"][0]["value"], "list:paired" - - def test_cannot_create_tool_modules_for_missing_tools(): trans = MockTrans() module = modules.module_factory.from_dict(trans, {"type": "tool", "tool_id": "cat1"}) From 3b42520959b67fac657d82eee58631d52f9284c8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 12 Dec 2024 19:05:04 -0500 Subject: [PATCH 03/10] Allow swapping handsontable with AG Grid Community in Rule Builder. --- client/package.json | 4 + .../src/components/RuleBuilder/RuleGrid.vue | 125 ++++++++++++++++++ .../src/components/RuleCollectionBuilder.vue | 15 +++ client/src/composables/useAgGrid.ts | 27 ++++ client/yarn.lock | 20 +++ 5 files changed, 191 insertions(+) create mode 100644 client/src/components/RuleBuilder/RuleGrid.vue create mode 100644 client/src/composables/useAgGrid.ts diff --git a/client/package.json b/client/package.json index de12b9f71852..86720d29ca7d 100644 --- a/client/package.json +++ b/client/package.json @@ -45,6 +45,8 @@ "@types/jest": "^29.5.12", "@vueuse/core": "^10.5.0", "@vueuse/math": "^10.9.0", + "ag-grid-community": "^30", + "ag-grid-vue": "^30", "assert": "^2.1.0", "axios": "^1.6.2", "babel-runtime": "^6.26.0", @@ -106,11 +108,13 @@ "vega-embed": "^6.26.0", "vega-lite": "^5.21.0", "vue": "^2.7.14", + "vue-class-component": "^7.2.6", "vue-echarts": "^7.0.3", "vue-infinite-scroll": "^2.0.2", "vue-multiselect": "^2.1.7", "vue-observe-visibility": "^1.0.0", "vue-prismjs": "^1.2.0", + "vue-property-decorator": "^9.1.2", "vue-router": "^3.6.5", "vue-rx": "^6.2.0", "vue-virtual-scroll-list": "^2.3.5", diff --git a/client/src/components/RuleBuilder/RuleGrid.vue b/client/src/components/RuleBuilder/RuleGrid.vue new file mode 100644 index 000000000000..90921457ba18 --- /dev/null +++ b/client/src/components/RuleBuilder/RuleGrid.vue @@ -0,0 +1,125 @@ + + + + + diff --git a/client/src/components/RuleCollectionBuilder.vue b/client/src/components/RuleCollectionBuilder.vue index 2b452512a9d9..b190899e7403 100644 --- a/client/src/components/RuleCollectionBuilder.vue +++ b/client/src/components/RuleCollectionBuilder.vue @@ -452,12 +452,20 @@
+
@@ -576,6 +584,7 @@ import RegularExpressionInput from "components/RuleBuilder/RegularExpressionInpu import RuleDefs from "components/RuleBuilder/rule-definitions"; import RuleComponent from "components/RuleBuilder/RuleComponent"; import RuleDisplay from "components/RuleBuilder/RuleDisplay"; +import RuleGrid from "components/RuleBuilder/RuleGrid"; import RuleModalFooter from "components/RuleBuilder/RuleModalFooter"; import RuleModalHeader from "components/RuleBuilder/RuleModalHeader"; import RuleModalMiddle from "components/RuleBuilder/RuleModalMiddle"; @@ -612,6 +621,7 @@ export default { components: { TooltipOnHover, HotTable, + RuleGrid, RuleComponent, RuleTargetComponent, SavedRulesSelector, @@ -675,6 +685,11 @@ export default { required: false, default: null, }, + gridImplementation: { + type: String, + required: false, + default: "aggrid", + }, }, data: function () { let orientation = "vertical"; diff --git a/client/src/composables/useAgGrid.ts b/client/src/composables/useAgGrid.ts new file mode 100644 index 000000000000..693cbb292d76 --- /dev/null +++ b/client/src/composables/useAgGrid.ts @@ -0,0 +1,27 @@ +import "ag-grid-community/styles/ag-grid.min.css"; +import "ag-grid-community/styles/ag-theme-alpine.min.css"; + +import { type ColumnApi, type GridApi, type GridReadyEvent } from "ag-grid-community"; +import { defineAsyncComponent, nextTick, ref } from "vue"; + +export function useAgGrid(forceGridSize: () => void) { + const gridApi = ref(null); + const columnApi = ref(null); + const theme = "ag-theme-alpine"; + function resizeOnNextTick() { + nextTick(forceGridSize); + } + + function onGridReady(params: GridReadyEvent) { + gridApi.value = params.api; + columnApi.value = params.columnApi; + forceGridSize(); + } + + const AgGridVue = defineAsyncComponent(async () => { + const { AgGridVue } = await import(/* webpackChunkName: "agGrid" */ "ag-grid-vue"); + return AgGridVue; + }); + + return { AgGridVue, gridApi, columnApi, resizeOnNextTick, onGridReady, theme }; +} diff --git a/client/yarn.lock b/client/yarn.lock index cd6c4bf5f55a..9f7eb64fc714 100644 --- a/client/yarn.lock +++ b/client/yarn.lock @@ -3183,6 +3183,16 @@ acorn@^8.1.0, acorn@^8.7.1, acorn@^8.8.0, acorn@^8.8.1, acorn@^8.8.2, acorn@^8.9 resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.10.0.tgz#8be5b3907a67221a81ab23c7889c4c5526b62ec5" integrity sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw== +ag-grid-community@^30: + version "30.2.1" + resolved "https://registry.yarnpkg.com/ag-grid-community/-/ag-grid-community-30.2.1.tgz#a83d153ad1dbec46402ebe89f74ebb4b0710b3c7" + integrity sha512-1slonXskJbbI9ybhTx//4YKfJpZVAEnHL8dui1rQJRSXKByUi+/f7XtvkLsbgBkawoWbqvRAySjYtvz80+kBfA== + +ag-grid-vue@^30: + version "30.2.1" + resolved "https://registry.yarnpkg.com/ag-grid-vue/-/ag-grid-vue-30.2.1.tgz#56395cf053ca2df70107123cf589618ddc3f84b3" + integrity sha512-dnyltXrVUPk0ALQ1PfwnjBtYk/GDOjRjyOMy8LVAiWxVQA6Tmnb/dTnS1yjym1uggu+dDKof2zgPxVKayIHtWg== + agent-base@6: version "6.0.2" resolved "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz" @@ -12355,6 +12365,11 @@ vinyl@^2.0.0: remove-trailing-separator "^1.0.1" replace-ext "^1.0.0" +vue-class-component@^7.2.6: + version "7.2.6" + resolved "https://registry.yarnpkg.com/vue-class-component/-/vue-class-component-7.2.6.tgz#8471e037b8e4762f5a464686e19e5afc708502e4" + integrity sha512-+eaQXVrAm/LldalI272PpDe3+i4mPis0ORiMYxF6Ae4hyuCh15W8Idet7wPUEs4N4YptgFHGys4UrgNQOMyO6w== + vue-demi@>=0.14.5, vue-demi@>=0.14.6: version "0.14.6" resolved "https://registry.yarnpkg.com/vue-demi/-/vue-demi-0.14.6.tgz#dc706582851dc1cdc17a0054f4fec2eb6df74c92" @@ -12446,6 +12461,11 @@ vue-prismjs@^1.2.0: dependencies: prismjs "^1.6.0" +vue-property-decorator@^9.1.2: + version "9.1.2" + resolved "https://registry.yarnpkg.com/vue-property-decorator/-/vue-property-decorator-9.1.2.tgz#266a2eac61ba6527e2e68a6933cfb98fddab5457" + integrity sha512-xYA8MkZynPBGd/w5QFJ2d/NM0z/YeegMqYTphy7NJQXbZcuU6FC6AOdUAcy4SXP+YnkerC6AfH+ldg7PDk9ESQ== + vue-router@^3.6.5: version "3.6.5" resolved "https://registry.yarnpkg.com/vue-router/-/vue-router-3.6.5.tgz#95847d52b9a7e3f1361cb605c8e6441f202afad8" From 8a631af519581411d25f937f4a3d213faf2bf9d7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 11 Dec 2024 10:22:00 -0500 Subject: [PATCH 04/10] Gray out repeat buttons that don't make sense. --- client/src/components/Form/FormRepeat.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/src/components/Form/FormRepeat.vue b/client/src/components/Form/FormRepeat.vue index b12b85bde084..9b938e27f7c2 100644 --- a/client/src/components/Form/FormRepeat.vue +++ b/client/src/components/Form/FormRepeat.vue @@ -132,6 +132,7 @@ const { keyObject } = useKeyedObjects(); Date: Wed, 11 Dec 2024 22:42:07 -0500 Subject: [PATCH 05/10] Reuse in FormData.vue. --- client/src/components/Form/Elements/FormData/FormData.vue | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/client/src/components/Form/Elements/FormData/FormData.vue b/client/src/components/Form/Elements/FormData/FormData.vue index 0695af21369b..b13dd87ae8e9 100644 --- a/client/src/components/Form/Elements/FormData/FormData.vue +++ b/client/src/components/Form/Elements/FormData/FormData.vue @@ -497,13 +497,15 @@ function canAcceptSrc(historyContentType: "dataset" | "dataset_collection", coll } } +const collectionTypesWithBuilders = ["list", "list:paired", "paired"]; + /** Allowed collection types for collection creation */ const effectiveCollectionTypes = props.collectionTypes?.filter((collectionType) => - ["list", "list:paired", "paired"].includes(collectionType) + collectionTypesWithBuilders.includes(collectionType) ); function buildNewCollection(collectionType: string) { - if (!["list", "list:paired", "paired"].includes(collectionType)) { + if (!collectionTypesWithBuilders.includes(collectionType)) { throw Error(`Unknown collection type: ${collectionType}`); } collectionModalType.value = collectionType as "list" | "list:paired" | "paired"; From a86b6e22d6afde199ab7d52f68a7b17addc166a7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 18 May 2020 14:26:38 -0400 Subject: [PATCH 06/10] [WIP] Implement records - heterogenous dataset collections. Existing dataset colleciton types are meant to be homogenous - all datasets of the same time. This introduces CWL-style record dataset collections. --- lib/galaxy/jobs/__init__.py | 5 +- lib/galaxy/managers/collections.py | 12 ++- lib/galaxy/managers/collections_util.py | 1 + lib/galaxy/model/__init__.py | 15 ++- .../model/dataset_collections/builder.py | 22 +++- .../model/dataset_collections/registry.py | 11 +- .../dataset_collections/type_description.py | 9 +- .../dataset_collections/types/__init__.py | 2 +- .../model/dataset_collections/types/list.py | 4 +- .../model/dataset_collections/types/paired.py | 8 +- .../model/dataset_collections/types/record.py | 45 ++++++++ lib/galaxy/tool_util/parser/output_objects.py | 8 +- lib/galaxy/tools/actions/__init__.py | 5 +- lib/galaxy/workflow/modules.py | 2 +- .../api/test_dataset_collections.py | 101 ++++++++++++++++++ 15 files changed, 223 insertions(+), 27 deletions(-) create mode 100644 lib/galaxy/model/dataset_collections/types/record.py diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index c26aa11dbc0a..70b648ffa57d 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1776,8 +1776,9 @@ def _finish_dataset( dataset.mark_unhidden() elif not purged: # If the tool was expected to set the extension, attempt to retrieve it - if dataset.ext == "auto": - dataset.extension = context.get("ext", "data") + context_ext = context.get("ext", "data") + if dataset.ext == "auto" or (dataset.ext == "data" and context_ext != "data"): + dataset.extension = context_ext dataset.init_meta(copy_from=dataset) # if a dataset was copied, it won't appear in our dictionary: # either use the metadata from originating output dataset, or call set_meta on the copies diff --git a/lib/galaxy/managers/collections.py b/lib/galaxy/managers/collections.py index fae6d0563347..e9042e3aac55 100644 --- a/lib/galaxy/managers/collections.py +++ b/lib/galaxy/managers/collections.py @@ -175,6 +175,7 @@ def create( flush=True, completed_job=None, output_name=None, + fields=None, ): """ PRECONDITION: security checks on ability to add to parent @@ -199,6 +200,7 @@ def create( hide_source_items=hide_source_items, copy_elements=copy_elements, history=history, + fields=fields, ) implicit_inputs = [] @@ -285,17 +287,20 @@ def create_dataset_collection( hide_source_items=None, copy_elements=False, history=None, + fields=None, ): # Make sure at least one of these is None. assert element_identifiers is None or elements is None - if element_identifiers is None and elements is None: raise RequestParameterInvalidException(ERROR_INVALID_ELEMENTS_SPECIFICATION) if not collection_type: raise RequestParameterInvalidException(ERROR_NO_COLLECTION_TYPE) - collection_type_description = self.collection_type_descriptions.for_collection_type(collection_type) + collection_type_description = self.collection_type_descriptions.for_collection_type( + collection_type, fields=fields + ) has_subcollections = collection_type_description.has_subcollections() + # If we have elements, this is an internal request, don't need to load # objects from identifiers. if elements is None: @@ -319,8 +324,9 @@ def create_dataset_collection( if elements is not self.ELEMENTS_UNINITIALIZED: type_plugin = collection_type_description.rank_type_plugin() - dataset_collection = builder.build_collection(type_plugin, elements) + dataset_collection = builder.build_collection(type_plugin, elements, fields=fields) else: + # TODO: Pass fields here - need test case first. dataset_collection = model.DatasetCollection(populated=False) dataset_collection.collection_type = collection_type return dataset_collection diff --git a/lib/galaxy/managers/collections_util.py b/lib/galaxy/managers/collections_util.py index 32feab23889d..7f129992c754 100644 --- a/lib/galaxy/managers/collections_util.py +++ b/lib/galaxy/managers/collections_util.py @@ -39,6 +39,7 @@ def api_payload_to_create_params(payload): name=payload.get("name", None), hide_source_items=string_as_bool(payload.get("hide_source_items", False)), copy_elements=string_as_bool(payload.get("copy_elements", False)), + fields=payload.get("fields", None), ) return params diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 8921f6ceaf6c..fb65c01516fb 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6545,12 +6545,21 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): populated_states = DatasetCollectionPopulatedState - def __init__(self, id=None, collection_type=None, populated=True, element_count=None): + def __init__( + self, + id=None, + collection_type=None, + populated=True, + element_count=None, + fields=None, + ): self.id = id self.collection_type = collection_type if not populated: self.populated_state = DatasetCollection.populated_states.NEW self.element_count = element_count + # TODO: persist fields... + self.fields = fields def _build_nested_collection_attributes_stmt( self, @@ -6725,6 +6734,10 @@ def populated_optimized(self): return self._populated_optimized + @property + def allow_implicit_mapping(self): + return self.collection_type != "record" + @property def populated(self): top_level_populated = self.populated_state == DatasetCollection.populated_states.OK diff --git a/lib/galaxy/model/dataset_collections/builder.py b/lib/galaxy/model/dataset_collections/builder.py index 2ae001f33a22..73af774904fe 100644 --- a/lib/galaxy/model/dataset_collections/builder.py +++ b/lib/galaxy/model/dataset_collections/builder.py @@ -4,25 +4,27 @@ from .type_description import COLLECTION_TYPE_DESCRIPTION_FACTORY -def build_collection(type, dataset_instances, collection=None, associated_identifiers=None): +def build_collection(type, dataset_instances, collection=None, associated_identifiers=None, fields=None): """ Build DatasetCollection with populated DatasetcollectionElement objects corresponding to the supplied dataset instances or throw exception if this is not a valid collection of the specified type. """ - dataset_collection = collection or model.DatasetCollection() + dataset_collection = collection or model.DatasetCollection(fields=fields) associated_identifiers = associated_identifiers or set() - set_collection_elements(dataset_collection, type, dataset_instances, associated_identifiers) + set_collection_elements(dataset_collection, type, dataset_instances, associated_identifiers, fields=fields) return dataset_collection -def set_collection_elements(dataset_collection, type, dataset_instances, associated_identifiers): +def set_collection_elements(dataset_collection, type, dataset_instances, associated_identifiers, fields=None): new_element_keys = OrderedSet(dataset_instances.keys()) - associated_identifiers new_dataset_instances = {k: dataset_instances[k] for k in new_element_keys} dataset_collection.element_count = dataset_collection.element_count or 0 element_index = dataset_collection.element_count elements = [] - for element in type.generate_elements(new_dataset_instances): + if fields == "auto": + fields = guess_fields(dataset_instances) + for element in type.generate_elements(new_dataset_instances, fields=fields): element.element_index = element_index add_object_to_object_session(element, dataset_collection) element.collection = dataset_collection @@ -35,6 +37,16 @@ def set_collection_elements(dataset_collection, type, dataset_instances, associa return dataset_collection +def guess_fields(dataset_instances): + fields = [] + for identifier, element in dataset_instances.items(): + # TODO: Make generic enough to handle nested record types. + assert element.history_content_type == "dataset" + fields.append({"type": "File", "name": identifier}) + + return fields + + class CollectionBuilder: """Purely functional builder pattern for building a dataset collection.""" diff --git a/lib/galaxy/model/dataset_collections/registry.py b/lib/galaxy/model/dataset_collections/registry.py index 9c849dfdad6f..bd148edafd2d 100644 --- a/lib/galaxy/model/dataset_collections/registry.py +++ b/lib/galaxy/model/dataset_collections/registry.py @@ -2,9 +2,14 @@ from .types import ( list, paired, + record, ) -PLUGIN_CLASSES = [list.ListDatasetCollectionType, paired.PairedDatasetCollectionType] +PLUGIN_CLASSES = [ + list.ListDatasetCollectionType, + paired.PairedDatasetCollectionType, + record.RecordDatasetCollectionType, +] class DatasetCollectionTypesRegistry: @@ -14,13 +19,13 @@ def __init__(self): def get(self, plugin_type): return self.__plugins[plugin_type] - def prototype(self, plugin_type): + def prototype(self, plugin_type, fields=None): plugin_type_object = self.get(plugin_type) if not hasattr(plugin_type_object, "prototype_elements"): raise Exception(f"Cannot pre-determine structure for collection of type {plugin_type}") dataset_collection = model.DatasetCollection() - for e in plugin_type_object.prototype_elements(): + for e in plugin_type_object.prototype_elements(fields=fields): e.collection = dataset_collection return dataset_collection diff --git a/lib/galaxy/model/dataset_collections/type_description.py b/lib/galaxy/model/dataset_collections/type_description.py index cade102453ca..172ac1976b27 100644 --- a/lib/galaxy/model/dataset_collections/type_description.py +++ b/lib/galaxy/model/dataset_collections/type_description.py @@ -9,9 +9,9 @@ def __init__(self, type_registry=DATASET_COLLECTION_TYPES_REGISTRY): # I think. self.type_registry = type_registry - def for_collection_type(self, collection_type): + def for_collection_type(self, collection_type, fields=None): assert collection_type is not None - return CollectionTypeDescription(collection_type, self) + return CollectionTypeDescription(collection_type, self, fields=fields) class CollectionTypeDescription: @@ -47,12 +47,15 @@ class CollectionTypeDescription: collection_type: str - def __init__(self, collection_type: Union[str, "CollectionTypeDescription"], collection_type_description_factory): + def __init__( + self, collection_type: Union[str, "CollectionTypeDescription"], collection_type_description_factory, fields=None + ): if isinstance(collection_type, CollectionTypeDescription): self.collection_type = collection_type.collection_type else: self.collection_type = collection_type self.collection_type_description_factory = collection_type_description_factory + self.fields = fields self.__has_subcollections = self.collection_type.find(":") > 0 def child_collection_type(self): diff --git a/lib/galaxy/model/dataset_collections/types/__init__.py b/lib/galaxy/model/dataset_collections/types/__init__.py index bfcf7bae79a6..c294f6957be6 100644 --- a/lib/galaxy/model/dataset_collections/types/__init__.py +++ b/lib/galaxy/model/dataset_collections/types/__init__.py @@ -11,7 +11,7 @@ class DatasetCollectionType(metaclass=ABCMeta): @abstractmethod - def generate_elements(self, dataset_instances): + def generate_elements(self, dataset_instances: dict, **kwds): """Generate DatasetCollectionElements with corresponding to the supplied dataset instances or throw exception if this is not a valid collection of the specified type. diff --git a/lib/galaxy/model/dataset_collections/types/list.py b/lib/galaxy/model/dataset_collections/types/list.py index 18ce4db76537..d4421d009c34 100644 --- a/lib/galaxy/model/dataset_collections/types/list.py +++ b/lib/galaxy/model/dataset_collections/types/list.py @@ -7,8 +7,8 @@ class ListDatasetCollectionType(BaseDatasetCollectionType): collection_type = "list" - def generate_elements(self, elements): - for identifier, element in elements.items(): + def generate_elements(self, dataset_instances, **kwds): + for identifier, element in dataset_instances.items(): association = DatasetCollectionElement( element=element, element_identifier=identifier, diff --git a/lib/galaxy/model/dataset_collections/types/paired.py b/lib/galaxy/model/dataset_collections/types/paired.py index 4ae95a1442a2..e774ab67aace 100644 --- a/lib/galaxy/model/dataset_collections/types/paired.py +++ b/lib/galaxy/model/dataset_collections/types/paired.py @@ -15,21 +15,21 @@ class PairedDatasetCollectionType(BaseDatasetCollectionType): collection_type = "paired" - def generate_elements(self, elements): - if forward_dataset := elements.get(FORWARD_IDENTIFIER): + def generate_elements(self, dataset_instances, **kwds): + if forward_dataset := dataset_instances.get(FORWARD_IDENTIFIER): left_association = DatasetCollectionElement( element=forward_dataset, element_identifier=FORWARD_IDENTIFIER, ) yield left_association - if reverse_dataset := elements.get(REVERSE_IDENTIFIER): + if reverse_dataset := dataset_instances.get(REVERSE_IDENTIFIER): right_association = DatasetCollectionElement( element=reverse_dataset, element_identifier=REVERSE_IDENTIFIER, ) yield right_association - def prototype_elements(self): + def prototype_elements(self, **kwds): left_association = DatasetCollectionElement( element=HistoryDatasetAssociation(), element_identifier=FORWARD_IDENTIFIER, diff --git a/lib/galaxy/model/dataset_collections/types/record.py b/lib/galaxy/model/dataset_collections/types/record.py new file mode 100644 index 000000000000..193509f439ee --- /dev/null +++ b/lib/galaxy/model/dataset_collections/types/record.py @@ -0,0 +1,45 @@ +from galaxy.exceptions import RequestParameterMissingException +from galaxy.model import ( + DatasetCollectionElement, + HistoryDatasetAssociation, +) +from ..types import BaseDatasetCollectionType + + +class RecordDatasetCollectionType(BaseDatasetCollectionType): + """Arbitrary CWL-style record type.""" + + collection_type = "record" + + def generate_elements(self, dataset_instances, **kwds): + fields = kwds.get("fields", None) + if fields is None: + raise RequestParameterMissingException("Missing or null parameter 'fields' required for record types.") + if len(dataset_instances) != len(fields): + self._validation_failed("Supplied element do not match fields.") + index = 0 + for identifier, element in dataset_instances.items(): + field = fields[index] + if field["name"] != identifier: + self._validation_failed("Supplied element do not match fields.") + + # TODO: validate type and such. + association = DatasetCollectionElement( + element=element, + element_identifier=identifier, + ) + yield association + index += 1 + + def prototype_elements(self, fields=None, **kwds): + if fields is None: + raise RequestParameterMissingException("Missing or null parameter 'fields' required for record types.") + for field in fields: + name = field.get("name", None) + assert name + assert field.get("type", "File") # NS: this assert doesn't make sense as it is + field_dataset = DatasetCollectionElement( + element=HistoryDatasetAssociation(), + element_identifier=name, + ) + yield field_dataset diff --git a/lib/galaxy/tool_util/parser/output_objects.py b/lib/galaxy/tool_util/parser/output_objects.py index 63148c1fb946..7825d6308197 100644 --- a/lib/galaxy/tool_util/parser/output_objects.py +++ b/lib/galaxy/tool_util/parser/output_objects.py @@ -402,12 +402,14 @@ def __init__( collection_type_from_rules: Optional[str] = None, structured_like: Optional[str] = None, dataset_collector_descriptions: Optional[List[DatasetCollectionDescription]] = None, + fields=None, ) -> None: self.collection_type = collection_type self.collection_type_source = collection_type_source self.collection_type_from_rules = collection_type_from_rules self.structured_like = structured_like self.dataset_collector_descriptions = dataset_collector_descriptions or [] + self.fields = fields if collection_type and collection_type_source: raise ValueError("Cannot set both type and type_source on collection output.") if ( @@ -424,6 +426,10 @@ def __init__( raise ValueError( "Cannot specify dynamic structure (discover_datasets) and collection type attributes structured_like or collection_type_from_rules." ) + if collection_type == "record" and fields is None: + raise ValueError("If record outputs are defined, fields must be defined as well.") + if fields is not None and collection_type != "record": + raise ValueError("If fields are specified for outputs, the collection type must be record.") self.dynamic = bool(dataset_collector_descriptions) def collection_prototype(self, inputs, type_registry): @@ -433,7 +439,7 @@ def collection_prototype(self, inputs, type_registry): else: collection_type = self.collection_type assert collection_type - collection_prototype = type_registry.prototype(collection_type) + collection_prototype = type_registry.prototype(collection_type, fields=self.fields) collection_prototype.collection_type = collection_type return collection_prototype diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 841eea988d49..f7a2138795a3 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -679,7 +679,10 @@ def handle_output(name, output, hidden=None): assert not element_identifiers # known_outputs must have been empty element_kwds = dict(elements=collections_manager.ELEMENTS_UNINITIALIZED) else: - element_kwds = dict(element_identifiers=element_identifiers) + element_kwds = dict( + element_identifiers=element_identifiers, + fields=output.structure.fields, + ) output_collections.create_collection( output=output, name=name, completed_job=completed_job, **element_kwds ) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 3d0f267ff3bb..ae5f8e31e920 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -551,7 +551,7 @@ def _find_collections_to_match(self, progress: "WorkflowProgress", step, all_inp for input_dict in all_inputs: name = input_dict["name"] data = progress.replacement_for_input(self.trans, step, input_dict) - can_map_over = hasattr(data, "collection") # and data.collection.allow_implicit_mapping + can_map_over = hasattr(data, "collection") and data.collection.allow_implicit_mapping if not can_map_over: continue diff --git a/lib/galaxy_test/api/test_dataset_collections.py b/lib/galaxy_test/api/test_dataset_collections.py index d7710c57b2fa..a4ba01877e5e 100644 --- a/lib/galaxy_test/api/test_dataset_collections.py +++ b/lib/galaxy_test/api/test_dataset_collections.py @@ -1,3 +1,4 @@ +import json import zipfile from io import BytesIO from typing import List @@ -100,6 +101,106 @@ def test_create_list_of_new_pairs(self): pair_1_element_1 = pair_elements[0] assert pair_1_element_1["element_index"] == 0 + def test_create_record(self, history_id): + contents = [ + ("condition", "1\t2\t3"), + ("control1", "4\t5\t6"), + ("control2", "7\t8\t9"), + ] + record_identifiers = self.dataset_collection_populator.list_identifiers(history_id, contents) + fields = [ + {"name": "condition", "type": "File"}, + {"name": "control1", "type": "File"}, + {"name": "control2", "type": "File"}, + ] + payload = dict( + name="a record", + instance_type="history", + history_id=history_id, + element_identifiers=json.dumps(record_identifiers), + collection_type="record", + fields=json.dumps(fields), + ) + create_response = self._post("dataset_collections", payload) + dataset_collection = self._check_create_response(create_response) + assert dataset_collection["collection_type"] == "record" + assert dataset_collection["name"] == "a record" + returned_collections = dataset_collection["elements"] + assert len(returned_collections) == 3, dataset_collection + record_pos_0_element = returned_collections[0] + self._assert_has_keys(record_pos_0_element, "element_index") + record_pos_0_object = record_pos_0_element["object"] + self._assert_has_keys(record_pos_0_object, "name", "history_content_type") + + def test_record_requires_fields(self, history_id): + contents = [ + ("condition", "1\t2\t3"), + ("control1", "4\t5\t6"), + ("control2", "7\t8\t9"), + ] + record_identifiers = self.dataset_collection_populator.list_identifiers(history_id, contents) + payload = dict( + name="a record", + instance_type="history", + history_id=history_id, + element_identifiers=json.dumps(record_identifiers), + collection_type="record", + ) + create_response = self._post("dataset_collections", payload) + self._assert_status_code_is(create_response, 400) + + def test_record_auto_fields(self, history_id): + contents = [ + ("condition", "1\t2\t3"), + ("control1", "4\t5\t6"), + ("control2", "7\t8\t9"), + ] + record_identifiers = self.dataset_collection_populator.list_identifiers(history_id, contents) + payload = dict( + name="a record", + instance_type="history", + history_id=history_id, + element_identifiers=json.dumps(record_identifiers), + collection_type="record", + fields="auto", + ) + create_response = self._post("dataset_collections", payload) + self._check_create_response(create_response) + + def test_record_field_validation(self, history_id): + contents = [ + ("condition", "1\t2\t3"), + ("control1", "4\t5\t6"), + ("control2", "7\t8\t9"), + ] + record_identifiers = self.dataset_collection_populator.list_identifiers(history_id, contents) + too_few_fields = [ + {"name": "condition", "type": "File"}, + {"name": "control1", "type": "File"}, + ] + too_many_fields = [ + {"name": "condition", "type": "File"}, + {"name": "control1", "type": "File"}, + {"name": "control2", "type": "File"}, + {"name": "control3", "type": "File"}, + ] + wrong_name_fields = [ + {"name": "condition", "type": "File"}, + {"name": "control1", "type": "File"}, + {"name": "control3", "type": "File"}, + ] + for fields in [too_few_fields, too_many_fields, wrong_name_fields]: + payload = dict( + name="a record", + instance_type="history", + history_id=history_id, + element_identifiers=json.dumps(record_identifiers), + collection_type="record", + fields=json.dumps(fields), + ) + create_response = self._post("dataset_collections", payload) + self._assert_status_code_is(create_response, 400) + def test_list_download(self): with self.dataset_populator.test_history(require_new=False) as history_id: fetch_response = self.dataset_collection_populator.create_list_in_history( From 137c8b413dff94e9a703cb779dc7b1def87ddacf Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 9 Dec 2024 10:51:37 -0500 Subject: [PATCH 07/10] DB migration for sample sheets. --- .../ec25b23b08e2_implement_sample_sheets.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 lib/galaxy/model/migrations/alembic/versions_gxy/ec25b23b08e2_implement_sample_sheets.py diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/ec25b23b08e2_implement_sample_sheets.py b/lib/galaxy/model/migrations/alembic/versions_gxy/ec25b23b08e2_implement_sample_sheets.py new file mode 100644 index 000000000000..558a485020cc --- /dev/null +++ b/lib/galaxy/model/migrations/alembic/versions_gxy/ec25b23b08e2_implement_sample_sheets.py @@ -0,0 +1,37 @@ +"""Implement sample sheets. + +Revision ID: ec25b23b08e2 +Revises: 75348cfb3715 +Create Date: 2024-12-09 10:28:54.902847 + +""" + +from sqlalchemy import Column + +from galaxy.model.custom_types import JSONType +from galaxy.model.migrations.util import ( + add_column, + drop_column, + transaction, +) + +# revision identifiers, used by Alembic. +revision = "ec25b23b08e2" +down_revision = "75348cfb3715" +branch_labels = None +depends_on = None + +dataset_collection_element_table = "dataset_collection_element" +dataset_collection_table = "dataset_collection" + + +def upgrade(): + with transaction(): + add_column(dataset_collection_table, Column("column_definitions", JSONType(), default=None)) + add_column(dataset_collection_element_table, Column("columns", JSONType(), default=None)) + + +def downgrade(): + with transaction(): + drop_column(dataset_collection_table, "column_definitions") + drop_column(dataset_collection_element_table, "columns") From 8f6154f8437ff490333b805be68feb573283fdd1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 9 Dec 2024 11:34:06 -0500 Subject: [PATCH 08/10] WIP: sample sheets... --- client/src/api/index.ts | 4 + .../Collections/CollectionCreatorModal.vue | 10 + .../SampleSheetCollectionCreator.vue | 32 ++++ .../Collections/sheet/SampleSheetGrid.vue | 159 ++++++++++++++++ .../Form/Elements/FormData/FormData.vue | 7 +- .../History/adapters/buildCollectionModal.ts | 2 +- .../Editor/Forms/FormCollectionType.vue | 1 + .../Editor/Forms/FormColumnDefinition.vue | 56 ++++++ .../Editor/Forms/FormColumnDefinitionType.vue | 48 +++++ .../Editor/Forms/FormColumnDefinitions.vue | 176 ++++++++++++++++++ .../Editor/Forms/FormInputCollection.vue | 15 ++ .../modules/collectionTypeDescription.ts | 2 +- client/src/stores/workflowStepStore.ts | 2 + lib/galaxy/managers/collections.py | 17 +- lib/galaxy/managers/collections_util.py | 6 + lib/galaxy/model/__init__.py | 16 +- .../model/dataset_collections/builder.py | 25 ++- .../model/dataset_collections/registry.py | 2 + .../dataset_collections/types/sample_sheet.py | 30 +++ .../types/sample_sheet_util.py | 103 ++++++++++ .../types/sample_sheet_workbook.py | 160 ++++++++++++++++ .../ec25b23b08e2_implement_sample_sheets.py | 2 + lib/galaxy/schema/schema.py | 42 +++++ lib/galaxy/tool_util/client/staging.py | 5 +- lib/galaxy/tool_util/cwl/util.py | 17 +- .../tool_util/parser/parameter_validators.py | 17 +- lib/galaxy/util/rules_dsl.py | 24 +++ lib/galaxy/util/rules_dsl_spec.yml | 22 +++ .../webapps/galaxy/api/dataset_collections.py | 45 +++++ lib/galaxy/workflow/modules.py | 5 + .../api/test_dataset_collections.py | 158 +++++++++++++++- lib/galaxy_test/api/test_tools.py | 3 + lib/galaxy_test/base/populators.py | 17 +- lib/galaxy_test/base/rules_test_data.py | 51 +++++ packages/data/setup.cfg | 1 + pyproject.toml | 1 + .../test_sample_sheet_util.py | 93 +++++++++ .../test_sample_sheet_workbook.py | 58 ++++++ 38 files changed, 1414 insertions(+), 20 deletions(-) create mode 100644 client/src/components/Collections/SampleSheetCollectionCreator.vue create mode 100644 client/src/components/Collections/sheet/SampleSheetGrid.vue create mode 100644 client/src/components/Workflow/Editor/Forms/FormColumnDefinition.vue create mode 100644 client/src/components/Workflow/Editor/Forms/FormColumnDefinitionType.vue create mode 100644 client/src/components/Workflow/Editor/Forms/FormColumnDefinitions.vue create mode 100644 lib/galaxy/model/dataset_collections/types/sample_sheet.py create mode 100644 lib/galaxy/model/dataset_collections/types/sample_sheet_util.py create mode 100644 lib/galaxy/model/dataset_collections/types/sample_sheet_workbook.py create mode 100644 test/unit/data/dataset_collections/test_sample_sheet_util.py create mode 100644 test/unit/data/dataset_collections/test_sample_sheet_workbook.py diff --git a/client/src/api/index.ts b/client/src/api/index.ts index 6f9ce9d0213d..27a4d572c797 100644 --- a/client/src/api/index.ts +++ b/client/src/api/index.ts @@ -312,4 +312,8 @@ export type ObjectExportTaskResponse = components["schemas"]["ObjectExportTaskRe export type ExportObjectRequestMetadata = components["schemas"]["ExportObjectRequestMetadata"]; export type ExportObjectResultMetadata = components["schemas"]["ExportObjectResultMetadata"]; +export type SampleSheetColumnDefinition = components["schemas"]["SampleSheetColumnDefinition"]; +export type SampleSheetColumnDefinitionType = SampleSheetColumnDefinition["type"]; +export type SampleSheetColumnDefinitions = SampleSheetColumnDefinition[] | null; + export type AsyncTaskResultSummary = components["schemas"]["AsyncTaskResultSummary"]; diff --git a/client/src/components/Collections/CollectionCreatorModal.vue b/client/src/components/Collections/CollectionCreatorModal.vue index 8db25d243c6d..dfd0ae54918d 100644 --- a/client/src/components/Collections/CollectionCreatorModal.vue +++ b/client/src/components/Collections/CollectionCreatorModal.vue @@ -16,6 +16,7 @@ import type { CollectionType, DatasetPair } from "../History/adapters/buildColle import ListCollectionCreator from "./ListCollectionCreator.vue"; import PairCollectionCreator from "./PairCollectionCreator.vue"; import PairedListCollectionCreator from "./PairedListCollectionCreator.vue"; +import SampleSheetCollectionCreator from "./SampleSheetCollectionCreator.vue"; import Heading from "@/components/Common/Heading.vue"; import GenericItem from "@/components/History/Content/GenericItem.vue"; import LoadingSpan from "@/components/LoadingSpan.vue"; @@ -313,6 +314,15 @@ function resetModal() { :extensions="props.extensions" @clicked-create="createPairedCollection" @on-cancel="hideModal" /> + diff --git a/client/src/components/Collections/SampleSheetCollectionCreator.vue b/client/src/components/Collections/SampleSheetCollectionCreator.vue new file mode 100644 index 000000000000..5781ad524fcc --- /dev/null +++ b/client/src/components/Collections/SampleSheetCollectionCreator.vue @@ -0,0 +1,32 @@ + + + diff --git a/client/src/components/Collections/sheet/SampleSheetGrid.vue b/client/src/components/Collections/sheet/SampleSheetGrid.vue new file mode 100644 index 000000000000..864ed374f3ad --- /dev/null +++ b/client/src/components/Collections/sheet/SampleSheetGrid.vue @@ -0,0 +1,159 @@ + + + diff --git a/client/src/components/Form/Elements/FormData/FormData.vue b/client/src/components/Form/Elements/FormData/FormData.vue index b13dd87ae8e9..0590639b7ebe 100644 --- a/client/src/components/Form/Elements/FormData/FormData.vue +++ b/client/src/components/Form/Elements/FormData/FormData.vue @@ -33,6 +33,7 @@ const COLLECTION_TYPE_TO_LABEL: Record = { list: "list", "list:paired": "list of dataset pairs", paired: "dataset pair", + sample_sheet: "sample sheet derived", }; type SelectOption = { @@ -88,7 +89,7 @@ const dragTarget: Ref = ref(null); // Collection creator modal settings const collectionModalShow = ref(false); -const collectionModalType = ref<"list" | "list:paired" | "paired">("list"); +const collectionModalType = ref<"list" | "list:paired" | "paired" | "sample_sheet">("list"); const { currentHistoryId } = storeToRefs(useHistoryStore()); const restrictsExtensions = computed(() => { const extensions = props.extensions; @@ -497,7 +498,7 @@ function canAcceptSrc(historyContentType: "dataset" | "dataset_collection", coll } } -const collectionTypesWithBuilders = ["list", "list:paired", "paired"]; +const collectionTypesWithBuilders = ["list", "list:paired", "paired", "sample_sheet"]; /** Allowed collection types for collection creation */ const effectiveCollectionTypes = props.collectionTypes?.filter((collectionType) => @@ -508,7 +509,7 @@ function buildNewCollection(collectionType: string) { if (!collectionTypesWithBuilders.includes(collectionType)) { throw Error(`Unknown collection type: ${collectionType}`); } - collectionModalType.value = collectionType as "list" | "list:paired" | "paired"; + collectionModalType.value = collectionType as "list" | "list:paired" | "paired" | "sample_sheet"; collectionModalShow.value = true; } diff --git a/client/src/components/History/adapters/buildCollectionModal.ts b/client/src/components/History/adapters/buildCollectionModal.ts index 53a41b424914..5b03fc060003 100644 --- a/client/src/components/History/adapters/buildCollectionModal.ts +++ b/client/src/components/History/adapters/buildCollectionModal.ts @@ -14,7 +14,7 @@ import jQuery from "jquery"; import type { HDASummary, HistoryItemSummary } from "@/api"; import RULE_BASED_COLLECTION_CREATOR from "@/components/Collections/RuleBasedCollectionCreatorModal"; -export type CollectionType = "list" | "paired" | "list:paired" | "rules"; +export type CollectionType = "list" | "paired" | "list:paired" | "rules" | "sample_sheet"; export type DatasetPair = { forward: HDASummary; reverse: HDASummary; diff --git a/client/src/components/Workflow/Editor/Forms/FormCollectionType.vue b/client/src/components/Workflow/Editor/Forms/FormCollectionType.vue index a93cb5ded6f1..77b5aa8161f2 100644 --- a/client/src/components/Workflow/Editor/Forms/FormCollectionType.vue +++ b/client/src/components/Workflow/Editor/Forms/FormCollectionType.vue @@ -23,6 +23,7 @@ const collectionTypeOptions = [ { value: "list", label: "List of Datasets" }, { value: "paired", label: "Dataset Pair" }, { value: "list:paired", label: "List of Dataset Pairs" }, + { value: "sample_sheet", label: "Sample Sheet of Datasets" }, ]; function updateValue(newValue: string | undefined) { diff --git a/client/src/components/Workflow/Editor/Forms/FormColumnDefinition.vue b/client/src/components/Workflow/Editor/Forms/FormColumnDefinition.vue new file mode 100644 index 000000000000..1e74cb43b04f --- /dev/null +++ b/client/src/components/Workflow/Editor/Forms/FormColumnDefinition.vue @@ -0,0 +1,56 @@ + + + diff --git a/client/src/components/Workflow/Editor/Forms/FormColumnDefinitionType.vue b/client/src/components/Workflow/Editor/Forms/FormColumnDefinitionType.vue new file mode 100644 index 000000000000..84aea614868e --- /dev/null +++ b/client/src/components/Workflow/Editor/Forms/FormColumnDefinitionType.vue @@ -0,0 +1,48 @@ + + + diff --git a/client/src/components/Workflow/Editor/Forms/FormColumnDefinitions.vue b/client/src/components/Workflow/Editor/Forms/FormColumnDefinitions.vue new file mode 100644 index 000000000000..229e92df209f --- /dev/null +++ b/client/src/components/Workflow/Editor/Forms/FormColumnDefinitions.vue @@ -0,0 +1,176 @@ + + + + + diff --git a/client/src/components/Workflow/Editor/Forms/FormInputCollection.vue b/client/src/components/Workflow/Editor/Forms/FormInputCollection.vue index ea17f09fb50d..ef5379b0e0d1 100644 --- a/client/src/components/Workflow/Editor/Forms/FormInputCollection.vue +++ b/client/src/components/Workflow/Editor/Forms/FormInputCollection.vue @@ -1,6 +1,7 @@