Support secrets in tool requirements

lib/galaxy/tool_util/cwl/parser.py

@@ -219,6 +219,9 @@ def software_requirements(self) -> List:
     def resource_requirements(self) -> List:
         return self.hints_or_requirements_of_class("ResourceRequirement")
 
+    def secrets(self) -> List:
+        return self.hints_or_requirements_of_class("Secrets")
+
 
 class CommandLineToolProxy(ToolProxy):
     _class = "CommandLineTool"

lib/galaxy/tool_util/deps/requirements.py

@@ -20,6 +20,7 @@
 
 from galaxy.util import (
     asbool,
+    string_as_bool,
     xml_text,
 )
 from galaxy.util.oset import OrderedSet
@@ -305,27 +306,78 @@ def resource_requirements_from_list(requirements: Iterable[Dict[str, Any]]) -> L
     return rr
 
 
+class SecretsRequirement:
+    def __init__(
+        self,
+        type: str,
+        user_preferences_key: str,
+        inject_as_env: str,
+        label: Optional[str] = "",
+        required: Optional[bool] = False,
+    ) -> None:
+        self.type = type
+        self.user_preferences_key = user_preferences_key
+        self.inject_as_env = inject_as_env
+        self.label = label
+        self.required = required
+        if not self.user_preferences_key:
+            raise ValueError("Missing user_preferences_key")
+        seperated_key = user_preferences_key.split("/")
+        if len(seperated_key) != 2 or not seperated_key[0] or not seperated_key[1]:
+            raise ValueError("Invalid user_preferences_key")
+        if self.type not in {"vault"}:
+            raise ValueError(f"Invalid secret type '{self.type}'")
+        if not self.inject_as_env:
+            raise ValueError("Missing inject_as_env")
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "type": self.type,
+            "user_preferences_key": self.user_preferences_key,
+            "inject_as_env": self.inject_as_env,
+            "label": self.label,
+            "required": self.required,
+        }
+
+    @classmethod
+    def from_dict(cls, dict: Dict[str, Any]) -> "SecretsRequirement":
+        type = dict["type"]
+        user_preferences_key = dict["user_preferences_key"]
+        inject_as_env = dict["inject_as_env"]
+        label = dict.get("label", "")
+        required = dict.get("required", False)
+        return cls(
+            type=type,
+            user_preferences_key=user_preferences_key,
+            inject_as_env=inject_as_env,
+            label=label,
+            required=required,
+        )
+
+
 def parse_requirements_from_lists(
     software_requirements: List[Union[ToolRequirement, Dict[str, Any]]],
     containers: Iterable[Dict[str, Any]],
     resource_requirements: Iterable[Dict[str, Any]],
-) -> Tuple[ToolRequirements, List[ContainerDescription], List[ResourceRequirement]]:
+    secrets: Iterable[Dict[str, Any]],
+) -> Tuple[ToolRequirements, List[ContainerDescription], List[ResourceRequirement], List[SecretsRequirement]]:
     return (
         ToolRequirements.from_list(software_requirements),
         [ContainerDescription.from_dict(c) for c in containers],
         resource_requirements_from_list(resource_requirements),
+        [SecretsRequirement.from_dict(s) for s in secrets],
     )
 
 
-def parse_requirements_from_xml(xml_root, parse_resources: bool = False):
+def parse_requirements_from_xml(xml_root, parse_resources_and_secrets: bool = False):
     """
     Parses requirements, containers and optionally resource requirements from Xml tree.
 
     >>> from galaxy.util import parse_xml_string
-    >>> def load_requirements(contents, parse_resources=False):
+    >>> def load_requirements(contents, parse_resources_and_secrets=False):
     ...     contents_document = '''<tool><requirements>%s</requirements></tool>'''
     ...     root = parse_xml_string(contents_document % contents)
-    ...     return parse_requirements_from_xml(root, parse_resources=parse_resources)
+    ...     return parse_requirements_from_xml(root, parse_resources_and_secrets=parse_resources_and_secrets)
     >>> reqs, containers = load_requirements('''<requirement>bwa</requirement>''')
     >>> reqs[0].name
     'bwa'
@@ -344,8 +396,10 @@ def parse_requirements_from_xml(xml_root, parse_resources: bool = False):
     requirements_elem = xml_root.find("requirements")
 
     requirement_elems = []
+    container_elems = []
     if requirements_elem is not None:
         requirement_elems = requirements_elem.findall("requirement")
+        container_elems = requirements_elem.findall("container")
 
     requirements = ToolRequirements()
     for requirement_elem in requirement_elems:
@@ -355,15 +409,13 @@ def parse_requirements_from_xml(xml_root, parse_resources: bool = False):
         requirement = ToolRequirement(name=name, type=type, version=version)
         requirements.append(requirement)
 
-    container_elems = []
-    if requirements_elem is not None:
-        container_elems = requirements_elem.findall("container")
-
     containers = [container_from_element(c) for c in container_elems]
-    if parse_resources:
+    if parse_resources_and_secrets:
         resource_elems = requirements_elem.findall("resource") if requirements_elem is not None else []
         resources = [resource_from_element(r) for r in resource_elems]
-        return requirements, containers, resources
+        secret_elems = requirements_elem.findall("secret") if requirements_elem is not None else []
+        secrets = [secret_from_element(s) for s in secret_elems]
+        return requirements, containers, resources, secrets
 
     return requirements, containers
 
@@ -386,3 +438,18 @@ def container_from_element(container_elem) -> ContainerDescription:
         shell=shell,
     )
     return container
+
+
+def secret_from_element(secret_elem) -> SecretsRequirement:
+    type = secret_elem.get("type")
+    user_preferences_key = secret_elem.get("user_preferences_key")
+    inject_as_env = secret_elem.get("inject_as_env")
+    label = secret_elem.get("label", "")
+    required = string_as_bool(secret_elem.get("required", "false"))
+    return SecretsRequirement(
+        type=type,
+        user_preferences_key=user_preferences_key,
+        inject_as_env=inject_as_env,
+        label=label,
+        required=required,
+    )

lib/galaxy/tool_util/linters/general.py

@@ -183,7 +183,7 @@ class RequirementNameMissing(Linter):
     @classmethod
     def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
         _, tool_node = _tool_xml_and_root(tool_source)
-        requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers()
+        requirements, containers, resource_requirements, secrets = tool_source.parse_requirements_and_containers()
         for r in requirements:
             if r.type != "package":
                 continue
@@ -195,7 +195,7 @@ class RequirementVersionMissing(Linter):
     @classmethod
     def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
         _, tool_node = _tool_xml_and_root(tool_source)
-        requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers()
+        requirements, containers, resource_requirements, secrets = tool_source.parse_requirements_and_containers()
         for r in requirements:
             if r.type != "package":
                 continue
@@ -207,7 +207,7 @@ class RequirementVersionWhitespace(Linter):
     @classmethod
     def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
         _, tool_node = _tool_xml_and_root(tool_source)
-        requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers()
+        requirements, containers, resource_requirements, secrets = tool_source.parse_requirements_and_containers()
         for r in requirements:
             if r.type != "package":
                 continue
@@ -223,7 +223,7 @@ class ResourceRequirementExpression(Linter):
     @classmethod
     def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
         _, tool_node = _tool_xml_and_root(tool_source)
-        requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers()
+        requirements, containers, resource_requirements, secrets = tool_source.parse_requirements_and_containers()
         for rr in resource_requirements:
             if rr.runtime_required:
                 lint_ctx.warn(

lib/galaxy/tool_util/parser/cwl.py

@@ -162,10 +162,12 @@ def parse_requirements_and_containers(self):
 
         software_requirements = self.tool_proxy.software_requirements()
         resource_requirements = self.tool_proxy.resource_requirements()
+        secrets = self.tool_proxy.secrets()
         return requirements.parse_requirements_from_lists(
             software_requirements=[{"name": r[0], "version": r[1], "type": "package"} for r in software_requirements],
             containers=containers,
             resource_requirements=resource_requirements,
+            secrets=secrets,
         )
 
     def parse_profile(self):

lib/galaxy/tool_util/parser/interface.py

@@ -35,6 +35,7 @@
     from galaxy.tool_util.deps.requirements import (
         ContainerDescription,
         ResourceRequirement,
+        SecretsRequirement,
         ToolRequirements,
     )
 
@@ -307,8 +308,10 @@ def parse_required_files(self) -> Optional["RequiredFiles"]:
     @abstractmethod
     def parse_requirements_and_containers(
         self,
-    ) -> Tuple["ToolRequirements", List["ContainerDescription"], List["ResourceRequirement"]]:
-        """Return triple of ToolRequirement, ContainerDescription and ResourceRequirement lists."""
+    ) -> Tuple[
+        "ToolRequirements", List["ContainerDescription"], List["ResourceRequirement"], List["SecretsRequirement"]
+    ]:
+        """Return triple of ToolRequirement, ContainerDescription, ResourceRequirement, and SecretsRequirement objects."""
 
     @abstractmethod
     def parse_input_pages(self) -> "PagesSource":

lib/galaxy/tool_util/parser/xml.py

@@ -413,7 +413,7 @@ def parse_include_exclude_list(tag_name):
         return RequiredFiles.from_dict(as_dict)
 
     def parse_requirements_and_containers(self):
-        return requirements.parse_requirements_from_xml(self.root, parse_resources=True)
+        return requirements.parse_requirements_from_xml(self.root, parse_resources_and_secrets=True)
 
     def parse_input_pages(self) -> "XmlPagesSource":
         return XmlPagesSource(self.root)

lib/galaxy/tool_util/parser/yaml.py

@@ -115,6 +115,7 @@ def parse_requirements_and_containers(self):
             software_requirements=[r for r in mixed_requirements if r.get("type") != "resource"],
             containers=self.root_dict.get("containers", []),
             resource_requirements=[r for r in mixed_requirements if r.get("type") == "resource"],
+            secrets=self.root_dict.get("secrets", []),
         )
 
     def parse_input_pages(self) -> PagesSource:

lib/galaxy/tool_util/xsd/galaxy.xsd

@@ -612,6 +612,7 @@ serve as complete descriptions of the runtime of a tool.
       <xs:element name="requirement" type="Requirement" minOccurs="0" maxOccurs="unbounded"/>
       <xs:element name="container" type="Container" minOccurs="0" maxOccurs="unbounded"/>
       <xs:element name="resource" type="Resource" minOccurs="0" maxOccurs="unbounded"/>
+      <xs:element name="secret" type="Secret" minOccurs="0" maxOccurs="unbounded"/>
     </xs:sequence>
   </xs:complexType>
   <xs:complexType name="Requirement">
@@ -725,6 +726,47 @@ Read more about configuring Galaxy to run Docker jobs
       </xs:extension>
     </xs:simpleContent>
   </xs:complexType>
+  <xs:complexType name="Secret">
+    <xs:annotation>
+      <xs:documentation xml:lang="en"><![CDATA[
+This is possible to add secrets into tools directly by adding a requirement of
+type ``secret``. This will inject the value of the requirement into the
+environment of the tool. The value of the requirement is the value of the
+environment variable to inject the secret into.
+
+```xml
+<requirements>
+    <secret type="vault" user_preferences_key="some_tool/api_key" inject_as_env="some_tool_api_key" label="API Key" required="true"/>
+</requirements>
+```
+]]></xs:documentation>
+    </xs:annotation>
+    <xs:attribute name="type" type="SecretType" use="required">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">The type of secret to inject. Valid value is ``vault`` for now.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="user_preferences_key" type="xs:string" use="required">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">The name of the user preference key to store the secret in.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="inject_as_env" type="xs:string" use="required">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">The name of the environment variable to inject the secret into.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="label" type="xs:string">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">The label of the secret.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="required" type="xs:boolean">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Whether the secret is required to run the tool.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+  </xs:complexType>
   <xs:simpleType name="HelpFormatType">
     <xs:annotation>
       <xs:documentation xml:lang="en">Document type of tool help</xs:documentation>
@@ -7848,6 +7890,14 @@ and ``bibtex`` are the only supported options.</xs:documentation>
       <xs:enumeration value="singularity"/>
     </xs:restriction>
   </xs:simpleType>
+  <xs:simpleType name="SecretType">
+    <xs:annotation>
+      <xs:documentation xml:lang="en">Type of secret for tool execution.</xs:documentation>
+    </xs:annotation>
+    <xs:restriction base="xs:string">
+      <xs:enumeration value="vault"/>
+    </xs:restriction>
+  </xs:simpleType>
   <xs:simpleType name="ToolTypeType">
     <xs:annotation>
       <xs:documentation xml:lang="en">Documentation for ToolTypeType</xs:documentation>

lib/galaxy/tools/__init__.py

@@ -1216,10 +1216,17 @@ def parse(self, tool_source: ToolSource, guid: Optional[str] = None, dynamic: bo
                     raise Exception(message)
 
         # Requirements (dependencies)
-        requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers()
+        requirements, containers, resource_requirements, secrets = tool_source.parse_requirements_and_containers()
         self.requirements = requirements
         self.containers = containers
         self.resource_requirements = resource_requirements
+        self.secrets = secrets
+        for secret in self.secrets:
+            preferences = self.app.config.user_preferences_extra["preferences"]
+            main_key, input_key = secret.user_preferences_key.split("/")
+            preferences_input = preferences.get(main_key, {}).get("inputs", [])
+            if not any(input_item.get("name") == input_key for input_item in preferences_input):
+                raise exceptions.ConfigurationError(f"User preferences key {secret.user_preferences_key} not found")
 
         required_files = tool_source.parse_required_files()
         if required_files is None:

lib/galaxy/tools/evaluation.py

@@ -9,6 +9,7 @@
 from typing import (
     Any,
     Callable,
+    cast,
     Dict,
     List,
     Optional,
@@ -28,9 +29,11 @@
 )
 from galaxy.model.none_like import NoneDataset
 from galaxy.security.object_wrapper import wrap_with_safe_string
+from galaxy.security.vault import UserVaultWrapper
 from galaxy.structured_app import (
     BasicSharedApp,
     MinimalToolApp,
+    StructuredApp,
 )
 from galaxy.tool_util.data import TabularToolDataTable
 from galaxy.tools.parameters import (
@@ -188,6 +191,17 @@ def set_compute_environment(self, compute_environment: ComputeEnvironment, get_s
         )
         self.execute_tool_hooks(inp_data=inp_data, out_data=out_data, incoming=incoming)
 
+        if self.tool.secrets:
+            app = cast(StructuredApp, self.app)
+            user_vault = UserVaultWrapper(app.vault, self._user)
+            for secret in self.tool.secrets:
+                vault_key = secret.user_preferences_key
+                secret_value = user_vault.read_secret("preferences/" + vault_key)
+                if secret_value is not None:
+                    self.environment_variables.append({"name": secret.inject_as_env, "value": secret_value})
+                else:
+                    log.warning("Failed to read secret from vault")
+
     def execute_tool_hooks(self, inp_data, out_data, incoming):
         # Certain tools require tasks to be completed prior to job execution
         # ( this used to be performed in the "exec_before_job" hook, but hooks are deprecated ).

test/functional/tools/secret_tool.xml

@@ -0,0 +1,15 @@
+<tool id="secret_tool" name="secret_tool" version="test" profile="23.0">
+    <requirements>
+        <secret type="vault" inject_as_env="secret_tool_api_key" user_preferences_key="secret_tool/api_key" label="secret API Key" required="true"/>
+    </requirements>
+    <command><![CDATA[
+echo \$secret_tool_api_key > '$output'
+    ]]></command>
+    <inputs>
+    </inputs>
+    <outputs>
+        <data name="output" format="txt"/>
+    </outputs>
+    <tests>
+    </tests>
+</tool>