Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
updating the credentials to the new format
Browse files Browse the repository at this point in the history
arash77 committed Dec 3, 2024
1 parent e07c413 commit 446e685
Showing 6 changed files with 126 additions and 60 deletions.
98 changes: 73 additions & 25 deletions lib/galaxy/tool_util/deps/requirements.py
Original file line number Diff line number Diff line change
@@ -306,70 +306,108 @@ def resource_requirements_from_list(requirements: Iterable[Dict[str, Any]]) -> L
return rr


class SecretOrVariable:
class Secret:
def __init__(
self,
name: str,
inject_as_env: str,
label: str = "",
description: str = "",
) -> None:
self.name = name
self.inject_as_env = inject_as_env
self.label = label
self.description = description
if not self.inject_as_env:
raise ValueError("Missing inject_as_env")

def to_dict(self) -> Dict[str, Any]:
return {
"name": self.name,
"inject_as_env": self.inject_as_env,
"label": self.label,
"description": self.description,
}

@classmethod
def from_element(cls, elem) -> "Secret":
return cls(
name=elem.get("name"),
inject_as_env=elem.get("inject_as_env"),
label=elem.get("label", ""),
description=elem.get("description", ""),
)

@classmethod
def from_dict(cls, dict: Dict[str, Any]) -> "Secret":
name = dict["name"]
inject_as_env = dict["inject_as_env"]
label = dict.get("label", "")
description = dict.get("description", "")
return cls(name=name, inject_as_env=inject_as_env, label=label, description=description)


class Variable:
def __init__(
self,
type: str,
name: str,
inject_as_env: str,
label: str = "",
description: str = "",
) -> None:
self.type = type
self.name = name
self.inject_as_env = inject_as_env
self.label = label
self.description = description
if self.type not in {"secret", "variable"}:
raise ValueError(f"Invalid credential type '{self.type}'")
if not self.inject_as_env:
raise ValueError("Missing inject_as_env")

def to_dict(self) -> Dict[str, Any]:
return {
"type": self.type,
"name": self.name,
"inject_as_env": self.inject_as_env,
"label": self.label,
"description": self.description,
}

@classmethod
def from_element(cls, elem) -> "SecretOrVariable":
def from_element(cls, elem) -> "Variable":
return cls(
type=elem.tag,
name=elem.get("name"),
inject_as_env=elem.get("inject_as_env"),
label=elem.get("label", ""),
description=elem.get("description", ""),
)

@classmethod
def from_dict(cls, dict: Dict[str, Any]) -> "SecretOrVariable":
type = dict["type"]
def from_dict(cls, dict: Dict[str, Any]) -> "Variable":
name = dict["name"]
inject_as_env = dict["inject_as_env"]
label = dict.get("label", "")
description = dict.get("description", "")
return cls(type=type, name=name, inject_as_env=inject_as_env, label=label, description=description)
return cls(name=name, inject_as_env=inject_as_env, label=label, description=description)


class CredentialsRequirement:
def __init__(
self,
name: str,
reference: str,
required: bool = False,
optional: bool = True,
label: str = "",
description: str = "",
secrets_and_variables: Optional[List[SecretOrVariable]] = None,
secrets: List[Secret] = [],
variables: List[Variable] = [],
multiple: bool = False,
) -> None:
self.name = name
self.reference = reference
self.required = required
self.optional = optional
self.label = label
self.description = description
self.secrets_and_variables = secrets_and_variables if secrets_and_variables is not None else []
self.secrets = secrets if secrets is not None else []
self.variables = variables if variables is not None else []
self.multiple = multiple

if not self.reference:
raise ValueError("Missing reference")
@@ -378,27 +416,33 @@ def to_dict(self) -> Dict[str, Any]:
return {
"name": self.name,
"reference": self.reference,
"required": self.required,
"optional": self.optional,
"label": self.label,
"description": self.description,
"secrets_and_variables": [s.to_dict() for s in self.secrets_and_variables],
"secrets": [s.to_dict() for s in self.secrets],
"variables": [v.to_dict() for v in self.variables],
"multiple": self.multiple,
}

@classmethod
def from_dict(cls, dict: Dict[str, Any]) -> "CredentialsRequirement":
name = dict["name"]
reference = dict["reference"]
required = dict.get("required", False)
optional = dict.get("optional", True)
label = dict.get("label", "")
description = dict.get("description", "")
secrets_and_variables = [SecretOrVariable.from_dict(s) for s in dict.get("secrets_and_variables", [])]
secrets = [Secret.from_dict(s) for s in dict.get("secrets", [])]
variables = [Variable.from_dict(v) for v in dict.get("variables", [])]
multiple = dict.get("multiple", False)
return cls(
name=name,
reference=reference,
required=required,
optional=optional,
label=label,
description=description,
secrets_and_variables=secrets_and_variables,
secrets=secrets,
variables=variables,
multiple=multiple,
)


@@ -490,15 +534,19 @@ def container_from_element(container_elem) -> ContainerDescription:
def credentials_from_element(credentials_elem) -> CredentialsRequirement:
name = credentials_elem.get("name")
reference = credentials_elem.get("reference")
required = string_as_bool(credentials_elem.get("required", "false"))
optional = string_as_bool(credentials_elem.get("optional", "true"))
label = credentials_elem.get("label", "")
description = credentials_elem.get("description", "")
secrets_and_variables = [SecretOrVariable.from_element(elem) for elem in credentials_elem.findall("*")]
multiple = string_as_bool(credentials_elem.get("multiple", "false"))
secrets = [Secret.from_element(elem) for elem in credentials_elem.findall("secret")]
variables = [Variable.from_element(elem) for elem in credentials_elem.findall("variable")]
return CredentialsRequirement(
name=name,
reference=reference,
required=required,
optional=optional,
label=label,
description=description,
secrets_and_variables=secrets_and_variables,
secrets=secrets,
variables=variables,
multiple=multiple,
)
19 changes: 12 additions & 7 deletions lib/galaxy/tool_util/xsd/galaxy.xsd
Original file line number Diff line number Diff line change
@@ -739,11 +739,11 @@ It can contain multiple ``variable`` and ``secret`` tags.
```xml
<requirements>
<credentials name="Apollo" reference="gmod.org/apollo" required="true" label="Apollo credential set" description="Please provide credentials for Apollo">
<variable name="server" inject_as_env="apollo_url" label="Your Apollo server" />
<secret name="username" inject_as_env="apollo_user" label="Your Apollo username" />
<secret name="password" inject_as_env="apollo_pass" label="Your Apollo password" />
</credentials>
<credentials name="service1" reference="service/54687" optional="false" multiple="true" label="Your credentials set" description="Optional description of the service using credentials">
<variable name="server" inject_as_env="service1_url" label="Your Service1 server" description="You can set the server..." />
<secret name="username" inject_as_env="service1_user" label="Your Service1 username" description="Your username is your email" />
<secret name="password" inject_as_env="service1_pass" label="Your Service1 password" description="This needs to be a strong password" />
</credentials>
</requirements>
```
]]></xs:documentation>
@@ -772,9 +772,14 @@ It can contain multiple ``variable`` and ``secret`` tags.
<xs:documentation xml:lang="en">The description of the credential set.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="required" type="xs:boolean">
<xs:attribute name="optional" type="xs:boolean">
<xs:annotation>
<xs:documentation xml:lang="en">Whether the credentials are required for the tool to run.</xs:documentation>
<xs:documentation xml:lang="en">Whether the credentials are optional for the tool to run.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="multiple" type="xs:boolean">
<xs:annotation>
<xs:documentation xml:lang="en">Indicates multiple sets of credentials can be provided.</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
22 changes: 22 additions & 0 deletions lib/galaxy/tools/__init__.py
Original file line number Diff line number Diff line change
@@ -2667,6 +2667,27 @@ def to_json(self, trans, kwd=None, job=None, workflow_building_mode=False, histo

state_inputs_json: ToolStateDumpedToJsonT = params_to_json(self.inputs, state_inputs, self.app)

credentials = []
for credential in self.credentials:
credential_dict = credential.to_dict()
credential_dict["variables"] = [
{
"name": variable.name,
"label": variable.label,
"description": variable.description,
}
for variable in credential.variables
]
credential_dict["secrets"] = [
{
"name": secret.name,
"label": secret.label,
"description": secret.description,
}
for secret in credential.secrets
]
credentials.append(credential_dict)

# update tool model
tool_model.update(
{
@@ -2679,6 +2700,7 @@ def to_json(self, trans, kwd=None, job=None, workflow_building_mode=False, histo
"warnings": tool_warnings,
"versions": self.tool_versions,
"requirements": [{"name": r.name, "version": r.version} for r in self.requirements],
"credentials": credentials,
"errors": state_errors,
"tool_errors": self.tool_errors,
"state_inputs": state_inputs_json,
19 changes: 6 additions & 13 deletions lib/galaxy/tools/evaluation.py
Original file line number Diff line number Diff line change
@@ -195,19 +195,12 @@ def set_compute_environment(self, compute_environment: ComputeEnvironment, get_s
# user_vault = UserVaultWrapper(app.vault, self._user)
for credentials in self.tool.credentials:
reference = credentials.reference
for secret_or_variable in credentials.secrets_and_variables:
if secret_or_variable.type == "variable":
# variable_value = self.param_dict.get(f"{reference}/{secret_or_variable.name}")
variable_value = f"A variable: {reference}/{secret_or_variable.name}"
self.environment_variables.append(
{"name": secret_or_variable.inject_as_env, "value": variable_value}
)
elif secret_or_variable.type == "secret":
# secret_value = user_vault.read_secret(f"{reference}/{secret_or_variable.name}")
secret_value = f"A secret: {reference}/{secret_or_variable.name}"
self.environment_variables.append(
{"name": secret_or_variable.inject_as_env, "value": secret_value}
)
for secret in credentials.secret:
secret_value = f"{reference}/{secret.name}"
self.environment_variables.append({"name": secret.inject_as_env, "value": secret_value})
for variable in credentials.variable:
variable_value = f"{reference}/{variable.name}"
self.environment_variables.append({"name": variable.inject_as_env, "value": variable_value})

def execute_tool_hooks(self, inp_data, out_data, incoming):
# Certain tools require tasks to be completed prior to job execution
12 changes: 6 additions & 6 deletions test/functional/tools/secret_tool.xml
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
<tool id="secret_tool" name="secret_tool" version="test" profile="23.0">
<requirements>
<secret type="vault" inject_as_env="secret_tool_api_key" user_preferences_key="secret_tool/api_key" label="secret API Key" required="true"/>
<credentials name="service1" reference="service/54687" optional="false" multiple="true" label="Your credentials set" description="Optional description of the service using credentials">
<variable name="server" inject_as_env="service1_url" label="Your Service1 server" description="You can set the server..."/>
<secret name="username" inject_as_env="service1_user" label="Your Service1 username" description="Your username is your email"/>
<secret name="password" inject_as_env="service1_pass" label="Your Service1 password" description="This needs to be a strong password"/>
</credentials>
</requirements>
<command><![CDATA[
echo \$secret_tool_api_key > '$output'
echo \$service1_url > '$output' && echo \$service1_user >> '$output' && echo \$service1_pass >> '$output'
]]></command>
<inputs>
</inputs>
<outputs>
<data name="output" format="txt"/>
</outputs>
<tests>
</tests>
</tool>
16 changes: 7 additions & 9 deletions test/unit/tool_util/test_parsing.py
Original file line number Diff line number Diff line change
@@ -50,10 +50,10 @@
<resource type="cuda_device_count_min">1</resource>
<resource type="cuda_device_count_max">2</resource>
<resource type="shm_size">67108864</resource>
<credentials name="Apollo" reference="gmod.org/apollo" required="true" label="Apollo credential set" description="Please provide credentials for Apollo">
<variable name="server" inject_as_env="apollo_url" label="Your Apollo server" />
<secret name="username" inject_as_env="apollo_user" label="Your Apollo username" />
<secret name="password" inject_as_env="apollo_pass" label="Your Apollo password" />
<credentials name="Apollo" reference="gmod.org/apollo" optional="true" multiple="false" label="Apollo credential set" description="Please provide credentials for Apollo">
<variable name="server" inject_as_env="apollo_url" label="Your Apollo server" description="URL of your Apollo server" />
<secret name="username" inject_as_env="apollo_user" label="Your Apollo username" description="Username for Apollo" />
<secret name="password" inject_as_env="apollo_pass" label="Your Apollo password" description="Password for Apollo" />
</credentials>
</requirements>
<outputs>
@@ -368,11 +368,9 @@ def test_credentials(self):
*_, credentials = self._tool_source.parse_requirements_and_containers()
assert credentials[0].name == "Apollo"
assert credentials[0].reference == "gmod.org/apollo"
assert credentials[0].required
assert len(credentials[0].secrets_and_variables) == 3
assert credentials[0].secrets_and_variables[0].type == "variable"
assert credentials[0].secrets_and_variables[1].type == "secret"
assert credentials[0].secrets_and_variables[2].type == "secret"
assert credentials[0].optional == True
assert len(credentials[0].secrets) == 2
assert len(credentials[0].variables) == 1

def test_outputs(self):
outputs, output_collections = self._tool_source.parse_outputs(object())

0 comments on commit 446e685

Please sign in to comment.