Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add polybox storage #527

Merged
merged 16 commits into from
Dec 3, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
feat: add polybox and switchDrive storage
andre-code committed Nov 14, 2024
commit 578240d15a5d2ffe7557baf66ad51d5a4984d4cb
4 changes: 4 additions & 0 deletions components/renku_data_services/data_connectors/api.spec.yaml
Original file line number Diff line number Diff line change
@@ -578,6 +578,10 @@ components:
type: string
description: The cloud provider the option is for (See 'provider' RCloneOption in the schema for potential values)
example: AWS
access_level:
type: string
description: Mode to connect to PolyBox and SwitchDrive it can be Public for shared folder or Private for own storage
example: Public
default:
oneOf:
- type: number
7 changes: 6 additions & 1 deletion components/renku_data_services/data_connectors/apispec.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: api.spec.yaml
# timestamp: 2024-10-28T20:03:14+00:00
# timestamp: 2024-11-14T10:23:28+00:00

from __future__ import annotations

@@ -44,6 +44,11 @@ class RCloneOption(BaseAPISpec):
description="The cloud provider the option is for (See 'provider' RCloneOption in the schema for potential values)",
example="AWS",
)
access_level: Optional[str] = Field(
None,
description="Mode to connect to PolyBox and SwitchDrive it can be Public for shared folder or Private for own storage",
example="Public",
)
default: Optional[Union[float, str, bool, Dict[str, Any], List]] = Field(
None, description="default value for the option"
)
4 changes: 4 additions & 0 deletions components/renku_data_services/storage/api.spec.yaml
Original file line number Diff line number Diff line change
@@ -379,6 +379,10 @@ components:
type: string
description: The cloud provider the option is for (See 'provider' RCloneOption in the schema for potential values)
example: AWS
access_level:
type: string
description: Mode to connect to PolyBox and SwitchDrive it can be Public for shared folder or Private for own storage
example: Public
default:
oneOf:
- type: number
7 changes: 6 additions & 1 deletion components/renku_data_services/storage/apispec.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: api.spec.yaml
# timestamp: 2024-10-28T17:26:56+00:00
# timestamp: 2024-11-14T07:52:23+00:00

from __future__ import annotations

@@ -49,6 +49,11 @@ class RCloneOption(BaseAPISpec):
description="The cloud provider the option is for (See 'provider' RCloneOption in the schema for potential values)",
example="AWS",
)
access_level: Optional[str] = Field(
None,
description="Mode to connect to PolyBox and SwitchDrive it can be Public for shared folder or Private for own storage",
example="Public",
)
default: Optional[Union[float, str, bool, Dict[str, Any], List]] = Field(
None, description="default value for the option"
)
120 changes: 119 additions & 1 deletion components/renku_data_services/storage/rclone.py
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@
import json
import tempfile
from collections.abc import Generator
from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING, Any, NamedTuple, Union, cast

@@ -54,7 +55,7 @@ def __init__(self) -> None:
provider_schema = RCloneProviderSchema.model_validate(provider_config)
self.providers[provider_schema.prefix] = provider_schema
except ValidationError:
logger.error("Couldn't load RClone config: %s", provider_config)
logger.error("🚀 Couldn't load RClone config: %s", provider_config)
raise

@staticmethod
@@ -156,6 +157,102 @@ def __patch_schema_remove_oauth_propeties(spec: list[dict[str, Any]]) -> None:
options.append(option)
storage["Options"] = options

@staticmethod
def __find_webdav_storage(spec: list[dict[str, Any]]) -> dict[str, Any]:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

issue: this is too specific to me, I think having _find_storage(prefix: str) would be cleaner and there might be some other places where we could use that as well,e.g. in __patch_schema_add_switch_provider

"""Find and return the WebDAV storage schema from the spec."""
webdav_storage = next((s for s in spec if s["Prefix"] == "webdav"), None)
if not webdav_storage:
raise errors.ValidationError(message="WebDAV storage not found in schema.")
return deepcopy(webdav_storage)

@staticmethod
def __add_webdav_based_storage(
spec: list[dict[str, Any]],
prefix: str,
name: str,
description: str,
url_value: str,
public_link_help: str,
) -> None:
"""Create a modified copy of WebDAV storage and add it to the schema."""
# Find WebDAV storage schema and create a modified copy
storage_copy = RCloneValidator.__find_webdav_storage(spec)
storage_copy.update({"Prefix": prefix, "Name": name, "Description": description})

# Define new custom options and append them
custom_options = [
{
"Name": "access_level",
"Help": "Choose the mode to access the data source.",
"Provider": "",
"AccessLevel": None,
"Default": "",
"Value": None,
"Examples": [
{"Value": "Private", "Help": "Use Private to connect a folder that only you use", "Provider": ""},
{
"Value": "Public",
"Help": "To connect a folder you share with others, both privately & publicly shared folders.",
"Provider": "",
},
],
"Required": True,
"Type": "string",
"ShortOpt": "",
"Hide": 0,
"IsPassword": False,
"NoPrefix": False,
"Advanced": False,
"Exclusive": False,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

issue: Exclusive should be True, this means that only the values in Examples are valid values

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, thanks for clarify that

"Sensitive": False,
"DefaultStr": "",
"ValueStr": "",
},
{
"Name": "publicLink",
"Help": public_link_help,
"Provider": "",
"AccessLevel": "Public",
"Default": "",
"Value": None,
"Examples": None,
"ShortOpt": "",
"Hide": 0,
"Required": True,
"IsPassword": False,
"NoPrefix": False,
"Advanced": False,
"Exclusive": False,
"Sensitive": False,
"DefaultStr": "",
"ValueStr": "",
"Type": "string",
},
]
storage_copy["Options"].extend(custom_options)

# Modify existing options
for option in storage_copy["Options"]:
if option["Name"] == "url":
option.update({"AccessLevel": "Private", "Value": url_value})
elif option["Name"] == "user":
option.update({"Help": "", "AccessLevel": "Private"})
elif option["Name"] == "pass":
option["Examples"] = [
{"Value": "", "Help": "", "AccessLevel": "Private", "Provider": ""},
{"Value": "", "Help": "", "AccessLevel": "Public", "Provider": ""},
]
elif option["Name"] in ["bearer_token", "bearer_token_command"]:
option["AccessLevel"] = "Private"

# Remove unwanted options
storage_copy["Options"] = [
o for o in storage_copy["Options"] if o["Name"] not in ["vendor", "nextcloud_chunk_size"]
]

# Append the customized storage configuration to the spec
spec.append(storage_copy)

def apply_patches(self, spec: list[dict[str, Any]]) -> None:
"""Apply patches to RClone schema."""
patches = [
@@ -167,6 +264,25 @@ def apply_patches(self, spec: list[dict[str, Any]]) -> None:
for patch in patches:
patch(spec)

# Apply patches for PolyBox and SwitchDrive to the schema.
self.__add_webdav_based_storage(
spec,
prefix="polybox",
name="PolyBox",
description="Polybox",
url_value="https://polybox.ethz.ch/remote.php/webdav/",
public_link_help="Shared folder link. E.g., https://polybox.ethz.ch/index.php/s/8NffJ3rFyHaVjgR",
)

self.__add_webdav_based_storage(
spec,
prefix="switchDrive",
name="SwitchDrive",
description="SwitchDrive",
url_value="https://drive.switch.ch/public.php/webdav/",
public_link_help="Shared folder link. E.g., https://drive.switch.ch/index.php/s/OPSd72zrs5JGCv6",
)

def validate(self, configuration: Union["RCloneConfig", dict[str, Any]], keep_sensitive: bool = False) -> None:
"""Validates an RClone config."""
provider = self.get_provider(configuration)
@@ -263,6 +379,7 @@ class RCloneExample(BaseModel):
value: str = Field(alias="Value")
help: str = Field(alias="Help")
provider: str = Field(alias="Provider")
access_level: str | None = Field(alias="AccessLevel", default=None)


class RCloneOption(BaseModel):
@@ -271,6 +388,7 @@ class RCloneOption(BaseModel):
name: str = Field(alias="Name")
help: str = Field(alias="Help")
provider: str = Field(alias="Provider")
access_level: str | None = Field(alias="AccessLevel", default=None)
default: str | int | bool | list[str] | RCloneTriState | None = Field(alias="Default")
value: str | int | bool | RCloneTriState | None = Field(alias="Value")
examples: list[RCloneExample] | None = Field(default=None, alias="Examples")