feat: add polybox and switchDrive storage

SwissDataScienceCenter · Nov 14, 2024 · 83b1acb · 83b1acb
1 parent 1609c03
commit 83b1acb
Show file tree

Hide file tree

Showing 5 changed files with 240 additions and 8 deletions.
diff --git a/components/renku_data_services/data_connectors/api.spec.yaml b/components/renku_data_services/data_connectors/api.spec.yaml
@@ -578,6 +578,10 @@ components:
           type: string
           description: The cloud provider the option is for (See 'provider' RCloneOption in the schema for potential values)
           example: AWS
+        access_level:
+          type: string
+          description: Mode to connect to PolyBox and SwitchDrive it can be Public for shared folder or Private for own storage
+          example: Public
         default:
           oneOf:
             - type: number

diff --git a/components/renku_data_services/data_connectors/apispec.py b/components/renku_data_services/data_connectors/apispec.py
@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api.spec.yaml
-#   timestamp: 2024-10-28T20:03:14+00:00
+#   timestamp: 2024-11-14T08:02:30+00:00
 
 from __future__ import annotations
 
@@ -44,6 +44,11 @@ class RCloneOption(BaseAPISpec):
         description="The cloud provider the option is for (See 'provider' RCloneOption in the schema for potential values)",
         example="AWS",
     )
+    access_level: Optional[str] = Field(
+        None,
+        description="Mode to connect to PolyBox and SwitchDrive it can be Public for shared folder or Private for own storage",
+        example="Public",
+    )
     default: Optional[Union[float, str, bool, Dict[str, Any], List]] = Field(
         None, description="default value for the option"
     )
@@ -301,10 +306,6 @@ class DataConnectorsGetQuery(PaginationRequest):
     namespace: str = Field("", description="A namespace, used as a filter.")
 
 
-class DataConnectorsGetParametersQuery(BaseAPISpec):
-    params: Optional[DataConnectorsGetQuery] = None
-
-
 class DataConnector(BaseAPISpec):
     model_config = ConfigDict(
         extra="forbid",
@@ -394,7 +395,7 @@ class DataConnectorPost(BaseAPISpec):
         pattern="^(?!.*\\.git$|.*\\.atom$|.*[\\-._][\\-._].*)[a-z0-9][a-z0-9\\-_.]*$",
     )
     storage: Union[CloudStorageCorePost, CloudStorageUrlV2]
-    visibility: Visibility = Visibility.private
+    visibility: Visibility = "private"
     description: Optional[str] = Field(
         None, description="A description for the resource", max_length=500
     )

diff --git a/components/renku_data_services/storage/api.spec.yaml b/components/renku_data_services/storage/api.spec.yaml
@@ -379,6 +379,10 @@ components:
           type: string
           description: The cloud provider the option is for (See 'provider' RCloneOption in the schema for potential values)
           example: AWS
+        access_level:
+          type: string
+          description: Mode to connect to PolyBox and SwitchDrive it can be Public for shared folder or Private for own storage
+          example: Public
         default:
           oneOf:
             - type: number

diff --git a/components/renku_data_services/storage/apispec.py b/components/renku_data_services/storage/apispec.py
@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api.spec.yaml
-#   timestamp: 2024-10-28T17:26:56+00:00
+#   timestamp: 2024-11-14T07:52:23+00:00
 
 from __future__ import annotations
 
@@ -49,6 +49,11 @@ class RCloneOption(BaseAPISpec):
         description="The cloud provider the option is for (See 'provider' RCloneOption in the schema for potential values)",
         example="AWS",
     )
+    access_level: Optional[str] = Field(
+        None,
+        description="Mode to connect to PolyBox and SwitchDrive it can be Public for shared folder or Private for own storage",
+        example="Public",
+    )
     default: Optional[Union[float, str, bool, Dict[str, Any], List]] = Field(
         None, description="default value for the option"
     )

diff --git a/components/renku_data_services/storage/rclone.py b/components/renku_data_services/storage/rclone.py
@@ -54,7 +54,7 @@ def __init__(self) -> None:
                 provider_schema = RCloneProviderSchema.model_validate(provider_config)
                 self.providers[provider_schema.prefix] = provider_schema
             except ValidationError:
-                logger.error("Couldn't load RClone config: %s", provider_config)
+                logger.error("🚀 Couldn't load RClone config: %s", provider_config)
                 raise
 
     @staticmethod
@@ -156,6 +156,222 @@ def __patch_schema_remove_oauth_propeties(spec: list[dict[str, Any]]) -> None:
                         options.append(option)
                 storage["Options"] = options
 
+    @staticmethod
+    def __patch_schema_add_polybox_storage(spec: list[dict[str, Any]]) -> None:
+        """Add PolyBox storage configuration as a copy of WebDAV with modified options."""
+        # Find the WebDAV storage schema
+        webdav_storage = next((s for s in spec if s["Prefix"] == "webdav"), None)
+
+        if not webdav_storage:
+            raise errors.ValidationError(message="WebDAV storage not found in schema.")
+
+        # Create a copy of WebDAV storage and modify it to be PolyBox
+        polybox_storage = webdav_storage.copy()
+        polybox_storage["Prefix"] = "polybox"
+        polybox_storage["Name"] = "PolyBox"
+        polybox_storage["Description"] = "Polybox"
+
+        # Add new options: access_level and publicLink
+        polybox_storage["Options"].append(
+            {
+                "Name": "access_level",
+                "Help": "Choose the mode to access the data source.",
+                "Provider": "",
+                "AccessLevel": None,
+                "Default": "",
+                "Value": None,
+                "Examples": [
+                    {"Value": "Private", "Help": "Use Private to connect a folder that only you use", "Provider": ""},
+                    {
+                        "Value": "Public",
+                        "Help": "To connect a folder you share with others, both privately & publicly shared folders.",
+                        "Provider": "",
+                    },
+                ],
+                "ShortOpt": "",
+                "Hide": 0,
+                "Required": True,
+                "IsPassword": False,
+                "NoPrefix": False,
+                "Advanced": False,
+                "Exclusive": False,
+                "Sensitive": False,
+                "DefaultStr": "",
+                "ValueStr": "",
+                "Type": "string",
+            }
+        )
+        polybox_storage["Options"].append(
+            {
+                "Name": "publicLink",
+                "Help": "shared folder link.\n\nE.g. https://polybox.ethz.ch/index.php/s/8NffJ3rFyHaVjgR",
+                "Provider": "",
+                "AccessLevel": "Public",
+                "Default": "",
+                "Value": None,
+                "Examples": None,
+                "ShortOpt": "",
+                "Hide": 0,
+                "Required": True,
+                "IsPassword": False,
+                "NoPrefix": False,
+                "Advanced": False,
+                "Exclusive": False,
+                "Sensitive": False,
+                "DefaultStr": "",
+                "ValueStr": "",
+                "Type": "string",
+            }
+        )
+
+        # Modify existing options for PolyBox
+        # Modify URL option
+        url_option = next(o for o in polybox_storage["Options"] if o["Name"] == "url")
+        url_option["AccessLevel"] = "Private"
+        url_option["Value"] = "https://polybox.ethz.ch/remote.php/webdav/"
+
+        # Modify user option
+        user_option = next(o for o in polybox_storage["Options"] if o["Name"] == "user")
+        user_option["Help"] = ""
+        user_option["AccessLevel"] = "Private"
+
+        # Modify pass option
+        pass_option = next(o for o in polybox_storage["Options"] if o["Name"] == "pass")
+        pass_option["Examples"] = [
+            {
+                "Value": "",
+                "Help": "",
+                "AccessLevel": "Private",
+                "Provider": "",
+            },
+            {
+                "Value": "",
+                "Help": "",
+                "AccessLevel": "Public",
+                "Provider": "",
+            },
+        ]
+
+        # Modify bearer_token and bearer_token_command options
+        for option_name in ["bearer_token", "bearer_token_command"]:
+            option = next(o for o in polybox_storage["Options"] if o["Name"] == option_name)
+            option["AccessLevel"] = "Private"
+
+        # Remove unwanted options
+        for option_name in ["vendor", "nextcloud_chunk_size"]:
+            polybox_storage["Options"] = [o for o in polybox_storage["Options"] if o["Name"] != option_name]
+
+        # Append PolyBox storage configuration to the schema
+        spec.append(polybox_storage)
+
+    @staticmethod
+    def __patch_schema_add_switchDrive_storage(spec: list[dict[str, Any]]) -> None:
+        """Add SwitchDrive storage configuration as a copy of WebDAV with modified options."""
+        # Find the WebDAV storage schema
+        webdav_storage = next((s for s in spec if s["Prefix"] == "webdav"), None)
+
+        if not webdav_storage:
+            raise errors.ValidationError(message="WebDAV storage not found in schema.")
+
+        # Create a copy of WebDAV storage and modify it to be SwitchDrive
+        switchDrive_storage = webdav_storage.copy()
+        switchDrive_storage["Prefix"] = "switchDrive"
+        switchDrive_storage["Name"] = "SwitchDrive"
+        switchDrive_storage["Description"] = "SwitchDrive"
+
+        # Add new options: access_level and publicLink
+        switchDrive_storage["Options"].append(
+            {
+                "Name": "access_level",
+                "Help": "Choose the mode to access the data source.",
+                "Provider": "",
+                "AccessLevel": None,
+                "Default": "",
+                "Value": None,
+                "Examples": [
+                    {"Value": "Private", "Help": "Use Private to connect a folder that only you use", "Provider": ""},
+                    {
+                        "Value": "Public",
+                        "Help": "To connect a folder you share with others, both privately & publicly shared folders.",
+                        "Provider": "",
+                    },
+                ],
+                "ShortOpt": "",
+                "Hide": 0,
+                "Required": True,
+                "IsPassword": False,
+                "NoPrefix": False,
+                "Advanced": False,
+                "Exclusive": False,
+                "Sensitive": False,
+                "DefaultStr": "",
+                "ValueStr": "",
+                "Type": "string",
+            }
+        )
+        switchDrive_storage["Options"].append(
+            {
+                "Name": "publicLink",
+                "Help": "shared folder link.\n\nE.g. https://drive.switch.ch/index.php/s/OPSd72zrs5JGCv6",
+                "Provider": "",
+                "AccessLevel": "Public",
+                "Default": "",
+                "Value": None,
+                "Examples": None,
+                "ShortOpt": "",
+                "Hide": 0,
+                "Required": True,
+                "IsPassword": False,
+                "NoPrefix": False,
+                "Advanced": False,
+                "Exclusive": False,
+                "Sensitive": False,
+                "DefaultStr": "",
+                "ValueStr": "",
+                "Type": "string",
+            }
+        )
+
+        # Modify existing options for PolyBox
+        # Modify URL option
+        url_option = next(o for o in switchDrive_storage["Options"] if o["Name"] == "url")
+        url_option["AccessLevel"] = "Private"
+        url_option["Value"] = "https://drive.switch.ch/public.php/webdav/"
+
+        # Modify user option
+        user_option = next(o for o in switchDrive_storage["Options"] if o["Name"] == "user")
+        user_option["Help"] = ""
+        user_option["AccessLevel"] = "Private"
+
+        # Modify pass option
+        pass_option = next(o for o in switchDrive_storage["Options"] if o["Name"] == "pass")
+        pass_option["Examples"] = [
+            {
+                "Value": "",
+                "Help": "",
+                "AccessLevel": "Private",
+                "Provider": "",
+            },
+            {
+                "Value": "",
+                "Help": "",
+                "AccessLevel": "Public",
+                "Provider": "",
+            },
+        ]
+
+        # Modify bearer_token and bearer_token_command options
+        for option_name in ["bearer_token", "bearer_token_command"]:
+            option = next(o for o in switchDrive_storage["Options"] if o["Name"] == option_name)
+            option["AccessLevel"] = "Private"
+
+        # Remove unwanted options
+        for option_name in ["vendor", "nextcloud_chunk_size"]:
+            switchDrive_storage["Options"] = [o for o in switchDrive_storage["Options"] if o["Name"] != option_name]
+
+        # Append SwitchDrive storage configuration to the schema
+        spec.append(switchDrive_storage)
+
     def apply_patches(self, spec: list[dict[str, Any]]) -> None:
         """Apply patches to RClone schema."""
         patches = [
@@ -263,6 +479,7 @@ class RCloneExample(BaseModel):
     value: str = Field(alias="Value")
     help: str = Field(alias="Help")
     provider: str = Field(alias="Provider")
+    access_level: str | None = Field(alias="AccessLevel", default=None)
 
 
 class RCloneOption(BaseModel):
@@ -271,6 +488,7 @@ class RCloneOption(BaseModel):
     name: str = Field(alias="Name")
     help: str = Field(alias="Help")
     provider: str = Field(alias="Provider")
+    access_level: str | None = Field(alias="AccessLevel", default=None)
     default: str | int | bool | list[str] | RCloneTriState | None = Field(alias="Default")
     value: str | int | bool | RCloneTriState | None = Field(alias="Value")
     examples: list[RCloneExample] | None = Field(default=None, alias="Examples")