feat: switch to rclone csi (#3380)

* switch to rclone csi * use csi-rclone chart * add tolerations config
SwissDataScienceCenter · Jan 24, 2024 · 5deb4e7 · 5deb4e7
1 parent 7dbd074
commit 5deb4e7
Show file tree

Hide file tree

Showing 8 changed files with 161 additions and 66 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,5 +1,83 @@
 .. _changelog:
 
+0.47.0
+------
+
+This release expands Renku's cloud storage functionality in two key ways: First, mounted storages
+ are now read **and write**, so you can use mounted storage as an active workspace for your data in a RenkuLab
+ session. Second, we have expanded the cloud storage services you can integrate with RenkuLab. You can now
+ mount not only S3 buckets, but also WebDAV-based storages and Azure Blobs. 
+
+If you use SSH sessions via the CLI, you can use cloud storage there too! Configure cloud storage for your
+project on RenkuLab.io, and those storages will be mounted in your remote session. Support for cloud
+storage in local Renku sessions is still on our roadmap. 
+
+This release also adds the ability to change which resource class your session uses when you unpause the
+session, in case the original resource class is now full.
+
+User-Facing Changes
+~~~~~~~~~~~~~~~~~~~
+
+**🌟 New Features**
+
+- 🖋 **Notebooks,Data Services,CSI**: Support for read and write storage mounting in sessions using a new rclone based storage driver
+  (`#1707 <https://github.com/SwissDataScienceCenter/renku-notebooks/pull/1707>`_,
+  `#92 <https://github.com/SwissDataScienceCenter/renku-data-services/pull/92>`_,
+  `#1 <https://github.com/SwissDataScienceCenter/csi-rclone/pull/1>`_).
+- 🔌 **UI**: add support for more storage services
+  (`#2908 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2908>`_,
+  `#2915 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2915>`_).
+
+**✨ Improvements**
+
+- 🖌️ **UI**: Improve the look and feel of the home page
+  (`#2968 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2968>`_,
+  `#2937 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2937>`_,
+  `#2927 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2927>`_).
+- 🔐 **UI**: Use password fields for credentials
+  (`#2920 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2920>`_).
+- 🔧 **UI**: Allow users to modify non running sessions
+  (`#2942 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2942>`_).
+- 🛑 **UI**: Improve feedback when starting sessions on outdated projects
+  (`#2985 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2985>`_).
+- 🖌️ **UI**: Update the Renku logo and Renku browser icons
+  (`#2848 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2848>`_).
+
+**🐞 Bug Fixes**
+
+- **UI**: Resize the feedback badge on the session settings page
+  (`#2953 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2953>`_).
+- **UI**: Fix the environment dropdown on the Start session page
+  (`#2949 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2949>`_).
+- **UI**: Improve string validation when trying to upload a dataset file by URL
+  (`#2834 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2834>`_).
+
+Internal Changes
+~~~~~~~~~~~~~~~~
+
+**Improvements**
+
+- **UI**: RenkuLab admins can now add tolerations and node affinities to resource classes
+  (`#2916 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2916>`_).
+- **UI**: RenkuLab admins can add multiple users to a resource pool at once via a list of emails
+  `#2910 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2910>`_).
+- **UI**: Use the renku-core API for session options
+  `#2947 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2947>`_).
+- **UI**: Specify a branch every time a renku-core API is invoked
+  `#2977 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2977>`_).
+
+Individual Components
+~~~~~~~~~~~~~~~~~~~~~
+
+- `amalthea 0.11.0 <https://github.com/SwissDataScienceCenter/amalthea/releases/tag/0.11.0>`_
+- `csi-rclone 0.1.5 <https://github.com/SwissDataScienceCenter/csi-rclone/releases/tag/v0.1.5>`_
+- `renku-data-services 0.4.0 <https://github.com/SwissDataScienceCenter/renku-data-services/releases/tag/v0.4.0>`_
+- `renku-notebooks 1.21.0 <https://github.com/SwissDataScienceCenter/renku-notebooks/releases/tag/1.21.0>`_
+- `renku-ui 3.18.0 <https://github.com/SwissDataScienceCenter/renku-ui/releases/tag/3.18.0>`_
+- `renku-ui 3.18.1 <https://github.com/SwissDataScienceCenter/renku-ui/releases/tag/3.18.1>`_
+- `renku-ui 3.19.0 <https://github.com/SwissDataScienceCenter/renku-ui/releases/tag/3.19.0>`_
+
+
 0.46.0
 ------
 

diff --git a/cypress-tests/cypress/e2e/useSession.cy.ts b/cypress-tests/cypress/e2e/useSession.cy.ts
@@ -241,37 +241,48 @@ describe("Basic public project functionality", () => {
         return;
       }
 
-      cy.getDataCy("settings-container")
+      cy.getDataCy("cloud-storage-section")
         .find("button")
         .contains("Add Cloud Storage")
         .should("be.visible")
         .click();
-
-      cy.get(".modal").contains("Add Cloud Storage").should("be.visible");
-
-      cy.get("label").contains("Name").click();
-      cy.get(":focused").type("data_s3");
-
-      cy.get(".modal")
-        .contains("For AWS S3 buckets, supported URLs are of the form")
+      cy.getDataCy("cloud-storage-edit-header")
+        .contains("Add Cloud Storage")
         .should("be.visible");
-      cy.get("label").contains("Endpoint URL").click();
-      cy.get(":focused").type("s3://giab");
 
-      cy.get("label")
-        .contains("Requires credentials")
-        .siblings("input")
-        .click()
-        .should("not.be.checked");
-
-      // NOTE: Temporarily removed this from the tests until we enable read-write
-      // cy.get("label")
-      //   .contains("Read-only")
-      //   .siblings("input")
-      //   .should("be.checked");
+      cy.getDataCy("cloud-storage-edit-schema")
+        .contains("s3")
+        .should("be.visible")
+        .click();
+      cy.getDataCy("cloud-storage-edit-providers")
+        .contains("AWS")
+        .should("be.visible")
+        .click();
+      cy.getDataCy("cloud-storage-edit-next-button").should("be.visible").click();
+
+      cy.getDataCy("cloud-storage-edit-options").should("be.visible");
+      cy.get("#sourcePath").should("have.value", "").type("giab");
+      cy.get("#endpoint")
+        .should("have.value", "")
+        .type("http://s3.amazonaws.com");
+      cy.getDataCy("cloud-storage-edit-next-button").should("be.visible").click();
+
+      cy.getDataCy("cloud-storage-edit-mount").should("be.visible");
+      cy.get("#name").should("have.value", "").type("data_s3");
+      cy.get("#mountPoint")
+        .should("have.value", "external_storage/data_s3")
+        .type("{selectAll}data_s3");
+      cy.get("#readOnly").should("not.be.checked").check();
+
+      cy.getDataCy("cloud-storage-edit-update-button")
+        .should("be.visible")
+        .contains("Add")
+        .click();
 
-      cy.get("button[type='submit']")
-        .contains("Add Storage")
+      cy.getDataCy("cloud-storage-edit-body").contains(
+        "storage data_s3 has been succesfully added"
+      );
+      cy.getDataCy("cloud-storage-edit-close-button")
         .should("be.visible")
         .click();
     });
@@ -289,6 +300,8 @@ describe("Basic public project functionality", () => {
     cy.get(".renku-container .badge.bg-success", { timeout: TIMEOUTS.vlong })
       .contains("available")
       .should("exist");
+    cy.getDataCy("cloud-storage-item").contains("data_s3").should("exist");
+    cy.get("#cloud-storage-data_s3-active").should("be.checked");
     cy.get(".renku-container button.btn-secondary", { timeout: TIMEOUTS.long })
       .contains("Start Session")
       .should("exist")

diff --git a/docs/renku-python b/docs/renku-python
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
@@ -4,6 +4,7 @@ adhoc
 admin
 Amalthea
 analytics
+api
 app
 aqs
 args
@@ -249,6 +250,7 @@ statuspage
 stderr
 stdin
 stdout
+storages
 subclass
 Subclass
 subclasses
@@ -301,6 +303,7 @@ unmapped
 unmerged
 Unmount
 unmount
+unpause
 unpushed
 unschedulable
 untracked

diff --git a/helm-chart/renku/requirements.yaml b/helm-chart/renku/requirements.yaml
@@ -23,8 +23,13 @@ dependencies:
   alias: jena
 - name: amalthea
   repository: "https://swissdatasciencecenter.github.io/helm-charts/"
-  version: "0.10.0"
+  version: "0.11.0"
 - name: dlf-chart
   repository: "https://swissdatasciencecenter.github.io/datashim/"
   version: "0.3.9-renku-2"
   condition: notebooks.cloudstorage.s3.installDatashim
+- name: csi-rclone
+  repository: "https://swissdatasciencecenter.github.io/helm-charts/"
+  version: "0.1.5"
+  condition: global.csi-rclone.install
+
diff --git a/helm-chart/renku/templates/notebooks/statefulset.yaml b/helm-chart/renku/templates/notebooks/statefulset.yaml
@@ -158,14 +158,10 @@ spec:
             {{- include "certificates.env.python" . | nindent 12 }}
             - name: NB_SESSIONS__ENFORCE_CPU_LIMITS
               value: {{ .Values.notebooks.enforceCPULimits | quote }}
-            - name: NB_CLOUD_STORAGE__S3__ENABLED
-              value: {{ .Values.notebooks.cloudstorage.s3.enabled | quote }}
-            - name: NB_CLOUD_STORAGE__S3__READ_ONLY
-              value: {{ .Values.notebooks.cloudstorage.s3.readOnly | quote }}
-            - name: NB_CLOUD_STORAGE__AZURE_BLOB__ENABLED
-              value: {{ .Values.notebooks.cloudstorage.azureBlob.enabled | quote }}
-            - name: NB_CLOUD_STORAGE__AZURE_BLOB__READ_ONLY
-              value: {{ .Values.notebooks.cloudstorage.azureBlob.readOnly | quote }}
+            - name: NB_CLOUD_STORAGE__ENABLED
+              value: {{ .Values.notebooks.cloudstorage.enabled | quote }}
+            - name: NB_CLOUD_STORAGE__STORAGE_CLASS
+              value: {{ .Values.notebooks.cloudstorage.storageClass | default "csi-rclone" | quote }}
             - name: NB_SESSIONS__TERMINATION_WARNING_DURATION_SECONDS
               value: {{ .Values.notebooks.sessionAutosave.terminationWarningDurationSeconds | quote }}
             - name: NB_VERSION

diff --git a/helm-chart/renku/templates/notebooks/test.yaml b/helm-chart/renku/templates/notebooks/test.yaml
@@ -85,14 +85,10 @@ spec:
         {{- end }}
         - name: NB_SESSIONS__ENFORCE_CPU_LIMITS
           value: {{ $.Values.notebooks.enforceCPULimits | quote }}
-        - name: NB_CLOUD_STORAGE__S3__ENABLED
-          value: {{ $.Values.notebooks.cloudstorage.s3.enabled | quote }}
-        - name: NB_CLOUD_STORAGE__S3__READ_ONLY
-          value: {{ $.Values.notebooks.cloudstorage.s3.readOnly | quote }}
-        - name: NB_CLOUD_STORAGE__AZURE_BLOB__ENABLED
-          value: {{ $.Values.notebooks.cloudstorage.azureBlob.enabled | quote }}
-        - name: NB_CLOUD_STORAGE__AZURE_BLOB__READ_ONLY
-          value: {{ $.Values.notebooks.cloudstorage.azureBlob.readOnly | quote }}
+        - name: NB_CLOUD_STORAGE__ENABLED
+          value: {{ $.Values.notebooks.cloudstorage.enabled | quote }}
+        - name: NB_CLOUD_STORAGE__STORAGE_CLASS
+          value: {{ $.Values.notebooks.cloudstorage.storageClass | default "csi-rclone" | quote }}
         - name: NB_SESSIONS__TERMINATION_WARNING_DURATION_SECONDS
           value: {{ $.Values.notebooks.sessionAutosave.terminationWarningDurationSeconds | quote }}
         - name: NB_VERSION

diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml
@@ -186,6 +186,10 @@ global:
       ## The contents of the 'password' key is used.
       ## The secret is not re-generated or modified in any way if it already exists.
       passwordSecretName: renku-db-common-password
+  csi-rclone:
+    # Set to true to install csi-rclone alongside renku
+    # Make sure to configure the csi-rclone section below correctly in that case.
+    install: false
 ## Ingress configuration
 ## See: https://kubernetes.io/docs/concepts/services-networking/ingress/
 ingress:
@@ -609,7 +613,7 @@ ui:
     replicaCount: 1
     image:
       repository: renku/renku-ui
-      tag: "3.17.3"
+      tag: "3.19.0"
       pullPolicy: IfNotPresent
       ## Optionally specify an array of imagePullSecrets.
       ## Secrets must be manually created in the namespace.
@@ -768,7 +772,7 @@ ui:
     keepCookies: []
     image:
       repository: renku/renku-ui-server
-      tag: "3.17.3"
+      tag: "3.19.0"
       pullPolicy: IfNotPresent
     imagePullSecrets: []
     nameOverride: ""
@@ -837,10 +841,6 @@ amalthea:
   scope:
     clusterWide: false
   deployCrd: true
-  ## If enabling S3 mounts uncomment the section below
-  # extraChildResources:
-  #   - name: datasets
-  #     group: com.ie.ibm.hpsys
 ## Used only if S3 Mounts are enabled
 dlf-chart:
   csi-sidecars-rbac:
@@ -854,20 +854,24 @@ dlf-chart:
     enabled: false
   dataset-operator-chart:
     enabled: true
+csi-rclone: {}
+  # This section is only relevant if you are installing csi-rclone as part of Renku
+  ## Name of the csi storage class to use for RClone/Cloudstorage. Should be unique per cluster.
+  # storageClassName: csi-rclone 
+  # csiNodepluginRclone:
+  #   nodeSelector: {}
+  # Set tolerations if you have taints on your user session nodes. The csi has to run on every node 
+  # where it is used.
+  #   tolerations: []
+  #   affinity: {}
 notebooks:
   cloudstorage:
-    ## If enabled this will allow mounting of buckets into user sessions
-    ## from any S3-compatible storage provider.
+    ## If enabled this will allow mounting of rclone cloudstorage
+    enabled: false
+    ## should match csi-rclone.storageClassName if you use a custom name
+    # storageClass: csi-rclone
     s3:
-      enabled: false
       installDatashim: false
-      readOnly: true
-    ## Azure blob storage mounting support is experimental. The CSI
-    ## storage driver for mounting azure blob storage has to be already installed
-    ## in the cluster if enabled.
-    azureBlob:
-      enabled: false
-      readOnly: true
   # configuration for user session persistent volumes
   userSessionPersistentVolumes:
     enabled: true
@@ -912,7 +916,7 @@ notebooks:
     targetCPUUtilizationPercentage: 50
   image:
     repository: renku/renku-notebooks
-    tag: "1.20.3"
+    tag: "1.21.0"
     pullPolicy: IfNotPresent
     ## Optionally specify an array of imagePullSecrets.
     ## Secrets must be manually created in the namespace.
@@ -1030,15 +1034,15 @@ notebooks:
   gitRpcServer:
     image:
       name: renku/git-rpc-server
-      tag: "1.20.2"
+      tag: "1.21.0"
   gitHttpsProxy:
     image:
       name: renku/git-https-proxy
-      tag: "1.20.2"
+      tag: "1.21.0"
   gitClone:
     image:
       name: renku/git-clone
-      tag: "1.20.2"
+      tag: "1.21.0"
   service:
     type: ClusterIP
     port: 80
@@ -1091,12 +1095,12 @@ notebooks:
     sessionTypes: ["registered"]
     image:
       repository: renku/renku-notebooks-tests
-      tag: "1.20.2"
+      tag: "1.21.0"
       pullPolicy: IfNotPresent
   k8sWatcher:
     image:
       repository: renku/k8s-watcher
-      tag: "1.20.2"
+      tag: "1.21.0"
       pullPolicy: IfNotPresent
     resources: {}
     replicaCount: 1
@@ -1109,7 +1113,7 @@ notebooks:
     enabled: false
     image:
       repository: renku/ssh-jump-host
-      tag: "1.20.2"
+      tag: "1.21.0"
       pullPolicy: IfNotPresent
     resources: {}
     replicaCount: 1
@@ -1476,7 +1480,7 @@ initDb:
 dataService:
   image:
     repository: renku/renku-data-service
-    tag: "0.3.0"
+    tag: "0.4.0"
     pullPolicy: IfNotPresent
   service:
     type: ClusterIP