diff --git a/.github/workflows/check-acceptance-test-code.yml b/.github/workflows/check-acceptance-test-code.yml index f272a5f77a..f50824df7f 100644 --- a/.github/workflows/check-acceptance-test-code.yml +++ b/.github/workflows/check-acceptance-test-code.yml @@ -8,7 +8,7 @@ jobs: name: Scala dependencies and code check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 - name: run test compile run: | cd acceptance-tests diff --git a/.github/workflows/check-acceptance-test-fmt.yml b/.github/workflows/check-acceptance-test-fmt.yml index be1dbdeadf..b69bded5ce 100644 --- a/.github/workflows/check-acceptance-test-fmt.yml +++ b/.github/workflows/check-acceptance-test-fmt.yml @@ -8,7 +8,7 @@ jobs: name: Scala formatting check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 - name: run scalafmt run: | cd acceptance-tests diff --git a/.github/workflows/create-release-branch.yml b/.github/workflows/create-release-branch.yml index 6f0e127148..3817e80d39 100644 --- a/.github/workflows/create-release-branch.yml +++ b/.github/workflows/create-release-branch.yml @@ -16,7 +16,7 @@ jobs: create-release-pr: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3.6.0 + - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 token: "${{ secrets.RENKUBOT_GITHUB_TOKEN }}" @@ -42,7 +42,7 @@ jobs: git commit -m "chore: create release ${{ github.event.inputs.version }}" git push - name: Create Pull Request - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: token: ${{ secrets.RENKUBOT_GITHUB_TOKEN }} script: | diff --git a/.github/workflows/cron-jobs.yaml b/.github/workflows/cron-jobs.yaml index 7a7d01e6ae..a5a233c429 100644 --- a/.github/workflows/cron-jobs.yaml +++ b/.github/workflows/cron-jobs.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: renku teardown - uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.9.1 + uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.10.0 env: GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }} RENKUBOT_KUBECONFIG: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} diff --git a/.github/workflows/generate-values-script.yaml b/.github/workflows/generate-values-script.yaml index dbc49a6160..3477797c07 100644 --- a/.github/workflows/generate-values-script.yaml +++ b/.github/workflows/generate-values-script.yaml @@ -15,7 +15,7 @@ jobs: - os: macos-11 - os: ubuntu-20.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - uses: actions/setup-python@v5 @@ -49,7 +49,7 @@ jobs: runs-on: ubuntu-20.04 needs: [test-script] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - uses: Azure/docker-login@v1 diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 8be7afa896..873d5b85a6 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -9,13 +9,13 @@ jobs: publish-chart: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - name: Set version id: vars run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.9.1 + - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.10.0 env: CHART_DIR: helm-chart/ CHART_NAME: renku @@ -45,11 +45,11 @@ jobs: needs: - "publish-chart" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - name: Rollout renku version - uses: SwissDataScienceCenter/renku-actions/rollout-renku-version@v1.9.1 + uses: SwissDataScienceCenter/renku-actions/rollout-renku-version@v1.10.0 env: CHART_VERSION: ${{ needs.publish-chart.outputs.chart-version }} GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }} diff --git a/.github/workflows/publish-master-merges.yaml b/.github/workflows/publish-master-merges.yaml index c424e89dde..7ebd9810b0 100644 --- a/.github/workflows/publish-master-merges.yaml +++ b/.github/workflows/publish-master-merges.yaml @@ -14,7 +14,7 @@ jobs: publish-chart: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - uses: azure/setup-helm@v3 @@ -35,7 +35,7 @@ jobs: - id: set-version run: | echo "publish_version=${{ steps.bump-semver.outputs.new_version }}.$(echo ${{ github.sha }} | cut -c 1-7)" >> $GITHUB_ENV - - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.9.1 + - uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.10.0 env: CHART_DIR: helm-chart/ CHART_TAG: "--tag ${{env.publish_version}}" diff --git a/.github/workflows/pull-request-test.yml b/.github/workflows/pull-request-test.yml index b1b923a4ec..9ef9a87571 100644 --- a/.github/workflows/pull-request-test.yml +++ b/.github/workflows/pull-request-test.yml @@ -18,7 +18,7 @@ jobs: if: github.event.action != 'closed' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 - uses: actions/setup-java@v4 with: distribution: "temurin" @@ -60,9 +60,9 @@ jobs: test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}} extra-values: ${{ steps.deploy-comment.outputs.extra-values}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 - id: deploy-comment - uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.9.1 + uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.10.0 with: string: /deploy pr_ref: ${{ github.event.number }} @@ -75,10 +75,10 @@ jobs: name: ci-renku-${{ github.event.number }} url: https://ci-renku-${{ github.event.number }}.dev.renku.ch steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 - name: renku build and deploy if: needs.check-deploy.outputs.pr-contains-string == 'true' - uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.9.1 + uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.10.0 env: DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }} @@ -100,7 +100,7 @@ jobs: extra_values: "${{ needs.check-deploy.outputs.extra-values }}" - name: Check existing renkubot comment if: needs.check-deploy.outputs.pr-contains-string == 'true' - uses: peter-evans/find-comment@v2 + uses: peter-evans/find-comment@v3 id: findcomment with: issue-number: ${{ github.event.pull_request.number }} @@ -108,7 +108,7 @@ jobs: - name: Create comment pre deploy if: ${{ steps.findcomment.outputs.comment-id == 0 && needs.check-deploy.outputs.pr-contains-string == 'true' }} - uses: peter-evans/create-or-update-comment@v3 + uses: peter-evans/create-or-update-comment@v4 with: token: ${{ secrets.RENKUBOT_GITHUB_TOKEN }} issue-number: ${{ github.event.pull_request.number }} @@ -120,7 +120,7 @@ jobs: needs: [check-deploy, deploy-pr] runs-on: ubuntu-22.04 steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.9.1 + - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.10.0 with: kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} renku-release: ci-renku-${{ github.event.number }} @@ -149,7 +149,7 @@ jobs: ] steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.9.1 + - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.10.0 if: github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true' with: e2e-target: ${{ matrix.tests }} @@ -162,7 +162,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: renku teardown - uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.9.1 + uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.10.0 env: HELM_RELEASE_REGEX: "^ci-renku-${{ github.event.number }}$" GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }} diff --git a/.github/workflows/renku-dev-test.yaml b/.github/workflows/renku-dev-test.yaml index 60b6b83e1c..9f922efd99 100644 --- a/.github/workflows/renku-dev-test.yaml +++ b/.github/workflows/renku-dev-test.yaml @@ -8,7 +8,7 @@ jobs: github.event.client_payload.message == 'Helm test succeeded' }} runs-on: ubuntu-20.04 steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.9.1 + - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.10.0 with: kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} renku-release: renku @@ -22,7 +22,7 @@ jobs: github.event.client_payload.message == 'Helm test succeeded' }} runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4.1.1 - uses: cypress-io/github-action@v5 id: cypress env: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index be377f495b..ed2249652f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,9 +1,28 @@ .. _changelog: -0.48.x +0.48.0 ------ +This release contains bug fixes to renku core service. +User-Facing Changes +~~~~~~~~~~~~~~~~~~~ + +**🐞 Bug Fixes** + +- **Core Service**: Fix issue with having to run project migration twice to migrate the Dockerfile/project template. + (`#3690 `__) + +Individual Components +~~~~~~~~~~~~~~~~~~~~~ + +- `renku-python 2.9.2 `_ + +0.47.1 +------ + +This release only includes changes to the documentation and updates to the acceptance tests. +It doesn't bring any new features or bug fixes. 0.47.0 ------ diff --git a/cypress-tests/cypress/e2e/useSession.cy.ts b/cypress-tests/cypress/e2e/useSession.cy.ts index 815522a43c..e402654566 100644 --- a/cypress-tests/cypress/e2e/useSession.cy.ts +++ b/cypress-tests/cypress/e2e/useSession.cy.ts @@ -224,20 +224,12 @@ describe("Basic public project functionality", () => { }); it("Start a new session with cloud storage attached.", () => { - cy.stopAllSessionsForProject(projectIdentifier); - - cy.intercept("/ui-server/api/data/storage*").as("getProjectCloudStorage"); - - cy.getProjectSection("Settings").click(); - cy.getDataCy("settings-navbar") - .contains("a.nav-link", "Cloud Storage") - .should("be.visible") - .click(); - - // Add a S3 storage configuration if it doesn't exist - cy.wait("@getProjectCloudStorage").then(({ response }) => { - const storages = response.body as { storage: { name: string } }[]; - if (storages.find(({ storage }) => storage.name === "data_s3")) { + // verify that cloud storage is supported in the deployment + cy.request({ url: "ui-server/api/notebooks/version" }).then((resp) => { + if (!resp.body.versions[0].data.cloudstorageEnabled) { + cy.log( + "Skipping the test since the deployment doesn't support Cloud Storage" + ); return; } @@ -285,13 +277,6 @@ describe("Basic public project functionality", () => { cy.getDataCy("cloud-storage-edit-close-button") .should("be.visible") .click(); - }); - - cy.getDataCy("more-menu").should("be.visible").click(); - cy.getProjectPageLink(projectIdentifier, "sessions/new") - .should("be.visible") - .first() - .click(); // Wait for the image to be ready and start a session cy.get(".renku-container") @@ -330,15 +315,57 @@ describe("Basic public project functionality", () => { cy.get(".jp-DirListing-item") .contains("README.s3_structure") .should("be.visible") - .dblclick(); - - cy.get(".jp-FileEditor", { timeout: TIMEOUTS.long }).should("be.visible"); - cy.get(".jp-FileEditor") - .contains("The GIAB s3 bucket and URLs") + .first() + .click(); + + // Wait for the image to be ready and start a session + cy.get(".renku-container") + .contains("A session gives you an environment") + .should("exist"); + cy.get(".renku-container .badge.bg-success", { timeout: TIMEOUTS.vlong }) + .contains("available") + .should("exist"); + cy.getDataCy("cloud-storage-item").contains("data_s3").should("exist"); + cy.get("#cloud-storage-data_s3-active").should("be.checked"); + cy.get(".renku-container button.btn-secondary", { timeout: TIMEOUTS.long }) + .contains("Start Session") + .should("exist") + .click(); + cy.get(".progress-box .progress-title").should("exist"); //.contains("Step 2 of 2"); + cy.get("button") + .contains(projectTestConfig.projectName) .should("be.visible"); + cy.get(".progress-box .progress-title") + .contains("Starting Session") + .should("exist"); + cy.get(".progress-box .progress-title", { timeout: TIMEOUTS.vlong }).should( + "not.exist" + ); + + // Verify that the S3 data is mounted + cy.getIframe("iframe#session-iframe").within(() => { + cy.get(".jp-DirListing-content", { timeout: TIMEOUTS.long }).should( + "be.visible" + ); + cy.get(".jp-DirListing-item") + .contains("data_s3") + .should("be.visible") + .dblclick(); + + cy.get(".jp-DirListing-item") + .contains("README.s3_structure") + .should("be.visible") + .dblclick(); + + cy.get(".jp-FileEditor", { timeout: TIMEOUTS.long }).should("be.visible"); + cy.get(".jp-FileEditor") + .contains("The GIAB s3 bucket and URLs") + .should("be.visible"); + }); + + cy.pauseSession(); + cy.deleteSession(); }); - - cy.pauseSession(); - cy.deleteSession(); }); + }); diff --git a/docs/_static/images/cloud_storage_1.png b/docs/_static/images/cloud_storage_1.png new file mode 100644 index 0000000000..38b7944145 Binary files /dev/null and b/docs/_static/images/cloud_storage_1.png differ diff --git a/docs/_static/images/cloud_storage_10.png b/docs/_static/images/cloud_storage_10.png new file mode 100644 index 0000000000..c8654ceb34 Binary files /dev/null and b/docs/_static/images/cloud_storage_10.png differ diff --git a/docs/_static/images/cloud_storage_11.png b/docs/_static/images/cloud_storage_11.png new file mode 100644 index 0000000000..ccb906b4c3 Binary files /dev/null and b/docs/_static/images/cloud_storage_11.png differ diff --git a/docs/_static/images/cloud_storage_2.png b/docs/_static/images/cloud_storage_2.png new file mode 100644 index 0000000000..80392abc0b Binary files /dev/null and b/docs/_static/images/cloud_storage_2.png differ diff --git a/docs/_static/images/cloud_storage_3.png b/docs/_static/images/cloud_storage_3.png new file mode 100644 index 0000000000..a868959d17 Binary files /dev/null and b/docs/_static/images/cloud_storage_3.png differ diff --git a/docs/_static/images/cloud_storage_4.png b/docs/_static/images/cloud_storage_4.png new file mode 100644 index 0000000000..e062628e73 Binary files /dev/null and b/docs/_static/images/cloud_storage_4.png differ diff --git a/docs/_static/images/cloud_storage_5.png b/docs/_static/images/cloud_storage_5.png new file mode 100644 index 0000000000..1a335f71e4 Binary files /dev/null and b/docs/_static/images/cloud_storage_5.png differ diff --git a/docs/_static/images/cloud_storage_6.png b/docs/_static/images/cloud_storage_6.png new file mode 100644 index 0000000000..a5fa7cc325 Binary files /dev/null and b/docs/_static/images/cloud_storage_6.png differ diff --git a/docs/_static/images/cloud_storage_7.png b/docs/_static/images/cloud_storage_7.png new file mode 100644 index 0000000000..1fb8907e6f Binary files /dev/null and b/docs/_static/images/cloud_storage_7.png differ diff --git a/docs/_static/images/cloud_storage_8.png b/docs/_static/images/cloud_storage_8.png new file mode 100644 index 0000000000..2a4deaa055 Binary files /dev/null and b/docs/_static/images/cloud_storage_8.png differ diff --git a/docs/_static/images/cloud_storage_9.png b/docs/_static/images/cloud_storage_9.png new file mode 100644 index 0000000000..10eb7aefd3 Binary files /dev/null and b/docs/_static/images/cloud_storage_9.png differ diff --git a/docs/how-to-guides/admin/sessions-cloud-storage.rst b/docs/how-to-guides/admin/sessions-cloud-storage.rst index b02a01565f..5ada83b8a0 100644 --- a/docs/how-to-guides/admin/sessions-cloud-storage.rst +++ b/docs/how-to-guides/admin/sessions-cloud-storage.rst @@ -3,17 +3,35 @@ Enable mounting cloud storage in users-sessions ----------------------------------------------- -Renkulab supports mounting cloud storage in users-sessions. -To enable the feature you need to set the following values in your `values-file.yaml`: +Renkulab supports mounting cloud storage in users-sessions. We use a CSI driver based on +`rclone `_. + +To enable the feature you need to configure it in your `values-file.yaml`: .. code-block:: yaml + # install the CSI driver + global: + csi-rclone: + install: true + notebooks: cloudstorage: - s3: - enabled: true - installDatashim: true + enabled: true + storageClass: csi-rclone + csi-rclone: + csiNodepluginRclone: + # optional tolerations if using node pools with taints + tolerations: + - key: renku.io/dedicated + operator: Equal + value: user + effect: NoSchedule + - key: renku.io/gpu + operator: Equal + value: "true" + effect: NoSchedule -The `installDatashim` option will deploy the Datashim Helm chart in the same -namespace as the `Renkulab` chart. +The ``global.csi-rclone.install`` option will install the ``csi-rclone`` Helm chart in the same namespace +as the renku chart. \ No newline at end of file diff --git a/docs/how-to-guides/renkulab/external-storage.rst b/docs/how-to-guides/renkulab/external-storage.rst new file mode 100644 index 0000000000..3c7bcac5be --- /dev/null +++ b/docs/how-to-guides/renkulab/external-storage.rst @@ -0,0 +1,235 @@ +.. _external_storage: + +External Storage in Renku Sessions +================================== + +Renku supports accessing data from external storage services like +S3, Azure Blob, WebDAV, etc. in interactive sessions. Access to external +storage can be configured via project settings. + +In the sections below we describe how to add and manage storage configuration to your projects +and highlight some current limitations: + +.. contents:: + :local: + +.. note:: + + Note that this feature is optional; it is enabled at + `renkulab.io `_ but other Renku deployments might not + support it. Contact your administrator for more details. + +.. warning:: + + We support both Read-only and Read/Write access modes. + Please note that explicitly marking an attached storage as Read-only is + **strongly** encouraged every time you don't have write permissions. This + prevents unexpected error messages when trying to access the storage. Furthermore, + please be aware that allowing write access to your data could result in data loss, data corruption or + data being overwritten. Consult the documentation of your cloud storage provider to learn how to + setup access controls and data versioning if you want to protect your data. + +Adding external storage to a project +------------------------------------ + +Currently, we support configuring external storage access per project. +Under the project's ``Settings`` tab, you can find the ``Cloud Storage`` +section. From there, you can click on the ``Add Cloud Storage`` button and follow the +instructions to go through the guided procedure. + +.. image:: ../../_static/images/cloud_storage_1.png + :align: center + :alt: Cloud Storage section in the project settings + +| + +Step-by-step mode +~~~~~~~~~~~~~~~~~ + +This is the default mode. It will guide you through the process of adding an +external storage to use in your sessions. + +Depending on the storage service, the list of options might be long, especially +when clicking the "Show full list" switch. Most of the time, the basic options +are sufficient. For more information, please refer to either +the `Rclone documentation `_ or the documentation from +the storage provider you are using. + + +Advanced Mode +~~~~~~~~~~~~~ + +If you are familiar with `Rclone `_, you can switch to +the "Advanced mode" and copy/paste from any Rclone configuration file. + +Mind that we require some additional information that cannot be inferred from +the configuration file, such as the storage name, the access mode, and the mount +point for the sessions. + + +Specific storage providers +-------------------------- + +The backend supports a variety of storage providers. It is based on `Rclone `_ +and we therefore support `most` of the storage services that Rclone supports, except for +anything requiring the OAuth flow or virtual rclone providers. The experience is optimized for the +most commonly used services, such as S3 and WebDAV. Please let us know if you require access to another +service that is not covered by the currently-available options! + +S3 +~~ + +Here is an example of attaching an S3 bucket to your project. +We pick the public dataset +`Genome in a Bottle (GIAB) `_ +hosted on AWS and show how to attach it to a Renku interactive session: + +- Navigate to the ``Settings: Cloud storage`` in one of your Renku projects. + +- Click on the ``Add Cloud Storage`` button and select ``S3`` for the ``Storage Type``, + then pick ``AWS`` on the provider list. Click on the ``Next`` button at the bottom. + + .. image:: ../../_static/images/cloud_storage_2.png + :align: center + :alt: Storage type and provider selection + +| + +- Fill in the form by using ``giab`` as the ``Source path``, and + ``http://s3.amazonaws.com`` as the endpoint. We don't need any credentials, and + the region is optional. Click on the ``"Next`` button. + + .. image:: ../../_static/images/cloud_storage_3.png + :align: center + :alt: GIAB bucket options configuration + +| + +- On the last page, pick any name for the storage (e.g. ``Giab``), then mark + this as Read-only. You can change the ``Mount point`` too; that determines the + virtual folder where this is mounted in the running sessions. + Click on the ``Add storage`` button. + + .. image:: ../../_static/images/cloud_storage_4.png + :align: center + :alt: GIAB bucket final steps + +| + +.. note:: + The ``mount point`` is relative to the root of the project path in the session. + +You can now start a new session and access the ``giab`` bucket at the ``external_storage/giab`` path in your project. + + +Azure Blob +~~~~~~~~~~ + +For this example, we host some data files in the folder ``test-data`` +hosted on Azure and show how to attach it to a Renku interactive session: + +- Navigate to the ``Settings - Cloud storage`` in your project. + +- Click on the ``Add Cloud Storage`` button and select ``azureblob`` for the + ``Storage Type``. Click on the ``Next`` button at the bottom. + + .. image:: ../../_static/images/cloud_storage_5.png + :align: center + :alt: Azure blob storage selection + +| + +- Fill in the form by filling the ``Source path`` to mount (keep it blank to + mount the default root, or specify the folder) and the Account Name. Fill + in the ``Shared Key``. Click on the ``Next`` button. + + .. image:: ../../_static/images/cloud_storage_6.png + :align: center + :alt: Azure blob options configuration + +| + +- On the last page, pick any name for the storage (e.g: `blob_storage``). + You can change the ``Mount point`` too; that determines the + virtual folder where this is mounted in the running sessions. + Click on the ``Add storage`` button. + + .. image:: ../../_static/images/cloud_storage_7.png + :align: center + :alt: Azure blob final steps + +| + +- When you start your session, the blob storage will be available at the specified mount point. + +WebDAV +~~~~~~ + +Many storage services, offer access via the WebDAV protocol. Renku supports +mounting WebDAV resources in interactive sessions. + +.. warning:: + We strongly encourage you to generate a token for accessing your private + OwnCloud/Nextcloud WebDAV shares to avoid sharing your account password. To create the token, go to + "Settings: Security" in your cloud storage application and generate a new + "Application pass-code" at the bottom of the page. + +Here we show an example of adding an `ETH Polybox `_ drive to +the session, but the procedure will be similar for any storage accessible via WebDAV. + +- Navigate to ``Settings - Cloud storage`` in your Renku project. +- Click on the "Add Cloud Storage" button and select ``webdav`` for + the "Storage Type". Click on the "Next" button at the bottom. + + .. image:: ../../_static/images/cloud_storage_8.png + :align: center + :alt: WebDAV storage selection + +| + +- Fill in the form by providing the ``Source path`` to mount (keep it blank to mount the + default root, or specify the folder), and the URL to specify the host to connect to + (``https://polybox.ethz.ch/remote.php/webdav/`` for `ETH Polybox `_). + Fill in the ``Username`` and the ``Token (or password)``. Click on the "Next" button. + + .. image:: ../../_static/images/cloud_storage_9.png + :align: center + :alt: WebDAV options configuration + +| + +- On the last page, pick any name for the storage (e.g: ``webdab_storage``). + You can change the ``Mount point`` too; that determines the + virtual folder where this is mounted in the running sessions. + Click on the "Add storage" button. + + .. image:: ../../_static/images/cloud_storage_10.png + :align: center + :alt: WebDAV final steps + +| + +- You will be able to access the cloud storage at the specified + mount point within your session. + + .. image:: ../../_static/images/cloud_storage_11.png + :align: center + :alt: Access to the mounted storage + +| + +Credentials +----------- + +For the time being, you cannot persist storage credentials in RenkuLab. If you need to +provide credentials to access storage, you will be asked to provide them +when starting a new session. The storage can be optionally disabled when launching a +session, if you do not wish to use it or if you do not have the required credentials. +If you pause and resume your session, you do not need to enter credentials again. + +Performance +----------- + +Performance varies based on the specific storage service, the access mode, +and the available bandwidth. Consider caching data locally if large amounts of +data need to be read or written repeatedly. diff --git a/docs/how-to-guides/renkulab/index.rst b/docs/how-to-guides/renkulab/index.rst index 407f2def81..137700f4d8 100644 --- a/docs/how-to-guides/renkulab/index.rst +++ b/docs/how-to-guides/renkulab/index.rst @@ -7,6 +7,6 @@ RenkuLab :maxdepth: 1 session-stopping-and-saving + external-storage ssh-into-sessions - s3 collaboration diff --git a/docs/how-to-guides/renkulab/s3.rst b/docs/how-to-guides/renkulab/s3.rst deleted file mode 100644 index 1ceed06345..0000000000 --- a/docs/how-to-guides/renkulab/s3.rst +++ /dev/null @@ -1,54 +0,0 @@ -.. _s3: - -Mount S3 Buckets in Renku Sessions -================================== - -Renku has an optional feature that allows users to mount and access data in -any S3-compatible object storage. This feature is enabled at `renkulab.io `_ -but please note that on other Renku deployment this may not be the case. -If your deployment does not have this feature enabled you should contact -your administrator. - -Amazon AWS initially came up with the S3 API for object storage. However, nowadays -almost all cloud providers have S3-compatible object storage. For example, Google Cloud Storage -is S3 compatible and can be used in Renku. Please refer to your cloud provider documentation -for more details. - -Step-by-step Instructions -------------------------- - -There are many public datasets that are hosted by AWS or other cloud providers on -S3-compatible object storage. One such dataset is the `Genome in a Bottle (GIAB) `_ -dataset hosted on AWS. We will use the GIAB dataset to demonstrate how to bring data in S3 to a Renku interactive session: - -1. Navigate to the ``Sessions`` tab in your Renku project. -2. Click the green ``New Session`` button on the right side of the page. -3. Click on the ``Do you want to select the branch, commit, image or configure cloud storage`` link. -4. Click on the ``Configure Cloud Storage`` button in the section that was revealed in the previous step. -5. In the form that opened, enter the ``Endpoint`` for the GIAB bucket as ``http://s3.amazonaws.com``. -6. The only other field you have to fill in is the ``Bucket Name`` field. For the GIAB bucket this is simply ``giab``. The ``Access Key`` and ``Secret Key`` fields can be left empty because this is a public bucket. -7. Click ``Save``. -8. Click ``Start session``. -9. Once you session is ready the bucket will be available inside your session at ``/cloudstorage/``. In this case for the GIAB bucket it will be available at ``/cloudstorage/giab``. - -Limitations ------------ - -There are a few limitations of this feature that users should be aware of. Please note that as we -further develop this feature some of these limitations will be relaxed or fully lifted. - -* **Accessing data in mounted buckets is potentially much slower than accessing data on disk.** - All content that is available in the bucket is fetched over the internet every time you need to - access it. Therefore, using this feature will be slower compared to when data is downloaded to disk. - However, the benefit is that mounting a bucket does not require any additional disk space. - So you could mount a bucket which has 1TB of data in your session and you would not need to request 1TB of - storage from Renku. We are investigating methods to provide caching or other means to improve the performance. -* **All mounted buckets have to have unique names.** There is no limit to how many buckets can be - mounted in a session but because all buckets are mounted in the same folder (i.e. ``/cloudstorage``) - if you try to mount two buckets with the same name then one mount would overwrite the other. - However, Renku will not let you get to this point and will prevent you from launching a session if - there are duplicate bucket names (regardless of the endpoint) across all the buckets you are trying - to mount. -* **Buckets can only be mounted in `read-only` mode** regardless of whether your credentials - (if provided) allow you to read and write in the bucket you are mounting. This is just a precaution - that may eventually be removed as the feature is further developed. diff --git a/docs/topic-guides/data/data.rst b/docs/topic-guides/data/data.rst index d2f37aa349..1197fa88b7 100644 --- a/docs/topic-guides/data/data.rst +++ b/docs/topic-guides/data/data.rst @@ -12,6 +12,12 @@ large files on the server and allows you to work with your repository without actually having a local copy of the data. You can pull the data from the server as it becomes needed, saving you time and resources. +However, git LFS is not efficient when working with a large amount of data. +Hence, we heavily recommend Renku users (especially when dealing with large amount of data) +to either store data files locally outside the Renku project folder in a +local filesystem, or even better +to provision :ref:`external_storage`, such as S3 or Azure Blob storage. + Using git LFS responsibly ------------------------- @@ -27,6 +33,17 @@ of data to each of their JupterLab sessions. Not only will this take a long time, but it might also eventually lead to resource starvation on the host node. + +Working with external cloud storage +----------------------------------- +If you setup external storage from the beginning of your project, you can more +easily share and handle your data. You need to control the access rights to +your external cloud storage data directly through your cloud storage provider. + + +The instructions to set up your cloud storage for a Renku project are in +:ref:`external_storage`. + Data in JupyterLab sessions --------------------------- diff --git a/helm-chart/renku/values.yaml b/helm-chart/renku/values.yaml index 61f4d8b985..ed38b2bfdb 100644 --- a/helm-chart/renku/values.yaml +++ b/helm-chart/renku/values.yaml @@ -62,7 +62,7 @@ global: fullnameOverride: "" image: repository: renku/renku-core - tag: "v2.9.1" + tag: "v2.9.2" pullPolicy: IfNotPresent uiserver: ## The client secret for the renku-ui client application registered in keycloak. @@ -1512,4 +1512,4 @@ affinity: {} versions: latest: image: - tag: v2.9.1 + tag: v2.9.2 diff --git a/helm-chart/values.yaml.changelog.md b/helm-chart/values.yaml.changelog.md index ad0e9ea23e..1aa578b668 100644 --- a/helm-chart/values.yaml.changelog.md +++ b/helm-chart/values.yaml.changelog.md @@ -5,6 +5,26 @@ For changes that require manual steps other than changing values, please check o Please follow this convention when adding a new row * ` - **:
` +## Upgrading to Renku 0.47.0 + +We completely overhauled how mounting cloud storage in sessions works, relying on a new CSI driver based on RClone +which has to be installed in the cluster for things to work. Either install it as part of Renku using the flag +mentioned below or install the csi-rclone chart manually and set the correct storage class in the values for the +notebooks service. + +* NEW `noteboks.cloudstorage.enabled` - set to `true` to enable mounting cloud storage in sessions. +* DELETE `notebooks.cloudstorage.s3.enabed` - superseeded by previous property. +* NEW `notebooks.cloudstorage.storageClass` - the storage class for the CSI Rclone chart, needed for new cloudstorage + to work. The default `csi-rclone` should be fine unless already in use. +* NEW `global.csi-rclone.install` - if `true` installs the csi-rclone chart alongside Renku. The chart is needed for + cloud storage in sessions to work. +* NEW `csi-rclone.storageClassName` - the storage class name the CSI drivers uses, should match what is configured in + the `storageClass` property mentioned above. +* NEW `csi-rclone.csiNodePlugin.tolerations` - Tolerations for the node plugin part of the CSI driver. Need to be set + in a way that allows it to be scheduled on user session nodes. By default this would mean `key=renku.io/dedicated`, + `operator=Equal`, `value=user` and `effect=NoSchedule` + + ## Upgrading to Renku 0.43.0 * DELETE `graph.gitlab.url` has been removed as graph services uses the `global.gitlab.url`.