Merge branch 'main' into andrea/3392-add-polybox-switchdrive-schema

components/renku_data_services/notebooks/api/classes/image.py

@@ -17,10 +17,15 @@ class ManifestTypes(Enum):
     """The mime types for docker image manifests."""
 
     docker_v2: str = "application/vnd.docker.distribution.manifest.v2+json"
+    docker_v2_list: str = "application/vnd.docker.distribution.manifest.list.v2+json"
     oci_v1_manifest: str = "application/vnd.oci.image.manifest.v1+json"
     oci_v1_index: str = "application/vnd.oci.image.index.v1+json"
 
 
+DEFAULT_PLATFORM_ARCHITECTURE = "amd64"
+DEFAULT_PLATFORM_OS = "linux"
+
+
 @dataclass
 class ImageRepoDockerAPI:
     """Used to query the docker image repository API.
@@ -62,7 +67,12 @@ async def _get_docker_token(self, image: "Image") -> Optional[str]:
         token_req = await self.client.get(realm, params=params, headers=headers)
         return str(token_req.json().get("token"))
 
-    async def get_image_manifest(self, image: "Image") -> Optional[dict[str, Any]]:
+    async def get_image_manifest(
+        self,
+        image: "Image",
+        platform_architecture: str = DEFAULT_PLATFORM_ARCHITECTURE,
+        platform_os: str = DEFAULT_PLATFORM_OS,
+    ) -> Optional[dict[str, Any]]:
         """Query the docker API to get the manifest of an image."""
         if image.hostname != self.hostname:
             raise errors.ValidationError(
@@ -80,16 +90,42 @@ async def get_image_manifest(self, image: "Image") -> Optional[dict[str, Any]]:
         if res.status_code != 200:
             headers["Accept"] = ManifestTypes.oci_v1_index.value
             res = await self.client.get(image_digest_url, headers=headers)
-            if res.status_code == 200:
-                index_parsed = res.json()
-                manifest = next(
-                    (man for man in index_parsed.get("manifests", []) if man.get("platform", {}).get("os") == "linux"),
-                    None,
-                )
-                manifest = cast(dict[str, Any] | None, manifest)
-                return manifest
         if res.status_code != 200:
             return None
+
+        content_type = res.headers.get("Content-Type")
+        if content_type in [ManifestTypes.docker_v2_list.value, ManifestTypes.oci_v1_index.value]:
+            index_parsed = res.json()
+
+            def platform_matches(manifest: dict[str, Any]) -> bool:
+                platform: dict[str, Any] = manifest.get("platform", {})
+                return platform.get("architecture") == platform_architecture and platform.get("os") == platform_os
+
+            manifest: dict[str, Any] = next(filter(platform_matches, index_parsed.get("manifests", [])), {})
+            image_digest: str | None = manifest.get("digest")
+            if not manifest or not image_digest:
+                return None
+            image_digest_url = f"https://{image.hostname}/v2/{image.name}/manifests/{image_digest}"
+            media_type = manifest.get("mediaType")
+            headers["Accept"] = ManifestTypes.docker_v2.value
+            if media_type in [
+                ManifestTypes.docker_v2.value,
+                ManifestTypes.oci_v1_manifest.value,
+            ]:
+                headers["Accept"] = media_type
+            res = await self.client.get(image_digest_url, headers=headers)
+            if res.status_code != 200:
+                headers["Accept"] = ManifestTypes.oci_v1_manifest.value
+                res = await self.client.get(image_digest_url, headers=headers)
+            if res.status_code != 200:
+                return None
+
+        if res.headers.get("Content-Type") not in [
+            ManifestTypes.docker_v2.value,
+            ManifestTypes.oci_v1_manifest.value,
+        ]:
+            return None
+
         return cast(dict[str, Any], res.json())
 
     async def image_exists(self, image: "Image") -> bool:

components/renku_data_services/notebooks/blueprints.py

@@ -54,6 +54,7 @@
     Ingress,
     InitContainer,
     Metadata,
+    ReconcileStrategy,
     Resources,
     SecretAsVolume,
     SecretAsVolumeItem,
@@ -437,6 +438,7 @@ async def _handler(
                 spec=AmaltheaSessionSpec(
                     codeRepositories=[],
                     hibernated=False,
+                    reconcileStrategy=ReconcileStrategy.whenFailedOrHibernated,
                     session=Session(
                         image=image,
                         urlPath=ui_path,

components/renku_data_services/notebooks/cr_amalthea_session.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  <stdin>
-#   timestamp: 2024-11-05T01:48:51+00:00
+#   timestamp: 2024-11-13T08:52:55+00:00
 
 from __future__ import annotations
 
@@ -3029,7 +3029,7 @@ class InitContainer(BaseCRD):
     )
 
 
-class ReconcileSrategy(Enum):
+class ReconcileStrategy(Enum):
     never = "never"
     always = "always"
     whenFailedOrHibernated = "whenFailedOrHibernated"
@@ -3221,7 +3221,7 @@ class Spec(BaseCRD):
         default=None,
         description="Selector which must match a node's labels for the pod to be scheduled on that node.\nPassed right through to the Statefulset used for the session.",
     )
-    reconcileSrategy: ReconcileSrategy = Field(
+    reconcileStrategy: ReconcileStrategy = Field(
         default="always",
         description="Indicates how Amalthea should reconcile the child resources for a session. This can be problematic because\nnewer versions of Amalthea may include new versions of the sidecars or other changes not reflected\nin the AmaltheaSession CRD, so simply updating Amalthea could cause existing sessions to restart\nbecause the sidecars will have a newer image or for other reasons because the code changed.\nHibernating the session and deleting it will always work as expected regardless of the strategy.\nThe status of the session and all hibernation or auto-cleanup functionality will always work as expected.\nA few values are possible:\n- never: Amalthea will never update any of the child resources and will ignore any changes to the CR\n- always: This is the expected method of operation for an operator, changes to the spec are always reconciled\n- whenHibernatedOrFailed: To avoid interrupting a running session, reconciliation of the child components\n  are only done when the session has a Failed or Hibernated status",
     )

components/renku_data_services/notebooks/crs.py

@@ -26,6 +26,7 @@
     NodeSelectorTerm,
     Preference,
     PreferredDuringSchedulingIgnoredDuringExecutionItem,
+    ReconcileStrategy,
     RequiredDuringSchedulingIgnoredDuringExecution,
     SecretRef,
     Session,

projects/renku_data_service/Dockerfile

@@ -1,11 +1,20 @@
 FROM python:3.12-bookworm as builder
 ARG DEV_BUILD=false
 ARG USER_UID=1000
+ARG RCLONE_VERSION=v1.65.2
+ARG RCLONE_ARCH=amd64
+ARG RCLONE_OS=linux
 ARG USER_GID=$USER_UID
 
 RUN groupadd --gid $USER_GID renku && \
     DEBIAN_FRONTEND=noninteractive adduser --gid $USER_GID --uid $USER_UID renku
-RUN curl -fsSL https://rclone.org/install.sh| bash
+RUN cd /tmp \
+    && wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-${RCLONE_OS}-${RCLONE_ARCH}.zip \
+    && unzip /tmp/rclone-${RCLONE_VERSION}-${RCLONE_OS}-${RCLONE_ARCH}.zip \
+    && mv /tmp/rclone-${RCLONE_VERSION}-${RCLONE_OS}-${RCLONE_ARCH}/rclone /usr/bin \
+    && chmod 755 /usr/bin/rclone \
+    && chown root:root /usr/bin/rclone \
+    && rm -r /tmp/rclone*
 USER $USER_UID:$USER_GID
 WORKDIR /app
 RUN python3 -m pip install --user pipx && \

test/components/renku_data_services/notebooks/test_notebooks_image_checks.py

@@ -198,8 +198,10 @@ async def test_public_image_check(image: str, exists_expected: bool) -> None:
 @pytest.mark.parametrize(
     "image,expected_path",
     [
+        ("jupyter/minimal-notebook", PurePosixPath("/home/jovyan")),
         ("jupyter/minimal-notebook:x86_64-python-3.11.6", PurePosixPath("/home/jovyan")),
-        ("nginx", None),
+        ("nginx", PurePosixPath("/")),
+        ("nginx@sha256:367678a80c0be120f67f3adfccc2f408bd2c1319ed98c1975ac88e750d0efe26", PurePosixPath("/")),
         ("madeuprepo/madeupproject:tag", None),
     ],
 )