Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: remove unused code and properly serialize v1 servers #569

Merged
merged 5 commits into from
Dec 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 1 addition & 21 deletions components/renku_data_services/notebooks/api.spec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,26 +71,6 @@ paths:
description: Server options such as CPU, memory, storage, etc.
tags:
- notebooks
"/notebooks/old/servers":
post:
summary: Launch a new session
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/LaunchNotebookRequestOld"
responses:
"201":
description: The project was created
content:
application/json:
schema:
$ref: "#/components/schemas/NotebookResponse"
default:
$ref: "#/components/responses/Error"
tags:
- notebooks
"/notebooks/servers":
post:
summary: Launch a new session
Expand All @@ -99,7 +79,7 @@ paths:
content:
application/json:
schema:
$ref: "#/components/schemas/LaunchNotebookRequest"
$ref: "#/components/schemas/LaunchNotebookRequestOld"
responses:
"201":
description: The project was created
Expand Down
71 changes: 0 additions & 71 deletions components/renku_data_services/notebooks/api/classes/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -486,74 +486,3 @@ def get_annotations(self) -> dict[str, str | None]:
annotations[f"{prefix}gitlabProjectId"] = str(self.gitlab_project.id)
annotations[f"{prefix}repository"] = self.gitlab_project.web_url
return annotations


class Renku2UserServer(UserServer):
"""Represents a Renku 2.0 server session."""

def __init__(
self,
user: AnonymousAPIUser | AuthenticatedAPIUser,
image: str,
project_id: str,
launcher_id: str,
server_name: str,
server_options: ServerOptions,
environment_variables: dict[str, str],
user_secrets: K8sUserSecrets | None,
cloudstorage: Sequence[ICloudStorageRequest],
k8s_client: K8sClient,
workspace_mount_path: PurePosixPath,
work_dir: PurePosixPath,
repositories: list[Repository],
config: NotebooksConfig,
internal_gitlab_user: APIUser,
using_default_image: bool = False,
is_image_private: bool = False,
**_: dict,
):
super().__init__(
user=user,
server_name=server_name,
image=image,
server_options=server_options,
environment_variables=environment_variables,
user_secrets=user_secrets,
cloudstorage=cloudstorage,
k8s_client=k8s_client,
workspace_mount_path=workspace_mount_path,
work_dir=work_dir,
using_default_image=using_default_image,
is_image_private=is_image_private,
repositories=repositories,
config=config,
internal_gitlab_user=internal_gitlab_user,
)

self.project_id = project_id
self.launcher_id = launcher_id

def get_labels(self) -> dict[str, str | None]:
"""Get the labels of the jupyter server."""
prefix = self._get_renku_annotation_prefix()
labels = super().get_labels()

# for validation purpose
for item in ["commit-sha", "gitlabProjectId"]:
labels[f"{prefix}{item}"] = ""

return labels

def get_annotations(self) -> dict[str, str | None]:
"""Get the annotations of the session."""
prefix = self._get_renku_annotation_prefix()
annotations = super().get_annotations()
annotations[f"{prefix}renkuVersion"] = "2.0"
annotations[f"{prefix}projectId"] = self.project_id
annotations[f"{prefix}launcherId"] = self.launcher_id

# for validation purpose
for item in ["commit-sha", "branch", "git-host", "namespace", "projectName", "gitlabProjectId", "repository"]:
annotations[f"{prefix}{item}"] = ""

return annotations
41 changes: 6 additions & 35 deletions components/renku_data_services/notebooks/blueprints.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,6 @@
from renku_data_services.notebooks.api.schemas.cloud_storage import RCloneStorage
from renku_data_services.notebooks.api.schemas.config_server_options import ServerOptionsEndpointResponse
from renku_data_services.notebooks.api.schemas.logs import ServerLogs
from renku_data_services.notebooks.api.schemas.servers_get import (
NotebookResponse,
ServersGetResponse,
)
from renku_data_services.notebooks.config import NotebooksConfig
from renku_data_services.notebooks.crs import (
Affinity,
Expand Down Expand Up @@ -110,7 +106,7 @@ async def _user_servers(
) -> JSONResponse:
filter_attrs = list(filter(lambda x: x[1] is not None, request.get_query_args()))
filtered_servers = await core.user_servers(self.nb_config, user, filter_attrs)
return json(ServersGetResponse().dump({"servers": filtered_servers}))
return core.serialize_v1_servers(filtered_servers, self.nb_config)

return "/notebooks/servers", ["GET"], _user_servers

Expand All @@ -122,47 +118,26 @@ async def _user_server(
request: Request, user: AnonymousAPIUser | AuthenticatedAPIUser, server_name: str
) -> JSONResponse:
server = await core.user_server(self.nb_config, user, server_name)
return json(NotebookResponse().dump(server))
return core.serialize_v1_server(server, self.nb_config)

return "/notebooks/servers/<server_name>", ["GET"], _user_server

def launch_notebook(self) -> BlueprintFactoryResponse:
"""Start a renku session."""

@authenticate_2(self.authenticator, self.internal_gitlab_authenticator)
@validate(json=apispec.LaunchNotebookRequest)
@validate(json=apispec.LaunchNotebookRequestOld)
async def _launch_notebook(
request: Request,
user: AnonymousAPIUser | AuthenticatedAPIUser,
internal_gitlab_user: APIUser,
body: apispec.LaunchNotebookRequest,
body: apispec.LaunchNotebookRequestOld,
) -> JSONResponse:
server, status_code = await core.launch_notebook(self.nb_config, user, internal_gitlab_user, body)
return json(NotebookResponse().dump(server), status_code)
return core.serialize_v1_server(server, self.nb_config, status_code)

return "/notebooks/servers", ["POST"], _launch_notebook

def launch_notebook_old(self) -> BlueprintFactoryResponse:
"""Start a renku session using the old operator."""

@authenticate_2(self.authenticator, self.internal_gitlab_authenticator)
@validate(json=apispec.LaunchNotebookRequestOld)
async def _launch_notebook_old(
request: Request,
user: AnonymousAPIUser | AuthenticatedAPIUser,
internal_gitlab_user: APIUser,
body: apispec.LaunchNotebookRequestOld,
) -> JSONResponse:
server, status_code = await core.launch_notebook_old(
self.nb_config,
user,
internal_gitlab_user,
body,
)
return json(NotebookResponse().dump(server), status_code)

return "/notebooks/old/servers", ["POST"], _launch_notebook_old

def patch_server(self) -> BlueprintFactoryResponse:
"""Patch a user server by name based on the query param."""

Expand All @@ -179,11 +154,7 @@ async def _patch_server(
raise AnonymousUserPatchError()

manifest = await core.patch_server(self.nb_config, user, internal_gitlab_user, server_name, body)
notebook_response = apispec.NotebookResponse.parse_obj(manifest)
return json(
notebook_response.model_dump(),
200,
)
return core.serialize_v1_server(manifest, self.nb_config)

return "/notebooks/servers/<server_name>", ["PATCH"], _patch_server

Expand Down
69 changes: 23 additions & 46 deletions components/renku_data_services/notebooks/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,29 +10,28 @@
from gitlab.const import Visibility as GitlabVisibility
from gitlab.v4.objects.projects import Project as GitlabProject
from sanic.log import logger
from sanic.response import JSONResponse

from renku_data_services.base_models import AnonymousAPIUser, APIUser, AuthenticatedAPIUser
from renku_data_services.base_models.validation import validated_json
from renku_data_services.errors import errors
from renku_data_services.notebooks import apispec
from renku_data_services.notebooks.api.classes.auth import GitlabToken, RenkuTokens
from renku_data_services.notebooks.api.classes.image import Image
from renku_data_services.notebooks.api.classes.repository import Repository
from renku_data_services.notebooks.api.classes.server import Renku1UserServer, Renku2UserServer, UserServer
from renku_data_services.notebooks.api.classes.server import Renku1UserServer, UserServer
from renku_data_services.notebooks.api.classes.server_manifest import UserServerManifest
from renku_data_services.notebooks.api.classes.user import NotebooksGitlabClient
from renku_data_services.notebooks.api.schemas.cloud_storage import RCloneStorage
from renku_data_services.notebooks.api.schemas.secrets import K8sUserSecrets
from renku_data_services.notebooks.api.schemas.server_options import ServerOptions
from renku_data_services.notebooks.api.schemas.servers_get import NotebookResponse
from renku_data_services.notebooks.api.schemas.servers_patch import PatchServerStatusEnum
from renku_data_services.notebooks.config import NotebooksConfig
from renku_data_services.notebooks.errors import intermittent
from renku_data_services.notebooks.errors import user as user_errors
from renku_data_services.notebooks.util import repository
from renku_data_services.notebooks.util.kubernetes_ import (
find_container,
renku_1_make_server_name,
renku_2_make_server_name,
)
from renku_data_services.notebooks.util.kubernetes_ import find_container, renku_1_make_server_name


def notebooks_info(config: NotebooksConfig) -> dict:
Expand Down Expand Up @@ -68,7 +67,7 @@ def notebooks_info(config: NotebooksConfig) -> dict:

async def user_servers(
config: NotebooksConfig, user: AnonymousAPIUser | AuthenticatedAPIUser, filter_attrs: list[dict]
) -> dict:
) -> dict[str, UserServerManifest]:
"""Returns a filtered list of servers for the given user."""

servers = [
Expand Down Expand Up @@ -553,45 +552,6 @@ async def _on_error(server_name: str, error_msg: str) -> None:


async def launch_notebook(
config: NotebooksConfig,
user: AnonymousAPIUser | AuthenticatedAPIUser,
internal_gitlab_user: APIUser,
launch_request: apispec.LaunchNotebookRequest,
) -> tuple[UserServerManifest, int]:
"""Starts a server."""

server_name = renku_2_make_server_name(
user=user, project_id=launch_request.project_id, launcher_id=launch_request.launcher_id
)
return await launch_notebook_helper(
nb_config=config,
server_name=server_name,
server_class=Renku2UserServer,
user=user,
image=launch_request.image or config.sessions.default_image,
resource_class_id=launch_request.resource_class_id,
storage=launch_request.storage,
environment_variables=launch_request.environment_variables,
user_secrets=launch_request.user_secrets,
default_url=config.server_options.default_url_default,
lfs_auto_fetch=config.server_options.lfs_auto_fetch_default,
cloudstorage=launch_request.cloudstorage,
server_options=None,
namespace=None,
project=None,
branch=None,
commit_sha=None,
notebook=None,
gl_project=None,
gl_project_path=None,
project_id=launch_request.project_id,
launcher_id=launch_request.launcher_id,
repositories=launch_request.repositories,
internal_gitlab_user=internal_gitlab_user,
)


async def launch_notebook_old(
config: NotebooksConfig,
user: AnonymousAPIUser | AuthenticatedAPIUser,
internal_gitlab_user: APIUser,
Expand Down Expand Up @@ -645,3 +605,20 @@ async def launch_notebook_old(
repositories=None,
internal_gitlab_user=internal_gitlab_user,
)


def serialize_v1_server(manifest: UserServerManifest, nb_config: NotebooksConfig, status: int = 200) -> JSONResponse:
"""Format and serialize a Renku v1 JupyterServer manifest."""
data = NotebookResponse().dump(NotebookResponse.format_user_pod_data(manifest, nb_config))
return validated_json(apispec.NotebookResponse, data, status=status)


def serialize_v1_servers(
manifests: dict[str, UserServerManifest], nb_config: NotebooksConfig, status: int = 200
) -> JSONResponse:
"""Format and serialize many Renku v1 JupyterServer manifests."""
data = {
manifest.server_name: NotebookResponse().dump(NotebookResponse.format_user_pod_data(manifest, nb_config))
for manifest in sorted(manifests.values(), key=lambda x: x.server_name)
}
return validated_json(apispec.ServersGetResponse, {"servers": data}, status=status)
38 changes: 30 additions & 8 deletions test/bases/renku_data_services/data_api/test_notebooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,17 @@ async def practice_jupyter_server(renku_image: str, server_name: str) -> AsyncIt
"renku.io/repository": "dummy",
},
},
"spec": {"jupyterServer": {"image": renku_image}},
"spec": {
"jupyterServer": {
"image": renku_image,
"resources": {
"requests": {
"cpu": 0.1,
"memory": 100_000_000,
},
},
},
},
}
)

Expand Down Expand Up @@ -263,17 +273,31 @@ async def test_user_server_list(
sanic_client: SanicASGITestClient,
request,
server_name,
jupyter_server,
authenticated_user_headers,
fake_gitlab,
):
"""Validate that the user server list endpoint answers correctly"""
data = {
"branch": "main",
"commit_sha": "ee4b1c9fedc99abe5892ee95320bbd8471c5985b",
"namespace": "test-namespace",
"project": "my-test",
"image": "alpine:3",
}
_, res = await sanic_client.post("/api/data/notebooks/servers/", json=data, headers=authenticated_user_headers)
assert res.status_code == 201, res.text
server_name = res.json["name"]

_, res = await sanic_client.get("/api/data/notebooks/servers", headers=authenticated_user_headers)

assert res.status_code == 200, res.text
assert "servers" in res.json
assert len(res.json["servers"]) == 1

_, res = await sanic_client.delete(
f"/api/data/notebooks/servers/{server_name}", headers=authenticated_user_headers
)
assert res.status_code == 204, res.text

@pytest.mark.asyncio
@pytest.mark.parametrize(
"server_name_fixture,expected_status_code", [("unknown_server_name", 404), ("server_name", 200)]
Expand Down Expand Up @@ -349,9 +373,7 @@ async def test_old_start_server(self, sanic_client: SanicASGITestClient, authent
"image": "alpine:3",
}

_, res = await sanic_client.post(
"/api/data/notebooks/old/servers/", json=data, headers=authenticated_user_headers
)
_, res = await sanic_client.post("/api/data/notebooks/servers/", json=data, headers=authenticated_user_headers)

assert res.status_code == 201, res.text

Expand All @@ -367,8 +389,8 @@ async def test_start_server(self, sanic_client: SanicASGITestClient, authenticat
data = {
"branch": "main",
"commit_sha": "ee4b1c9fedc99abe5892ee95320bbd8471c5985b",
"project_id": "test-namespace/my-test",
"launcher_id": "test_launcher",
"namespace": "test-namespace",
"project": "my-test",
"image": "alpine:3",
}

Expand Down
Loading