Merge pull request #22 from katulu-io/GerardoGR/lean-and-mean

Keep only services needed from kubeflow

components/centraldashboard/src/app/config.service.ts

@@ -1,25 +1,13 @@
 import { Injectable } from '@angular/core';
 
 const APPS = [
-  { path: '_/jupyter', location: '/jupyter/', label: 'Notebooks', icon: 'book' },
+  { path: '_/jupyter', location: '/jupyter/', label: 'Notebook', icon: 'book' },
   {
     path: '_/edges',
     location: '/edges/',
     label: 'Edges',
     icon: 'developer_board',
   },
-  {
-    path: '_/volumes',
-    location: '/volumes/',
-    label: 'Volumes',
-    icon: 'storage',
-  },
-  {
-    path: '_/models',
-    location: '/models/',
-    label: 'Models',
-    icon: 'widgets',
-  },
   {
     label: 'Experiments (KFP)',
     path: '_/pipeline/#/experiments',

components/kubeflow-notebooks/.gitignore

@@ -1 +1,2 @@
-dist
+dist
+/kustomization.yaml

components/kubeflow-notebooks/Makefile

@@ -3,6 +3,8 @@
 MAKEVAR_REGISTRY?=localhost:5000
 MAKEVAR_VERSION?=0.0.0-dev-$(shell git log -n 1 --pretty=format:%h)
 CONTAINER_TAG = $(shell echo "${MAKEVAR_VERSION}" | sed "s/\+[0-9a-zA-Z-]*//")
+IMAGE_NAME = katulu-fl-jupyter-full
+IMAGE_URL = ${MAKEVAR_REGISTRY}/${IMAGE_NAME}:${CONTAINER_TAG}
 
 PIP_VERSION = $(shell echo "${MAKEVAR_VERSION}" | sed -e "s/-/+/" -e "s/-/./g")
 
@@ -18,23 +20,32 @@ build:
 	@echo "build not implemented"
 .PHONY: build
 
-dist:
+dist: manifests
 	mkdir -p dist
 	cp ../kubeflow-pipeline/python/dist/fl_suite-${PIP_VERSION}-py3-none-any.whl dist/
 	sed -e "s/\\\$$\\\$$container-tag\\\$$\\\$$/${PIP_VERSION}/g" Dockerfile.tpl > dist/Dockerfile
 	sed -e "s/\\\$$\\\$$container-tag\\\$$\\\$$/${PIP_VERSION}/g" requirements.tpl.txt > dist/requirements.txt
-	DOCKER_BUILDKIT=1 docker build -t ${MAKEVAR_REGISTRY}/katulu-fl-jupyter-full:${CONTAINER_TAG} dist
+	DOCKER_BUILDKIT=1 docker build -t ${IMAGE_URL} dist
 .PHONY: dist
 
+manifests:
+	cp kustomization.yaml.tpl kustomization.yaml
+	kustomize edit set image ${IMAGE_NAME}=${IMAGE_URL}
+.PHONY: manifests
+
 push:
-	docker push ${MAKEVAR_REGISTRY}/katulu-fl-jupyter-full:${CONTAINER_TAG}
+	docker push ${IMAGE_URL}
 .PHONY: push
 
 # Handy install target to iterate development over a single component
 install:
-	kustomize build kustomize/ | kubectl apply -f -
+	kustomize build . | kubectl apply -f -
 .PHONY: install
 
+uninstall:
+	kustomize build . | kubectl delete -f -
+.PHONY: uninstall
+
 clean:
 	@rm -rf dist
 .PHONY: clean

components/kubeflow-notebooks/kustomization.yaml.tpl

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: katulu-fl
+resources:
+- kustomize/overlays/istio
+commonLabels:
+  app: jupyter-notebook
+  app.kubernetes.io/component: jupyter-notebook
+  app.kubernetes.io/name: jupyter-notebook
+  kustomize.component: jupyter-notebook

components/kubeflow-notebooks/kustomize/base/deployment.yaml

@@ -0,0 +1,63 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: jupyter-notebook
+  labels:
+    app: jupyter-notebook
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: jupyter-notebook
+  template:
+    metadata:
+      labels:
+        app: jupyter-notebook
+    spec:
+      serviceAccountName: fl-notebook
+      containers:
+        - env:
+          - name: NB_PREFIX
+            value: /jupyter
+          - name: KF_PIPELINES_SA_TOKEN_PATH
+            value: /var/run/secrets/kubeflow/pipelines/token
+          image: katulu-fl-jupyter-full
+          imagePullPolicy: IfNotPresent
+          name: katulu-fl-jupyter-full
+          ports:
+            - containerPort: 8888
+              name: notebook-port
+              protocol: TCP
+          resources:
+            limits:
+              cpu: 600m
+              memory: 1Gi
+            requests:
+              cpu: 500m
+              memory: 1Gi
+          workingDir: /home/jovyan
+          volumeMounts:
+              - mountPath: /var/run/secrets/kubeflow/pipelines
+                name: volume-kf-pipeline-token
+                readOnly: true
+              - name: minio-credentials
+                mountPath: /minio
+                readOnly: true
+              - mountPath: /home/jovyan
+                name: workspace
+      volumes:
+        - name: workspace
+          persistentVolumeClaim:
+            claimName: jupyter-notebook-workspace
+        - name: volume-kf-pipeline-token
+          projected:
+            sources:
+              - serviceAccountToken:
+                  path: token
+                  expirationSeconds: 7200
+                  audience: pipelines.kubeflow.org
+        - name: minio-credentials
+          secret:
+            secretName: mlpipeline-minio-artifact
+            optional: false

components/kubeflow-notebooks/kustomize/kustomization.yaml → components/kubeflow-notebooks/kustomize/base/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 namespace: katulu-fl
 
 resources:
-  - allow-pipeline-access.yaml
   - rolebinding.yaml
   - serviceaccount.yaml
+  - pv-claim.yaml
+  - deployment.yaml
+  - service.yaml

components/kubeflow-notebooks/kustomize/base/pv-claim.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: jupyter-notebook-workspace
+  labels:
+    app: jupyter-notebook
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 20Gi

components/kubeflow-notebooks/kustomize/base/service.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: jupyter-notebook
+spec:
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: 8888
+  selector:
+    app: jupyter-notebook
+  type: ClusterIP

components/kubeflow-notebooks/kustomize/overlays/istio/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: katulu-fl
+
+resources:
+  - ../../base
+  - virtualservice.yaml

components/kubeflow-notebooks/kustomize/overlays/istio/virtualservice.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: networking.istio.io/v1beta1
+kind: VirtualService
+metadata:
+  name: jupyter-notebook
+spec:
+  gateways:
+    - kubeflow/kubeflow-gateway
+  hosts:
+    - '*'
+  http:
+    - headers:
+        request:
+          add:
+            x-forwarded-prefix: /jupyter
+      match:
+      - uri:
+          prefix: /jupyter/
+      rewrite:
+        uri: /jupyter/
+
+      route:
+        - destination:
+            host: jupyter-notebook.katulu-fl.svc.cluster.local
+            port:
+              number: 80

docs/installation-guide.md

@@ -112,7 +112,7 @@ You can now open your browser and visit http://localhost:8080/. You should get t
 
 <br />
 
-![](../images/fl-suite-notebooks.png)
+![](../images/fl-suite-notebook.png)
 
 <br />
 

docs/quickstart.md

@@ -19,20 +19,10 @@ To do that, we need to describe how a Federated Learning client will train and t
 
 At least two edge clients need to be connected to FL Suite before running these steps, as documented in the [installation guide](./installation-guide.md).
 
-## Starting a new notebook
-
-First, we create a new notebook from the "Notebooks" section in FL Suite. The new notebook needs to use the custom Jupyter image `ghcr.io/katulu-io/fl-suite/katulu-fl-jupyter-full:1.0.0-beta.3` to have all necessary packages available.
-
-![](../images/new-notebook-custom-image.png)
-
-Furthermore, the "Allow access to Kubeflow Pipelines" configuration has to be enabled.
-
-![](../images/new-notebook-configurations.png)
-
-With this notebook launched, we can prepare a new document there.
-
 ## Preparing and running the pipeline
 
+Following up from the [installation guide](./installation-guide.md) a fl-suite jupyter notebook will be accessible at: http://localhost:8080. The next steps will use such notebook to start an FL pipeline.
+
 We start by importing the necessary package:
 
 ```python

kustomize/fl-suite/base/kustomization.yaml

@@ -7,10 +7,6 @@ resources:
   - ../../../components/vendor/kubeflow-manifests/common/oidc-authservice/base
   # Dex
   - ../../../components/katulu-dex
-  # KNative
-  - ../../../components/vendor/kubeflow-manifests/common/knative/knative-serving/overlays/gateways
-  - ../../../components/vendor/kubeflow-manifests/common/knative/knative-eventing/base
-  - ../../../components/vendor/kubeflow-manifests/common/istio-1-11/cluster-local-gateway/base
   # Kubeflow namespace
   - ../../../components/vendor/kubeflow-manifests/common/kubeflow-namespace/base
   # Kubeflow Roles
@@ -22,18 +18,8 @@ resources:
   - ../../../components/vendor/kubeflow-manifests/apps/pipeline/upstream/env/platform-agnostic-multi-user-emissary
   # Central Dashboard
   - ../../../components/centraldashboard
-  # Admission Webhook
-  - ../../../components/vendor/kubeflow-manifests/apps/admission-webhook/upstream/overlays/cert-manager
-  # Notebook Controller
-  - ../../../components/vendor/kubeflow-manifests/apps/jupyter/jupyter-web-app/upstream/overlays/istio
-  # Jupyter Web App
-  - ../../../components/vendor/kubeflow-manifests/apps/jupyter/notebook-controller/upstream/overlays/kubeflow
   # Profiles + KFAM
   - ../../../components/vendor/kubeflow-manifests/apps/profiles/upstream/overlays/kubeflow
-  # Volumes Web App
-  - ../../../components/vendor/kubeflow-manifests/apps/volumes-web-app/upstream/overlays/istio
-  # Training Operator
-  - ../../../components/vendor/kubeflow-manifests/apps/training-operator/upstream/overlays/kubeflow
   # User namespace
   - ../../../components/kubeflow-profile/kustomize
   # Edges Web app
@@ -47,7 +33,7 @@ resources:
   - ../../../components/spire/kustomize/spire-namespace
   - ../../../components/spire/kustomize/spire-agent
 
-  - ../../../components/kubeflow-notebooks/kustomize
+  - ../../../components/kubeflow-notebooks
   - ../../../components/kubeflow-pipeline/kustomize
   - ../../../components/fl-orchestrator
   - ../../../components/client-image-builder/kustomize