From 2a5f927a9764e03cbbe408f42833536ba18c4df3 Mon Sep 17 00:00:00 2001 From: nagar-ajay Date: Fri, 27 Oct 2023 18:27:26 +0530 Subject: [PATCH] Image pull using docker credentials support (#17) * move ntnx pipeline overlays to overlays/pipeline directory * add support to pull docker images using docker secrets * add AWS_REGION in ml-pipeline-ui deployment * update README and website * use docker creds in kubeflow-user-example-com namespace --- kubeflow/README.md | 4 +- kubeflow/install.sh | 40 +++++- .../overlays/docker/docker-credentials.env | 4 + .../docker/service-account-patch.yaml | 135 ++++++++++++++++++ .../overlays/ntnx/pipeline-install-config.env | 3 - kubeflow/overlays/{ => pipeline}/ntnx/config | 0 .../{ => pipeline}/ntnx/kustomization.yaml | 0 .../ntnx/ntnx-config-patch.yaml | 5 + .../ntnx/object-store-secrets.env | 0 .../pipeline/ntnx/pipeline-install-config.env | 4 + .../ntnx/viewer-pod-template.json | 0 .../pipeline-kustomization.yaml | 0 website/content/en/docs/install-kubeflow.md | 4 +- 13 files changed, 186 insertions(+), 13 deletions(-) create mode 100644 kubeflow/overlays/docker/docker-credentials.env create mode 100644 kubeflow/overlays/docker/service-account-patch.yaml delete mode 100644 kubeflow/overlays/ntnx/pipeline-install-config.env rename kubeflow/overlays/{ => pipeline}/ntnx/config (100%) rename kubeflow/overlays/{ => pipeline}/ntnx/kustomization.yaml (100%) rename kubeflow/overlays/{ => pipeline}/ntnx/ntnx-config-patch.yaml (87%) rename kubeflow/overlays/{ => pipeline}/ntnx/object-store-secrets.env (100%) create mode 100644 kubeflow/overlays/pipeline/ntnx/pipeline-install-config.env rename kubeflow/overlays/{ => pipeline}/ntnx/viewer-pod-template.json (100%) rename kubeflow/overlays/{ => pipeline}/pipeline-kustomization.yaml (100%) diff --git a/kubeflow/README.md b/kubeflow/README.md index 1922ff1..55b18f6 100644 --- a/kubeflow/README.md +++ b/kubeflow/README.md @@ -17,8 +17,8 @@ #### Kubeflow on NKE * Configure the object store by replacing the following variables: - * put object store `accesskey` and `secretkey` in `overlays/ntnx/object-store-secrets.env` - * put `objStoreHost` in `overlays/ntnx/pipeline-install-config.env` + * put object store `accesskey` and `secretkey` in `overlays/pipeline/ntnx/object-store-secrets.env` + * put `objStoreHost` in `overlays/pipeline/ntnx/pipeline-install-config.env` * Run the following make command from the root of the github repo diff --git a/kubeflow/install.sh b/kubeflow/install.sh index d353367..ddf1e7d 100755 --- a/kubeflow/install.sh +++ b/kubeflow/install.sh @@ -1,18 +1,20 @@ #!/bin/bash -KF_VERSION=v1.8.0-rc.1 +KF_VERSION=v1.8.0-rc.4 helpFunction() { echo "" - echo "Usage: install.sh [OPTIONAL -v]" + echo "Usage: install.sh [OPTIONAL -v] [OPTIONAL -d]" echo "-v vanilla kubeflow" + echo "-d use docker credentials" exit 1 # Exit script after printing help } -while getopts ":v" option; do +while getopts "vd" option; do case $option in v ) vanilla_kubeflow="vanilla_kubeflow" ;; + d ) use_docker_creds="use_docker_creds" ;; ? ) helpFunction ;; esac done @@ -29,13 +31,39 @@ if [ -z "$vanilla_kubeflow" ] then echo "Using nutanix object store" # Patch kubeflow pipelines - cp overlays/pipeline-kustomization.yaml manifests/apps/pipeline/upstream/env/cert-manager/platform-agnostic-multi-user/kustomization.yaml + cp overlays/pipeline/pipeline-kustomization.yaml manifests/apps/pipeline/upstream/env/cert-manager/platform-agnostic-multi-user/kustomization.yaml mkdir -p manifests/apps/pipeline/upstream/env/ntnx - cp -r overlays/ntnx manifests/apps/pipeline/upstream/env + cp -r overlays/pipeline/ntnx manifests/apps/pipeline/upstream/env +fi + +if [ -n "$use_docker_creds" ] +then + echo "Using docker imagePullSecrets" + source overlays/docker/docker-credentials.env + kubectl create namespace kubeflow + kubectl create namespace istio-system + kubectl create secret docker-registry kf-docker-cred --docker-server=$DOCKER_SERVER --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL -n kubeflow + kubectl create secret docker-registry kf-docker-cred --docker-server=$DOCKER_SERVER --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL -n istio-system + kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "kf-docker-cred"}]}' -n kubeflow + + cp overlays/docker/service-account-patch.yaml manifests/example/service-account-patch.yaml + + cat << EOF >> manifests/example/kustomization.yaml + +patchesStrategicMerge: +- service-account-patch.yaml +EOF + fi # Install kubeflow while ! kustomize build manifests/example | kubectl apply -f -; do echo "Retrying to apply resources"; sleep 10; done # Remove kubeflow manifests -rm -rf manifests \ No newline at end of file +rm -rf manifests + +if [ -n "$use_docker_creds" ] +then + kubectl create secret docker-registry kf-docker-cred --docker-server=$DOCKER_SERVER --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL -n kubeflow-user-example-com + kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "kf-docker-cred"}]}' -n kubeflow-user-example-com +fi \ No newline at end of file diff --git a/kubeflow/overlays/docker/docker-credentials.env b/kubeflow/overlays/docker/docker-credentials.env new file mode 100644 index 0000000..1663fbf --- /dev/null +++ b/kubeflow/overlays/docker/docker-credentials.env @@ -0,0 +1,4 @@ +DOCKER_SERVER=https://index.docker.io/v1/ +DOCKER_USERNAME=YOUR_DOCKER_USERNAME +DOCKER_PASSWORD=YOUR_DOCKER_PASSWORD +DOCKER_EMAIL=YOUR_DOCKER_EMAIL diff --git a/kubeflow/overlays/docker/service-account-patch.yaml b/kubeflow/overlays/docker/service-account-patch.yaml new file mode 100644 index 0000000..3ce26ff --- /dev/null +++ b/kubeflow/overlays/docker/service-account-patch.yaml @@ -0,0 +1,135 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admission-webhook-service-account + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: centraldashboard + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: jupyter-web-app-service-account + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: katib-controller + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: katib-ui + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kserve-controller-manager + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kserve-models-web-app + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: meta-controller-service + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: notebook-controller-service-account + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: profiles-controller-service-account + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pvcviewer-controller-manager + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tensorboard-controller-controller-manager + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tensorboards-web-app-service-account + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: training-operator + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: volumes-web-app-service-account + namespace: kubeflow +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: istiod + namespace: istio-system +imagePullSecrets: +- name: kf-docker-cred +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: authservice + namespace: istio-system +imagePullSecrets: +- name: kf-docker-cred \ No newline at end of file diff --git a/kubeflow/overlays/ntnx/pipeline-install-config.env b/kubeflow/overlays/ntnx/pipeline-install-config.env deleted file mode 100644 index 08868f1..0000000 --- a/kubeflow/overlays/ntnx/pipeline-install-config.env +++ /dev/null @@ -1,3 +0,0 @@ -bucketName=mlpipeline -insecure=true -objStoreHost=YOUR_NTNX_OBJECT_STORE_HOST \ No newline at end of file diff --git a/kubeflow/overlays/ntnx/config b/kubeflow/overlays/pipeline/ntnx/config similarity index 100% rename from kubeflow/overlays/ntnx/config rename to kubeflow/overlays/pipeline/ntnx/config diff --git a/kubeflow/overlays/ntnx/kustomization.yaml b/kubeflow/overlays/pipeline/ntnx/kustomization.yaml similarity index 100% rename from kubeflow/overlays/ntnx/kustomization.yaml rename to kubeflow/overlays/pipeline/ntnx/kustomization.yaml diff --git a/kubeflow/overlays/ntnx/ntnx-config-patch.yaml b/kubeflow/overlays/pipeline/ntnx/ntnx-config-patch.yaml similarity index 87% rename from kubeflow/overlays/ntnx/ntnx-config-patch.yaml rename to kubeflow/overlays/pipeline/ntnx/ntnx-config-patch.yaml index c0db13b..72bc731 100644 --- a/kubeflow/overlays/ntnx/ntnx-config-patch.yaml +++ b/kubeflow/overlays/pipeline/ntnx/ntnx-config-patch.yaml @@ -30,6 +30,11 @@ spec: configMapKeyRef: name: pipeline-install-config key: objStoreHost + - name: AWS_REGION + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: objStoreRegion --- apiVersion: apps/v1 kind: Deployment diff --git a/kubeflow/overlays/ntnx/object-store-secrets.env b/kubeflow/overlays/pipeline/ntnx/object-store-secrets.env similarity index 100% rename from kubeflow/overlays/ntnx/object-store-secrets.env rename to kubeflow/overlays/pipeline/ntnx/object-store-secrets.env diff --git a/kubeflow/overlays/pipeline/ntnx/pipeline-install-config.env b/kubeflow/overlays/pipeline/ntnx/pipeline-install-config.env new file mode 100644 index 0000000..2c579f4 --- /dev/null +++ b/kubeflow/overlays/pipeline/ntnx/pipeline-install-config.env @@ -0,0 +1,4 @@ +bucketName=mlpipeline +insecure=true +objStoreHost=YOUR_NTNX_OBJECT_STORE_HOST +objStoreRegion=ap-northeast-1 \ No newline at end of file diff --git a/kubeflow/overlays/ntnx/viewer-pod-template.json b/kubeflow/overlays/pipeline/ntnx/viewer-pod-template.json similarity index 100% rename from kubeflow/overlays/ntnx/viewer-pod-template.json rename to kubeflow/overlays/pipeline/ntnx/viewer-pod-template.json diff --git a/kubeflow/overlays/pipeline-kustomization.yaml b/kubeflow/overlays/pipeline/pipeline-kustomization.yaml similarity index 100% rename from kubeflow/overlays/pipeline-kustomization.yaml rename to kubeflow/overlays/pipeline/pipeline-kustomization.yaml diff --git a/website/content/en/docs/install-kubeflow.md b/website/content/en/docs/install-kubeflow.md index 6a9f6f7..8a1128b 100644 --- a/website/content/en/docs/install-kubeflow.md +++ b/website/content/en/docs/install-kubeflow.md @@ -28,8 +28,8 @@ weight = 4 3. Configure the object store in kubeflow manifests: - * put object store `accesskey` and `secretkey` in `kubeflow/overlays/ntnx/object-store-secrets.env` - * put `objStoreHost` in `kubeflow/overlays/ntnx/pipeline-install-config.env` + * put object store `accesskey` and `secretkey` in `kubeflow/overlays/pipeline/ntnx/object-store-secrets.env` + * put `objStoreHost` in `kubeflow/overlays/pipeline/ntnx/pipeline-install-config.env` 4. Run the following make command from the root of the github repository.