feat: add kustomize manifests for kubeflow, fixes RHOAIENG-1947 (kube…

…flow#292)
dhirajsb · Feb 19, 2024 · ec8618e · ec8618e
1 parent 78b0971
commit ec8618e
Show file tree

Hide file tree

Showing 23 changed files with 486 additions and 0 deletions.
diff --git a/manifests/kustomize/base/kustomization.yaml b/manifests/kustomize/base/kustomization.yaml
@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - model-registry-configmap.yaml
+  - model-registry-deployment.yaml
+  - model-registry-service.yaml
+  - model-registry-sa.yaml
+images:
+  - name: gcr.io/ml-pipeline/metadata-envoy
+    newTag: 2.0.5
diff --git a/manifests/kustomize/base/model-registry-configmap.yaml b/manifests/kustomize/base/model-registry-configmap.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: model-registry-configmap
+  labels:
+    component: model-registry-server
+data:
+  MODEL_REGISTRY_REST_SERVICE_HOST: "model-registry-service"
+  MODEL_REGISTRY_REST_SERVICE_PORT: "8080"
+  MODEL_REGISTRY_GRPC_SERVICE_HOST: "model-registry-service"
+  MODEL_REGISTRY_GRPC_SERVICE_PORT: "9090"
diff --git a/manifests/kustomize/base/model-registry-deployment.yaml b/manifests/kustomize/base/model-registry-deployment.yaml
@@ -0,0 +1,103 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: model-registry-deployment
+  labels:
+    component: model-registry-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: model-registry-server
+  template:
+    metadata:
+      labels:
+        component: model-registry-server
+    spec:
+      containers:
+        - name: rest-container
+          args:
+            - --hostname=0.0.0.0
+            - --port=8080
+            - --mlmd-hostname=localhost
+            - --mlmd-port=9090
+          command:
+            - /model-registry
+            - proxy
+          image: quay.io/opendatahub/model-registry:latest
+          # empty placeholder environment for patching
+          env: []
+          ports:
+            - name: http-api
+              containerPort: 8080
+          livenessProbe:
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            tcpSocket:
+              port: http-api
+            timeoutSeconds: 2
+          readinessProbe:
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            tcpSocket:
+              port: http-api
+            timeoutSeconds: 2
+        - name: grpc-container
+          # ! Sync to the same MLMD version:
+          # * backend/metadata_writer/requirements.in and requirements.txt
+          # * @kubeflow/frontend/src/mlmd/generated
+          # * .cloudbuild.yaml and .release.cloudbuild.yaml
+          # * manifests/kustomize/base/metadata/base/model-registry-deployment.yaml
+          # * test/tag_for_hosted.sh
+          image: gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0
+          env:
+          - name: DBCONFIG_USER
+            valueFrom:
+              secretKeyRef:
+                name: mysql-secret
+                key: username
+          - name: DBCONFIG_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: mysql-secret
+                key: password
+          - name: MYSQL_DATABASE
+            valueFrom:
+              configMapKeyRef:
+                name: pipeline-install-config
+                key: mlmdDb
+          - name: MYSQL_HOST
+            valueFrom:
+              configMapKeyRef:
+                name: pipeline-install-config
+                key: dbHost
+          - name: MYSQL_PORT
+            valueFrom:
+              configMapKeyRef:
+                name: pipeline-install-config
+                key: dbPort
+          command: ["/bin/metadata_store_server"]
+          args: ["--grpc_port=9090",
+                 "--mysql_config_database=$(MYSQL_DATABASE)",
+                 "--mysql_config_host=$(MYSQL_HOST)",
+                 "--mysql_config_port=$(MYSQL_PORT)",
+                 "--mysql_config_user=$(DBCONFIG_USER)",
+                 "--mysql_config_password=$(DBCONFIG_PASSWORD)",
+                 "--enable_database_upgrade=true"
+                 ]
+          ports:
+          - name: grpc-api
+            containerPort: 9090
+          livenessProbe:
+            tcpSocket:
+              port: grpc-api
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
+          readinessProbe:
+            tcpSocket:
+              port: grpc-api
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
+      serviceAccountName: model-registry-server
diff --git a/manifests/kustomize/base/model-registry-sa.yaml b/manifests/kustomize/base/model-registry-sa.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: model-registry-server
diff --git a/manifests/kustomize/base/model-registry-service.yaml b/manifests/kustomize/base/model-registry-service.yaml
@@ -0,0 +1,17 @@
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+    app: metadata
+  name: model-registry-service
+spec:
+  selector:
+    component: model-registry-server
+  type: ClusterIP
+  ports:
+  - port: 8080
+    protocol: TCP
+    name: http-api
+  - port: 9090
+    protocol: TCP
+    name: grpc-api
diff --git a/manifests/kustomize/options/istio/destination-rule.yaml b/manifests/kustomize/options/istio/destination-rule.yaml
@@ -0,0 +1,9 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: DestinationRule
+metadata:
+  name: model-registry-service
+spec:
+  host: model-registry-service.kubeflow.svc.cluster.local
+  trafficPolicy:
+    tls:
+      mode: ISTIO_MUTUAL
diff --git a/manifests/kustomize/options/istio/istio-authorization-policy.yaml b/manifests/kustomize/options/istio/istio-authorization-policy.yaml
@@ -0,0 +1,11 @@
+apiVersion: security.istio.io/v1beta1
+kind: AuthorizationPolicy
+metadata:
+  name: model-registry-service
+spec:
+  action: ALLOW
+  selector:
+    matchLabels:
+      component: model-registry-server
+  rules:
+  - {}
diff --git a/manifests/kustomize/options/istio/kustomization.yaml b/manifests/kustomize/options/istio/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- istio-authorization-policy.yaml
+- destination-rule.yaml
+- virtual-service.yaml
diff --git a/manifests/kustomize/options/istio/virtual-service.yaml b/manifests/kustomize/options/istio/virtual-service.yaml
@@ -0,0 +1,21 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: model-registry
+  namespace: kubeflow
+spec:
+  gateways:
+  - kubeflow-gateway
+  hosts:
+  - '*'
+  http:
+  - match:
+    - uri:
+        prefix: /ml_metadata
+    rewrite:
+      uri: /ml_metadata
+    route:
+    - destination:
+        host: metadata-envoy-service.kubeflow.svc.cluster.local
+        port:
+          number: 9090
diff --git a/manifests/kustomize/overlays/db/kustomization.yaml b/manifests/kustomize/overlays/db/kustomization.yaml
@@ -0,0 +1,38 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+
+resources:
+- model-registry-db-pvc.yaml
+- model-registry-db-deployment.yaml
+- model-registry-db-service.yaml
+- ../../base
+
+patchesStrategicMerge:
+- patches/model-registry-deployment.yaml
+
+configMapGenerator:
+- envs:
+  - params.env
+  name: model-registry-db-parameters
+secretGenerator:
+- envs:
+  - secrets.env
+  name: model-registry-db-secrets
+generatorOptions:
+  disableNameSuffixHash: true
+
+
+images:
+- name: mysql
+  newName: mysql
+  newTag: 8.0.3
+
+vars:
+- fieldref:
+    fieldPath: metadata.name
+  name: MLMD_DB_HOST
+  objref:
+    apiVersion: v1
+    kind: Service
+    name: model-registry-db
diff --git a/manifests/kustomize/overlays/db/model-registry-db-deployment.yaml b/manifests/kustomize/overlays/db/model-registry-db-deployment.yaml
@@ -0,0 +1,52 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: model-registry-db
+  labels:
+    component: db
+spec:
+  selector:
+    matchLabels:
+      component: db
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      name: db
+      labels:
+        component: db
+      annotations:
+        sidecar.istio.io/inject: "false"
+    spec:
+      containers:
+      - name: db-container
+        image: mysql:8.0.3
+        args:
+        - --datadir
+        - /var/lib/mysql/datadir
+        - --default-authentication-plugin=mysql_native_password
+        envFrom:
+        - configMapRef:
+            name: model-registry-db-parameters
+        - secretRef:
+            name: model-registry-db-secrets
+        ports:
+        - name: dbapi
+          containerPort: 3306
+        readinessProbe:
+          exec:
+            command:
+            - "/bin/bash"
+            - "-c"
+            - "mysql -D $$MYSQL_DATABASE -u$$MYSQL_USER_NAME -p$$MYSQL_ROOT_PASSWORD -e 'SELECT 1'"
+          initialDelaySeconds: 5
+          periodSeconds: 2
+          timeoutSeconds: 1
+        volumeMounts:
+        - name: metadata-mysql
+          mountPath: /var/lib/mysql
+      volumes:
+      - name: metadata-mysql
+        persistentVolumeClaim:
+          claimName: metadata-mysql
diff --git a/manifests/kustomize/overlays/db/model-registry-db-pvc.yaml b/manifests/kustomize/overlays/db/model-registry-db-pvc.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: metadata-mysql
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
diff --git a/manifests/kustomize/overlays/db/model-registry-db-service.yaml b/manifests/kustomize/overlays/db/model-registry-db-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: model-registry-db
+  labels:
+    component: db
+spec:
+  type: ClusterIP
+  ports:
+    - port: 3306
+      protocol: TCP
+      name: dbapi
+  selector:
+    component: db
diff --git a/manifests/kustomize/overlays/db/params.env b/manifests/kustomize/overlays/db/params.env
@@ -0,0 +1,3 @@
+MYSQL_DATABASE=metadb
+MYSQL_PORT=3306
+MYSQL_ALLOW_EMPTY_PASSWORD=true
diff --git a/manifests/kustomize/overlays/db/patches/model-registry-deployment.yaml b/manifests/kustomize/overlays/db/patches/model-registry-deployment.yaml
@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: model-registry-deployment
+spec:
+  template:
+    spec:
+      containers:
+        - name: rest-container
+          # Remove existing environment variables
+          env:
+          - $patch: replace
+          envFrom:
+          - configMapRef:
+              name: model-registry-configmap
+          args:
+            - --hostname=0.0.0.0
+            - --port=$(MODEL_REGISTRY_REST_SERVICE_PORT)
+            - --mlmd-hostname=localhost
+            - --mlmd-port=$(MODEL_REGISTRY_GRPC_SERVICE_PORT)
+        - name: grpc-container
+          # Remove existing environment variables
+          env:
+          - $patch: replace
+          envFrom:
+          - configMapRef:
+              name: model-registry-db-parameters
+          - secretRef:
+              name: model-registry-db-secrets
+          - configMapRef:
+              name: model-registry-configmap
+          args: ["--grpc_port=$(MODEL_REGISTRY_GRPC_SERVICE_PORT)",
+                 "--mysql_config_host=$(MLMD_DB_HOST)",
+                 "--mysql_config_database=$(MYSQL_DATABASE)",
+                 "--mysql_config_port=$(MYSQL_PORT)",
+                 "--mysql_config_user=$(MYSQL_USER_NAME)",
+                 "--mysql_config_password=$(MYSQL_ROOT_PASSWORD)"]
diff --git a/manifests/kustomize/overlays/db/secrets.env b/manifests/kustomize/overlays/db/secrets.env
@@ -0,0 +1,2 @@
+MYSQL_USER_NAME=root
+MYSQL_ROOT_PASSWORD=test