pipelines first pass without rbac changes

Signed-off-by: David van der Spek <[email protected]>

kubeflow/helm/pipelines/templates/api-server/deployment.yaml

@@ -143,6 +143,18 @@ spec:
             initialDelaySeconds: 3
             periodSeconds: 5
             timeoutSeconds: 2
+          startupProbe:
+            exec:
+              command:
+                - wget
+                - -q
+                - -S
+                - -O
+                - '-'
+                - http://localhost:8888/apis/v1beta1/healthz
+            failureThreshold: 12
+            periodSeconds: 5
+            timeoutSeconds: 2
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}

kubeflow/helm/pipelines/templates/argo-workflow-controller/deployment.yaml

@@ -38,7 +38,7 @@ spec:
             - --configmap
             - {{ include "pipelines.fullname" . }}-argo-workflow-controller
             - --executor-image
-            - gcr.io/ml-pipeline/argoexec:v3.2.3-license-compliance
+            - "{{ .Values.argoWorkflowController.executorImage.repository }}:{{ .Values.argoWorkflowController.executorImage.tag }}"
           command:
             - workflow-controller
           env:
@@ -64,6 +64,7 @@ spec:
             timeoutSeconds: 30
           resources:
             {{- toYaml .Values.argoWorkflowController.resources | nindent 12 }}
+      priorityClassName: {{ include "pipelines.fullname" . }}-argo-workflow-controller
       {{- with .Values.nodeSelector }}
       nodeSelector:
         kubernetes.io/os: linux

kubeflow/helm/pipelines/templates/argo-workflow-controller/priority-class.yaml

@@ -0,0 +1,7 @@
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: {{ include "pipelines.fullname" . }}-argo-workflow-controller
+  labels:
+    {{- include "pipelines.labels" . | nindent 4 }}
+value: 1000000

kubeflow/helm/pipelines/templates/cache/server/deployment.yaml

@@ -42,9 +42,20 @@ spec:
             - --db_user=$(DBCONFIG_USER)
             - --db_password=$(DBCONFIG_PASSWORD)
             - --namespace_to_watch=$(NAMESPACE_TO_WATCH)
+            - --listen_ports=$(WEBHOOK_PORT)
           env:
             - name: NAMESPACE_TO_WATCH
               value: ""
+            - name: DEFAULT_CACHE_STALENESS
+              valueFrom:
+                configMapKeyRef:
+                  key: defaultCacheStaleness
+                  name: pipeline-install-config
+            - name: MAXIMUM_CACHE_STALENESS
+              valueFrom:
+                configMapKeyRef:
+                  key: maximumCacheStaleness
+                  name: pipeline-install-config
             - name: CACHE_IMAGE
               valueFrom:
                 configMapKeyRef:
@@ -82,6 +93,8 @@ spec:
                 secretKeyRef:
                   key: PASSWORD
                   name: pipelines-db-user
+            - name: WEBHOOK_PORT
+              value: "8443"
           ports:
             - name: webhook-api
               containerPort: 8443

kubeflow/helm/pipelines/templates/configmap.yaml

@@ -1,21 +1,24 @@
 apiVersion: v1
 data:
   appName: pipeline
-  appVersion: 1.8.1
-  autoUpdatePipelineDefaultVersion: "true"
+  ConMaxLifeTime: 120s # TODO: make configurable
+  defaultCacheStaleness: "" # TODO: make configurable
+  maximumCacheStaleness: "" # TODO: make configurable
+  appVersion: {{ .Chart.AppVersion }}
+  autoUpdatePipelineDefaultVersion: "true" # TODO: make configurable
   bucketHost: {{ .Values.config.objectStore.bucketHost }}
   bucketRegion: {{ .Values.config.objectStore.bucketRegion }}
   bucketName: {{ .Values.config.objectStore.bucketName }}
   defaultPipelineRoot: s3://{{ .Values.config.objectStore.bucketName }}/pipelines
   cacheDb: {{ .Values.config.databases.cacheDB }}
-  cacheImage: gcr.io/google-containers/busybox
-  cacheNodeRestrictions: "false"
-  cronScheduleTimezone: UTC
+  cacheImage: gcr.io/google-containers/busybox # TODO: make configurable
+  cacheNodeRestrictions: "false" # TODO: make configurable
+  cronScheduleTimezone: UTC # TODO: make configurable
   dbHost: {{ .Values.config.databases.connection.host }}
   dbPort: {{ .Values.config.databases.connection.port | quote }}
-  dbConMaxLifeTime: 120s
-  dbDriverName: mysql
-  dbGroupConcatMaxLen: "4194304"
+  dbConMaxLifeTime: 120s # TODO: make configurable
+  dbDriverName: mysql # TODO: make configurable
+  dbGroupConcatMaxLen: "4194304" # TODO: make configurable
   mlmdDb: {{ .Values.config.databases.metadataDB }}
   pipelineDb: {{ .Values.config.databases.pipelineDB }}
 kind: ConfigMap
@@ -24,4 +27,4 @@ metadata:
     app.kubernetes.io/component: ml-pipeline
     app.kubernetes.io/name: kubeflow-pipelines
     application-crd-id: kubeflow-pipelines
-  name: pipeline-install-config
+  name: pipeline-install-config # TODO: don't hardcode name

kubeflow/helm/pipelines/templates/persistence-agent/configmap.yaml

@@ -13,3 +13,6 @@ data:
   API_SERVER_BASE_PATH: "/apis/v1beta1"
   MAX_CLIENT_QPS: "5"
   MAX_CLIENT_BURST: "10"
+  KUBEFLOW_USERID_HEADER: {{ .Values.global.userIDHeader }}
+  KUBEFLOW_USERID_PREFIX: {{ .Values.global.userIDPrefix }}
+  MULTIUSER: "true" # TODO: should this be configurable?

kubeflow/helm/pipelines/templates/persistence-agent/deployment.yaml

@@ -35,7 +35,7 @@ spec:
           image: "{{ .Values.persistenceAgent.image.repository }}:{{ .Values.persistenceAgent.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.persistenceAgent.image.pullPolicy }}
           args:
-            - --logtostderr=true
+            - --logtostderr=true # TODO: why are we specifying this while the upstream manifest doesn't?
             - --namespace=$(NAMESPACE)
             - --ttlSecondsAfterWorkflowFinish=$(TTL_SECONDS_AFTER_WORKFLOW_FINISH)
             - --numWorker=$(NUM_WORKERS)
@@ -47,11 +47,25 @@ spec:
             - --clientBurst=$(MAX_CLIENT_BURST)
           command:
             - persistence_agent
+          env:
+            - name: NAMESPACE # TODO: should this be specified? Or should it only be specified in single user mode?
+              value: ""
           envFrom:
             - configMapRef:
                 name: {{ include "pipelines.fullname" . }}-persistence-agent-parameters
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+            - mountPath: /var/run/secrets/kubeflow/tokens
+              name: persistenceagent-sa-token
+      volumes:
+        - name: persistenceagent-sa-token
+          projected:
+            sources:
+              - serviceAccountToken:
+                  audience: pipelines.kubeflow.org
+                  expirationSeconds: 3600
+                  path: persistenceagent-sa-token
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

kubeflow/helm/pipelines/values.yaml

@@ -35,7 +35,7 @@ image:
   repository: gcr.io/ml-pipeline/frontend
   pullPolicy: IfNotPresent
   # Overrides the image tag whose default is the chart appVersion.
-  tag: 1.8.1
+  tag: 2.0.1
 
 imagePullSecrets: []
 nameOverride: ""
@@ -80,7 +80,7 @@ apiServer:
     repository: gcr.io/ml-pipeline/api-server
     pullPolicy: IfNotPresent
     # Overrides the image tag whose default is the chart appVersion.
-    tag: 1.8.1
+    tag: 2.0.1
   service:
     http:
       port: 8888
@@ -92,28 +92,28 @@ persistenceAgent:
     repository: gcr.io/ml-pipeline/persistenceagent
     pullPolicy: IfNotPresent
     # Overrides the image tag whose default is the chart appVersion.
-    tag: 1.8.1
+    tag: 2.0.1
 
 scheduledWorkflow:
   image:
     repository: gcr.io/ml-pipeline/scheduledworkflow
     pullPolicy: IfNotPresent
     # Overrides the image tag whose default is the chart appVersion.
-    tag: 1.8.1
+    tag: 2.0.1
 
 viewerController:
   image:
     repository: gcr.io/ml-pipeline/viewer-crd-controller
     pullPolicy: IfNotPresent
     # Overrides the image tag whose default is the chart appVersion.
-    tag: 1.8.1
+    tag: 2.0.1
 
 visualizationServer:
   image:
     repository: gcr.io/ml-pipeline/visualization-server
     pullPolicy: IfNotPresent
     # Overrides the image tag whose default is the chart appVersion.
-    tag: 1.8.1
+    tag: 2.0.1
   service:
     port: 8888
 
@@ -122,7 +122,11 @@ argoWorkflowController:
     repository: gcr.io/ml-pipeline/workflow-controller
     pullPolicy: IfNotPresent
     # Overrides the image tag whose default is the chart appVersion.
-    tag: v3.2.3-license-compliance
+    tag: v3.3.10-license-compliance
+  executorImage:
+    repository: gcr.io/ml-pipeline/argoexec
+    pullPolicy: IfNotPresent
+    tag: v3.3.10-license-compliance
   resources:
     requests:
       cpu: 100m
@@ -154,23 +158,23 @@ cache:
       repository: gcr.io/ml-pipeline/cache-server
       pullPolicy: IfNotPresent
       # Overrides the image tag whose default is the chart appVersion.
-      tag: 1.8.1
+      tag: 2.0.1
     service:
       port: 443
   deployer:
     image:
       repository: gcr.io/ml-pipeline/cache-deployer
       pullPolicy: IfNotPresent
       # Overrides the image tag whose default is the chart appVersion.
-      tag: 1.8.1
+      tag: 2.0.1
 
 metadata:
   envoy:
     image:
       repository: gcr.io/ml-pipeline/metadata-envoy
       pullPolicy: IfNotPresent
       # Overrides the image tag whose default is the chart appVersion.
-      tag: 1.8.1
+      tag: 2.0.1
     service:
       port: 9090
       admin:
@@ -195,7 +199,7 @@ metadata:
       repository: gcr.io/ml-pipeline/metadata-writer
       pullPolicy: IfNotPresent
       # Overrides the image tag whose default is the chart appVersion.
-      tag: 1.8.1
+      tag: 2.0.1
 
 resources: {}
   # We usually recommend not to specify default resources and to leave this as a conscious