add mlflow

catalogs/data/airbyte/README.md

@@ -6,7 +6,7 @@ This is a baseline, prod ready airbyte installation using Plural.  It includes a
 * RDS/Google Cloud Sql, Azure Flexible Server to handle postgres. This gives you a robust RDBMS service to hold airbyte's core transactional data.
 * Plural OIDC to handle authentication to Airbyte. Airbyte does not support this natively, and so we use oauth-proxy as a middleware to handle authentication.
 
-In addtion, there are a few common customizations you might want to do.
+In addition, there are a few common customizations you might want to do.
 
 ## Configure Basic Auth
 
@@ -80,4 +80,4 @@ when building your application.
 
 ## Contributing
 
-If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds 
+If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds 

catalogs/data/airbyte/airbyte-servicedeployment.yaml

@@ -28,6 +28,7 @@ spec:
     cluster: {{ context.cluster }}
     hostname: {{ context.hostname }}
     bucket: {{ context.bucket }}
+    region: {{ context.region }}
   clusterRef:
     kind: Cluster
     name: {{ context.cluster }}

catalogs/data/airbyte/helm/airbyte.yaml.liquid

@@ -11,7 +11,7 @@ global:
     type: S3
     storageSecretName: airbyte-airbyte-secrets
     s3:
-      region: us-east-2
+      region: {{ configuration.region }}
       authenticationType: credentials
       accessKeyId: {{ imports[imports_airbyte_key].access_key_id }}
       accessKeyIdSecretKey: AWS_ACCESS_KEY_ID

catalogs/data/mlflow/README.md

@@ -0,0 +1,7 @@
+# Mlflow
+
+This is a baseline, prod-ready Mlflow installation using Plural.
+
+## Contributing
+
+If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds.

catalogs/data/mlflow/helm/mlflow.yaml.liquid

@@ -0,0 +1,43 @@
+{% raw %}
+{% assign imports_mlflow_key =  'mlflow-' | append: configuration.cluster %}
+
+tracking:
+  enabled: true
+  podAnnotations:
+    security.plural.sh/oauth-env-secret: mlflow-proxy-config
+  podLabels:
+    security.plural.sh/inject-oauth-sidecar: "true"
+  auth:
+    enabled: false
+  service:
+    type: ClusterIP
+
+externalDatabase:
+  dialectDriver: "postgresql"
+  host: {{ imports[imports_mlflow_key].postgres_host }}
+  port: '5432'
+  user: 'mlflow'
+  database: 'mlflow'
+  authDatabase: 'mlflow_auth'
+  password: {{ imports[imports_airbyte_key].postgres_password }}
+
+externalS3:
+  host: {{ configuration.region }}.amazonaws.com
+  port: 443
+  useCredentialsInSecret: true
+  accessKeyID: {{imports[imports_airbyte_key].access_key_id}}
+  accessKeySecret: {{imports[imports_airbyte_key].secret_access_key}}
+  existingSecret: ~
+  existingSecretAccessKeyIDKey: AWS_ACCESS_KEY_ID
+  existingSecretKeySecretKey: AWS_SECRET_ACCESS_KEY
+  protocol: "https"
+  bucket: {{ configuration.bucket }}
+  serveArtifacts: true
+
+postgresql:
+  enabled: false
+
+minio:
+  enabled: false
+
+{% endraw %}

catalogs/data/mlflow/helm/oauth-proxy-config.yaml.liquid

@@ -0,0 +1,28 @@
+{% raw %}
+{% assign imports_mlflow_key =  'mlflow-' | append: configuration.cluster %}
+
+service:
+  name: mlflow-oauth2-proxy
+  selector:
+    app.kubernetes.io/instance: mlflow
+    app.kubernetes.io/name: webapp
+
+secret:
+  clientID: {{ imports[imports_mlflow_key].oidc_client_id }}
+  clientSecret: {{ imports[imports_mlflow_key].oidc_client_secret }}
+  cookieSecret: {{ imports[imports_mlflow_key].oidc_cookie_secret }}
+  issuer: https://oidc.plural.sh/
+  upstream: http://localhost:8080
+  name: mlflow-proxy-config
+  env:
+    OAUTH2_PROXY_UPSTREAM_TIMEOUT: '120s'
+
+{% if configuration["basicAuth"] %}
+{% assign basicAuth = configuration["basicAuth"] | from_json %}
+users:
+{% for user in basicAuth %}
+  {{ user[0] }}: {{ user[1] }}
+{% endfor %}
+{% endif %}
+
+{% endraw %}

catalogs/data/mlflow/mlflow-raw-servicedeployment.yaml

@@ -0,0 +1,20 @@
+apiVersion: deployments.plural.sh/v1alpha1
+kind: ServiceDeployment
+metadata:
+  name: mlflow-raw-{{ context.cluster }}
+  namespace: apps
+spec:
+  namespace: mlflow
+  git:
+    folder: services/apps/mlflow
+    ref: main
+  repositoryRef:
+    kind: GitRepository
+    name: infra
+    namespace: infra
+  configuration:
+    hostname: {{ context.hostname }}
+  clusterRef:
+    kind: Cluster
+    name: {{ context.cluster }}
+    namespace: infra

catalogs/data/mlflow/mlflow-servicedeployment.yaml

@@ -0,0 +1,35 @@
+apiVersion: deployments.plural.sh/v1alpha1
+kind: ServiceDeployment
+metadata:
+  name: mlflow-{{ context.cluster }}
+  namespace: apps
+spec:
+  namespace: mflow
+  git:
+    folder: helm/mlflow/{{ context.cluster }}
+    ref: main
+  repositoryRef:
+    kind: GitRepository
+    name: infra
+    namespace: infra
+  helm:
+    version: "2.x.x"
+    chart: mlflow
+    release: mlflow
+    ignoreHooks: false
+    url: https://charts.bitnami.com/bitnami
+    valuesFiles:
+      - mlflow.yaml.liquid
+  imports:
+    - stackRef:
+        name: mlflow-{{ context.cluster }}
+        namespace: apps
+  configuration:
+    cluster: {{ context.cluster }}
+    hostname: {{ context.hostname }}
+    bucket: {{ context.bucket }}
+    region: {{ context.region }}
+  clusterRef:
+    kind: Cluster
+    name: {{ context.cluster }}
+    namespace: infra

catalogs/data/mlflow/mlflow-stack.yaml

@@ -0,0 +1,22 @@
+apiVersion: deployments.plural.sh/v1alpha1
+kind: InfrastructureStack
+metadata:
+  name: mlflow-{{ context.cluster }}
+  namespace: apps
+spec:
+  detach: false
+  type: TERRAFORM
+  approval: true
+  manageState: true
+  actor: [email protected]
+  git:
+    ref: main
+    folder: terraform/apps/mlflow/{{ context.cluster }}
+  repositoryRef:
+    name: infra
+    namespace: infra
+  configuration:
+    version: '1.8'
+  clusterRef:
+    name: {{ context.cluster }}
+    namespace: infra

catalogs/data/mlflow/oauth-proxy-config-servicedeployment.yaml

@@ -0,0 +1,31 @@
+apiVersion: deployments.plural.sh/v1alpha1
+kind: ServiceDeployment
+metadata:
+  name: mlflow-oauth-proxy-config-{{ context.cluster }}
+  namespace: apps
+spec:
+  namespace: mlflow
+  git:
+    folder: helm/mlflow/{{ context.cluster }}
+    ref: main
+  repositoryRef:
+    kind: GitRepository
+    name: infra
+    namespace: infra
+  helm:
+    version: "x.x.x"
+    chart: oidc-config
+    url: https://pluralsh.github.io/module-library
+    valuesFiles:
+    - oauth-proxy-config.yaml.liquid
+  imports:
+  - stackRef:
+      name: mlflow-{{ context.cluster }}
+      namespace: apps
+  configuration:
+    cluster: {{ context.cluster }}
+    hostname: {{ context.hostname }}
+  clusterRef:
+    kind: Cluster
+    name: {{ context.cluster }}
+    namespace: infra

catalogs/data/mlflow/services/oauth-proxy-ingress.yaml.liquid

@@ -0,0 +1,33 @@
+{% raw %}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: mlflow-webapp-proxy
+  namespace: mlflow
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+    kubernetes.io/tls-acme: "true"
+    # Extend timeout to allow long running queries.
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/use-regex: "true"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: {{ configuration.hostname }}
+      http:
+        paths:
+          - backend:
+              service:
+                name: mlflow-oauth2-proxy
+                port:
+                  number: 80
+            path: /.*
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - {{ configuration.hostname }}
+      secretName: mlflow-tls
+{% endraw %}

catalogs/data/mlflow/terraform/aws/iam.tf

@@ -0,0 +1,37 @@
+
+resource "aws_iam_policy" "mlflow" {
+  name_prefix = "mlflow"
+  description = "policy for the plural admin mlflow"
+  policy      = data.aws_iam_policy_document.mlflow.json
+}
+
+resource "aws_iam_user" "mlflow" {
+  name = "${data.plural_cluster.cluster.name}-mlflow"
+
+  depends_on = [ data.plural_cluster.cluster ]
+}
+
+resource "aws_iam_access_key" "mlflow" {
+  user = aws_iam_user.mlflow.name
+}
+
+data "aws_iam_policy_document" "mlflow" {
+  statement {
+    sid    = "admin"
+    effect = "Allow"
+    actions = ["s3:*"]
+
+    resources = [
+      "arn:aws:s3:::${var.mlflow_bucket}",
+      "arn:aws:s3:::${var.mlflow_bucket}/*",
+    ]
+  }
+}
+
+resource "aws_iam_policy_attachment" "mlflow-user" {
+  name = "${data.plural_cluster.cluster.name}-mlflow-policy"
+  users = [aws_iam_user.mlflow.name]
+  policy_arn = aws_iam_policy.mlflow.arn
+
+  depends_on = [ data.plural_cluster.cluster ]
+}

catalogs/data/mlflow/terraform/aws/oidc.tf

@@ -0,0 +1,15 @@
+resource "random_password" "oidc_cookie" {
+  length      = 24
+  min_lower   = 1
+  min_numeric = 1
+  min_upper   = 1
+  special     = false
+}
+
+resource "plural_oidc_provider" "mlflow" {
+  name = "mlflow-{{ context.cluster }}"
+  auth_method = "BASIC"
+  type = "PLURAL"
+  description = "OIDC provider for mlflow deployed to the {{ context.cluster }} cluster"
+  redirect_uris = ["https://{{ context.hostname }}/oauth2/callback"]
+}

catalogs/data/mlflow/terraform/aws/outputs.tf

@@ -0,0 +1,32 @@
+output "access_key_id" {
+  value = aws_iam_access_key.mlflow.id
+}
+
+output "secret_access_key" {
+  value = aws_iam_access_key.mlflow.secret
+  sensitive = true
+}
+
+output "postgres_host" {
+  value = try(module.db.db_instance_address, "")
+}
+
+output "postgres_password" {
+  value = random_password.password.result
+  sensitive = true
+}
+
+output "oidc_cookie_secret" {
+  value = random_password.oidc_cookie.result
+  sensitive = true
+}
+
+output "oidc_client_id" {
+  value = plural_oidc_provider.mlflow.client_id
+  sensitive = true
+}
+
+output "oidc_client_secret" {
+  value = plural_oidc_provider.mlflow.client_secret
+  sensitive = true
+}

catalogs/data/mlflow/terraform/aws/plural.tf

@@ -0,0 +1,3 @@
+data "plural_cluster" "cluster" {
+  handle = var.cluster_name
+}