diff --git a/.github/workflows/test-catalogs.yaml b/.github/workflows/test-catalogs.yaml index 5ae8e4f2..354dde31 100644 --- a/.github/workflows/test-catalogs.yaml +++ b/.github/workflows/test-catalogs.yaml @@ -18,6 +18,6 @@ jobs: - uses: extractions/setup-just@v2 - uses: pluralsh/setup-plural@v0.4.0 with: - vsn: 0.9.24 + vsn: 0.11.0 - run: just test diff --git a/catalogs/data/airbyte/README.md b/catalogs/data/airbyte/README.md index 41dbfc5e..d273998b 100644 --- a/catalogs/data/airbyte/README.md +++ b/catalogs/data/airbyte/README.md @@ -6,7 +6,7 @@ This is a baseline, prod ready airbyte installation using Plural. It includes a * RDS/Google Cloud Sql, Azure Flexible Server to handle postgres. This gives you a robust RDBMS service to hold airbyte's core transactional data. * Plural OIDC to handle authentication to Airbyte. Airbyte does not support this natively, and so we use oauth-proxy as a middleware to handle authentication. -In addtion, there are a few common customizations you might want to do. +In addition, there are a few common customizations you might want to do. ## Configure Basic Auth @@ -80,4 +80,4 @@ when building your application. ## Contributing -If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds \ No newline at end of file +If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds diff --git a/catalogs/data/airbyte/airbyte-raw-servicedeployment.yaml b/catalogs/data/airbyte/airbyte-raw-servicedeployment.yaml new file mode 100644 index 00000000..9a2aaf3a --- /dev/null +++ b/catalogs/data/airbyte/airbyte-raw-servicedeployment.yaml @@ -0,0 +1,20 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: airbyte-raw-{{ context.cluster }} + namespace: apps +spec: + namespace: airbyte + git: + folder: services/apps/airbyte + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + configuration: + hostname: {{ context.hostname }} + clusterRef: + kind: Cluster + name: {{ context.cluster }} + namespace: infra diff --git a/catalogs/data/airbyte/airbyte-servicedeployment.yaml b/catalogs/data/airbyte/airbyte-servicedeployment.yaml new file mode 100644 index 00000000..94ee5553 --- /dev/null +++ b/catalogs/data/airbyte/airbyte-servicedeployment.yaml @@ -0,0 +1,35 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: airbyte-{{ context.cluster }} + namespace: apps +spec: + namespace: airbyte + git: + folder: helm/airbyte/{{ context.cluster }} + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + helm: + version: "1.x.x" + chart: airbyte + release: airbyte + ignoreHooks: false + url: https://airbytehq.github.io/helm-charts + valuesFiles: + - airbyte.yaml.liquid + imports: + - stackRef: + name: airbyte-{{ context.cluster }} + namespace: apps + configuration: + cluster: {{ context.cluster }} + hostname: {{ context.hostname }} + bucket: {{ context.bucket }} + region: {{ context.region }} + clusterRef: + kind: Cluster + name: {{ context.cluster }} + namespace: infra diff --git a/test/outputs/catalogs/data/airbyte/stack.yaml.liquid b/catalogs/data/airbyte/airbyte-stack.yaml similarity index 76% rename from test/outputs/catalogs/data/airbyte/stack.yaml.liquid rename to catalogs/data/airbyte/airbyte-stack.yaml index 94c96633..a2abd4e7 100644 --- a/test/outputs/catalogs/data/airbyte/stack.yaml.liquid +++ b/catalogs/data/airbyte/airbyte-stack.yaml @@ -2,20 +2,21 @@ apiVersion: deployments.plural.sh/v1alpha1 kind: InfrastructureStack metadata: name: airbyte-{{ context.cluster }} + namespace: apps spec: detach: false type: TERRAFORM approval: true manageState: true actor: console@plural.sh - configuration: - version: '1.8' + git: + ref: main + folder: terraform/apps/airbyte/{{ context.cluster }} repositoryRef: name: infra namespace: infra + configuration: + version: '1.8' clusterRef: - name: mgmt + name: {{ context.stackCluster }} namespace: infra - git: - ref: main - folder: terraform/apps/airbyte/{{ context.cloud }} \ No newline at end of file diff --git a/catalogs/data/airbyte/helm/airbyte.yaml.liquid b/catalogs/data/airbyte/helm/airbyte.yaml.liquid new file mode 100644 index 00000000..c4dfff80 --- /dev/null +++ b/catalogs/data/airbyte/helm/airbyte.yaml.liquid @@ -0,0 +1,57 @@ +{% raw %} +{% assign imports_airbyte_key = 'airbyte-' | append: configuration.cluster %} + +global: + deploymentMode: oss + edition: community + + airbyteUrl: {{ configuration.hostname }} + + storage: + type: S3 + storageSecretName: airbyte-airbyte-secrets + s3: + region: {{ configuration.region }} + authenticationType: credentials + accessKeyId: {{ imports[imports_airbyte_key].access_key_id }} + accessKeyIdSecretKey: AWS_ACCESS_KEY_ID + secretAccessKey: {{ imports[imports_airbyte_key].secret_access_key }} + secretAccessKeySecretKey: AWS_SECRET_ACCESS_KEY + bucket: + log: {{ configuration.bucket }} + state: {{ configuration.bucket }} + workloadOutput: {{ configuration.bucket }} + + database: + type: external + database: airbyte + host: {{ imports[imports_airbyte_key].postgres_host }} + port: "5432" + secretName: airbyte-airbyte-secrets + user: airbyte + userSecretKey: DATABASE_USER + password: {{ imports[imports_airbyte_key].postgres_password }} + passwordSecretKey: DATABASE_PASSWORD + +postgresql: + enabled: false + +externalDatabase: + database: airbyte + host: {{ imports[imports_airbyte_key].postgres_host }} + user: airbyte + existingSecret: ~ + password: {{ imports[imports_airbyte_key].postgres_password }} + port: 5432 + +webapp: + ingress: + enabled: false + podAnnotations: + security.plural.sh/oauth-env-secret: airbyte-proxy-config + {% if configuration["basicAuth"] %} + security.plural.sh/htpasswd-secret: httpaswd-users + {% endif %} + podLabels: + security.plural.sh/inject-oauth-sidecar: "true" +{% endraw %} diff --git a/catalogs/data/airbyte/helm/oauth-proxy-config.yaml.liquid b/catalogs/data/airbyte/helm/oauth-proxy-config.yaml.liquid new file mode 100644 index 00000000..d0168f3e --- /dev/null +++ b/catalogs/data/airbyte/helm/oauth-proxy-config.yaml.liquid @@ -0,0 +1,28 @@ +{% raw %} +{% assign imports_airbyte_key = 'airbyte-' | append: configuration.cluster %} + +service: + name: airbyte-oauth2-proxy + selector: + app.kubernetes.io/instance: airbyte + app.kubernetes.io/name: webapp + +secret: + clientID: {{ imports[imports_airbyte_key].oidc_client_id }} + clientSecret: {{ imports[imports_airbyte_key].oidc_client_secret }} + cookieSecret: {{ imports[imports_airbyte_key].oidc_cookie_secret }} + issuer: https://oidc.plural.sh/ + upstream: http://localhost:8080 + name: airbyte-proxy-config + env: + OAUTH2_PROXY_UPSTREAM_TIMEOUT: '120s' + +{% if configuration["basicAuth"] %} +{% assign basicAuth = configuration["basicAuth"] | from_json %} +users: +{% for user in basicAuth %} + {{ user[0] }}: {{ user[1] }} +{% endfor %} +{% endif %} + +{% endraw %} diff --git a/catalogs/data/airbyte/helm/values.yaml.liquid b/catalogs/data/airbyte/helm/values.yaml.liquid deleted file mode 100644 index 12263d0d..00000000 --- a/catalogs/data/airbyte/helm/values.yaml.liquid +++ /dev/null @@ -1,89 +0,0 @@ -{% raw %} -global: - deploymentMode: "oss" - application: - links: - - description: airbyte web ui - url: {{ context.hostname }} - logs: - accessKey: - existingSecret: airbyte-airbyte-secrets - existingSecretKey: AWS_ACCESS_KEY_ID - password: {{ imports["airbyte-{{ context.cluster }}"].access_key_id }} - s3: - bucket: plrl-leadstreams-plrl-mgmt-airbyte - bucketRegion: us-east-2 - enabled: true - secretKey: - existingSecret: airbyte-airbyte-secrets - existingSecretKey: AWS_SECRET_ACCESS_KEY - password: {{ imports["airbyte-{{ context.cluster }}"].secret_access_key }} - storage: - type: S3 - state: - storage: - type: S3 - - database: - secretName: airbyte-airbyte-secrets - secretValue: DATABASE_PASSWORD - host: {{ imports["airbyte-{{ context.cluster }}"].postgres_host }} - -airbyte: - externalDatabase: - database: airbyte - host: {{ imports["airbyte-{{ context.cluster }}"].postgres_host }} - user: airbyte - existingSecret: ~ - password: {{ imports["airbyte-{{ context.cluster }}"].postgres_password }} - port: 5432 - webapp: - ingress: - hosts: - - host: {{ context.hostname }} - paths: - - path: /.* - pathType: ImplementationSpecific - tls: - - hosts: - - {{ context.hostname }} - secretName: airbyte-tls - podAnnotations: - security.plural.sh/oauth-env-secret: airbyte-proxy-config - {% if configuration["basicAuth"] %} - security.plural.sh/htpasswd-secret: httpaswd-users - {% endif %} - podLabels: - security.plural.sh/inject-oauth-sidecar: "true" -oidc-config: - enabled: true - secret: - clientID: {{ imports["airbyte-{{ context.cluster }}"].oidc_client_id }} - clientSecret: {{ imports["airbyte-{{ context.cluster }}"].oidc_client_secret }} - cookieSecret: {{ imports["airbyte-{{ context.cluster }}"].oidc_cookie_secret }} - issuer: https://oidc.plural.sh/ - name: airbyte-proxy-config - -{% if configuration["basicAuth"] %} -{% assign basicAuth = configuration["basicAuth"] | from_json %} - users: - {% for user in basicAuth %} - {{ user[0] }}: {{ user[1] }} - {% endfor %} -{% endif %} - -postgres: - enabled: false -private: - ingress: - enabled: true - hosts: - - host: {{ context.apiHostname }} - paths: - - path: /.* - pathType: ImplementationSpecific - tls: - - hosts: - - {{ context.apiHostname }} - secretName: airbyte-private-tls -{% endraw %} \ No newline at end of file diff --git a/catalogs/data/airbyte/servicedeployment.yaml.liquid b/catalogs/data/airbyte/oauth-proxy-config-servicedeployment.yaml similarity index 53% rename from catalogs/data/airbyte/servicedeployment.yaml.liquid rename to catalogs/data/airbyte/oauth-proxy-config-servicedeployment.yaml index 020d16e8..218a62f7 100644 --- a/catalogs/data/airbyte/servicedeployment.yaml.liquid +++ b/catalogs/data/airbyte/oauth-proxy-config-servicedeployment.yaml @@ -1,12 +1,12 @@ apiVersion: deployments.plural.sh/v1alpha1 kind: ServiceDeployment metadata: - name: airbyte-{{ context.cluster }} - namespace: infra + name: airbyte-oauth-proxy-config-{{ context.cluster }} + namespace: apps spec: namespace: airbyte git: - folder: helm/airbyte + folder: helm/airbyte/{{ context.cluster }} ref: main repositoryRef: kind: GitRepository @@ -14,14 +14,17 @@ spec: namespace: infra helm: version: "x.x.x" - chart: airbyte - url: https://app.plural.sh/cm/airbyte + chart: oidc-config + url: https://pluralsh.github.io/module-library valuesFiles: - - {{ context.cluster }}.yaml.liquid + - oauth-proxy-config.yaml.liquid imports: - stackRef: name: airbyte-{{ context.cluster }} - namespace: infra + namespace: apps + configuration: + cluster: {{ context.cluster }} + hostname: {{ context.hostname }} clusterRef: kind: Cluster name: {{ context.cluster }} diff --git a/catalogs/data/airbyte/services/oauth-proxy-ingress.yaml.liquid b/catalogs/data/airbyte/services/oauth-proxy-ingress.yaml.liquid new file mode 100644 index 00000000..2ba7d998 --- /dev/null +++ b/catalogs/data/airbyte/services/oauth-proxy-ingress.yaml.liquid @@ -0,0 +1,33 @@ +{% raw %} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: airbyte-webapp-proxy + namespace: airbyte + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + kubernetes.io/tls-acme: "true" + # Extend timeout to allow long running queries. + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/use-regex: "true" +spec: + ingressClassName: nginx + rules: + - host: {{ configuration.hostname }} + http: + paths: + - backend: + service: + name: airbyte-oauth2-proxy + port: + number: 80 + path: /.* + pathType: ImplementationSpecific + tls: + - hosts: + - {{ configuration.hostname }} + secretName: airbyte-tls +{% endraw %} diff --git a/catalogs/data/airbyte/terraform/aws/iam.tf b/catalogs/data/airbyte/terraform/aws/iam.tf index 5f199f73..c8afdc5e 100644 --- a/catalogs/data/airbyte/terraform/aws/iam.tf +++ b/catalogs/data/airbyte/terraform/aws/iam.tf @@ -6,7 +6,9 @@ resource "aws_iam_policy" "airbyte" { } resource "aws_iam_user" "airbyte" { - name = "${var.cluster_name}-airbyte" + name = "${data.plural_cluster.cluster.name}-airbyte" + + depends_on = [ data.plural_cluster.cluster ] } resource "aws_iam_access_key" "airbyte" { @@ -27,7 +29,9 @@ data "aws_iam_policy_document" "airbyte" { } resource "aws_iam_policy_attachment" "airbyte-user" { - name = "${var.cluster_name}-airbyte-policy" + name = "${data.plural_cluster.cluster.name}-airbyte-policy" users = [aws_iam_user.airbyte.name] policy_arn = aws_iam_policy.airbyte.arn + + depends_on = [ data.plural_cluster.cluster ] } diff --git a/catalogs/data/airbyte/terraform/aws/oidc.tf b/catalogs/data/airbyte/terraform/aws/oidc.tf index ca2084d8..8af5cc3a 100644 --- a/catalogs/data/airbyte/terraform/aws/oidc.tf +++ b/catalogs/data/airbyte/terraform/aws/oidc.tf @@ -1,5 +1,5 @@ resource "random_password" "oidc_cookie" { - length = 20 + length = 24 min_lower = 1 min_numeric = 1 min_upper = 1 @@ -12,4 +12,4 @@ resource "plural_oidc_provider" "airbyte" { type = "PLURAL" description = "OIDC provider for airbyte deployed to the {{ context.cluster }} cluster" redirect_uris = ["https://{{ context.hostname }}/oauth2/callback"] -} \ No newline at end of file +} diff --git a/catalogs/data/airbyte/terraform/aws/outputs.tf b/catalogs/data/airbyte/terraform/aws/outputs.tf index 9e83e5c1..9466a3df 100644 --- a/catalogs/data/airbyte/terraform/aws/outputs.tf +++ b/catalogs/data/airbyte/terraform/aws/outputs.tf @@ -4,6 +4,7 @@ output "access_key_id" { output "secret_access_key" { value = aws_iam_access_key.airbyte.secret + sensitive = true } output "postgres_host" { @@ -28,4 +29,4 @@ output "oidc_client_id" { output "oidc_client_secret" { value = plural_oidc_provider.airbyte.client_secret sensitive = true -} \ No newline at end of file +} diff --git a/catalogs/data/airbyte/terraform/aws/plural.tf b/catalogs/data/airbyte/terraform/aws/plural.tf new file mode 100644 index 00000000..c30d03d4 --- /dev/null +++ b/catalogs/data/airbyte/terraform/aws/plural.tf @@ -0,0 +1,3 @@ +data "plural_cluster" "cluster" { + handle = var.cluster_name +} diff --git a/catalogs/data/airbyte/terraform/aws/postgres.tf b/catalogs/data/airbyte/terraform/aws/postgres.tf index 6099d659..8c339459 100644 --- a/catalogs/data/airbyte/terraform/aws/postgres.tf +++ b/catalogs/data/airbyte/terraform/aws/postgres.tf @@ -7,7 +7,9 @@ resource "random_password" "password" { } data "aws_eks_cluster" "mgmt" { - name = var.cluster_name + name = data.plural_cluster.cluster.name + + depends_on = [ data.plural_cluster.cluster ] } data "aws_vpc" "mgmt" { diff --git a/catalogs/data/airbyte/terraform/aws/s3.tf b/catalogs/data/airbyte/terraform/aws/s3.tf index 5f6d8fc8..cba4d9c1 100644 --- a/catalogs/data/airbyte/terraform/aws/s3.tf +++ b/catalogs/data/airbyte/terraform/aws/s3.tf @@ -12,7 +12,3 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "airbyte" { } } } - -data "aws_iam_role" "postgres" { - name = "${var.cluster_name}-postgres" -} diff --git a/catalogs/data/dagster/README.md b/catalogs/data/dagster/README.md new file mode 100644 index 00000000..0216da2a --- /dev/null +++ b/catalogs/data/dagster/README.md @@ -0,0 +1,7 @@ +# Dagster + +This is a baseline, prod-ready Dagster installation using Plural. + +## Contributing + +If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds. \ No newline at end of file diff --git a/catalogs/data/dagster/dagster-aws-env.yaml b/catalogs/data/dagster/dagster-aws-env.yaml new file mode 100644 index 00000000..207fb089 --- /dev/null +++ b/catalogs/data/dagster/dagster-aws-env.yaml @@ -0,0 +1,24 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: dagster-aws-env-{{ context.cluster }} + namespace: apps +spec: + namespace: dagster + git: + folder: services/apps/dagster + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + imports: + - stackRef: + name: dagster-{{ context.cluster }} + namespace: apps + configuration: + cluster: {{ context.cluster }} + clusterRef: + kind: Cluster + name: {{ context.cluster }} + namespace: infra diff --git a/catalogs/data/dagster/dagster.yaml b/catalogs/data/dagster/dagster.yaml new file mode 100644 index 00000000..0dcde1c3 --- /dev/null +++ b/catalogs/data/dagster/dagster.yaml @@ -0,0 +1,55 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: InfrastructureStack +metadata: + name: dagster-{{ context.cluster }} + namespace: apps +spec: + detach: false + type: TERRAFORM + approval: true + manageState: true + actor: console@plural.sh + git: + ref: main + folder: terraform/apps/dagster/{{ context.cluster }} + repositoryRef: + name: infra + namespace: infra + configuration: + version: '1.8' + clusterRef: + name: {{ context.stackCluster }} + namespace: infra +--- +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: dagster-{{ context.cluster }} + namespace: apps +spec: + namespace: dagster + git: + folder: helm/dagster/{{ context.cluster }} + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + helm: + url: https://dagster-io.github.io/helm + version: "1.x.x" + chart: dagster + valuesFiles: + - dagster.yaml.liquid + imports: + - stackRef: + name: dagster-{{ context.cluster }} + namespace: apps + configuration: + cluster: {{ context.cluster }} + hostname: {{ context.hostname }} + bucket: {{ context.bucket }} + clusterRef: + kind: Cluster + name: {{ context.cluster }} + namespace: infra diff --git a/catalogs/data/dagster/helm/dagster.yaml.liquid b/catalogs/data/dagster/helm/dagster.yaml.liquid new file mode 100644 index 00000000..cf0aa8b2 --- /dev/null +++ b/catalogs/data/dagster/helm/dagster.yaml.liquid @@ -0,0 +1,70 @@ +{% raw %} +{% assign imports_dagster_key = 'dagster-' | append: configuration.cluster %} + +dagsterWebserver: + annotations: + security.plural.sh/oauth-env-secret: dagster-proxy-config + {% if configuration["basicAuth"] %} + security.plural.sh/htpasswd-secret: httpaswd-users + {% endif %} + labels: + security.plural.sh/inject-oauth-sidecar: "true" + envSecrets: + - name: dagster-aws-env + +postgresql: + enabled: false + postgresqlHost: {{ imports[imports_dagster_key].postgres_host }} + postgresqlUsername: dagster + postgresqlPassword: {{ imports[imports_dagster_key].postgres_password }} + postgresqlDatabase: dagster + postgresqlParams: + sslmode: require + +computeLogManager: + type: S3ComputeLogManager + config: + s3ComputeLogManager: + bucket: {{ configuration.bucket }} + +ingress: + enabled: true + ingressClassName: nginx + dagsterWebserver: + host: {{ configuration.hostname }} + tls: + enabled: true + secretName: dagster-tls + precedingPaths: + - path: /.* + pathType: ImplementationSpecific + serviceName: dagster-oauth2-proxy + servicePort: http-oauth + annotations: + kubernetes.io/tls-acme: "true" + cert-manager.io/cluster-issuer: letsencrypt-prod + # Extend timeout to allow long running queries. + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + nginx.ingress.kubernetes.io/use-regex: "true" + +runLauncher: + config: + k8sRunLauncher: + envSecrets: + - name: dagster-aws-env + +dagster-user-deployments: + enabled: true + deployments: + - name: "k8s-example-user-code-1" + image: + repository: "docker.io/dagster/user-code-example" + tag: latest + pullPolicy: Always + dagsterApiGrpcArgs: + - "--python-file" + - "/example_project/example_repo/repo.py" + port: 3030 +{% endraw %} diff --git a/catalogs/data/dagster/helm/oauth-proxy-config.yaml.liquid b/catalogs/data/dagster/helm/oauth-proxy-config.yaml.liquid new file mode 100644 index 00000000..fca62409 --- /dev/null +++ b/catalogs/data/dagster/helm/oauth-proxy-config.yaml.liquid @@ -0,0 +1,28 @@ +{% raw %} +{% assign imports_dagster_key = 'dagster-' | append: configuration.cluster %} + +service: + name: dagster-oauth2-proxy + selector: + app.kubernetes.io/name: dagster + component: dagster-webserver + +secret: + clientID: {{ imports[imports_dagster_key].oidc_client_id }} + clientSecret: {{ imports[imports_dagster_key].oidc_client_secret }} + cookieSecret: {{ imports[imports_dagster_key].oidc_cookie_secret }} + issuer: https://oidc.plural.sh/ + upstream: http://localhost:80 + name: dagster-proxy-config + env: + OAUTH2_PROXY_UPSTREAM_TIMEOUT: '120s' + +{% if configuration["basicAuth"] %} +{% assign basicAuth = configuration["basicAuth"] | from_json %} +users: +{% for user in basicAuth %} + {{ user[0] }}: {{ user[1] }} +{% endfor %} +{% endif %} + +{% endraw %} diff --git a/catalogs/data/dagster/oauth-proxy-config.yaml b/catalogs/data/dagster/oauth-proxy-config.yaml new file mode 100644 index 00000000..be247072 --- /dev/null +++ b/catalogs/data/dagster/oauth-proxy-config.yaml @@ -0,0 +1,31 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: dagster-oauth-proxy-config-{{ context.cluster }} + namespace: apps +spec: + namespace: dagster + git: + folder: helm/dagster/{{ context.cluster }} + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + helm: + version: "x.x.x" + chart: oidc-config + url: https://pluralsh.github.io/module-library + valuesFiles: + - oauth-proxy-config.yaml.liquid + imports: + - stackRef: + name: dagster-{{ context.cluster }} + namespace: apps + configuration: + cluster: {{ context.cluster }} + hostname: {{ context.hostname }} + clusterRef: + kind: Cluster + name: {{ context.cluster }} + namespace: infra diff --git a/catalogs/data/dagster/services/dagster-aws-env.yaml.liquid b/catalogs/data/dagster/services/dagster-aws-env.yaml.liquid new file mode 100644 index 00000000..0df8498c --- /dev/null +++ b/catalogs/data/dagster/services/dagster-aws-env.yaml.liquid @@ -0,0 +1,11 @@ +{% raw %} +{% assign imports_dagster_key = 'dagster-' | append: configuration.cluster %} +apiVersion: v1 +kind: Secret +metadata: + name: dagster-aws-env + namespace: dagster +stringData: + AWS_ACCESS_KEY_ID: {{ imports[imports_dagster_key].access_key_id }} + AWS_SECRET_ACCESS_KEY: {{ imports[imports_dagster_key].secret_access_key }} +{% endraw %} diff --git a/catalogs/data/dagster/terraform/aws/iam.tf b/catalogs/data/dagster/terraform/aws/iam.tf new file mode 100644 index 00000000..8cd52cbe --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/iam.tf @@ -0,0 +1,35 @@ +data "aws_iam_policy_document" "dagster" { + statement { + sid = "admin" + effect = "Allow" + actions = ["s3:*"] + + resources = [ + "arn:aws:s3:::${var.dagster_bucket}", + "arn:aws:s3:::${var.dagster_bucket}/*", + ] + } +} + +resource "aws_iam_policy" "dagster" { + name_prefix = "dagster" + description = "policy for the plural admin dagster" + policy = data.aws_iam_policy_document.dagster.json +} + +resource "aws_iam_user" "dagster" { + name = "${data.plural_cluster.cluster.name}-dagster" + + depends_on = [ data.plural_cluster.cluster ] + +} + +resource "aws_iam_access_key" "dagster" { + user = aws_iam_user.dagster.name +} + +resource "aws_iam_policy_attachment" "dagster-user" { + name = "${data.plural_cluster.cluster.name}-dagster-policy" + users = [aws_iam_user.dagster.name] + policy_arn = aws_iam_policy.dagster.arn +} diff --git a/catalogs/data/dagster/terraform/aws/oidc.tf b/catalogs/data/dagster/terraform/aws/oidc.tf new file mode 100644 index 00000000..5425c2bd --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/oidc.tf @@ -0,0 +1,15 @@ +resource "random_password" "oidc_cookie" { + length = 24 + min_lower = 1 + min_numeric = 1 + min_upper = 1 + special = false +} + +resource "plural_oidc_provider" "dagster" { + name = "dagster-{{ context.cluster }}" + auth_method = "BASIC" + type = "PLURAL" + description = "OIDC provider for Dagster deployed to the {{ context.cluster }} cluster" + redirect_uris = ["https://{{ context.hostname }}/oauth2/callback"] +} diff --git a/catalogs/data/dagster/terraform/aws/outputs.tf b/catalogs/data/dagster/terraform/aws/outputs.tf new file mode 100644 index 00000000..076908be --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/outputs.tf @@ -0,0 +1,36 @@ +output "iam_user" { + value = aws_iam_user.dagster +} + +output "access_key_id" { + value = aws_iam_access_key.dagster.id +} + +output "secret_access_key" { + value = aws_iam_access_key.dagster.secret + sensitive = true +} + +output "postgres_host" { + value = try(module.db.db_instance_address, "") +} + +output "postgres_password" { + value = random_password.password.result + sensitive = true +} + +output "oidc_cookie_secret" { + value = random_password.oidc_cookie.result + sensitive = true +} + +output "oidc_client_id" { + value = plural_oidc_provider.dagster.client_id + sensitive = true +} + +output "oidc_client_secret" { + value = plural_oidc_provider.dagster.client_secret + sensitive = true +} diff --git a/catalogs/data/dagster/terraform/aws/plural.tf b/catalogs/data/dagster/terraform/aws/plural.tf new file mode 100644 index 00000000..c30d03d4 --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/plural.tf @@ -0,0 +1,3 @@ +data "plural_cluster" "cluster" { + handle = var.cluster_name +} diff --git a/catalogs/data/dagster/terraform/aws/postgres.tf b/catalogs/data/dagster/terraform/aws/postgres.tf new file mode 100644 index 00000000..f04449de --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/postgres.tf @@ -0,0 +1,86 @@ +resource "random_password" "password" { + length = 20 + min_lower = 1 + min_numeric = 1 + min_upper = 1 + special = false +} + +data "aws_eks_cluster" "mgmt" { + name = data.plural_cluster.cluster.name + + depends_on = [ data.plural_cluster.cluster ] +} + +data "aws_vpc" "mgmt" { + id = one(data.aws_eks_cluster.mgmt.vpc_config).vpc_id +} + +module "db" { + source = "terraform-aws-modules/rds/aws" + version = "~> 6.3" + + identifier = var.db_name + + engine = "postgres" + engine_version = var.postgres_vsn + family = "postgres14" + major_engine_version = var.postgres_vsn + instance_class = var.db_instance_class + allocated_storage = var.db_storage + + db_name = "dagster" + username = "dagster" + password = random_password.password.result + manage_master_user_password = false + + maintenance_window = "Mon:00:00-Mon:03:00" + backup_window = "03:00-06:00" + backup_retention_period = var.backup_retention_period + + monitoring_interval = "30" + monitoring_role_name = "${var.db_name}-PluralRDSMonitoringRole" + create_monitoring_role = true + apply_immediately = true + + multi_az = true + + create_db_subnet_group = true + subnet_ids = one(data.aws_eks_cluster.mgmt.vpc_config).subnet_ids + vpc_security_group_ids = [module.security_group.security_group_id] + + create_cloudwatch_log_group = true + enabled_cloudwatch_logs_exports = ["postgresql"] + + parameters = [ + { + name = "autovacuum" + value = 1 + }, + { + name = "client_encoding" + value = "utf8" + } + ] + + deletion_protection = var.deletion_protection +} + +module "security_group" { + source = "terraform-aws-modules/security-group/aws" + version = "~> 5.0" + + name = "${var.db_name}-db-security-group" + description = "security group for your plural console db" + vpc_id = data.aws_vpc.mgmt.id + + ingress_with_cidr_blocks = [ + { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + description = "PostgreSQL access from within VPC" + cidr_blocks = data.aws_vpc.mgmt.cidr_block + }, + ] +} diff --git a/catalogs/data/dagster/terraform/aws/s3.tf b/catalogs/data/dagster/terraform/aws/s3.tf new file mode 100644 index 00000000..2f2fb32e --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/s3.tf @@ -0,0 +1,14 @@ +resource "aws_s3_bucket" "dagster" { + bucket = var.dagster_bucket + force_destroy = var.force_destroy_bucket +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "dagster" { + bucket = aws_s3_bucket.dagster.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} diff --git a/catalogs/data/dagster/terraform/aws/variables.tf b/catalogs/data/dagster/terraform/aws/variables.tf new file mode 100644 index 00000000..38c1e755 --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/variables.tf @@ -0,0 +1,46 @@ +variable "namespace" { + type = string + default = "dagster" +} + +variable "cluster_name" { + type = string + default = "{{ context.cluster }}" +} + +variable "dagster_bucket" { + type = string + default = "{{ context.bucket }}" +} + +variable "force_destroy_bucket" { + type = bool + default = true + description = "If true, the bucket will be deleted even if it contains objects." +} + +variable "db_name" { + default = "plrl-{{ context.cluster }}-dagster" +} + +variable "postgres_vsn" { + default = "14" +} + +variable "db_storage" { + default = 20 +} + +variable "deletion_protection" { + type = bool + default = true +} + +variable "backup_retention_period" { + type = number + default = 7 +} + +variable "db_instance_class" { + default = "db.t4g.large" +} \ No newline at end of file diff --git a/catalogs/data/dagster/terraform/aws/versions.tf b/catalogs/data/dagster/terraform/aws/versions.tf new file mode 100644 index 00000000..a265c4d5 --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.57" + } + plural = { + source = "pluralsh/plural" + version = ">= 0.2.1" + } + } +} + +provider "plural" {} + +provider "aws" { + region = "{{ context.region }}" +} \ No newline at end of file diff --git a/catalogs/data/mlflow/README.md b/catalogs/data/mlflow/README.md new file mode 100644 index 00000000..929995a0 --- /dev/null +++ b/catalogs/data/mlflow/README.md @@ -0,0 +1,7 @@ +# Mlflow + +This is a baseline, prod-ready Mlflow installation using Plural. + +## Contributing + +If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds. diff --git a/catalogs/data/mlflow/helm/mlflow.yaml.liquid b/catalogs/data/mlflow/helm/mlflow.yaml.liquid new file mode 100644 index 00000000..4922bf0d --- /dev/null +++ b/catalogs/data/mlflow/helm/mlflow.yaml.liquid @@ -0,0 +1,43 @@ +{% raw %} +{% assign imports_mlflow_key = 'mlflow-' | append: configuration.cluster %} + +tracking: + enabled: true + podAnnotations: + security.plural.sh/oauth-env-secret: mlflow-proxy-config + podLabels: + security.plural.sh/inject-oauth-sidecar: "true" + auth: + enabled: false + service: + type: ClusterIP + +externalDatabase: + dialectDriver: "postgresql" + host: {{ imports[imports_mlflow_key].postgres_host }} + port: 5432 + user: 'mlflow' + database: 'mlflow' + authDatabase: 'mlflow_auth' + password: {{ imports[imports_mlflow_key].postgres_password }} + +externalS3: + host: {{ configuration.region }}.amazonaws.com + port: 443 + useCredentialsInSecret: true + accessKeyID: {{imports[imports_mlflow_key].access_key_id}} + accessKeySecret: {{imports[imports_mlflow_key].secret_access_key}} + existingSecret: ~ + existingSecretAccessKeyIDKey: AWS_ACCESS_KEY_ID + existingSecretKeySecretKey: AWS_SECRET_ACCESS_KEY + protocol: "https" + bucket: {{ configuration.bucket }} + serveArtifacts: true + +postgresql: + enabled: false + +minio: + enabled: false + +{% endraw %} diff --git a/catalogs/data/mlflow/helm/oauth-proxy-config.yaml.liquid b/catalogs/data/mlflow/helm/oauth-proxy-config.yaml.liquid new file mode 100644 index 00000000..7b5a2fe5 --- /dev/null +++ b/catalogs/data/mlflow/helm/oauth-proxy-config.yaml.liquid @@ -0,0 +1,21 @@ +{% raw %} +{% assign imports_mlflow_key = 'mlflow-' | append: configuration.cluster %} + +service: + name: mlflow-oauth2-proxy + selector: + app.kubernetes.io/instance: mlflow + app.kubernetes.io/component: tracking + app.kubernetes.io/name: mlflow + +secret: + clientID: {{ imports[imports_mlflow_key].oidc_client_id }} + clientSecret: {{ imports[imports_mlflow_key].oidc_client_secret }} + cookieSecret: {{ imports[imports_mlflow_key].oidc_cookie_secret }} + issuer: https://oidc.plural.sh/ + upstream: http://localhost:5000 + name: mlflow-proxy-config + env: + OAUTH2_PROXY_UPSTREAM_TIMEOUT: '120s' + +{% endraw %} diff --git a/catalogs/data/mlflow/mlflow-helmrepository.yaml b/catalogs/data/mlflow/mlflow-helmrepository.yaml new file mode 100644 index 00000000..66df2e30 --- /dev/null +++ b/catalogs/data/mlflow/mlflow-helmrepository.yaml @@ -0,0 +1,10 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: bitnami + namespace: apps +spec: + type: oci + interval: 5m0s + url: oci://registry-1.docker.io/bitnamicharts + diff --git a/catalogs/data/mlflow/mlflow-raw-servicedeployment.yaml b/catalogs/data/mlflow/mlflow-raw-servicedeployment.yaml new file mode 100644 index 00000000..9aca6769 --- /dev/null +++ b/catalogs/data/mlflow/mlflow-raw-servicedeployment.yaml @@ -0,0 +1,20 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: mlflow-raw-{{ context.cluster }} + namespace: apps +spec: + namespace: mlflow + git: + folder: services/apps/mlflow + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + configuration: + hostname: {{ context.hostname }} + clusterRef: + kind: Cluster + name: {{ context.cluster }} + namespace: infra diff --git a/catalogs/data/mlflow/mlflow-servicedeployment.yaml b/catalogs/data/mlflow/mlflow-servicedeployment.yaml new file mode 100644 index 00000000..18ca6553 --- /dev/null +++ b/catalogs/data/mlflow/mlflow-servicedeployment.yaml @@ -0,0 +1,35 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: mlflow-{{ context.cluster }} + namespace: apps +spec: + namespace: mlflow + git: + folder: helm/mlflow/{{ context.cluster }} + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + helm: + version: "2.x.x" + chart: mlflow + release: mlflow + ignoreHooks: false + url: https://charts.bitnami.com/bitnami + valuesFiles: + - mlflow.yaml.liquid + imports: + - stackRef: + name: mlflow-{{ context.cluster }} + namespace: apps + configuration: + cluster: {{ context.cluster }} + hostname: {{ context.hostname }} + bucket: {{ context.bucket }} + region: {{ context.region }} + clusterRef: + kind: Cluster + name: {{ context.cluster }} + namespace: infra diff --git a/catalogs/data/mlflow/mlflow-stack.yaml b/catalogs/data/mlflow/mlflow-stack.yaml new file mode 100644 index 00000000..fcc25737 --- /dev/null +++ b/catalogs/data/mlflow/mlflow-stack.yaml @@ -0,0 +1,22 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: InfrastructureStack +metadata: + name: mlflow-{{ context.cluster }} + namespace: apps +spec: + detach: false + type: TERRAFORM + approval: true + manageState: true + actor: console@plural.sh + git: + ref: main + folder: terraform/apps/mlflow/{{ context.cluster }} + repositoryRef: + name: infra + namespace: infra + configuration: + version: '1.8' + clusterRef: + name: {{ context.stackCluster }} + namespace: infra diff --git a/catalogs/data/mlflow/oauth-proxy-config-servicedeployment.yaml b/catalogs/data/mlflow/oauth-proxy-config-servicedeployment.yaml new file mode 100644 index 00000000..e6f07ddd --- /dev/null +++ b/catalogs/data/mlflow/oauth-proxy-config-servicedeployment.yaml @@ -0,0 +1,31 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: mlflow-oauth-proxy-config-{{ context.cluster }} + namespace: apps +spec: + namespace: mlflow + git: + folder: helm/mlflow/{{ context.cluster }} + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + helm: + version: "x.x.x" + chart: oidc-config + url: https://pluralsh.github.io/module-library + valuesFiles: + - oauth-proxy-config.yaml.liquid + imports: + - stackRef: + name: mlflow-{{ context.cluster }} + namespace: apps + configuration: + cluster: {{ context.cluster }} + hostname: {{ context.hostname }} + clusterRef: + kind: Cluster + name: {{ context.cluster }} + namespace: infra diff --git a/catalogs/data/mlflow/services/oauth-proxy-ingress.yaml.liquid b/catalogs/data/mlflow/services/oauth-proxy-ingress.yaml.liquid new file mode 100644 index 00000000..44eeb700 --- /dev/null +++ b/catalogs/data/mlflow/services/oauth-proxy-ingress.yaml.liquid @@ -0,0 +1,33 @@ +{% raw %} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: mlflow-webapp-proxy + namespace: mlflow + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + kubernetes.io/tls-acme: "true" + # Extend timeout to allow long running queries. + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/use-regex: "true" +spec: + ingressClassName: nginx + rules: + - host: {{ configuration.hostname }} + http: + paths: + - backend: + service: + name: mlflow-oauth2-proxy + port: + number: 80 + path: /.* + pathType: ImplementationSpecific + tls: + - hosts: + - {{ configuration.hostname }} + secretName: mlflow-tls +{% endraw %} diff --git a/catalogs/data/mlflow/terraform/aws/iam.tf b/catalogs/data/mlflow/terraform/aws/iam.tf new file mode 100644 index 00000000..b3af2f6c --- /dev/null +++ b/catalogs/data/mlflow/terraform/aws/iam.tf @@ -0,0 +1,37 @@ + +resource "aws_iam_policy" "mlflow" { + name_prefix = "mlflow" + description = "policy for the plural admin mlflow" + policy = data.aws_iam_policy_document.mlflow.json +} + +resource "aws_iam_user" "mlflow" { + name = "${data.plural_cluster.cluster.name}-mlflow" + + depends_on = [ data.plural_cluster.cluster ] +} + +resource "aws_iam_access_key" "mlflow" { + user = aws_iam_user.mlflow.name +} + +data "aws_iam_policy_document" "mlflow" { + statement { + sid = "admin" + effect = "Allow" + actions = ["s3:*"] + + resources = [ + "arn:aws:s3:::${var.mlflow_bucket}", + "arn:aws:s3:::${var.mlflow_bucket}/*", + ] + } +} + +resource "aws_iam_policy_attachment" "mlflow-user" { + name = "${data.plural_cluster.cluster.name}-mlflow-policy" + users = [aws_iam_user.mlflow.name] + policy_arn = aws_iam_policy.mlflow.arn + + depends_on = [ data.plural_cluster.cluster ] +} diff --git a/catalogs/data/mlflow/terraform/aws/oidc.tf b/catalogs/data/mlflow/terraform/aws/oidc.tf new file mode 100644 index 00000000..7da02f53 --- /dev/null +++ b/catalogs/data/mlflow/terraform/aws/oidc.tf @@ -0,0 +1,15 @@ +resource "random_password" "oidc_cookie" { + length = 24 + min_lower = 1 + min_numeric = 1 + min_upper = 1 + special = false +} + +resource "plural_oidc_provider" "mlflow" { + name = "mlflow-{{ context.cluster }}" + auth_method = "BASIC" + type = "PLURAL" + description = "OIDC provider for mlflow deployed to the {{ context.cluster }} cluster" + redirect_uris = ["https://{{ context.hostname }}/oauth2/callback"] +} diff --git a/catalogs/data/mlflow/terraform/aws/outputs.tf b/catalogs/data/mlflow/terraform/aws/outputs.tf new file mode 100644 index 00000000..c807cedf --- /dev/null +++ b/catalogs/data/mlflow/terraform/aws/outputs.tf @@ -0,0 +1,32 @@ +output "access_key_id" { + value = aws_iam_access_key.mlflow.id +} + +output "secret_access_key" { + value = aws_iam_access_key.mlflow.secret + sensitive = true +} + +output "postgres_host" { + value = try(module.db.db_instance_address, "") +} + +output "postgres_password" { + value = random_password.password.result + sensitive = true +} + +output "oidc_cookie_secret" { + value = random_password.oidc_cookie.result + sensitive = true +} + +output "oidc_client_id" { + value = plural_oidc_provider.mlflow.client_id + sensitive = true +} + +output "oidc_client_secret" { + value = plural_oidc_provider.mlflow.client_secret + sensitive = true +} diff --git a/catalogs/data/mlflow/terraform/aws/plural.tf b/catalogs/data/mlflow/terraform/aws/plural.tf new file mode 100644 index 00000000..c30d03d4 --- /dev/null +++ b/catalogs/data/mlflow/terraform/aws/plural.tf @@ -0,0 +1,3 @@ +data "plural_cluster" "cluster" { + handle = var.cluster_name +} diff --git a/catalogs/data/mlflow/terraform/aws/postgres.tf b/catalogs/data/mlflow/terraform/aws/postgres.tf new file mode 100644 index 00000000..5a9acf73 --- /dev/null +++ b/catalogs/data/mlflow/terraform/aws/postgres.tf @@ -0,0 +1,87 @@ +resource "random_password" "password" { + length = 20 + min_lower = 1 + min_numeric = 1 + min_upper = 1 + special = false +} + +data "aws_eks_cluster" "mgmt" { + name = data.plural_cluster.cluster.name + + depends_on = [ data.plural_cluster.cluster ] +} + +data "aws_vpc" "mgmt" { + id = one(data.aws_eks_cluster.mgmt.vpc_config).vpc_id +} + +module "db" { + source = "terraform-aws-modules/rds/aws" + version = "~> 6.3" + + identifier = var.db_name + + engine = "postgres" + engine_version = var.postgres_vsn + family = "postgres14" + major_engine_version = var.postgres_vsn + instance_class = var.db_instance_class + allocated_storage = var.db_storage + + db_name = "mlflow" + username = "mlflow" + password = random_password.password.result + manage_master_user_password = false + + maintenance_window = "Mon:00:00-Mon:03:00" + backup_window = "03:00-06:00" + backup_retention_period = var.backup_retention_period + + monitoring_interval = "30" + monitoring_role_name = "${var.db_name}-PluralRDSMonitoringRole" + create_monitoring_role = true + apply_immediately = true + + multi_az = true + + create_db_subnet_group = true + subnet_ids = one(data.aws_eks_cluster.mgmt.vpc_config).subnet_ids + vpc_security_group_ids = [module.security_group.security_group_id] + + create_cloudwatch_log_group = true + enabled_cloudwatch_logs_exports = ["postgresql"] + + parameters = [ + { + name = "autovacuum" + value = 1 + }, + { + name = "client_encoding" + value = "utf8" + } + ] + + # Database Deletion Protection + deletion_protection = var.deletion_protection +} + +module "security_group" { + source = "terraform-aws-modules/security-group/aws" + version = "~> 5.0" + + name = "${var.db_name}-db-security-group" + description = "security group for your plural console db" + vpc_id = data.aws_vpc.mgmt.id + + ingress_with_cidr_blocks = [ + { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + description = "PostgreSQL access from within VPC" + cidr_blocks = data.aws_vpc.mgmt.cidr_block + }, + ] +} diff --git a/catalogs/data/mlflow/terraform/aws/s3.tf b/catalogs/data/mlflow/terraform/aws/s3.tf new file mode 100644 index 00000000..1436b2b2 --- /dev/null +++ b/catalogs/data/mlflow/terraform/aws/s3.tf @@ -0,0 +1,14 @@ +resource "aws_s3_bucket" "mlflow" { + bucket = var.mlflow_bucket + force_destroy = var.force_destroy_bucket +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "mlflow" { + bucket = aws_s3_bucket.mlflow.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} diff --git a/catalogs/data/mlflow/terraform/aws/variables.tf b/catalogs/data/mlflow/terraform/aws/variables.tf new file mode 100644 index 00000000..2f447826 --- /dev/null +++ b/catalogs/data/mlflow/terraform/aws/variables.tf @@ -0,0 +1,41 @@ +variable "cluster_name" { + type = string + default = "{{ context.cluster }}" +} + +variable "mlflow_bucket" { + type = string + default = "{{ context.bucket }}" +} + +variable "force_destroy_bucket" { + type = bool + default = false + description = "If true, the bucket will be deleted even if it contains objects." +} + +variable "db_name" { + default = "plrl-{{ context.cluster }}-mlflow" +} + +variable "postgres_vsn" { + default = "14" +} + +variable "db_storage" { + default = 20 +} + +variable "deletion_protection" { + type = bool + default = true +} + +variable "backup_retention_period" { + type = number + default = 7 +} + +variable "db_instance_class" { + default = "db.t4g.large" +} diff --git a/catalogs/data/mlflow/terraform/aws/versions.tf b/catalogs/data/mlflow/terraform/aws/versions.tf new file mode 100644 index 00000000..54f47de0 --- /dev/null +++ b/catalogs/data/mlflow/terraform/aws/versions.tf @@ -0,0 +1,19 @@ + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.57" + } + plural = { + source = "pluralsh/plural" + version = ">= 0.2.1" + } + } +} + +provider "plural" {} + +provider "aws" { + region = "{{ context.region }}" +} \ No newline at end of file diff --git a/catalogs/devops/grafana/README.md b/catalogs/devops/grafana/README.md new file mode 100644 index 00000000..40077f8c --- /dev/null +++ b/catalogs/devops/grafana/README.md @@ -0,0 +1,2 @@ +# Plural Grafana + diff --git a/catalogs/devops/grafana/grafana.yaml.liquid b/catalogs/devops/grafana/grafana.yaml.liquid new file mode 100644 index 00000000..192e3d08 --- /dev/null +++ b/catalogs/devops/grafana/grafana.yaml.liquid @@ -0,0 +1,23 @@ +ingress: + enabled: true + annotations: + cert-manager.io/cluster-issuer: plural + ingressClassName: nginx + + hosts: + - {{ context.hostname }} + + tls: + - hosts: + - {{ context.hostname }} + secretName: grafana-{{ context.cluster }}-tls + +persistence: + enabled: true + +plugins: + - https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.8.2/victoriametrics-datasource-v0.8.2.zip;victoriametrics-datasource + +grafana.ini: + plugins: + allow_loading_unsigned_plugins: victoriametrics-datasource \ No newline at end of file diff --git a/catalogs/devops/grafana/servicedeployment.yaml.liquid b/catalogs/devops/grafana/servicedeployment.yaml.liquid new file mode 100644 index 00000000..217df246 --- /dev/null +++ b/catalogs/devops/grafana/servicedeployment.yaml.liquid @@ -0,0 +1,26 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: grafana-{{ context.cluster }} + namespace: apps +spec: + namespace: grafana + git: + folder: helm/grafana/{{ context.cluster }} + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + helm: + version: "8.6.x" + chart: grafana + url: https://grafana.github.io/helm-charts + valuesFiles: + - grafana.yaml.liquid + configuration: + cluster: {{ context.cluster }} + clusterRef: + kind: Cluster + name: mgmt + namespace: infra diff --git a/catalogs/devops/kubecost/README.md b/catalogs/devops/kubecost/README.md new file mode 100644 index 00000000..7bd57b16 --- /dev/null +++ b/catalogs/devops/kubecost/README.md @@ -0,0 +1,2 @@ +# Plural Kubecost + diff --git a/catalogs/devops/kubecost/kubecost.yaml.liquid b/catalogs/devops/kubecost/kubecost.yaml.liquid new file mode 100644 index 00000000..3a634ff7 --- /dev/null +++ b/catalogs/devops/kubecost/kubecost.yaml.liquid @@ -0,0 +1,14 @@ +global: + grafana: + enabled: false + +kubecostFrontend: + enabled: false + +serviceMonitor: + enabled: true + +server: + global: + external_labels: + cluster_id: {{ context.cluster }} diff --git a/catalogs/devops/kubecost/servicedeployment.yaml.liquid b/catalogs/devops/kubecost/servicedeployment.yaml.liquid new file mode 100644 index 00000000..6ae5159c --- /dev/null +++ b/catalogs/devops/kubecost/servicedeployment.yaml.liquid @@ -0,0 +1,26 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: kubecost-{{ context.cluster }} + namespace: apps +spec: + namespace: kubecost + git: + folder: helm/kubecost/{{ context.cluster }} + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + helm: + version: "2.5.x" + chart: kubecost + url: https://grafana.github.io/helm-charts + valuesFiles: + - kubecost.yaml.liquid + configuration: + cluster: { ? { context.cluster } } + clusterRef: + kind: Cluster + name: mgmt + namespace: infra diff --git a/catalogs/infra/cluster/aws/stack.yaml b/catalogs/infra/cluster/aws/stack.yaml new file mode 100644 index 00000000..7baafd71 --- /dev/null +++ b/catalogs/infra/cluster/aws/stack.yaml @@ -0,0 +1,40 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: InfrastructureStack +metadata: + name: cluster-{{ context.name }} + namespace: infra +spec: + name: cluster-{{ context.name }} + detach: false + type: TERRAFORM + approval: true + manageState: true + actor: console@plural.sh + configuration: + version: '1.8' + repositoryRef: + name: infra + namespace: infra + clusterRef: + name: mgmt + namespace: infra + git: + ref: main + folder: terraform/modules/clusters/aws + environment: + - name: TF_VAR_cluster + value: {{ context.name }} + - name: TF_VAR_fleet + value: {{ context.fleet }} + - name: TF_VAR_tier + value: {{ context.tier }} + - name: TF_VAR_region + value: {{ context.region }} + - name: AWS_ACCESS_KEY_ID + secretKeyRef: + name: {{ context.credentialSecret }} + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + secretKeyRef: + name: {{ context.credentialSecret }} + key: AWS_SECRET_ACCESS_KEY diff --git a/catalogs/infra/cluster/cluster.yaml b/catalogs/infra/cluster/cluster.yaml new file mode 100644 index 00000000..2dc9578e --- /dev/null +++ b/catalogs/infra/cluster/cluster.yaml @@ -0,0 +1,7 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: Cluster +metadata: + name: {{ context.name }} + namespace: infra +spec: + handle: {{ context.name }} \ No newline at end of file diff --git a/catalogs/infra/cluster/servicedeployment.yaml b/catalogs/infra/cluster/servicedeployment.yaml new file mode 100644 index 00000000..97de920c --- /dev/null +++ b/catalogs/infra/cluster/servicedeployment.yaml @@ -0,0 +1,17 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: clusters + namespace: infra +spec: + namespace: infra + git: + folder: services/infra/clusters + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + clusterRef: + name: mgmt + namespace: infra diff --git a/catalogs/security/gatekeeper/README.md b/catalogs/security/gatekeeper/README.md new file mode 100644 index 00000000..2e6b806b --- /dev/null +++ b/catalogs/security/gatekeeper/README.md @@ -0,0 +1,13 @@ +# Gatekeeper + +This is a baseline, prod-ready OPA Gatekeeper installation using Plural. Besides Gatekeeper installation, it includes a policy bundle and set of constraints. + +You might want to slightly tweak the default setup for a few reasons: + +- only want to set up policy enforcement on a subset of your fleet (it's fleet-wide by default) +- prefer to choose a different policy bundle +- tweaking namespace names, crd names, etc. for your organization's preferences + +## Contributing + +If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds. \ No newline at end of file diff --git a/catalogs/security/gatekeeper/gatekeeper-constraints.yaml b/catalogs/security/gatekeeper/gatekeeper-constraints.yaml new file mode 100644 index 00000000..9f406a94 --- /dev/null +++ b/catalogs/security/gatekeeper/gatekeeper-constraints.yaml @@ -0,0 +1,14 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: GlobalService +metadata: + name: gatekeeper-constraints +spec: + template: + name: gatekeeper-constraints + namespace: policy + git: + folder: resources/policy/constraints + ref: main + repositoryRef: + kind: GitRepository + name: bootstrap diff --git a/catalogs/security/gatekeeper/gatekeeper-policy-bundle.yaml b/catalogs/security/gatekeeper/gatekeeper-policy-bundle.yaml new file mode 100644 index 00000000..ada1b26f --- /dev/null +++ b/catalogs/security/gatekeeper/gatekeeper-policy-bundle.yaml @@ -0,0 +1,15 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: GlobalService +metadata: + name: gatekeeper-policy-bundle + namespace: apps +spec: + template: + name: gatekeeper-policy-bundle + namespace: policy + git: + folder: resources/policy/bundles/{{ context.bundle }} + ref: main + repositoryRef: + kind: GitRepository + name: bootstrap diff --git a/catalogs/security/gatekeeper/gatekeeper.yaml b/catalogs/security/gatekeeper/gatekeeper.yaml new file mode 100644 index 00000000..0acf3b57 --- /dev/null +++ b/catalogs/security/gatekeeper/gatekeeper.yaml @@ -0,0 +1,13 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: GlobalService +metadata: + name: gatekeeper + namespace: apps +spec: + template: + name: gatekeeper + namespace: policy + helm: + url: https://open-policy-agent.github.io/gatekeeper/charts + version: 3.15.1 + chart: gatekeeper \ No newline at end of file diff --git a/catalogs/security/trivy-operator/README.md b/catalogs/security/trivy-operator/README.md new file mode 100644 index 00000000..a8e1f3bf --- /dev/null +++ b/catalogs/security/trivy-operator/README.md @@ -0,0 +1,7 @@ +# Trivy Operator + +This is a baseline, prod-ready Trivy Operator installation using Plural. + +## Contributing + +If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds. \ No newline at end of file diff --git a/catalogs/security/trivy-operator/trivy-operator.yaml b/catalogs/security/trivy-operator/trivy-operator.yaml new file mode 100644 index 00000000..27a20de7 --- /dev/null +++ b/catalogs/security/trivy-operator/trivy-operator.yaml @@ -0,0 +1,16 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: GlobalService +metadata: + name: trivy-operator + namespace: apps +spec: + template: + name: trivy-operator + namespace: trivy-system + helm: + url: https://aquasecurity.github.io/helm-charts/ + chart: trivy-operator + version: 'x.x.x' + values: + trivy: + additionalVulnerabilityReportFields: Description,Links,CVSS,Target diff --git a/setup/catalogs/catalogs.yaml b/setup/catalogs/catalogs.yaml new file mode 100644 index 00000000..dde39e6f --- /dev/null +++ b/setup/catalogs/catalogs.yaml @@ -0,0 +1,47 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: Catalog +metadata: + name: data-engineering +spec: + name: data-engineering + category: data + icon: https://docs.plural.sh/favicon-128.png + author: Plural + description: | + Sets up OSS data infrastructure using Plural +--- +apiVersion: deployments.plural.sh/v1alpha1 +kind: Catalog +metadata: + name: security +spec: + name: security + category: security + icon: https://docs.plural.sh/favicon-128.png + author: Plural + description: | + Sets up OSS security infrastructure using Plural +--- +apiVersion: deployments.plural.sh/v1alpha1 +kind: Catalog +metadata: + name: devops +spec: + name: devops + category: devops + icon: https://docs.plural.sh/favicon-128.png + author: Plural + description: | + Sets up OSS devops infrastructure using Plural +--- +apiVersion: deployments.plural.sh/v1alpha1 +kind: Catalog +metadata: + name: infra +spec: + name: infra + category: data + icon: https://docs.plural.sh/favicon-128.png + author: Plural + description: | + Sets up infrastructure using Plural diff --git a/setup/catalogs/data/airbyte.yaml b/setup/catalogs/data/airbyte.yaml new file mode 100644 index 00000000..fa4478f3 --- /dev/null +++ b/setup/catalogs/data/airbyte.yaml @@ -0,0 +1,71 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: PrAutomation +metadata: + name: airbyte +spec: + name: airbyte + icon: https://plural-assets.s3.us-east-2.amazonaws.com/uploads/repos/d79a69b7-dfcd-480a-a51d-518865fd6e7c/airbyte.png + documentation: | + Sets up an airbyte instance for a given cloud + creates: + git: + ref: sebastian/prod-2981-set-up-catalog-pipeline # TODO set to main + folder: catalogs/data/airbyte + templates: + - source: helm + destination: helm/airbyte/{{ context.cluster }} + external: true + - source: services/oauth-proxy-ingress.yaml.liquid + destination: services/apps/airbyte/oauth-proxy-ingress.yaml.liquid + external: true + - source: "terraform/{{ context.cloud }}" + destination: "terraform/apps/airbyte/{{ context.cluster }}" + external: true + - source: airbyte-raw-servicedeployment.yaml + destination: "bootstrap/apps/airbyte/{{ context.cluster }}/airbyte-raw-servicedeployment.yaml" + external: true + - source: airbyte-servicedeployment.yaml + destination: "bootstrap/apps/airbyte/{{ context.cluster }}/airbyte-servicedeployment.yaml" + external: true + - source: airbyte-stack.yaml + destination: "bootstrap/apps/airbyte/{{ context.cluster }}/airbyte-stack.yaml" + external: true + - source: oauth-proxy-config-servicedeployment.yaml + destination: "bootstrap/apps/airbyte/{{ context.cluster }}/oauth-proxy-config-servicedeployment.yaml" + external: true + - source: README.md + destination: documentation/airbyte/README.md + external: true + repositoryRef: + name: scaffolds + catalogRef: + name: data-engineering + scmConnectionRef: + name: plural # you'll need to add this ScmConnection manually before this is functional + title: "Setting up airbyte on cluster {{ context.cluster }} for {{ context.cloud }}" + message: | + Set up airbyte on {{ context.cluster }} ({{ context.cloud }}) + + Will set up an airbyte deployment, including object storage and postgres setup + configuration: + - name: cluster + type: STRING + documentation: Handle of the cluster you want to deploy airbyte to. + - name: stackCluster + type: STRING + documentation: Handle of the cluster used to run Infrastructure Stacks for provisioning the infrastructure. Defaults to the management cluster. + default: mgmt + - name: cloud + type: ENUM + documentation: Cloud provider you want to deploy airbyte to. + values: + - aws + - name: bucket + type: STRING + documentation: The name of the S3/GCS/Azure Blob bucket you'll use for airbyte logs. This must be globally unique. + - name: hostname + type: STRING + documentation: The DNS name you'll host airbyte under. + - name: region + type: STRING + documentation: The cloud provider region you're going to use to deploy cloud resources. diff --git a/setup/catalogs/data/dagster.yaml b/setup/catalogs/data/dagster.yaml new file mode 100644 index 00000000..804db6b7 --- /dev/null +++ b/setup/catalogs/data/dagster.yaml @@ -0,0 +1,65 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: PrAutomation +metadata: + name: dagster +spec: + name: dagster + icon: https://dagster.io/images/brand/logos/dagster-primary-mark.png + documentation: Sets up Dagster instance for given cloud. + creates: + git: + ref: sebastian/prod-2981-set-up-catalog-pipeline # TODO set to main + folder: catalogs/data/dagster + templates: + - source: README.md + destination: documentation/dagster/README.md + external: true + - source: dagster.yaml + destination: "bootstrap/apps/dagster/{{ context.cluster }}/dagster.yaml" + external: true + - source: helm + destination: helm/dagster/{{ context.cluster }} + external: true + - source: "terraform/{{ context.cloud }}" + destination: "terraform/apps/dagster/{{ context.cluster }}" + external: true + - source: dagster-aws-env.yaml + destination: "bootstrap/apps/dagster/{{ context.cluster }}/dagster-aws-env.yaml" + external: true + - source: oauth-proxy-config.yaml + destination: "bootstrap/apps/dagster/{{ context.cluster }}/oauth-proxy-config.yaml" + external: true + - source: services/dagster-aws-env.yaml.liquid + destination: services/apps/dagster/dagster-aws-env.yaml.liquid + external: true + repositoryRef: + name: scaffolds + catalogRef: + name: data-engineering + scmConnectionRef: + name: plural # you'll need to add this ScmConnection manually before this is functional + title: Dagster setup ({{ context.cluster }}) + message: Sets up Dagster on {{ context.cluster }} cluster. + configuration: + - name: cluster + type: STRING + documentation: The cluster you want to deploy to. + - name: stackCluster + type: STRING + documentation: Handle of the cluster used to run Infrastructure Stacks for provisioning the infrastructure. Defaults to the management cluster. + default: mgmt + - name: cloud + type: ENUM + documentation: The cloud you want to deploy to. + values: + - aws + - name: region + type: STRING + documentation: The cloud provider region you want to use to deploy cloud resources. + - name: bucket + type: STRING + documentation: The name of the bucket you want to use. This must be globally unique. + - name: hostname + type: STRING + documentation: The DNS name you want to host Dagster under. + diff --git a/setup/catalogs/data/mlflow.yaml b/setup/catalogs/data/mlflow.yaml new file mode 100644 index 00000000..a8971d8c --- /dev/null +++ b/setup/catalogs/data/mlflow.yaml @@ -0,0 +1,74 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: PrAutomation +metadata: + name: mlflow +spec: + name: mlflow + icon: https://cdn.icon-icons.com/icons2/3913/PNG/512/mlflow_logo_icon_248445.png + documentation: | + Sets up an mlflow instance for a given cloud + creates: + git: + ref: sebastian/prod-2981-set-up-catalog-pipeline # TODO set to main + folder: catalogs/data/mlflow + templates: + - source: helm + destination: helm/mlflow/{{ context.cluster }} + external: true + - source: services/oauth-proxy-ingress.yaml.liquid + destination: services/apps/mlflow/oauth-proxy-ingress.yaml.liquid + external: true + - source: "terraform/{{ context.cloud }}" + destination: "terraform/apps/mlflow/{{ context.cluster }}" + external: true + - source: mlflow-helmrepository.yaml + destination: "bootstrap/apps/mlflow/{{ context.cluster }}/mlflow-helmrepository.yaml" + external: true + - source: mlflow-raw-servicedeployment.yaml + destination: "bootstrap/apps/mlflow/{{ context.cluster }}/mlflow-raw-servicedeployment.yaml" + external: true + - source: mlflow-servicedeployment.yaml + destination: "bootstrap/apps/mlflow/{{ context.cluster }}/mlflow-servicedeployment.yaml" + external: true + - source: mlflow-stack.yaml + destination: "bootstrap/apps/mlflow/{{ context.cluster }}/mlflow-stack.yaml" + external: true + - source: oauth-proxy-config-servicedeployment.yaml + destination: "bootstrap/apps/mlflow/{{ context.cluster }}/oauth-proxy-config-servicedeployment.yaml" + external: true + - source: README.md + destination: documentation/mlflow/README.md + external: true + repositoryRef: + name: scaffolds + catalogRef: + name: data-engineering + scmConnectionRef: + name: plural # you'll need to add this ScmConnection manually before this is functional + title: "Setting up mlflow on cluster {{ context.cluster }} for {{ context.cloud }}" + message: | + Set up mlflow on {{ context.cluster }} ({{ context.cloud }}) + + Will set up an mlflow deployment, including object storage and postgres setup + configuration: + - name: cluster + type: STRING + documentation: Handle of the cluster you want to deploy mlflow to. + - name: stackCluster + type: STRING + documentation: Handle of the cluster used to run Infrastructure Stacks for provisioning the infrastructure. Defaults to the management cluster. + default: mgmt + - name: cloud + type: ENUM + documentation: Cloud provider you want to deploy mlflow to. + values: + - aws + - name: bucket + type: STRING + documentation: The name of the S3/GCS/Azure Blob bucket you'll use for mlflow logs. This must be globally unique. + - name: hostname + type: STRING + documentation: The DNS name you'll host mlflow under. + - name: region + type: STRING + documentation: The cloud provider region you're going to use to deploy cloud resources. diff --git a/setup/catalogs/devops/grafana.yaml b/setup/catalogs/devops/grafana.yaml new file mode 100644 index 00000000..37844902 --- /dev/null +++ b/setup/catalogs/devops/grafana.yaml @@ -0,0 +1,40 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: PrAutomation +metadata: + name: grafana +spec: + name: grafana + icon: https://cdn.icon-icons.com/icons2/2699/PNG/512/grafana_logo_icon_171048.png + documentation: | + Sets up Grafana Open Observability Platform. + creates: + git: + ref: sebastian/prod-2981-set-up-catalog-pipeline # TODO set to main + folder: catalogs/devops/grafana + templates: + - source: README.md + destination: documentation/grafana/README.md + external: true + - source: grafana.yaml.liquid + destination: "helm/grafana/{{ context.cluster }}/grafana.yaml.liquid" + external: true + - source: servicedeployment.yaml.liquid + destination: "bootstrap/apps/grafana/{{ context.cluster }}/servicedeployment.yaml" + external: true + repositoryRef: + name: scaffolds + catalogRef: + name: devops + scmConnectionRef: + name: plural # you'll need to add this ScmConnection manually before this is functional + title: "Grafana setup ({{ context.cluster }})" + message: | + Sets up Grafana on {{ context.cluster }} cluster. + identifier: pluralsh/plrl-dev-aws # FIXME + configuration: + - name: cluster + type: STRING + documentation: the cluster you want to deploy to + - name: hostname + type: STRING + documentation: the hostname of the grafana instance diff --git a/setup/catalogs/devops/kubecost.yaml b/setup/catalogs/devops/kubecost.yaml new file mode 100644 index 00000000..07c40cb2 --- /dev/null +++ b/setup/catalogs/devops/kubecost.yaml @@ -0,0 +1,37 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: PrAutomation +metadata: + name: kubecost +spec: + name: kubecost + icon: https://github.com/pluralsh/plural-artifacts/blob/main/kubecost/plural/icons/kubecost.png?raw=true + documentation: | + Sets up Kubecost Kubernetes Cost Monitoring and Management. + creates: + git: + ref: sebastian/prod-2981-set-up-catalog-pipeline # TODO set to main + folder: catalogs/devops/kubecost + templates: + - source: README.md + destination: documentation/kubecost/README.md + external: true + - source: kubecost.yaml.liquid + destination: "helm/kubecost/{{ context.cluster }}/kubecost.yaml.liquid" + external: true + - source: servicedeployment.yaml.liquid + destination: "bootstrap/apps/kubecost/{{ context.cluster }}/servicedeployment.yaml" + external: true + repositoryRef: + name: scaffolds + catalogRef: + name: devops + scmConnectionRef: + name: plural # you'll need to add this ScmConnection manually before this is functional + title: "Kubecost setup ({{ context.cluster }})" + message: | + Sets up Kubecost on {{ context.cluster }} cluster. + identifier: pluralsh/plrl-dev-aws # FIXME + configuration: + - name: cluster + type: STRING + documentation: the cluster you want to deploy to diff --git a/setup/catalogs/infra/cluster-eks.yaml b/setup/catalogs/infra/cluster-eks.yaml new file mode 100644 index 00000000..57723d9f --- /dev/null +++ b/setup/catalogs/infra/cluster-eks.yaml @@ -0,0 +1,50 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: PrAutomation +metadata: + name: cluster-eks +spec: + name: cluster-eks + icon: https://cdn.icon-icons.com/icons2/2699/PNG/512/amazon_eks_logo_icon_168659.png + documentation: | + Sets up a PR to provision EKS cluster for a fleet + stage + creates: + git: + ref: sebastian/prod-2981-set-up-catalog-pipeline # TODO set to main + folder: catalogs/infra/cluster + templates: + - source: aws/stack.yaml + destination: "services/infra/clusters/aws/stacks/{{ context.name }}.yaml" + external: true + - source: cluster.yaml + destination: "services/infra/clusters/aws/{{ context.name }}.yaml" + external: true + - source: servicedeployment.yaml + destination: "bootstrap/infra/clusters/servicedeployment.yaml" + external: true + repositoryRef: + name: scaffolds + catalogRef: + name: infra + scmConnectionRef: + name: plural # you'll need to add this ScmConnection manually before this is functional + title: "Adding EKS cluster: {{ context.name }}" + message: "Adding EKS cluster {{ context.name }} and registering it with Plural" + configuration: + - name: name + type: STRING + documentation: Name of the cluster. + - name: fleet + type: STRING + documentation: Name for the fleet you want this cluster to belong to. + - name: tier + type: ENUM + documentation: What tier to place this cluster in. + values: + - dev + - prd + - name: region + type: STRING + documentation: Region where the cluster should be created. + - name: credentialSecret + type: STRING + documentation: Name of the Kubernetes secret on the mgmt cluster in the infra namespace with the credentials that can be used to access the AWS cloud. It should contain 'AWS_ACCESS_KEY_ID' and 'AWS_SECRET_ACCESS_KEY' keys. diff --git a/setup/catalogs/security/gatekeeper.yaml b/setup/catalogs/security/gatekeeper.yaml new file mode 100644 index 00000000..90af0401 --- /dev/null +++ b/setup/catalogs/security/gatekeeper.yaml @@ -0,0 +1,44 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: PrAutomation +metadata: + name: gatekeeper +spec: + name: gatekeeper + icon: https://www.openpolicyagent.org/img/logos/opa-no-text-color.png + documentation: Sets up Gatekeeper policy controller. + creates: + git: + ref: sebastian/prod-2981-set-up-catalog-pipeline # TODO set to main + folder: catalogs/security/gatekeeper + templates: + - source: README.md + destination: documentation/gatekeeper/README.md + external: true + - source: gatekeeper.yaml + destination: bootstrap/apps/gatekeeper/gatekeeper.yaml + external: true + - source: gatekeeper-constraints.yaml + destination: bootstrap/apps/gatekeeper/gatekeeper-constraints.yaml + external: true + - source: gatekeeper-policy-bundle.yaml + destination: bootstrap/apps/gatekeeper/gatekeeper-policy-bundle.yaml + external: true + repositoryRef: + name: scaffolds + catalogRef: + name: security + scmConnectionRef: + name: plural # you'll need to add this ScmConnection manually before this is functional + title: Gatekeeper setup + message: Sets up Gatekeeper, policy bundle and a set of constraints. + configuration: + - name: bundle + type: ENUM + documentation: The policy bundle you want to install. + values: + - asm-policy-v0.0.1 + - cis-k8s-v1.5.1 + - policy-essentials-v2022 + - psp-v2022 + - pss-baseline-v2022 + diff --git a/setup/catalogs/security/trivy-operator.yaml b/setup/catalogs/security/trivy-operator.yaml new file mode 100644 index 00000000..c0f55441 --- /dev/null +++ b/setup/catalogs/security/trivy-operator.yaml @@ -0,0 +1,27 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: PrAutomation +metadata: + name: trivy-operator +spec: + name: trivy-operator + icon: https://raw.githubusercontent.com/aquasecurity/trivy-vscode-extension/refs/tags/0.7.1/media/trivy.svg + documentation: Sets up Trivy Operator security toolkit. + creates: + git: + ref: sebastian/prod-2981-set-up-catalog-pipeline # TODO set to main + folder: catalogs/security/trivy-operator + templates: + - source: README.md + destination: documentation/trivy-operator/README.md + external: true + - source: trivy-operator.yaml + destination: bootstrap/apps/trivy-operator/trivy-operator.yaml + external: true + repositoryRef: + name: scaffolds + catalogRef: + name: security + scmConnectionRef: + name: plural # you'll need to add this ScmConnection manually before this is functional + title: Trivy Operator setup + message: Sets up Trivy Operator. diff --git a/setup/data.yaml b/setup/data.yaml deleted file mode 100644 index 2e93a175..00000000 --- a/setup/data.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: deployments.plural.sh/v1alpha1 -kind: Catalog -metadata: - name: data-engineering -spec: - name: data-engineering - category: data - icon: https://docs.plural.sh/favicon-128.png - author: Plural - description: | - Sets up OSS data infrastructure using Plural - \ No newline at end of file diff --git a/setup/data/airbyte.yaml b/setup/data/airbyte.yaml deleted file mode 100644 index 923513a9..00000000 --- a/setup/data/airbyte.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: deployments.plural.sh/v1alpha1 -kind: PrAutomation -metadata: - name: airbyte -spec: - name: airbyte - icon: https://plural-assets.s3.us-east-2.amazonaws.com/uploads/repos/d79a69b7-dfcd-480a-a51d-518865fd6e7c/airbyte.png - documentation: | - Sets up an airbyte instance for a given cloud - creates: - git: - ref: main - folder: catalogs/data/airbyte - templates: - - source: stack.yaml.liquid - destination: "bootstrap/apps/airbyte/{{ context.cluster }}/stack.yaml" - external: true - - source: servicedeployment.yaml.liquid - destination: "bootstrap/apps/airbyte/{{ context.cluster }}/servicedeployment.yaml" - external: true - - source: README.md - destination: documentation/airbyte/README.md - external: true - - source: helm/values.yaml.liquid - destination: helm/airbyte/{{ context.cluster }}.yaml.liquid - external: true - - source: terraform - destination: terraform/apps/airbyte - external: true - repositoryRef: - name: scaffolds - namespace: infra - catalogRef: - name: data-engineering - namespace: infra - scmConnectionRef: - name: github # you'll need to add this ScmConnection manually before this is functional - title: "Setting up airbyte on cluster {{ context.cluster }} for {{ context.cloud }}" - message: | - Set up airbyte on {{ context.cluster }} ({{ context.cloud }}) - - Will set up an airbyte deployment, including object storage and postgres setup - identifier: pluralsh/scaffolds # REPLACEME with your own repo slug - configuration: - - name: cluster - type: STRING - documentation: the cluster you want to deploy to - - name: cloud - type: ENUM - documentation: the cloud you can deploy airbyte to - values: - - aws - - name: bucket - type: STRING - documentation: the name of the S3/GCS/Azure Blob bucket you'll use for airbyte logs. This must be globally unique - - name: hostname - type: STRING - documentation: the DNS name you'll host airbyte under - - name: region - type: STRING - documentation: the cloud region you're going to use for this instance \ No newline at end of file diff --git a/test/contexts/airbyte.yaml b/test/contexts/airbyte.yaml index e018c9d9..edd30936 100644 --- a/test/contexts/airbyte.yaml +++ b/test/contexts/airbyte.yaml @@ -1,5 +1,6 @@ cluster: mgmt +stackCluster: mgmt cloud: aws bucket: plrl-airbyte-logs hostname: airbyte.plural.sh -region: us-east-2 \ No newline at end of file +region: us-east-2 diff --git a/test/contracts.yaml b/test/contracts.yaml index b808a887..230a5c44 100644 --- a/test/contracts.yaml +++ b/test/contracts.yaml @@ -5,8 +5,8 @@ metadata: spec: workdir: test/outputs templates: - from: catalogs - to: test/outputs/catalogs + from: catalogs/data/airbyte + to: test/outputs/catalogs/data/airbyte automations: - - file: ../../setup/data/airbyte.yaml - context: ../contexts/airbyte.yaml \ No newline at end of file + - file: ../../setup/catalogs/data/airbyte.yaml + context: ../contexts/airbyte.yaml diff --git a/test/outputs/bootstrap/apps/airbyte/mgmt/airbyte-raw-servicedeployment.yaml b/test/outputs/bootstrap/apps/airbyte/mgmt/airbyte-raw-servicedeployment.yaml new file mode 100644 index 00000000..a9684b1a --- /dev/null +++ b/test/outputs/bootstrap/apps/airbyte/mgmt/airbyte-raw-servicedeployment.yaml @@ -0,0 +1,20 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: airbyte-raw-mgmt + namespace: apps +spec: + namespace: airbyte + git: + folder: services/apps/airbyte + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + configuration: + hostname: airbyte.plural.sh + clusterRef: + kind: Cluster + name: mgmt + namespace: infra diff --git a/test/outputs/bootstrap/apps/airbyte/mgmt/airbyte-servicedeployment.yaml b/test/outputs/bootstrap/apps/airbyte/mgmt/airbyte-servicedeployment.yaml new file mode 100644 index 00000000..3c887fef --- /dev/null +++ b/test/outputs/bootstrap/apps/airbyte/mgmt/airbyte-servicedeployment.yaml @@ -0,0 +1,35 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: airbyte-mgmt + namespace: apps +spec: + namespace: airbyte + git: + folder: helm/airbyte/mgmt + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + helm: + version: "1.x.x" + chart: airbyte + release: airbyte + ignoreHooks: false + url: https://airbytehq.github.io/helm-charts + valuesFiles: + - airbyte.yaml.liquid + imports: + - stackRef: + name: airbyte-mgmt + namespace: apps + configuration: + cluster: mgmt + hostname: airbyte.plural.sh + bucket: plrl-airbyte-logs + region: us-east-2 + clusterRef: + kind: Cluster + name: mgmt + namespace: infra diff --git a/test/outputs/bootstrap/apps/airbyte/mgmt/stack.yaml b/test/outputs/bootstrap/apps/airbyte/mgmt/airbyte-stack.yaml similarity index 86% rename from test/outputs/bootstrap/apps/airbyte/mgmt/stack.yaml rename to test/outputs/bootstrap/apps/airbyte/mgmt/airbyte-stack.yaml index d54970cf..7a296741 100644 --- a/test/outputs/bootstrap/apps/airbyte/mgmt/stack.yaml +++ b/test/outputs/bootstrap/apps/airbyte/mgmt/airbyte-stack.yaml @@ -2,20 +2,21 @@ apiVersion: deployments.plural.sh/v1alpha1 kind: InfrastructureStack metadata: name: airbyte-mgmt + namespace: apps spec: detach: false type: TERRAFORM approval: true manageState: true actor: console@plural.sh - configuration: - version: '1.8' + git: + ref: main + folder: terraform/apps/airbyte/mgmt repositoryRef: name: infra namespace: infra + configuration: + version: '1.8' clusterRef: name: mgmt namespace: infra - git: - ref: main - folder: terraform/apps/airbyte/aws \ No newline at end of file diff --git a/test/outputs/bootstrap/apps/airbyte/mgmt/servicedeployment.yaml b/test/outputs/bootstrap/apps/airbyte/mgmt/oauth-proxy-config-servicedeployment.yaml similarity index 56% rename from test/outputs/bootstrap/apps/airbyte/mgmt/servicedeployment.yaml rename to test/outputs/bootstrap/apps/airbyte/mgmt/oauth-proxy-config-servicedeployment.yaml index ea6e8cae..cb1606b3 100644 --- a/test/outputs/bootstrap/apps/airbyte/mgmt/servicedeployment.yaml +++ b/test/outputs/bootstrap/apps/airbyte/mgmt/oauth-proxy-config-servicedeployment.yaml @@ -1,12 +1,12 @@ apiVersion: deployments.plural.sh/v1alpha1 kind: ServiceDeployment metadata: - name: airbyte-mgmt - namespace: infra + name: airbyte-oauth-proxy-config-mgmt + namespace: apps spec: namespace: airbyte git: - folder: helm/airbyte + folder: helm/airbyte/mgmt ref: main repositoryRef: kind: GitRepository @@ -14,14 +14,17 @@ spec: namespace: infra helm: version: "x.x.x" - chart: airbyte - url: https://app.plural.sh/cm/airbyte + chart: oidc-config + url: https://pluralsh.github.io/module-library valuesFiles: - - mgmt.yaml.liquid + - oauth-proxy-config.yaml.liquid imports: - stackRef: name: airbyte-mgmt - namespace: infra + namespace: apps + configuration: + cluster: mgmt + hostname: airbyte.plural.sh clusterRef: kind: Cluster name: mgmt diff --git a/test/outputs/catalogs/data/airbyte/README.md b/test/outputs/catalogs/data/airbyte/README.md index 41dbfc5e..d273998b 100644 --- a/test/outputs/catalogs/data/airbyte/README.md +++ b/test/outputs/catalogs/data/airbyte/README.md @@ -6,7 +6,7 @@ This is a baseline, prod ready airbyte installation using Plural. It includes a * RDS/Google Cloud Sql, Azure Flexible Server to handle postgres. This gives you a robust RDBMS service to hold airbyte's core transactional data. * Plural OIDC to handle authentication to Airbyte. Airbyte does not support this natively, and so we use oauth-proxy as a middleware to handle authentication. -In addtion, there are a few common customizations you might want to do. +In addition, there are a few common customizations you might want to do. ## Configure Basic Auth @@ -80,4 +80,4 @@ when building your application. ## Contributing -If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds \ No newline at end of file +If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds diff --git a/test/outputs/catalogs/data/airbyte/airbyte-raw-servicedeployment.yaml b/test/outputs/catalogs/data/airbyte/airbyte-raw-servicedeployment.yaml new file mode 100644 index 00000000..9a2aaf3a --- /dev/null +++ b/test/outputs/catalogs/data/airbyte/airbyte-raw-servicedeployment.yaml @@ -0,0 +1,20 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: airbyte-raw-{{ context.cluster }} + namespace: apps +spec: + namespace: airbyte + git: + folder: services/apps/airbyte + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + configuration: + hostname: {{ context.hostname }} + clusterRef: + kind: Cluster + name: {{ context.cluster }} + namespace: infra diff --git a/test/outputs/catalogs/data/airbyte/airbyte-servicedeployment.yaml b/test/outputs/catalogs/data/airbyte/airbyte-servicedeployment.yaml new file mode 100644 index 00000000..94ee5553 --- /dev/null +++ b/test/outputs/catalogs/data/airbyte/airbyte-servicedeployment.yaml @@ -0,0 +1,35 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: airbyte-{{ context.cluster }} + namespace: apps +spec: + namespace: airbyte + git: + folder: helm/airbyte/{{ context.cluster }} + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + helm: + version: "1.x.x" + chart: airbyte + release: airbyte + ignoreHooks: false + url: https://airbytehq.github.io/helm-charts + valuesFiles: + - airbyte.yaml.liquid + imports: + - stackRef: + name: airbyte-{{ context.cluster }} + namespace: apps + configuration: + cluster: {{ context.cluster }} + hostname: {{ context.hostname }} + bucket: {{ context.bucket }} + region: {{ context.region }} + clusterRef: + kind: Cluster + name: {{ context.cluster }} + namespace: infra diff --git a/catalogs/data/airbyte/stack.yaml.liquid b/test/outputs/catalogs/data/airbyte/airbyte-stack.yaml similarity index 76% rename from catalogs/data/airbyte/stack.yaml.liquid rename to test/outputs/catalogs/data/airbyte/airbyte-stack.yaml index 94c96633..a2abd4e7 100644 --- a/catalogs/data/airbyte/stack.yaml.liquid +++ b/test/outputs/catalogs/data/airbyte/airbyte-stack.yaml @@ -2,20 +2,21 @@ apiVersion: deployments.plural.sh/v1alpha1 kind: InfrastructureStack metadata: name: airbyte-{{ context.cluster }} + namespace: apps spec: detach: false type: TERRAFORM approval: true manageState: true actor: console@plural.sh - configuration: - version: '1.8' + git: + ref: main + folder: terraform/apps/airbyte/{{ context.cluster }} repositoryRef: name: infra namespace: infra + configuration: + version: '1.8' clusterRef: - name: mgmt + name: {{ context.stackCluster }} namespace: infra - git: - ref: main - folder: terraform/apps/airbyte/{{ context.cloud }} \ No newline at end of file diff --git a/test/outputs/catalogs/data/airbyte/helm/airbyte.yaml.liquid b/test/outputs/catalogs/data/airbyte/helm/airbyte.yaml.liquid new file mode 100644 index 00000000..c4dfff80 --- /dev/null +++ b/test/outputs/catalogs/data/airbyte/helm/airbyte.yaml.liquid @@ -0,0 +1,57 @@ +{% raw %} +{% assign imports_airbyte_key = 'airbyte-' | append: configuration.cluster %} + +global: + deploymentMode: oss + edition: community + + airbyteUrl: {{ configuration.hostname }} + + storage: + type: S3 + storageSecretName: airbyte-airbyte-secrets + s3: + region: {{ configuration.region }} + authenticationType: credentials + accessKeyId: {{ imports[imports_airbyte_key].access_key_id }} + accessKeyIdSecretKey: AWS_ACCESS_KEY_ID + secretAccessKey: {{ imports[imports_airbyte_key].secret_access_key }} + secretAccessKeySecretKey: AWS_SECRET_ACCESS_KEY + bucket: + log: {{ configuration.bucket }} + state: {{ configuration.bucket }} + workloadOutput: {{ configuration.bucket }} + + database: + type: external + database: airbyte + host: {{ imports[imports_airbyte_key].postgres_host }} + port: "5432" + secretName: airbyte-airbyte-secrets + user: airbyte + userSecretKey: DATABASE_USER + password: {{ imports[imports_airbyte_key].postgres_password }} + passwordSecretKey: DATABASE_PASSWORD + +postgresql: + enabled: false + +externalDatabase: + database: airbyte + host: {{ imports[imports_airbyte_key].postgres_host }} + user: airbyte + existingSecret: ~ + password: {{ imports[imports_airbyte_key].postgres_password }} + port: 5432 + +webapp: + ingress: + enabled: false + podAnnotations: + security.plural.sh/oauth-env-secret: airbyte-proxy-config + {% if configuration["basicAuth"] %} + security.plural.sh/htpasswd-secret: httpaswd-users + {% endif %} + podLabels: + security.plural.sh/inject-oauth-sidecar: "true" +{% endraw %} diff --git a/test/outputs/catalogs/data/airbyte/helm/oauth-proxy-config.yaml.liquid b/test/outputs/catalogs/data/airbyte/helm/oauth-proxy-config.yaml.liquid new file mode 100644 index 00000000..d0168f3e --- /dev/null +++ b/test/outputs/catalogs/data/airbyte/helm/oauth-proxy-config.yaml.liquid @@ -0,0 +1,28 @@ +{% raw %} +{% assign imports_airbyte_key = 'airbyte-' | append: configuration.cluster %} + +service: + name: airbyte-oauth2-proxy + selector: + app.kubernetes.io/instance: airbyte + app.kubernetes.io/name: webapp + +secret: + clientID: {{ imports[imports_airbyte_key].oidc_client_id }} + clientSecret: {{ imports[imports_airbyte_key].oidc_client_secret }} + cookieSecret: {{ imports[imports_airbyte_key].oidc_cookie_secret }} + issuer: https://oidc.plural.sh/ + upstream: http://localhost:8080 + name: airbyte-proxy-config + env: + OAUTH2_PROXY_UPSTREAM_TIMEOUT: '120s' + +{% if configuration["basicAuth"] %} +{% assign basicAuth = configuration["basicAuth"] | from_json %} +users: +{% for user in basicAuth %} + {{ user[0] }}: {{ user[1] }} +{% endfor %} +{% endif %} + +{% endraw %} diff --git a/test/outputs/catalogs/data/airbyte/helm/values.yaml.liquid b/test/outputs/catalogs/data/airbyte/helm/values.yaml.liquid deleted file mode 100644 index 12263d0d..00000000 --- a/test/outputs/catalogs/data/airbyte/helm/values.yaml.liquid +++ /dev/null @@ -1,89 +0,0 @@ -{% raw %} -global: - deploymentMode: "oss" - application: - links: - - description: airbyte web ui - url: {{ context.hostname }} - logs: - accessKey: - existingSecret: airbyte-airbyte-secrets - existingSecretKey: AWS_ACCESS_KEY_ID - password: {{ imports["airbyte-{{ context.cluster }}"].access_key_id }} - s3: - bucket: plrl-leadstreams-plrl-mgmt-airbyte - bucketRegion: us-east-2 - enabled: true - secretKey: - existingSecret: airbyte-airbyte-secrets - existingSecretKey: AWS_SECRET_ACCESS_KEY - password: {{ imports["airbyte-{{ context.cluster }}"].secret_access_key }} - storage: - type: S3 - state: - storage: - type: S3 - - database: - secretName: airbyte-airbyte-secrets - secretValue: DATABASE_PASSWORD - host: {{ imports["airbyte-{{ context.cluster }}"].postgres_host }} - -airbyte: - externalDatabase: - database: airbyte - host: {{ imports["airbyte-{{ context.cluster }}"].postgres_host }} - user: airbyte - existingSecret: ~ - password: {{ imports["airbyte-{{ context.cluster }}"].postgres_password }} - port: 5432 - webapp: - ingress: - hosts: - - host: {{ context.hostname }} - paths: - - path: /.* - pathType: ImplementationSpecific - tls: - - hosts: - - {{ context.hostname }} - secretName: airbyte-tls - podAnnotations: - security.plural.sh/oauth-env-secret: airbyte-proxy-config - {% if configuration["basicAuth"] %} - security.plural.sh/htpasswd-secret: httpaswd-users - {% endif %} - podLabels: - security.plural.sh/inject-oauth-sidecar: "true" -oidc-config: - enabled: true - secret: - clientID: {{ imports["airbyte-{{ context.cluster }}"].oidc_client_id }} - clientSecret: {{ imports["airbyte-{{ context.cluster }}"].oidc_client_secret }} - cookieSecret: {{ imports["airbyte-{{ context.cluster }}"].oidc_cookie_secret }} - issuer: https://oidc.plural.sh/ - name: airbyte-proxy-config - -{% if configuration["basicAuth"] %} -{% assign basicAuth = configuration["basicAuth"] | from_json %} - users: - {% for user in basicAuth %} - {{ user[0] }}: {{ user[1] }} - {% endfor %} -{% endif %} - -postgres: - enabled: false -private: - ingress: - enabled: true - hosts: - - host: {{ context.apiHostname }} - paths: - - path: /.* - pathType: ImplementationSpecific - tls: - - hosts: - - {{ context.apiHostname }} - secretName: airbyte-private-tls -{% endraw %} \ No newline at end of file diff --git a/test/outputs/catalogs/data/airbyte/servicedeployment.yaml.liquid b/test/outputs/catalogs/data/airbyte/oauth-proxy-config-servicedeployment.yaml similarity index 53% rename from test/outputs/catalogs/data/airbyte/servicedeployment.yaml.liquid rename to test/outputs/catalogs/data/airbyte/oauth-proxy-config-servicedeployment.yaml index 020d16e8..218a62f7 100644 --- a/test/outputs/catalogs/data/airbyte/servicedeployment.yaml.liquid +++ b/test/outputs/catalogs/data/airbyte/oauth-proxy-config-servicedeployment.yaml @@ -1,12 +1,12 @@ apiVersion: deployments.plural.sh/v1alpha1 kind: ServiceDeployment metadata: - name: airbyte-{{ context.cluster }} - namespace: infra + name: airbyte-oauth-proxy-config-{{ context.cluster }} + namespace: apps spec: namespace: airbyte git: - folder: helm/airbyte + folder: helm/airbyte/{{ context.cluster }} ref: main repositoryRef: kind: GitRepository @@ -14,14 +14,17 @@ spec: namespace: infra helm: version: "x.x.x" - chart: airbyte - url: https://app.plural.sh/cm/airbyte + chart: oidc-config + url: https://pluralsh.github.io/module-library valuesFiles: - - {{ context.cluster }}.yaml.liquid + - oauth-proxy-config.yaml.liquid imports: - stackRef: name: airbyte-{{ context.cluster }} - namespace: infra + namespace: apps + configuration: + cluster: {{ context.cluster }} + hostname: {{ context.hostname }} clusterRef: kind: Cluster name: {{ context.cluster }} diff --git a/test/outputs/catalogs/data/airbyte/services/oauth-proxy-ingress.yaml.liquid b/test/outputs/catalogs/data/airbyte/services/oauth-proxy-ingress.yaml.liquid new file mode 100644 index 00000000..2ba7d998 --- /dev/null +++ b/test/outputs/catalogs/data/airbyte/services/oauth-proxy-ingress.yaml.liquid @@ -0,0 +1,33 @@ +{% raw %} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: airbyte-webapp-proxy + namespace: airbyte + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + kubernetes.io/tls-acme: "true" + # Extend timeout to allow long running queries. + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/use-regex: "true" +spec: + ingressClassName: nginx + rules: + - host: {{ configuration.hostname }} + http: + paths: + - backend: + service: + name: airbyte-oauth2-proxy + port: + number: 80 + path: /.* + pathType: ImplementationSpecific + tls: + - hosts: + - {{ configuration.hostname }} + secretName: airbyte-tls +{% endraw %} diff --git a/test/outputs/catalogs/data/airbyte/terraform/aws/iam.tf b/test/outputs/catalogs/data/airbyte/terraform/aws/iam.tf index 5f199f73..c8afdc5e 100644 --- a/test/outputs/catalogs/data/airbyte/terraform/aws/iam.tf +++ b/test/outputs/catalogs/data/airbyte/terraform/aws/iam.tf @@ -6,7 +6,9 @@ resource "aws_iam_policy" "airbyte" { } resource "aws_iam_user" "airbyte" { - name = "${var.cluster_name}-airbyte" + name = "${data.plural_cluster.cluster.name}-airbyte" + + depends_on = [ data.plural_cluster.cluster ] } resource "aws_iam_access_key" "airbyte" { @@ -27,7 +29,9 @@ data "aws_iam_policy_document" "airbyte" { } resource "aws_iam_policy_attachment" "airbyte-user" { - name = "${var.cluster_name}-airbyte-policy" + name = "${data.plural_cluster.cluster.name}-airbyte-policy" users = [aws_iam_user.airbyte.name] policy_arn = aws_iam_policy.airbyte.arn + + depends_on = [ data.plural_cluster.cluster ] } diff --git a/test/outputs/catalogs/data/airbyte/terraform/aws/oidc.tf b/test/outputs/catalogs/data/airbyte/terraform/aws/oidc.tf index ca2084d8..8af5cc3a 100644 --- a/test/outputs/catalogs/data/airbyte/terraform/aws/oidc.tf +++ b/test/outputs/catalogs/data/airbyte/terraform/aws/oidc.tf @@ -1,5 +1,5 @@ resource "random_password" "oidc_cookie" { - length = 20 + length = 24 min_lower = 1 min_numeric = 1 min_upper = 1 @@ -12,4 +12,4 @@ resource "plural_oidc_provider" "airbyte" { type = "PLURAL" description = "OIDC provider for airbyte deployed to the {{ context.cluster }} cluster" redirect_uris = ["https://{{ context.hostname }}/oauth2/callback"] -} \ No newline at end of file +} diff --git a/test/outputs/catalogs/data/airbyte/terraform/aws/outputs.tf b/test/outputs/catalogs/data/airbyte/terraform/aws/outputs.tf index 9e83e5c1..9466a3df 100644 --- a/test/outputs/catalogs/data/airbyte/terraform/aws/outputs.tf +++ b/test/outputs/catalogs/data/airbyte/terraform/aws/outputs.tf @@ -4,6 +4,7 @@ output "access_key_id" { output "secret_access_key" { value = aws_iam_access_key.airbyte.secret + sensitive = true } output "postgres_host" { @@ -28,4 +29,4 @@ output "oidc_client_id" { output "oidc_client_secret" { value = plural_oidc_provider.airbyte.client_secret sensitive = true -} \ No newline at end of file +} diff --git a/test/outputs/catalogs/data/airbyte/terraform/aws/plural.tf b/test/outputs/catalogs/data/airbyte/terraform/aws/plural.tf new file mode 100644 index 00000000..c30d03d4 --- /dev/null +++ b/test/outputs/catalogs/data/airbyte/terraform/aws/plural.tf @@ -0,0 +1,3 @@ +data "plural_cluster" "cluster" { + handle = var.cluster_name +} diff --git a/test/outputs/catalogs/data/airbyte/terraform/aws/postgres.tf b/test/outputs/catalogs/data/airbyte/terraform/aws/postgres.tf index 6099d659..8c339459 100644 --- a/test/outputs/catalogs/data/airbyte/terraform/aws/postgres.tf +++ b/test/outputs/catalogs/data/airbyte/terraform/aws/postgres.tf @@ -7,7 +7,9 @@ resource "random_password" "password" { } data "aws_eks_cluster" "mgmt" { - name = var.cluster_name + name = data.plural_cluster.cluster.name + + depends_on = [ data.plural_cluster.cluster ] } data "aws_vpc" "mgmt" { diff --git a/test/outputs/catalogs/data/airbyte/terraform/aws/s3.tf b/test/outputs/catalogs/data/airbyte/terraform/aws/s3.tf index 5f6d8fc8..cba4d9c1 100644 --- a/test/outputs/catalogs/data/airbyte/terraform/aws/s3.tf +++ b/test/outputs/catalogs/data/airbyte/terraform/aws/s3.tf @@ -12,7 +12,3 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "airbyte" { } } } - -data "aws_iam_role" "postgres" { - name = "${var.cluster_name}-postgres" -} diff --git a/test/outputs/documentation/airbyte/README.md b/test/outputs/documentation/airbyte/README.md index 41dbfc5e..d273998b 100644 --- a/test/outputs/documentation/airbyte/README.md +++ b/test/outputs/documentation/airbyte/README.md @@ -6,7 +6,7 @@ This is a baseline, prod ready airbyte installation using Plural. It includes a * RDS/Google Cloud Sql, Azure Flexible Server to handle postgres. This gives you a robust RDBMS service to hold airbyte's core transactional data. * Plural OIDC to handle authentication to Airbyte. Airbyte does not support this natively, and so we use oauth-proxy as a middleware to handle authentication. -In addtion, there are a few common customizations you might want to do. +In addition, there are a few common customizations you might want to do. ## Configure Basic Auth @@ -80,4 +80,4 @@ when building your application. ## Contributing -If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds \ No newline at end of file +If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds diff --git a/test/outputs/helm/airbyte/mgmt.yaml.liquid b/test/outputs/helm/airbyte/mgmt.yaml.liquid deleted file mode 100644 index b2a3dbe0..00000000 --- a/test/outputs/helm/airbyte/mgmt.yaml.liquid +++ /dev/null @@ -1,88 +0,0 @@ - -global: - deploymentMode: "oss" - application: - links: - - description: airbyte web ui - url: {{ context.hostname }} - logs: - accessKey: - existingSecret: airbyte-airbyte-secrets - existingSecretKey: AWS_ACCESS_KEY_ID - password: {{ imports["airbyte-{{ context.cluster }}"].access_key_id }} - s3: - bucket: plrl-leadstreams-plrl-mgmt-airbyte - bucketRegion: us-east-2 - enabled: true - secretKey: - existingSecret: airbyte-airbyte-secrets - existingSecretKey: AWS_SECRET_ACCESS_KEY - password: {{ imports["airbyte-{{ context.cluster }}"].secret_access_key }} - storage: - type: S3 - state: - storage: - type: S3 - - database: - secretName: airbyte-airbyte-secrets - secretValue: DATABASE_PASSWORD - host: {{ imports["airbyte-{{ context.cluster }}"].postgres_host }} - -airbyte: - externalDatabase: - database: airbyte - host: {{ imports["airbyte-{{ context.cluster }}"].postgres_host }} - user: airbyte - existingSecret: ~ - password: {{ imports["airbyte-{{ context.cluster }}"].postgres_password }} - port: 5432 - webapp: - ingress: - hosts: - - host: {{ context.hostname }} - paths: - - path: /.* - pathType: ImplementationSpecific - tls: - - hosts: - - {{ context.hostname }} - secretName: airbyte-tls - podAnnotations: - security.plural.sh/oauth-env-secret: airbyte-proxy-config - {% if configuration["basicAuth"] %} - security.plural.sh/htpasswd-secret: httpaswd-users - {% endif %} - podLabels: - security.plural.sh/inject-oauth-sidecar: "true" -oidc-config: - enabled: true - secret: - clientID: {{ imports["airbyte-{{ context.cluster }}"].oidc_client_id }} - clientSecret: {{ imports["airbyte-{{ context.cluster }}"].oidc_client_secret }} - cookieSecret: {{ imports["airbyte-{{ context.cluster }}"].oidc_cookie_secret }} - issuer: https://oidc.plural.sh/ - name: airbyte-proxy-config - -{% if configuration["basicAuth"] %} -{% assign basicAuth = configuration["basicAuth"] | from_json %} - users: - {% for user in basicAuth %} - {{ user[0] }}: {{ user[1] }} - {% endfor %} -{% endif %} - -postgres: - enabled: false -private: - ingress: - enabled: true - hosts: - - host: {{ context.apiHostname }} - paths: - - path: /.* - pathType: ImplementationSpecific - tls: - - hosts: - - {{ context.apiHostname }} - secretName: airbyte-private-tls diff --git a/test/outputs/helm/airbyte/mgmt/airbyte.yaml.liquid b/test/outputs/helm/airbyte/mgmt/airbyte.yaml.liquid new file mode 100644 index 00000000..e0cc77e5 --- /dev/null +++ b/test/outputs/helm/airbyte/mgmt/airbyte.yaml.liquid @@ -0,0 +1,57 @@ + +{% assign imports_airbyte_key = 'airbyte-' | append: configuration.cluster %} + +global: + deploymentMode: oss + edition: community + + airbyteUrl: {{ configuration.hostname }} + + storage: + type: S3 + storageSecretName: airbyte-airbyte-secrets + s3: + region: {{ configuration.region }} + authenticationType: credentials + accessKeyId: {{ imports[imports_airbyte_key].access_key_id }} + accessKeyIdSecretKey: AWS_ACCESS_KEY_ID + secretAccessKey: {{ imports[imports_airbyte_key].secret_access_key }} + secretAccessKeySecretKey: AWS_SECRET_ACCESS_KEY + bucket: + log: {{ configuration.bucket }} + state: {{ configuration.bucket }} + workloadOutput: {{ configuration.bucket }} + + database: + type: external + database: airbyte + host: {{ imports[imports_airbyte_key].postgres_host }} + port: "5432" + secretName: airbyte-airbyte-secrets + user: airbyte + userSecretKey: DATABASE_USER + password: {{ imports[imports_airbyte_key].postgres_password }} + passwordSecretKey: DATABASE_PASSWORD + +postgresql: + enabled: false + +externalDatabase: + database: airbyte + host: {{ imports[imports_airbyte_key].postgres_host }} + user: airbyte + existingSecret: ~ + password: {{ imports[imports_airbyte_key].postgres_password }} + port: 5432 + +webapp: + ingress: + enabled: false + podAnnotations: + security.plural.sh/oauth-env-secret: airbyte-proxy-config + {% if configuration["basicAuth"] %} + security.plural.sh/htpasswd-secret: httpaswd-users + {% endif %} + podLabels: + security.plural.sh/inject-oauth-sidecar: "true" + diff --git a/test/outputs/helm/airbyte/mgmt/oauth-proxy-config.yaml.liquid b/test/outputs/helm/airbyte/mgmt/oauth-proxy-config.yaml.liquid new file mode 100644 index 00000000..c469f9dc --- /dev/null +++ b/test/outputs/helm/airbyte/mgmt/oauth-proxy-config.yaml.liquid @@ -0,0 +1,28 @@ + +{% assign imports_airbyte_key = 'airbyte-' | append: configuration.cluster %} + +service: + name: airbyte-oauth2-proxy + selector: + app.kubernetes.io/instance: airbyte + app.kubernetes.io/name: webapp + +secret: + clientID: {{ imports[imports_airbyte_key].oidc_client_id }} + clientSecret: {{ imports[imports_airbyte_key].oidc_client_secret }} + cookieSecret: {{ imports[imports_airbyte_key].oidc_cookie_secret }} + issuer: https://oidc.plural.sh/ + upstream: http://localhost:8080 + name: airbyte-proxy-config + env: + OAUTH2_PROXY_UPSTREAM_TIMEOUT: '120s' + +{% if configuration["basicAuth"] %} +{% assign basicAuth = configuration["basicAuth"] | from_json %} +users: +{% for user in basicAuth %} + {{ user[0] }}: {{ user[1] }} +{% endfor %} +{% endif %} + + diff --git a/test/outputs/services/apps/airbyte/oauth-proxy-ingress.yaml.liquid b/test/outputs/services/apps/airbyte/oauth-proxy-ingress.yaml.liquid new file mode 100644 index 00000000..cad4886a --- /dev/null +++ b/test/outputs/services/apps/airbyte/oauth-proxy-ingress.yaml.liquid @@ -0,0 +1,33 @@ + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: airbyte-webapp-proxy + namespace: airbyte + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + kubernetes.io/tls-acme: "true" + # Extend timeout to allow long running queries. + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/use-regex: "true" +spec: + ingressClassName: nginx + rules: + - host: {{ configuration.hostname }} + http: + paths: + - backend: + service: + name: airbyte-oauth2-proxy + port: + number: 80 + path: /.* + pathType: ImplementationSpecific + tls: + - hosts: + - {{ configuration.hostname }} + secretName: airbyte-tls + diff --git a/test/outputs/terraform/apps/airbyte/aws/iam.tf b/test/outputs/terraform/apps/airbyte/mgmt/iam.tf similarity index 76% rename from test/outputs/terraform/apps/airbyte/aws/iam.tf rename to test/outputs/terraform/apps/airbyte/mgmt/iam.tf index 5f199f73..c8afdc5e 100644 --- a/test/outputs/terraform/apps/airbyte/aws/iam.tf +++ b/test/outputs/terraform/apps/airbyte/mgmt/iam.tf @@ -6,7 +6,9 @@ resource "aws_iam_policy" "airbyte" { } resource "aws_iam_user" "airbyte" { - name = "${var.cluster_name}-airbyte" + name = "${data.plural_cluster.cluster.name}-airbyte" + + depends_on = [ data.plural_cluster.cluster ] } resource "aws_iam_access_key" "airbyte" { @@ -27,7 +29,9 @@ data "aws_iam_policy_document" "airbyte" { } resource "aws_iam_policy_attachment" "airbyte-user" { - name = "${var.cluster_name}-airbyte-policy" + name = "${data.plural_cluster.cluster.name}-airbyte-policy" users = [aws_iam_user.airbyte.name] policy_arn = aws_iam_policy.airbyte.arn + + depends_on = [ data.plural_cluster.cluster ] } diff --git a/test/outputs/terraform/apps/airbyte/aws/oidc.tf b/test/outputs/terraform/apps/airbyte/mgmt/oidc.tf similarity index 94% rename from test/outputs/terraform/apps/airbyte/aws/oidc.tf rename to test/outputs/terraform/apps/airbyte/mgmt/oidc.tf index eaecbdca..b803dffc 100644 --- a/test/outputs/terraform/apps/airbyte/aws/oidc.tf +++ b/test/outputs/terraform/apps/airbyte/mgmt/oidc.tf @@ -1,5 +1,5 @@ resource "random_password" "oidc_cookie" { - length = 20 + length = 24 min_lower = 1 min_numeric = 1 min_upper = 1 @@ -12,4 +12,4 @@ resource "plural_oidc_provider" "airbyte" { type = "PLURAL" description = "OIDC provider for airbyte deployed to the mgmt cluster" redirect_uris = ["https://airbyte.plural.sh/oauth2/callback"] -} \ No newline at end of file +} diff --git a/test/outputs/terraform/apps/airbyte/aws/outputs.tf b/test/outputs/terraform/apps/airbyte/mgmt/outputs.tf similarity index 96% rename from test/outputs/terraform/apps/airbyte/aws/outputs.tf rename to test/outputs/terraform/apps/airbyte/mgmt/outputs.tf index 9e83e5c1..9466a3df 100644 --- a/test/outputs/terraform/apps/airbyte/aws/outputs.tf +++ b/test/outputs/terraform/apps/airbyte/mgmt/outputs.tf @@ -4,6 +4,7 @@ output "access_key_id" { output "secret_access_key" { value = aws_iam_access_key.airbyte.secret + sensitive = true } output "postgres_host" { @@ -28,4 +29,4 @@ output "oidc_client_id" { output "oidc_client_secret" { value = plural_oidc_provider.airbyte.client_secret sensitive = true -} \ No newline at end of file +} diff --git a/test/outputs/terraform/apps/airbyte/mgmt/plural.tf b/test/outputs/terraform/apps/airbyte/mgmt/plural.tf new file mode 100644 index 00000000..c30d03d4 --- /dev/null +++ b/test/outputs/terraform/apps/airbyte/mgmt/plural.tf @@ -0,0 +1,3 @@ +data "plural_cluster" "cluster" { + handle = var.cluster_name +} diff --git a/test/outputs/terraform/apps/airbyte/aws/postgres.tf b/test/outputs/terraform/apps/airbyte/mgmt/postgres.tf similarity index 95% rename from test/outputs/terraform/apps/airbyte/aws/postgres.tf rename to test/outputs/terraform/apps/airbyte/mgmt/postgres.tf index 6099d659..8c339459 100644 --- a/test/outputs/terraform/apps/airbyte/aws/postgres.tf +++ b/test/outputs/terraform/apps/airbyte/mgmt/postgres.tf @@ -7,7 +7,9 @@ resource "random_password" "password" { } data "aws_eks_cluster" "mgmt" { - name = var.cluster_name + name = data.plural_cluster.cluster.name + + depends_on = [ data.plural_cluster.cluster ] } data "aws_vpc" "mgmt" { diff --git a/test/outputs/terraform/apps/airbyte/aws/s3.tf b/test/outputs/terraform/apps/airbyte/mgmt/s3.tf similarity index 81% rename from test/outputs/terraform/apps/airbyte/aws/s3.tf rename to test/outputs/terraform/apps/airbyte/mgmt/s3.tf index 5f6d8fc8..cba4d9c1 100644 --- a/test/outputs/terraform/apps/airbyte/aws/s3.tf +++ b/test/outputs/terraform/apps/airbyte/mgmt/s3.tf @@ -12,7 +12,3 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "airbyte" { } } } - -data "aws_iam_role" "postgres" { - name = "${var.cluster_name}-postgres" -} diff --git a/test/outputs/terraform/apps/airbyte/aws/variables.tf b/test/outputs/terraform/apps/airbyte/mgmt/variables.tf similarity index 100% rename from test/outputs/terraform/apps/airbyte/aws/variables.tf rename to test/outputs/terraform/apps/airbyte/mgmt/variables.tf diff --git a/test/outputs/terraform/apps/airbyte/aws/versions.tf b/test/outputs/terraform/apps/airbyte/mgmt/versions.tf similarity index 100% rename from test/outputs/terraform/apps/airbyte/aws/versions.tf rename to test/outputs/terraform/apps/airbyte/mgmt/versions.tf