diff --git a/catalogs/data/airbyte/airbyte-raw-servicedeployment.yaml b/catalogs/data/airbyte/airbyte-raw-servicedeployment.yaml new file mode 100644 index 00000000..5a10d6a8 --- /dev/null +++ b/catalogs/data/airbyte/airbyte-raw-servicedeployment.yaml @@ -0,0 +1,20 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: airbyte-raw-{{ context.cluster }} + namespace: apps +spec: + namespace: airbyte + git: + folder: services/ingress/airbyte + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + configuration: + hostname: {{ context.hostname }} + clusterRef: + kind: Cluster + name: {{ cluster.handle }} + namespace: infra diff --git a/catalogs/data/airbyte/servicedeployment.yaml.liquid b/catalogs/data/airbyte/airbyte-servicedeployment.yaml similarity index 70% rename from catalogs/data/airbyte/servicedeployment.yaml.liquid rename to catalogs/data/airbyte/airbyte-servicedeployment.yaml index 4788e118..3903cd07 100644 --- a/catalogs/data/airbyte/servicedeployment.yaml.liquid +++ b/catalogs/data/airbyte/airbyte-servicedeployment.yaml @@ -6,7 +6,7 @@ metadata: spec: namespace: airbyte git: - folder: helm/airbyte + folder: helm/airbyte/{{ context.cluster }} ref: main repositoryRef: kind: GitRepository @@ -15,11 +15,11 @@ spec: helm: version: "1.x.x" chart: airbyte - repository: - name: airbyte - namespace: apps + release: airbyte + ignoreHooks: false + url: https://airbytehq.github.io/helm-charts valuesFiles: - - {{ context.cluster }}.yaml.liquid + - airbyte.yaml.liquid imports: - stackRef: name: airbyte-{{ context.cluster }} @@ -27,7 +27,8 @@ spec: configuration: cluster: {{ context.cluster }} hostname: {{ context.hostname }} + bucket: {{ context.bucket }} clusterRef: kind: Cluster - name: {{ context.cluster }} + name: {{ cluster.handle }} namespace: infra diff --git a/catalogs/data/airbyte/stack.yaml.liquid b/catalogs/data/airbyte/airbyte-stack.yaml similarity index 77% rename from catalogs/data/airbyte/stack.yaml.liquid rename to catalogs/data/airbyte/airbyte-stack.yaml index 94c96633..ebecdf23 100644 --- a/catalogs/data/airbyte/stack.yaml.liquid +++ b/catalogs/data/airbyte/airbyte-stack.yaml @@ -2,20 +2,21 @@ apiVersion: deployments.plural.sh/v1alpha1 kind: InfrastructureStack metadata: name: airbyte-{{ context.cluster }} + namespace: apps spec: detach: false type: TERRAFORM approval: true manageState: true actor: console@plural.sh - configuration: - version: '1.8' + git: + ref: main + folder: terraform/apps/airbyte/{{ context.cluster }} repositoryRef: name: infra namespace: infra + configuration: + version: '1.8' clusterRef: - name: mgmt + name: {{ cluster.handle }} namespace: infra - git: - ref: main - folder: terraform/apps/airbyte/{{ context.cloud }} \ No newline at end of file diff --git a/catalogs/data/airbyte/helm/airbyte.yaml.liquid b/catalogs/data/airbyte/helm/airbyte.yaml.liquid new file mode 100644 index 00000000..4051f35e --- /dev/null +++ b/catalogs/data/airbyte/helm/airbyte.yaml.liquid @@ -0,0 +1,57 @@ +{% raw %} +{% assign imports_airbyte_key = 'airbyte-' | append: configuration.cluster %} + +global: + deploymentMode: oss + edition: community + + airbyteUrl: {{ configuration.hostname }} + + storage: + type: S3 + storageSecretName: airbyte-airbyte-secrets + s3: + region: us-east-2 + authenticationType: credentials + accessKeyId: {{ imports[imports_airbyte_key].access_key_id }} + accessKeyIdSecretKey: AWS_ACCESS_KEY_ID + secretAccessKey: {{ imports[imports_airbyte_key].secret_access_key }} + secretAccessKeySecretKey: AWS_SECRET_ACCESS_KEY + bucket: + log: {{ configuration.bucket }} + state: {{ configuration.bucket }} + workloadOutput: {{ configuration.bucket }} + + database: + type: external + database: airbyte + host: {{ imports[imports_airbyte_key].postgres_host }} + port: "5432" + secretName: airbyte-airbyte-secrets + user: airbyte + userSecretKey: DATABASE_USER + password: {{ imports[imports_airbyte_key].postgres_password }} + passwordSecretKey: DATABASE_PASSWORD + +postgresql: + enabled: false + +externalDatabase: + database: airbyte + host: {{ imports[imports_airbyte_key].postgres_host }} + user: airbyte + existingSecret: ~ + password: {{ imports[imports_airbyte_key].postgres_password }} + port: 5432 + +webapp: + ingress: + enabled: false + podAnnotations: + security.plural.sh/oauth-env-secret: airbyte-proxy-config + {% if configuration["basicAuth"] %} + security.plural.sh/htpasswd-secret: httpaswd-users + {% endif %} + podLabels: + security.plural.sh/inject-oauth-sidecar: "true" +{% endraw %} diff --git a/catalogs/data/airbyte/helm/oauth-proxy-config.yaml.liquid b/catalogs/data/airbyte/helm/oauth-proxy-config.yaml.liquid new file mode 100644 index 00000000..7a49aea2 --- /dev/null +++ b/catalogs/data/airbyte/helm/oauth-proxy-config.yaml.liquid @@ -0,0 +1,28 @@ +{% raw %} +{% assign imports_airbyte_key = 'airbyte-' | append: configuration.cluster %} + +service: + name: airbyte-oauth2-proxy + selector: + app.kubernetes.io/instance: airbyte + app.kubernetes.io/name: webapp + +secret: + clientID: {{ imports[imports_airbyte_key].oidc_client_id }} + clientSecret: {{ imports[imports_airbyte_key].oidc_client_secret }} + cookieSecret: {{ imports[imports_airbyte_key].oidc_cookie_secret }} + issuer: https://oidc.plural.sh/ + upstream: http://localhost:8080 + name: airbyte-proxy-config + env: + OAUTH2_PROXY_UPSTREAM_TIMEOUT: '120s' + +{% if configuration["basicAuth"] %} +{% assign basicAuth = configuration["basicAuth"] | from_json %} + users: +{% for user in basicAuth %} + {{ user[0] }}: {{ user[1] }} +{% endfor %} +{% endif %} + +{% endraw %} diff --git a/catalogs/data/airbyte/helm/values.yaml.liquid b/catalogs/data/airbyte/helm/values.yaml.liquid deleted file mode 100644 index 7a2cbe3c..00000000 --- a/catalogs/data/airbyte/helm/values.yaml.liquid +++ /dev/null @@ -1,96 +0,0 @@ -{% raw %} -global: - deploymentMode: "oss" - edition: "community" - - application: - links: - - description: airbyte web ui - url: {{ context.hostname }} - - logs: - accessKey: - existingSecret: airbyte-airbyte-secrets - existingSecretKey: AWS_ACCESS_KEY_ID - password: {{ imports["airbyte-{{ context.cluster }}"].access_key_id }} - s3: - bucket: plrl-leadstreams-plrl-mgmt-airbyte - bucketRegion: us-east-2 - enabled: true - secretKey: - existingSecret: airbyte-airbyte-secrets - existingSecretKey: AWS_SECRET_ACCESS_KEY - password: {{ imports["airbyte-{{ context.cluster }}"].secret_access_key }} - storage: - type: S3 - - state: - storage: - type: S3 - - database: - secretName: airbyte-airbyte-secrets - secretValue: DATABASE_PASSWORD - host: {{ imports["airbyte-{{ context.cluster }}"].postgres_host }} - -airbyte: - externalDatabase: - database: airbyte - host: {{ imports["airbyte-{{ context.cluster }}"].postgres_host }} - user: airbyte - existingSecret: ~ - password: {{ imports["airbyte-{{ context.cluster }}"].postgres_password }} - port: 5432 - - webapp: - ingress: - hosts: - - host: {{ context.hostname }} - paths: - - path: /.* - pathType: ImplementationSpecific - tls: - - hosts: - - {{ context.hostname }} - secretName: airbyte-tls - podAnnotations: - security.plural.sh/oauth-env-secret: airbyte-proxy-config - {% if configuration["basicAuth"] %} - security.plural.sh/htpasswd-secret: httpaswd-users - {% endif %} - podLabels: - security.plural.sh/inject-oauth-sidecar: "true" - -oidc-config: - enabled: true - secret: - clientID: {{ imports["airbyte-{{ context.cluster }}"].oidc_client_id }} - clientSecret: {{ imports["airbyte-{{ context.cluster }}"].oidc_client_secret }} - cookieSecret: {{ imports["airbyte-{{ context.cluster }}"].oidc_cookie_secret }} - issuer: https://oidc.plural.sh/ - name: airbyte-proxy-config - -{% if configuration["basicAuth"] %} -{% assign basicAuth = configuration["basicAuth"] | from_json %} - users: - {% for user in basicAuth %} - {{ user[0] }}: {{ user[1] }} - {% endfor %} -{% endif %} - -postgres: - enabled: false - -private: - ingress: - enabled: true - hosts: - - host: {{ context.apiHostname }} - paths: - - path: /.* - pathType: ImplementationSpecific - tls: - - hosts: - - {{ context.apiHostname }} - secretName: airbyte-private-tls -{% endraw %} diff --git a/catalogs/data/airbyte/helmrepository.yaml b/catalogs/data/airbyte/helmrepository.yaml deleted file mode 100644 index c258a051..00000000 --- a/catalogs/data/airbyte/helmrepository.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: source.toolkit.fluxcd.io/v1beta2 -kind: HelmRepository -metadata: - name: airbyte - namespace: apps -spec: - interval: 5m0s - url: https://airbytehq.github.io/helm-charts diff --git a/catalogs/data/airbyte/oauth-proxy-config-servicedeployment.yaml b/catalogs/data/airbyte/oauth-proxy-config-servicedeployment.yaml new file mode 100644 index 00000000..59b9b7bc --- /dev/null +++ b/catalogs/data/airbyte/oauth-proxy-config-servicedeployment.yaml @@ -0,0 +1,31 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: airbyte-oauth-proxy-config-{{ context.cluster }} + namespace: apps +spec: + namespace: airbyte + git: + folder: helm/airbyte/{{ context.cluster }} + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + helm: + version: "x.x.x" + chart: oidc-config + url: https://pluralsh.github.io/module-library + valuesFiles: + - oauth-proxy-config.yaml.liquid + imports: + - stackRef: + name: airbyte-{{ context.cluster }} + namespace: infra + configuration: + cluster: {{ context.cluster }} + hostname: {{ context.hostname }} + clusterRef: + kind: Cluster + name: {{ cluster.handle }} + namespace: infra diff --git a/catalogs/data/airbyte/services/oauth-proxy-ingress.yaml.liquid b/catalogs/data/airbyte/services/oauth-proxy-ingress.yaml.liquid new file mode 100644 index 00000000..2ba7d998 --- /dev/null +++ b/catalogs/data/airbyte/services/oauth-proxy-ingress.yaml.liquid @@ -0,0 +1,33 @@ +{% raw %} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: airbyte-webapp-proxy + namespace: airbyte + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + kubernetes.io/tls-acme: "true" + # Extend timeout to allow long running queries. + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/use-regex: "true" +spec: + ingressClassName: nginx + rules: + - host: {{ configuration.hostname }} + http: + paths: + - backend: + service: + name: airbyte-oauth2-proxy + port: + number: 80 + path: /.* + pathType: ImplementationSpecific + tls: + - hosts: + - {{ configuration.hostname }} + secretName: airbyte-tls +{% endraw %} diff --git a/setup/catalogs/data/airbyte.yaml b/setup/catalogs/data/airbyte.yaml index f7bd9bb0..5da9e133 100644 --- a/setup/catalogs/data/airbyte.yaml +++ b/setup/catalogs/data/airbyte.yaml @@ -12,20 +12,29 @@ spec: ref: main folder: catalogs/data/airbyte templates: - - source: stack.yaml.liquid - destination: "bootstrap/apps/airbyte/{{ context.cluster }}/stack.yaml" + - source: helm + destination: helm/airbyte/{{ context.cluster }} external: true - - source: servicedeployment.yaml.liquid - destination: "bootstrap/apps/airbyte/{{ context.cluster }}/servicedeployment.yaml" + - source: services/oauth-proxy-ingress.yaml.liquid + destination: services/ingress/airbyte/oauth-proxy-ingress.yaml.liquid external: true - - source: README.md - destination: documentation/airbyte/README.md + - source: terraform + destination: terraform/apps/airbyte/{{ context.cluster }} external: true - - source: helm/values.yaml.liquid - destination: helm/airbyte/{{ context.cluster }}.yaml.liquid + - source: airbyte-raw-servicedeployment.yaml + destination: "bootstrap/apps/airbyte/{{ context.cluster }}/airbyte-raw-servicedeployment.yaml" external: true - - source: terraform - destination: terraform/apps/airbyte + - source: airbyte-servicedeployment.yaml + destination: "bootstrap/apps/airbyte/{{ context.cluster }}/airbyte-servicedeployment.yaml" + external: true + - source: airbyte-stack.yaml + destination: "bootstrap/apps/airbyte/{{ context.cluster }}/airbyte-stack.yaml" + external: true + - source: oauth-proxy-config-servicedeployment.yaml + destination: "bootstrap/apps/airbyte/{{ context.cluster }}/oauth-proxy-config-servicedeployment.yaml" + external: true + - source: README.md + destination: documentation/airbyte/README.md external: true repositoryRef: name: scaffolds