Add initial version of ppdb-replication application

Add the initial version of an application for replicating data between
the APDB and the PPDB.

applications/ppdb-replication/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

applications/ppdb-replication/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v2
+appVersion: 0.1.0
+description: Replicates data from the APDB to the PPDB
+name: ppdb-replication
+sources:
+- https://github.com/lsst/dax_ppdb.git
+type: application
+version: 1.0.0

applications/ppdb-replication/README.md

@@ -0,0 +1,44 @@
+# ppdb-replication
+
+Replicates data from the APDB to the PPDB
+
+## Source Code
+
+* <https://github.com/lsst/dax_ppdb.git>
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` | Affinity rules for the ppdb-replication deployment pod |
+| config.additionalS3ProfileName | string | `nil` | Additional S3 profile name |
+| config.additionalS3ProfileUrl | string | `nil` | Additional S3 profile URL |
+| config.apdbConfig | string | `nil` | APDB config file resource |
+| config.apdbIndexUri | string | `nil` | APDB index URI |
+| config.checkInterval | string | `nil` | Time to wait before checking for new chunks, if no chunk appears |
+| config.disableBucketValidation | int | `1` | Disable bucket validation in LSST S3 tools |
+| config.logLevel | string | `"INFO"` | Logging level |
+| config.logProfile | string | `"production"` | Logging profile (`production` for JSON, `development` for human-friendly) |
+| config.maxWaitTime | string | `nil` | Maximum time to wait before replicating a chunk after next chunk appears |
+| config.minWaitTime | string | `nil` | Minimum time to wait before replicating a chunk after next chunk appears |
+| config.monLogger | string | `"lsst.dax.ppdb.monitor"` | Name of logger for monitoring |
+| config.monRules | string | `nil` | Comma-separated list of monitoring filter rules |
+| config.pathPrefix | string | `"/ppdb-replication"` | URL path prefix |
+| config.persistentVolumeClaims | list | `[]` | Persistent volume claims |
+| config.ppdbConfig | string | `nil` | PPDB config file resource |
+| config.s3EndpointUrl | string | `nil` | S3 endpoint URL |
+| config.updateExisting | bool | `false` | Allow updates to already replicated data |
+| config.volumeMounts | list | `[]` | Volume mounts |
+| config.volumes | list | `[]` | Volumes specific to the environment |
+| global.baseUrl | string | Set by Argo CD | Base URL for the environment |
+| global.host | string | Set by Argo CD | Host name for ingress |
+| global.vaultSecretsPath | string | Set by Argo CD | Base path for Vault secrets |
+| image.pullPolicy | string | `"Always"` | Pull policy for the ppdb-replication image |
+| image.repository | string | `"ghcr.io/lsst/ppdb-replication"` | Image to use in the ppdb-replication deployment |
+| image.tag | string | The appVersion of the chart | Tag of image to use |
+| ingress.annotations | object | `{}` | Additional annotations for the ingress rule |
+| nodeSelector | object | `{}` | Node selection rules for the ppdb-replication deployment pod |
+| podAnnotations | object | `{}` | Annotations for the ppdb-replication deployment pod |
+| replicaCount | int | `1` | Number of deployment pods to start |
+| resources | object | see `values.yaml` | Resource limits and requests for the ppdb-replication deployment pod |
+| tolerations | list | `[]` | Tolerations for the ppdb-replication deployment pod |

applications/ppdb-replication/secrets.yaml

@@ -0,0 +1,9 @@
+"aws-credentials.ini":
+  description: >-
+    AWS credentials required for acessing configuration files in S3.
+"db-auth.yaml":
+  description: >-
+    Cassandra database credentials for the APDB.
+"postgres-credentials.txt":
+  description: >-
+    PostgreSQL credentials in its pgpass format for the PPDB database.

applications/ppdb-replication/templates/_helpers.tpl

@@ -0,0 +1,44 @@
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "ppdb-replication.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "ppdb-replication.labels" -}}
+helm.sh/chart: {{ include "ppdb-replication.chart" . }}
+{{ include "ppdb-replication.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "ppdb-replication.selectorLabels" -}}
+app.kubernetes.io/name: "ppdb-replication"
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "ppdb-replication.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}

applications/ppdb-replication/templates/configmap.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "ppdb-replication"
+  labels:
+    {{- include "ppdb-replication.labels" . | nindent 4 }}
+data:
+  DAX_APDB_INDEX_URI: {{ .Values.config.apdbIndexUri | quote }}
+  PPDB_REPLICATION_LOG_LEVEL: {{ .Values.config.logLevel | quote }}
+  PPDB_REPLICATION_PATH_PREFIX: {{ .Values.config.pathPrefix | quote }}
+  PPDB_REPLICATION_PROFILE: {{ .Values.config.logProfile | quote }}
+  PPDB_REPLICATION_APDB_CONFIG: {{ .Values.config.apdbConfig | quote }}
+  PPDB_REPLICATION_PPDB_CONFIG: {{ .Values.config.ppdbConfig | quote }}
+  PPDB_REPLICATION_MON_LOGGER: {{ .Values.config.monLogger | quote }}
+  PPDB_REPLICATION_MON_RULES: {{ .Values.config.monRules | quote }}
+  PPDB_REPLICATION_UPDATE_EXISTING: {{ .Values.config.updateExisting | quote}}
+  PPDB_REPLICATION_MIN_WAIT_TIME: {{ .Values.config.minWaitTime | quote }}
+  PPDB_REPLICATION_MAX_WAIT_TIME: {{ .Values.config.maxWaitTime | quote }}
+  PPDB_REPLICATION_CHECK_INTERVAL: {{ .Values.config.checkInterval | quote}}

applications/ppdb-replication/templates/deployment.yaml

@@ -0,0 +1,89 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "ppdb-replication.fullname" . }}
+  labels:
+    {{- include "ppdb-replication.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "ppdb-replication.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "ppdb-replication.selectorLabels" . | nindent 8 }}
+      annotations:
+        # Force the pod to restart when the config maps are updated.
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      volumes:
+        - name: "ppdb-replication-secrets-raw"
+          secret:
+            secretName: {{ include "ppdb-replication.fullname" . }}
+        - name: "ppdb-replication-secrets"
+          emptyDir:
+            sizeLimit: "100Mi"
+        {{- with .Values.config.volumes }}
+        {{- . | toYaml | nindent 8 }}
+        {{- end }}
+      initContainers:
+        - name: fix-secret-permissions
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
+          command:
+            - "/bin/sh"
+            - "-c"
+            - |
+              cp -RL /tmp/ppdb-replication-secrets-raw/* /app/secrets/
+              chmod 0400 /app/secrets/*
+          securityContext:
+            runAsNonRoot: false
+            runAsUser: 0
+            runAsGroup: 0
+          volumeMounts:
+            - name: "ppdb-replication-secrets"
+              mountPath: "/app/secrets"
+            - name: "ppdb-replication-secrets-raw"
+              mountPath: "/tmp/ppdb-replication-secrets-raw"
+              readOnly: true
+      containers:
+        - name: {{ .Chart.Name }}
+          envFrom:
+            - configMapRef:
+                name: "ppdb-replication"
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+            - name: AWS_SHARED_CREDENTIALS_FILE
+              value: "/app/secrets/aws-credentials.ini"
+            - name: PGPASSFILE
+              value: "/app/secrets/postgres-credentials.txt"
+            - name: LSST_DB_AUTH
+              value: "/app/secrets/db-auth.yaml"
+            - name: S3_ENDPOINT_URL
+              value: {{ .Values.config.s3EndpointUrl | quote }}
+            - name: LSST_RESOURCES_S3_PROFILE_{{ .Values.config.additionalS3ProfileName }}
+              value: {{ .Values.config.additionalS3ProfileUrl | quote }}
+            - name: LSST_DISABLE_BUCKET_VALIDATION
+              value: {{ .Values.config.disableBucketValidation | quote }}
+          volumeMounts:
+            - name: "ppdb-replication-secrets"
+              mountPath: "/app/secrets"
+              readOnly: true
+            {{- with .Values.config.volumeMounts }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

applications/ppdb-replication/templates/ingress.yaml

@@ -0,0 +1,30 @@
+apiVersion: gafaelfawr.lsst.io/v1alpha1
+kind: GafaelfawrIngress
+metadata:
+  name: "ppdb-replication"
+  labels:
+    {{- include "ppdb-replication.labels" . | nindent 4 }}
+config:
+  baseUrl: {{ .Values.global.baseUrl | quote }}
+  scopes:
+    all:
+      - "read:image"
+template:
+  metadata:
+    name: "ppdb-replication"
+    {{- with .Values.ingress.annotations }}
+    annotations:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+  spec:
+    rules:
+      - host: {{ required "global.host must be set" .Values.global.host | quote }}
+        http:
+          paths:
+            - path: {{ .Values.config.pathPrefix | quote }}
+              pathType: "Prefix"
+              backend:
+                service:
+                  name: "ppdb-replication"
+                  port:
+                    number: 8080

applications/ppdb-replication/templates/networkpolicy.yaml

@@ -0,0 +1,21 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: "ppdb-replication"
+spec:
+  podSelector:
+    matchLabels:
+      {{- include "ppdb-replication.selectorLabels" . | nindent 6 }}
+  policyTypes:
+    - "Ingress"
+  ingress:
+    # Allow inbound access from pods (in any namespace) labeled
+    # gafaelfawr.lsst.io/ingress: true.
+    - from:
+        - namespaceSelector: {}
+          podSelector:
+            matchLabels:
+              gafaelfawr.lsst.io/ingress: "true"
+      ports:
+        - protocol: "TCP"
+          port: 8080

applications/ppdb-replication/templates/pvc.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.config.persistentVolumeClaims }}
+{{- $top := . -}}
+{{- range $index, $pvc := .Values.config.persistentVolumeClaims }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: "{{ $pvc.name }}"
+spec:
+  storageClassName: "{{ $pvc.storageClassName }}"
+  accessModes:
+    - ReadOnlyMany
+  resources:
+    requests:
+      storage: 100Mi
+{{- end }}
+{{- end }}
+

applications/ppdb-replication/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: "ppdb-replication"
+  labels:
+    {{- include "ppdb-replication.labels" . | nindent 4 }}
+spec:
+  type: "ClusterIP"
+  ports:
+    - port: 8080
+      targetPort: "http"
+      protocol: "TCP"
+      name: "http"
+  selector:
+    {{- include "ppdb-replication.selectorLabels" . | nindent 4 }}

applications/ppdb-replication/templates/vault-secrets.yaml

@@ -0,0 +1,9 @@
+apiVersion: ricoberger.de/v1alpha1
+kind: VaultSecret
+metadata:
+  name: {{ include "ppdb-replication.fullname" . }}
+  labels:
+    {{- include "ppdb-replication.labels" . | nindent 4 }}
+spec:
+  path: "{{ .Values.global.vaultSecretsPath }}/ppdb-replication"
+  type: Opaque

applications/ppdb-replication/values-usdfdev.yaml

@@ -0,0 +1,44 @@
+config:
+
+  # -- Logging level
+  logLevel: "INFO"
+
+  # -- Logging profile (`production` for JSON, `development` for
+  # human-friendly)
+  logProfile: "development"
+
+  # -- APDB config file resource
+  apdbConfig: "label:pp-prod:lsstcomcamsim-or4"
+
+  # -- PPDB config file resource
+  ppdbConfig: "/sdf/group/rubin/user/jeremym/ppdb-replication/config/ppdb-replication-test-1.yaml"
+
+  # -- APDB index URI
+  apdbIndexUri: "/sdf/group/rubin/shared/apdb_config/apdb-index.yaml"
+
+  # -- S3 endpoint URL
+  s3EndpointUrl: https://s3dfrgw.slac.stanford.edu
+
+  # -- S3 profile name for additional S3 profile
+  additionalS3ProfileName: "embargo"
+
+  # -- S3 profile URL for additional S3 profile
+  additionalS3ProfileUrl: "https://sdfembs3.sdf.slac.stanford.edu"
+
+  volumes:
+    - name: sdf-group-rubin
+      persistentVolumeClaim:
+        claimName: sdf-group-rubin
+    - name: sdf-data-rubin
+      persistentVolumeClaim:
+        claimName: sdf-data-rubin
+  volumeMounts:
+    - name: sdf-group-rubin
+      mountPath: /sdf/group/rubin
+    - name: sdf-data-rubin
+      mountPath: /sdf/data/rubin
+  persistentVolumeClaims:
+    - name: sdf-group-rubin
+      storageClassName: sdf-group-rubin
+    - name: sdf-data-rubin
+      storageClassName: sdf-data-rubin