From 9a381dffa21b1108565f09be8189d5e3ee73ae0c Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Tue, 10 Dec 2024 14:05:37 -0800 Subject: [PATCH] instructions & standby updates --- docs/database/CLUSTER_DB.MD | 20 ++++-- .../scripts/oc_provision_crunchy_standby.sh | 6 +- openshift/templates/crunchy_standby.yaml | 64 +++++++++++++++---- 3 files changed, 70 insertions(+), 20 deletions(-) diff --git a/docs/database/CLUSTER_DB.MD b/docs/database/CLUSTER_DB.MD index bde57717a..4ae50229b 100644 --- a/docs/database/CLUSTER_DB.MD +++ b/docs/database/CLUSTER_DB.MD @@ -49,7 +49,7 @@ Source: https://access.crunchydata.com/documentation/postgres-operator/latest/ar When the repo-standby is spun, it reads and applies the WAL in order to replicate the state of the database and runs as it's own cluster, separate from the primary. Spin up the Repo-Standby Cluster with: -`PROJ_TARGET= BUCKET= bash openshift/scripts/oc_provision_crunchy_standby.sh apply` +`PROJ_TARGET= BUCKET= DATE= bash openshift/scripts/oc_provision_crunchy_standby.sh apply` - Anecdotally, spinning up a standby cluster for a 15GB database took about 5 minutes @@ -61,16 +61,24 @@ Promote the standby cluster by editing the [crunchy_standby.yaml](../../openshif More details here: +### Setting secrets + +The promoted standby cluster created it's own secrets for connecting to pgbouncer and it has created a new user in the database with the same name as the cluster. Ex. if the standby cluster is named "wps-crunchy-16-2024-12-10", the user will have the same name. +Once the standby has been promoted, the easiest way to update user privileges is to reassign table ownership from the old user to the new user. +`REASSIGN OWNED BY "" TO "";` + +Once this is done, the deployment using the crunchy secrets will need to be updated. This can be done by manually editing the deployment YAML and changing all config that referenced the original crunchy secrets to the newly promoted standby cluster secrets. Once this is done, new pods should roll out successfully. + ## Cluster Restore From pg_dump In the event that the cluster can't be restored from pgbackrest you can create a new cluster and restore using a pg_dump from S3. ##### Deploy new cluster - ``` - oc login --token= --server= - PROJ_TARGET= BUCKET= CPU_REQUEST=75m CPU_LIMIT=2000m MEMORY_REQUEST=2Gi MEMORY_LIMIT=16Gi DATA_SIZE=65Gi WAL_SIZE=45Gi bash ./oc_provision_crunchy.sh apply - ``` +``` +oc login --token= --server= +PROJ_TARGET= BUCKET= CPU_REQUEST=75m CPU_LIMIT=2000m MEMORY_REQUEST=2Gi MEMORY_LIMIT=16Gi DATA_SIZE=65Gi WAL_SIZE=45Gi bash ./oc_provision_crunchy.sh apply +``` ##### Set superuser permissions in new cluster via OpenShift web GUI @@ -90,7 +98,7 @@ PGUSER=$(oc get secrets -n " "" -o go-template='{{.data.dbname | base64decode}}') oc -n port-forward "${PG_CLUSTER_PRIMARY_POD}" 5432:5432 ``` - + ##### Restore sql dump into new cluster in another shell Download the latest SQL dump from S3 storage and unzip it. diff --git a/openshift/scripts/oc_provision_crunchy_standby.sh b/openshift/scripts/oc_provision_crunchy_standby.sh index 3b71fe574..cbb523a42 100755 --- a/openshift/scripts/oc_provision_crunchy_standby.sh +++ b/openshift/scripts/oc_provision_crunchy_standby.sh @@ -25,6 +25,9 @@ source "$(dirname ${0})/common/common" # Target project override for Dev or Prod deployments # PROJ_TARGET="${PROJ_TARGET:-${PROJ_DEV}}" + +# Set DATE to today's date if it isn't set +DATE=${DATE:-$(date +"%Y-%m-%d")} # Prepare names for crunchy ephemeral instance for this PR. IMAGE_STREAM_NAMESPACE=${IMAGE_STREAM_NAMESPACE:-${PROJ_TOOLS}} @@ -35,7 +38,8 @@ OC_PROCESS="oc -n ${PROJ_TARGET} process -f ${TEMPLATE_PATH}/crunchy_standby.yam -p SUFFIX=${SUFFIX} \ -p TARGET_NAMESPACE=${PROJ_TARGET} \ -p BUCKET=${BUCKET} \ --p DATA_SIZE=45Gi \ +-p DATE=${DATE} \ +-p DATA_SIZE=65Gi \ -p WAL_SIZE=15Gi \ ${IMAGE_NAME:+ " -p IMAGE_NAME=${IMAGE_NAME}"} \ ${IMAGE_TAG:+ " -p IMAGE_TAG=${IMAGE_TAG}"} \ diff --git a/openshift/templates/crunchy_standby.yaml b/openshift/templates/crunchy_standby.yaml index baf163c80..da7dbce1d 100644 --- a/openshift/templates/crunchy_standby.yaml +++ b/openshift/templates/crunchy_standby.yaml @@ -1,12 +1,12 @@ apiVersion: template.openshift.io/v1 kind: Template metadata: - name: wps-crunchydb-standby + name: ${APP_NAME}-${DATE} annotations: - "openshift.io/display-name": wps-crunchydb-standby + "openshift.io/display-name": ${APP_NAME}-${DATE} labels: - app.kubernetes.io/part-of: wps-crunchydb-standby - app: wps-crunchydb-standby + app.kubernetes.io/part-of: ${APP_NAME}-${DATE} + app: ${APP_NAME}-${DATE} parameters: - description: Namespace in which database resides displayName: Target Namespace @@ -15,6 +15,13 @@ parameters: - name: BUCKET description: S3 bucket name required: true + - name: APP_NAME + description: Application name (wps - wildfire predictive services) + value: wps-crunchydb-16 + required: true + - name: DATE + description: Date the standby was created + required: true - name: DATA_SIZE description: Data PVC size required: true @@ -60,23 +67,17 @@ objects: - apiVersion: postgres-operator.crunchydata.com/v1beta1 kind: PostgresCluster metadata: - name: wps-crunchydb-standby + name: ${APP_NAME}-${DATE} spec: postgresVersion: 16 postGISVersion: "3.3" metadata: - name: wps-crunchydb-standby + name: ${APP_NAME}-${DATE} labels: - app: wps-crunchydb-standby + app: ${APP_NAME}-${DATE} databaseInitSQL: key: init.sql name: wps-init-sql - users: - - name: wps - databases: - - postgres - - wps - options: "SUPERUSER" instances: - name: crunchy replicas: 1 @@ -104,6 +105,10 @@ objects: backups: pgbackrest: image: artifacts.developer.gov.bc.ca/bcgov-docker-local/crunchy-pgbackrest:ubi8-2.41-4 + manual: + repoName: repo1 + options: + - --type=full configuration: - secret: name: crunchy-pgbackrest @@ -111,13 +116,46 @@ objects: - key: conf path: s3.conf global: + repo1-retention-full: "3" + repo1-retention-full-type: count repo1-path: /pgbackrest/${SUFFIX}/repo1 repos: - name: repo1 + schedules: + full: "0 1 * * 0" + differential: "0 1 * * 1-6" s3: bucket: ${BUCKET} endpoint: nrs.objectstore.gov.bc.ca region: "ca-central-1" + proxy: + pgBouncer: + image: artifacts.developer.gov.bc.ca/bcgov-docker-local/crunchy-pgbouncer:ubi8-1.21-0 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + postgres-operator.crunchydata.com/cluster: db + postgres-operator.crunchydata.com/role: pgbouncer + topologyKey: kubernetes.io/hostname + weight: 1 + config: + global: + pool_mode: transaction + ignore_startup_parameters: options, extra_float_digits + max_prepared_statements: "10" + max_client_conn: "1000" + port: 5432 + replicas: 1 + resources: + limits: + cpu: 500m + memory: 3Gi + requests: + cpu: 100m + memory: 1Gi standby: enabled: true repoName: repo1