From b759d11f18b23eb8bcb00b3e03c829ec0bb1bc9e Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Wed, 15 Nov 2023 10:06:57 -0800 Subject: [PATCH 1/8] Install pgbackrest and configure --- openshift/patroni-postgis/README.md | 4 +-- openshift/patroni-postgis/docker/Dockerfile | 1 + openshift/patroni-postgis/docker/init_postgis | 25 +++++++++++++++++++ .../patroni-postgis/openshift/build.yaml | 14 ++++++++++- 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/openshift/patroni-postgis/README.md b/openshift/patroni-postgis/README.md index 8393eff89..ad60a696b 100644 --- a/openshift/patroni-postgis/README.md +++ b/openshift/patroni-postgis/README.md @@ -11,7 +11,7 @@ The WPS pipeline currently assumes the existence of an appropriately tagged patr ```bash # Build a patroni imagestream: -oc -n e1e498-tools process -f openshift/build.yaml | oc -n e1e498-tools apply -f - +oc -n e1e498-tools process -f openshift/build.yaml -p OBJECT_STORE_SERVER="server" -p OBJECT_STORE_BUCKET="bucket" -p OBJECT_STORE_USER_ID="uid" -p OBJECT_STORE_SECRET="sec" | oc -n e1e498-tools apply -f - # Tag the old imagestream so we can keep it around if we need to revert: oc -n e1e498-tools tag patroni-postgres:v12 patroni-postgres:v12- # Tag the new imagestream (it won't be used until the pods get re-created): @@ -69,4 +69,4 @@ SELECT postgis_extensions_upgrade(); This script cannot be run as part of the alembic scripts in the api project, the api uses a `wps` user that does not have the appropriate priveleges. Upgrading the image may this require manually running `SELECT postgis_extensions_upgrade();` on the target database once the new image has been applied. -The PostGIS version can be checked with `SELECT postgis_full_version();` \ No newline at end of file +The PostGIS version can be checked with `SELECT postgis_full_version();` diff --git a/openshift/patroni-postgis/docker/Dockerfile b/openshift/patroni-postgis/docker/Dockerfile index 23bc25d83..38ce5b89b 100644 --- a/openshift/patroni-postgis/docker/Dockerfile +++ b/openshift/patroni-postgis/docker/Dockerfile @@ -14,6 +14,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION \ postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts=$POSTGIS_VERSION \ + pgbackrest \ && rm -rf /var/lib/apt/lists/* # Add the POSTGIS command to the end of the post_init script. diff --git a/openshift/patroni-postgis/docker/init_postgis b/openshift/patroni-postgis/docker/init_postgis index 829d96150..ac4297308 100644 --- a/openshift/patroni-postgis/docker/init_postgis +++ b/openshift/patroni-postgis/docker/init_postgis @@ -3,3 +3,28 @@ if [[ (! -z "$APP_USER") && (! -z "$APP_PASSWORD") && (! -z "$APP_DATABASE")]]; echo "Loading PostGIS extensions into $APP_DATABASE" psql "$1" -w -c "\c ${APP_DATABASE}" -c "CREATE EXTENSION IF NOT EXISTS postgis;" fi + +cat < Date: Wed, 15 Nov 2023 10:35:56 -0800 Subject: [PATCH 2/8] Bump postgis --- openshift/patroni-postgis/README.md | 2 +- openshift/patroni-postgis/docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openshift/patroni-postgis/README.md b/openshift/patroni-postgis/README.md index ad60a696b..17854720e 100644 --- a/openshift/patroni-postgis/README.md +++ b/openshift/patroni-postgis/README.md @@ -11,7 +11,7 @@ The WPS pipeline currently assumes the existence of an appropriately tagged patr ```bash # Build a patroni imagestream: -oc -n e1e498-tools process -f openshift/build.yaml -p OBJECT_STORE_SERVER="server" -p OBJECT_STORE_BUCKET="bucket" -p OBJECT_STORE_USER_ID="uid" -p OBJECT_STORE_SECRET="sec" | oc -n e1e498-tools apply -f - +oc -n e1e498-tools process -f openshift/build.yaml -p OBJECT_STORE_SERVER="server" -p OBJECT_STORE_BUCKET="bucket" -p OBJECT_STORE_USER_ID="uid" -p OBJECT_STORE_SECRET="sec" -p GIT_REF="" | oc -n e1e498-tools apply -f - # Tag the old imagestream so we can keep it around if we need to revert: oc -n e1e498-tools tag patroni-postgres:v12 patroni-postgres:v12- # Tag the new imagestream (it won't be used until the pods get re-created): diff --git a/openshift/patroni-postgis/docker/Dockerfile b/openshift/patroni-postgis/docker/Dockerfile index 38ce5b89b..5887109be 100644 --- a/openshift/patroni-postgis/docker/Dockerfile +++ b/openshift/patroni-postgis/docker/Dockerfile @@ -3,7 +3,7 @@ FROM image-registry.openshift-image-registry.svc:5000/bcgov/patroni-postgres:12.4-latest ENV POSTGIS_MAJOR 3 -ENV POSTGIS_VERSION 3.3.1+dfsg-1.pgdg100+1 +ENV POSTGIS_VERSION 3.4.0+dfsg-1.pgdg110+1 # Switch to root user for package installs USER 0 From 442814164a8892eb218753e3474f1689cb09cfb8 Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Wed, 15 Nov 2023 11:00:23 -0800 Subject: [PATCH 3/8] Try again --- openshift/patroni-postgis/docker/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openshift/patroni-postgis/docker/Dockerfile b/openshift/patroni-postgis/docker/Dockerfile index 5887109be..ef9e8889d 100644 --- a/openshift/patroni-postgis/docker/Dockerfile +++ b/openshift/patroni-postgis/docker/Dockerfile @@ -12,8 +12,12 @@ USER 0 RUN apt-get update \ && apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ && apt-get install -y --no-install-recommends \ + # ca-certificates: for accessing remote raster files; + # fix: https://github.com/postgis/docker-postgis/issues/307 + ca-certificates \ + \ postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts=$POSTGIS_VERSION \ + postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ pgbackrest \ && rm -rf /var/lib/apt/lists/* From 85b628852a354861498f91cf92fb6beb39e8d3c7 Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Wed, 15 Nov 2023 11:19:24 -0800 Subject: [PATCH 4/8] Again --- openshift/patroni-postgis/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openshift/patroni-postgis/docker/Dockerfile b/openshift/patroni-postgis/docker/Dockerfile index ef9e8889d..5843d3c21 100644 --- a/openshift/patroni-postgis/docker/Dockerfile +++ b/openshift/patroni-postgis/docker/Dockerfile @@ -3,7 +3,7 @@ FROM image-registry.openshift-image-registry.svc:5000/bcgov/patroni-postgres:12.4-latest ENV POSTGIS_MAJOR 3 -ENV POSTGIS_VERSION 3.4.0+dfsg-1.pgdg110+1 +ENV POSTGIS_VERSION 3.3.4+dfsg-1.pgdg100+1 # Switch to root user for package installs USER 0 From 642b7bb18b6ed55f77d556dd7c2243b9a9814c71 Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Wed, 15 Nov 2023 11:58:24 -0800 Subject: [PATCH 5/8] Point patroni to new image --- openshift/templates/patroni.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openshift/templates/patroni.yaml b/openshift/templates/patroni.yaml index 53166ae15..ef8502596 100644 --- a/openshift/templates/patroni.yaml +++ b/openshift/templates/patroni.yaml @@ -287,12 +287,12 @@ parameters: - name: IMAGE_NAME description: | The Patroni image stream name - value: patroni-postgres + value: patroni-postgres-pgbackrest - name: IMAGE_TAG description: | The image tag used to specify which image you would like deployed. Don't use `latest`. - value: "v12-2022-09-12" + value: "v12-latest" - name: PVC_SIZE description: The size of the persistent volume to create. displayName: Persistent Volume Size From e1e4ab88953a3bbdb090da0f9907c8adce7fb0ab Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Wed, 15 Nov 2023 12:54:38 -0800 Subject: [PATCH 6/8] Make object store docker args --- openshift/patroni-postgis/README.md | 2 +- openshift/patroni-postgis/docker/Dockerfile | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/openshift/patroni-postgis/README.md b/openshift/patroni-postgis/README.md index 17854720e..892e9a726 100644 --- a/openshift/patroni-postgis/README.md +++ b/openshift/patroni-postgis/README.md @@ -11,7 +11,7 @@ The WPS pipeline currently assumes the existence of an appropriately tagged patr ```bash # Build a patroni imagestream: -oc -n e1e498-tools process -f openshift/build.yaml -p OBJECT_STORE_SERVER="server" -p OBJECT_STORE_BUCKET="bucket" -p OBJECT_STORE_USER_ID="uid" -p OBJECT_STORE_SECRET="sec" -p GIT_REF="" | oc -n e1e498-tools apply -f - +oc -n e1e498-tools process -f openshift/build.yaml| oc -n e1e498-tools apply -f - # Tag the old imagestream so we can keep it around if we need to revert: oc -n e1e498-tools tag patroni-postgres:v12 patroni-postgres:v12- # Tag the new imagestream (it won't be used until the pods get re-created): diff --git a/openshift/patroni-postgis/docker/Dockerfile b/openshift/patroni-postgis/docker/Dockerfile index 5843d3c21..1ea47fe1d 100644 --- a/openshift/patroni-postgis/docker/Dockerfile +++ b/openshift/patroni-postgis/docker/Dockerfile @@ -2,6 +2,12 @@ # For details, see: https://github.com/bcgov/patroni-postgres-container/ FROM image-registry.openshift-image-registry.svc:5000/bcgov/patroni-postgres:12.4-latest +# The below must be supplied for a build +ARG OBJECT_STORE_SERVER=server +ARG OBJECT_STORE_BUCKET=bucket +ARG OBJECT_STORE_USER_ID=uid +ARG OBJECT_STORE_SECRET=sec + ENV POSTGIS_MAJOR 3 ENV POSTGIS_VERSION 3.3.4+dfsg-1.pgdg100+1 From 08e7ab13e9d2087602e3a897a716817fb70c5397 Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Wed, 15 Nov 2023 12:55:58 -0800 Subject: [PATCH 7/8] Remove required object store fields from build --- .../patroni-postgis/openshift/build.yaml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/openshift/patroni-postgis/openshift/build.yaml b/openshift/patroni-postgis/openshift/build.yaml index d4fe5ae70..cc2aceea6 100644 --- a/openshift/patroni-postgis/openshift/build.yaml +++ b/openshift/patroni-postgis/openshift/build.yaml @@ -23,18 +23,18 @@ parameters: value: main - name: POSTGRES_VERSION value: "12" - - name: OBJECT_STORE_SERVER - description: Backup object store server - required: true - - name: OBJECT_STORE_BUCKET - description: Backup object store bucket - required: true - - name: OBJECT_STORE_USER_ID - description: Backup object store user id - required: true - - name: OBJECT_STORE_SECRET - description: Backup object store secret - required: true + # - name: OBJECT_STORE_SERVER + # description: Backup object store server + # required: true + # - name: OBJECT_STORE_BUCKET + # description: Backup object store bucket + # required: true + # - name: OBJECT_STORE_USER_ID + # description: Backup object store user id + # required: true + # - name: OBJECT_STORE_SECRET + # description: Backup object store secret + # required: true objects: - apiVersion: v1 kind: ImageStream From 109d55e54ad1689d70c49a68e2cf03dbe5d003de Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Wed, 15 Nov 2023 13:56:45 -0800 Subject: [PATCH 8/8] Try with build args --- openshift/patroni-postgis/README.md | 2 +- .../patroni-postgis/openshift/build.yaml | 33 ++++++++++++------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/openshift/patroni-postgis/README.md b/openshift/patroni-postgis/README.md index 892e9a726..17854720e 100644 --- a/openshift/patroni-postgis/README.md +++ b/openshift/patroni-postgis/README.md @@ -11,7 +11,7 @@ The WPS pipeline currently assumes the existence of an appropriately tagged patr ```bash # Build a patroni imagestream: -oc -n e1e498-tools process -f openshift/build.yaml| oc -n e1e498-tools apply -f - +oc -n e1e498-tools process -f openshift/build.yaml -p OBJECT_STORE_SERVER="server" -p OBJECT_STORE_BUCKET="bucket" -p OBJECT_STORE_USER_ID="uid" -p OBJECT_STORE_SECRET="sec" -p GIT_REF="" | oc -n e1e498-tools apply -f - # Tag the old imagestream so we can keep it around if we need to revert: oc -n e1e498-tools tag patroni-postgres:v12 patroni-postgres:v12- # Tag the new imagestream (it won't be used until the pods get re-created): diff --git a/openshift/patroni-postgis/openshift/build.yaml b/openshift/patroni-postgis/openshift/build.yaml index cc2aceea6..4b73103b6 100644 --- a/openshift/patroni-postgis/openshift/build.yaml +++ b/openshift/patroni-postgis/openshift/build.yaml @@ -23,18 +23,18 @@ parameters: value: main - name: POSTGRES_VERSION value: "12" - # - name: OBJECT_STORE_SERVER - # description: Backup object store server - # required: true - # - name: OBJECT_STORE_BUCKET - # description: Backup object store bucket - # required: true - # - name: OBJECT_STORE_USER_ID - # description: Backup object store user id - # required: true - # - name: OBJECT_STORE_SECRET - # description: Backup object store secret - # required: true + - name: OBJECT_STORE_SERVER + description: Backup object store server + required: true + - name: OBJECT_STORE_BUCKET + description: Backup object store bucket + required: true + - name: OBJECT_STORE_USER_ID + description: Backup object store user id + required: true + - name: OBJECT_STORE_SECRET + description: Backup object store secret + required: true objects: - apiVersion: v1 kind: ImageStream @@ -67,6 +67,15 @@ objects: type: Git strategy: dockerStrategy: + buildArgs: + - name: "OBJECT_STORE_SERVER" + value: "${OBJECT_STORE_SERVER}" + - name: "OBJECT_STORE_BUCKET" + value: "${OBJECT_STORE_BUCKET}" + - name: "OBJECT_STORE_USER_ID" + value: "${OBJECT_STORE_USER_ID}" + - name: "OBJECT_STORE_SECRET" + value: "${OBJECT_STORE_SECRET}" type: Docker triggers: - type: ConfigChange