Version bump backup-util to v0.1.9

.github/workflows/deploy-application.yml

@@ -77,7 +77,7 @@ jobs:
           cf_password: ${{ secrets.CF_PASSWORD }}
           cf_org: gsa-tts-oros-fac
           cf_space: ${{ env.space }}
-          command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup --command "./fac-backup-util.sh v0.1.5 deploy_backup"
+          command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup --command "./fac-backup-util.sh v0.1.9 deploy_backup"
 
       - name: Deploy Preview to cloud.gov
         if: ${{ inputs.environment == 'preview' }}
@@ -123,5 +123,5 @@ jobs:
     secrets: inherit
     with:
       environment: ${{ inputs.environment }}
-      util_version: "v0.1.8"
+      util_version: "v0.1.9"
       backup_operation: "check_tables"

.github/workflows/fac-api-standup-scheduler.yml

@@ -20,6 +20,6 @@ jobs:
     secrets: inherit
     with:
       environment: ${{ matrix.environment.name }}
-      util_version: "v0.1.8"
+      util_version: "v0.1.9"
       backup_operation: "rds_backup"
 

.github/workflows/fac-api-standup.yml

@@ -1,8 +1,8 @@
 ---
 name: Standup the API
 ### Common Commands:
-# ./fac-backup-util.sh v0.1.8 scheduled_backup
-# ./fac-backup-util.sh v0.1.8 daily_backup
+# ./fac-backup-util.sh v0.1.9 scheduled_backup
+# ./fac-backup-util.sh v0.1.9 daily_backup
 on:
   workflow_call:
     inputs:
@@ -31,6 +31,7 @@ on:
         description: Version for fac backup utility to use (ex. vX.Y.Z)
         required: true
         type: string
+        default: "v0.1.9"
       backup_operation:
         description: Operation for fac-backup-utility
         required: true
@@ -58,4 +59,4 @@ jobs:
           #cf_space: ${{ env.space }}
           cf_space: preview
           #command: cf run-task gsa-fac -k 3G -m 3G --name standup_api --command "./util/nightly_api_refresh.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}"
-          command: cf run-task gsa-fac -k 2G -m 2G --name api_refresh --command "./util/nightly_api_refresh.sh v0.1.8 rds_backup"
+          command: cf run-task gsa-fac -k 2G -m 2G --name api_refresh --command "./util/nightly_api_refresh.sh v0.1.9 rds_backup"

.github/workflows/fac-backup-util-scheduled.yml

@@ -1,8 +1,8 @@
 ---
 name: Backup the database with fac-backup-utility
 ### Common Commands:
-# ./fac-backup-util.sh v0.1.8 scheduled_backup
-# ./fac-backup-util.sh v0.1.8 daily_backup
+# ./fac-backup-util.sh v0.1.9 scheduled_backup
+# ./fac-backup-util.sh v0.1.9 daily_backup
 on:
   workflow_call:
     inputs:

.github/workflows/fac-backup-util.yml

@@ -1,8 +1,8 @@
 ---
 name: Backup the database with fac-backup-utility
 ### Common Commands:
-# ./fac-backup-util.sh v0.1.8 initial_backup
-# ./fac-backup-util.sh v0.1.8 deploy_backup
+# ./fac-backup-util.sh v0.1.9 initial_backup
+# ./fac-backup-util.sh v0.1.9 deploy_backup
 on:
   workflow_dispatch:
     inputs:
@@ -18,7 +18,7 @@ on:
         description: Version for fac backup utility to use (ex. vX.Y.Z)
         required: true
         type: string
-        default: "v0.1.8"
+        default: "v0.1.9"
       backup_operation:
         description: Operation for fac-backup-utility
         required: true

.github/workflows/fac-check-tables-scheduler.yml

@@ -20,5 +20,5 @@ jobs:
     secrets: inherit
     with:
       environment: ${{ matrix.environment.name }}
-      util_version: "v0.1.8"
+      util_version: "v0.1.9"
       backup_operation: "check_tables"

.github/workflows/fac-check-tables.yml

@@ -1,7 +1,7 @@
 ---
 name: Check existing tables in an environment
 ### Common Commands:
-# ./fac-backup-util.sh v0.1.8 check_tables
+# ./fac-backup-util.sh v0.1.9 check_tables
 on:
   workflow_dispatch:
     inputs:

backend/tools/setup_local_env.sh

@@ -4,7 +4,7 @@ function setup_local_env {
 
     if [[ "${ENV}" == "LOCAL" || "${ENV}" == "TESTING" ]]; then
         startup_log "LOCAL_ENV" "We are in a local envirnoment."
-        
+
         # Load a fake VCAP_SERVICES file into the environment variable,
         # so we can mimic the cloud.gov setup.
         export VCAP_SERVICES=$(cat config/vcap_services_for_containers.json)
@@ -23,16 +23,16 @@ function setup_local_env {
         # export AWS_PRIVATE_BUCKET_NAME="fac-private-s3"
 
         export AWS_PRIVATE_BUCKET_NAME=$(echo $VCAP_SERVICES \
-            | jq --raw-output '.s3 
-                | map(select(.instance_name 
-                            | contains("fac-private-s3"))) 
+            | jq --raw-output '.s3
+                | map(select(.instance_name
+                            | contains("fac-private-s3")))
                 | .[] .credentials.bucket')
         check_env_var_not_empty "AWS_PRIVATE_BUCKET_NAME"
-        
+
         export AWS_PUBLIC_BUCKET_NAME=$(echo $VCAP_SERVICES \
-            | jq --raw-output '.s3 
-                | map(select(.instance_name 
-                            | contains("fac-public-s3"))) 
+            | jq --raw-output '.s3
+                | map(select(.instance_name
+                            | contains("fac-public-s3")))
                 | .[] .credentials.bucket')
 
 
@@ -51,7 +51,7 @@ function setup_local_env {
         get_aws_s3 "fac-private-s3" "endpoint"
         export AWS_S3_PRIVATE_ENDPOINT=$_GET_AWS_RESULT
         check_env_var_not_empty "AWS_S3_PRIVATE_ENDPOINT"
-        
+
         get_aws_s3 "fac-public-s3" "access_key_id"
         export AWS_PUBLIC_ACCESS_KEY_ID=$_GET_AWS_RESULT
         check_env_var_not_empty "AWS_PUBLIC_ACCESS_KEY_ID"
@@ -63,7 +63,7 @@ function setup_local_env {
         get_aws_s3 "fac-public-s3" "endpoint"
         export AWS_S3_PUBLIC_ENDPOINT=$_GET_AWS_RESULT
         check_env_var_not_empty "AWS_S3_PUBLIC_ENDPOINT"
-        
+
         #export MC_HOST_<alias>=https://<Access Key>:<Secret Key>:<Session Token>@<YOUR-S3-ENDPOINT>
         export MC_HOST_myminio="http://${AWS_PRIVATE_ACCESS_KEY_ID}:${AWS_PRIVATE_SECRET_ACCESS_KEY}@minio:9000"
         # mc alias set myminio ${AWS_S3_PRIVATE_ENDPOINT} ${AWS_PRIVATE_ACCESS_KEY_ID} ${AWS_PRIVATE_ACCESS_KEY_ID}
@@ -72,13 +72,13 @@ function setup_local_env {
         # https://min.io/docs/minio/linux/reference/minio-mc/mc-mb.html
         mc mb --ignore-existing myminio/${AWS_PUBLIC_BUCKET_NAME}
         mc mb --ignore-existing myminio/${AWS_PRIVATE_BUCKET_NAME}
-        
+
         # MCJ 20241016 FIXME: Is this even needed locally? I don't think so.
         # mc admin user svcacct add \
         #     --access-key="${AWS_PRIVATE_ACCESS_KEY_ID}" \
         #     --secret-key="${AWS_PRIVATE_SECRET_ACCESS_KEY}" \
         #     myminio minioadmin
-        
+
         # For database work
         export FAC_DB_URI=${DATABASE_URL} #?sslmode=disable
         export FAC_SNAPSHOT_URI=${SNAPSHOT_URL}
@@ -98,13 +98,13 @@ function setup_local_env {
 
         # And we need cgov-util
         pushd /tmp
-            local CGOV_VERSION=v0.1.8
+            local CGOV_VERSION=v0.1.9
             curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/${CGOV_VERSION}/gov.gsa.fac.cgov-util-${CGOV_VERSION}-linux-amd64.tar.gz
             tar xvzf gov.gsa.fac.cgov-util-${CGOV_VERSION}-linux-amd64.tar.gz gov.gsa.fac.cgov-util
             chmod 755 gov.gsa.fac.cgov-util
             mv gov.gsa.fac.cgov-util /bin/cgov-util
         popd
-                
+
         export SLING_EXE='/bin/sling'
         export CGOV_UTIL_EXE='/bin/cgov-util'
 

backend/util/load_public_dissem_data/Dockerfile

@@ -17,9 +17,9 @@ RUN curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download
   && chmod +x sling \
   && mv sling /bin/sling
 
-RUN curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/v0.1.8/gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz \
-    && tar xvzf gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz gov.gsa.fac.cgov-util \
-    && chmod 755 gov.gsa.fac.cgov-util \ 
+RUN curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/v0.1.9/gov.gsa.fac.cgov-util-v0.1.9-linux-amd64.tar.gz \
+    && tar xvzf gov.gsa.fac.cgov-util-v0.1.9-linux-amd64.tar.gz gov.gsa.fac.cgov-util \
+    && chmod 755 gov.gsa.fac.cgov-util \
     && mv gov.gsa.fac.cgov-util /layered/cgov-util
 
 # Where do we need the config? Try home?

backend/util/load_public_dissem_data/README.md

@@ -50,15 +50,15 @@ This is containerized, so it should work on all platforms. To build the containe
 make build
 ```
 
-Then, to run the container, 
+Then, to run the container,
 
 ```
 make run
 ```
 
 You need to run this from the current directory.
 
-NOTE: The docker command in the Makefile uses the `--network` flag. The `--network` flag tells Docker to run this container on the same network as your currently running stack. It assumes that you did a `docker compose up` on the FAC stack, and that the web container has the default name of `backend-web-1`. If this does not work, you will need to... 
+NOTE: The docker command in the Makefile uses the `--network` flag. The `--network` flag tells Docker to run this container on the same network as your currently running stack. It assumes that you did a `docker compose up` on the FAC stack, and that the web container has the default name of `backend-web-1`. If this does not work, you will need to...
 
 ```
 make NETWORK=<container-name> run
@@ -68,10 +68,10 @@ where `<container-name>` is the name of your web container. This should allow th
 
 ## When to rebuild this container
 
-Note this is pinned to v0.1.8 of the cgov-util.
+Note this is pinned to v0.1.9 of the cgov-util.
 
 https://github.com/GSA-TTS/fac-backup-utility
 
 If that gets updated, you'll need to update the dockerfile.
 
-It also copies in the YAML for sling from `dissemination/sql/sling`. If that changes, you'll want to 
+It also copies in the YAML for sling from `dissemination/sql/sling`. If that changes, you'll want to

backend/util/load_public_dissem_data/load_public_data_locally.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-export CGOV_UTIL_VERSION=v0.1.8
+export CGOV_UTIL_VERSION=v0.1.9
 export FAC_DB_URI=postgresql://postgres@db:5432/postgres?sslmode=disable
 export FAC_SNAPSHOT_URI=postgresql://postgres@db2:5432/postgres?sslmode=disable
 
@@ -42,7 +42,7 @@ echo "Unzipping data."
 unzip db_dissem_dump.zip
 popd
 
-# Truncate the dissemination_* tables if they exist. 
+# Truncate the dissemination_* tables if they exist.
 # CASCADE as well. This makes sure we don't duplicate data,
 # which causes PK/FK problems.
 echo "select 'TRUNCATE TABLE '||tablename||' CASCADE;' FROM pg_tables WHERE tablename LIKE 'dissemination_%'" | \

backend/util/nightly_api_refresh.sh

@@ -7,7 +7,7 @@ source tools/sql_pre_post.sh
 setup_env
 
 # Run an RDS backup, to refresh dissemination_* in FAC_SNAPSHOT from FAC_DB
-#./../fac-backup-util.sh "v0.1.8" "rds_backup"
+#./../fac-backup-util.sh "v0.1.9" "rds_backup"
 # Deploy test
 ./fac-backup-util.sh "$1" "$2"
 gonogo "fac-backup-util"

docs/backups_and_restores.md

@@ -34,7 +34,7 @@ Information regarding the fac-backup-utility can be found [at the repository](ht
 Database backups occur in the following ways:
 1. An initial backup, where a backup has not been run in the target environment. This input of `initial_backup` is important, as when it does a the `db_to_db` command, it will not truncate the target table, as the table does not exist in the destination database.
 ```bash
-./fac-backup-util.sh v0.1.5 initial_backup
+./fac-backup-util.sh v0.1.9 initial_backup
 # Curl the utility
 # Install AWS
 # DB to S3 table dump (backups)
@@ -44,7 +44,7 @@ Database backups occur in the following ways:
 
 2. A deploy backup, where the `db_to_db` function is not called. This is a standard backup strategy before the application deploys, to ensure the s3 contents of the primary s3 are sync'd to the backups bucket, and a table dump is stored in the backups bucket.
 ```bash
-./fac-backup-util.sh v0.1.5 deploy_backup
+./fac-backup-util.sh v0.1.9 deploy_backup
 # Curl the utility
 # Install AWS
 # DB to S3 table dump (backups)
@@ -53,7 +53,7 @@ Database backups occur in the following ways:
 
 3. A scheduled backup is run every two hours, across each environment, ensuring that we have a clean backup in s3, rds, and the bucket contents are in sync.
 ```bash
-./fac-backup-util.sh v0.1.5 scheduled_backup
+./fac-backup-util.sh v0.1.9 scheduled_backup
 # Curl the utility
 # Install AWS
 # DB to S3 table dump (fac-db -> backups)
@@ -66,7 +66,7 @@ Restoring from backups can be run via workflow, from designated individuals. The
 
 1. S3 Restore takes a `operation-mm-DD-HH` input (ex `scheduled-06-04-10`), and is required for the backups to be restored. The utility looks in `s3://${bucket}/backups/operation-mm-DD-HH/` for its table dumps, and without supplying the target backups, it will not restore. Once it does a `--data-only` restoration, it will then sync the files from the backups bucket to the application bucket. We do this to ensure the contents of the application bucket are up to date, relative to the data in the database. We know that if we use the latest folder in `/backups/` then the contents of the s3 are the latest available, from the prior backup.
 ```bash
-./fac-restore-util.sh v0.1.5 s3_restore scheduled-06-04-10
+./fac-restore-util.sh v0.1.9 s3_restore scheduled-06-04-10
 # Curl the utility
 # Install AWS
 # DB to S3 table dump (backups -> fac-db) [Truncate target table before --data-only pg_restore]
@@ -81,7 +81,7 @@ daily-mm-dd
 
 2. Database to database restoration also can occur as well, using `psql` to dump the tables from the cold store database to the live database.
 ```bash
-./fac-restore-util.sh v0.1.5 db_restore
+./fac-restore-util.sh v0.1.9 db_restore
 # Curl the utility
 # Install AWS
 # DB to DB table dump (fac-snapshot-db -> fac-db) [Truncate target table before dump]