diff --git a/.github/workflows/deploy-application.yml b/.github/workflows/deploy-application.yml index 67a9288b9..dd19d7b79 100644 --- a/.github/workflows/deploy-application.yml +++ b/.github/workflows/deploy-application.yml @@ -77,7 +77,7 @@ jobs: cf_password: ${{ secrets.CF_PASSWORD }} cf_org: gsa-tts-oros-fac cf_space: ${{ env.space }} - command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup --command "./fac-backup-util.sh v0.1.5 deploy_backup" + command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup --command "./fac-backup-util.sh v0.1.9 deploy_backup" - name: Deploy Preview to cloud.gov if: ${{ inputs.environment == 'preview' }} @@ -123,5 +123,5 @@ jobs: secrets: inherit with: environment: ${{ inputs.environment }} - util_version: "v0.1.8" + util_version: "v0.1.9" backup_operation: "check_tables" diff --git a/.github/workflows/fac-api-standup-scheduler.yml b/.github/workflows/fac-api-standup-scheduler.yml index d25a589ea..85c04be32 100644 --- a/.github/workflows/fac-api-standup-scheduler.yml +++ b/.github/workflows/fac-api-standup-scheduler.yml @@ -20,6 +20,6 @@ jobs: secrets: inherit with: environment: ${{ matrix.environment.name }} - util_version: "v0.1.8" + util_version: "v0.1.9" backup_operation: "rds_backup" diff --git a/.github/workflows/fac-api-standup.yml b/.github/workflows/fac-api-standup.yml index 7b77845b5..580dbb6ac 100644 --- a/.github/workflows/fac-api-standup.yml +++ b/.github/workflows/fac-api-standup.yml @@ -1,8 +1,8 @@ --- name: Standup the API ### Common Commands: -# ./fac-backup-util.sh v0.1.8 scheduled_backup -# ./fac-backup-util.sh v0.1.8 daily_backup +# ./fac-backup-util.sh v0.1.9 scheduled_backup +# ./fac-backup-util.sh v0.1.9 daily_backup on: workflow_call: inputs: @@ -31,6 +31,7 @@ on: description: Version for fac backup utility to use (ex. vX.Y.Z) required: true type: string + default: "v0.1.9" backup_operation: description: Operation for fac-backup-utility required: true @@ -58,4 +59,4 @@ jobs: #cf_space: ${{ env.space }} cf_space: preview #command: cf run-task gsa-fac -k 3G -m 3G --name standup_api --command "./util/nightly_api_refresh.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}" - command: cf run-task gsa-fac -k 2G -m 2G --name api_refresh --command "./util/nightly_api_refresh.sh v0.1.8 rds_backup" + command: cf run-task gsa-fac -k 2G -m 2G --name api_refresh --command "./util/nightly_api_refresh.sh v0.1.9 rds_backup" diff --git a/.github/workflows/fac-backup-util-scheduled.yml b/.github/workflows/fac-backup-util-scheduled.yml index d5992fac3..4509e38fb 100644 --- a/.github/workflows/fac-backup-util-scheduled.yml +++ b/.github/workflows/fac-backup-util-scheduled.yml @@ -1,8 +1,8 @@ --- name: Backup the database with fac-backup-utility ### Common Commands: -# ./fac-backup-util.sh v0.1.8 scheduled_backup -# ./fac-backup-util.sh v0.1.8 daily_backup +# ./fac-backup-util.sh v0.1.9 scheduled_backup +# ./fac-backup-util.sh v0.1.9 daily_backup on: workflow_call: inputs: diff --git a/.github/workflows/fac-backup-util.yml b/.github/workflows/fac-backup-util.yml index 8e9bf804d..11a7caadc 100644 --- a/.github/workflows/fac-backup-util.yml +++ b/.github/workflows/fac-backup-util.yml @@ -1,8 +1,8 @@ --- name: Backup the database with fac-backup-utility ### Common Commands: -# ./fac-backup-util.sh v0.1.8 initial_backup -# ./fac-backup-util.sh v0.1.8 deploy_backup +# ./fac-backup-util.sh v0.1.9 initial_backup +# ./fac-backup-util.sh v0.1.9 deploy_backup on: workflow_dispatch: inputs: @@ -18,7 +18,7 @@ on: description: Version for fac backup utility to use (ex. vX.Y.Z) required: true type: string - default: "v0.1.8" + default: "v0.1.9" backup_operation: description: Operation for fac-backup-utility required: true diff --git a/.github/workflows/fac-check-tables-scheduler.yml b/.github/workflows/fac-check-tables-scheduler.yml index 111ca0029..a1b22b559 100644 --- a/.github/workflows/fac-check-tables-scheduler.yml +++ b/.github/workflows/fac-check-tables-scheduler.yml @@ -20,5 +20,5 @@ jobs: secrets: inherit with: environment: ${{ matrix.environment.name }} - util_version: "v0.1.8" + util_version: "v0.1.9" backup_operation: "check_tables" diff --git a/.github/workflows/fac-check-tables.yml b/.github/workflows/fac-check-tables.yml index f45d0f600..0ce9c2d14 100644 --- a/.github/workflows/fac-check-tables.yml +++ b/.github/workflows/fac-check-tables.yml @@ -1,7 +1,7 @@ --- name: Check existing tables in an environment ### Common Commands: -# ./fac-backup-util.sh v0.1.8 check_tables +# ./fac-backup-util.sh v0.1.9 check_tables on: workflow_dispatch: inputs: diff --git a/backend/tools/setup_local_env.sh b/backend/tools/setup_local_env.sh index d5f96d777..2c9da7621 100644 --- a/backend/tools/setup_local_env.sh +++ b/backend/tools/setup_local_env.sh @@ -4,7 +4,7 @@ function setup_local_env { if [[ "${ENV}" == "LOCAL" || "${ENV}" == "TESTING" ]]; then startup_log "LOCAL_ENV" "We are in a local envirnoment." - + # Load a fake VCAP_SERVICES file into the environment variable, # so we can mimic the cloud.gov setup. export VCAP_SERVICES=$(cat config/vcap_services_for_containers.json) @@ -23,16 +23,16 @@ function setup_local_env { # export AWS_PRIVATE_BUCKET_NAME="fac-private-s3" export AWS_PRIVATE_BUCKET_NAME=$(echo $VCAP_SERVICES \ - | jq --raw-output '.s3 - | map(select(.instance_name - | contains("fac-private-s3"))) + | jq --raw-output '.s3 + | map(select(.instance_name + | contains("fac-private-s3"))) | .[] .credentials.bucket') check_env_var_not_empty "AWS_PRIVATE_BUCKET_NAME" - + export AWS_PUBLIC_BUCKET_NAME=$(echo $VCAP_SERVICES \ - | jq --raw-output '.s3 - | map(select(.instance_name - | contains("fac-public-s3"))) + | jq --raw-output '.s3 + | map(select(.instance_name + | contains("fac-public-s3"))) | .[] .credentials.bucket') @@ -51,7 +51,7 @@ function setup_local_env { get_aws_s3 "fac-private-s3" "endpoint" export AWS_S3_PRIVATE_ENDPOINT=$_GET_AWS_RESULT check_env_var_not_empty "AWS_S3_PRIVATE_ENDPOINT" - + get_aws_s3 "fac-public-s3" "access_key_id" export AWS_PUBLIC_ACCESS_KEY_ID=$_GET_AWS_RESULT check_env_var_not_empty "AWS_PUBLIC_ACCESS_KEY_ID" @@ -63,7 +63,7 @@ function setup_local_env { get_aws_s3 "fac-public-s3" "endpoint" export AWS_S3_PUBLIC_ENDPOINT=$_GET_AWS_RESULT check_env_var_not_empty "AWS_S3_PUBLIC_ENDPOINT" - + #export MC_HOST_=https://::@ export MC_HOST_myminio="http://${AWS_PRIVATE_ACCESS_KEY_ID}:${AWS_PRIVATE_SECRET_ACCESS_KEY}@minio:9000" # mc alias set myminio ${AWS_S3_PRIVATE_ENDPOINT} ${AWS_PRIVATE_ACCESS_KEY_ID} ${AWS_PRIVATE_ACCESS_KEY_ID} @@ -72,13 +72,13 @@ function setup_local_env { # https://min.io/docs/minio/linux/reference/minio-mc/mc-mb.html mc mb --ignore-existing myminio/${AWS_PUBLIC_BUCKET_NAME} mc mb --ignore-existing myminio/${AWS_PRIVATE_BUCKET_NAME} - + # MCJ 20241016 FIXME: Is this even needed locally? I don't think so. # mc admin user svcacct add \ # --access-key="${AWS_PRIVATE_ACCESS_KEY_ID}" \ # --secret-key="${AWS_PRIVATE_SECRET_ACCESS_KEY}" \ # myminio minioadmin - + # For database work export FAC_DB_URI=${DATABASE_URL} #?sslmode=disable export FAC_SNAPSHOT_URI=${SNAPSHOT_URL} @@ -98,13 +98,13 @@ function setup_local_env { # And we need cgov-util pushd /tmp - local CGOV_VERSION=v0.1.8 + local CGOV_VERSION=v0.1.9 curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/${CGOV_VERSION}/gov.gsa.fac.cgov-util-${CGOV_VERSION}-linux-amd64.tar.gz tar xvzf gov.gsa.fac.cgov-util-${CGOV_VERSION}-linux-amd64.tar.gz gov.gsa.fac.cgov-util chmod 755 gov.gsa.fac.cgov-util mv gov.gsa.fac.cgov-util /bin/cgov-util popd - + export SLING_EXE='/bin/sling' export CGOV_UTIL_EXE='/bin/cgov-util' diff --git a/backend/util/load_public_dissem_data/Dockerfile b/backend/util/load_public_dissem_data/Dockerfile index 686278867..68d76caca 100644 --- a/backend/util/load_public_dissem_data/Dockerfile +++ b/backend/util/load_public_dissem_data/Dockerfile @@ -17,9 +17,9 @@ RUN curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download && chmod +x sling \ && mv sling /bin/sling -RUN curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/v0.1.8/gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz \ - && tar xvzf gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz gov.gsa.fac.cgov-util \ - && chmod 755 gov.gsa.fac.cgov-util \ +RUN curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/v0.1.9/gov.gsa.fac.cgov-util-v0.1.9-linux-amd64.tar.gz \ + && tar xvzf gov.gsa.fac.cgov-util-v0.1.9-linux-amd64.tar.gz gov.gsa.fac.cgov-util \ + && chmod 755 gov.gsa.fac.cgov-util \ && mv gov.gsa.fac.cgov-util /layered/cgov-util # Where do we need the config? Try home? diff --git a/backend/util/load_public_dissem_data/README.md b/backend/util/load_public_dissem_data/README.md index 3134d062a..62a62f117 100644 --- a/backend/util/load_public_dissem_data/README.md +++ b/backend/util/load_public_dissem_data/README.md @@ -50,7 +50,7 @@ This is containerized, so it should work on all platforms. To build the containe make build ``` -Then, to run the container, +Then, to run the container, ``` make run @@ -58,7 +58,7 @@ make run You need to run this from the current directory. -NOTE: The docker command in the Makefile uses the `--network` flag. The `--network` flag tells Docker to run this container on the same network as your currently running stack. It assumes that you did a `docker compose up` on the FAC stack, and that the web container has the default name of `backend-web-1`. If this does not work, you will need to... +NOTE: The docker command in the Makefile uses the `--network` flag. The `--network` flag tells Docker to run this container on the same network as your currently running stack. It assumes that you did a `docker compose up` on the FAC stack, and that the web container has the default name of `backend-web-1`. If this does not work, you will need to... ``` make NETWORK= run @@ -68,10 +68,10 @@ where `` is the name of your web container. This should allow th ## When to rebuild this container -Note this is pinned to v0.1.8 of the cgov-util. +Note this is pinned to v0.1.9 of the cgov-util. https://github.com/GSA-TTS/fac-backup-utility If that gets updated, you'll need to update the dockerfile. -It also copies in the YAML for sling from `dissemination/sql/sling`. If that changes, you'll want to +It also copies in the YAML for sling from `dissemination/sql/sling`. If that changes, you'll want to diff --git a/backend/util/load_public_dissem_data/load_public_data_locally.sh b/backend/util/load_public_dissem_data/load_public_data_locally.sh index 63d85effb..b71ced961 100755 --- a/backend/util/load_public_dissem_data/load_public_data_locally.sh +++ b/backend/util/load_public_dissem_data/load_public_data_locally.sh @@ -1,6 +1,6 @@ #!/bin/bash -export CGOV_UTIL_VERSION=v0.1.8 +export CGOV_UTIL_VERSION=v0.1.9 export FAC_DB_URI=postgresql://postgres@db:5432/postgres?sslmode=disable export FAC_SNAPSHOT_URI=postgresql://postgres@db2:5432/postgres?sslmode=disable @@ -42,7 +42,7 @@ echo "Unzipping data." unzip db_dissem_dump.zip popd -# Truncate the dissemination_* tables if they exist. +# Truncate the dissemination_* tables if they exist. # CASCADE as well. This makes sure we don't duplicate data, # which causes PK/FK problems. echo "select 'TRUNCATE TABLE '||tablename||' CASCADE;' FROM pg_tables WHERE tablename LIKE 'dissemination_%'" | \ diff --git a/backend/util/nightly_api_refresh.sh b/backend/util/nightly_api_refresh.sh index e3f32c26b..e78609105 100755 --- a/backend/util/nightly_api_refresh.sh +++ b/backend/util/nightly_api_refresh.sh @@ -7,7 +7,7 @@ source tools/sql_pre_post.sh setup_env # Run an RDS backup, to refresh dissemination_* in FAC_SNAPSHOT from FAC_DB -#./../fac-backup-util.sh "v0.1.8" "rds_backup" +#./../fac-backup-util.sh "v0.1.9" "rds_backup" # Deploy test ./fac-backup-util.sh "$1" "$2" gonogo "fac-backup-util" diff --git a/docs/backups_and_restores.md b/docs/backups_and_restores.md index 9019cae01..720c82a34 100644 --- a/docs/backups_and_restores.md +++ b/docs/backups_and_restores.md @@ -34,7 +34,7 @@ Information regarding the fac-backup-utility can be found [at the repository](ht Database backups occur in the following ways: 1. An initial backup, where a backup has not been run in the target environment. This input of `initial_backup` is important, as when it does a the `db_to_db` command, it will not truncate the target table, as the table does not exist in the destination database. ```bash -./fac-backup-util.sh v0.1.5 initial_backup +./fac-backup-util.sh v0.1.9 initial_backup # Curl the utility # Install AWS # DB to S3 table dump (backups) @@ -44,7 +44,7 @@ Database backups occur in the following ways: 2. A deploy backup, where the `db_to_db` function is not called. This is a standard backup strategy before the application deploys, to ensure the s3 contents of the primary s3 are sync'd to the backups bucket, and a table dump is stored in the backups bucket. ```bash -./fac-backup-util.sh v0.1.5 deploy_backup +./fac-backup-util.sh v0.1.9 deploy_backup # Curl the utility # Install AWS # DB to S3 table dump (backups) @@ -53,7 +53,7 @@ Database backups occur in the following ways: 3. A scheduled backup is run every two hours, across each environment, ensuring that we have a clean backup in s3, rds, and the bucket contents are in sync. ```bash -./fac-backup-util.sh v0.1.5 scheduled_backup +./fac-backup-util.sh v0.1.9 scheduled_backup # Curl the utility # Install AWS # DB to S3 table dump (fac-db -> backups) @@ -66,7 +66,7 @@ Restoring from backups can be run via workflow, from designated individuals. The 1. S3 Restore takes a `operation-mm-DD-HH` input (ex `scheduled-06-04-10`), and is required for the backups to be restored. The utility looks in `s3://${bucket}/backups/operation-mm-DD-HH/` for its table dumps, and without supplying the target backups, it will not restore. Once it does a `--data-only` restoration, it will then sync the files from the backups bucket to the application bucket. We do this to ensure the contents of the application bucket are up to date, relative to the data in the database. We know that if we use the latest folder in `/backups/` then the contents of the s3 are the latest available, from the prior backup. ```bash -./fac-restore-util.sh v0.1.5 s3_restore scheduled-06-04-10 +./fac-restore-util.sh v0.1.9 s3_restore scheduled-06-04-10 # Curl the utility # Install AWS # DB to S3 table dump (backups -> fac-db) [Truncate target table before --data-only pg_restore] @@ -81,7 +81,7 @@ daily-mm-dd 2. Database to database restoration also can occur as well, using `psql` to dump the tables from the cold store database to the live database. ```bash -./fac-restore-util.sh v0.1.5 db_restore +./fac-restore-util.sh v0.1.9 db_restore # Curl the utility # Install AWS # DB to DB table dump (fac-snapshot-db -> fac-db) [Truncate target table before dump]