Integration of fac-backup-utility (#3916)

* Add secondary db to tf to run db_to_db backup * add secondary db to local stack * Update command in makefile * bump s3 version and add tag * add dev maintained s3 * update service bindings * Remove binding steps from deploy * Add historic data load * Remove s3 bucket sharing We opted to remove this as the decision was made that staging would have its own dedicated backups bucket, and if files in the prod s3 bucket are to be shared, we can create a new bucket for the specific purpose of syncing then sharing * Add dedicated backups bucket in each environment * give org_name for backups bucket * Preliminary bash script for backups * chmod +x * File Rename * Function Modifications * Add restore script * Add backup workflow * fix typo * chmod +x * Small modifications to ensure util works properly * Have the version be an input * Update db2db operation * Testing workflow * scheduled_backup workflow test version bump to v0.1.2 of util * s3_restore workflow test * db_restore workflow test * Backup workflow Run via workflow_dispatch: * Quote to prevent globbing * Delete - File no longer used * Rename and replace workflow call Potentially going to delete * Update pre-deploy backup call * Add restore workflow * New scheduled backup workflow Now with a matrix, for all environments * CODEOWNERS update * Add docs * Point source to correct repo Though the redirect will still happen, the repo was moved to gsa-tts org * Version bump and modify backup logic * Change folder path * Add daily backup option * Update verbiage and workflow options * change pathing for s3 dumps * deploy_backup task test * scheduled_backup task test * Increase task instance size * scheduled_backup task test v2 * daily_backup task test * typo fixes * s3_restore task test * s3_restore task test v2 * db_restore task test * Final cleanup * remove (restore test) * typo fix * remove restore workflows per discussion with matt/tim * workflow cleanup and removal of unused items * Fix a small rebase issue
GSA-TTS · Jun 24, 2024 · 60a89b1 · 60a89b1
1 parent c5c6684
commit 60a89b1
Show file tree

Hide file tree

Showing 33 changed files with 654 additions and 622 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,7 +1,7 @@
-# This file is sensitive since rules below may be used for restricting who can 
-# make access control changes.
-/.github/CODEOWNERS @GSA-TTS/FAC-admins
-
-
-# Changes to the following Terraform directory will impact access control in cloud.gov spaces. Any PR involving these files should get a review from someone in FAC-admins.
-/terraform/meta/ @GSA-TTS/FAC-admins
+# This file is sensitive since rules below may be used for restricting who can
+# make access control changes.
+/.github/CODEOWNERS @GSA-TTS/FAC-admins
+/.github/workflows/fac-restore-util.yml @GSA-TTS/FAC-admins @asteel-gsa
+
+# Changes to the following Terraform directory will impact access control in cloud.gov spaces. Any PR involving these files should get a review from someone in FAC-admins.
+/terraform/meta/ @GSA-TTS/FAC-admins
diff --git a/.github/workflows/deploy-application.yml b/.github/workflows/deploy-application.yml
@@ -1,148 +1,128 @@
----
-name: Deploy application to cloud.gov
-on:
-  workflow_call:
-    inputs:
-      environment:
-        required: true
-        type: string
-
-jobs:
-  push-with-creds:
-    name: Deploy to cloud.gov with updated credentials
-    runs-on: ubuntu-latest
-    environment: ${{ inputs.environment }}
-    env:
-      space: ${{ inputs.environment }}
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Restore npm cache
-        uses: actions/cache@v4
-        id: cache-npm
-        with:
-          path: ~/.npm
-          key: fac-build-npm-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            fac-build-npm-
-            fac-build-
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - name: Restore pip cache
-        uses: actions/cache@v4
-        id: cache-pip
-        with:
-          path: |
-            ~/.cache/pip
-            /opt/hostedtoolcache/Python/
-          key: fac-build-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/dev-requirements.txt') }}
-          restore-keys: |
-            fac-build-pip-${{ hashFiles('**/requirements.txt') }}-
-            fac-build-pip-
-            fac-build-
-
-      - name: Install npm dependencies
-        working-directory: ./backend
-        run: npm ci --production
-
-      - name: Compile JS/CSS assets
-        working-directory: ./backend
-        run: npm run build
-
-      - name: Update service keys
-        uses: cloud-gov/cg-cli-tools@main
-        env:
-          SAM_API_KEY: ${{ secrets.SAM_API_KEY }}
-          DJANGO_SECRET_LOGIN_KEY: $${{ secrets.DJANGO_SECRET_LOGIN_KEY }}
-          LOGIN_CLIENT_ID: $${{ secrets.LOGIN_CLIENT_ID }}
-          SECRET_KEY: ${{ secrets.SECRET_KEY }}
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          cf_command: update-user-provided-service fac-key-service -p '"{\"SAM_API_KEY\":\"${{ secrets.SAM_API_KEY }}\", \"DJANGO_SECRET_LOGIN_KEY\":\"${{ secrets.DJANGO_SECRET_LOGIN_KEY }}\", \"LOGIN_CLIENT_ID\":\"${{ secrets.LOGIN_CLIENT_ID }}\", \"SECRET_KEY\":\"${{ secrets.SECRET_KEY}}\"}"'
-
-      - name: Bind backup s3 bucket to prod app
-        if: startsWith(github.ref, 'refs/tags/v1.')
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          command: cf bind-service gsa-fac backups
-
-      - name: Backup the database (Prod Only)
-        if: startsWith(github.ref, 'refs/tags/v1.')
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          command: cf run-task gsa-fac -k 2G -m 2G --name pg_backup --command "./backup_database.sh ${{ env.space }}"
-
-      - name: Deploy Preview to cloud.gov
-        if: ${{ inputs.environment == 'preview' }}
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          cf_manifest: backend/manifests/manifest-preview.yml
-          cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml
-          command: bin/ops/deploy_preview.sh
-
-      - name: Deploy fac to cloud.gov
-        if: ${{ inputs.environment  != 'preview' }}
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          cf_manifest: backend/manifests/manifest-fac.yml
-          cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml
-          command: bin/ops/deploy.sh
-
-      - name: Unbind backup s3 bucket from prod app
-        if: startsWith(github.ref, 'refs/tags/v1.')
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          command: cf unbind-service gsa-fac backups
-
-      - name: Load historical data
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          command: cf run-task gsa-fac -k 6G -m 1G --name load_data --command "./load_data.sh"
-
-      # This has to happen after an application deployment because the manifest (currently) is responsible
-      # for binding the "logdrain service" to the "gsa-fac application". This also needs to be done
-      # based on the suspicion that fluentbit cannot register the incoming logs when it is initially
-      # created, resulting in a 502. Restarting the application after everything is configured results
-      # in a 201, or, the expected status when transmitting logs.
-      - name: Restart the logshipper application
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          command: cf restart logshipper
-
+---
+name: Deploy application to cloud.gov
+on:
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
+
+jobs:
+  push-with-creds:
+    name: Deploy to cloud.gov with updated credentials
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    env:
+      space: ${{ inputs.environment }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Restore npm cache
+        uses: actions/cache@v4
+        id: cache-npm
+        with:
+          path: ~/.npm
+          key: fac-build-npm-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            fac-build-npm-
+            fac-build-
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Restore pip cache
+        uses: actions/cache@v4
+        id: cache-pip
+        with:
+          path: |
+            ~/.cache/pip
+            /opt/hostedtoolcache/Python/
+          key: fac-build-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/dev-requirements.txt') }}
+          restore-keys: |
+            fac-build-pip-${{ hashFiles('**/requirements.txt') }}-
+            fac-build-pip-
+            fac-build-
+
+      - name: Install npm dependencies
+        working-directory: ./backend
+        run: npm ci --production
+
+      - name: Compile JS/CSS assets
+        working-directory: ./backend
+        run: npm run build
+
+      - name: Update service keys
+        uses: cloud-gov/cg-cli-tools@main
+        env:
+          SAM_API_KEY: ${{ secrets.SAM_API_KEY }}
+          DJANGO_SECRET_LOGIN_KEY: $${{ secrets.DJANGO_SECRET_LOGIN_KEY }}
+          LOGIN_CLIENT_ID: $${{ secrets.LOGIN_CLIENT_ID }}
+          SECRET_KEY: ${{ secrets.SECRET_KEY }}
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          cf_command: update-user-provided-service fac-key-service -p '"{\"SAM_API_KEY\":\"${{ secrets.SAM_API_KEY }}\", \"DJANGO_SECRET_LOGIN_KEY\":\"${{ secrets.DJANGO_SECRET_LOGIN_KEY }}\", \"LOGIN_CLIENT_ID\":\"${{ secrets.LOGIN_CLIENT_ID }}\", \"SECRET_KEY\":\"${{ secrets.SECRET_KEY}}\"}"'
+
+      - name: Backup the database
+        # if: startsWith(github.ref, 'refs/tags/v1.')
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup --command "./fac-backup-util.sh v0.1.3 deploy_backup"
+
+      - name: Deploy Preview to cloud.gov
+        if: ${{ inputs.environment == 'preview' }}
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          cf_manifest: backend/manifests/manifest-preview.yml
+          cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml
+          command: bin/ops/deploy_preview.sh
+
+      - name: Deploy fac to cloud.gov
+        if: ${{ inputs.environment  != 'preview' }}
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          cf_manifest: backend/manifests/manifest-fac.yml
+          cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml
+          command: bin/ops/deploy.sh
+
+      - name: Load historical data
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf run-task gsa-fac -k 6G -m 1G --name load_data --command "./load_data.sh"
+
+      # This has to happen after an application deployment because the manifest (currently) is responsible
+      # for binding the "logdrain service" to the "gsa-fac application". This also needs to be done
+      # based on the suspicion that fluentbit cannot register the incoming logs when it is initially
+      # created, resulting in a 502. Restarting the application after everything is configured results
+      # in a 201, or, the expected status when transmitting logs.
+      - name: Restart the logshipper application
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf restart logshipper
+
diff --git a/.github/workflows/fac-backup-scheduler.yml b/.github/workflows/fac-backup-scheduler.yml
@@ -0,0 +1,24 @@
+---
+name: Backup Environments
+on:
+  schedule:
+    # Invoke every 2 hours
+    - cron: '0 */2 * * *'
+  workflow_dispatch: null
+
+jobs:
+  backup-environment:
+    strategy:
+      fail-fast: false
+      matrix:
+        environment:
+          - name: dev
+          - name: staging
+          - name: production
+    uses: ./.github/workflows/fac-backup-util-scheduled.yml
+    secrets: inherit
+    with:
+      environment: ${{ matrix.environment.name }}
+      util_version: "v0.1.3"
+      backup_operation: "scheduled_backup"
+
diff --git a/.github/workflows/fac-backup-util-scheduled.yml b/.github/workflows/fac-backup-util-scheduled.yml
@@ -0,0 +1,36 @@
+---
+name: Backup the database with fac-backup-utility
+### Common Commands:
+# ./fac-backup-util.sh v0.1.3 scheduled_backup
+# ./fac-backup-util.sh v0.1.3 daily_backup
+on:
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
+      util_version:
+        description: Version for fac backup utility to use (ex. vX.Y.Z)
+        required: true
+        type: string
+      backup_operation:
+        description: Operation for fac-backup-utility
+        required: true
+        type: string
+jobs:
+  fac-backup-scheduled:
+    name: Perform Database Backup
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    env:
+      space: ${{ inputs.environment }}
+    steps:
+      - name: Backup FAC Database and Sync Media
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf run-task gsa-fac -k 7G -m 3G --name backup_util_scheduled --command "./fac-backup-util.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}"
+
diff --git a/.github/workflows/fac-backup-util.yml b/.github/workflows/fac-backup-util.yml
@@ -0,0 +1,43 @@
+---
+name: Backup the database with fac-backup-utility
+### Common Commands:
+# ./fac-backup-util.sh v0.1.3 initial_backup
+# ./fac-backup-util.sh v0.1.3 deploy_backup
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        required: true
+        type: choice
+        options:
+          - 'dev'
+          - 'preview'
+          - 'staging'
+          - 'production'
+      util_version:
+        description: Version for fac backup utility to use (ex. vX.Y.Z)
+        required: true
+        type: string
+      backup_operation:
+        description: Operation for fac-backup-utility
+        required: true
+        type: choice
+        options:
+          - 'initial_backup'
+          - 'deploy_backup'
+jobs:
+  fac-backup:
+    name: Perform Database Backup
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    env:
+      space: ${{ inputs.environment }}
+    steps:
+      - name: Backup FAC Database and Sync Media
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup_util --command "./fac-backup-util.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}"