Add management command and workflow for generating live test data fro…

…m historical exports (#2569)

* First commit of E2E improvements

For running in production-like environments.

* add e2e generator workflow and profile changes

* lint

* double os

* more linting somehow

* don't do generation via profile

* Linting.

Runs defaults

* invoke via workflow instead of profile

* Apply suggestions from code review

* lint

---------

Co-authored-by: Matt Jadud <[email protected]>

.github/workflows/deploy-development.yml

@@ -66,3 +66,11 @@ jobs:
     with:
       url: "https://fac-dev.app.cloud.gov/"
 
+  generate-e2e-test-data:
+    needs:
+      - deploy-dev
+    name: 
+    uses: ./.github/workflows/end-to-end-test-data-generator.yml
+    secrets: inherit
+    with:
+      environment: "dev"

.github/workflows/deploy-staging.yml

@@ -57,3 +57,12 @@ jobs:
     with:
       url: "https://fac-staging.app.cloud.gov"
       environment: "staging"
+
+  generate-e2e-test-data:
+    needs:
+      - deploy-staging
+    name: 
+    uses: ./.github/workflows/end-to-end-test-data-generator.yml
+    secrets: inherit
+    with:
+      environment: "staging"

.github/workflows/end-to-end-test-data-generator.yml

@@ -0,0 +1,41 @@
+---
+name: "Task: run Django command to generate and disseminate test data for non-production environments."
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        required: true
+        type: choice
+        description: Which environment should the workflow run on?
+        options:
+          - dev
+          - staging
+      dbkeys:
+        required: false
+        type: string
+        description: Comma-separated list of report-IDs.
+      years:
+        required: false
+        type: string
+        description: Comma-separated list of report years. The list of years needs to be the same lenght as the list of dbkeys.
+
+jobs:
+  end-to-end-test-data:
+    name: Generate and disseminate end-to-end test data in ${{ inputs.environment }} database
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    env:
+      space: ${{ inputs.environment }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run Django command to generate and disseminate end-to-end test data in ${{ inputs.environment }}
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf run-task gsa-fac -k 2G -m 2G --name generate_e2e_data --command "python manage.py end_to_end_test_data_generator --dbkeys ${{ inputs.dbkeys }} --years ${{ inputs.years }}"

backend/config/settings.py

@@ -171,7 +171,13 @@
     ),
 }
 
-POSTGREST = {"URL": env.str("POSTGREST_URL", "http://api:3000")}
+POSTGREST = {
+    "URL": env.str("POSTGREST_URL", "http://api:3000"),
+    "LOCAL": env.str("POSTGREST_URL", "http://api:3000"),
+    "DEVELOPMENT": "https://api-dev.fac.gov",
+    "STAGING": "https://api-staging.fac.gov",
+    "PRODUCTION": "https://api.fac.gov",
+}
 
 
 # Password validation

backend/dissemination/management/commands/end_to_end_test_data_generator.py

@@ -0,0 +1,61 @@
+import os
+import logging
+import sys
+
+from config.settings import ENVIRONMENT
+from django.core.management.base import BaseCommand
+from dissemination.workbooklib.end_to_end_core import run_end_to_end
+
+CYPRESS_TEST_EMAIL_ADDR = os.getenv("CYPRESS_LOGIN_TEST_EMAIL_AUDITEE")
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument("--email", type=str, required=False)
+        parser.add_argument("--dbkeys", type=str, required=False, default="")
+        parser.add_argument("--years", type=str, required=False, default="")
+
+    def handle(self, *args, **options):
+        dbkeys_str = options["dbkeys"]
+        years_str = options["years"]
+        dbkeys = dbkeys_str.split(",")
+        years = years_str.split(",")
+
+        if len(dbkeys) != len(years):
+            logger.error(
+                "Received {} dbkeys and {} years. Must be equal. Exiting.".format(
+                    len(dbkeys), len(years)
+                )
+            )
+            sys.exit(-1)
+
+        lengths = [len(s) == 2 for s in years]
+        if dbkeys_str and years_str and (not all(lengths)):
+            logger.error("Years must be two digits. Exiting.")
+            sys.exit(-2)
+
+        email = options.get("email", CYPRESS_TEST_EMAIL_ADDR)
+
+        defaults = [
+            (182926, 22),
+            (181744, 22),
+            (191734, 22),
+        ]
+
+        if ENVIRONMENT in ["LOCAL", "DEVELOPMENT", "PREVIEW", "STAGING"]:
+            if dbkeys_str and years_str:
+                logger.info(
+                    f"Generating test reports for DBKEYS: {dbkeys_str} and YEARS: {years_str}"
+                )
+                for dbkey, year in zip(dbkeys, years):
+                    run_end_to_end(email, dbkey, year)
+            else:
+                for pair in defaults:
+                    logger.info("Running {}-{} end-to-end".format(pair[0], pair[1]))
+                    run_end_to_end(email, str(pair[0]), str(pair[1]))
+        else:
+            logger.error(
+                "Cannot run end-to-end workbook generation in production. Exiting."
+            )
+            sys.exit(-3)

backend/dissemination/management/commands/end_to_end_workbook_test.py → backend/dissemination/workbooklib/end_to_end_core.py

@@ -1,4 +1,3 @@
-from django.core.management.base import BaseCommand
 from users.models import User
 import argparse
 import logging
@@ -90,7 +89,12 @@ def call_api(api_url, endpoint, rid, field):
     )
     full_request = f"{api_url}/{endpoint}?report_id=eq.{rid}&select={field}"
     response = requests.get(
-        full_request, headers={"Authorization": f"Bearer {encoded_jwt}"}, timeout=10
+        full_request,
+        headers={
+            "Authorization": f"Bearer {encoded_jwt}",
+            "X-Api-Key": os.getenv("CYPRESS_API_GOV_KEY"),
+        },
+        timeout=10,
     )
     return response
 
@@ -126,7 +130,7 @@ def check_equality(in_wb, in_json):
 
 
 def get_api_values(endpoint, rid, field):
-    api_url = settings.POSTGREST.get("URL")
+    api_url = settings.POSTGREST.get(settings.ENVIRONMENT)
     res = call_api(api_url, endpoint, rid, field)
 
     if res.status_code == 200:
@@ -222,16 +226,3 @@ def run_end_to_end(email, dbkey, year):
         logger.info("No user found for %s, have you logged in once?", email)
         return
     generate_workbooks(user, email, dbkey, year)
-
-
-class Command(BaseCommand):
-    def add_arguments(self, parser):
-        parser.add_argument("--email", type=str, required=True)
-        parser.add_argument("--dbkey", type=str, required=True)
-        parser.add_argument("--year", type=str, default="22")
-
-    def handle(self, *args, **options):
-        email = options["email"]
-        dbkey = options["dbkey"]
-        year = options["year"]
-        run_end_to_end(email, dbkey, year)