feat: add migrate approvers and compare files scripts (#206)

* Update API response to match previous JSON format

- Add "ensure_ascii": false param to account for special characters
- Add more formatting to directly match the old JSON format.
- Fix formatting error with subgroups object
- Change how names are displayed.
- Only add approvers array if there are approvers.

* linting and update tests

* Update tests.py

* Add JSON Migration Script

- Add JSON file approver migration script to migrate approver values into the DB from the existing JSON file.
- Updated docs and links to reflect the change.
- Rename load.py to csv_load.py

* Add JSON compare file

- Formatting
- Add script to compare JSON file and API response saved to JSON file.

Pipfile

@@ -22,6 +22,7 @@ watchtower = "3.*"
 boto3 = "1.*"
 boto3-stubs = "1.*"
 django-cors-headers = "4.*"
+deepdiff = "8.*"
 
 [dev-packages]
 black = "==24.*"

doc/tutorial.md

@@ -20,11 +20,11 @@ oriented towards learning how rather than learning what.
 - Run the development server: `cd src`, then `python manage.py runserver`.
 - Install [yamlfmt](https://github.com/google/yamlfmt): `brew install yamlfmt`.
 
-## Running the load.py script to import CSV data into the database
+## Running the load.py script to import data into the database
 
 - Make sure all dependencies are synced: `pipenv sync --dev`.
-- Save CSV into `scripts` folder in directory.
-- Run `python manage.py runscript load`.
+- Save file into `scripts` folder in directory.
+- Run `python manage.py runscript {script_name}`.
 
 ## Running database backup
 

src/scripts/affils_compare.py

@@ -0,0 +1,46 @@
+"""
+Script to be run to compare JSON file and a JSON file of API response.
+
+Both files need to be saved in the `scripts` folder in directory before running.
+
+You can then run this script by running: 
+`python manage.py runscript affils_compare` in the command line from the directory.
+
+Follow steps outlined in [tutorial.md](
+doc/tutorial.md/#running-the-loadpy-script-to-import-data-into-the-database).
+"""
+
+from pathlib import Path
+import os
+import json
+from deepdiff import DeepDiff  # type: ignore
+
+CURRENT_DIR = os.path.dirname(__file__)
+FILENAME = os.path.join(CURRENT_DIR, "affils_diff_output.txt")
+
+AFFIL_JSON_PATH = Path(__file__).parent / "affiliations.json"
+AFFIL_RESPONSE_PATH = Path(__file__).parent / "affils_response.json"
+
+
+def run():
+    """Compare JSON file to API response and return a txt file of any differences."""
+    with open(AFFIL_JSON_PATH, encoding="utf-8") as f, open(
+        AFFIL_RESPONSE_PATH, encoding="utf-8"
+    ) as f2:
+        affils_json = json.load(f)
+        affils_response = json.load(f2)
+        affils_json_dict = {}
+        affils_response_dict = {}
+
+        # Build dict for each file.
+        for affil in affils_json:
+            affil_id = affil["affiliation_id"]
+            affils_json_dict[affil_id] = affil
+        for affil in affils_response:
+            affil_id = affil["affiliation_id"]
+            affils_response_dict[affil_id] = affil
+
+        # Compare both files
+        diff = DeepDiff(affils_json_dict, affils_response_dict)
+        with open(FILENAME, "w", encoding="utf-8") as f:
+            print(diff, file=f)

src/scripts/load.py → src/scripts/csv_load.py

@@ -5,10 +5,10 @@
 CSV needs to be saved in the `scripts` folder in directory before running.
 
 You can then run this script by running: 
-`python manage.py runscript load` in the command line from the directory.
+`python manage.py runscript csv_load` in the command line from the directory.
 
 Follow steps outlined in [tutorial.md](
-doc/tutorial.md/#running-the-loadpy-script-to-import-csv-data-into-the-database).
+doc/tutorial.md/#running-the-loadpy-script-to-import-data-into-the-database).
 """
 
 from pathlib import Path

src/scripts/json_load.py

@@ -0,0 +1,65 @@
+"""
+Script to be run to insert existing data from 
+the affiliations JSON file to the database.
+
+JSON file needs to be saved in the `scripts` folder in directory before running.
+
+You can then run this script by running: 
+`python manage.py runscript json_load` in the command line from the directory.
+
+Follow steps outlined in [tutorial.md](
+doc/tutorial.md/#running-the-loadpy-script-to-import-data-into-the-database).
+"""
+
+from pathlib import Path
+import json
+from affiliations.models import Affiliation, Approver
+
+FILEPATH = Path(__file__).parent / "affiliations.json"
+
+
+def run():
+    """Iterate through JSON file and update Affiliation with Approver
+    objects in the DB."""
+    count = 0
+    with open(FILEPATH, encoding="utf-8") as json_file:
+
+        data = json.load(json_file)
+        for item in data:
+            if "approver" in item:
+                affiliation_id = item["affiliation_id"]
+                approver = item["approver"]
+                if "subgroups" in item:
+                    if "vcep" in item["subgroups"]:
+                        vcep_id = item["subgroups"]["vcep"]["id"]
+                        affil_obj = Affiliation.objects.get(
+                            affiliation_id=affiliation_id, expert_panel_id=vcep_id
+                        )
+                        create_approver_model(approver, affil_obj, count)
+
+                    if "gcep" in item["subgroups"]:
+                        gcep_id = item["subgroups"]["gcep"]["id"]
+                        affil_obj = Affiliation.objects.get(
+                            affiliation_id=affiliation_id, expert_panel_id=gcep_id
+                        )
+                        create_approver_model(approver, affil_obj, count)
+                else:
+                    affil_obj = Affiliation.objects.get(affiliation_id=affiliation_id)
+                    create_approver_model(approver, affil_obj, count)
+        print(count, "changed")
+
+
+def create_approver_model(approver, affil_obj, count):
+    """Check if approver exists, if not create approver foreign key model."""
+    for approver_name in approver:
+        if not (
+            Approver.objects.filter(
+                affiliation=affil_obj,
+                approver_name=approver_name,
+            ).exists()
+        ):
+            count += 1
+            Approver.objects.create(
+                affiliation=affil_obj,
+                approver_name=approver_name,
+            )