feat: add migrate approvers and compare files scripts

Pipfile

@@ -22,6 +22,7 @@ watchtower = "3.*"
 boto3 = "1.*"
 boto3-stubs = "1.*"
 django-cors-headers = "4.*"
+deepdiff = "8.*"
 
 [dev-packages]
 black = "==24.*"

doc/tutorial.md

@@ -20,11 +20,11 @@ oriented towards learning how rather than learning what.
 - Run the development server: `cd src`, then `python manage.py runserver`.
 - Install [yamlfmt](https://github.com/google/yamlfmt): `brew install yamlfmt`.
 
-## Running the load.py script to import CSV data into the database
+## Running the load.py script to import data into the database
 
 - Make sure all dependencies are synced: `pipenv sync --dev`.
-- Save CSV into `scripts` folder in directory.
-- Run `python manage.py runscript load`.
+- Save file into `scripts` folder in directory.
+- Run `python manage.py runscript {script_name}`.
 
 ## Running database backup
 

src/affiliations/tests.py

@@ -30,7 +30,7 @@ def setUpTestData(cls):
             "full_name": "Test Success Result Affil",
             "short_name": "Successful",
             "status": "Inactive",
-            "type": "Gene Curation Expert Panel",
+            "type": "GCEP",
             "clinical_domain_working_group": "Neurodevelopmental Disorders",
             "members": "Bulbasaur, Charmander, Squirtle",
             "is_deleted": False,
@@ -191,17 +191,15 @@ def test_detail_affiliation_json_call(self):
             response.json(),
             [
                 {
-                    "affiliation_id": 10000,
+                    "affiliation_id": "10000",
                     "affiliation_fullname": "Test Success Result Affil",
                     "subgroups": {
-                        "gene curation expert panel": {
-                            "id": 40000,
+                        "gcep": {
+                            "id": "40000",
                             "fullname": "Test Success Result Affil",
                         }
                     },
-                    "approver": [
-                        "Mew",
-                    ],
+                    "approver": ["Mew"],
                 }
             ],
         )

src/affiliations/views.py

@@ -35,39 +35,67 @@ def affiliations_list_json_format(request):  # pylint: disable=unused-argument
     response_obj = {}
     for affil in affils_queryset:
         affil_type = affil["type"].lower()
-        if affil["affiliation_id"] not in response_obj:
-            old_json_format = {
-                "affiliation_id": affil["affiliation_id"],
-                "affiliation_fullname": affil["full_name"],
-                "subgroups": {
-                    affil_type: {
-                        "id": affil["expert_panel_id"],
-                        "fullname": affil["full_name"],
+        # In old JSON, SC-VCEPS are only considered VCEPS.
+        if affil_type == "sc_vcep":
+            affil_type = "vcep"
+        # In old JSON, Affiliation IDs and EP Ids are in string format.
+        affil_id = str(affil["affiliation_id"])
+        ep_id = str(affil["expert_panel_id"])
+
+        if affil_id not in response_obj:
+            if affil_type in ["vcep", "gcep"]:
+                old_json_format = {
+                    "affiliation_id": affil_id,
+                    "affiliation_fullname": affil["full_name"],
+                    "subgroups": {
+                        affil_type: {
+                            "id": ep_id,
+                            "fullname": affil["full_name"],
+                        },
                     },
-                },
-                "approver": [],
-            }
-            response_obj[affil["affiliation_id"]] = old_json_format
-
-        elif affil_type not in response_obj[affil["affiliation_id"]]["subgroups"]:
-            response_obj[affil["affiliation_id"]]["affiliation_fullname"] = (
-                response_obj[affil["affiliation_id"]]["affiliation_fullname"]
-                + "/"
-                + affil["type"]
-            )
-            response_obj[affil["affiliation_id"]]["subgroups"][affil_type] = {
-                affil_type: {
-                    "id": affil["expert_panel_id"],
-                    "fullname": affil["full_name"],
-                },
+                }
+            # Independent curation group format
+            else:
+                old_json_format = {
+                    "affiliation_id": affil_id,
+                    "affiliation_fullname": affil["full_name"],
+                }
+            response_obj[affil_id] = old_json_format
+        elif affil_type not in response_obj[affil_id]["subgroups"]:
+            # If VCEP or GCEP in full name, add other subgroup to end of name.
+            if ("VCEP" in response_obj[affil_id]["affiliation_fullname"]) or (
+                "GCEP" in response_obj[affil_id]["affiliation_fullname"]
+            ):
+                response_obj[affil_id]["affiliation_fullname"] = (
+                    response_obj[affil_id]["affiliation_fullname"] + "/" + affil["type"]
+                )
+            # Else append affiliation subgroup name to full name
+            else:
+                response_obj[affil_id]["affiliation_fullname"] = (
+                    response_obj[affil_id]["affiliation_fullname"]
+                    + "/"
+                    + affil["full_name"]
+                )
+
+            response_obj[affil_id]["subgroups"][affil_type] = {
+                "id": ep_id,
+                "fullname": affil["full_name"],
             }
+        # If there are approvers, add them to the object.
         approvers_queryset = Approver.objects.filter(
             affiliation_id=affil["id"]
         ).values_list("approver_name", flat=True)
+        if approvers_queryset and "approver" not in response_obj[affil_id]:
+            response_obj[affil_id]["approver"] = []
         for name in approvers_queryset:
-            response_obj[affil["affiliation_id"]]["approver"].append(name)
+            response_obj[affil_id]["approver"].append(name)
 
-    return JsonResponse(list(response_obj.values()), status=200, safe=False)
+    return JsonResponse(
+        list(response_obj.values()),
+        status=200,
+        safe=False,
+        json_dumps_params={"ensure_ascii": False},
+    )
 
 
 @login_required
@@ -81,36 +109,63 @@ def affiliation_detail_json_format(request):
     response_obj = {}
     for affil in affils_queryset:
         affil_type = affil["type"].lower()
-        if affil["affiliation_id"] not in response_obj:
-            old_json_format = {
-                "affiliation_id": affil["affiliation_id"],
-                "affiliation_fullname": affil["full_name"],
-                "subgroups": {
-                    affil_type: {
-                        "id": affil["expert_panel_id"],
-                        "fullname": affil["full_name"],
+        # In old JSON, SC-VCEPS are only considered VCEPS.
+        if affil_type == "sc_vcep":
+            affil_type = "vcep"
+        # In old JSON, Affiliation IDs and EP Ids are in string format.
+        affil_id = str(affil["affiliation_id"])
+        ep_id = str(affil["expert_panel_id"])
+
+        if affil_id not in response_obj:
+            if affil_type in ["vcep", "gcep"]:
+                old_json_format = {
+                    "affiliation_id": affil_id,
+                    "affiliation_fullname": affil["full_name"],
+                    "subgroups": {
+                        affil_type: {
+                            "id": ep_id,
+                            "fullname": affil["full_name"],
+                        },
                     },
-                },
-                "approver": [],
-            }
-            response_obj[affil["affiliation_id"]] = old_json_format
-
-        elif affil_type not in response_obj[affil["affiliation_id"]]["subgroups"]:
-            response_obj[affil["affiliation_id"]]["affiliation_fullname"] = (
-                response_obj[affil["affiliation_id"]]["affiliation_fullname"]
-                + "/"
-                + affil["full_name"]
-            )
-            response_obj[affil["affiliation_id"]]["subgroups"][affil_type] = {
-                affil_type: {
-                    "id": affil["expert_panel_id"],
-                    "fullname": affil["full_name"],
-                },
+                }
+            # Independent curation group format
+            else:
+                old_json_format = {
+                    "affiliation_id": affil_id,
+                    "affiliation_fullname": affil["full_name"],
+                }
+            response_obj[affil_id] = old_json_format
+        elif affil_type not in response_obj[affil_id]["subgroups"]:
+            # If VCEP or GCEP in full name, add other subgroup to end of name.
+            if ("VCEP" in response_obj[affil_id]["affiliation_fullname"]) or (
+                "GCEP" in response_obj[affil_id]["affiliation_fullname"]
+            ):
+                response_obj[affil_id]["affiliation_fullname"] = (
+                    response_obj[affil_id]["affiliation_fullname"] + "/" + affil["type"]
+                )
+            # Else append affiliation subgroup name to full name
+            else:
+                response_obj[affil_id]["affiliation_fullname"] = (
+                    response_obj[affil_id]["affiliation_fullname"]
+                    + "/"
+                    + affil["full_name"]
+                )
+
+            response_obj[affil_id]["subgroups"][affil_type] = {
+                "id": ep_id,
+                "fullname": affil["full_name"],
             }
+        # If there are approvers, add them to the object.
         approvers_queryset = Approver.objects.filter(
             affiliation_id=affil["id"]
         ).values_list("approver_name", flat=True)
+        if approvers_queryset and "approver" not in response_obj[affil_id]:
+            response_obj[affil_id]["approver"] = []
         for name in approvers_queryset:
-            response_obj[affil["affiliation_id"]]["approver"].append(name)
-
-    return JsonResponse(list(response_obj.values()), status=200, safe=False)
+            response_obj[affil_id]["approver"].append(name)
+    return JsonResponse(
+        list(response_obj.values()),
+        status=200,
+        safe=False,
+        json_dumps_params={"ensure_ascii": False},
+    )

src/scripts/affils_compare.py

@@ -0,0 +1,46 @@
+"""
+Script to be run to compare JSON file and a JSON file of API response.
+
+Both files need to be saved in the `scripts` folder in directory before running.
+
+You can then run this script by running: 
+`python manage.py runscript affils_compare` in the command line from the directory.
+
+Follow steps outlined in [tutorial.md](
+doc/tutorial.md/#running-the-loadpy-script-to-import-data-into-the-database).
+"""
+
+from pathlib import Path
+import os
+import json
+from deepdiff import DeepDiff  # type: ignore
+
+CURRENT_DIR = os.path.dirname(__file__)
+FILENAME = os.path.join(CURRENT_DIR, "affils_diff_output.txt")
+
+AFFIL_JSON_PATH = Path(__file__).parent / "affiliations.json"
+AFFIL_RESPONSE_PATH = Path(__file__).parent / "affils_response.json"
+
+
+def run():
+    """Compare JSON file to API response and return a txt file of any differences."""
+    with open(AFFIL_JSON_PATH, encoding="utf-8") as f, open(
+        AFFIL_RESPONSE_PATH, encoding="utf-8"
+    ) as f2:
+        affils_json = json.load(f)
+        affils_response = json.load(f2)
+        affils_json_dict = {}
+        affils_response_dict = {}
+
+        # Build dict for each file.
+        for affil in affils_json:
+            affil_id = affil["affiliation_id"]
+            affils_json_dict[affil_id] = affil
+        for affil in affils_response:
+            affil_id = affil["affiliation_id"]
+            affils_response_dict[affil_id] = affil
+
+        # Compare both files
+        diff = DeepDiff(affils_json_dict, affils_response_dict)
+        with open(FILENAME, "w", encoding="utf-8") as f:
+            print(diff, file=f)

src/scripts/load.py → src/scripts/csv_load.py

@@ -5,10 +5,10 @@
 CSV needs to be saved in the `scripts` folder in directory before running.
 
 You can then run this script by running: 
-`python manage.py runscript load` in the command line from the directory.
+`python manage.py runscript csv_load` in the command line from the directory.
 
 Follow steps outlined in [tutorial.md](
-doc/tutorial.md/#running-the-loadpy-script-to-import-csv-data-into-the-database).
+doc/tutorial.md/#running-the-loadpy-script-to-import-data-into-the-database).
 """
 
 from pathlib import Path

src/scripts/json_load.py

@@ -0,0 +1,65 @@
+"""
+Script to be run to insert existing data from 
+the affiliations JSON file to the database.
+
+JSON file needs to be saved in the `scripts` folder in directory before running.
+
+You can then run this script by running: 
+`python manage.py runscript json_load` in the command line from the directory.
+
+Follow steps outlined in [tutorial.md](
+doc/tutorial.md/#running-the-loadpy-script-to-import-data-into-the-database).
+"""
+
+from pathlib import Path
+import json
+from affiliations.models import Affiliation, Approver
+
+FILEPATH = Path(__file__).parent / "affiliations.json"
+
+
+def run():
+    """Iterate through JSON file and update Affiliation with Approver
+    objects in the DB."""
+    count = 0
+    with open(FILEPATH, encoding="utf-8") as json_file:
+
+        data = json.load(json_file)
+        for item in data:
+            if "approver" in item:
+                affiliation_id = item["affiliation_id"]
+                approver = item["approver"]
+                if "subgroups" in item:
+                    if "vcep" in item["subgroups"]:
+                        vcep_id = item["subgroups"]["vcep"]["id"]
+                        affil_obj = Affiliation.objects.get(
+                            affiliation_id=affiliation_id, expert_panel_id=vcep_id
+                        )
+                        create_approver_model(approver, affil_obj, count)
+
+                    if "gcep" in item["subgroups"]:
+                        gcep_id = item["subgroups"]["gcep"]["id"]
+                        affil_obj = Affiliation.objects.get(
+                            affiliation_id=affiliation_id, expert_panel_id=gcep_id
+                        )
+                        create_approver_model(approver, affil_obj, count)
+                else:
+                    affil_obj = Affiliation.objects.get(affiliation_id=affiliation_id)
+                    create_approver_model(approver, affil_obj, count)
+        print(count, "changed")
+
+
+def create_approver_model(approver, affil_obj, count):
+    """Check if approver exists, if not create approver foreign key model."""
+    for approver_name in approver:
+        if not (
+            Approver.objects.filter(
+                affiliation=affil_obj,
+                approver_name=approver_name,
+            ).exists()
+        ):
+            count += 1
+            Approver.objects.create(
+                affiliation=affil_obj,
+                approver_name=approver_name,
+            )