Merge pull request #1061 from rcpch:csv-totals

total-registered-vs-total

epilepsy12/common_view_functions/aggregate_by.py

@@ -958,7 +958,7 @@ def create_KPI_aggregation_dataframe(
     return pd.DataFrame.from_dict(final_list)
 
 
-def create_reference_dataframe(trusts, health_boards, networks, icbs):
+def create_reference_dataframe(trusts, health_boards, networks, icbs, totals=False):
     """
     INPUTS:
     - trusts: TRUSTS should be passed in. Contains a list of trust objects
@@ -1027,3 +1027,73 @@ def create_reference_dataframe(trusts, health_boards, networks, icbs):
         final_list.append(item)
 
     return pd.DataFrame.from_dict(final_list)
+
+def create_totals_dataframe(cohort, abstraction_level):
+    """
+    create a dataframe for all organisations with totals of all registered cases vs cases included in aggregation
+    """
+
+    from ..common_view_functions import all_registered_cases_for_cohort_and_abstraction_level
+    Organisation = apps.get_model("epilepsy12", "Organisation")
+    Trust = apps.get_model("epilepsy12", "Trust")
+    LocalHealthBoard = apps.get_model("epilepsy12", "LocalHealthBoard")
+    IntegratedCareBoard = apps.get_model("epilepsy12", "IntegratedCareBoard")
+    NHSEnglandRegion = apps.get_model("epilepsy12", "NHSEnglandRegion")
+    OPENUKNetwork = apps.get_model("epilepsy12", "OPENUKNetwork")
+    Country = apps.get_model("epilepsy12", "Country")
+
+    abs_level = []
+
+
+    if abstraction_level == "trust":
+        query_set = Trust.objects.annotate(organisation_count=Count('organisation')).filter(organisation_count__gt=0)
+    elif abstraction_level == "local_health_board":
+        query_set = LocalHealthBoard.objects.annotate(organisation_count=Count('organisation')).filter(organisation_count__gt=0)
+    elif abstraction_level == "icb":
+        query_set = IntegratedCareBoard.objects.annotate(organisation_count=Count('organisation')).filter(organisation_count__gt=0)
+    elif abstraction_level == "nhs_england_region":
+        query_set = NHSEnglandRegion.objects.annotate(organisation_count=Count('organisation')).filter(organisation_count__gt=0)
+    elif abstraction_level == "open_uk":
+        query_set = OPENUKNetwork.objects.annotate(organisation_count=Count('organisation')).filter(organisation_count__gt=0)
+    elif abstraction_level == "country":
+        query_set = Country.objects.annotate(organisation_count=Count('organisation')).filter(organisation_count__gt=0)
+
+
+    for abstraction_item in query_set:
+        if abstraction_level == "trust":
+            organisation_instance = Organisation.objects.filter(trust=abstraction_item).first()
+        elif abstraction_level == "local_health_board":
+            organisation_instance = Organisation.objects.filter(local_health_board=abstraction_item).first()
+        elif abstraction_level == "icb":
+            organisation_instance = Organisation.objects.filter(integrated_care_board=abstraction_item).first()
+        elif abstraction_level == "nhs_england_region":
+            organisation_instance = Organisation.objects.filter(nhs_england_region=abstraction_item).first()
+        elif abstraction_level == "open_uk":
+            organisation_instance = Organisation.objects.filter(openuk_network=abstraction_item).first()
+        elif abstraction_level == "country":
+            organisation_instance = Organisation.objects.filter(country=abstraction_item).first()
+
+        all_cases = all_registered_cases_for_cohort_and_abstraction_level(
+            organisation_instance=organisation_instance,
+            cohort=cohort, 
+            case_complete=False, 
+            abstraction_level=abstraction_level
+        ).count()
+        all_registered_cases = all_registered_cases_for_cohort_and_abstraction_level(
+            organisation_instance=organisation_instance,
+            cohort=cohort, 
+            case_complete=True, 
+            abstraction_level=abstraction_level
+        ).count()
+
+        abs_level.append({
+            "name": abstraction_item.name,
+            "all_cases": all_cases,
+            "all_registered_cases": all_registered_cases,
+        })
+
+    return pd.DataFrame.from_dict(abs_level)
+
+
+
+

epilepsy12/common_view_functions/report_queries.py

@@ -136,7 +136,7 @@ def all_registered_cases_for_cohort_and_abstraction_level(
         )
     else:
         raise ValueError(
-            f"Incorrect or invalid abstraction error f{abstraction_level} supplied."
+            f"Incorrect or invalid abstraction error {abstraction_level} supplied."
         )
 
     return all_cases_for_cohort.filter(q_filter)

epilepsy12/kpi.py

@@ -17,9 +17,21 @@
     create_KPI_aggregation_dataframe,
     create_reference_dataframe,
     create_kpi_report_row,
-    get_kpi_aggregation_rows
+    get_kpi_aggregation_rows,
+    create_totals_dataframe,
+)
+from epilepsy12.models import (
+    Organisation,
+    Trust,
+    KPI,
+    ICBKPIAggregation,
+    OpenUKKPIAggregation,
+    CountryKPIAggregation,
+    LocalHealthBoardKPIAggregation,
+    TrustKPIAggregation,
+    NHSEnglandRegionKPIAggregation,
+    NationalKPIAggregation,
 )
-from epilepsy12.models import (Organisation, Trust, KPI, ICBKPIAggregation, OpenUKKPIAggregation, CountryKPIAggregation, LocalHealthBoardKPIAggregation, TrustKPIAggregation, NHSEnglandRegionKPIAggregation, NationalKPIAggregation)
 
 
 def download_kpi_summary_as_csv(cohort):
@@ -39,93 +51,83 @@ def download_kpi_summary_as_csv(cohort):
 
     # The summary only includes some measures
     measures = [
-        KPI._meta.get_field('paediatrician_with_expertise_in_epilepsies'),
-        KPI._meta.get_field('epilepsy_specialist_nurse'),
-        KPI._meta.get_field('tertiary_input'),
-        KPI._meta.get_field('epilepsy_surgery_referral'),
-        KPI._meta.get_field('ecg'),
-        KPI._meta.get_field('mri'),
-        KPI._meta.get_field('assessment_of_mental_health_issues'),
-        KPI._meta.get_field('mental_health_support'),
-        KPI._meta.get_field('sodium_valproate'),
-        KPI._meta.get_field('comprehensive_care_planning_agreement'),
-        KPI._meta.get_field('comprehensive_care_planning_content'),
-        KPI._meta.get_field('school_individual_healthcare_plan'),
+        KPI._meta.get_field("paediatrician_with_expertise_in_epilepsies"),
+        KPI._meta.get_field("epilepsy_specialist_nurse"),
+        KPI._meta.get_field("tertiary_input"),
+        KPI._meta.get_field("epilepsy_surgery_referral"),
+        KPI._meta.get_field("ecg"),
+        KPI._meta.get_field("mri"),
+        KPI._meta.get_field("assessment_of_mental_health_issues"),
+        KPI._meta.get_field("mental_health_support"),
+        KPI._meta.get_field("sodium_valproate"),
+        KPI._meta.get_field("comprehensive_care_planning_agreement"),
+        KPI._meta.get_field("comprehensive_care_planning_content"),
+        KPI._meta.get_field("school_individual_healthcare_plan"),
     ]
 
     # Many trusts do not have organisations that participate in the audit
     # The list of trusts in the database includes all of them
     # The list of organisations however is curated to just those that participate
-    trusts_from_organisations = Organisation.objects.values('trust').distinct()
-    trusts_participating_in_the_audit = Trust.objects.filter(id__in=trusts_from_organisations).values()
+    trusts_from_organisations = Organisation.objects.values("trust").distinct()
+    trusts_participating_in_the_audit = Trust.objects.filter(
+        id__in=trusts_from_organisations
+    ).values()
 
     # COUNTRY - SHEET 1
     # create a dataframe with a row for each measure of each country, and a column for each of ["Measure", "Percentage", "Numerator", "Denominator"]
 
     all_country_rows = get_kpi_aggregation_rows(
-        CountryKPIAggregation,
-        cohort,
-        abstraction_key_field="name"
+        CountryKPIAggregation, cohort, abstraction_key_field="name"
     )
-    
+
     # Only England and Wales participate in the audit but we have entries in the country table for NI and Scotland
     # Even if a participating country doesn't have any data yet we'd still like to include a blank row for it
     # We keep a row for Wales as we need it later on for SHEET 4 - NHS Region level
     england_rows = [row for row in all_country_rows if row["key_field"] == "England"]
     wales_rows = [row for row in all_country_rows if row["key_field"] == "Wales"]
 
     country_df = create_KPI_aggregation_dataframe(
-        england_rows + wales_rows,
-        measures,
-        title="Country"
+        england_rows + wales_rows, measures, title="Country"
     )
 
     # HBT (Trusts & Health Boards) - SHEET 2
 
     # Only in Wales
     local_health_board_rows = get_kpi_aggregation_rows(
-        LocalHealthBoardKPIAggregation,
-        cohort,
-        abstraction_key_field="ods_code"
+        LocalHealthBoardKPIAggregation, cohort, abstraction_key_field="ods_code"
     )
 
     # Only in England
     trust_rows = get_kpi_aggregation_rows(
-        TrustKPIAggregation,
-        cohort,
-        abstraction_key_field="ods_code"
+        TrustKPIAggregation, cohort, abstraction_key_field="ods_code"
     )
 
     # Only those trusts that participate in the audit
-    ods_codes_for_trusts_participating_in_the_audit = [trust["ods_code"] for trust in trusts_participating_in_the_audit]
-    trust_rows = [row for row in trust_rows if row["key_field"] in ods_codes_for_trusts_participating_in_the_audit]
+    ods_codes_for_trusts_participating_in_the_audit = [
+        trust["ods_code"] for trust in trusts_participating_in_the_audit
+    ]
+    trust_rows = [
+        row
+        for row in trust_rows
+        if row["key_field"] in ods_codes_for_trusts_participating_in_the_audit
+    ]
 
     trust_hb_df = create_KPI_aggregation_dataframe(
-        local_health_board_rows + trust_rows,
-        measures,
-        title="HBT"
+        local_health_board_rows + trust_rows, measures, title="HBT"
     )
 
     # ICB (Integrated Care Board) - SHEET 3
 
     icb_rows = get_kpi_aggregation_rows(
-        ICBKPIAggregation,
-        cohort,
-        abstraction_key_field="name"
+        ICBKPIAggregation, cohort, abstraction_key_field="name"
     )
 
-    icb_df = create_KPI_aggregation_dataframe(
-        icb_rows,
-        measures,
-        title="ICB"
-    )
+    icb_df = create_KPI_aggregation_dataframe(icb_rows, measures, title="ICB")
 
     # NHS region level - SHEET 4
 
     nhs_england_regional_rows = get_kpi_aggregation_rows(
-        NHSEnglandRegionKPIAggregation,
-        cohort,
-        abstraction_key_field="name"
+        NHSEnglandRegionKPIAggregation, cohort, abstraction_key_field="name"
     )
 
     # Treat Wales as a single region
@@ -140,29 +142,24 @@ def download_kpi_summary_as_csv(cohort):
     # NETWORKS - SHEET 5
 
     network_rows = get_kpi_aggregation_rows(
-        OpenUKKPIAggregation,
-        cohort,
-        abstraction_key_field="boundary_identifier"
+        OpenUKKPIAggregation, cohort, abstraction_key_field="boundary_identifier"
     )
 
     network_df = create_KPI_aggregation_dataframe(
-        network_rows,
-        measures,
-        title="Network"
+        network_rows, measures, title="Network"
     )
 
     # NATIONAL - SHEET 6
     # create a dataframe with a row for each measure, and column for each of ["Measure", "Percentage", "Numerator", "Denominator"]
     # note rows are named ["1. Paediatrician with expertise","2. Epilepsy specialist nurse","3a. Tertiary involvement","3b. Epilepsy surgery referral","4. ECG","5. MRI","6. Assessment of mental health issues","7. Mental health support","8. Sodium valproate","9a. Comprehensive care planning agreement","9b. Comprehensive care planning content","10. School Individual Health Care Plan"]
-    
+
     # NationalKPIAggregation has no abstraction relation to walk so add the key field manually
-    national_rows = [row | {"key_field": "England and Wales"} for row in get_kpi_aggregation_rows(NationalKPIAggregation, cohort)]
+    national_rows = [
+        row | {"key_field": "England and Wales"}
+        for row in get_kpi_aggregation_rows(NationalKPIAggregation, cohort)
+    ]
 
-    national_df = create_KPI_aggregation_dataframe(
-        national_rows,
-        measures,
-        title="uk"
-    )
+    national_df = create_KPI_aggregation_dataframe(national_rows, measures, title="uk")
 
     # REFERENCE - SHEET 7
 
@@ -173,6 +170,32 @@ def download_kpi_summary_as_csv(cohort):
         INTEGRATED_CARE_BOARDS_LOCAL_AUTHORITIES,
     )
 
+    # TOTALS - SHEET 8
+    trust_totals_df = create_totals_dataframe(
+        cohort=cohort,
+        abstraction_level="trust",
+    )
+    local_health_board_totals_df = create_totals_dataframe(
+        cohort=cohort,
+        abstraction_level="local_health_board",
+    )
+    icb_totals_df = create_totals_dataframe(
+        cohort=cohort,
+        abstraction_level="icb",
+    )
+    nhs_england_region_totals_df = create_totals_dataframe(
+        cohort=cohort,
+        abstraction_level="nhs_england_region",
+    )
+    openuk_network_totals_df = create_totals_dataframe(
+        cohort=cohort,
+        abstraction_level="open_uk",
+    )
+    country_totals_df = create_totals_dataframe(
+        cohort=cohort,
+        abstraction_level="country",
+    )
+
     return (
         country_df,
         trust_hb_df,
@@ -181,4 +204,10 @@ def download_kpi_summary_as_csv(cohort):
         network_df,
         national_df,
         reference_df,
+        trust_totals_df,
+        local_health_board_totals_df,
+        icb_totals_df,
+        nhs_england_region_totals_df,
+        openuk_network_totals_df,
+        country_totals_df,
     )

epilepsy12/views/organisation_views.py

@@ -536,6 +536,12 @@ def kpi_download_file(request):
         network_df,
         national_df,
         reference_df,
+        trust_totals_df,
+        local_health_board_totals_df,
+        icb_totals_df,
+        nhs_england_region_totals_df,
+        openuk_network_totals_df,
+        country_totals_df,
     ) = download_kpi_summary_as_csv(cohort=6)
 
     with pd.ExcelWriter("kpi_export.xlsx") as writer:
@@ -546,6 +552,24 @@ def kpi_download_file(request):
         network_df.to_excel(writer, sheet_name="Network_level", index=False)
         national_df.to_excel(writer, sheet_name="National_level", index=False)
         reference_df.to_excel(writer, sheet_name="Reference", index=False)
+        trust_totals_df.to_excel(
+            writer, sheet_name="Registered vs Total - Trust", index=False
+        )
+        local_health_board_totals_df.to_excel(
+            writer, sheet_name="Registered vs Total - LHB", index=False
+        )
+        icb_totals_df.to_excel(
+            writer, sheet_name="Registered vs Total - ICB", index=False
+        )
+        nhs_england_region_totals_df.to_excel(
+            writer, sheet_name="Registered vs Total - NHSregion", index=False
+        )
+        openuk_network_totals_df.to_excel(
+            writer, sheet_name="Registered vs Total - Network", index=False
+        )
+        country_totals_df.to_excel(
+            writer, sheet_name="Registered vs Total - Country", index=False
+        )
 
     with open("kpi_export.xlsx", "rb") as file:
         excel_data = file.read()