From 37fbe89c74e9f3387a64d61d0d8ccae19d001c0e Mon Sep 17 00:00:00 2001 From: eatyourpeas Date: Mon, 7 Oct 2024 22:30:27 +0100 Subject: [PATCH] adds separate sheets to count total cases vs total registered --- .../common_view_functions/aggregate_by.py | 72 ++++++++- .../common_view_functions/report_queries.py | 2 +- epilepsy12/kpi.py | 145 +++++++++++------- epilepsy12/views/organisation_views.py | 24 +++ 4 files changed, 183 insertions(+), 60 deletions(-) diff --git a/epilepsy12/common_view_functions/aggregate_by.py b/epilepsy12/common_view_functions/aggregate_by.py index 532f3ad8..a1ac75f3 100644 --- a/epilepsy12/common_view_functions/aggregate_by.py +++ b/epilepsy12/common_view_functions/aggregate_by.py @@ -958,7 +958,7 @@ def create_KPI_aggregation_dataframe( return pd.DataFrame.from_dict(final_list) -def create_reference_dataframe(trusts, health_boards, networks, icbs): +def create_reference_dataframe(trusts, health_boards, networks, icbs, totals=False): """ INPUTS: - trusts: TRUSTS should be passed in. Contains a list of trust objects @@ -1027,3 +1027,73 @@ def create_reference_dataframe(trusts, health_boards, networks, icbs): final_list.append(item) return pd.DataFrame.from_dict(final_list) + +def create_totals_dataframe(cohort, abstraction_level): + """ + create a dataframe for all organisations with totals of all registered cases vs cases included in aggregation + """ + + from ..common_view_functions import all_registered_cases_for_cohort_and_abstraction_level + Organisation = apps.get_model("epilepsy12", "Organisation") + Trust = apps.get_model("epilepsy12", "Trust") + LocalHealthBoard = apps.get_model("epilepsy12", "LocalHealthBoard") + IntegratedCareBoard = apps.get_model("epilepsy12", "IntegratedCareBoard") + NHSEnglandRegion = apps.get_model("epilepsy12", "NHSEnglandRegion") + OPENUKNetwork = apps.get_model("epilepsy12", "OPENUKNetwork") + Country = apps.get_model("epilepsy12", "Country") + + abs_level = [] + + + if abstraction_level == "trust": + query_set = Trust.objects.annotate(organisation_count=Count('organisation')).filter(organisation_count__gt=0) + elif abstraction_level == "local_health_board": + query_set = LocalHealthBoard.objects.annotate(organisation_count=Count('organisation')).filter(organisation_count__gt=0) + elif abstraction_level == "icb": + query_set = IntegratedCareBoard.objects.annotate(organisation_count=Count('organisation')).filter(organisation_count__gt=0) + elif abstraction_level == "nhs_england_region": + query_set = NHSEnglandRegion.objects.annotate(organisation_count=Count('organisation')).filter(organisation_count__gt=0) + elif abstraction_level == "open_uk": + query_set = OPENUKNetwork.objects.annotate(organisation_count=Count('organisation')).filter(organisation_count__gt=0) + elif abstraction_level == "country": + query_set = Country.objects.annotate(organisation_count=Count('organisation')).filter(organisation_count__gt=0) + + + for abstraction_item in query_set: + if abstraction_level == "trust": + organisation_instance = Organisation.objects.filter(trust=abstraction_item).first() + elif abstraction_level == "local_health_board": + organisation_instance = Organisation.objects.filter(local_health_board=abstraction_item).first() + elif abstraction_level == "icb": + organisation_instance = Organisation.objects.filter(integrated_care_board=abstraction_item).first() + elif abstraction_level == "nhs_england_region": + organisation_instance = Organisation.objects.filter(nhs_england_region=abstraction_item).first() + elif abstraction_level == "open_uk": + organisation_instance = Organisation.objects.filter(openuk_network=abstraction_item).first() + elif abstraction_level == "country": + organisation_instance = Organisation.objects.filter(country=abstraction_item).first() + + all_cases = all_registered_cases_for_cohort_and_abstraction_level( + organisation_instance=organisation_instance, + cohort=cohort, + case_complete=False, + abstraction_level=abstraction_level + ).count() + all_registered_cases = all_registered_cases_for_cohort_and_abstraction_level( + organisation_instance=organisation_instance, + cohort=cohort, + case_complete=True, + abstraction_level=abstraction_level + ).count() + + abs_level.append({ + "name": abstraction_item.name, + "all_cases": all_cases, + "all_registered_cases": all_registered_cases, + }) + + return pd.DataFrame.from_dict(abs_level) + + + + diff --git a/epilepsy12/common_view_functions/report_queries.py b/epilepsy12/common_view_functions/report_queries.py index 8f7d258c..3bc50f70 100644 --- a/epilepsy12/common_view_functions/report_queries.py +++ b/epilepsy12/common_view_functions/report_queries.py @@ -136,7 +136,7 @@ def all_registered_cases_for_cohort_and_abstraction_level( ) else: raise ValueError( - f"Incorrect or invalid abstraction error f{abstraction_level} supplied." + f"Incorrect or invalid abstraction error {abstraction_level} supplied." ) return all_cases_for_cohort.filter(q_filter) diff --git a/epilepsy12/kpi.py b/epilepsy12/kpi.py index f1aba184..2eeaf262 100644 --- a/epilepsy12/kpi.py +++ b/epilepsy12/kpi.py @@ -17,9 +17,21 @@ create_KPI_aggregation_dataframe, create_reference_dataframe, create_kpi_report_row, - get_kpi_aggregation_rows + get_kpi_aggregation_rows, + create_totals_dataframe, +) +from epilepsy12.models import ( + Organisation, + Trust, + KPI, + ICBKPIAggregation, + OpenUKKPIAggregation, + CountryKPIAggregation, + LocalHealthBoardKPIAggregation, + TrustKPIAggregation, + NHSEnglandRegionKPIAggregation, + NationalKPIAggregation, ) -from epilepsy12.models import (Organisation, Trust, KPI, ICBKPIAggregation, OpenUKKPIAggregation, CountryKPIAggregation, LocalHealthBoardKPIAggregation, TrustKPIAggregation, NHSEnglandRegionKPIAggregation, NationalKPIAggregation) def download_kpi_summary_as_csv(cohort): @@ -39,35 +51,35 @@ def download_kpi_summary_as_csv(cohort): # The summary only includes some measures measures = [ - KPI._meta.get_field('paediatrician_with_expertise_in_epilepsies'), - KPI._meta.get_field('epilepsy_specialist_nurse'), - KPI._meta.get_field('tertiary_input'), - KPI._meta.get_field('epilepsy_surgery_referral'), - KPI._meta.get_field('ecg'), - KPI._meta.get_field('mri'), - KPI._meta.get_field('assessment_of_mental_health_issues'), - KPI._meta.get_field('mental_health_support'), - KPI._meta.get_field('sodium_valproate'), - KPI._meta.get_field('comprehensive_care_planning_agreement'), - KPI._meta.get_field('comprehensive_care_planning_content'), - KPI._meta.get_field('school_individual_healthcare_plan'), + KPI._meta.get_field("paediatrician_with_expertise_in_epilepsies"), + KPI._meta.get_field("epilepsy_specialist_nurse"), + KPI._meta.get_field("tertiary_input"), + KPI._meta.get_field("epilepsy_surgery_referral"), + KPI._meta.get_field("ecg"), + KPI._meta.get_field("mri"), + KPI._meta.get_field("assessment_of_mental_health_issues"), + KPI._meta.get_field("mental_health_support"), + KPI._meta.get_field("sodium_valproate"), + KPI._meta.get_field("comprehensive_care_planning_agreement"), + KPI._meta.get_field("comprehensive_care_planning_content"), + KPI._meta.get_field("school_individual_healthcare_plan"), ] # Many trusts do not have organisations that participate in the audit # The list of trusts in the database includes all of them # The list of organisations however is curated to just those that participate - trusts_from_organisations = Organisation.objects.values('trust').distinct() - trusts_participating_in_the_audit = Trust.objects.filter(id__in=trusts_from_organisations).values() + trusts_from_organisations = Organisation.objects.values("trust").distinct() + trusts_participating_in_the_audit = Trust.objects.filter( + id__in=trusts_from_organisations + ).values() # COUNTRY - SHEET 1 # create a dataframe with a row for each measure of each country, and a column for each of ["Measure", "Percentage", "Numerator", "Denominator"] all_country_rows = get_kpi_aggregation_rows( - CountryKPIAggregation, - cohort, - abstraction_key_field="name" + CountryKPIAggregation, cohort, abstraction_key_field="name" ) - + # Only England and Wales participate in the audit but we have entries in the country table for NI and Scotland # Even if a participating country doesn't have any data yet we'd still like to include a blank row for it # We keep a row for Wales as we need it later on for SHEET 4 - NHS Region level @@ -75,57 +87,47 @@ def download_kpi_summary_as_csv(cohort): wales_rows = [row for row in all_country_rows if row["key_field"] == "Wales"] country_df = create_KPI_aggregation_dataframe( - england_rows + wales_rows, - measures, - title="Country" + england_rows + wales_rows, measures, title="Country" ) # HBT (Trusts & Health Boards) - SHEET 2 # Only in Wales local_health_board_rows = get_kpi_aggregation_rows( - LocalHealthBoardKPIAggregation, - cohort, - abstraction_key_field="ods_code" + LocalHealthBoardKPIAggregation, cohort, abstraction_key_field="ods_code" ) # Only in England trust_rows = get_kpi_aggregation_rows( - TrustKPIAggregation, - cohort, - abstraction_key_field="ods_code" + TrustKPIAggregation, cohort, abstraction_key_field="ods_code" ) # Only those trusts that participate in the audit - ods_codes_for_trusts_participating_in_the_audit = [trust["ods_code"] for trust in trusts_participating_in_the_audit] - trust_rows = [row for row in trust_rows if row["key_field"] in ods_codes_for_trusts_participating_in_the_audit] + ods_codes_for_trusts_participating_in_the_audit = [ + trust["ods_code"] for trust in trusts_participating_in_the_audit + ] + trust_rows = [ + row + for row in trust_rows + if row["key_field"] in ods_codes_for_trusts_participating_in_the_audit + ] trust_hb_df = create_KPI_aggregation_dataframe( - local_health_board_rows + trust_rows, - measures, - title="HBT" + local_health_board_rows + trust_rows, measures, title="HBT" ) # ICB (Integrated Care Board) - SHEET 3 icb_rows = get_kpi_aggregation_rows( - ICBKPIAggregation, - cohort, - abstraction_key_field="name" + ICBKPIAggregation, cohort, abstraction_key_field="name" ) - icb_df = create_KPI_aggregation_dataframe( - icb_rows, - measures, - title="ICB" - ) + icb_df = create_KPI_aggregation_dataframe(icb_rows, measures, title="ICB") # NHS region level - SHEET 4 nhs_england_regional_rows = get_kpi_aggregation_rows( - NHSEnglandRegionKPIAggregation, - cohort, - abstraction_key_field="name" + NHSEnglandRegionKPIAggregation, cohort, abstraction_key_field="name" ) # Treat Wales as a single region @@ -140,29 +142,24 @@ def download_kpi_summary_as_csv(cohort): # NETWORKS - SHEET 5 network_rows = get_kpi_aggregation_rows( - OpenUKKPIAggregation, - cohort, - abstraction_key_field="boundary_identifier" + OpenUKKPIAggregation, cohort, abstraction_key_field="boundary_identifier" ) network_df = create_KPI_aggregation_dataframe( - network_rows, - measures, - title="Network" + network_rows, measures, title="Network" ) # NATIONAL - SHEET 6 # create a dataframe with a row for each measure, and column for each of ["Measure", "Percentage", "Numerator", "Denominator"] # note rows are named ["1. Paediatrician with expertise","2. Epilepsy specialist nurse","3a. Tertiary involvement","3b. Epilepsy surgery referral","4. ECG","5. MRI","6. Assessment of mental health issues","7. Mental health support","8. Sodium valproate","9a. Comprehensive care planning agreement","9b. Comprehensive care planning content","10. School Individual Health Care Plan"] - + # NationalKPIAggregation has no abstraction relation to walk so add the key field manually - national_rows = [row | {"key_field": "England and Wales"} for row in get_kpi_aggregation_rows(NationalKPIAggregation, cohort)] + national_rows = [ + row | {"key_field": "England and Wales"} + for row in get_kpi_aggregation_rows(NationalKPIAggregation, cohort) + ] - national_df = create_KPI_aggregation_dataframe( - national_rows, - measures, - title="uk" - ) + national_df = create_KPI_aggregation_dataframe(national_rows, measures, title="uk") # REFERENCE - SHEET 7 @@ -173,6 +170,32 @@ def download_kpi_summary_as_csv(cohort): INTEGRATED_CARE_BOARDS_LOCAL_AUTHORITIES, ) + # TOTALS - SHEET 8 + trust_totals_df = create_totals_dataframe( + cohort=cohort, + abstraction_level="trust", + ) + local_health_board_totals_df = create_totals_dataframe( + cohort=cohort, + abstraction_level="local_health_board", + ) + icb_totals_df = create_totals_dataframe( + cohort=cohort, + abstraction_level="icb", + ) + nhs_england_region_totals_df = create_totals_dataframe( + cohort=cohort, + abstraction_level="nhs_england_region", + ) + openuk_network_totals_df = create_totals_dataframe( + cohort=cohort, + abstraction_level="open_uk", + ) + country_totals_df = create_totals_dataframe( + cohort=cohort, + abstraction_level="country", + ) + return ( country_df, trust_hb_df, @@ -181,4 +204,10 @@ def download_kpi_summary_as_csv(cohort): network_df, national_df, reference_df, + trust_totals_df, + local_health_board_totals_df, + icb_totals_df, + nhs_england_region_totals_df, + openuk_network_totals_df, + country_totals_df, ) diff --git a/epilepsy12/views/organisation_views.py b/epilepsy12/views/organisation_views.py index 12cebd15..88145b08 100644 --- a/epilepsy12/views/organisation_views.py +++ b/epilepsy12/views/organisation_views.py @@ -536,6 +536,12 @@ def kpi_download_file(request): network_df, national_df, reference_df, + trust_totals_df, + local_health_board_totals_df, + icb_totals_df, + nhs_england_region_totals_df, + openuk_network_totals_df, + country_totals_df, ) = download_kpi_summary_as_csv(cohort=6) with pd.ExcelWriter("kpi_export.xlsx") as writer: @@ -546,6 +552,24 @@ def kpi_download_file(request): network_df.to_excel(writer, sheet_name="Network_level", index=False) national_df.to_excel(writer, sheet_name="National_level", index=False) reference_df.to_excel(writer, sheet_name="Reference", index=False) + trust_totals_df.to_excel( + writer, sheet_name="Registered vs Total - Trust", index=False + ) + local_health_board_totals_df.to_excel( + writer, sheet_name="Registered vs Total - LHB", index=False + ) + icb_totals_df.to_excel( + writer, sheet_name="Registered vs Total - ICB", index=False + ) + nhs_england_region_totals_df.to_excel( + writer, sheet_name="Registered vs Total - NHSregion", index=False + ) + openuk_network_totals_df.to_excel( + writer, sheet_name="Registered vs Total - Network", index=False + ) + country_totals_df.to_excel( + writer, sheet_name="Registered vs Total - Country", index=False + ) with open("kpi_export.xlsx", "rb") as file: excel_data = file.read()