From f076466a6b75006c5f41f8c5620a020a0111c33a Mon Sep 17 00:00:00 2001 From: Matthew Jadud Date: Mon, 25 Mar 2024 18:27:18 -0400 Subject: [PATCH] Search, Advanced Search, Oh My! (#3552) * Enabling advanced search This leaves a basic search in place, and adds an advanced search. Per https://github.com/GSA-TTS/FAC/issues/3526 for more information and documentation at the top of search.html: All input fields besides UEI/EIN are commented out pending a performance investigation and fix. Summary report downloads are also disabled. ** SUMMARY REPORTS HAVE BEEN RE-ENABLED** However, this commit missed the SF-SAC on overview pages. TBD. Alerts have been put in place. ** ALERTS HAVE BEEN COMMENTED OUT ** To undo this action, complete the following steps. Search filters: 1. Uncomment the search filters ** SEARCH FILTERS UNCOMMENTED FROM search/advanced ** Warnings: 1. Remove search-alert-performance.html and its imports. ** THIS WAS COMMENTED OUT ** 2. Remove the warning banner in header.html ** THIS WAS ALSO COMMENTED OUT ** Summary report downloads: 1. Uncomment the summary report download button here ** THIS WAS UNUNCOMMENTED ** 2. Uncomment the SF-SAC download button in summary.html ** THIS WAS NOT DONE - NEXT COMMIT ** 3. Remove the forced Http404 in SingleSummaryReportDownloadView (dissemination/views.py) ** I HAVE NO IDEA IF THIS WAS DONE... TBD ** * Uncommenting the SF-SAC on summary pages * Correctly flip between basic/advanced Adds a flag for advanced search. * Linting results. * Fix linting * A fix on the adv search code And test. * Updating text * Update search-alert-info.html Copy edits to "Basic Search" and "Advanced Search" sentences. * Enabling single summary report download * cleanup * update views for advanced search * lint * general params and MP logic * invert the general params * invert the inversion * update test_search * fix summary download tests in test_views --------- Co-authored-by: Hassan D. M. Sambo Co-authored-by: Laura H <123114420+lauraherring@users.noreply.github.com> Co-authored-by: Daniel Swick <2365503+danswick@users.noreply.github.com> --- backend/dissemination/forms.py | 75 ++- backend/dissemination/search.py | 47 +- .../searchlib/search_major_program.py | 4 +- backend/dissemination/templates/advanced.html | 438 ++++++++++++++++++ .../templates/search-alert-info.html | 18 +- .../templates/search-alert-performance.html | 14 - .../templates/search-table-header.html | 1 + backend/dissemination/templates/search.html | 166 +------ backend/dissemination/templates/summary.html | 4 +- backend/dissemination/test_search.py | 42 +- backend/dissemination/urls.py | 1 + backend/dissemination/views.py | 140 +++++- backend/templates/includes/header.html | 3 - 13 files changed, 710 insertions(+), 243 deletions(-) create mode 100644 backend/dissemination/templates/advanced.html delete mode 100644 backend/dissemination/templates/search-alert-performance.html diff --git a/backend/dissemination/forms.py b/backend/dissemination/forms.py index 760e0cca0c..a0f0606726 100644 --- a/backend/dissemination/forms.py +++ b/backend/dissemination/forms.py @@ -1,7 +1,7 @@ from django import forms -class SearchForm(forms.Form): +class AdvancedSearchForm(forms.Form): # Multiple choice field mappings findings_field_mapping = { "field_name": [ @@ -134,3 +134,76 @@ def clean_limit(self): Default page limit to 30. """ return int(self.cleaned_data["limit"] or 30) + + +class SearchForm(forms.Form): + # Multiple choice field Tuples. "choices" variable in field declaration. + AY_choices = (("all_years", "All years"),) + tuple( + (x, str(x)) for x in reversed(range(2016, 2024)) + ) + + # Query params + entity_name = forms.CharField(required=False) + uei_or_ein = forms.CharField(required=False) + start_date = forms.DateField(required=False) + end_date = forms.DateField(required=False) + agency_name = forms.CharField(required=False) + audit_year = forms.MultipleChoiceField( + choices=AY_choices, initial=[2023], required=False + ) + auditee_state = forms.CharField(required=False) + + # Display params + limit = forms.CharField(required=False) + page = forms.CharField(required=False) + order_by = forms.CharField(required=False) + order_direction = forms.CharField(required=False) + + # Variables for cleaning functions + text_input_delimiters = [ + ",", + ":", + ";", + "-", + " ", + ] + + def clean_entity_name(self): + """ + Clean the name field. We can't trust that separators aren't a part of a name somewhere, + so just split on newlines. + """ + text_input = self.cleaned_data["entity_name"] + return text_input.splitlines() + + def clean_uei_or_ein(self): + """ + Clean up the UEI/EIN field. Replace common separators with a newline. + Split on the newlines. Strip all the resulting elements. + """ + text_input = self.cleaned_data["uei_or_ein"] + for delimiter in self.text_input_delimiters: + text_input = text_input.replace(delimiter, "\n") + text_input = [x.strip() for x in text_input.splitlines()] + return text_input + + def clean_audit_year(self): + """ + If "All years" is selected, don't include any years. + """ + audit_year = self.cleaned_data["audit_year"] + if "all_years" in audit_year: + return [] + return audit_year + + def clean_page(self): + """ + Default page number to one. + """ + return int(self.cleaned_data["page"] or 1) + + def clean_limit(self): + """ + Default page limit to 30. + """ + return int(self.cleaned_data["limit"] or 30) diff --git a/backend/dissemination/search.py b/backend/dissemination/search.py index bb35f8a562..980852d559 100644 --- a/backend/dissemination/search.py +++ b/backend/dissemination/search.py @@ -6,7 +6,7 @@ from .searchlib.search_findings import search_findings from .searchlib.search_direct_funding import search_direct_funding from .searchlib.search_major_program import search_major_program -from dissemination.models import DisseminationCombined +from dissemination.models import DisseminationCombined, General logger = logging.getLogger(__name__) @@ -16,22 +16,26 @@ # Their ORM cookbook looks to be useful reading. # https://books.agiliq.com/projects/django-orm-cookbook/en/latest/subquery.html +# {'alns': [], -- DisseminationCombined +# 'names': ['AWESOME'], -- General +# 'uei_or_eins': [], -- General +# 'start_date': None, -- General +# 'end_date': None, -- General +# 'cog_or_oversight': '', -- General, but not wanted +# 'agency_name': '', -- NO IDEA +# 'audit_years': [], -- General +# 'findings': [], -- DisseminationCombined +# 'direct_funding': [], -- DisseminationCombined +# 'major_program': [], -- DisseminationCombined +# 'auditee_state': '', -- General +# 'order_by': -- General +# 'fac_accepted_date', -- General +# 'order_direction': -- General +# 'descending', 'LIMIT': 1000} -- General -def is_only_general_params(params_dict): - params_set = set(list(params_dict.keys())) - gen_set = set( - [ - "audit_years", - "auditee_state", - "names", - "uei_or_eins", - "start_date", - "end_date", - "agency_name", - "cog_or_oversight", - ] - ) - return params_set.issubset(gen_set) + +def is_advanced_search(params_dict): + return params_dict.get("advanced_search_flag") def search(params): @@ -49,16 +53,19 @@ def search(params): ############## # GENERAL - if is_only_general_params(params): - results = search_general(DisseminationCombined, params) - results = _sort_results(results, params) - else: + logger.info(params) + if is_advanced_search(params): + logger.info("search Searching `DisseminationCombined`") results = search_general(DisseminationCombined, params) results = _sort_results(results, params) results = search_alns(results, params) results = search_findings(results, params) results = search_direct_funding(results, params) results = search_major_program(results, params) + else: + logger.info("search Searching `General`") + results = search_general(General, params) + results = _sort_results(results, params) results = results.distinct("report_id", params.get("order_by", "fac_accepted_date")) diff --git a/backend/dissemination/searchlib/search_major_program.py b/backend/dissemination/searchlib/search_major_program.py index 3dbd241b53..3e11e35e43 100644 --- a/backend/dissemination/searchlib/search_major_program.py +++ b/backend/dissemination/searchlib/search_major_program.py @@ -15,9 +15,9 @@ def search_major_program(general_results, params): q = Q() major_program_fields = params.get("major_program", []) - if True in major_program_fields: + if "True" in major_program_fields: q |= Q(is_major="Y") - elif False in major_program_fields: + elif "False" in major_program_fields: q |= Q(is_major="N") filtered_general_results = general_results.filter(q).distinct() diff --git a/backend/dissemination/templates/advanced.html b/backend/dissemination/templates/advanced.html new file mode 100644 index 0000000000..0e36ab728e --- /dev/null +++ b/backend/dissemination/templates/advanced.html @@ -0,0 +1,438 @@ +{% extends "base.html" %} +{% load static %} +{% load sprite_helper %} + +{% block content %} +
+
+
+

Filters

+
+ {% csrf_token %} + {% comment %} Submission {% endcomment %} +
+ + +
+
+ {% comment %} Audit Year {% endcomment %} + +
+ {% for value, text in form.audit_year.field.choices %} +
+ + +
+ {% endfor %} +
+ + {% comment %} UEI/EIN {% endcomment %} + + + + {% comment %} ALN/CFDA {% endcomment %} + + + + {% comment %} Name {% endcomment %} + + + + {% comment %} Release Date(s) {% endcomment %} + +
+
+ +
mm/dd/yyyy
+
+ +
+
+
+ +
mm/dd/yyyy
+
+ +
+
+
+ + {% comment %} State {% endcomment %} + +
+ + +
+ + {% comment %} Cog/Over {% endcomment %} + +
+ + + + +
+ + {% comment %} Findings {% endcomment %} + +
+ {% for value, text in form.findings.field.choices %} +
+ + +
+ {% endfor %} +
+ + {% comment %} Direct funding {% endcomment %} + +
+ {% for value, text in form.direct_funding.field.choices %} +
+ + +
+ {% endfor %} +
+ + {% comment %} Major program {% endcomment %} + +
+ {% for value, text in form.major_program.field.choices %} +
+ + +
+ {% endfor %} +
+
+ + {% comment %} Submission {% endcomment %} +
+ + +
+ + {% comment %} Hidden page input for use when clicking pagination buttons {% endcomment %} + + + {% comment %} Hidden order and direction inputs for use when clicking a sort button in the table header {% endcomment %} + + +
+
+
+

Search single audit reports

+ {% include "search-alert-info.html"%} + {% if results|length > 0 %} +
+

+ Results: {{ results_count }} + showing {{ limit }} per page +

+ {% comment %} SF-SAC generation {% endcomment %} + {% if results_count <= summary_report_download_limit %} + + {% endif %} +
+ +
+ + + + {% include "search-table-header.html" with friendly_title="Name" field_name="auditee_name" %} + {% include "search-table-header.html" with friendly_title="UEI or EIN" field_name="auditee_uei" %} + {% include "search-table-header.html" with friendly_title="Acc Date" field_name="fac_accepted_date" %} + {% include "search-table-header.html" with friendly_title="AY" field_name="audit_year" %} + {% include "search-table-header.html" with friendly_title="Cog or Over" field_name="cog_over" %} + + + {% if results.0.finding_my_aln is not None %} + {% include "search-table-header.html" with friendly_title="Finding my ALN" field_name="findings_my_aln"%} + {% include "search-table-header.html" with friendly_title="Finding other ALN" field_name="findings_all_aln"%} + {% endif %} + + + + {% for result in results %} + + + {% comment %} Display UEI. If it's "GSA_MIGRATION", use the EIN instead. If no EIN, just show "GSA_MIGRATION". {% endcomment %} + + {% comment %} Sorts ascending/descending by the numeric date string (i.e. 20231231) {% endcomment %} + + + + + + {% if result.finding_my_aln is not None%} + + + {% endif %} + + {% endfor %} + +
ViewPDF
{{ result.auditee_name }} + {% if result.auditee_uei != "GSA_MIGRATION" %} + {{ result.auditee_uei }} + {% elif result.auditee_ein %} + {{ result.auditee_ein }} + {% else %} + {{ result.auditee_uei }} + {% endif %} + {{ result.fac_accepted_date }} + {{ result.audit_year }} + {% if result.oversight_agency %} + OVER-{{ result.oversight_agency }} + {% elif result.cognizant_agency %} + COG-{{ result.cognizant_agency }} + {% else %} + N/A + {% endif %} + + + + + + {% if result.is_public or include_private %} + + + + {% endif %} + + + {% if result.finding_my_aln %} + Y + {% else %} + N + {% endif %} + + + + {% if result.finding_all_aln %} + Y + {% else %} + N + {% endif %} + +
+
+ + {% elif results is not None %} +
+ an arrow points left, toward the search form +

+ No results found. +

+
+ {% else %} +
+ an arrow points left, toward the search form +

+ Enter your filters and select Search to begin +

+
+ {% endif %} +
+
+
+
+
+
+

Please rotate your device

+
+

Search works best with your device in landscape orientation.

+
+
+ +
+
+ + + +{% endblock content %} diff --git a/backend/dissemination/templates/search-alert-info.html b/backend/dissemination/templates/search-alert-info.html index 2c27829c95..5001b56e1b 100644 --- a/backend/dissemination/templates/search-alert-info.html +++ b/backend/dissemination/templates/search-alert-info.html @@ -4,18 +4,32 @@

Searching the FAC database

- Learn more about how our search filters work on our Search Resources page. + + Learn more about search on our Search Resources page.

+

Summary reports

-

+

For search results of {{ summary_report_download_limit|intcomma }} submissions or less, you can download a combined spreadsheet of all data. If you need to download more than {{ summary_report_download_limit|intcomma }} submissions, try limiting your search parameters to download in batches.

+ {% if "advanced" in request.path %} +

Basic search

+

+ We update advanced search nightly with new submissions. Basic search is always up-to-date. This is best for users wanting to confirm their submissions are complete. +

+ {% else %} +

Advanced search

+

+ Audit resolution officials may want to use the advanced search for additional filters. +

+ {% endif %}
diff --git a/backend/dissemination/templates/search-alert-performance.html b/backend/dissemination/templates/search-alert-performance.html deleted file mode 100644 index 329f190b82..0000000000 --- a/backend/dissemination/templates/search-alert-performance.html +++ /dev/null @@ -1,14 +0,0 @@ -{% load sprite_helper %} -
-
-

Warning status

-

- Audit search is currently limited to UEI-only. You can search your UEI to confirm your audit submission is complete. The PDF report will still be available for download but summary report workbooks won't. -

-

- For more information, visit - the FAC system status page - . -

-
-
diff --git a/backend/dissemination/templates/search-table-header.html b/backend/dissemination/templates/search-table-header.html index 755a7376b3..8008eebaf3 100644 --- a/backend/dissemination/templates/search-table-header.html +++ b/backend/dissemination/templates/search-table-header.html @@ -1,3 +1,4 @@ + {% load sprite_helper %}
@@ -42,7 +23,6 @@

Filters

{% comment %} Audit Year {% endcomment %} - {% comment %}
{% endfor %}
- {% endcomment %} {% comment %} UEI/EIN {% endcomment %} - - {% comment %} ALN/CFDA {% endcomment %} - {% comment %} - - - {% endcomment %} {% comment %} Name {% endcomment %} - {% comment %} - {% endcomment %} {% comment %} Release Date(s) {% endcomment %} - {% comment %} -
- - - - -
- {% endcomment %} - - {% comment %} Findings {% endcomment %} - {% comment %} - -
- {% for value, text in form.findings.field.choices %} -
- - -
- {% endfor %} -
- {% endcomment %} - - {% comment %} Direct funding {% endcomment %} - {% comment %} - -
- {% for value, text in form.direct_funding.field.choices %} -
- - -
- {% endfor %} -
- {% endcomment %} - - {% comment %} Major program {% endcomment %} - {% comment %} - -
- {% for value, text in form.major_program.field.choices %} -
- - -
- {% endfor %} -
- {% endcomment %} - + {% comment %} Submission {% endcomment %} - {% comment %}
- {% endcomment %} {% comment %} Hidden page input for use when clicking pagination buttons {% endcomment %} Filters

Search single audit reports

- {% include "search-alert-performance.html"%} {% include "search-alert-info.html"%} {% if results|length > 0 %}
@@ -304,13 +174,8 @@

Search single audit reports

{% include "search-table-header.html" with friendly_title="UEI or EIN" field_name="auditee_uei" %} {% include "search-table-header.html" with friendly_title="Acc Date" field_name="fac_accepted_date" %} {% include "search-table-header.html" with friendly_title="AY" field_name="audit_year" %} - {% include "search-table-header.html" with friendly_title="Cog or Over" field_name="cog_over" %} View PDF - {% if results.0.finding_my_aln is not None %} - {% include "search-table-header.html" with friendly_title="Finding my ALN" field_name="findings_my_aln"%} - {% include "search-table-header.html" with friendly_title="Finding other ALN" field_name="findings_all_aln"%} - {% endif %} @@ -331,15 +196,6 @@

Search single audit reports

{{ result.fac_accepted_date }} {{ result.audit_year }} - - {% if result.oversight_agency %} - OVER-{{ result.oversight_agency }} - {% elif result.cognizant_agency %} - COG-{{ result.cognizant_agency }} - {% else %} - N/A - {% endif %} - Search single audit reports {% endif %} - {% if result.finding_my_aln is not None%} - - - {% if result.finding_my_aln %} - Y - {% else %} - N - {% endif %} - - - - - {% if result.finding_all_aln %} - Y - {% else %} - N - {% endif %} - - - {% endif %} {% endfor %} diff --git a/backend/dissemination/templates/summary.html b/backend/dissemination/templates/summary.html index a6fffe3c89..15f2330601 100644 --- a/backend/dissemination/templates/summary.html +++ b/backend/dissemination/templates/summary.html @@ -31,7 +31,7 @@
- {% comment %} + @@ -42,7 +42,7 @@

SF-SAC

 - {% endcomment %} + {% if general.is_public or include_private %} ", views.AuditSummaryView.as_view(), name="Summary"), ] diff --git a/backend/dissemination/views.py b/backend/dissemination/views.py index fe65785202..964f177565 100644 --- a/backend/dissemination/views.py +++ b/backend/dissemination/views.py @@ -17,6 +17,7 @@ from dissemination.file_downloads import get_download_url, get_filename from dissemination.forms import SearchForm +from dissemination.forms import AdvancedSearchForm from dissemination.search import search from dissemination.mixins import ReportAccessRequiredMixin from dissemination.models import ( @@ -44,24 +45,27 @@ def _add_search_params_to_newrelic(search_parameters): + is_advanced = search_parameters["advanced_search_flag"] singles = [ "start_date", "end_date", - "cog_or_oversight", "agency_name", "auditee_state", ] - newrelic.agent.add_custom_attributes( - [(f"request.search.{k}", str(search_parameters[k])) for k in singles] - ) - multis = [ "uei_or_eins", - "alns", "names", ] + if is_advanced: + singles.append("cog_or_oversight") + multis.append("alns") + + newrelic.agent.add_custom_attributes( + [(f"request.search.{k}", str(search_parameters[k])) for k in singles] + ) + newrelic.agent.add_custom_attributes( [(f"request.search.{k}", ",".join(search_parameters[k])) for k in multis] ) @@ -91,29 +95,129 @@ def run_search(form_data): Returns the results QuerySet. """ - # As a dictionary, this is easily extensible. - search_parameters = { - "alns": form_data["aln"], + basic_parameters = { "names": form_data["entity_name"], "uei_or_eins": form_data["uei_or_ein"], "start_date": form_data["start_date"], "end_date": form_data["end_date"], - "cog_or_oversight": form_data["cog_or_oversight"], "agency_name": form_data["agency_name"], "audit_years": form_data["audit_year"], - "findings": form_data["findings"], - "direct_funding": form_data["direct_funding"], - "major_program": form_data["major_program"], "auditee_state": form_data["auditee_state"], "order_by": form_data["order_by"], "order_direction": form_data["order_direction"], } + search_parameters = basic_parameters.copy() + + search_parameters["advanced_search_flag"] = form_data["advanced_search_flag"] + if search_parameters["advanced_search_flag"]: + advanced_parameters = { + "alns": form_data["aln"], + "findings": form_data["findings"], + "direct_funding": form_data["direct_funding"], + "major_program": form_data["major_program"], + "cog_or_oversight": form_data["cog_or_oversight"], + } + search_parameters.update(advanced_parameters) _add_search_params_to_newrelic(search_parameters) return search(search_parameters) +class AdvancedSearch(View): + @method_decorator(csrf_exempt) + def dispatch(self, *args, **kwargs): + return super(AdvancedSearch, self).dispatch(*args, **kwargs) + + def get(self, request, *args, **kwargs): + """ + When accessing the search page through get, return the blank search page. + """ + form = AdvancedSearchForm() + + return render( + request, + "advanced.html", + { + "form": form, + "form_user_input": {"audit_year": ["2023"]}, + "state_abbrevs": STATE_ABBREVS, + "summary_report_download_limit": SUMMARY_REPORT_DOWNLOAD_LIMIT, + }, + ) + + @newrelic_timing_metric("search-advanced") + def post(self, request, *args, **kwargs): + """ + When accessing the search page through post, run a search and display the results. + """ + time_starting_post = time.time() + + form = AdvancedSearchForm(request.POST) + results = [] + context = {} + + # form_data = clean_form_data(form) + if form.is_valid(): + form_data = form.cleaned_data + form_user_input = { + k: v[0] if len(v) == 1 else v for k, v in form.data.lists() + } + else: + raise ValidationError(f"Form error in Search POST. {form.errors}") + + # Tells the backend we're running advanced search. + form_data["advanced_search_flag"] = True + + logger.info(f"Advanced searching on fields: {form_data}") + + include_private = include_private_results(request) + + results = run_search(form_data) + + results_count = results.count() + + # Reset page to one if the page number surpasses how many pages there actually are + page = form_data["page"] + ceiling = math.ceil(results_count / form_data["limit"]) + if page > ceiling: + page = 1 + + logger.info(f"TOTAL: results_count: [{results_count}]") + + # The paginator object handles splicing the results to a one-page iterable and calculates which page numbers to show. + paginator = Paginator(object_list=results, per_page=form_data["limit"]) + paginator_results = paginator.get_page(page) + paginator_results.adjusted_elided_pages = paginator.get_elided_page_range( + page, on_each_side=1 + ) + + # Reformat these so the date-picker elements in HTML prepopulate + if form_data["start_date"]: + form_data["start_date"] = form_data["start_date"].strftime("%Y-%m-%d") + if form_data["end_date"]: + form_data["end_date"] = form_data["end_date"].strftime("%Y-%m-%d") + + context = context | { + "form": form, + "form_user_input": form_user_input, + "state_abbrevs": STATE_ABBREVS, + "limit": form_data["limit"], + "results": paginator_results, + "include_private": include_private, + "results_count": results_count, + "page": page, + "order_by": form_data["order_by"], + "order_direction": form_data["order_direction"], + "summary_report_download_limit": SUMMARY_REPORT_DOWNLOAD_LIMIT, + } + time_beginning_render = time.time() + logger.info( + f"Total time between post and render {int(math.ceil((time_beginning_render - time_starting_post) * 1000))}ms" + ) + return render(request, "advanced.html", context) + + class Search(View): @method_decorator(csrf_exempt) def dispatch(self, *args, **kwargs): @@ -156,6 +260,9 @@ def post(self, request, *args, **kwargs): else: raise ValidationError(f"Form error in Search POST. {form.errors}") + # Tells the backend we're running basic search. + form_data["advanced_search_flag"] = False + logger.info(f"Searching on fields: {form_data}") include_private = include_private_results(request) @@ -349,10 +456,6 @@ def get(self, request, report_id): Given a report_id in the URL, generate the summary report in S3 and redirect to its download link. """ - raise Http404( - "SF-SAC downloads are temporarily disabled. See the FAC status page for more details." - ) - sac = get_object_or_404(General, report_id=report_id) include_private = include_private_results(request) filename = generate_summary_report([sac.report_id], include_private) @@ -369,11 +472,12 @@ def post(self, request): 3. Generate a summary report with the report_ids, which goes into into S3 4. Redirect to the download url of this new report """ - form = SearchForm(request.POST) + form = AdvancedSearchForm(request.POST) try: if form.is_valid(): form_data = form.cleaned_data + form_data["advanced_search_flag"] = True else: raise ValidationError("Form error in Search POST.") diff --git a/backend/templates/includes/header.html b/backend/templates/includes/header.html index be44c2ea6c..35db286f2c 100644 --- a/backend/templates/includes/header.html +++ b/backend/templates/includes/header.html @@ -8,9 +8,6 @@
{% endif %} -
-
Audit search is currently limited to UEI-only. For more information, visit the FAC system status page.
-