Search MVP (#2419)

* Add basic search form

* Add dummy results table

* move search to dissem app, impl basic form (#2431)

* Basic styles for search sidebar

* Style up results column

* [DRAFT] Search Summary (#2480)

* Search Summary

* Some title info, make tables take up the full width of the page

* Linting - py whitespace & imports, html closed tag

* Summary view tests (#2512)

* Flesh out Search view (#2494)

* fleshing out search view

* rm leftover list comprehension

* cog-over search fix, only search public

* more tests

* quick pass at displaying search results

* Add explainer text

* Remove filter chicklets (for now)

* Add pagination component

* Rm download all results button

* Add alt text

* Search - Pre-populate Form, Summary Link, Link Icons, Formatting (#2538)

* URLS - Add a backslash to the search url

* Search - Link to summary, icon size, formatting

* Summary - remove "back" button, formatting.

* Search - prepopulate form after making a search

* Remove usused import

* Don't run methods on empty data, kids

* Search - Move UEI and ALN above the name field

* Djlint reformatting

* Search - Audit Year (#2547)

* URLS - Add a backslash to the search url

* Search - Link to summary, icon size, formatting

* Summary - remove "back" button, formatting.

* Search - prepopulate form after making a search

* Remove usused import

* Don't run methods on empty data, kids

* Search - Move UEI and ALN above the name field

* Djlint reformatting

* Search - Add audit year

* search_general params default to none

* Tests - test_audit_year, search_general cleanups

* Revert "Merge branch 'mh/implement-mvp-search-form-2369' into jp/search-audit-year"

This reverts commit 7c99f95, reversing
changes made to 30b2743.

* Remove unused Y/N choices

* PDF downloads via Search (#2520)

* first pass at pdf downloads via search

* re-add AWS_S3_ENDPOINT_URL

* re-add download link

* linter

* fix cog_or_over field name mismatch

* fix search tests

* check if file exists in s3, else 404

* log warning if file not found in S3

---------

Co-authored-by: Matt Henry <[email protected]>
Co-authored-by: Tim Ballard <[email protected]>
Co-authored-by: Tim Ballard <[email protected]>
Co-authored-by: James Person <[email protected]>
Co-authored-by: James Person <[email protected]>

backend/audit/file_downloads.py

@@ -0,0 +1,74 @@
+import logging
+
+from django.conf import settings
+from django.http import Http404
+from django.shortcuts import get_object_or_404
+
+from boto3 import client as boto3_client
+from botocore.client import ClientError, Config
+
+from audit.models import ExcelFile, SingleAuditReportFile
+
+logger = logging.getLogger(__name__)
+
+
+def get_filename(sac, file_type):
+    if file_type == "report":
+        file_obj = get_object_or_404(SingleAuditReportFile, sac=sac)
+        return f"singleauditreport/{file_obj.filename}"
+    else:
+        file_obj = get_object_or_404(ExcelFile, sac=sac, form_section=file_type)
+        return f"excel/{file_obj.filename}"
+
+
+def file_exists(filename):
+    # this client uses the internal endpoint url because we're making a request to S3 from within the app
+    s3_client = boto3_client(
+        service_name="s3",
+        region_name=settings.AWS_S3_PRIVATE_REGION_NAME,
+        aws_access_key_id=settings.AWS_PRIVATE_ACCESS_KEY_ID,
+        aws_secret_access_key=settings.AWS_PRIVATE_SECRET_ACCESS_KEY,
+        endpoint_url=settings.AWS_S3_PRIVATE_ENDPOINT,
+        config=Config(signature_version="s3v4"),
+    )
+
+    try:
+        s3_client.head_object(
+            Bucket=settings.AWS_PRIVATE_STORAGE_BUCKET_NAME,
+            Key=filename,
+        )
+
+        return True
+    except ClientError:
+        logger.warn(f"Unable to locate file {filename} in S3!")
+        return False
+
+
+def get_download_url(filename):
+    try:
+        # this client uses the external endpoint url because we're generating a request URL that is eventually triggered from outside the app
+        s3_client = boto3_client(
+            service_name="s3",
+            region_name=settings.AWS_S3_PRIVATE_REGION_NAME,
+            aws_access_key_id=settings.AWS_PRIVATE_ACCESS_KEY_ID,
+            aws_secret_access_key=settings.AWS_PRIVATE_SECRET_ACCESS_KEY,
+            endpoint_url=settings.AWS_S3_PRIVATE_EXTERNAL_ENDPOINT,
+            config=Config(signature_version="s3v4"),
+        )
+
+        if file_exists(filename):
+            response = s3_client.generate_presigned_url(
+                ClientMethod="get_object",
+                Params={
+                    "Bucket": settings.AWS_PRIVATE_STORAGE_BUCKET_NAME,
+                    "Key": filename,
+                    "ResponseContentDisposition": f"attachment;filename={filename}",
+                },
+                ExpiresIn=30,
+            )
+
+            return response
+        else:
+            raise Http404("File not found")
+    except ClientError:
+        raise Http404("File not found")

backend/config/settings.py

@@ -257,6 +257,7 @@
     )
 
     AWS_S3_ENDPOINT_URL = AWS_S3_PRIVATE_ENDPOINT
+    AWS_S3_PRIVATE_EXTERNAL_ENDPOINT = "http://localhost:9001"
 
     DISABLE_AUTH = env.bool("DISABLE_AUTH", default=False)
 
@@ -308,6 +309,7 @@
 
             AWS_S3_PRIVATE_ENDPOINT = s3_creds["endpoint"]
             AWS_S3_ENDPOINT_URL = f"https://{AWS_S3_PRIVATE_ENDPOINT}"
+            AWS_S3_PRIVATE_EXTERNAL_ENDPOINT = AWS_S3_ENDPOINT_URL
 
             AWS_PRIVATE_LOCATION = "static"
             AWS_PRIVATE_DEFAULT_ACL = "private"

backend/config/urls.py

@@ -79,6 +79,7 @@
         name="sprite",
     ),
     path("audit/", include("audit.urls")),
+    path("dissemination/", include("dissemination.urls")),
     # Keep last so we can use short urls for content pages like home page etc.
     path("", include("cms.urls")),
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

backend/dissemination/forms.py

@@ -0,0 +1,16 @@
+from django import forms
+
+
+class SearchForm(forms.Form):
+    AY_choices = (
+        (x, str(x)) for x in range(2016, 2024)
+    )  # ((2016, "2016"), (2017, "2017"), ..., (2023, "2023"))
+
+    entity_name = forms.CharField(required=False)
+    uei_or_ein = forms.CharField(required=False)
+    aln = forms.CharField(required=False)
+    start_date = forms.DateField(required=False)
+    end_date = forms.DateField(required=False)
+    cog_or_oversight = forms.CharField(required=False)
+    agency_name = forms.CharField(required=False)
+    audit_year = forms.MultipleChoiceField(choices=AY_choices, required=False)

backend/dissemination/search.py

@@ -0,0 +1,51 @@
+from django.db.models import Q
+
+from dissemination.models import General
+
+
+def search_general(
+    names=None,
+    uei_or_eins=None,
+    start_date=None,
+    end_date=None,
+    cog_or_oversight=None,
+    agency_name=None,
+    audit_years=None,
+):
+    query = Q(is_public=True)
+
+    # TODO: use something like auditee_name__contains
+    # SELECT * WHERE auditee_name LIKE '%SomeString%'
+    if names:
+        names_match = Q(Q(auditee_name__in=names) | Q(auditor_firm_name__in=names))
+        query.add(names_match, Q.AND)
+
+    if uei_or_eins:
+        uei_or_ein_match = Q(
+            Q(auditee_uei__in=uei_or_eins) | Q(auditee_ein__in=uei_or_eins)
+        )
+        query.add(uei_or_ein_match, Q.AND)
+
+    if start_date:
+        start_date_match = Q(fac_accepted_date__gte=start_date)
+        query.add(start_date_match, Q.AND)
+
+    if end_date:
+        end_date_match = Q(fac_accepted_date__lte=end_date)
+        query.add(end_date_match, Q.AND)
+
+    if cog_or_oversight:
+        if cog_or_oversight.lower() == "cog":
+            cog_match = Q(cognizant_agency__in=[agency_name])
+            query.add(cog_match, Q.AND)
+        elif cog_or_oversight.lower() == "oversight":
+            oversight_match = Q(oversight_agency__in=[agency_name])
+            query.add(oversight_match, Q.AND)
+
+    if audit_years:
+        fiscal_year_match = Q(audit_year__in=audit_years)
+        query.add(fiscal_year_match, Q.AND)
+
+    results = General.objects.filter(query)
+
+    return results