Merge pull request #3021 from fedspendingtransparency/staging

Sprint 127 Prod Deploy (w/o Warmfixes)

usaspending_api/accounts/v2/filters/account_download.py

@@ -47,7 +47,7 @@
     get_submission_ids_for_periods,
 )
 
-AWARD_URL = f"{HOST}/#/award/" if "localhost" in HOST else f"https://{HOST}/#/award/"
+AWARD_URL = f"{HOST}/award/" if "localhost" in HOST else f"https://{HOST}/award/"
 
 
 def account_download_filter(account_type, download_table, filters, account_level="treasury_account"):

usaspending_api/api_contracts/contracts/v2/download/count.md

@@ -48,7 +48,7 @@ Returns the number of transactions that would be included in a download request
         + `calculated_transaction_count` (required, number)
             The calculated count of all transactions which would be included in the download files.
         + `maximum_transaction_limit` (required, number)
-            The current allowed maximum number of transactions in a row-limited download. Visit https://www.usaspending.gov/#/download_center/custom_award_data to download larger volumes of data.
+            The current allowed maximum number of transactions in a row-limited download. Visit https://www.usaspending.gov/download_center/custom_award_data to download larger volumes of data.
         + `messages` (optional, array[string])
             An array of warnings or instructional directives to aid consumers of this endpoint with development and debugging.
     + Body

usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md

@@ -36,7 +36,7 @@ This endpoint returns an overview list of government agencies submission data.
         + Members
             + `toptier_code`
             + `current_total_budget_authority_amount`
-            + `missing_tas_accounts_total`
+            + `tas_accounts_total`
             + `missing_tas_accounts_count`
             + `agency_name`
             + `obligation_difference`
@@ -82,8 +82,8 @@ This endpoint returns an overview list of government agencies submission data.
                             "missing_tas_accounts_count": 20
                         },
                         "obligation_difference": 436376232652.87,
-                        "unlinked_contract_award_count": 0,
-                        "unlinked_assistance_award_count": 0,
+                        "unlinked_contract_award_count": 3,
+                        "unlinked_assistance_award_count": 2,
                         "assurance_statement_url": "https://files-nonprod.usaspending.gov/agency_submissions/Raw%20DATA%20Act%20Files/2020/P09/075%20-%20Department%20of%20Health%20and%20Human%20Services%20(HHS)/2020-P09-075_Department%20of%20Health%20and%20Human%20Services%20(HHS)-Assurance_Statement.txt"
                     },
                     {

usaspending_api/api_contracts/contracts/v2/reporting/agencies/toptier_code/fiscal_year/fiscal_period/unlinked_awards/type.md

@@ -0,0 +1,37 @@
+FORMAT: 1A
+HOST: https://api.usaspending.gov
+
+# Agencies' Unlinked Awards [/api/v2/reporting/agencies/{toptier_code}/{fiscal_year}/{fiscal_period}/unlinked_awards/{type}/]
+
+This endpoint is used to power USAspending.gov's About the Data \| Agencies unlinked data modals.
+
+## GET
+
+This endpoint returns the number of unlinked and linked awards for the agency in the provided fiscal year and period.
+
++ Parameters
+    + `toptier_code`: `020` (required, string)
+        The specific agency code.
+    + `fiscal_year`: 2020 (required, number)
+        The fiscal year of the submission
+    + `fiscal_period`: 10 (required, number)
+        The fiscal period of the submission. valid values: 2-12 (2 = November ... 12 = September)
+        For retrieving quarterly submissions, provide the period which equals 'quarter * 3' (e.g. Q2 = P6)
+    + `type`: `assistance` (required, enum[string])
+        + Members
+            + `assistance`
+            + `procurement`
+
++ Response 200 (application/json)
+
+    + Attributes (object)
+        + `unlinked_file_c_award_count` (required, number)
+        + `unlinked_file_d_award_count` (required, number)
+        + `total_linked_award_count` (required, number)
+    + Body
+
+            {
+                "unlinked_file_c_award_count": 123213,
+                "unlinked_file_d_award_count": 43543,
+                "total_linked_award_count": 12321312
+            }

usaspending_api/api_contracts/contracts/v2/reporting/agencies/toptier_code/overview.md

@@ -32,7 +32,7 @@ This endpoint returns an overview of government agency submission data.
             + `fiscal_period`
             + `fiscal_year`
             + `missing_tas_accounts_count`
-            + `missing_tas_accounts_total`
+            + `tas_accounts_total`
             + `obligation_difference`
             + `percent_of_total_budgetary_resources`
             + `recent_publication_date`
@@ -78,8 +78,8 @@ This endpoint returns an overview of government agency submission data.
                             "missing_tas_accounts_count": 9
                         },
                         "obligation_difference": 12581114.45,
-                        "unlinked_contract_award_count": 0,
-                        "unlinked_assistance_award_count": 0,
+                        "unlinked_contract_award_count": 2,
+                        "unlinked_assistance_award_count": 5,
                         "assurance_statement_url": "https://files.usaspending.gov/agency_submissions/Raw%20DATA%20Act%20Files/2020/P07/020%20-%20Department%20of%20the%20Treasury%20(TREAS)/2020-P07-020_Department%20of%20the%20Treasury%20(TREAS)-Assurance_Statement.txt"
                     },
                     {

usaspending_api/api_contracts/contracts/v2/subawards.md

@@ -47,15 +47,37 @@ This endpoint returns a filtered set of subawards.
         + `results` (required, array[SubawardResponse], fixed-type)
         + `page_metadata` (required, PageMetadataObject)
 
+    + Body
+
+            {
+                "page_metadata": {
+                    "page": 1,
+                    "next": 2,
+                    "previous": null,
+                    "hasNext": true,
+                    "hasPrevious": false
+                },
+                "results": [
+                    {
+                        "id": 119270129,
+                        "subaward_number": "Z981002",
+                        "description": "DEVELOPMENT OF A SELF-SUSTAINED WIRELESS INTEGRATED STRUCTURAL HEALTH MONITORING SYSTEM FOR HIGHWAY BRIDGES",
+                        "action_date": "2011-10-27",
+                        "amount": 110000.0,
+                        "recipient_name": "URS GROUP, INC."
+                    }
+                ]
+            }
+
 # Data Structures
 
 ## SubawardResponse (object)
-+ `subaward_number` (required, string)
-+ `amount` (required, number)
 + `id` (required, number)
++ `subaward_number` (required, string)
++ `description` (required, string)
 + `action_date` (required, string)
++ `amount` (required, number)
 + `recipient_name` (required, string)
-+ `description` (required, string)
 
 ## PageMetadataObject (object)
 + `page` (required, number)

usaspending_api/api_docs/management/commands/generate_model_markdown.py

@@ -7,7 +7,7 @@ class Command(BaseCommand):
     help = "Generates a markdown file of a model's fields and help text \
             for use in documentation \
             Usage: `python manage.py generate_model_markdown <MODEL>`"
-    logger = logging.getLogger("console")
+    logger = logging.getLogger("script")
 
     friendly_names = {
         "ForeignKey": "Relation",

usaspending_api/api_docs/markdown/endpoints.md

@@ -147,6 +147,7 @@ The currently available endpoints are listed in the following table.
 |[/api/v2/reporting/agencies/overview/](/api/v2/reporting/agencies/overview/)|GET| Returns About the Data information about all agencies with submissions in a provided fiscal year and period|
 |[/api/v2/reporting/agencies/publish_dates/](/api/v2/reporting/agencies/publish_dates/)|GET| Returns submission publication and certification information about all agencies with submissions in a provided fiscal year and period|
 |[/api/v2/reporting/agencies/<TOPTIER_CODE\>/<FISCAL_YEAR\>/<FISCAL_PERIOD\>/submission_history/](/api/v2/reporting/agencies/020/2020/12/submission_history/)|GET| Returns a list of submission publication dates and certified dates for the provided agency for the provided fiscal year and period. |
+|[/api/v2/reporting/agencies/<TOPTIER_CODE\>/<FISCAL_YEAR\>/<FISCAL_PERIOD\>/unlinked_awards/<TYPE\>/](/api/v2/reporting/agencies/020/2020/12/unlinked_awards/procurement/)|GET| Returns counts of an agency's linked and unlinked awards for a given period. |
 |[/api/v2/search/new_awards_over_time/](/api/v2/search/new_awards_over_time/)|POST| Returns a list of time periods with the new awards in the appropriate period within the provided time range |
 |[/api/v2/search/spending_by_award/](/api/v2/search/spending_by_award/)|POST| Returns the fields of the filtered awards |
 |[/api/v2/search/spending_by_award_count/](/api/v2/search/spending_by_award_count/)|POST| Returns the number of awards in each award type (Contracts, IDV, Loans, Direct Payments, Grants, and Other) |

usaspending_api/api_docs/unused_markdown/request_recipes.md

@@ -96,4 +96,4 @@ POST
 
 # Postman Collections <a name="postman"></a>
 
-[Postman](https://www.getpostman.com/) is a free app for making easy API requests. You can also use it to import and inspect a collection of pre-generated API requests. [Here is a postman collection](https://raw.githubusercontent.com/fedspendingtransparency/usaspending-api/master/usaspending_api/static_doc_files/docs/usaspending_searchpage_postmancollection.json) you can use to see how we generate the visualizations on [the search page](https://www.usaspending.gov/#/search/).
+[Postman](https://www.getpostman.com/) is a free app for making easy API requests. You can also use it to import and inspect a collection of pre-generated API requests. [Here is a postman collection](https://raw.githubusercontent.com/fedspendingtransparency/usaspending-api/master/usaspending_api/static_doc_files/docs/usaspending_searchpage_postmancollection.json) you can use to see how we generate the visualizations on [the search page](https://www.usaspending.gov/search/).

usaspending_api/api_docs/unused_markdown/using_the_api.md

@@ -365,11 +365,11 @@ The endpoints described in this section generate files that reflect the site's u
 
 #### Award Data Archive
 
-On a monthly basis, the website pre-generates a series of commonly used files based on the agency, fiscal year, and award type. You can find these on the [Award Data Archive](https://www.usaspending.gov/#/download_center/award_data_archive) page. You can also access this information via the API's [List Downloads Endpoint](https://api.usaspending.gov/api/v2/bulk_download/list_monthly_files/).
+On a monthly basis, the website pre-generates a series of commonly used files based on the agency, fiscal year, and award type. You can find these on the [Award Data Archive](https://www.usaspending.gov/download_center/award_data_archive) page. You can also access this information via the API's [List Downloads Endpoint](https://api.usaspending.gov/api/v2/bulk_download/list_monthly_files/).
 
 #### Generating Download Files
 
-**Reminder**: Before using these endpoints, check the  [Award Data Archive](https://usaspending.gov/#/download_center/award_data_archive) for pre-generated files
+**Reminder**: Before using these endpoints, check the  [Award Data Archive](https://usaspending.gov/download_center/award_data_archive) for pre-generated files
 
 There are several downloadable endpoints, all with different features/constraints.
 

usaspending_api/awards/management/commands/fix_missing_agencies.py

@@ -10,7 +10,7 @@
 from django.core.management.base import BaseCommand
 from django.db import connection
 
-logger = logging.getLogger("console")
+logger = logging.getLogger("script")
 
 BATCH_SIZE = 10000
 

usaspending_api/awards/management/commands/load_subawards.py

@@ -9,7 +9,7 @@
 from usaspending_api.etl.operations.subaward.update_city_county import update_subaward_city_county
 
 
-logger = logging.getLogger("console")
+logger = logging.getLogger("script")
 
 
 class Command(mixins.ETLMixin, BaseCommand):

usaspending_api/awards/management/commands/restock_parent_award.py

@@ -8,7 +8,7 @@
 class Command(BaseCommand):
 
     help = "Empty and repopulate parent_award table with IDV aggregates and counts"
-    logger = logging.getLogger("console")
+    logger = logging.getLogger("script")
 
     def add_arguments(self, parser):
 

usaspending_api/awards/management/commands/update_transaction_fiscal_year.py

@@ -6,7 +6,7 @@
 class Command(BaseCommand):
     help = "Updates the fiscal year for all transactions based on their individual action dates"
 
-    logger = logging.getLogger("console")
+    logger = logging.getLogger("script")
 
     def handle(self, *args, **options):
         all_transactions = TransactionNormalized.objects.all()

usaspending_api/awards/tests/integration/test_subaward_endpoint.py

@@ -8,22 +8,22 @@
     subaward_1,
     subaward_2,
     subaward_3,
+    subaward_10,
+    subaward_11,
     subaward_12,
 )
 
 
 @pytest.mark.django_db
-def test_subaward_success(client):
-
-    resp = client.post(
-        "/api/v2/subawards/", content_type="application/json", data=json.dumps({"order": "desc", "limit": 100})
-    )
+def test_subaward_no_params(client):
+    create_subaward_test_data(subaward_1, subaward_2, subaward_3)
+    resp = client.post("/api/v2/subawards/", content_type="application/json")
     assert resp.status_code == status.HTTP_200_OK
+    assert len(json.loads(resp.content.decode("utf-8"))["results"]) == 3
 
 
 @pytest.mark.django_db
 def test_subaward_failure(client):
-
     resp = client.post(
         "/api/v2/subawards/",
         content_type="application/json",
@@ -33,33 +33,54 @@ def test_subaward_failure(client):
 
 
 @pytest.mark.django_db
-def test_subaward_query_1(client):
+def test_subaward_limit(client):
     create_subaward_test_data(subaward_1, subaward_2, subaward_3)
     resp = client.post(
         "/api/v2/subawards/",
         content_type="application/json",
-        data=json.dumps({"order": "desc", "limit": 100, "award_id": 99}),
+        data=json.dumps({"limit": 2}),
     )
-    assert len(json.loads(resp.content.decode("utf-8"))["results"]) == 3
+    assert len(json.loads(resp.content.decode("utf-8"))["results"]) == 2
 
 
 @pytest.mark.django_db
-def test_subaward_query_2(client):
-    create_subaward_test_data(subaward_12)
+def test_subaward_filters(client):
+    create_subaward_test_data(subaward_1, subaward_2, subaward_3, subaward_12, subaward_11)
+    resp = client.post(
+        "/api/v2/subawards/",
+        content_type="application/json",
+        data=json.dumps({"award_id": 99}),
+    )
+    assert len(json.loads(resp.content.decode("utf-8"))["results"]) == 4
+
+    resp = client.post(
+        "/api/v2/subawards/",
+        content_type="application/json",
+        data=json.dumps({"award_id": 88}),
+    )
+    results = json.loads(resp.content.decode("utf-8"))["results"]
+    assert len(results) == 1
+    assert results[0]["id"] == 12
+
     resp = client.post(
         "/api/v2/subawards/",
         content_type="application/json",
-        data=json.dumps({"order": "desc", "limit": 100, "award_id": 88}),
+        data=json.dumps({"award_id": "generated_unique_award_id_for_88"}),
     )
-    assert json.loads(resp.content.decode("utf-8"))["results"][0]["id"] == 12
+    results = json.loads(resp.content.decode("utf-8"))["results"]
+    assert len(results) == 1
+    assert results[0]["id"] == 12
 
 
 @pytest.mark.django_db
-def test_subaward_query_3(client):
-    create_subaward_test_data(subaward_12)
+def test_subaward_sorting(client):
+    create_subaward_test_data(subaward_1, subaward_12, subaward_10, subaward_2, subaward_3)
     resp = client.post(
         "/api/v2/subawards/",
         content_type="application/json",
-        data=json.dumps({"order": "desc", "limit": 100, "award_id": "generated_unique_award_id_for_88"}),
+        data=json.dumps({"sort": "description", "order": "asc"}),
     )
-    assert json.loads(resp.content.decode("utf-8"))["results"][0]["id"] == 12
+    results = json.loads(resp.content.decode("utf-8"))["results"]
+    assert len(results) == 5
+    assert results[0]["id"] == 1
+    assert results[4]["id"] == 12

usaspending_api/awards/v2/views/subawards.py

@@ -66,10 +66,15 @@ def _business_logic(self, request_data):
 
         queryset = queryset.values(*list(self.subaward_lookup.values()))
 
+        # always secondary-sort by PK in case a repeating value (e.g. subaward_number) crosses pages, so suborder isn't abitrary
         if request_data["order"] == "desc":
-            queryset = queryset.order_by(F(self.subaward_lookup[request_data["sort"]]).desc(nulls_last=True))
+            queryset = queryset.order_by(
+                F(self.subaward_lookup[request_data["sort"]]).desc(nulls_last=True), F("subaward_id").desc()
+            )
         else:
-            queryset = queryset.order_by(F(self.subaward_lookup[request_data["sort"]]).asc(nulls_first=True))
+            queryset = queryset.order_by(
+                F(self.subaward_lookup[request_data["sort"]]).asc(nulls_first=True), F("subaward_id").asc()
+            )
 
         rows = list(queryset[lower_limit : upper_limit + 1])
         return [{k: row[v] for k, v in self.subaward_lookup.items()} for row in rows]
@@ -79,7 +84,5 @@ def post(self, request):
         request_data = self._parse_and_validate_request(request.data)
         results = self._business_logic(request_data)
         page_metadata = get_simple_pagination_metadata(len(results), request_data["limit"], request_data["page"])
-
         response = {"page_metadata": page_metadata, "results": results[: request_data["limit"]]}
-
         return Response(response)

usaspending_api/broker/helpers/delete_fabs_transactions.py

@@ -5,7 +5,7 @@
 from usaspending_api.common.helpers.timing_helpers import timer
 
 
-logger = logging.getLogger("console")
+logger = logging.getLogger("script")
 
 
 def delete_fabs_transactions(ids_to_delete):

usaspending_api/broker/helpers/delete_stale_fabs.py

@@ -7,7 +7,7 @@
 from usaspending_api.broker.helpers.find_related_awards import find_related_awards
 
 
-logger = logging.getLogger("console")
+logger = logging.getLogger("script")
 
 
 @transaction.atomic

usaspending_api/broker/helpers/last_load_date.py

@@ -1,9 +1,13 @@
+import logging
+
 from datetime import timedelta
 
 from usaspending_api.broker import lookups
 from usaspending_api.broker.models import ExternalDataLoadDate
 from usaspending_api.common.helpers.date_helper import cast_datetime_to_utc
 
+logger = logging.getLogger("script")
+
 
 def get_last_load_date(key, lookback_minutes=None, default=None):
     """
@@ -25,7 +29,10 @@ def get_last_load_date(key, lookback_minutes=None, default=None):
         .first()
     )
     if last_load_date is None:
+        logger.warning(f"No record of a previous run for `{key}` was found!")
         return default
+    else:
+        logger.info(f"Value for previous `{key}` ETL: {last_load_date}")
     if lookback_minutes is not None:
         last_load_date -= timedelta(minutes=lookback_minutes)
     return last_load_date