Merge pull request #3043 from fedspendingtransparency/staging

Sprint 128 Production Deploy

Dockerfile

@@ -40,4 +40,5 @@ RUN python3 -m pip install -r requirements/requirements.txt
 ##### Copy the rest of the project files into the container
 COPY . /dockermount
 
-ENV PYTHONUNBUFFERED=0
+##### Ensure Python STDOUT gets sent to container logs
+ENV PYTHONUNBUFFERED=1

requirements/requirements-app.txt

@@ -4,7 +4,7 @@ attrs==20.*
 boto3==1.16.*
 certifi==2020.12.5
 dataclasses-json==0.5.*
-ddtrace==0.37.1
+ddtrace==0.46.0
 dj-database-url==0.5.0
 django-cors-headers==2.5.3
 django-debug-toolbar==3.2

usaspending_api/accounts/v2/filters/account_download.py

@@ -129,21 +129,22 @@ def get_submission_filter(account_type, filters):
         )
 
         submission_filter = Q(
-            outlay_filter & (Q(gross_outlay_amount_by_award_cpe__gt=0) | Q(gross_outlay_amount_by_award_cpe__lt=0))
+            outlay_filter
+            & Q(
+                Q(gross_outlay_amount_by_award_cpe__gt=0)
+                | Q(gross_outlay_amount_by_award_cpe__lt=0)
+                | Q(ussgl487200_down_adj_pri_ppaid_undel_orders_oblig_refund_cpe__gt=0)
+                | Q(ussgl487200_down_adj_pri_ppaid_undel_orders_oblig_refund_cpe__lt=0)
+                | Q(ussgl497200_down_adj_pri_paid_deliv_orders_oblig_refund_cpe__gt=0)
+                | Q(ussgl497200_down_adj_pri_paid_deliv_orders_oblig_refund_cpe__lt=0)
+            )
         ) | Q(obligation_filter & Q(Q(transaction_obligated_amount__gt=0) | Q(transaction_obligated_amount__lt=0)))
 
     return submission_filter
 
 
-def generate_gross_outlay_amount_derived_field(filters, account_type):
-    column_name = {
-        "account_balances": "gross_outlay_amount_by_tas_cpe",
-        "object_class_program_activity": "gross_outlay_amount_by_program_object_class_cpe",
-        "award_financial": "gross_outlay_amount_by_award_cpe",
-    }[account_type]
-
+def _generate_closed_period_for_derived_field(filters, column_name):
     filter_year = filters.get("fy")
-
     closed_periods = get_last_closed_periods_per_year()
 
     q = Q()
@@ -163,6 +164,26 @@ def generate_gross_outlay_amount_derived_field(filters, account_type):
     return Cast(Value(None), DecimalField(max_digits=23, decimal_places=2))
 
 
+def generate_ussgl487200_derived_field(filters):
+    column_name = "ussgl487200_down_adj_pri_ppaid_undel_orders_oblig_refund_cpe"
+    return _generate_closed_period_for_derived_field(filters, column_name)
+
+
+def generate_ussgl497200_derived_field(filters):
+    column_name = "ussgl497200_down_adj_pri_paid_deliv_orders_oblig_refund_cpe"
+    return _generate_closed_period_for_derived_field(filters, column_name)
+
+
+def generate_gross_outlay_amount_derived_field(filters, account_type):
+    column_name = {
+        "account_balances": "gross_outlay_amount_by_tas_cpe",
+        "object_class_program_activity": "gross_outlay_amount_by_program_object_class_cpe",
+        "award_financial": "gross_outlay_amount_by_award_cpe",
+    }[account_type]
+
+    return _generate_closed_period_for_derived_field(filters, column_name)
+
+
 def generate_treasury_account_query(queryset, account_type, tas_id, filters):
     """ Derive necessary fields for a treasury account-grouped query """
     derived_fields = {
@@ -176,6 +197,19 @@ def generate_treasury_account_query(queryset, account_type, tas_id, filters):
     }
 
     lmd = "last_modified_date" + NAMING_CONFLICT_DISCRIMINATOR
+
+    if account_type != "account_balances":
+        derived_fields.update(
+            {
+                "downward_adj_prior_yr_ppaid_undeliv_orders_oblig_refunds_cpe": Sum(
+                    generate_ussgl487200_derived_field(filters)
+                ),
+                "downward_adj_prior_yr_paid_delivered_orders_oblig_refunds_cpe": Sum(
+                    generate_ussgl497200_derived_field(filters)
+                ),
+            }
+        )
+
     if account_type == "award_financial":
         # Separating out last_modified_date like this prevents unnecessary grouping in the full File
         # C TAS download.  Keeping it as MAX caused grouping on every single column in the SQL statement.
@@ -201,6 +235,17 @@ def generate_federal_account_query(queryset, account_type, tas_id, filters):
         "gross_outlay_amount_fyb_to_period_end": Sum(generate_gross_outlay_amount_derived_field(filters, account_type)),
     }
 
+    if account_type != "account_balances":
+        derived_fields.update(
+            {
+                "downward_adj_prior_yr_ppaid_undeliv_orders_oblig_refunds_cpe": Sum(
+                    generate_ussgl487200_derived_field(filters)
+                ),
+                "downward_adj_prior_yr_paid_delivered_orders_oblig_refunds_cpe": Sum(
+                    generate_ussgl497200_derived_field(filters)
+                ),
+            }
+        )
     if account_type == "award_financial":
         derived_fields = award_financial_derivations(derived_fields)
 

usaspending_api/agency/tests/integration/conftest.py

@@ -6,17 +6,53 @@
 
 @pytest.fixture
 def agency_account_data():
+    dabs = mommy.make("submissions.DABSSubmissionWindowSchedule", submission_reveal_date="2020-10-09")
+
     ta1 = mommy.make("references.ToptierAgency", toptier_code="007")
     ta2 = mommy.make("references.ToptierAgency", toptier_code="008")
     ta3 = mommy.make("references.ToptierAgency", toptier_code="009")
     ta4 = mommy.make("references.ToptierAgency", toptier_code="010")
+
+    mommy.make("references.Agency", id=1, toptier_flag=True, toptier_agency=ta1)
+    mommy.make("references.Agency", id=2, toptier_flag=True, toptier_agency=ta2)
+    mommy.make("references.Agency", id=3, toptier_flag=True, toptier_agency=ta3)
+    mommy.make("references.Agency", id=4, toptier_flag=True, toptier_agency=ta4)
+
     sub1 = mommy.make(
-        "submissions.SubmissionAttributes", reporting_fiscal_year=current_fiscal_year(), is_final_balances_for_fy=True
+        "submissions.SubmissionAttributes",
+        reporting_fiscal_year=current_fiscal_year(),
+        toptier_code=ta1.toptier_code,
+        is_final_balances_for_fy=True,
+        submission_window_id=dabs.id,
+    )
+    sub2 = mommy.make(
+        "submissions.SubmissionAttributes",
+        reporting_fiscal_year=2017,
+        toptier_code=ta2.toptier_code,
+        is_final_balances_for_fy=True,
+        submission_window_id=dabs.id,
+    )
+    sub3 = mommy.make(
+        "submissions.SubmissionAttributes",
+        reporting_fiscal_year=2018,
+        toptier_code=ta3.toptier_code,
+        is_final_balances_for_fy=True,
+        submission_window_id=dabs.id,
+    )
+    sub4 = mommy.make(
+        "submissions.SubmissionAttributes",
+        reporting_fiscal_year=2019,
+        toptier_code=ta4.toptier_code,
+        is_final_balances_for_fy=True,
+        submission_window_id=dabs.id,
+    )
+    sub5 = mommy.make(
+        "submissions.SubmissionAttributes",
+        reporting_fiscal_year=2016,
+        toptier_code=ta1.toptier_code,
+        is_final_balances_for_fy=True,
+        submission_window_id=dabs.id,
     )
-    sub2 = mommy.make("submissions.SubmissionAttributes", reporting_fiscal_year=2017, is_final_balances_for_fy=True)
-    sub3 = mommy.make("submissions.SubmissionAttributes", reporting_fiscal_year=2018, is_final_balances_for_fy=True)
-    sub4 = mommy.make("submissions.SubmissionAttributes", reporting_fiscal_year=2019, is_final_balances_for_fy=True)
-    sub5 = mommy.make("submissions.SubmissionAttributes", reporting_fiscal_year=2016, is_final_balances_for_fy=True)
     fa1 = mommy.make("accounts.FederalAccount", federal_account_code="001-0000", account_title="FA 1")
     fa2 = mommy.make("accounts.FederalAccount", federal_account_code="002-0000", account_title="FA 2")
     fa3 = mommy.make("accounts.FederalAccount", federal_account_code="003-0000", account_title="FA 3")

usaspending_api/agency/tests/integration/test_agency.py

@@ -34,6 +34,9 @@ def agency_data():
     mommy.make("accounts.AppropriationAccountBalances", treasury_account_identifier=tas1)
     mommy.make("accounts.AppropriationAccountBalances", treasury_account_identifier=tas2)
     mommy.make("awards.TransactionNormalized", awarding_agency=a1, fiscal_year=current_fiscal_year())
+    dabs = mommy.make("submissions.DABSSubmissionWindowSchedule", submission_reveal_date="2020-10-09")
+    mommy.make("submissions.SubmissionAttributes", toptier_code=ta1.toptier_code, submission_window_id=dabs.id)
+    mommy.make("submissions.SubmissionAttributes", toptier_code=ta2.toptier_code, submission_window_id=dabs.id)
 
 
 @pytest.mark.django_db

usaspending_api/agency/tests/integration/test_agency_budgetary_resources.py

@@ -13,15 +13,48 @@
 
 @pytest.fixture
 def data_fixture():
+    dabs = mommy.make("submissions.DABSSubmissionWindowSchedule", submission_reveal_date="2020-10-09")
     ta1 = mommy.make("references.ToptierAgency", toptier_code="001")
     ta2 = mommy.make("references.ToptierAgency", toptier_code="002")
+    mommy.make("references.Agency", toptier_flag=True, toptier_agency=ta1)
+    mommy.make("references.Agency", toptier_flag=True, toptier_agency=ta2)
     tas1 = mommy.make("accounts.TreasuryAppropriationAccount", funding_toptier_agency=ta1)
     tas2 = mommy.make("accounts.TreasuryAppropriationAccount", funding_toptier_agency=ta2)
-    sa1_3 = mommy.make("submissions.SubmissionAttributes", reporting_fiscal_year=FY, reporting_fiscal_period=3)
-    sa1_6 = mommy.make("submissions.SubmissionAttributes", reporting_fiscal_year=FY, reporting_fiscal_period=6)
-    sa1_9 = mommy.make("submissions.SubmissionAttributes", reporting_fiscal_year=FY, reporting_fiscal_period=9)
-    sa1_12 = mommy.make("submissions.SubmissionAttributes", reporting_fiscal_year=FY, reporting_fiscal_period=12)
-    sa2_12 = mommy.make("submissions.SubmissionAttributes", reporting_fiscal_year=PRIOR_FY, reporting_fiscal_period=12)
+    sa1_3 = mommy.make(
+        "submissions.SubmissionAttributes",
+        reporting_fiscal_year=FY,
+        reporting_fiscal_period=3,
+        submission_window_id=dabs.id,
+        toptier_code=ta1.toptier_code,
+    )
+    sa1_6 = mommy.make(
+        "submissions.SubmissionAttributes",
+        reporting_fiscal_year=FY,
+        reporting_fiscal_period=6,
+        submission_window_id=dabs.id,
+        toptier_code=ta1.toptier_code,
+    )
+    sa1_9 = mommy.make(
+        "submissions.SubmissionAttributes",
+        reporting_fiscal_year=FY,
+        reporting_fiscal_period=9,
+        submission_window_id=dabs.id,
+        toptier_code=ta1.toptier_code,
+    )
+    sa1_12 = mommy.make(
+        "submissions.SubmissionAttributes",
+        reporting_fiscal_year=FY,
+        reporting_fiscal_period=12,
+        submission_window_id=dabs.id,
+        toptier_code=ta1.toptier_code,
+    )
+    sa2_12 = mommy.make(
+        "submissions.SubmissionAttributes",
+        reporting_fiscal_year=PRIOR_FY,
+        reporting_fiscal_period=12,
+        submission_window_id=dabs.id,
+        toptier_code=ta2.toptier_code,
+    )
 
     mommy.make(
         "accounts.AppropriationAccountBalances",

usaspending_api/agency/v2/views/agency_base.py

@@ -64,7 +64,9 @@ def fiscal_period(self):
 
     @cached_property
     def toptier_agency(self):
-        toptier_agency = ToptierAgency.objects.account_agencies().filter(toptier_code=self.toptier_code).first()
+        toptier_agency = ToptierAgency.objects.filter(
+            toptieragencypublisheddabsview__toptier_code=self.toptier_code
+        ).first()
         if not toptier_agency:
             raise NotFound(f"Agency with a toptier code of '{self.toptier_code}' does not exist")
         return toptier_agency

usaspending_api/awards/migrations/0079_auto_20210315_1946.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.2.9 on 2021-03-15 19:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('awards', '0078_auto_20210211_2237'),
+    ]
+
+    operations = [
+        migrations.RemoveIndex(
+            model_name='financialaccountsbyawards',
+            name='faba_subid_awardkey_sums_idx',
+        ),
+        migrations.AddIndex(
+            model_name='financialaccountsbyawards',
+            index=models.Index(condition=models.Q(disaster_emergency_fund__in=['L', 'M', 'N', 'O', 'P', 'U']), fields=['submission', 'distinct_award_key', 'piid', 'transaction_obligated_amount', 'gross_outlay_amount_by_award_cpe', 'ussgl497200_down_adj_pri_paid_deliv_orders_oblig_refund_cpe', 'ussgl487200_down_adj_pri_ppaid_undel_orders_oblig_refund_cpe'], name='faba_subid_awardkey_sums_idx'),
+        ),
+    ]

usaspending_api/awards/models/financial_accounts_by_awards.py

@@ -157,6 +157,8 @@ class Meta:
                     "piid",
                     "transaction_obligated_amount",
                     "gross_outlay_amount_by_award_cpe",
+                    "ussgl497200_down_adj_pri_paid_deliv_orders_oblig_refund_cpe",
+                    "ussgl487200_down_adj_pri_ppaid_undel_orders_oblig_refund_cpe",
                 ],
                 name="faba_subid_awardkey_sums_idx",
                 condition=Q(disaster_emergency_fund__in=["L", "M", "N", "O", "P", "U"]),

usaspending_api/awards/tests/test_awards_v2.py

@@ -1218,6 +1218,44 @@ def test_file_c_data(client, awards_and_transactions):
     assert json.loads(resp.content.decode("utf-8"))["total_account_outlay"] == 110.0
 
 
+def test_outlay_calculations(client, awards_and_transactions):
+    defc = mommy.make("references.DisasterEmergencyFundCode", code="L", group_name="covid_19")
+    mommy.make(
+        "submissions.DABSSubmissionWindowSchedule",
+        submission_fiscal_year=2019,
+        submission_fiscal_month=12,
+        is_quarter=True,
+        submission_reveal_date="2020-04-01",
+        period_start_date="2020-04-01",
+    )
+    mommy.make(
+        "submissions.SubmissionAttributes",
+        pk=4,
+        reporting_fiscal_period=12,
+        reporting_fiscal_year=2019,
+        reporting_period_end="2020-06-30",
+        quarter_format_flag=True,
+        is_final_balances_for_fy=True,
+        reporting_period_start="2020-04-01",
+    )
+    mommy.make(
+        "awards.FinancialAccountsByAwards",
+        award_id=1,
+        transaction_obligated_amount=10,
+        gross_outlay_amount_by_award_cpe=10,
+        ussgl487200_down_adj_pri_ppaid_undel_orders_oblig_refund_cpe=-1,
+        ussgl497200_down_adj_pri_paid_deliv_orders_oblig_refund_cpe=-2,
+        disaster_emergency_fund=defc,
+        submission_id=4,
+    )
+    resp = client.get("/api/v2/awards/1/")
+    assert resp.status_code == status.HTTP_200_OK
+    assert json.loads(resp.content.decode("utf-8"))["account_obligations_by_defc"] == [{"code": "L", "amount": 10.0}]
+    assert json.loads(resp.content.decode("utf-8"))["account_outlays_by_defc"] == [{"code": "L", "amount": 7.0}]
+    assert json.loads(resp.content.decode("utf-8"))["total_account_obligation"] == 10.0
+    assert json.loads(resp.content.decode("utf-8"))["total_account_outlay"] == 7.0
+
+
 expected_response_asst = {
     "id": 1,
     "record_type": 111,

usaspending_api/awards/v2/data_layer/sql.py

@@ -1,7 +1,9 @@
 defc_sql = """
     SELECT
         DISTINCT disaster_emergency_fund_code,
-        COALESCE(sum(CASE WHEN sa.is_final_balances_for_fy = TRUE THEN faba.gross_outlay_amount_by_award_cpe END), 0) AS total_outlay,
+        COALESCE(sum(CASE WHEN sa.is_final_balances_for_fy = TRUE THEN (COALESCE(faba.gross_outlay_amount_by_award_cpe,0)
+            + COALESCE(faba.ussgl487200_down_adj_pri_ppaid_undel_orders_oblig_refund_cpe, 0)
+            + COALESCE(faba.ussgl497200_down_adj_pri_paid_deliv_orders_oblig_refund_cpe, 0)) END), 0) AS total_outlay,
         COALESCE(sum(faba.transaction_obligated_amount), 0) AS obligated_amount
     FROM
         financial_accounts_by_awards faba

usaspending_api/common/helpers/s3_helpers.py

@@ -3,6 +3,7 @@
 import logging
 import math
 
+from boto3.s3.transfer import TransferConfig, S3Transfer
 from django.conf import settings
 from pathlib import Path
 from typing import List
@@ -52,6 +53,6 @@ def multipart_upload(bucketname, regionname, source_path, keyname):
     source_size = Path(source_path).stat().st_size
     # Sets the chunksize at minimum ~5MB to sqrt(5MB) * sqrt(source size)
     bytes_per_chunk = max(int(math.sqrt(5242880) * math.sqrt(source_size)), 5242880)
-    config = boto3.s3.transfer.TransferConfig(multipart_chunksize=bytes_per_chunk)
-    transfer = boto3.s3.transfer.S3Transfer(s3client, config)
+    config = TransferConfig(multipart_chunksize=bytes_per_chunk)
+    transfer = S3Transfer(s3client, config)
     transfer.upload_file(source_path, bucketname, Path(keyname).name, extra_args={"ACL": "bucket-owner-full-control"})

usaspending_api/common/query_with_filters.py

@@ -469,23 +469,6 @@ def generate_elasticsearch_query(
         return ES_Q("bool", should=non_zero_queries, minimum_should_match=1)
 
 
-class _NonzeroNestedSumFields(_Filter):
-    """Query for when the sum of nested fields should not be zero"""
-
-    underscore_name = "nonzero_sum_fields"
-
-    @classmethod
-    def generate_elasticsearch_query(
-        cls, filter_values: List[str], query_type: _QueryType, nested_path: str = ""
-    ) -> ES_Q:
-        non_zero_queries = []
-        for field in filter_values:
-            field_name = f"{nested_path}{'.' if nested_path else ''}{field}"
-            non_zero_queries.append(ES_Q("range", **{field_name: {"gt": 0}}))
-            non_zero_queries.append(ES_Q("range", **{field_name: {"lt": 0}}))
-        return ES_Q("bool", should=non_zero_queries, minimum_should_match=1)
-
-
 class QueryWithFilters:
 
     filter_lookup = {