merge: from main

backend/data_tools/Pipfile

@@ -21,7 +21,7 @@ azure-identity = "==1.19.0"
 azure-keyvault-secrets = "==4.9.0"
 loguru = "==0.7.2"
 click = "==8.1.7"
-aiohttp = "==3.11.2"
+aiohttp = "==3.11.4"
 
 [dev-packages]
 nox = "==2024.10.9"

backend/data_tools/src/load_cans/utils.py

@@ -1,12 +1,12 @@
 from csv import DictReader
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import List
 
 from loguru import logger
 from sqlalchemy import and_, select
 from sqlalchemy.orm import Session
 
-from models import CAN, BaseModel, CANFundingDetails, CANFundingSource, CANMethodOfTransfer, Portfolio, User
+from models import CAN, CANFundingDetails, CANFundingSource, CANMethodOfTransfer, Portfolio, User
 
 
 @dataclass
@@ -120,52 +120,96 @@ def create_models(data: CANData, sys_user: User, session: Session) -> None:
 
         can.portfolio = portfolio
 
-        # get or create funding details
-        fiscal_year = int(data.FUND[6:10])
-        fund_code = data.FUND
-        allowance = data.ALLOWANCE
-        sub_allowance = data.SUB_ALLOWANCE
-        allotment = data.ALLOTMENT_ORG
-        appropriation = "-".join([data.APPROP_PREFIX or "", data.APPROP_YEAR[0:2] or "", data.APPROP_POSTFIX or ""])
-        method_of_transfer = CANMethodOfTransfer[data.METHOD_OF_TRANSFER]
-        funding_source = CANFundingSource[data.FUNDING_SOURCE]
-
-        existing_funding_details = session.execute(select(CANFundingDetails).where(
-            and_(
-                CANFundingDetails.fiscal_year == fiscal_year,
-                CANFundingDetails.fund_code == fund_code,
-                CANFundingDetails.allowance == allowance,
-                CANFundingDetails.sub_allowance == sub_allowance,
-                CANFundingDetails.allotment == allotment,
-                CANFundingDetails.appropriation == appropriation,
-                CANFundingDetails.method_of_transfer == method_of_transfer,
-                CANFundingDetails.funding_source == funding_source,
-            ))).scalar_one_or_none()
-
-        if not existing_funding_details:
-            funding_details = CANFundingDetails(
-                fiscal_year=fiscal_year,
-                fund_code=fund_code,
-                allowance=allowance,
-                sub_allowance=sub_allowance,
-                allotment=allotment,
-                appropriation=appropriation,
-                method_of_transfer=method_of_transfer,
-                funding_source=funding_source,
-                created_by=sys_user.id,
-            )
-            session.add(funding_details)
-            session.commit()
-            can.funding_details = funding_details
-        else:
-            can.funding_details = existing_funding_details
+        try:
+            validate_fund_code(data)
+            can.funding_details = get_or_create_funding_details(data, sys_user, session)
+        except ValueError as e:
+            logger.info(f"Skipping creating funding details for {data} due to invalid fund code. {e}")
 
         session.merge(can)
         session.commit()
     except Exception as e:
         logger.error(f"Error creating models for {data}")
         raise e
 
+
+def get_or_create_funding_details(data: CANData, sys_user: User, session: Session) -> CANFundingDetails:
+    """
+    Get or create a CANFundingDetails instance.
+
+    :param data: The CANData instance to use.
+    :param sys_user: The system user to use.
+    :param session: The database session to use.
+
+    :return: A CANFundingDetails instance.
+    """
+    fiscal_year = int(data.FUND[6:10])
+    fund_code = data.FUND
+    allowance = data.ALLOWANCE
+    sub_allowance = data.SUB_ALLOWANCE
+    allotment = data.ALLOTMENT_ORG
+
+    appropriation_year = data.APPROP_YEAR[0:2] if data.APPROP_YEAR else ""
+    appropriation = "-".join([data.APPROP_PREFIX or "", appropriation_year, data.APPROP_POSTFIX or ""])
+
+    method_of_transfer = CANMethodOfTransfer[data.METHOD_OF_TRANSFER]
+    funding_source = CANFundingSource[data.FUNDING_SOURCE]
+    existing_funding_details = session.execute(select(CANFundingDetails).where(
+        and_(
+            CANFundingDetails.fiscal_year == fiscal_year,
+            CANFundingDetails.fund_code == fund_code,
+            CANFundingDetails.allowance == allowance,
+            CANFundingDetails.sub_allowance == sub_allowance,
+            CANFundingDetails.allotment == allotment,
+            CANFundingDetails.appropriation == appropriation,
+            CANFundingDetails.method_of_transfer == method_of_transfer,
+            CANFundingDetails.funding_source == funding_source,
+        ))).scalar_one_or_none()
+    if not existing_funding_details:
+        funding_details = CANFundingDetails(
+            fiscal_year=fiscal_year,
+            fund_code=fund_code,
+            allowance=allowance,
+            sub_allowance=sub_allowance,
+            allotment=allotment,
+            appropriation=appropriation,
+            method_of_transfer=method_of_transfer,
+            funding_source=funding_source,
+            created_by=sys_user.id,
+        )
+        return funding_details
+    else:
+        return existing_funding_details
+
+
+def validate_fund_code(data: CANData) -> None:
+    """
+    Validate the fund code in a CANData instance.
+
+    :param data: The CANData instance to validate.
+
+    :return: None
+    :raises ValueError: If the fund code is invalid.
+    """
+    if not data.FUND:
+        raise ValueError("Fund code is required.")
+    if len(data.FUND) != 14:
+        raise ValueError(f"Invalid fund code length {data.FUND}")
+    int(data.FUND[6:10])
+    length_of_appropriation = data.FUND[10]
+    if length_of_appropriation not in ["0", "1", "5"]:
+        raise ValueError(f"Invalid length of appropriation {length_of_appropriation}")
+    direct_or_reimbursable = data.FUND[11]
+    if direct_or_reimbursable not in ["D", "R"]:
+        raise ValueError(f"Invalid direct or reimbursable {direct_or_reimbursable}")
+    category = data.FUND[12]
+    if category not in ["A", "B", "C"]:
+        raise ValueError(f"Invalid category {category}")
+    discretionary_or_mandatory = data.FUND[13]
+    if discretionary_or_mandatory not in ["D", "M"]:
+        raise ValueError(f"Invalid discretionary or mandatory {discretionary_or_mandatory}")
+
+
 def create_all_models(data: List[CANData], sys_user: User, session: Session) -> None:
     """
     Convert a list of CanData instances to a list of BaseModel instances.

backend/data_tools/tests/load_cans/test_load_cans.py

@@ -6,7 +6,14 @@
 from data_tools.src.common.utils import get_or_create_sys_user
 from data_tools.src.import_static_data.import_data import get_config
 from data_tools.src.load_cans.main import main
-from data_tools.src.load_cans.utils import CANData, create_can_data, create_models, validate_all, validate_data
+from data_tools.src.load_cans.utils import (
+    CANData,
+    create_can_data,
+    create_models,
+    validate_all,
+    validate_data,
+    validate_fund_code,
+)
 from sqlalchemy import and_, text
 
 from models import *  # noqa: F403, F401
@@ -295,7 +302,7 @@ def test_create_models_upsert(db_with_portfolios):
         SYS_CAN_ID=500,
         CAN_NBR="G99HRF3",
         CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
-        FUND="AAXXXX20231DAE",
+        FUND="AAXXXX20231DAM",
         ALLOWANCE="0000000001",
         ALLOTMENT_ORG="YZC6S1JUGUN",
         SUB_ALLOWANCE="9KRZ2ND",
@@ -357,7 +364,7 @@ def test_create_models_upsert(db_with_portfolios):
     assert can_1.portfolio == db_with_portfolios.execute(
         select(Portfolio).where(Portfolio.abbreviation == "HMRF")).scalar_one_or_none()
     assert can_1.funding_details.id == db_with_portfolios.execute(
-        select(CANFundingDetails).where(CANFundingDetails.fund_code == "AAXXXX20231DAE")).scalar_one_or_none().id
+        select(CANFundingDetails).where(CANFundingDetails.fund_code == "AAXXXX20231DAM")).scalar_one_or_none().id
     assert can_1.created_by == sys_user.id
 
     assert len(db_with_portfolios.execute(select(CAN)).scalars().all()) == 1
@@ -370,3 +377,197 @@ def test_create_models_upsert(db_with_portfolios):
     db_with_portfolios.execute(text("DELETE FROM can_funding_details_version"))
     db_with_portfolios.execute(text("DELETE FROM ops_db_history"))
     db_with_portfolios.execute(text("DELETE FROM ops_db_history_version"))
+
+def test_validate_fund_code():
+    data = CANData(
+        SYS_CAN_ID=500,
+        CAN_NBR="G99HRF2",
+        CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
+        FUND="AAXXXX20231DAD",
+        ALLOWANCE="0000000001",
+        ALLOTMENT_ORG="YZC6S1JUGUN",
+        SUB_ALLOWANCE="9KRZ2ND",
+        CURRENT_FY_FUNDING_YTD=880000.0,
+        APPROP_PREFIX="XX",
+        APPROP_POSTFIX="XXXX",
+        APPROP_YEAR="23",
+        PORTFOLIO="HMRF",
+        FUNDING_SOURCE="OPRE",
+        METHOD_OF_TRANSFER="DIRECT",
+        NICK_NAME="HMRF-OPRE",
+    )
+    validate_fund_code(data)
+
+def test_validate_fund_code_length():
+    data = CANData(
+        SYS_CAN_ID=500,
+        CAN_NBR="G99HRF2",
+        CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
+        FUND="AAXXXX20231DADDDDDDD",
+        ALLOWANCE="0000000001",
+        ALLOTMENT_ORG="YZC6S1JUGUN",
+        SUB_ALLOWANCE="9KRZ2ND",
+        CURRENT_FY_FUNDING_YTD=880000.0,
+        APPROP_PREFIX="XX",
+        APPROP_POSTFIX="XXXX",
+        APPROP_YEAR="23",
+        PORTFOLIO="HMRF",
+        FUNDING_SOURCE="OPRE",
+        METHOD_OF_TRANSFER="DIRECT",
+        NICK_NAME="HMRF-OPRE",
+    )
+    with pytest.raises(ValueError) as e_info:
+        validate_fund_code(data)
+    assert e_info.value.args[0] == "Invalid fund code length AAXXXX20231DADDDDDDD"
+
+def test_validate_fund_code_fy():
+    data = CANData(
+        SYS_CAN_ID=500,
+        CAN_NBR="G99HRF2",
+        CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
+        FUND="AAXXXX20FY1DAD",
+        ALLOWANCE="0000000001",
+        ALLOTMENT_ORG="YZC6S1JUGUN",
+        SUB_ALLOWANCE="9KRZ2ND",
+        CURRENT_FY_FUNDING_YTD=880000.0,
+        APPROP_PREFIX="XX",
+        APPROP_POSTFIX="XXXX",
+        APPROP_YEAR="23",
+        PORTFOLIO="HMRF",
+        FUNDING_SOURCE="OPRE",
+        METHOD_OF_TRANSFER="DIRECT",
+        NICK_NAME="HMRF-OPRE",
+    )
+    with pytest.raises(ValueError) as e_info:
+        validate_fund_code(data)
+    assert e_info.value.args[0] == "invalid literal for int() with base 10: '20FY'"
+
+def test_validate_fund_code_length_of_appropriation():
+    data = CANData(
+        SYS_CAN_ID=500,
+        CAN_NBR="G99HRF2",
+        CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
+        FUND="AAXXXX20233DAD",
+        ALLOWANCE="0000000001",
+        ALLOTMENT_ORG="YZC6S1JUGUN",
+        SUB_ALLOWANCE="9KRZ2ND",
+        CURRENT_FY_FUNDING_YTD=880000.0,
+        APPROP_PREFIX="XX",
+        APPROP_POSTFIX="XXXX",
+        APPROP_YEAR="23",
+        PORTFOLIO="HMRF",
+        FUNDING_SOURCE="OPRE",
+        METHOD_OF_TRANSFER="DIRECT",
+        NICK_NAME="HMRF-OPRE",
+    )
+    with pytest.raises(ValueError) as e_info:
+        validate_fund_code(data)
+    assert e_info.value.args[0] == "Invalid length of appropriation 3"
+
+
+def test_validate_fund_code_direct_or_reimbursable():
+    data = CANData(
+        SYS_CAN_ID=500,
+        CAN_NBR="G99HRF2",
+        CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
+        FUND="AAXXXX20231OAD",
+        ALLOWANCE="0000000001",
+        ALLOTMENT_ORG="YZC6S1JUGUN",
+        SUB_ALLOWANCE="9KRZ2ND",
+        CURRENT_FY_FUNDING_YTD=880000.0,
+        APPROP_PREFIX="XX",
+        APPROP_POSTFIX="XXXX",
+        APPROP_YEAR="23",
+        PORTFOLIO="HMRF",
+        FUNDING_SOURCE="OPRE",
+        METHOD_OF_TRANSFER="DIRECT",
+        NICK_NAME="HMRF-OPRE",
+    )
+    with pytest.raises(ValueError) as e_info:
+        validate_fund_code(data)
+    assert e_info.value.args[0] == "Invalid direct or reimbursable O"
+
+def test_validate_fund_code_category():
+    data = CANData(
+        SYS_CAN_ID=500,
+        CAN_NBR="G99HRF2",
+        CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
+        FUND="AAXXXX20231DDD",
+        ALLOWANCE="0000000001",
+        ALLOTMENT_ORG="YZC6S1JUGUN",
+        SUB_ALLOWANCE="9KRZ2ND",
+        CURRENT_FY_FUNDING_YTD=880000.0,
+        APPROP_PREFIX="XX",
+        APPROP_POSTFIX="XXXX",
+        APPROP_YEAR="23",
+        PORTFOLIO="HMRF",
+        FUNDING_SOURCE="OPRE",
+        METHOD_OF_TRANSFER="DIRECT",
+        NICK_NAME="HMRF-OPRE",
+    )
+    with pytest.raises(ValueError) as e_info:
+        validate_fund_code(data)
+    assert e_info.value.args[0] == "Invalid category D"
+
+def test_validate_fund_code_discretionary_or_mandatory():
+    data = CANData(
+        SYS_CAN_ID=500,
+        CAN_NBR="G99HRF2",
+        CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
+        FUND="AAXXXX20231DAR",
+        ALLOWANCE="0000000001",
+        ALLOTMENT_ORG="YZC6S1JUGUN",
+        SUB_ALLOWANCE="9KRZ2ND",
+        CURRENT_FY_FUNDING_YTD=880000.0,
+        APPROP_PREFIX="XX",
+        APPROP_POSTFIX="XXXX",
+        APPROP_YEAR="23",
+        PORTFOLIO="HMRF",
+        FUNDING_SOURCE="OPRE",
+        METHOD_OF_TRANSFER="DIRECT",
+        NICK_NAME="HMRF-OPRE",
+    )
+    with pytest.raises(ValueError) as e_info:
+        validate_fund_code(data)
+    assert e_info.value.args[0] == "Invalid discretionary or mandatory R"
+
+def test_create_models_invalid_fund_code(db_with_portfolios):
+    data = CANData(
+        SYS_CAN_ID=500,
+        CAN_NBR="G99HRF2",
+        CAN_DESCRIPTION="Healthy Marriages Responsible Fatherhood - OPRE",
+        FUND="AAXXXX20231DADDDDDD",
+        ALLOWANCE="0000000001",
+        ALLOTMENT_ORG="YZC6S1JUGUN",
+        SUB_ALLOWANCE="9KRZ2ND",
+        CURRENT_FY_FUNDING_YTD=880000.0,
+        APPROP_PREFIX="XX",
+        APPROP_POSTFIX="XXXX",
+        APPROP_YEAR="23",
+        PORTFOLIO="HMRF",
+        FUNDING_SOURCE="OPRE",
+        METHOD_OF_TRANSFER="DIRECT",
+        NICK_NAME="HMRF-OPRE",
+    )
+
+    sys_user = User(
+        email="system.admin@localhost",
+    )
+    create_models(data, sys_user, db_with_portfolios)
+
+    can_model = db_with_portfolios.get(CAN, 500)
+
+    assert can_model.id == 500
+    assert can_model.number == "G99HRF2"
+    assert can_model.description == "Healthy Marriages Responsible Fatherhood - OPRE"
+    assert can_model.nick_name == "HMRF-OPRE"
+    assert can_model.portfolio == db_with_portfolios.execute(select(Portfolio).where(Portfolio.abbreviation == "HMRF")).scalar_one_or_none()
+    assert can_model.funding_details is None
+
+    # Cleanup
+    db_with_portfolios.execute(text("DELETE FROM can"))
+    db_with_portfolios.execute(text("DELETE FROM can_funding_details"))
+    db_with_portfolios.execute(text("DELETE FROM can_version"))
+    db_with_portfolios.execute(text("DELETE FROM can_funding_details_version"))
+    db_with_portfolios.execute(text("DELETE FROM ops_db_history"))
+    db_with_portfolios.execute(text("DELETE FROM ops_db_history_version"))