From 0cb4d5b5a5aaf70acf6106804132c935a8f60827 Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Wed, 23 Oct 2024 17:06:18 -0700 Subject: [PATCH 1/2] Refactors gherkin suite to parametrized suite --- api/app/tests/fba_calc/test_fba_error.feature | 91 ----- api/app/tests/fba_calc/test_fba_error.py | 313 ------------------ .../tests/fba_calc/test_fba_error_redapp.py | 147 ++++++++ api/pyproject.toml | 3 + 4 files changed, 150 insertions(+), 404 deletions(-) delete mode 100644 api/app/tests/fba_calc/test_fba_error.feature delete mode 100644 api/app/tests/fba_calc/test_fba_error.py create mode 100644 api/app/tests/fba_calc/test_fba_error_redapp.py diff --git a/api/app/tests/fba_calc/test_fba_error.feature b/api/app/tests/fba_calc/test_fba_error.feature deleted file mode 100644 index b4b482ab9..000000000 --- a/api/app/tests/fba_calc/test_fba_error.feature +++ /dev/null @@ -1,91 +0,0 @@ -Feature: /fbc/ - - Scenario: Fire Behaviour Calculation - vs. REDapp - Given REDapp input , , , , , , , , , , , , , , , - Then ROS is within () - And CFB is within () - And HFI is within () - And 1 HR Size is within () - And ROS_t is within range () - And CFB_t is within range () - And HFI_t is within range () - And (1 HR Size)_t is within range () - And Log it - - # C1 Notes: - # - # C1 ROS in CFFDRS is correct (Based on investigation, the result differs from REDapp slightly - # but the way we call CFFDRS is correct.) - # C1 CFB requires the date of minimum foliar moisture content to be set to 144 to match the REDapp - # result. - # C1 in the coastal spreadsheet is wrong. - # - # C3 redapp error margin is off - # C6 The redapp error margin is horrible (71%-80%)! This must be improved! - # M1 The redapp error margin is HUGE! - # M2 The redapp error margin is HUGE! - # M4 Redapp margin bad. - # O1A Spreadsheet bad - # O1B Spreadsheet bad - # Current target for margin of error: 1% (or 0.01) - - # one_hr_em :- 1 Hour Error Margin - # cfb_em :- Crown Fraction Burned Error Margin - # hfi_em :- Head Fire Intensity Error Margin - Examples: - | fuel_type | elevation | latitude | longitude | time_of_interest | wind_speed | wind_direction | percentage_conifer | percentage_dead_balsam_fir | grass_cure | crown_base_height | isi | bui | ffmc | dmc | dc | one_hr_em | ros_em | hfi_em | cfb_em | note | - | C1 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | 0 | 0 | 2 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | C2 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | None | None | 3 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | C3 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | None | None | 8 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.02 | REDapp | - | C3 | 3244 | 55.207218202444906 | -128.73536255317552 | 2021-05-21 | 23.835537134445126 | 168.4979596024838 | 100.0 | None | None | 8.0 | 13.74041206802971 | 55.2112620090575 | 89.7461720614907 | 33.22303813883521 | 408.17004615391113 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | C3 | 3075 | 49.002377849288145 | -129.53731237443654 | 2021-02-10 | 17.3906100267642 | 163.34053427851433 | 100.0 | None | None | 8.0 | 14.132196449385695 | 66.1696880962494 | 92.22387770168902 | 45.65846482284549 | 300.351667830179 | 0.14 | 0.01 | 0.05 | 0.08 | REDapp | - | C4 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | None | None | 4 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | C5 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | None | None | 18 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | C6 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | None | None | 7 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.02 | 0.01 | 0.01 | 0.01 | REDapp | - | C6 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | None | None | 2 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | C7 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | None | None | 10 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | D1 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | None | None | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | M1 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 75 | None | None | 6 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | M1 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 50 | None | None | 6 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | M1 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 25 | None | None | 6 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.39 | 0.01 | 0.07 | 0.15 | REDapp | - | M2 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 75 | None | None | 6 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | M2 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 50 | None | None | 6 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | M2 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 25 | None | None | 6 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | M3 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | 30 | None | 6 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | M3 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | 60 | None | 6 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | M3 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | 100 | None | 6 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | M4 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | 30 | None | 6 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | M4 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | 60 | None | 6 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | M4 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | 100 | None | 6 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | O1A | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | 25 | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | O1A | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | 50 | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | O1A | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | 100 | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | O1B | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | 25 | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.12 | 0.01 | 0.01 | 0.01 | REDapp | - | O1B | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | 50 | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | O1B | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | 100 | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | S1 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | None | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | S2 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | None | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - | S3 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | None | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.01 | 0.01 | 0.01 | 0.01 | REDapp | - - - Scenario: Fire Behaviour Calculation - vs. Spreadsheet - Given Spreadsheet input , , , , , , , , , , , , , , , , , , - Then ROS is within () - And CFB is within () - And HFI is within () - - Examples: - | fuel_type | elevation | latitude | longitude | time_of_interest | wind_speed | wind_direction | percentage_conifer | percentage_dead_balsam_fir | grass_cure | crown_base_height | isi | bui | ffmc | dmc | dc | ros | ros_em | hfi | hfi_em | cfb | cfb_em | note | - | C2 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | None | None | 3 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 19.26 | 0.01 | 30072.67 | 0.01 | 1.0 | 0.011 | n/a | - | C3 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | None | None | 8 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 8.21 | 0.01 | 13097.75 | 0.02 | 0.7 | 0.84 | n/a | - | C4 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | None | None | 4 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 18.77 | 0.01 | 31944.15 | 0.01 | 1.0 | 0.03 | n/a | - | C5 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | None | None | 18 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 3.19 | 0.01 | 4081.48 | 0.01 | -7.9 | 0.01 | Spreadsheet gives me -7.9 for CFB! | - | C7 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | 100 | None | None | 10 | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 4.07 | 0.01 | 4089.57 | 0.0105 | -0.9 | 0.01 | Spreadsheet has a negative CFB! | - | O1A | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | 25 | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.63 | 0.01 | 56.94 | 0.17 | None | 0.01 | n/a | - | O1A | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | 50 | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 3.54 | 0.01 | 318.58 | 0.17 | None | 0.01 | n/a | - | O1A | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | 100 | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 35.19 | 0.01 | 3167.51 | 0.17 | None | 0.01 | n/a | - | O1B | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | 25 | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 0.63 | 0.1 | 56.94 | 0.27 | None | 0.01 | n/a | - | O1B | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | 50 | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 3.54 | 0.09 | 318.58 | 0.27 | None | 0.01 | n/a | - | O1B | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | 100 | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 17.60 | 1.18 | 1583.75 | 1.54 | None | 0.01 | n/a | - | S3 | 780 | 50.6733333 | -120.4816667 | 2021-07-12 | 6.2 | 3 | None | None | None | None | 11.5 | 186.8 | 94.8 | 126.1 | 900.3 | 17.05 | 0.01 | 158929.49 | 0.01 | None | 0.01 | n/a | - diff --git a/api/app/tests/fba_calc/test_fba_error.py b/api/app/tests/fba_calc/test_fba_error.py deleted file mode 100644 index 5c444752f..000000000 --- a/api/app/tests/fba_calc/test_fba_error.py +++ /dev/null @@ -1,313 +0,0 @@ -""" -Unit tests for fire behavour calculator. -""" -from datetime import date -import logging -from pytest_bdd import scenario, given, then, parsers -from app import configure_logging -from app.schemas.fba_calc import FuelTypeEnum -from app.utils.time import get_hour_20_from_date -from app.fire_behaviour.advisory import calculate_fire_behaviour_advisory, FBACalculatorWeatherStation -from app.utils.redapp import FBPCalculateStatisticsCOM -from app.tests.common import str2float -from app.tests.fba_calc import (check_metric, acceptable_margin_of_error, - fire_size_acceptable_margin_of_error) -import pytest - - -configure_logging() - -logger = logging.getLogger(__name__) - - -@pytest.mark.usefixtures('mock_jwt_decode') -@scenario('test_fba_error.feature', 'Fire Behaviour Calculation - vs. REDapp') -def test_fire_behaviour_calculator_scenario(): - """ BDD Scenario. """ - - -@given( - parsers.parse("""REDapp input {elevation}, {latitude}, {longitude}, {time_of_interest}, {wind_speed}, {wind_direction}, """ - """{percentage_conifer}, {percentage_dead_balsam_fir}, {grass_cure}, {crown_base_height}, """ - """{isi}, {bui}, {ffmc}, {dmc}, {dc}, {fuel_type}"""), - converters=dict(elevation=float, - latitude=float, - longitude=float, - time_of_interest=date.fromisoformat, - wind_speed=float, - wind_direction=str2float, - percentage_conifer=str2float, - percentage_dead_balsam_fir=str2float, - grass_cure=str2float, - crown_base_height=str2float, - isi=float, - bui=float, - ffmc=float, - dmc=float, - dc=float, - fuel_type=str), - target_fixture='result') -def given_red_app_input(elevation: float, - latitude: float, longitude: float, time_of_interest: date, - wind_speed: float, wind_direction: float, - percentage_conifer: float, - percentage_dead_balsam_fir: float, - grass_cure: float, - crown_base_height: float, - isi: float, bui: float, ffmc: float, dmc: float, dc: float, fuel_type: str): - """ Take input and calculate actual and expected results """ - # get python result: - python_input = FBACalculatorWeatherStation(elevation=elevation, - fuel_type=FuelTypeEnum[fuel_type], - time_of_interest=time_of_interest, - percentage_conifer=percentage_conifer, - percentage_dead_balsam_fir=percentage_dead_balsam_fir, - grass_cure=grass_cure, - crown_base_height=crown_base_height, - crown_fuel_load=None, - lat=latitude, long=longitude, bui=bui, - ffmc=ffmc, isi=isi, fwi=None, wind_speed=wind_speed, - wind_direction=wind_direction, - temperature=20.0, # temporary fix so tests don't break - relative_humidity=20.0, - precipitation=2.0, - status='Forecasted', - prev_day_daily_ffmc=90.0, - last_observed_morning_rh_values={ - 7.0: 61.0, 8.0: 54.0, 9.0: 43.0, 10.0: 38.0, - 11.0: 34.0, 12.0: 23.0}) - python_fba = calculate_fire_behaviour_advisory(python_input) - # get REDapp result from java: - java_fbp = FBPCalculateStatisticsCOM(elevation=elevation, - latitude=latitude, - longitude=longitude, - time_of_interest=get_hour_20_from_date(time_of_interest), - fuel_type=fuel_type, - ffmc=ffmc, - dmc=dmc, - dc=dc, - bui=bui, - wind_speed=wind_speed, - wind_direction=wind_direction, - percentage_conifer=percentage_conifer, - percentage_dead_balsam_fir=percentage_dead_balsam_fir, - grass_cure=grass_cure, - crown_base_height=crown_base_height) - - # NOTE: REDapp has a ros_eq and a ros_t ; - # assumptions: - # ros_eq == ROScalc - # ros_t == ROStcalc - expected = { - 'ros': java_fbp.ros_eq, - 'ros_t': java_fbp.ros_t, - 'cfb': java_fbp.cfb / 100.0, # CFFDRS gives cfb as a fraction - 'hfi': java_fbp.hfi, - 'area': java_fbp.area - } - - error_dict = { - 'fuel_type': fuel_type - } - - return { - 'python': python_fba, - 'expected': expected, - 'fuel_type': fuel_type, - 'error': error_dict - } - - -@then(parsers.parse("ROS is within {ros_em} ({note})"), converters={'ros_em': float, 'note': str}) -def then_ros(result: dict, ros_em: float, note: str): - """ check the relative error of the ros """ - error = check_metric('ROS', - result['fuel_type'], - result['python'].ros, - result['expected']['ros'], - ros_em, - note) - result['error']['ros_em'] = error - - -@then(parsers.parse("ROS_t is within range ({note})"), converters={'note': str}) -def then_ros_t(result: dict, note: str): - """ check the relative error of the ros """ - check_metric('ROS_t', - result['fuel_type'], - result['python'].ros_t, - result['expected']['ros_t'], - acceptable_margin_of_error, - note) - - -@then(parsers.parse("CFB is within {cfb_em} ({note})"), converters={'cfb_em': float, 'note': str}) -def then_cfb(result: dict, cfb_em: float, note: str): - """ check the relative error of the cfb """ - error = check_metric('CFB', - result['fuel_type'], - result['python'].cfb, - result['expected']['cfb'], - cfb_em, - note) - result['error']['cfb_em'] = error - - -@then(parsers.parse("CFB_t is within range ({note})"), converters={'note': str}) -def then_cfb_t(result: dict, note: str): - """ check the relative error of the ros """ - check_metric('CFB_t', - result['fuel_type'], - result['python'].cfb_t, - result['expected']['cfb'], - acceptable_margin_of_error, - note) - - -@then(parsers.parse("HFI is within {hfi_em} ({note})"), converters={'hfi_em': float, 'note': str}) -def then_hfi(result: dict, hfi_em: float, note: str): - """ check the relative error of the hfi """ - error = check_metric('HFI', - result['fuel_type'], - result['python'].hfi, - result['expected']['hfi'], - hfi_em, - note) - result['error']['hfi_em'] = error - - -@then(parsers.parse("HFI_t is within range ({note})"), converters={'note': str}) -def then_hfi_t(result: dict, note: str): - """ check the relative error of the ros """ - check_metric('HFI_t', - result['fuel_type'], - result['python'].hfi_t, - result['expected']['hfi'], - acceptable_margin_of_error, - note) - - -@then(parsers.parse("1 HR Size is within {one_hr_em} ({note})"), converters={'one_hr_em': float, 'note': str}) -def then_one_hr(result: dict, one_hr_em: float, note: str): - """ check the relative error of the a hour fire size""" - error = check_metric('1 HR Size', - result['fuel_type'], - result['python'].sixty_minute_fire_size, - result['expected']['area'], - one_hr_em, - note) - result['error']['one_hr_em'] = error - - -@then(parsers.parse("(1 HR Size)_t is within range ({note})"), converters={'note': str}) -def then_one_hr_t(result: dict, note: str): - """ check the relative error of the a hour fire size""" - check_metric('1 HR Size t', - result['fuel_type'], - result['python'].sixty_minute_fire_size_t, - result['expected']['area'], - fire_size_acceptable_margin_of_error, - note) - - -@then("Log it") -def log_it(result: dict): - """ Log a string matching the scenario input - useful when improving values. """ - error_dict = result.get('error') - header = '| fuel_type | elevation | latitude | longitude | time_of_interest | wind_speed | wind_direction | percentage_conifer | percentage_dead_balsam_fir | grass_cure | crown_base_height | isi | bui | ffmc | dmc | dc | one_hr_em | ros_em | hfi_em | cfb_em | note |' - header = header.strip('|') - header = header.split('|') - header = [x.strip() for x in header] - - line = '|' - for key in header: - value = error_dict.get(key, key) - if isinstance(value, float): - line += f'{value:.2f}|' - else: - line += f'{value}|' - logger.debug(line) - - -@pytest.mark.usefixtures('mock_jwt_decode') -@scenario('test_fba_error.feature', 'Fire Behaviour Calculation - vs. Spreadsheet') -def test_fire_behaviour_calculator_spreadsheet_scenario(): - """ BDD Scenario. """ - - -@given(parsers.parse("""Spreadsheet input {elevation}, {latitude}, {longitude}, {time_of_interest}, {wind_speed}, {wind_direction}, """ - """{percentage_conifer}, {percentage_dead_balsam_fir}, {grass_cure}, {crown_base_height}, {isi}, """ - """{bui}, {ffmc}, {dmc}, {dc}, {fuel_type}, {ros}, {hfi}, {cfb}"""), - converters=dict(elevation=float, - latitude=float, - longitude=float, - percentage_conifer=str2float, - percentage_dead_balsam_fir=str2float, - crown_base_height=str2float, - grass_cure=str2float, - isi=float, - bui=float, - ffmc=float, - dmc=float, - dc=float, - fuel_type=str, - cfb=str2float, - hfi=str2float, - ros=str2float, - time_of_interest=date.fromisoformat, - wind_direction=float, - wind_speed=float), - target_fixture='result') -def given_spreadsheet_input(elevation: float, - latitude: float, longitude: float, time_of_interest: str, - wind_speed: float, wind_direction: float, - percentage_conifer: float, percentage_dead_balsam_fir: float, grass_cure: float, - crown_base_height: float, - isi: float, bui: float, ffmc: float, dmc: float, dc: float, fuel_type: str, - ros: float, hfi: float, cfb: float): - """ Take input and calculate actual and expected results """ - # get python result: - fwi = None - python_input = FBACalculatorWeatherStation(elevation=elevation, - fuel_type=FuelTypeEnum[fuel_type], - time_of_interest=time_of_interest, - percentage_conifer=percentage_conifer, - percentage_dead_balsam_fir=percentage_dead_balsam_fir, - grass_cure=grass_cure, - crown_base_height=crown_base_height, - crown_fuel_load=None, - lat=latitude, - long=longitude, - bui=bui, - ffmc=ffmc, - isi=isi, - fwi=fwi, - wind_speed=wind_speed, - wind_direction=wind_direction, - temperature=20.0, # temporary fix so tests don't break - relative_humidity=20.0, - precipitation=2.0, - status='Forecasted', - prev_day_daily_ffmc=90.0, - last_observed_morning_rh_values={ - 7.0: 61.0, 8.0: 54.0, 9.0: 43.0, 10.0: 38.0, - 11.0: 34.0, 12.0: 23.0}) - python_fba = calculate_fire_behaviour_advisory(python_input) - - expected = { - 'ros': ros, - 'cfb': cfb, - 'hfi': hfi, - 'area': None - } - - error_dict = { - 'fuel_type': fuel_type - } - - return { - 'python': python_fba, - 'expected': expected, - 'fuel_type': fuel_type, - 'error': error_dict - } diff --git a/api/app/tests/fba_calc/test_fba_error_redapp.py b/api/app/tests/fba_calc/test_fba_error_redapp.py new file mode 100644 index 000000000..b03b9e88f --- /dev/null +++ b/api/app/tests/fba_calc/test_fba_error_redapp.py @@ -0,0 +1,147 @@ +import pytest +import logging +from datetime import date +from app.schemas.fba_calc import FuelTypeEnum +from app.utils.time import get_hour_20_from_date +from app.fire_behaviour.advisory import calculate_fire_behaviour_advisory, FBACalculatorWeatherStation +from app.utils.redapp import FBPCalculateStatisticsCOM +from app.tests.fba_calc import check_metric, acceptable_margin_of_error, fire_size_acceptable_margin_of_error + +logger = logging.getLogger(__name__) + +# C1 Notes: +# +# C1 ROS in CFFDRS is correct (Based on investigation, the result differs from REDapp slightly +# but the way we call CFFDRS is correct.) +# C1 CFB requires the date of minimum foliar moisture content to be set to 144 to match the REDapp +# result. +# C1 in the coastal spreadsheet is wrong. +# +# C3 redapp error margin is off +# C6 The redapp error margin is horrible (71%-80%)! This must be improved! +# M1 The redapp error margin is HUGE! +# M2 The redapp error margin is HUGE! +# M4 Redapp margin bad. +# O1A Spreadsheet bad +# O1B Spreadsheet bad +# Current target for margin of error: 1% (or 0.01) + +# one_hr_em :- 1 Hour Error Margin +# cfb_em :- Crown Fraction Burned Error Margin +# hfi_em :- Head Fire Intensity Error Margin + +@pytest.mark.parametrize( + "fuel_type,elevation,latitude,longitude,time_of_interest,wind_speed,wind_direction,percentage_conifer,percentage_dead_balsam_fir,grass_cure,crown_base_height,isi,bui,ffmc,dmc,dc,one_hr_em,ros_em,hfi_em,cfb_em", + [ + ("C1", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 100, 0, 0, 2, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("C2", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 100, None, None, 3, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("C3", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 100, None, None, 8, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.02), + ("C3", 3244, 55.207218202444906, -128.73536255317552, date.fromisoformat("2021-05-21"), 23.835537134445126, 168.4979596024838, 100.0, None, None, 8.0, 13.74041206802971, 55.2112620090575, 89.7461720614907, 33.22303813883521, 408.17004615391113, 0.01, 0.01, 0.01, 0.01), + ("C3",3075,49.002377849288145,-129.53731237443654,date.fromisoformat("2021-02-10"),17.3906100267642,163.34053427851433,100.0,None,None,8.0,14.132196449385695,66.1696880962494,92.22387770168902,45.65846482284549,300.351667830179,0.14,0.01,0.05,0.08), + ("C4", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 100, None, None, 4, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("C5", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 100, None, None, 18, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("C6", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 100, None, None, 7, 11.5, 186.8, 94.8, 126.1, 900.3, 0.02, 0.01, 0.01, 0.01), + ("C6", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 100, None, None, 2, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("C7", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 100, None, None, 10, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("D1", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 100, None, None, None, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("M1", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 75, None, None, 6, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("M1", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 50, None, None, 6, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("M1", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 25, None, None, 6, 11.5, 186.8, 94.8, 126.1, 900.3, 0.39, 0.01, 0.07, 0.15), + ("M2", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 75, None, None, 6, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("M2", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 50, None, None, 6, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("M2", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, 25, None, None, 6, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("M3", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, 30, None, 6, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("M3", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, 60, None, 6, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("M3", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, 100, None, 6, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("M4", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, 30, None, 6, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("M4", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, 60, None, 6, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("M4", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, 100, None, 6, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("O1A", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, None, 25, None, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("O1A", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, None, 50, None, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("O1A", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, None, 100, None, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("O1B", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, None, 25, None, 11.5, 186.8, 94.8, 126.1, 900.3, 0.12, 0.01, 0.01, 0.01), + ("O1B", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, None, 50, None, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("O1B", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, None, 100, None, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("S1", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, None, None, None, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("S2", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, None, None, None, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01), + ("S3", 780, 50.6733333, -120.4816667, date.fromisoformat("2021-07-12"), 6.2, 3, None, None, None, None, 11.5, 186.8, 94.8, 126.1, 900.3, 0.01, 0.01, 0.01, 0.01) + ], +) +def test_redapp_vs_fba( + fuel_type, + elevation, + latitude, + longitude, + time_of_interest, + wind_speed, + wind_direction, + percentage_conifer, + percentage_dead_balsam_fir, + grass_cure, + crown_base_height, + isi, + bui, + ffmc, + dmc, + dc, + one_hr_em, + ros_em, + hfi_em, + cfb_em, +): + # get python result: + python_input = FBACalculatorWeatherStation( + elevation=elevation, + fuel_type=FuelTypeEnum[fuel_type], + time_of_interest=time_of_interest, + percentage_conifer=percentage_conifer, + percentage_dead_balsam_fir=percentage_dead_balsam_fir, + grass_cure=grass_cure, + crown_base_height=crown_base_height, + crown_fuel_load=None, + lat=latitude, + long=longitude, + bui=bui, + ffmc=ffmc, + isi=isi, + fwi=None, + wind_speed=wind_speed, + wind_direction=wind_direction, + temperature=20.0, # temporary fix so tests don't break + relative_humidity=20.0, + precipitation=2.0, + status="Forecasted", + prev_day_daily_ffmc=90.0, + last_observed_morning_rh_values={7.0: 61.0, 8.0: 54.0, 9.0: 43.0, 10.0: 38.0, 11.0: 34.0, 12.0: 23.0}, + ) + python_fba = calculate_fire_behaviour_advisory(python_input) + # NOTE: REDapp has a ros_eq and a ros_t ; + # assumptions: + # ros_eq == ROScalc + # ros_t == ROStcalc + # get REDapp result from java: + java_fbp = FBPCalculateStatisticsCOM( + elevation=elevation, + latitude=latitude, + longitude=longitude, + time_of_interest=get_hour_20_from_date(time_of_interest), + fuel_type=fuel_type, + ffmc=ffmc, + dmc=dmc, + dc=dc, + bui=bui, + wind_speed=wind_speed, + wind_direction=wind_direction, + percentage_conifer=percentage_conifer, + percentage_dead_balsam_fir=percentage_dead_balsam_fir, + grass_cure=grass_cure, + crown_base_height=crown_base_height, + ) + check_metric("ROS", fuel_type, python_fba.ros, java_fbp.ros_eq, ros_em, "REDapp") + check_metric('ROS_t', fuel_type, python_fba.ros_t, java_fbp.ros_t, acceptable_margin_of_error,"REDapp") + check_metric('CFB', fuel_type, python_fba.cfb, java_fbp.cfb / 100.0, cfb_em, "REDapp") # CFFDRS gives cfb as a fraction + check_metric('CFB_t', fuel_type, python_fba.cfb_t, java_fbp.cfb / 100.0, cfb_em, "REDapp") # CFFDRS gives cfb as a fraction + check_metric('HFI', fuel_type, python_fba.hfi, java_fbp.hfi, hfi_em, "REDapp") + check_metric('HFI_t', fuel_type, python_fba.hfi_t, java_fbp.hfi, acceptable_margin_of_error, "REDapp") + check_metric('1 HR Size', fuel_type, python_fba.sixty_minute_fire_size, java_fbp.area, one_hr_em, "REDapp") + check_metric('1 HR Size t', fuel_type, python_fba.sixty_minute_fire_size_t, java_fbp.area, fire_size_acceptable_margin_of_error, "REDapp") \ No newline at end of file diff --git a/api/pyproject.toml b/api/pyproject.toml index aded22c23..95abc7e00 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -71,6 +71,7 @@ requires = ["poetry>=1.1.11"] build-backend = "poetry.masonry.api" [tool.ruff] +exclude = ["app/tests/fba_calc/test_fba_error_redapp.py"] per-file-ignores = { "alembic/versions/00df3c7b5cba_rethink_classification.py" = [ "E501", ], "alembic/versions/39806f02cdec_wfwx_update_date_part_of_unique_.py" = [ @@ -83,6 +84,8 @@ per-file-ignores = { "alembic/versions/00df3c7b5cba_rethink_classification.py" = "E501", ], "app/tests/fba_calc/test_fba_error.py" = [ "E501", +], "app/tests/fba_calc/test_fba_error_redapp.py" = [ + "E501", ], "app/tests/hfi/test_hfi.py" = [ "F811", ], "app/auto_spatial_advisory/nats_consumer.py" = [ From 3466da4e0469ef055e73aacac18493e0a1eebae7 Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Wed, 23 Oct 2024 17:19:55 -0700 Subject: [PATCH 2/2] undo config change --- api/pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/pyproject.toml b/api/pyproject.toml index 95abc7e00..b7131ffa0 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -84,8 +84,6 @@ per-file-ignores = { "alembic/versions/00df3c7b5cba_rethink_classification.py" = "E501", ], "app/tests/fba_calc/test_fba_error.py" = [ "E501", -], "app/tests/fba_calc/test_fba_error_redapp.py" = [ - "E501", ], "app/tests/hfi/test_hfi.py" = [ "F811", ], "app/auto_spatial_advisory/nats_consumer.py" = [