From 318dc7c93ad5e605632d3c660832b9003decd4be Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Mon, 25 Nov 2024 14:26:36 -0800 Subject: [PATCH] Refactor FBA random sample tests (#4134) Remove pytest-bdd and replace with parametrized tests --- .../test_fba_error_random_sample.feature | 47 --- .../fba_calc/test_fba_error_random_sample.py | 350 ++++++------------ 2 files changed, 123 insertions(+), 274 deletions(-) delete mode 100644 api/app/tests/fba_calc/test_fba_error_random_sample.feature diff --git a/api/app/tests/fba_calc/test_fba_error_random_sample.feature b/api/app/tests/fba_calc/test_fba_error_random_sample.feature deleted file mode 100644 index 8ab3f9322..000000000 --- a/api/app/tests/fba_calc/test_fba_error_random_sample.feature +++ /dev/null @@ -1,47 +0,0 @@ -Feature: /fbc/ - - Scenario: Fire Behaviour Calculation - Given , , , and for - Then ROS is within compared to REDapp - And HFI is within compared to REDapp - And CFB is within compared to REDapp - And 1 Hour Spread is within compared to REDapp - And ROS_t is within range - And CFB_t is within range of compared to REDapp - And HFI_t is within range - And (1 HR Size)_t is within range - And Log it - - Examples: - | fuel_type | percentage_conifer | percentage_dead_balsam_fir | grass_cure | crown_base_height | ros_margin_of_error | hfi_margin_of_error | cfb_margin_of_error | cfb_t_margin_of_error | one_hour_spread_margin_of_error | num_iterations | - | C1 | 100 | None | None | 2 | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - | C2 | 100 | None | None | 3 | 0.01 | 0.12 | 0.19 | 0.01 | 0.17 | 20 | - | C3 | 100 | None | None | 8 | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - # C4 and C5 seem to have some issues with CFB - # | C4 | 100 | None | None | 8 | 0.01 | 0.02 | 1.00 | 0.01 | 0.28 | 20 | - # | C5 | 100 | None | None | 8 | 0.01 | 0.01 | 0.04 | 0.01 | 0.01 | 20 | - | C6 | 100 | None | None | 7 | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - | C7 | 100 | None | None | 8 | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - | D1 | 100 | None | None | None | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - # M1, M2, M3 & M4 are failing on 1Ha Fire Size (though not that bad!) - # | M1_75C | 75 | None | None | 6 | 0.01 | 0.02 | 0.02 | 0.01 | 0.05 | 20 | - # | M1_50C | 50 | None | None | 6 | 0.01 | 0.21 | 0.61 | 0.01 | 0.10 | 20 | - | M1 | 25 | None | None | 6 | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - # | M2_75C | 75 | None | None | 6 | 0.01 | 0.03 | 0.03 | 0.01 | 0.07 | 20 | - # | M2_50C | 50 | None | None | 6 | 0.01 | 0.11 | 0.39 | 0.01 | 0.13 | 20 | - | M2 | 25 | None | None | 6 | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - | M3 | None | 30 | None | 6 | 0.01 | 0.01 | 0.19 | 0.12 | 0.01 | 20 | - | M3 | None | 60 | None | 6 | 0.01 | 0.19 | 0.48 | 0.01 | 0.35 | 20 | - # | M3_100D | None | 100 | None | 6 | 0.01 | 0.17 | 0.32 | 0.01 | 0.38 | 20 | - # | M4_30D | None | 30 | None | 6 | 0.01 | 0.19 | 0.36 | 0.01 | 0.21 | 20 | - | M4 | None | 60 | None | 6 | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - | M4 | None | 100 | None | 6 | 0.01 | 0.03 | 0.12 | 0.01 | 0.02 | 20 | - | O1A | None | None | 25 | None | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - | O1A | None | None | 50 | None | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - | O1A | None | None | 100 | None | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - | O1B | None | None | 25 | None | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - | O1B | None | None | 50 | None | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - | O1B | None | None | 100 | None | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - | S1 | None | None | None | None | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - | S2 | None | None | None | None | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | - | S3 | None | None | None | None | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | diff --git a/api/app/tests/fba_calc/test_fba_error_random_sample.py b/api/app/tests/fba_calc/test_fba_error_random_sample.py index 0dd0c38e7..7f2806ee7 100644 --- a/api/app/tests/fba_calc/test_fba_error_random_sample.py +++ b/api/app/tests/fba_calc/test_fba_error_random_sample.py @@ -1,11 +1,11 @@ """ -Unit tests for fire behavour calculator. +Unit tests for fire behaviour calculator. """ + from datetime import datetime, timezone as dt_tz import random from typing import Final import logging -from pytest_bdd import scenario, given, then, parsers from app import configure_logging from app.schemas.fba_calc import FuelTypeEnum from app.utils.time import get_hour_20_from_date @@ -13,7 +13,6 @@ from app.fire_behaviour.cffdrs import fire_weather_index, initial_spread_index, bui_calc from app.utils.redapp import FBPCalculateStatisticsCOM from app.tests.fba_calc import check_metric, fire_size_acceptable_margin_of_error -from app.tests.common import str2float import pytest @@ -23,41 +22,68 @@ def _random_date(): - start = datetime.fromisoformat('2021-01-01') - end = datetime.fromisoformat('2021-12-31') + start = datetime.fromisoformat("2021-01-01") + end = datetime.fromisoformat("2021-12-31") return datetime.fromtimestamp(random.uniform(start.timestamp(), end.timestamp()), tz=dt_tz.utc) acceptable_margin_of_error: Final = 0.01 -@pytest.mark.usefixtures('mock_jwt_decode') -@scenario('test_fba_error_random_sample.feature', 'Fire Behaviour Calculation') -def test_fire_behaviour_calculator_scenario(): - """ BDD Scenario. """ - - -@given( - parsers.parse("""{fuel_type}, {percentage_conifer}, {percentage_dead_balsam_fir}, {grass_cure} and """ - """{crown_base_height} for {num_iterations}"""), - converters=dict(crown_base_height=str2float, - fuel_type=str, - percentage_conifer=str2float, - percentage_dead_balsam_fir=str2float, - grass_cure=str2float, - num_iterations=int), - target_fixture='results' +@pytest.mark.parametrize( + "fuel_type, percentage_conifer, percentage_dead_balsam_fir, grass_cure, crown_base_height, ros_margin_of_error, hfi_margin_of_error, cfb_margin_of_error, cfb_t_margin_of_error, one_hour_spread_margin_of_error, num_iterations,", + [ + ("C1", 100, None, None, 2, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + ("C2", 100, None, None, 3, 0.01, 0.12, 0.19, 0.01, 0.17, 20), + ("C3", 100, None, None, 8, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + # C4 and C5 seem to have some issues with CFB + # | C4 | 100 | None | None | 8 | 0.01 | 0.02 | 1.00 | 0.01 | 0.28 | 20 | + # | C5 | 100 | None | None | 8 | 0.01 | 0.01 | 0.04 | 0.01 | 0.01 | 20 | | C6 | 100 | None | None | 7 | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 | 20 | + ("C7", 100, None, None, 8, 0.01, 0.01, 0.01, 0.01, 0.01, 0), + ("D1", 100, None, None, None, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + # M1, M2, M3 & M4 are failing on 1Ha Fire Size (though not that bad!) + # | M1_75C | 75 | None | None | 6 | 0.01 | 0.02 | 0.02 | 0.01 | 0.05 | 20 | + # | M1_50C | 50 | None | None | 6 | 0.01 | 0.21 | 0.61 | 0.01 | 0.10 | 20 | + ("M1", 25, None, None, 6, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + # | M2_75C | 75 | None | None | 6 | 0.01 | 0.03 | 0.03 | 0.01 | 0.07 | 20 | + # | M2_50C | 50 | None | None | 6 | 0.01 | 0.11 | 0.39 | 0.01 | 0.13 | 20 | + ("M2", 25, None, None, 6, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + ("M3", None, 30, None, 6, 0.01, 0.01, 0.19, 0.12, 0.01, 20), + ("M3", None, 60, None, 6, 0.01, 0.19, 0.48, 0.01, 0.35, 20), + # | M3_100D | None | 100 | None | 6 | 0.01 | 0.17 | 0.32 | 0.01 | 0.38 | 20 | + # | M4_30D | None | 30 | None | 6 | 0.01 | 0.19 | 0.36 | 0.01 | 0.21 | 20 | + ("M4", None, 60, None, 6, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + ("M4", None, 100, None, 6, 0.01, 0.03, 0.12, 0.01, 0.02, 20), + ("O1A", None, None, 25, None, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + ("O1A", None, None, 50, None, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + ("O1A", None, None, 100, None, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + ("O1B", None, None, 25, None, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + ("O1B", None, None, 50, None, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + ("O1B", None, None, 100, None, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + ("S1", None, None, None, None, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + ("S2", None, None, None, None, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + ("S3", None, None, None, None, 0.01, 0.01, 0.01, 0.01, 0.01, 20), + ], ) -def given_input(fuel_type: str, percentage_conifer: float, percentage_dead_balsam_fir: float, - grass_cure: float, crown_base_height: float, num_iterations: int): - """ Take input and calculate actual and expected results """ - +def test_get_endpoints_unauthorized( + fuel_type, + percentage_conifer, + percentage_dead_balsam_fir, + grass_cure, + crown_base_height, + ros_margin_of_error, + hfi_margin_of_error, + cfb_margin_of_error, + cfb_t_margin_of_error, + one_hour_spread_margin_of_error, + num_iterations, +): + """Calculate actual and expected outputs""" # get python result: # seed = time() seed = 43 - logger.info('using random seed: %s', seed) + logger.info("using random seed: %s", seed) random.seed(seed) - results = [] for index in range(num_iterations): elevation = random.randint(0, 4019) latitude = random.uniform(45, 60) @@ -80,211 +106,81 @@ def given_input(fuel_type: str, percentage_conifer: float, percentage_dead_balsa isi = initial_spread_index(ffmc, wind_speed) fwi = fire_weather_index(isi, bui) - message = (f"""({index}) elevation:{elevation} ; lat: {latitude} ; lon: {longitude}; """ - f"""toi: {time_of_interest}; ws: {wind_speed}; wd: {wind_direction}; """ - f"""temperature: {temperature}; relative_humidity: {relative_humidity}; """ - f"""precipitation: {precipitation}; dc: {dc}; dmc: {dmc}; bui: {bui}; """ - f"""ffmc: {ffmc}; isi: {isi}""") + message = ( + f"""({index}) elevation:{elevation} ; lat: {latitude} ; lon: {longitude}; """ + f"""toi: {time_of_interest}; ws: {wind_speed}; wd: {wind_direction}; """ + f"""temperature: {temperature}; relative_humidity: {relative_humidity}; """ + f"""precipitation: {precipitation}; dc: {dc}; dmc: {dmc}; bui: {bui}; """ + f"""ffmc: {ffmc}; isi: {isi}""" + ) logger.debug(message) - test_entry = (f"""({index}) | {fuel_type} | {elevation} | {latitude} | {longitude} | """ - f"""{time_of_interest} | {wind_speed} | {wind_direction} | {percentage_conifer} | """ - f"""{percentage_dead_balsam_fir} | {grass_cure} | {crown_base_height} | {isi} | """ - f"""{bui} | {ffmc} | {dmc} | {dc} | 0.01 | 0.01 | 0.01 | 0.01 | None | 0.01 | """ - f"""None | 0.01 | None | 0.01 | |""") + test_entry = ( + f"""({index}) | {fuel_type} | {elevation} | {latitude} | {longitude} | """ + f"""{time_of_interest} | {wind_speed} | {wind_direction} | {percentage_conifer} | """ + f"""{percentage_dead_balsam_fir} | {grass_cure} | {crown_base_height} | {isi} | """ + f"""{bui} | {ffmc} | {dmc} | {dc} | 0.01 | 0.01 | 0.01 | 0.01 | None | 0.01 | """ + f"""None | 0.01 | None | 0.01 | |""" + ) logger.debug(test_entry) - python_input = FBACalculatorWeatherStation(elevation=elevation, - fuel_type=FuelTypeEnum[fuel_type], - time_of_interest=time_of_interest, - percentage_conifer=percentage_conifer, - percentage_dead_balsam_fir=percentage_dead_balsam_fir, - grass_cure=grass_cure, - crown_base_height=crown_base_height, - crown_fuel_load=None, - lat=latitude, - long=longitude, - bui=bui, - ffmc=ffmc, - isi=isi, - fwi=fwi, - wind_speed=wind_speed, - wind_direction=wind_direction, - temperature=temperature, - relative_humidity=relative_humidity, - precipitation=precipitation, - status='Forecasted', - prev_day_daily_ffmc=None, - last_observed_morning_rh_values=None) + python_input = FBACalculatorWeatherStation( + elevation=elevation, + fuel_type=FuelTypeEnum[fuel_type], + time_of_interest=time_of_interest, + percentage_conifer=percentage_conifer, + percentage_dead_balsam_fir=percentage_dead_balsam_fir, + grass_cure=grass_cure, + crown_base_height=crown_base_height, + crown_fuel_load=None, + lat=latitude, + long=longitude, + bui=bui, + ffmc=ffmc, + isi=isi, + fwi=fwi, + wind_speed=wind_speed, + wind_direction=wind_direction, + temperature=temperature, + relative_humidity=relative_humidity, + precipitation=precipitation, + status="Forecasted", + prev_day_daily_ffmc=None, + last_observed_morning_rh_values=None, + ) python_fba = calculate_fire_behaviour_advisory(python_input) # get REDapp result from java: - java_fbp = FBPCalculateStatisticsCOM(elevation=elevation, - latitude=latitude, - longitude=longitude, - time_of_interest=get_hour_20_from_date(time_of_interest), - fuel_type=fuel_type, - ffmc=ffmc, - dmc=dmc, - dc=dc, - bui=bui, - wind_speed=wind_speed, - wind_direction=wind_direction, - percentage_conifer=percentage_conifer, - percentage_dead_balsam_fir=percentage_dead_balsam_fir, - grass_cure=grass_cure, - crown_base_height=crown_base_height) - - error_dict = { - 'fuel_type': fuel_type - } - results.append( - { - 'input': {'isi': isi, 'bui': bui, 'wind_speed': wind_speed, 'ffmc': ffmc}, - 'python': python_fba, - 'java': java_fbp, - 'fuel_type': fuel_type, - 'error': error_dict - } + java_fbp = FBPCalculateStatisticsCOM( + elevation=elevation, + latitude=latitude, + longitude=longitude, + time_of_interest=get_hour_20_from_date(time_of_interest), + fuel_type=fuel_type, + ffmc=ffmc, + dmc=dmc, + dc=dc, + bui=bui, + wind_speed=wind_speed, + wind_direction=wind_direction, + percentage_conifer=percentage_conifer, + percentage_dead_balsam_fir=percentage_dead_balsam_fir, + grass_cure=grass_cure, + crown_base_height=crown_base_height, ) - return results - - -@then(parsers.parse("ROS is within {ros_margin_of_error} compared to REDapp"), - converters={'ros_margin_of_error': float}) -def then_ros_good(results: list, ros_margin_of_error: float): - """ check the relative error of ROS """ - for index, result in enumerate(results): - wind_speed = result['input']['wind_speed'] - ffmc = result['input']['ffmc'] - isi = result['input']['isi'] - bui = result['input']['bui'] - java_isi = result['java'].isi - # assumptions: - # ros_eq == ROScalc - # ros_t == ROStcalc - error = check_metric('ROS', - result['fuel_type'], - result['python'].ros, - result['java'].ros_eq, - ros_margin_of_error, - f"""({index}) input- isi:{isi}; bui:{bui}; wind_speed:{wind_speed}; ffmc:{ffmc}; """ - f"""java - isi:{java_isi}""") - result['error']['ros_margin_of_error'] = error - - -@then("ROS_t is within range") -def then_ros_t(results: list): - """ check the relative error of the ros """ - for result in results: - check_metric('ROS_t', - result['fuel_type'], - result['python'].ros_t, - result['java'].ros_t, - acceptable_margin_of_error) - - -@then(parsers.parse("HFI is within {hfi_margin_of_error} compared to REDapp"), - converters={'hfi_margin_of_error': float}) -def then_hfi_good(results: list, hfi_margin_of_error: float): - """ check the relative error of HFI """ - for index, result in enumerate(results): - error = check_metric('HFI', - result['fuel_type'], - result['python'].hfi, - result['java'].hfi, - hfi_margin_of_error, - f'({index})') - result['error']['hfi_margin_of_error'] = error - - -@then("HFI_t is within range") -def then_hfi_t(results: list): - """ check the relative error of the ros """ - for result in results: - check_metric('HFI_t', - result['fuel_type'], - result['python'].hfi_t, - result['java'].hfi, - acceptable_margin_of_error) - - -@then(parsers.parse("CFB is within {cfb_margin_of_error} compared to REDapp"), - converters={'cfb_margin_of_error': float}) -def then_cfb_good(results: list, cfb_margin_of_error: float): - """ check the relative error of HFI """ - for index, result in enumerate(results): - error = check_metric('CFB', - result['fuel_type'], - result['python'].cfb * 100.0, - result['java'].cfb, cfb_margin_of_error, - f'({index})') - result['error']['cfb_margin_of_error'] = error - - -@then(parsers.parse("CFB_t is within range of {cfb_t_margin_of_error} compared to REDapp"), - converters={'cfb_t_margin_of_error': float}) -def then_cfb_t(results: list, cfb_t_margin_of_error: float): - """ check the relative error of the ros """ - for result in results: - check_metric('CFB_t', - result['fuel_type'], - result['python'].cfb_t * 100.0, - result['java'].cfb, - cfb_t_margin_of_error) - - -@then(parsers.parse("1 Hour Spread is within {one_hour_spread_margin_of_error} compared to REDapp"), - converters={'one_hour_spread_margin_of_error': float}) -def then_1_hour_spread_good(results: list, one_hour_spread_margin_of_error: float): - """ check the relative error of HFI """ - for index, result in enumerate(results): - error = check_metric('one_hour_size', - result['fuel_type'], - result['python'].sixty_minute_fire_size, - result['java'].area, one_hour_spread_margin_of_error, - f'({index})') - result['error']['one_hour_spread_margin_of_error'] = error - - -@then("(1 HR Size)_t is within range") -def then_one_hour_size_t(results: list): - """ check the relative error of the ros """ - for result in results: - check_metric('1 HR Size t', - result['fuel_type'], - result['python'].sixty_minute_fire_size_t, - result['java'].area, - fire_size_acceptable_margin_of_error) - - -@then("Log it") -def log_it(results: list): - """ Log a string matching the scenario input - useful when improving values. """ - keys = ('one_hour_spread_margin_of_error', 'cfb_margin_of_error', - 'hfi_margin_of_error', 'ros_margin_of_error') - values = {} - for key in keys: - values[key] = 0.01 - for result in results: - error_dict = result.get('error') - for key in keys: - old_value = values.get(key) - if error_dict[key] > old_value: - values[key] = error_dict[key] - values['fuel_type'] = error_dict['fuel_type'] - - header = ("""| fuel_type | percentage_conifer | percentage_dead_balsam_fir | grass_cure | """ - """crown_base_height | ros_margin_of_error | hfi_margin_of_error | cfb_margin_of_error | """ - """one_hour_spread_margin_of_error | num_iterations |""") - header = header.strip('|') - header = header.split('|') - header = [x.strip() for x in header] + check_metric( + "ROS", + fuel_type, + python_fba.ros, + java_fbp.ros_eq, + ros_margin_of_error, + f"""({index}) input- isi:{isi}; bui:{bui}; wind_speed:{wind_speed}; ffmc:{ffmc}; """ f"""java - isi:{java_fbp.isi}""", + ) - line = '|' - for key in header: - value = values.get(key, key) - if isinstance(value, float): - line += f'{value:.2f}|' - else: - line += f'{value}|' - logger.debug(line) + check_metric("ROS_t", fuel_type, python_fba.ros_t, java_fbp.ros_t, acceptable_margin_of_error) + check_metric("HFI", fuel_type, python_fba.hfi, java_fbp.hfi, hfi_margin_of_error, f"({index})") + check_metric("HFI", fuel_type, python_fba.hfi, java_fbp.hfi, hfi_margin_of_error, f"({index})") + check_metric("HFI_t", fuel_type, python_fba.hfi_t, java_fbp.hfi, acceptable_margin_of_error) + check_metric("CFB", fuel_type, python_fba.cfb * 100.0, java_fbp.cfb, cfb_margin_of_error, f"({index})") + check_metric("CFB_t", fuel_type, python_fba.cfb_t * 100.0, java_fbp.cfb, cfb_t_margin_of_error) + check_metric("one_hour_size", fuel_type, python_fba.sixty_minute_fire_size, java_fbp.area, one_hour_spread_margin_of_error, f"({index})") + check_metric("1 HR Size t", fuel_type, python_fba.sixty_minute_fire_size_t, java_fbp.area, fire_size_acceptable_margin_of_error)