From ef09612c6c0cb3c3e118a7c43b40717a20a89e30 Mon Sep 17 00:00:00 2001 From: Eva Sebestova <73821679+esebesto@users.noreply.github.com> Date: Tue, 26 Mar 2024 14:41:17 +0100 Subject: [PATCH] [COST-4175] Fix AWS first of the month issue (#487) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Just get the values view Do not both returning and then disccarding the keys view. * Simplfy generated_end_date determination - Use nested .get() calls instead of conditionals - Store the value instead of doing repeated attribute lookups - Default to today() * Reduce attribute lookups and improve legibility It’s expensive in CPython to keep looking up attributes, plus it hurts legibility greatly here. Lookup up the attributes once and store the values to make the conditionals much easier to read and improve performance. Compare both the month and day of month_end and end_date in order to determine if they are the same. It may be sufficient to compare only day, but it seems. more accurate to compare both month and day. * Increase test coverage * Extend test_create_generator_for_dates_from_yaml_first_month Co-authored-by: Sam Doran --- nise/__init__.py | 2 +- nise/__main__.py | 21 ++++++++---- nise/report.py | 47 +++++++++++++++------------ tests/test_report.py | 76 ++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 115 insertions(+), 31 deletions(-) diff --git a/nise/__init__.py b/nise/__init__.py index 85525933..13044923 100644 --- a/nise/__init__.py +++ b/nise/__init__.py @@ -1,3 +1,3 @@ -__version__ = "4.4.15" +__version__ = "4.4.16" VERSION = __version__.split(".") diff --git a/nise/__main__.py b/nise/__main__.py index 73a3c97f..046c1776 100644 --- a/nise/__main__.py +++ b/nise/__main__.py @@ -693,18 +693,25 @@ def _load_static_report_data(options): end_dates = [] static_report_data = load_yaml(static_file) for generator_dict in static_report_data.get("generators"): - for _, attributes in generator_dict.items(): + for attributes in generator_dict.values(): start_date = get_start_date(attributes, options) generated_start_date = calculate_start_date(start_date) start_dates.append(generated_start_date) - if attributes.get("end_date"): - generated_end_date = calculate_end_date(generated_start_date, attributes.get("end_date")) - elif options.get("end_date") and options.get("end_date").date() != today().date(): - generated_end_date = calculate_end_date(generated_start_date, options.get("end_date")) - else: - generated_end_date = today() + + end_date = attributes.get("end_date", options.get("end_date")) + generated_end_date = today() + if end_date and ( + end_date != today().date() + or ( + isinstance(end_date, datetime.datetime) + and (end_date.date() != today().date() or end_date.hour != 0) + ) + ): + generated_end_date = calculate_end_date(generated_start_date, end_date) + if options.get("provider") == "azure": generated_end_date += datetime.timedelta(hours=24) + end_dates.append(generated_end_date) attributes["start_date"] = str(generated_start_date) diff --git a/nise/report.py b/nise/report.py index 54d3ad57..e08f44a7 100644 --- a/nise/report.py +++ b/nise/report.py @@ -501,33 +501,39 @@ def _create_generator_dates_from_yaml(attributes, month): gen_start_date = None gen_end_date = None + start_date = attributes.get("start_date") + end_date = attributes.get("end_date") + month_start = month.get("start") + month_end = month.get("end") + + # Use a different variable for the end of month comparison. This matters + # when the end_date is the same as the month_end, which can happen based + # on the specified --end-date parameter or on the first of the month. + month_end_compare = month_end + if month_end.day != 1: + # Create a new datetime object and store it, leaving the original + # month_end object unmodified. + month_end_compare = month_end_compare.replace(hour=23, minute=59, second=59) + # Generator range is larger then current month on both start and end - if attributes.get("start_date") < month.get("start") and attributes.get("end_date") > month.get("end").replace( - hour=23, minute=59, second=59 - ): - gen_start_date = month.get("start") - gen_end_date = month.get("end") + if start_date < month_start and end_date > month_end_compare: + gen_start_date = month_start + gen_end_date = month_end # Generator starts before month start and ends within month - if attributes.get("start_date") <= month.get("start") and attributes.get("end_date") <= month.get("end").replace( - hour=23, minute=59, second=59 - ): - gen_start_date = month.get("start") - gen_end_date = attributes.get("end_date") + elif start_date < month_start and end_date <= month_end_compare: + gen_start_date = month_start + gen_end_date = end_date # Generator is within month - if attributes.get("start_date") >= month.get("start") and attributes.get("end_date") <= month.get("end").replace( - hour=23, minute=59, second=59 - ): - gen_start_date = attributes.get("start_date") - gen_end_date = attributes.get("end_date") + elif start_date >= month_start and end_date <= month_end_compare: + gen_start_date = start_date + gen_end_date = end_date # Generator starts within month and ends in next month - if attributes.get("start_date") >= month.get("start") and attributes.get("end_date") > month.get("end").replace( - hour=23, minute=59, second=59 - ): - gen_start_date = attributes.get("start_date") - gen_end_date = month.get("end") + elif start_date >= month_start and end_date > month_end_compare: + gen_start_date = start_date + gen_end_date = month_end return gen_start_date, gen_end_date @@ -593,7 +599,6 @@ def aws_create_marketplace_report(options): # noqa: C901 def aws_create_report(options): # noqa: C901 """Create a cost usage report file.""" - data = [] start_date = options.get("start_date") end_date = options.get("end_date") aws_finalize_report = options.get("aws_finalize_report") diff --git a/tests/test_report.py b/tests/test_report.py index eae39723..0bd419d9 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -120,7 +120,6 @@ def test_write_manifest(self): def test_create_generator_for_dates_from_yaml(self): """Test helper function for generating dates.""" month = { - "name": "June", "start": datetime.datetime(2021, 6, 30, 0, 0), "end": datetime.datetime(2021, 6, 30, 23, 59), } @@ -135,10 +134,83 @@ def test_create_generator_for_dates_from_yaml(self): self.assertEqual(start_date, datetime.datetime(2021, 6, 30, 0, 0)) self.assertEqual(end_date, datetime.datetime(2021, 6, 30, 23, 59)) + def test_create_generator_for_dates_from_yaml_within_month(self): + month = { + "start": datetime.datetime(2023, 5, 1, 0, 0), + "end": datetime.datetime(2023, 5, 31, 23, 59), + } + + attributes = { + "start_date": datetime.datetime(2023, 5, 5, 0, 0), + "end_date": datetime.datetime(2023, 5, 15, 15, 0), + } + + start_date, end_date = _create_generator_dates_from_yaml(attributes, month) + + self.assertEqual(start_date, datetime.datetime(2023, 5, 5, 0, 0)) + self.assertEqual(end_date, datetime.datetime(2023, 5, 15, 15, 0)) + + def test_create_generator_for_dates_from_yaml_within_month_before_month_start(self): + month = { + "start": datetime.datetime(2023, 5, 1, 0, 0), + "end": datetime.datetime(2023, 5, 31, 23, 59), + } + + attributes = { + "start_date": datetime.datetime(2023, 4, 5, 0, 0), + "end_date": datetime.datetime(2023, 5, 15, 15, 0), + } + + start_date, end_date = _create_generator_dates_from_yaml(attributes, month) + + self.assertEqual(start_date, datetime.datetime(2023, 5, 1, 0, 0)) + self.assertEqual(end_date, datetime.datetime(2023, 5, 15, 15, 0)) + + def test_create_generator_for_dates_from_yaml_within_month_ends_next_month(self): + month = { + "start": datetime.datetime(2023, 5, 1, 0, 0), + "end": datetime.datetime(2023, 5, 31, 23, 59), + } + + attributes = { + "start_date": datetime.datetime(2023, 5, 5, 0, 0), + "end_date": datetime.datetime(2023, 8, 15, 15, 0), + } + + start_date, end_date = _create_generator_dates_from_yaml(attributes, month) + + self.assertEqual(start_date, datetime.datetime(2023, 5, 5, 0, 0)) + self.assertEqual(end_date, datetime.datetime(2023, 5, 31, 23, 59)) + + def test_create_generator_for_dates_from_yaml_first_month(self): + """Test that correct dates are generated on the first of the month""" + previous_month = { + "start": datetime.datetime(2024, 2, 1, 0, 0, tzinfo=datetime.timezone.utc), + "end": datetime.datetime(2024, 3, 1, 0, 0, tzinfo=datetime.timezone.utc), + } + current_month = { + "start": datetime.datetime(2024, 3, 1, 0, 0, tzinfo=datetime.timezone.utc), + "end": datetime.datetime(2024, 4, 1, 0, 0, tzinfo=datetime.timezone.utc), + } + + attributes = { + "start_date": datetime.datetime(2024, 2, 1, 0, 0, tzinfo=datetime.timezone.utc), + "end_date": datetime.datetime(2024, 3, 1, 23, 0, tzinfo=datetime.timezone.utc), + } + + start_date_previous, end_date_previous = _create_generator_dates_from_yaml(attributes, previous_month) + + self.assertEqual(start_date_previous, datetime.datetime(2024, 2, 1, 0, 0, tzinfo=datetime.timezone.utc)) + self.assertEqual(end_date_previous, datetime.datetime(2024, 3, 1, 0, 0, tzinfo=datetime.timezone.utc)) + + start_date_current, end_date_current = _create_generator_dates_from_yaml(attributes, current_month) + + self.assertEqual(start_date_current, datetime.datetime(2024, 3, 1, 0, 0, tzinfo=datetime.timezone.utc)) + self.assertEqual(end_date_current, datetime.datetime(2024, 3, 1, 23, 0, tzinfo=datetime.timezone.utc)) + def test_create_generator_for_dates_from_yaml_middle_month(self): """Test helper function for generating dates verifying the middle month in a 3 month range.""" month = { - "name": "June", "start": datetime.datetime(2021, 6, 30, 0, 0), "end": datetime.datetime(2021, 6, 30, 23, 59), }