[COST-4175] Fix AWS first of the month issue (#487)

* Just get the values view
  Do not both returning and then disccarding the keys view.

* Simplfy generated_end_date determination
- Use nested .get() calls instead of conditionals
- Store the value instead of doing repeated attribute lookups
- Default to today()

* Reduce attribute lookups and improve legibility
  It’s expensive in CPython to keep looking up attributes, plus it hurts
  legibility greatly here. Lookup up the attributes once and store the
  values to make the conditionals much easier to read and improve performance.

  Compare both the month and day of month_end and end_date in order to determine
  if they are the same. It may be sufficient to compare only day, but it seems.
  more accurate to compare both month and day.

* Increase test coverage
* Extend test_create_generator_for_dates_from_yaml_first_month

Co-authored-by: Sam Doran <[email protected]>

nise/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "4.4.15"
+__version__ = "4.4.16"
 
 VERSION = __version__.split(".")

nise/__main__.py

@@ -693,18 +693,25 @@ def _load_static_report_data(options):
     end_dates = []
     static_report_data = load_yaml(static_file)
     for generator_dict in static_report_data.get("generators"):
-        for _, attributes in generator_dict.items():
+        for attributes in generator_dict.values():
             start_date = get_start_date(attributes, options)
             generated_start_date = calculate_start_date(start_date)
             start_dates.append(generated_start_date)
-            if attributes.get("end_date"):
-                generated_end_date = calculate_end_date(generated_start_date, attributes.get("end_date"))
-            elif options.get("end_date") and options.get("end_date").date() != today().date():
-                generated_end_date = calculate_end_date(generated_start_date, options.get("end_date"))
-            else:
-                generated_end_date = today()
+
+            end_date = attributes.get("end_date", options.get("end_date"))
+            generated_end_date = today()
+            if end_date and (
+                end_date != today().date()
+                or (
+                    isinstance(end_date, datetime.datetime)
+                    and (end_date.date() != today().date() or end_date.hour != 0)
+                )
+            ):
+                generated_end_date = calculate_end_date(generated_start_date, end_date)
+
             if options.get("provider") == "azure":
                 generated_end_date += datetime.timedelta(hours=24)
+
             end_dates.append(generated_end_date)
 
             attributes["start_date"] = str(generated_start_date)

nise/report.py

@@ -501,33 +501,39 @@ def _create_generator_dates_from_yaml(attributes, month):
     gen_start_date = None
     gen_end_date = None
 
+    start_date = attributes.get("start_date")
+    end_date = attributes.get("end_date")
+    month_start = month.get("start")
+    month_end = month.get("end")
+
+    # Use a different variable for the end of month comparison. This matters
+    # when the end_date is the same as the month_end, which can happen based
+    # on the specified --end-date parameter or on the first of the month.
+    month_end_compare = month_end
+    if month_end.day != 1:
+        # Create a new datetime object and store it, leaving the original
+        # month_end object unmodified.
+        month_end_compare = month_end_compare.replace(hour=23, minute=59, second=59)
+
     # Generator range is larger then current month on both start and end
-    if attributes.get("start_date") < month.get("start") and attributes.get("end_date") > month.get("end").replace(
-        hour=23, minute=59, second=59
-    ):
-        gen_start_date = month.get("start")
-        gen_end_date = month.get("end")
+    if start_date < month_start and end_date > month_end_compare:
+        gen_start_date = month_start
+        gen_end_date = month_end
 
     # Generator starts before month start and ends within month
-    if attributes.get("start_date") <= month.get("start") and attributes.get("end_date") <= month.get("end").replace(
-        hour=23, minute=59, second=59
-    ):
-        gen_start_date = month.get("start")
-        gen_end_date = attributes.get("end_date")
+    elif start_date < month_start and end_date <= month_end_compare:
+        gen_start_date = month_start
+        gen_end_date = end_date
 
     # Generator is within month
-    if attributes.get("start_date") >= month.get("start") and attributes.get("end_date") <= month.get("end").replace(
-        hour=23, minute=59, second=59
-    ):
-        gen_start_date = attributes.get("start_date")
-        gen_end_date = attributes.get("end_date")
+    elif start_date >= month_start and end_date <= month_end_compare:
+        gen_start_date = start_date
+        gen_end_date = end_date
 
     # Generator starts within month and ends in next month
-    if attributes.get("start_date") >= month.get("start") and attributes.get("end_date") > month.get("end").replace(
-        hour=23, minute=59, second=59
-    ):
-        gen_start_date = attributes.get("start_date")
-        gen_end_date = month.get("end")
+    elif start_date >= month_start and end_date > month_end_compare:
+        gen_start_date = start_date
+        gen_end_date = month_end
 
     return gen_start_date, gen_end_date
 
@@ -593,7 +599,6 @@ def aws_create_marketplace_report(options):  # noqa: C901
 
 def aws_create_report(options):  # noqa: C901
     """Create a cost usage report file."""
-    data = []
     start_date = options.get("start_date")
     end_date = options.get("end_date")
     aws_finalize_report = options.get("aws_finalize_report")

tests/test_report.py

@@ -120,7 +120,6 @@ def test_write_manifest(self):
     def test_create_generator_for_dates_from_yaml(self):
         """Test helper function for generating dates."""
         month = {
-            "name": "June",
             "start": datetime.datetime(2021, 6, 30, 0, 0),
             "end": datetime.datetime(2021, 6, 30, 23, 59),
         }
@@ -135,10 +134,83 @@ def test_create_generator_for_dates_from_yaml(self):
         self.assertEqual(start_date, datetime.datetime(2021, 6, 30, 0, 0))
         self.assertEqual(end_date, datetime.datetime(2021, 6, 30, 23, 59))
 
+    def test_create_generator_for_dates_from_yaml_within_month(self):
+        month = {
+            "start": datetime.datetime(2023, 5, 1, 0, 0),
+            "end": datetime.datetime(2023, 5, 31, 23, 59),
+        }
+
+        attributes = {
+            "start_date": datetime.datetime(2023, 5, 5, 0, 0),
+            "end_date": datetime.datetime(2023, 5, 15, 15, 0),
+        }
+
+        start_date, end_date = _create_generator_dates_from_yaml(attributes, month)
+
+        self.assertEqual(start_date, datetime.datetime(2023, 5, 5, 0, 0))
+        self.assertEqual(end_date, datetime.datetime(2023, 5, 15, 15, 0))
+
+    def test_create_generator_for_dates_from_yaml_within_month_before_month_start(self):
+        month = {
+            "start": datetime.datetime(2023, 5, 1, 0, 0),
+            "end": datetime.datetime(2023, 5, 31, 23, 59),
+        }
+
+        attributes = {
+            "start_date": datetime.datetime(2023, 4, 5, 0, 0),
+            "end_date": datetime.datetime(2023, 5, 15, 15, 0),
+        }
+
+        start_date, end_date = _create_generator_dates_from_yaml(attributes, month)
+
+        self.assertEqual(start_date, datetime.datetime(2023, 5, 1, 0, 0))
+        self.assertEqual(end_date, datetime.datetime(2023, 5, 15, 15, 0))
+
+    def test_create_generator_for_dates_from_yaml_within_month_ends_next_month(self):
+        month = {
+            "start": datetime.datetime(2023, 5, 1, 0, 0),
+            "end": datetime.datetime(2023, 5, 31, 23, 59),
+        }
+
+        attributes = {
+            "start_date": datetime.datetime(2023, 5, 5, 0, 0),
+            "end_date": datetime.datetime(2023, 8, 15, 15, 0),
+        }
+
+        start_date, end_date = _create_generator_dates_from_yaml(attributes, month)
+
+        self.assertEqual(start_date, datetime.datetime(2023, 5, 5, 0, 0))
+        self.assertEqual(end_date, datetime.datetime(2023, 5, 31, 23, 59))
+
+    def test_create_generator_for_dates_from_yaml_first_month(self):
+        """Test that correct dates are generated on the first of the month"""
+        previous_month = {
+            "start": datetime.datetime(2024, 2, 1, 0, 0, tzinfo=datetime.timezone.utc),
+            "end": datetime.datetime(2024, 3, 1, 0, 0, tzinfo=datetime.timezone.utc),
+        }
+        current_month = {
+            "start": datetime.datetime(2024, 3, 1, 0, 0, tzinfo=datetime.timezone.utc),
+            "end": datetime.datetime(2024, 4, 1, 0, 0, tzinfo=datetime.timezone.utc),
+        }
+
+        attributes = {
+            "start_date": datetime.datetime(2024, 2, 1, 0, 0, tzinfo=datetime.timezone.utc),
+            "end_date": datetime.datetime(2024, 3, 1, 23, 0, tzinfo=datetime.timezone.utc),
+        }
+
+        start_date_previous, end_date_previous = _create_generator_dates_from_yaml(attributes, previous_month)
+
+        self.assertEqual(start_date_previous, datetime.datetime(2024, 2, 1, 0, 0, tzinfo=datetime.timezone.utc))
+        self.assertEqual(end_date_previous, datetime.datetime(2024, 3, 1, 0, 0, tzinfo=datetime.timezone.utc))
+
+        start_date_current, end_date_current = _create_generator_dates_from_yaml(attributes, current_month)
+
+        self.assertEqual(start_date_current, datetime.datetime(2024, 3, 1, 0, 0, tzinfo=datetime.timezone.utc))
+        self.assertEqual(end_date_current, datetime.datetime(2024, 3, 1, 23, 0, tzinfo=datetime.timezone.utc))
+
     def test_create_generator_for_dates_from_yaml_middle_month(self):
         """Test helper function for generating dates verifying the middle month in a 3 month range."""
         month = {
-            "name": "June",
             "start": datetime.datetime(2021, 6, 30, 0, 0),
             "end": datetime.datetime(2021, 6, 30, 23, 59),
         }