fix(mapper): produce correct release date in UTC format

nightingale/mapper.py

@@ -1,11 +1,11 @@
 import logging
-from datetime import datetime
-from typing import Any
+from typing import Any, Dict
 
 import dict_hash
 
 from .config import Config
 from .mapping.v1 import Mapping
+from .utils import get_iso_now, remove_dicts_without_id
 
 logger = logging.getLogger(__name__)
 
@@ -88,6 +88,7 @@ def transform_data(self, data: list[dict[Any, Any]], mapping: Mapping) -> list[d
                 self.tag_tags(curr_release)
                 self.make_release_id(curr_release)
                 logger.info(f"Release mapped: {curr_release['ocid']}")
+                curr_release = self.remove_empty_id_arrays(curr_release)
                 mapped.append(curr_release)
                 curr_release = {}
                 continue
@@ -217,7 +218,7 @@ def date_release(self, curr_row: dict) -> None:
         :param curr_row: The current release row dictionary.
         :type curr_row: dict
         """
-        curr_row["date"] = datetime.now().isoformat()
+        curr_row["date"] = get_iso_now()
 
     def tag_initiation_type(self, curr_row: dict) -> None:
         """
@@ -249,3 +250,13 @@ def tag_tags(self, curr_row) -> None:
             curr_row["tag"].append("award")
         if "contract" in curr_row:
             curr_row["tag"].append("contract")
+
+    def remove_empty_id_arrays(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Recursively remove arrays that do not contain an 'id' field.
+
+        :param data: The data dictionary to process.
+        :type data: dict[str, Any]
+        """
+
+        return remove_dicts_without_id(data)

tests/test_unflatten.py

@@ -365,5 +365,56 @@ def test_map(mock_config, mocker):
     loader.load.assert_called_once_with("selector1")
 
 
+@pytest.mark.parametrize(
+    "input_data, output_data",
+    [
+        (
+            {
+                "id": "1",
+                "tender": {
+                    "items": [
+                        {"id": "item1", "description": "Item 1"},
+                        {"description": "Item without ID"},
+                    ],
+                    "documents": [
+                        {"id": "doc1", "title": "Document 1"},
+                        {"title": "Document without ID"},
+                    ],
+                },
+            },
+            {
+                "id": "1",
+                "tender": {
+                    "items": [
+                        {"id": "item1", "description": "Item 1"},
+                    ],
+                    "documents": [
+                        {"id": "doc1", "title": "Document 1"},
+                    ],
+                },
+            },
+        ),
+        (
+            {
+                "id": "2",
+                "tender": {
+                    "items": [
+                        {"description": "Item without ID"},
+                    ],
+                    "documents": [],
+                },
+            },
+            {
+                "id": "2",
+            },
+        ),
+    ],
+)
+def test_remove_empty_id_arrays(input_data, output_data, mock_config):
+    mapper = OCDSDataMapper(mock_config)
+    data = mapper.remove_empty_id_arrays(input_data)
+    assert data == output_data
+
+
 if __name__ == "__main__":
     pytest.main()