diff --git a/docs/examples.rst b/docs/examples.rst index a778430..17078d2 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -36,13 +36,13 @@ Example 1: Basic Transformation [output] directory = 'output' -2. **Sample Mapping File (`mapping.xlsx`):** +2. **Sample MappingTemplate File (`mapping.xlsx`):** **General Sheet:** .. code-block:: text - id | Title | Description | Path | Status | Mapping | Comment + id | Title | Description | Path | Status | MappingTemplate | Comment ----|----------------|----------------|-----------------------------|----------|------------------------------------ | ------- 1 | OCID | unique ID | ocid | Required | example_table (id) | - 2 | Party Identifier | Identifier | parties/[0]/identifier/id | Required | party_table (identifier) | - @@ -53,7 +53,7 @@ Example 1: Basic Transformation .. code-block:: text - id | Title | Description | Path | Status | Mapping | Comment + id | Title | Description | Path | Status | MappingTemplate | Comment ----|----------------|-----------------|----------------------------|----------|------------------------------------ | ------- 1 | Tender Title | Name of tender | tender/title | Optional | example_table (name) | - 2 | Value | Tender value | tender/value/amount | Optional | example_table (value) | - @@ -101,7 +101,7 @@ Example 2: Transformation with Packaging [output] directory = 'output' -2. **Sample Mapping File (`mapping.xlsx`):** +2. **Sample MappingTemplate File (`mapping.xlsx`):** Use the same `mapping.xlsx` as in Example 1. @@ -154,13 +154,13 @@ You may need to manipulate data within the SQL query itself before it is fed int [output] directory = 'output' -2. **Sample Mapping File (`mapping.xlsx`):** +2. **Sample MappingTemplate File (`mapping.xlsx`):** **General Sheet:** .. code-block:: text - id | Title | Description | Path | Status | Mapping | Comment + id | Title | Description | Path | Status | MappingTemplate | Comment ----|----------------|----------------|-----------------------------|----------|------------------------------------ | ------- 1 | OCID | unique ID | ocid | Required | example_table (id) | - 2 | Party Identifier | Identifier | parties/[0]/identifier/id | Required | party_table (identifier) | - @@ -171,7 +171,7 @@ You may need to manipulate data within the SQL query itself before it is fed int .. code-block:: text - id | Title | Description | Path | Status | Mapping | Comment + id | Title | Description | Path | Status | MappingTemplate | Comment ----|----------------|-----------------|----------------------------|----------|------------------------------------ | ------- 1 | Tender Title | Tender title | tender/title | Optional | example_table (name) | - 2 | Value | Tender value | tender/value/amount | Optional | example_table (value) | - @@ -272,13 +272,13 @@ If the required data spans across multiple tables, you can use SQL JOINs to comb [output] directory = 'output' -3. **Sample Mapping File (`mapping.xlsx`):** +3. **Sample MappingTemplate File (`mapping.xlsx`):** **General Sheet:** .. code-block:: text - id | Title | Description | Path | Status | Mapping | Comment + id | Title | Description | Path | Status | MappingTemplate | Comment ----|----------------|----------------|-----------------------------|----------|------------------------------------ | ------- 1 | OCID | unique ID | ocid | Required | example_table (id) | - 2 | Party Identifier | Identifier | parties/[0]/identifier/id | Required | party_table (identifier) | - @@ -289,7 +289,7 @@ If the required data spans across multiple tables, you can use SQL JOINs to comb .. code-block:: text - id | Title | Description | Path | Status | Mapping | Comment + id | Title | Description | Path | Status | MappingTemplate | Comment ----|----------------|--------------|----------------------------|----------|------------------------------------ | ------- 1 | Tender Title | Tender title | tender/title | Optional | example_table (name) | - 2 | Value | Tender value | tender/value/amount | Optional | example_table (value) | - @@ -298,7 +298,7 @@ If the required data spans across multiple tables, you can use SQL JOINs to comb .. code-block:: text - id | Title | Description | Path | Status | Mapping | Comment + id | Title | Description | Path | Status | MappingTemplate | Comment ----|----------------|--------------------------------|-------------------------------|----------|----------------------------------- | ------- 1 | Description | Description from another table | contracts/[0]/description | Optional | another_table (description) | - diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 642c1f2..0ee95dc 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -92,7 +92,7 @@ Setup [output] directory = 'output' -3. **Prepare the Mapping File:** +3. **Prepare the MappingTemplate File:** Use the following configuration for `mapping.xlsx` based on the `ocds field level mapping` template: @@ -100,7 +100,7 @@ Setup .. code-block:: text - | Title | Description | Path | Status | Mapping | Comment + | Title | Description | Path | Status | MappingTemplate | Comment |---------------|---------------|------------------|----------|------------------------------------ | ------- | OCID | unique ID | ocid | Required | example_table (id) | - | Party ID | Party ID | parties/id | Optional | party_table (identifier) | - @@ -112,7 +112,7 @@ Setup .. code-block:: text - id | Title | Description | Path | Status | Mapping | Comment + id | Title | Description | Path | Status | MappingTemplate | Comment ----|----------------|--------------|-------------------|----------|------------------------------------ | ------- 1 | Tender Title | Tender title | tender/title | Optional | example_table (name) | - 2 | Value | Tender value | tender/value/amount | Optional | example_table (value) | - @@ -129,17 +129,17 @@ Run the transformation using the CLI: This will produce an output file in the `output` directory. -Mapping Configuration +MappingTemplate Configuration ---------------------- -Field-level mapping is specified in the `mapping.xlsx` file. It is formed from stardard `"OCDS Field Level Mapping template" `_. -For more information about how to fill the mapping file, refer to the `OCDS Field Level Mapping template guidance `_. +Field-level mapping is specified in the `mapping.xlsx` file. It is formed from stardard `"OCDS Field Level MappingTemplate template" `_. +For more information about how to fill the mapping file, refer to the `OCDS Field Level MappingTemplate template guidance `_. Here the bried description of the columns from mapping sheets in the mapping file: * **Path**: The path in the OCDS release schema where the field value should be placed. * **Title**: A human-readable title for the field. * **Description**: A description of what the field represents. - * **Mapping**: The field in the source data that maps to the OCDS path. + * **MappingTemplate**: The field in the source data that maps to the OCDS path. Understanding these mappings will help you configure the transformation correctly for your data. diff --git a/nightingale/cli.py b/nightingale/cli.py index 65dd4e6..0564e89 100644 --- a/nightingale/cli.py +++ b/nightingale/cli.py @@ -48,7 +48,7 @@ def run(config_file, package, validate_mapping, loglevel): config = Config.from_file(config_file) mapper = OCDSDataMapper(config) writer = DataWriter(config.output) - logger.info("Mapping data...") + logger.info("MappingTemplate data...") ocds_data = mapper.map(DataLoader(config.datasource), validate_mapping=validate_mapping) if package: diff --git a/nightingale/mapper.py b/nightingale/mapper.py index 8cda0f8..c8b0525 100644 --- a/nightingale/mapper.py +++ b/nightingale/mapper.py @@ -4,7 +4,7 @@ import dict_hash from .config import Config -from .mapping.v1 import Mapping, MappingTemplateValidator +from .mapping_template.v09 import MappingTemplate, MappingTemplateValidator from .utils import get_iso_now, is_new_array, remove_dicts_without_id logger = logging.getLogger(__name__) @@ -49,7 +49,7 @@ def map(self, loader: Any, validate_mapping: bool = False) -> list[dict[str, Any :rtype: list[dict[str, Any]] """ config = self.config.mapping - mapping = Mapping(config) + mapping = MappingTemplate(config) data = loader.load(config.selector) if validate_mapping: validator = MappingTemplateValidator(loader, mapping) @@ -58,14 +58,14 @@ def map(self, loader: Any, validate_mapping: bool = False) -> list[dict[str, Any logger.info("Start mapping data") return self.transform_data(data, mapping) - def transform_data(self, data: list[dict[Any, Any]], mapping: Mapping) -> list[dict[str, Any]]: + def transform_data(self, data: list[dict[Any, Any]], mapping: MappingTemplate) -> list[dict[str, Any]]: """ Transform the input data to the OCDS format. :param data: List of input data dictionaries. :type data: list[dict[Any, Any]] :param mapping: Mapping configuration object. - :type mapping: Mapping + :type mapping: MappingTemplate :return: List of transformed release dictionaries. :rtype: list[dict[str, Any]] """ @@ -103,7 +103,7 @@ def finish_release(self, curr_ocid, curr_release, mapped): def transform_row( self, input_data: dict[Any, Any], - mapping_config: Mapping, + mapping_config: MappingTemplate, flattened_schema: dict[str, Any], result: dict = None, ) -> dict: @@ -113,7 +113,7 @@ def transform_row( :param input_data: Dictionary of input data. :type input_data: dict[Any, Any] :param mapping_config: Mapping configuration object. - :type mapping_config: Mapping + :type mapping_config: MappingTemplate :param flattened_schema: Flattened schema dictionary. :type flattened_schema: dict[str, Any] :param result: Existing result dictionary to update. diff --git a/nightingale/mapping/v1/__init__.py b/nightingale/mapping/v1/__init__.py deleted file mode 100644 index 8c2834e..0000000 --- a/nightingale/mapping/v1/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .config import Mapping # noqa -from .validator import MappingTemplateValidator # noqa diff --git a/nightingale/mapping/__init__.py b/nightingale/mapping_template/__init__.py similarity index 100% rename from nightingale/mapping/__init__.py rename to nightingale/mapping_template/__init__.py diff --git a/nightingale/mapping/v1/config.py b/nightingale/mapping_template/v09/__init__.py similarity index 97% rename from nightingale/mapping/v1/config.py rename to nightingale/mapping_template/v09/__init__.py index 1183800..615fd71 100644 --- a/nightingale/mapping/v1/config.py +++ b/nightingale/mapping_template/v09/__init__.py @@ -2,6 +2,9 @@ import openpyxl +from nightingale.mapping_template.validator import MappingTemplateValidator # noqa +from nightingale.utils import get_longest_array_path + logger = logging.getLogger(__name__) @@ -19,7 +22,7 @@ SCHEMA_SHEET = "OCDS Schema" -class Mapping: +class MappingTemplate: def __init__(self, config): self.config = config self.wb = openpyxl.load_workbook(self.config.file, data_only=True) @@ -197,9 +200,3 @@ def get_ocid_mapping(self): def get_containing_array_path(self, path): return get_longest_array_path(self.get_arrays(), path) - - -def get_longest_array_path(arrays, path): # extract for testing - for array in reversed(sorted(arrays, key=len)): - if path.startswith(array): - return array diff --git a/nightingale/mapping/v1/validator.py b/nightingale/mapping_template/validator.py similarity index 100% rename from nightingale/mapping/v1/validator.py rename to nightingale/mapping_template/validator.py diff --git a/nightingale/utils.py b/nightingale/utils.py index 089f818..471925c 100644 --- a/nightingale/utils.py +++ b/nightingale/utils.py @@ -67,3 +67,9 @@ def is_new_array(array_counters, child_path, array_key, array_value, array_path) if array_key == "id" and "/" + array_key == child_path and array_counters[array_path] != array_value: return True return False + + +def get_longest_array_path(arrays, path): # extract for testing + for array in reversed(sorted(arrays, key=len)): + if path.startswith(array): + return array diff --git a/tests/test_mapping_loader.py b/tests/test_mapping_loader.py index 586ad17..e68157f 100644 --- a/tests/test_mapping_loader.py +++ b/tests/test_mapping_loader.py @@ -3,8 +3,8 @@ import openpyxl import pytest -from nightingale.mapping.v1 import Mapping -from nightingale.mapping.v1.config import get_longest_array_path +from nightingale.mapping_template.v09 import MappingTemplate +from nightingale.utils import get_longest_array_path def test_sheets(): @@ -136,7 +136,7 @@ class Config: @patch("openpyxl.load_workbook") def test_mapping_init(mock_load_workbook, mock_workbook, mock_config): mock_load_workbook.return_value = mock_workbook - mapping = Mapping(mock_config) + mapping = MappingTemplate(mock_config) assert mapping.wb == mock_workbook @@ -150,7 +150,7 @@ def test_normmalize_mapping_column(mock_config): {"mapping": "field1 field2"}, {"mapping": "field3 field4"}, ] - mapping = Mapping(mock_config) + mapping = MappingTemplate(mock_config) result = mapping.normmalize_mapping_column(mappings) assert result == expected_result @@ -160,7 +160,7 @@ def test_read_data_elements_sheet(mock_load_workbook, mock_workbook, mock_config mock_workbook.__getitem__ = lambda self, x: getter(x) mock_load_workbook.return_value = mock_workbook - mapping = Mapping(mock_config) + mapping = MappingTemplate(mock_config) expected_data_elements = { "element1": { @@ -193,7 +193,7 @@ def test_read_schema_sheet(mock_load_workbook, mock_workbook, mock_config): mock_workbook.__getitem__ = lambda self, x: getter(x) mock_load_workbook.return_value = mock_workbook - mapping = Mapping(mock_config) + mapping = MappingTemplate(mock_config) schema = mapping.get_schema() expected_schema = { @@ -232,7 +232,7 @@ def test_enforce_mapping_structure(mock_load_workbook, mock_workbook, mock_confi {"path": "general/item6", "mapping": "map6"}, ] - mapping = Mapping(mock_config) + mapping = MappingTemplate(mock_config) result = mapping.enforce_mapping_structure(mappings) expected_result = [ @@ -252,7 +252,7 @@ def test_read_mappings(mock_load_workbook, mock_workbook, mock_config): mock_workbook.__getitem__ = lambda self, x: getter(x) mock_load_workbook.return_value = mock_workbook - mapping = Mapping(mock_config) + mapping = MappingTemplate(mock_config) expected_mappings = [ { @@ -491,7 +491,7 @@ def test_get_element_by_mapping(mock_load_workbook, mock_workbook, mock_config): mock_workbook.__getitem__.side_effect = lambda name: mock_sheet mock_load_workbook.return_value = mock_workbook - mapping = Mapping(mock_config) + mapping = MappingTemplate(mock_config) element = mapping.get_element_by_mapping("mapping1 (element1)") expected_element = { @@ -565,7 +565,7 @@ def test_get_paths_for_mapping(mock_load_workbook, mock_workbook, mock_config): }, ] - mapping = Mapping(mock_config) + mapping = MappingTemplate(mock_config) mapping.mappings = mock_mappings paths = mapping.get_paths_for_mapping("mapping1 (element1)") @@ -580,7 +580,7 @@ def test_is_array_path(mock_load_workbook, mock_workbook, mock_config): mock_workbook.__getitem__ = lambda self, x: getter(x) mock_load_workbook.return_value = mock_workbook - mapping = Mapping(mock_config) + mapping = MappingTemplate(mock_config) mapping.schema = mapping.read_schema_sheet() assert mapping.is_array_path("/array_path") is True assert mapping.is_array_path("/path2") is False diff --git a/tests/test_unflatten.py b/tests/test_unflatten.py index e8a9423..44c21b3 100644 --- a/tests/test_unflatten.py +++ b/tests/test_unflatten.py @@ -6,7 +6,7 @@ from nightingale.config import Config, Datasource, Mapping, Output, Publishing from nightingale.mapper import OCDSDataMapper -from nightingale.mapping.v1.config import get_longest_array_path +from nightingale.utils import get_longest_array_path class MockMappingConfig: @@ -354,8 +354,8 @@ def test_map(mock_config, mocker): loader = mocker.Mock() loader.load.return_value = [{"ocid": "1", "field": "value"}] - # Mock the Mapping class to avoid reading from an actual file - with mock.patch("nightingale.mapper.Mapping") as MockMapping: + # Mock the MappingTemplate class to avoid reading from an actual file + with mock.patch("nightingale.mapper.MappingTemplate") as MockMapping: MockMapping.return_value = MockMappingConfig({}) mapper = OCDSDataMapper(mock_config) result = mapper.map(loader) @@ -423,7 +423,7 @@ def test_map_with_validation(mock_config): mock_validator_instance = MockValidator.return_value mock_validator_instance.validate_data_elements = mock.MagicMock() mock_validator_instance.validate_selector = mock.MagicMock() - with mock.patch("nightingale.mapper.Mapping") as mock_mapping_template: + with mock.patch("nightingale.mapper.MappingTemplate") as mock_mapping_template: mapper = OCDSDataMapper(mock_config) mapper.map(loader, validate_mapping=True) MockValidator.assert_called_with(loader, mock_mapping_template()) @@ -472,7 +472,7 @@ def test_finish_release(mock_get_iso_now, mock_config): @mock.patch("nightingale.mapper.get_iso_now") -@mock.patch("nightingale.mapper.Mapping") +@mock.patch("nightingale.mapper.MappingTemplate") def test_finish_release_with_tags(mock_mapping, mock_get_iso_now, mock_config): mock_get_iso_now.return_value = "2022-01-01T00:00:00Z" mock_mapping.get_ocid_mapping.return_value = "ocid" diff --git a/tests/test_validator.py b/tests/test_validator.py index c9c4d99..effb37e 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import Mock, patch -from nightingale.mapping.v1.validator import MappingTemplateValidator +from nightingale.mapping_template.validator import MappingTemplateValidator class TestMappingTemplateValidator(unittest.TestCase): @@ -13,7 +13,7 @@ def setUp(self): # Create instance of the class self.validator = MappingTemplateValidator(self.loader, self.mapping_template) - @patch("nightingale.mapping.v1.validator.logger") + @patch("nightingale.mapping_template.validator.logger") def test_validate_data_elements_all_columns_described(self, mock_logger): # Mock the cursor and database table structure cursor = Mock() @@ -62,7 +62,7 @@ def test_validate_data_elements_all_columns_described(self, mock_logger): # Ensure no warnings were logged mock_logger.warning.assert_not_called() - @patch("nightingale.mapping.v1.validator.logger") + @patch("nightingale.mapping_template.validator.logger") def test_validate_data_elements_missing_columns(self, mock_logger): # Mock the cursor and database table structure cursor = Mock() @@ -107,7 +107,7 @@ def test_validate_data_elements_missing_columns(self, mock_logger): # Check if a warning was logged for the missing column mock_logger.warning.assert_called_with("Column table1/missing is not described in data elements") - @patch("nightingale.mapping.v1.validator.logger") + @patch("nightingale.mapping_template.validator.logger") def test_validate_selector_all_columns_mapped(self, mock_logger): # Mock the mapping template data elements self.mapping_template.get_data_elements.return_value = { @@ -142,7 +142,7 @@ def test_validate_selector_all_columns_mapped(self, mock_logger): # Ensure no warnings were logged mock_logger.warning.assert_not_called() - @patch("nightingale.mapping.v1.validator.logger") + @patch("nightingale.mapping_template.validator.logger") def test_validate_selector_missing_columns(self, mock_logger): # Mock the mapping template data elements self.mapping_template.get_data_elements.return_value = {