From d84f25e7e683ec34cbc0253647f5d786822e0014 Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Tue, 8 Oct 2024 14:13:00 +0200 Subject: [PATCH 01/11] Add CDS / ODS selector --- parameter-sets/basic-auth/parameter-set.json | 16 ++++++++++++++++ parameter-sets/login/parameter-set.json | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/parameter-sets/basic-auth/parameter-set.json b/parameter-sets/basic-auth/parameter-set.json index a3c009a..0b9a575 100644 --- a/parameter-sets/basic-auth/parameter-set.json +++ b/parameter-sets/basic-auth/parameter-set.json @@ -45,6 +45,22 @@ } ] }, + { + "name": "sap_mode", + "label": "SAP mode", + "type": "SELECT", + "defaultValue" : "cds", + "selectChoices": [ + { + "value": "cds", + "label": "CDS" + }, + { + "value": "odp", + "label": "ODP" + } + ] + }, { "name": "sap_client", "label": "Client", diff --git a/parameter-sets/login/parameter-set.json b/parameter-sets/login/parameter-set.json index 409be2e..9c5053b 100644 --- a/parameter-sets/login/parameter-set.json +++ b/parameter-sets/login/parameter-set.json @@ -44,6 +44,22 @@ } ] }, + { + "name": "sap_mode", + "label": "SAP mode", + "type": "SELECT", + "defaultValue" : "cds", + "selectChoices": [ + { + "value": "cds", + "label": "CDS" + }, + { + "value": "odp", + "label": "ODP" + } + ] + }, { "name": "odata_username", "label": "Username", From e78d5262363dbd15fcb760610f7315cdc8bc79dc Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Tue, 8 Oct 2024 14:13:27 +0200 Subject: [PATCH 02/11] Set bulk_size min --- python-connectors/sap_odata/connector.json | 1 + 1 file changed, 1 insertion(+) diff --git a/python-connectors/sap_odata/connector.json b/python-connectors/sap_odata/connector.json index db8e0d4..198d398 100644 --- a/python-connectors/sap_odata/connector.json +++ b/python-connectors/sap_odata/connector.json @@ -129,6 +129,7 @@ "label": " ", "description": "Bulk size", "type": "INT", + "minI": 0, "defaultValue": 1000, "visibilityCondition": "model.show_advanced_parameters == true" } From 570dcb13ff528f79e38278574669a47c6f23d65e Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Tue, 8 Oct 2024 14:14:13 +0200 Subject: [PATCH 03/11] Implement server side pagination --- python-connectors/sap_odata/connector.py | 45 +++++++++++++++++------- python-lib/odata_client.py | 2 +- python-lib/odata_common.py | 20 +++++++++++ python-lib/odata_constants.py | 1 + 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/python-connectors/sap_odata/connector.py b/python-connectors/sap_odata/connector.py index 7fd1c7f..166e73e 100644 --- a/python-connectors/sap_odata/connector.py +++ b/python-connectors/sap_odata/connector.py @@ -1,7 +1,7 @@ from dataiku.connector import Connector from dataikuapi.utils import DataikuException from odata_client import ODataClient -from odata_common import get_clean_row_method, get_list_title +from odata_common import get_clean_row_method, get_list_title, RecordsLimit, get_sap_mode import logging @@ -21,10 +21,11 @@ def __init__(self, config, plugin_config): object 'plugin_config' to the constructor """ Connector.__init__(self, config, plugin_config) - logger.info("Starting SAP-OData v1.0.3") + logger.info("Starting SAP-OData v1.0.4-beta.2") self.odata_list_title = get_list_title(config) self.bulk_size = config.get("bulk_size", 1000) self.odata_filter_query = "" + self.sap_mode = get_sap_mode(config) if config.get("show_advanced_parameters", False): self.odata_filter_query = config.get("odata_filter_query", "") @@ -66,10 +67,9 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None, The dataset schema and partitioning are given for information purpose. """ - skip = 0 - bulk_size = self.bulk_size - if records_limit > 0: - bulk_size = records_limit if records_limit < bulk_size else bulk_size + limit = RecordsLimit(records_limit=records_limit) + skip = None + bulk_size = self.get_bulk_size(records_limit=records_limit) items, next_page_url = self.client.get_entity_collections( entity=self.odata_list_title, top=bulk_size, @@ -77,19 +77,40 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None, filter=self.odata_filter_query ) while items: + number_of_items = len(items) for item in items: yield self.clean_row(item) - skip = skip + bulk_size - if records_limit > 0: - if skip >= records_limit: - break - if skip + bulk_size > records_limit: - bulk_size = records_limit - skip + if limit.is_reached(): + logger.info("Limit is reached") + return + if self.is_client_side_pagination(): + if skip is None: + skip = 0 + skip = skip + number_of_items + else: + if not next_page_url: + # Server side pagination with no next_page_url + # -> time to quit + return items, next_page_url = self.client.get_entity_collections( entity=self.odata_list_title, top=bulk_size, skip=skip, page_url=next_page_url, filter=self.odata_filter_query ) + def get_bulk_size(self, records_limit=None): + if self.is_client_side_pagination(): + if self.bulk_size == 0: + return None + bulk_size = self.bulk_size + if records_limit > 0: + bulk_size = records_limit if records_limit < bulk_size else bulk_size + else: + bulk_size = None + return bulk_size + + def is_client_side_pagination(self): + return self.sap_mode == "cds" + def get_schema_set(self, set_name): for one_set in self.client.schema.entity_sets: if one_set.name == set_name: diff --git a/python-lib/odata_client.py b/python-lib/odata_client.py index 86fc4d1..3eb0549 100644 --- a/python-lib/odata_client.py +++ b/python-lib/odata_client.py @@ -94,7 +94,7 @@ def get_entity_collections(self, entity="", top=None, skip=None, page_url=None, response = self.get(url) self.assert_response(response) data = response.json() - next_page_url = data.get(ODataConstants.NEXT_LINK, None) + next_page_url = data.get(ODataConstants.NEXT_LINK_SAP, data.get(ODataConstants.NEXT_LINK, None)) item = data.get(ODataConstants.DATA_CONTAINER_V4, data.get(ODataConstants.DATA_CONTAINER_V2, {})) return self.format(item), next_page_url diff --git a/python-lib/odata_common.py b/python-lib/odata_common.py index 94cb7f3..84f4f27 100644 --- a/python-lib/odata_common.py +++ b/python-lib/odata_common.py @@ -83,6 +83,13 @@ def get_list_title(config): return odata_list_title +def get_sap_mode(config): + auth_type = config.get("auth_type", "login") + login_config = config.get(ODataConstants.LOGIN, {}) if auth_type == "login" else config.get("sap-odata_user-account", {}) + sap_mode = login_config.get("sap_mode", "cds") + return sap_mode + + class DSSSelectorChoices(object): def __init__(self): self.choices = [] @@ -118,3 +125,16 @@ def text_message(self, text_message): def to_dss(self): return self._build_select_choices(self.choices) + + +class RecordsLimit(): + def __init__(self, records_limit=-1): + self.has_no_limit = (records_limit == -1) + self.records_limit = records_limit + self.counter = 0 + + def is_reached(self): + if self.has_no_limit: + return False + self.counter += 1 + return self.counter > self.records_limit diff --git a/python-lib/odata_constants.py b/python-lib/odata_constants.py index 60165fb..6711be0 100644 --- a/python-lib/odata_constants.py +++ b/python-lib/odata_constants.py @@ -9,6 +9,7 @@ class ODataConstants(object): LIST_TITLE = "odata_list_title" LOGIN = "sap-odata_login" NEXT_LINK = "@odata.nextLink" + NEXT_LINK_SAP = "__next" OAUTH = "odata_oauth" ODATA_V2 = "v2" ODATA_V3 = "v3" From 2dd14b3db5fa2880ff019f7e8aa01cdcd00c8dea Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Tue, 8 Oct 2024 14:14:36 +0200 Subject: [PATCH 04/11] set version 1.0.4 --- plugin.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.json b/plugin.json index 97a5267..6454a37 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "id": "sap-odata", - "version": "1.0.3", + "version": "1.0.4", "meta": { "label": "SAP OData", "description": "Import data from your SAP account", From 9105720d5c0050bffc37f4907292659367b340b0 Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Tue, 8 Oct 2024 14:14:45 +0200 Subject: [PATCH 05/11] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c824c85..f49e2fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [Version 1.0.4](https://github.com/dataiku/dss-plugin-sap-odata/releases/tag/v1.0.4) - Feature - 2024-10-04 + +- Add server side pagination + ## [Version 1.0.3](https://github.com/dataiku/dss-plugin-sap-odata/releases/tag/v1.0.3) - Feature - 2023-10-20 - Add filter field From 0e468f2df5b962b257306da082542be930b33c3f Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Tue, 8 Oct 2024 17:17:04 +0200 Subject: [PATCH 06/11] Add integration tests --- tests/python/integration/pytest.ini | 2 ++ tests/python/integration/requirements.txt | 3 +++ tests/python/integration/test_scenario.py | 12 ++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 tests/python/integration/pytest.ini create mode 100755 tests/python/integration/requirements.txt create mode 100644 tests/python/integration/test_scenario.py diff --git a/tests/python/integration/pytest.ini b/tests/python/integration/pytest.ini new file mode 100644 index 0000000..597fa3a --- /dev/null +++ b/tests/python/integration/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +usefixtures = plugin dss_target \ No newline at end of file diff --git a/tests/python/integration/requirements.txt b/tests/python/integration/requirements.txt new file mode 100755 index 0000000..cac3188 --- /dev/null +++ b/tests/python/integration/requirements.txt @@ -0,0 +1,3 @@ +pytest==6.2.1 +dataiku-api-client +git+git://github.com/dataiku/dataiku-plugin-tests-utils.git@master#egg=dataiku-plugin-tests-utils diff --git a/tests/python/integration/test_scenario.py b/tests/python/integration/test_scenario.py new file mode 100644 index 0000000..3723c5c --- /dev/null +++ b/tests/python/integration/test_scenario.py @@ -0,0 +1,12 @@ +from dku_plugin_test_utils import dss_scenario + + +TEST_PROJECT_KEY = "PLUGINTESTSAPODATA" + + +def test_run_sap_odata_read(user_dss_clients): + dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="SAP") + + +def test_run_sap_odata_v4(user_dss_clients): + dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="SAP_V4") From 8874996fa38f0cb52772331ede593be228492a02 Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Tue, 8 Oct 2024 17:23:00 +0200 Subject: [PATCH 07/11] Add integration tests --- Jenkinsfile | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..75575a3 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,47 @@ +pipeline { + options { disableConcurrentBuilds() } + agent { label 'dss-plugin-tests'} + environment { + PLUGIN_INTEGRATION_TEST_INSTANCE="/home/jenkins-agent/instance_config.json" + } + stages { + stage('Run Unit Tests') { + steps { + sh 'echo "Running unit tests"' + catchError(stageResult: 'FAILURE') { + sh """ + make unit-tests + """ + } + sh 'echo "Done with unit tests"' + } + } + stage('Run Integration Tests') { + steps { + sh 'echo "Running integration tests"' + catchError(stageResult: 'FAILURE') { + sh """ + make integration-tests + """ + } + sh 'echo "Done with integration tests"' + } + } + } + post { + always { + script { + allure([ + includeProperties: false, + jdk: '', + properties: [], + reportBuildPolicy: 'ALWAYS', + results: [[path: 'tests/allure_report']] + ]) + + def status = currentBuild.currentResult + sh "file_name=\$(echo ${env.JOB_NAME} | tr '/' '-').status; touch \$file_name; echo \"${env.BUILD_URL};${env.CHANGE_TITLE};${env.CHANGE_AUTHOR};${env.CHANGE_URL};${env.BRANCH_NAME};${status};\" >> $HOME/daily-statuses/\$file_name" + } + } + } +} \ No newline at end of file From beb24ea433039a54134100200b833febaac7628a Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Tue, 8 Oct 2024 17:30:37 +0200 Subject: [PATCH 08/11] fix back compatibility issue --- python-lib/odata_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python-lib/odata_client.py b/python-lib/odata_client.py index 3eb0549..4be40e1 100644 --- a/python-lib/odata_client.py +++ b/python-lib/odata_client.py @@ -85,6 +85,8 @@ def get_session(self, config, odata_version): return session def get_entity_collections(self, entity="", top=None, skip=None, page_url=None, filter=None): + if entity is None: + entity = "" if self.odata_list_title is None or self.odata_list_title == "": top = None # SAP will complain if $top is present in a request to list entities query_options = self.get_base_query_options(top=top, skip=skip, filter=filter) From 37c7388c37ce2c2cdfa3691934c4c64044caa75f Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Wed, 9 Oct 2024 12:05:34 +0200 Subject: [PATCH 09/11] Align error 400 handling with odata plugin --- python-connectors/sap_odata/connector.py | 2 +- python-lib/odata_client.py | 22 ++++++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/python-connectors/sap_odata/connector.py b/python-connectors/sap_odata/connector.py index 166e73e..a5b8186 100644 --- a/python-connectors/sap_odata/connector.py +++ b/python-connectors/sap_odata/connector.py @@ -94,7 +94,7 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None, return items, next_page_url = self.client.get_entity_collections( entity=self.odata_list_title, top=bulk_size, skip=skip, - page_url=next_page_url, filter=self.odata_filter_query + page_url=next_page_url, filter=self.odata_filter_query, can_raise=False ) def get_bulk_size(self, records_limit=None): diff --git a/python-lib/odata_client.py b/python-lib/odata_client.py index 4be40e1..93f8183 100644 --- a/python-lib/odata_client.py +++ b/python-lib/odata_client.py @@ -84,7 +84,7 @@ def get_session(self, config, odata_version): ) return session - def get_entity_collections(self, entity="", top=None, skip=None, page_url=None, filter=None): + def get_entity_collections(self, entity="", top=None, skip=None, page_url=None, filter=None, can_raise=True): if entity is None: entity = "" if self.odata_list_title is None or self.odata_list_title == "": @@ -93,8 +93,12 @@ def get_entity_collections(self, entity="", top=None, skip=None, page_url=None, url = page_url if page_url else self.odata_instance + '/' + entity.strip("/") + self.get_query_string(query_options) data = None while self._should_retry(data): + logger.info("requests get url {}".format(url)) response = self.get(url) - self.assert_response(response) + if self.assert_response_ok(response, can_raise=can_raise): + data = response.json() + else: + return {}, None data = response.json() next_page_url = data.get(ODataConstants.NEXT_LINK_SAP, data.get(ODataConstants.NEXT_LINK, None)) item = data.get(ODataConstants.DATA_CONTAINER_V4, data.get(ODataConstants.DATA_CONTAINER_V2, {})) @@ -186,11 +190,21 @@ def get_query_string(self, query_options): else: return "" - def assert_response(self, response): + def assert_response_ok(self, response, can_raise=True): status_code = response.status_code + return_code = True if status_code == 404: - raise DataikuException("This entity does not exist") + return_code = False + logger.error("Error 404, response={}".format(response.content)) + if can_raise: + raise DataikuException("This entity does not exist") if status_code == 403: raise DataikuException("{}".format(response)) if status_code == 401: raise DataikuException("Forbidden access") + if status_code == 400: + return_code = False + logger.error("Error 400, response={}".format(response.content)) + if can_raise: + raise DataikuException("Error 400: {}".format(response)) + return return_code From af027c922138cf1a16fb043f50ab4919e55bf400 Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Mon, 14 Oct 2024 09:43:25 +0200 Subject: [PATCH 10/11] fix __next key in items rather than response --- python-lib/odata_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python-lib/odata_client.py b/python-lib/odata_client.py index 93f8183..7201922 100644 --- a/python-lib/odata_client.py +++ b/python-lib/odata_client.py @@ -100,8 +100,9 @@ def get_entity_collections(self, entity="", top=None, skip=None, page_url=None, else: return {}, None data = response.json() - next_page_url = data.get(ODataConstants.NEXT_LINK_SAP, data.get(ODataConstants.NEXT_LINK, None)) + next_page_url = data.get(ODataConstants.NEXT_LINK, None) item = data.get(ODataConstants.DATA_CONTAINER_V4, data.get(ODataConstants.DATA_CONTAINER_V2, {})) + next_page_url = item.get(ODataConstants.NEXT_LINK_SAP, next_page_url) return self.format(item), next_page_url def _should_retry(self, data): From 340680b21115ff637e8149268aed07bd0ab847c9 Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Mon, 14 Oct 2024 09:44:20 +0200 Subject: [PATCH 11/11] beta.3 --- python-connectors/sap_odata/connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-connectors/sap_odata/connector.py b/python-connectors/sap_odata/connector.py index a5b8186..5c4ebd2 100644 --- a/python-connectors/sap_odata/connector.py +++ b/python-connectors/sap_odata/connector.py @@ -21,7 +21,7 @@ def __init__(self, config, plugin_config): object 'plugin_config' to the constructor """ Connector.__init__(self, config, plugin_config) - logger.info("Starting SAP-OData v1.0.4-beta.2") + logger.info("Starting SAP-OData v1.0.4-beta.3") self.odata_list_title = get_list_title(config) self.bulk_size = config.get("bulk_size", 1000) self.odata_filter_query = ""