fix(ingest/tableau): apply page_size regardless of object count (#1…

…2026)
datahub-project · Dec 12, 2024 · 47bd446 · 47bd446
1 parent 93c8ae2
commit 47bd446
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 3 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py
@@ -979,7 +979,6 @@ def get_filter_pages(query_filter: dict, page_size: int) -> List[dict]:
         len(query_filter.keys()) == 1
         and query_filter.get(c.ID_WITH_IN)
         and isinstance(query_filter[c.ID_WITH_IN], list)
-        and len(query_filter[c.ID_WITH_IN]) > 100 * page_size
     ):
         ids = query_filter[c.ID_WITH_IN]
         filter_pages = [

diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py
@@ -60,7 +60,7 @@
     "site": "acryl",
     "projects": ["default", "Project 2", "Samples"],
     "extract_project_hierarchy": False,
-    "page_size": 10,
+    "page_size": 1000,
     "ingest_tags": True,
     "ingest_owner": True,
     "ingest_tables_external": True,
@@ -673,7 +673,7 @@ def test_tableau_ingest_with_platform_instance(
         "site": "acryl",
         "platform_instance": "acryl_site1",
         "projects": ["default", "Project 2"],
-        "page_size": 10,
+        "page_size": 1000,
         "ingest_tags": True,
         "ingest_owner": True,
         "ingest_tables_external": True,