Merge branch 'main' into feature/filter-spec-implementation

src/databricks/labs/lsql/deployment.py

@@ -1,7 +1,11 @@
+import datetime as dt
 import logging
 import pkgutil
 from typing import Any
 
+from databricks.sdk.errors import InternalError
+from databricks.sdk.retries import retried
+
 from databricks.labs.lsql.backends import Dataclass, SqlBackend
 
 logger = logging.getLogger(__name__)
@@ -13,6 +17,8 @@ def __init__(self, sql_backend: SqlBackend, inventory_schema: str, mod: Any):
         self._inventory_schema = inventory_schema
         self._module = mod
 
+    # InternalError are retried for resilience on sporadic Databricks issues
+    @retried(on=[InternalError], timeout=dt.timedelta(minutes=1))
     def deploy_schema(self):
         logger.info(f"Ensuring {self._inventory_schema} database exists")
         self._sql_backend.execute(f"CREATE SCHEMA IF NOT EXISTS hive_metastore.{self._inventory_schema}")

tests/integration/test_structs.py

@@ -1,6 +1,8 @@
 import datetime
 from dataclasses import dataclass
 
+import pytest
+
 from databricks.labs.lsql.backends import StatementExecutionBackend
 
 
@@ -26,6 +28,7 @@ class Nesting:
     struct_array: list[NestedWithDict]
 
 
+@pytest.mark.skip(reason="Missing permissions to create table")
 def test_appends_complex_types(ws, env_or_skip, make_random) -> None:
     sql_backend = StatementExecutionBackend(ws, env_or_skip("TEST_DEFAULT_WAREHOUSE_ID"))
     today = datetime.date.today()

tests/unit/test_dashboards.py

@@ -4,6 +4,7 @@
 import string
 from pathlib import Path
 from unittest.mock import create_autospec
+from zipfile import ZipFile
 
 import pytest
 import sqlglot
@@ -1741,7 +1742,7 @@ def test_dashboards_get_dashboard_url():
     assert dashboard_url == dashboard_url_expected
 
 
-def test_dashboards_export_to_zipped_csv(tmp_path):
+def test_dashboards_export_to_zipped_csv_contains_csv_for_query(tmp_path):
     query = {
         "SELECT\n  one\nFROM ucx.external_locations": [
             Row(location="s3://bucket1/folder1", table_count=1),
@@ -1753,13 +1754,13 @@ def test_dashboards_export_to_zipped_csv(tmp_path):
     mock_backend = MockBackend(rows=query)
 
     (tmp_path / "external_locations.sql").write_text(list(query.keys())[0])
-    export_path = tmp_path / "export"
-    export_path.mkdir(parents=True, exist_ok=True)
+    export_path = tmp_path / "export.zip"
 
     dash_metadata = DashboardMetadata(display_name="External Locations")
 
     dash = dash_metadata.from_path(tmp_path)
     dash = dash.replace_database(catalog="hive_metastore", database="ucx")
     dash.export_to_zipped_csv(mock_backend, export_path)
 
-    assert len(list(export_path.glob("export_to_zipped_csv.zip"))) == 1
+    zipped_files = ZipFile(export_path).namelist()
+    assert zipped_files == ["external_locations.csv"]