Fix dashboard deployment/creation (#230)

Fixes the dashboard deployment by:
- Raising an error when dashboard is invalid
- Deprecating the `deploy_dashboard` for the `create_dashboard` as this
is more accurate and available after refactoring Lakeview dashboard
creation to be part of `DashboardMetadata`
- Add `publish` flag to `create_dashboard`

# Related issues
- Resolves #222
- Resolves #229
- Partially resolves #220: leaves the return type to be discussed

Below the dashboard created through the deprecated `deploy_dashboard`:

![Screenshot 2024-07-19 at 14 30
25](https://github.com/user-attachments/assets/e40fe707-f7c6-44ed-afd6-9072d9d6b2be)

src/databricks/labs/lsql/cli.py

@@ -28,8 +28,7 @@ def create_dashboard(
         catalog=catalog or None,
         database=database or None,
     )
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
-    sdk_dashboard = lakeview_dashboards.deploy_dashboard(lakeview_dashboard)
+    sdk_dashboard = lakeview_dashboards.create_dashboard(dashboard_metadata)
     if not no_open:
         assert sdk_dashboard.dashboard_id is not None
         dashboard_url = lakeview_dashboards.get_url(sdk_dashboard.dashboard_id)

src/databricks/labs/lsql/dashboards.py

@@ -6,6 +6,8 @@
 import math
 import re
 import shlex
+import tempfile
+import warnings
 from argparse import ArgumentParser
 from collections import defaultdict
 from collections.abc import Callable, Iterable, Sized
@@ -324,6 +326,15 @@ def position(self) -> Position:
         height = self.metadata.height or self._position.height
         return Position(self._position.x, self._position.y, width, height)
 
+    def validate(self) -> None:
+        """Validate the tile
+
+        Raises:
+            ValueError : If the tile is invalid.
+        """
+        if len(self.content) == 0:
+            raise ValueError(f"Tile has empty content: {self}")
+
     def get_layouts(self) -> Iterable[Layout]:
         """Get the layout(s) reflecting this tile in the dashboard."""
         widget = Widget(name=self.metadata.id, textbox_spec=self.content)
@@ -366,6 +377,16 @@ def __repr__(self):
 class MarkdownTile(Tile):
     _position: Position = dataclasses.field(default_factory=lambda: Position(0, 0, _MAXIMUM_DASHBOARD_WIDTH, 3))
 
+    def validate(self) -> None:
+        """Validate the tile
+
+        Raises:
+            ValueError : If the tile is invalid.
+        """
+        super().validate()
+        if not self.metadata.is_markdown():
+            raise ValueError(f"Tile is not a markdown file: {self}")
+
 
 @dataclass
 class QueryTile(Tile):
@@ -376,6 +397,20 @@ class QueryTile(Tile):
     _DIALECT = sqlglot.dialects.Databricks
     _FILTER_HEIGHT = 1
 
+    def validate(self) -> None:
+        """Validate the tile
+
+        Raises:
+            ValueError : If the tile is invalid.
+        """
+        super().validate()
+        if not self.metadata.is_query():
+            raise ValueError(f"Tile is not a query file: {self}")
+        try:
+            sqlglot.parse_one(self.content, dialect=self._DIALECT)
+        except sqlglot.ParseError as e:
+            raise ValueError(f"Invalid query content: {self.content}") from e
+
     @staticmethod
     def format(query: str, max_text_width: int = 120) -> str:
         try:
@@ -701,8 +736,7 @@ def validate(self) -> None:
         """
         tile_ids = []
         for tile in self.tiles:
-            if len(tile.content) == 0:
-                raise ValueError(f"Tile has empty content: {tile}")
+            tile.validate()
             tile_ids.append(tile.metadata.id)
         counter = collections.Counter(tile_ids)
         for tile_id, id_count in counter.items():
@@ -851,39 +885,61 @@ def save_to_folder(self, dashboard: Dashboard, local_path: Path) -> Dashboard:
         dashboard = self._with_better_names(dashboard)
         for dataset in dashboard.datasets:
             query = QueryTile.format(dataset.query)
-            with (local_path / f"{dataset.name}.sql").open("w") as f:
-                f.write(query)
+            (local_path / f"{dataset.name}.sql").write_text(query)
         for page in dashboard.pages:
             with (local_path / f"{page.name}.yml").open("w") as f:
                 yaml.safe_dump(page.as_dict(), f)
+            for layout in page.layout:
+                if layout.widget.textbox_spec is not None:
+                    (local_path / f"{layout.widget.name}.md").write_text(layout.widget.textbox_spec)
         return dashboard
 
-    def deploy_dashboard(
+    def create_dashboard(
         self,
-        lakeview_dashboard: Dashboard,
+        dashboard_metadata: DashboardMetadata,
         *,
         parent_path: str | None = None,
         dashboard_id: str | None = None,
         warehouse_id: str | None = None,
     ) -> SDKDashboard:
-        """Deploy a lakeview dashboard."""
-        serialized_dashboard = json.dumps(lakeview_dashboard.as_dict())
-        display_name = lakeview_dashboard.pages[0].display_name or lakeview_dashboard.pages[0].name
+        """Create a Lakeview dashboard.
+
+        Parameters :
+            dashboard_metadata : DashboardMetadata
+                The dashboard metadata
+            parent_path : str | None (default: None)
+                The folder in the Databricks workspace to store the dashboard file in
+            dashboard_id : str | None (default: None)
+                The id of the dashboard to update
+            warehouse_id : str | None (default: None)
+                The id of the warehouse to use
+        """
+        dashboard_metadata.validate()
+        serialized_dashboard = json.dumps(dashboard_metadata.as_lakeview().as_dict())
         if dashboard_id is not None:
-            dashboard = self._ws.lakeview.update(
+            sdk_dashboard = self._ws.lakeview.update(
                 dashboard_id,
-                display_name=display_name,
+                display_name=dashboard_metadata.display_name,
                 serialized_dashboard=serialized_dashboard,
                 warehouse_id=warehouse_id,
             )
         else:
-            dashboard = self._ws.lakeview.create(
-                display_name,
+            sdk_dashboard = self._ws.lakeview.create(
+                dashboard_metadata.display_name,
                 parent_path=parent_path,
                 serialized_dashboard=serialized_dashboard,
                 warehouse_id=warehouse_id,
             )
-        return dashboard
+        return sdk_dashboard
+
+    def deploy_dashboard(self, dashboard: Dashboard, **kwargs) -> SDKDashboard:
+        """Legacy method use :meth:create_dashboard instead."""
+        warnings.warn("Deprecated method use `create_dashboard` instead.", category=DeprecationWarning)
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory)
+            self.save_to_folder(dashboard, path)
+            dashboard_metadata = DashboardMetadata.from_path(path)
+            return self.create_dashboard(dashboard_metadata, **kwargs)
 
     def _with_better_names(self, dashboard: Dashboard) -> Dashboard:
         """Replace names with human-readable names."""

tests/integration/test_dashboards.py

@@ -62,22 +62,21 @@ def tmp_path(tmp_path, make_random):
     The folder name becomes the dashboard name, which then becomes the Lakeview file name with the
     `.lvdash.json` extension. `tmp_path` last subfolder contains the test name cut off at thirty characters plus a
     number starting at zero indicating the test run. `tmp_path` adds randomness in the parent folders. Because most test
-    start with `test_dashboards_deploys_dashboard_`, the dashboard name for most tests ends up being
+    start with `test_dashboards_creates_dashboard_`, the dashboard name for most tests ends up being
     `test_dashboard_deploys_dashboa0.lvdash.json`, causing collisions. This is solved by adding a random subfolder name.
     """
     folder = tmp_path / f"created_by_lsql_{make_random()}"
     folder.mkdir(parents=True, exist_ok=True)
     return folder
 
 
-def test_dashboards_deploys_exported_dashboard_definition(ws, make_dashboard):
+def test_dashboards_creates_exported_dashboard_definition(ws, make_dashboard):
     dashboards = Dashboards(ws)
     sdk_dashboard = make_dashboard()
+    dashboard_content = (Path(__file__).parent / "dashboards" / "dashboard.lvdash.json").read_text()
 
-    dashboard_file = Path(__file__).parent / "dashboards" / "dashboard.lvdash.json"
-    lakeview_dashboard = Dashboard.from_dict(json.loads(dashboard_file.read_text()))
-
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    ws.lakeview.update(sdk_dashboard.dashboard_id, serialized_dashboard=dashboard_content)
+    lakeview_dashboard = Dashboard.from_dict(json.loads(dashboard_content))
     new_dashboard = dashboards.get_dashboard(sdk_dashboard.path)
 
     assert (
@@ -92,13 +91,12 @@ def test_dashboard_deploys_dashboard_the_same_as_created_dashboard(ws, make_dash
 
     (tmp_path / "counter.sql").write_text("SELECT 10 AS count")
     dashboard_metadata = DashboardMetadata.from_path(tmp_path)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
 
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
     new_dashboard = dashboards.get_dashboard(sdk_dashboard.path)
 
     assert (
-        dashboards._with_better_names(lakeview_dashboard).as_dict()
+        dashboards._with_better_names(dashboard_metadata.as_lakeview()).as_dict()
         == dashboards._with_better_names(new_dashboard).as_dict()
     )
 
@@ -110,9 +108,8 @@ def test_dashboard_deploys_dashboard_with_ten_counters(ws, make_dashboard, tmp_p
     for i in range(10):
         (tmp_path / f"counter_{i}.sql").write_text(f"SELECT {i} AS count")
     dashboard_metadata = DashboardMetadata.from_path(tmp_path)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
 
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
 
     assert ws.lakeview.get(sdk_dashboard.dashboard_id)
 
@@ -124,9 +121,8 @@ def test_dashboard_deploys_dashboard_with_display_name(ws, make_dashboard, tmp_p
     (tmp_path / "dashboard.yml").write_text("display_name: Counter")
     (tmp_path / "counter.sql").write_text("SELECT 102132 AS count")
     dashboard_metadata = DashboardMetadata.from_path(tmp_path)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
 
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
 
     assert ws.lakeview.get(sdk_dashboard.dashboard_id)
 
@@ -137,9 +133,8 @@ def test_dashboard_deploys_dashboard_with_counter_variation(ws, make_dashboard,
 
     (tmp_path / "counter.sql").write_text("SELECT 10 AS `Something Else Than Count`")
     dashboard_metadata = DashboardMetadata.from_path(tmp_path)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
 
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
 
     assert ws.lakeview.get(sdk_dashboard.dashboard_id)
 
@@ -151,14 +146,13 @@ def test_dashboard_deploys_dashboard_with_big_widget(ws, make_dashboard, tmp_pat
     query = """-- --width 6 --height 3\nSELECT 82917019218921 AS big_number_needs_big_widget"""
     (tmp_path / "counter.sql").write_text(query)
     dashboard_metadata = DashboardMetadata.from_path(tmp_path)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
 
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
 
     assert ws.lakeview.get(sdk_dashboard.dashboard_id)
 
 
-def test_dashboards_deploys_dashboard_with_order_overwrite_in_query_header(ws, make_dashboard, tmp_path):
+def test_dashboards_creates_dashboard_with_order_overwrite_in_query_header(ws, make_dashboard, tmp_path):
     dashboards = Dashboards(ws)
     sdk_dashboard = make_dashboard()
 
@@ -168,14 +162,13 @@ def test_dashboards_deploys_dashboard_with_order_overwrite_in_query_header(ws, m
     # order tiebreaker the query name decides the final order.
     (tmp_path / "4.sql").write_text("-- --order 1\nSELECT 4 AS count")
     dashboard_metadata = DashboardMetadata.from_path(tmp_path)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
 
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
 
     assert ws.lakeview.get(sdk_dashboard.dashboard_id)
 
 
-def test_dashboards_deploys_dashboard_with_order_overwrite_in_dashboard_yaml(ws, make_dashboard, tmp_path):
+def test_dashboards_creates_dashboard_with_order_overwrite_in_dashboard_yaml(ws, make_dashboard, tmp_path):
     dashboards = Dashboards(ws)
     sdk_dashboard = make_dashboard()
 
@@ -192,9 +185,8 @@ def test_dashboards_deploys_dashboard_with_order_overwrite_in_dashboard_yaml(ws,
     for query_name in range(6):
         (tmp_path / f"query_{query_name}.sql").write_text(f"SELECT {query_name} AS count")
     dashboard_metadata = DashboardMetadata.from_path(tmp_path)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
 
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
 
     assert ws.lakeview.get(sdk_dashboard.dashboard_id)
 
@@ -205,58 +197,40 @@ def test_dashboard_deploys_dashboard_with_table(ws, make_dashboard):
 
     dashboard_folder = Path(__file__).parent / "dashboards" / "one_table"
     dashboard_metadata = DashboardMetadata.from_path(dashboard_folder)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
 
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
 
     assert ws.lakeview.get(sdk_dashboard.dashboard_id)
 
 
-def test_dashboards_deploys_dashboard_with_invalid_query(ws, make_dashboard, tmp_path):
-    dashboards = Dashboards(ws)
-    sdk_dashboard = make_dashboard()
-
-    for query_name in range(6):
-        (tmp_path / f"{query_name}.sql").write_text(f"SELECT {query_name} AS count")
-    (tmp_path / "4.sql").write_text("SELECT COUNT(* AS invalid_column")
-    dashboard_metadata = DashboardMetadata.from_path(tmp_path)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
-
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
-
-    assert ws.lakeview.get(sdk_dashboard.dashboard_id)
-
-
-def test_dashboards_deploys_dashboard_with_markdown_header(ws, make_dashboard, tmp_path):
+def test_dashboards_creates_dashboard_with_markdown_header(ws, make_dashboard, tmp_path):
     dashboards = Dashboards(ws)
     sdk_dashboard = make_dashboard()
 
     for count, query_name in enumerate("abcdef"):
         (tmp_path / f"{query_name}.sql").write_text(f"SELECT {count} AS count")
     (tmp_path / "z_description.md").write_text("---\norder: -1\n---\nBelow you see counters.")
     dashboard_metadata = DashboardMetadata.from_path(tmp_path)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
 
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
 
     assert ws.lakeview.get(sdk_dashboard.dashboard_id)
 
 
-def test_dashboards_deploys_dashboard_with_widget_title_and_description(ws, make_dashboard, tmp_path):
+def test_dashboards_creates_dashboard_with_widget_title_and_description(ws, make_dashboard, tmp_path):
     dashboards = Dashboards(ws)
     sdk_dashboard = make_dashboard()
 
     description = "-- --title 'Counting' --description 'The answer to life'\nSELECT 42"
     (tmp_path / "counter.sql").write_text(description)
     dashboard_metadata = DashboardMetadata.from_path(tmp_path)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
 
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
 
     assert ws.lakeview.get(sdk_dashboard.dashboard_id)
 
 
-def test_dashboards_deploys_dashboard_from_query_with_cte(ws, make_dashboard, tmp_path):
+def test_dashboards_creates_dashboard_from_query_with_cte(ws, make_dashboard, tmp_path):
     dashboards = Dashboards(ws)
     sdk_dashboard = make_dashboard()
 
@@ -269,24 +243,22 @@ def test_dashboards_deploys_dashboard_from_query_with_cte(ws, make_dashboard, tm
     )
     (tmp_path / "table.sql").write_text(query_with_cte)
     dashboard_metadata = DashboardMetadata.from_path(tmp_path)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
 
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
 
     assert ws.lakeview.get(sdk_dashboard.dashboard_id)
 
 
-def test_dashboards_deploys_dashboard_with_filters(ws, make_dashboard, tmp_path):
+def test_dashboards_creates_dashboard_with_filters(ws, make_dashboard, tmp_path):
     dashboards = Dashboards(ws)
     sdk_dashboard = make_dashboard()
 
     table_query_path = Path(__file__).parent / "dashboards/one_table/databricks_office_locations.sql"
     office_locations = table_query_path.read_text()
     (tmp_path / "table.sql").write_text(f"-- --width 2 --filter City State Country\n{office_locations}")
     dashboard_metadata = DashboardMetadata.from_path(tmp_path)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
 
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
 
     assert ws.lakeview.get(sdk_dashboard.dashboard_id)
 
@@ -298,8 +270,22 @@ def test_dashboard_deploys_dashboard_with_empty_title(ws, make_dashboard, tmp_pa
     query = '-- --overrides \'{"spec": {"frame": {"showTitle": true}}}\'\nSELECT 102132 AS count'
     (tmp_path / "counter.sql").write_text(query)
     dashboard_metadata = DashboardMetadata.from_path(tmp_path)
-    lakeview_dashboard = dashboard_metadata.as_lakeview()
 
-    sdk_dashboard = dashboards.deploy_dashboard(lakeview_dashboard, dashboard_id=sdk_dashboard.dashboard_id)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
+
+    assert ws.lakeview.get(sdk_dashboard.dashboard_id)
+
+
+def test_dashboards_creates_dashboard_via_legacy_method(ws, make_dashboard, tmp_path):
+    dashboards = Dashboards(ws)
+    sdk_dashboard = make_dashboard()
+
+    (tmp_path / "a.md").write_text("Below you see counters.")
+    for count, query_name in enumerate("bcdefg"):
+        (tmp_path / f"{query_name}.sql").write_text(f"SELECT {count} AS count")
+    dashboard_metadata = DashboardMetadata.from_path(tmp_path)
+    dashboard = dashboard_metadata.as_lakeview()
+
+    sdk_dashboard = dashboards.deploy_dashboard(dashboard, dashboard_id=sdk_dashboard.dashboard_id)
 
     assert ws.lakeview.get(sdk_dashboard.dashboard_id)