Skip to content

Commit

Permalink
updates
Browse files Browse the repository at this point in the history
  • Loading branch information
mayurinehate committed Dec 20, 2024
1 parent 1ca0d35 commit 3d839a3
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 7 deletions.
2 changes: 1 addition & 1 deletion docs/how/updating-datahub.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
This file documents any backwards-incompatible changes in DataHub and assists people when migrating to a new version.

## Next
- #12179 - Config `include_view_lineage` is removed from snowflake ingestion source. View and External Table DDL lineage will always be ingested when definitions are available.
- #12191 - Configs `include_view_lineage` and `include_view_column_lineage` is removed from snowflake ingestion source. View and External Table DDL lineage will always be ingested when definitions are available.
- #11560 - The PowerBI ingestion source configuration option include_workspace_name_in_dataset_urn determines whether the workspace name is included in the PowerBI dataset's URN.<br/> PowerBI allows to have identical name of semantic model and their tables across the workspace, It will overwrite the semantic model in-case of multi-workspace ingestion.<br/>
Entity urn with `include_workspace_name_in_dataset_urn: false`

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ class SnowflakeConfig(
)

_include_view_lineage = pydantic_removed_field("include_view_lineage")
_include_view_column_lineage = pydantic_removed_field("include_view_column_lineage")

ignore_start_time_lineage: bool = False
upstream_lineage_in_report: bool = False
Expand Down Expand Up @@ -208,13 +209,8 @@ class SnowflakeV2Config(
description="Populates table->table and view->table column lineage. Requires appropriate grants given to the role and the Snowflake Enterprise Edition or above.",
)

include_view_column_lineage: bool = Field(
default=True,
description="Populates view->view and table->view column lineage using DataHub's sql parser.",
)

use_queries_v2: bool = Field(
default=True,
default=False,
description="If enabled, uses the new queries extractor to extract queries from snowflake.",
)

Expand Down
12 changes: 12 additions & 0 deletions metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -653,3 +653,15 @@ def test_create_snowsight_base_url_ap_northeast_1():

def test_snowflake_utils() -> None:
assert_doctest(datahub.ingestion.source.snowflake.snowflake_utils)


def test_using_removed_fields_causes_no_error() -> None:
assert SnowflakeV2Config.parse_obj(
{
"account_id": "test",
"username": "snowflake",
"password": "snowflake",
"include_view_lineage": "true",
"include_view_column_lineage": "true",
}
)

0 comments on commit 3d839a3

Please sign in to comment.