From 0cd9551765d44deed58da4fbb31b1b3dc01cf29a Mon Sep 17 00:00:00 2001 From: Siddique Bagwan Date: Thu, 19 Dec 2024 09:24:58 +0530 Subject: [PATCH] lookml parameter support --- .../source/looker/looker_template_language.py | 31 ++- .../tests/integration/lookml/test_lookml.py | 1 + .../data.model.lkml | 4 + .../star_award_winner.view.lkml | 23 +++ .../vv_lineage_liquid_template_golden.json | 186 ++++++++++++++++++ 5 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py index 6d49d57e077435..b1beb96b7eedab 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py @@ -198,9 +198,9 @@ def __init__(self, source_config: LookMLSourceConfig): def transform(self, view: dict) -> dict: value_to_transform: Optional[str] = None - # is_attribute_supported check is required because not all transformer works on all attributes in current - # case mostly all transformer works on sql_table_name and derived.sql attributes, - # however IncompleteSqlTransformer only transform the derived.sql attribute + # is_attribute_supported check is required because not all transformers work on all attributes in the current + # case, mostly all transformers work on sql_table_name and derived.sql attributes; + # however, IncompleteSqlTransformer only transform the derived.sql attribute if SQL_TABLE_NAME in view and self.is_attribute_supported(SQL_TABLE_NAME): # Give precedence to already processed transformed view.sql_table_name to apply more transformation value_to_transform = view.get( @@ -287,7 +287,7 @@ def _apply_transformation(self, value: str, view: dict) -> str: class DropDerivedViewPatternTransformer(LookMLViewTransformer): """ - drop ${} from datahub_transformed_sql_table_name and view["derived_table"]["datahub_transformed_sql_table_name"] values. + drop ${} from datahub_transformed_sql_table_name and view["derived_table"]["datahub_transformed_sql_table_name"] values. Example: transform ${employee_income_source.SQL_TABLE_NAME} to employee_income_source.SQL_TABLE_NAME """ @@ -335,6 +335,26 @@ def _apply_transformation(self, value: str, view: dict) -> str: return self._apply_regx(value) +class LookmlParameterTransformer(LookMLViewTransformer): + """ + Replace the lookml parameter @{variable} with their values. + """ + + PATTERN = r"@{(\w+)}" + + def resolve_lookml_parameter(self, text: str) -> str: + return re.sub( + LookmlParameterTransformer.PATTERN, + lambda match: self.source_config.liquid_variable.get( + match.group(1), match.group(0) + ), # Replace or keep original + text, + ) + + def _apply_transformation(self, value: str, view: dict) -> str: + return self.resolve_lookml_parameter(text=value) + + class TransformedLookMlView: """ TransformedLookMlView is collecting output of LookMLViewTransformer and creating a new transformed LookML view. @@ -401,6 +421,9 @@ def process_lookml_template_language( LiquidVariableTransformer( source_config=source_config ), # Now resolve liquid variables + LookmlParameterTransformer( + source_config=source_config + ), # Remove @{variable} with its corresponding value DropDerivedViewPatternTransformer( source_config=source_config ), # Remove any ${} symbol diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index 940e7f36675f79..66b49cdb6eaaa1 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -873,6 +873,7 @@ def test_view_to_view_lineage_and_liquid_template(pytestconfig, tmp_path, mock_t "_is_selected": True, }, "source_region": "ap-south-1", + "star_award_winner_year": "public.winner_2025", } pipeline = Pipeline.create(new_recipe) diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml index d570e0ecdb5b22..ae6f70443ab567 100644 --- a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml @@ -10,6 +10,7 @@ include: "environment_activity_logs.view.lkml" include: "employee_income_source_as_per_env.view.lkml" include: "rent_as_employee_income_source.view.lkml" include: "child_view.view.lkml" +include: "star_award_winner.view.lkml" explore: activity_logs { } @@ -39,4 +40,7 @@ explore: rent_as_employee_income_source { } explore: child_view { +} + +explore: star_award_winner { } \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml new file mode 100644 index 00000000000000..e8f1a8aa48f95c --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml @@ -0,0 +1,23 @@ +view: star_award_winner { + sql_table_name: @{star_award_winner_year} ;; + + dimension: id { + label: "id" + primary_key: yes + type: number + sql: ${TABLE}.id ;; + } + + parameter: star_award_winner_year { + type: string + allowed_value: { + label: "Star Award Winner Of 2025" + value: "public.winner_2025" + } + allowed_value: { + label: "Star Award Winner Of 2024" + value: "public.winner_2024" + } + } + +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json index 468450310c2abc..9af13f2a6b0a1a 100644 --- a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json +++ b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json @@ -2827,6 +2827,192 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "view: star_award_winner {\n sql_table_name: @{star_award_winner_year} ;;\n\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: number\n sql: ${TABLE}.id ;;\n }\n\n parameter: star_award_winner_year {\n type: string\n allowed_value: {\n label: \"Star Award Winner Of 2025\"\n value: \"public.winner_2025\"\n }\n allowed_value: {\n label: \"Star Award Winner Of 2024\"\n value: \"public.winner_2024\"\n }\n }\n\n}", + "viewLanguage": "lookml" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Develop/lkml_samples/" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.public.winner_2025,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.winner_2025,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD),id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "star_award_winner", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "description": "", + "label": "id", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": true + } + ], + "primaryKeys": [ + "id" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "star_award_winner.view.lkml", + "looker.model": "data" + }, + "name": "star_award_winner", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:Dimension",