Add meta to seeds/snapshots (#237)

* add meta to seeds/snapshots * remove quotes - bigquery * add parse_json * type_array -> type_json
brooklyn-data · Nov 2, 2022 · e640c06 · e640c06
1 parent c07c8b2
commit e640c06
Show file tree

Hide file tree

Showing 8 changed files with 24 additions and 12 deletions.
diff --git a/macros/upload_seeds.sql b/macros/upload_seeds.sql
@@ -19,7 +19,8 @@
             {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }},
             {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }},
             {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }},
-            {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}
+            {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }},
+            {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(10)) }}
         from values
         {% for seed in seeds -%}
             (
@@ -31,7 +32,8 @@
                 '{{ seed.name }}', {# name #}
                 '{{ seed.package_name }}', {# package_name #}
                 '{{ seed.original_file_path | replace('\\', '\\\\') }}', {# path #}
-                '{{ seed.checksum.checksum }}' {# checksum #}
+                '{{ seed.checksum.checksum }}', {# checksum #}
+                '{{ tojson(seed.config.meta) }}' {# meta #}
             )
             {%- if not loop.last %},{%- endif %}
         {%- endfor %}
@@ -55,7 +57,8 @@
                     '{{ seed.name }}', {# name #}
                     '{{ seed.package_name }}', {# package_name #}
                     '{{ seed.original_file_path | replace('\\', '\\\\') }}', {# path #}
-                    '{{ seed.checksum.checksum }}' {# checksum #}
+                    '{{ seed.checksum.checksum }}', {# checksum #}
+                    parse_json('{{ tojson(seed.config.meta) }}') {# meta #}
                 )
                 {%- if not loop.last %},{%- endif %}
             {%- endfor %}

diff --git a/macros/upload_snapshots.sql b/macros/upload_snapshots.sql
@@ -22,7 +22,8 @@
             {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }},
             {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }},
             {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }},
-            {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}
+            {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }},
+            {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(12)) }}
         from values
         {% for snapshot in snapshots -%}
             (
@@ -36,7 +37,8 @@
                 '{{ snapshot.package_name }}', {# package_name #}
                 '{{ snapshot.original_file_path | replace('\\', '\\\\') }}', {# path #}
                 '{{ snapshot.checksum.checksum }}', {# checksum #}
-                '{{ snapshot.config.strategy }}' {# strategy #}
+                '{{ snapshot.config.strategy }}', {# strategy #}
+                '{{ tojson(snapshot.config.meta) }}' {# meta #}
             )
             {%- if not loop.last %},{%- endif %}
         {%- endfor %}
@@ -62,7 +64,8 @@
                     '{{ snapshot.package_name }}', {# package_name #}
                     '{{ snapshot.original_file_path | replace('\\', '\\\\') }}', {# path #}
                     '{{ snapshot.checksum.checksum }}', {# checksum #}
-                    '{{ snapshot.config.strategy }}' {# strategy #}
+                    '{{ snapshot.config.strategy }}', {# strategy #}
+                    parse_json('{{ tojson(snapshot.config.meta) }}') {# meta #}
                 )
                 {%- if not loop.last %},{%- endif %}
             {%- endfor %}

diff --git a/models/dim_dbt__seeds.sql b/models/dim_dbt__seeds.sql
@@ -17,7 +17,8 @@ seeds as (
         name,
         package_name,
         path,
-        checksum
+        checksum,
+        meta
     from base
 
 )

diff --git a/models/dim_dbt__snapshots.sql b/models/dim_dbt__snapshots.sql
@@ -19,7 +19,8 @@ snapshots as (
         package_name,
         path,
         checksum,
-        strategy
+        strategy,
+        meta
     from base
 
 )

diff --git a/models/sources/seeds.sql b/models/sources/seeds.sql
@@ -12,6 +12,7 @@ select
     cast(null as {{ type_string() }}) as name,
     cast(null as {{ type_string() }}) as package_name,
     cast(null as {{ type_string() }}) as path,
-    cast(null as {{ type_string() }}) as checksum
+    cast(null as {{ type_string() }}) as checksum,
+    cast(null as {{ type_json() }}) as meta
 from dummy_cte
 where 1 = 0
diff --git a/models/sources/snapshots.sql b/models/sources/snapshots.sql
@@ -14,6 +14,7 @@ select
     cast(null as {{ type_string() }}) as package_name,
     cast(null as {{ type_string() }}) as path,
     cast(null as {{ type_string() }}) as checksum,
-    cast(null as {{ type_string() }}) as strategy
+    cast(null as {{ type_string() }}) as strategy,
+    cast(null as {{ type_json() }}) as meta
 from dummy_cte
 where 1 = 0
diff --git a/models/staging/stg_dbt__seeds.sql b/models/staging/stg_dbt__seeds.sql
@@ -17,7 +17,8 @@ enhanced as (
         name,
         package_name,
         path,
-        checksum
+        checksum,
+        meta
     from base
 
 )

diff --git a/models/staging/stg_dbt__snapshots.sql b/models/staging/stg_dbt__snapshots.sql
@@ -19,7 +19,8 @@ enhanced as (
         package_name,
         path,
         checksum,
-        strategy
+        strategy,
+        meta
     from base
 
 )