feat!: migrating to TheLook Ecommerce dataset

.github/renovate.json

@@ -30,15 +30,23 @@
     },
     {"matchDepTypes": ["module"], "groupName": "TF modules"},
     {
-      "matchDepTypes": ["require"],
+      "matchDepTypes": [
+        "require"
+      ],
       "groupName": "GO modules",
-      "postUpdateOptions": ["gomodTidy"]
+      "postUpdateOptions": [
+        "gomodTidy"
+      ]
     },
     {
-      "matchDatasources": ["golang-version"],
+      "matchDatasources": [
+        "golang-version"
+      ],
       "rangeStrategy": "bump",
       "allowedVersions": "<1.21.0",
-      "postUpdateOptions": ["gomodTidy"]
+      "postUpdateOptions": [
+        "gomodTidy"
+      ]
     },
     {
       "matchDepNames": ["google", "google-beta"],

modules/data_warehouse/README.md

@@ -12,7 +12,7 @@ The resources/services/activations/deletions that this module will create/trigge
 - Creates a BigQuery Dataset
 - Creates a BigQuery Table
 - Creates a Google Cloud Storage bucket
-- Loads the Google Cloud Storage bucket with data from https://console.cloud.google.com/marketplace/product/city-of-new-york/nyc-tlc-trips
+- Loads the Google Cloud Storage bucket with data from [TheLook eCommerce Public Dataset](https://console.cloud.google.com/marketplace/product/bigquery-public-data/thelook-ecommerce)
 - Provides SQL examples
 - Creates and inferences with a BigQuery ML model
 - Creates a Looker Studio report

modules/data_warehouse/bigquery.tf

@@ -24,6 +24,8 @@ resource "google_bigquery_dataset" "ds_edw" {
   location                   = var.region
   labels                     = var.labels
   delete_contents_on_destroy = var.force_destroy
+
+  depends_on = [time_sleep.wait_after_apis]
 }
 
 # # Create a BigQuery connection
@@ -33,6 +35,7 @@ resource "google_bigquery_connection" "ds_connection" {
   location      = var.region
   friendly_name = "Storage Bucket Connection"
   cloud_resource {}
+  depends_on = [time_sleep.wait_after_apis]
 }
 
 # # Grant IAM access to the BigQuery Connection account for Cloud Storage
@@ -48,22 +51,127 @@ resource "google_storage_bucket_iam_binding" "bq_connection_iam_object_viewer" {
   ]
 }
 
-# # Create a BigQuery external table
-resource "google_bigquery_table" "tbl_edw_taxi" {
+# # Create a Biglake table for events with metadata caching
+resource "google_bigquery_table" "tbl_edw_events" {
+  dataset_id          = google_bigquery_dataset.ds_edw.dataset_id
+  table_id            = "events"
+  project             = module.project-services.project_id
+  deletion_protection = var.deletion_protection
+  # max_staleness =  "1:0:0"
+
+  schema = file("${path.module}/src/schema/events_schema.json")
+
+  external_data_configuration {
+    autodetect    = true
+    connection_id = google_bigquery_connection.ds_connection.name
+    source_format = "PARQUET"
+    source_uris   = ["gs://${google_storage_bucket.raw_bucket.name}/thelook-ecommerce/events.parquet"]
+    # metadata_cache_mode = "AUTOMATIC"
+  }
+
+  labels = var.labels
+
+  depends_on = [
+    google_bigquery_connection.ds_connection,
+    google_storage_bucket.raw_bucket,
+  ]
+}
+
+# # Create a Biglake table for inventory_items
+resource "google_bigquery_table" "tbl_edw_inventory_items" {
   dataset_id          = google_bigquery_dataset.ds_edw.dataset_id
-  table_id            = "taxi_trips"
+  table_id            = "inventory_items"
   project             = module.project-services.project_id
   deletion_protection = var.deletion_protection
+  # max_staleness =  "1:0:0"
+
+  schema = file("${path.module}/src/schema/inventory_items_schema.json")
 
   external_data_configuration {
     autodetect    = true
-    connection_id = "${module.project-services.project_id}.${var.region}.ds_connection"
+    connection_id = google_bigquery_connection.ds_connection.name
     source_format = "PARQUET"
-    source_uris   = ["gs://${google_storage_bucket.raw_bucket.name}/new-york-taxi-trips/tlc-yellow-trips-2022/taxi-*.Parquet"]
+    source_uris   = ["gs://${google_storage_bucket.raw_bucket.name}/thelook-ecommerce/inventory_items.parquet"]
+    # metadata_cache_mode = "AUTOMATIC"
+  }
+
+  labels = var.labels
+
+  depends_on = [
+    google_bigquery_connection.ds_connection,
+    google_storage_bucket.raw_bucket,
+  ]
+}
+
+# # Create a Biglake table with metadata caching for order_items
+resource "google_bigquery_table" "tbl_edw_order_items" {
+  dataset_id          = google_bigquery_dataset.ds_edw.dataset_id
+  table_id            = "order_items"
+  project             = module.project-services.project_id
+  deletion_protection = var.deletion_protection
+  # max_staleness =  "1:0:0"
+
+  schema = file("${path.module}/src/schema/order_items_schema.json")
+
+  external_data_configuration {
+    autodetect    = true
+    connection_id = google_bigquery_connection.ds_connection.name
+    source_format = "PARQUET"
+    source_uris   = ["gs://${google_storage_bucket.raw_bucket.name}/thelook-ecommerce/order_items.parquet"]
+    # metadata_cache_mode = "AUTOMATIC"
+  }
+
+  labels = var.labels
+
+  depends_on = [
+    google_bigquery_connection.ds_connection,
+    google_storage_bucket.raw_bucket,
+  ]
+}
+
+# # Create a Biglake table for orders
+resource "google_bigquery_table" "tbl_edw_orders" {
+  dataset_id          = google_bigquery_dataset.ds_edw.dataset_id
+  table_id            = "orders"
+  project             = module.project-services.project_id
+  deletion_protection = var.deletion_protection
+  # max_staleness =  "1:0:0"
 
+  schema = file("${path.module}/src/schema/orders_schema.json")
+
+  external_data_configuration {
+    autodetect    = true
+    connection_id = google_bigquery_connection.ds_connection.name
+    source_format = "PARQUET"
+    source_uris   = ["gs://${google_storage_bucket.raw_bucket.name}/thelook-ecommerce/orders.parquet"]
+    # metadata_cache_mode = "AUTOMATIC"
   }
 
-  schema = file("${path.module}/src/taxi_trips_schema.json")
+  labels = var.labels
+
+  depends_on = [
+    google_bigquery_connection.ds_connection,
+    google_storage_bucket.raw_bucket,
+  ]
+}
+
+# # Create a Biglake table for products
+resource "google_bigquery_table" "tbl_edw_products" {
+  dataset_id          = google_bigquery_dataset.ds_edw.dataset_id
+  table_id            = "products"
+  project             = module.project-services.project_id
+  deletion_protection = var.deletion_protection
+  # max_staleness =  "1:0:0"
+
+  schema = file("${path.module}/src/schema/products_schema.json")
+
+  external_data_configuration {
+    autodetect    = true
+    connection_id = google_bigquery_connection.ds_connection.name
+    source_format = "PARQUET"
+    source_uris   = ["gs://${google_storage_bucket.raw_bucket.name}/thelook-ecommerce/products.parquet"]
+    # metadata_cache_mode = "AUTOMATIC"
+  }
 
   labels = var.labels
 
@@ -73,8 +181,33 @@ resource "google_bigquery_table" "tbl_edw_taxi" {
   ]
 }
 
+# # Create a Biglake table for products
+resource "google_bigquery_table" "tbl_edw_users" {
+  dataset_id          = google_bigquery_dataset.ds_edw.dataset_id
+  table_id            = "users"
+  project             = module.project-services.project_id
+  deletion_protection = var.deletion_protection
+  # max_staleness =  "1:0:0"
+
+  schema = file("${path.module}/src/schema/users_schema.json")
+
+  external_data_configuration {
+    autodetect    = true
+    connection_id = google_bigquery_connection.ds_connection.name
+    source_format = "PARQUET"
+    source_uris   = ["gs://${google_storage_bucket.raw_bucket.name}/thelook-ecommerce/users.parquet"]
+    # metadata_cache_mode = "AUTOMATIC"
+  }
+
+  labels = var.labels
+    depends_on = [
+    google_bigquery_connection.ds_connection,
+    google_storage_bucket.raw_bucket,
+  ]
+}
+
 # Load Queries for Stored Procedure Execution
-# # Load Lookup Data Tables
+# # Load Distribution Center Lookup Data Tables
 resource "google_bigquery_routine" "sp_provision_lookup_tables" {
   project         = module.project-services.project_id
   dataset_id      = google_bigquery_dataset.ds_edw.dataset_id
@@ -88,9 +221,8 @@ resource "google_bigquery_routine" "sp_provision_lookup_tables" {
   ]
 }
 
-
-# # Add Looker Studio Data Report Procedure
-resource "google_bigquery_routine" "sproc_sp_demo_datastudio_report" {
+# Add Looker Studio Data Report Procedure
+resource "google_bigquery_routine" "sproc_sp_demo_lookerstudio_report" {
   project         = module.project-services.project_id
   dataset_id      = google_bigquery_dataset.ds_edw.dataset_id
   routine_id      = "sp_lookerstudio_report"
@@ -99,7 +231,7 @@ resource "google_bigquery_routine" "sproc_sp_demo_datastudio_report" {
   definition_body = templatefile("${path.module}/src/sql/sp_lookerstudio_report.sql", { project_id = module.project-services.project_id })
 
   depends_on = [
-    google_bigquery_table.tbl_edw_taxi,
+    google_bigquery_table.tbl_edw_inventory_items,
   ]
 }
 
@@ -113,11 +245,12 @@ resource "google_bigquery_routine" "sp_sample_queries" {
   definition_body = templatefile("${path.module}/src/sql/sp_sample_queries.sql", { project_id = module.project-services.project_id })
 
   depends_on = [
-    google_bigquery_table.tbl_edw_taxi,
+    google_bigquery_table.tbl_edw_inventory_items,
   ]
 }
 
-# # Add Bigquery ML Model
+
+# Add Bigquery ML Model
 resource "google_bigquery_routine" "sp_bigqueryml_model" {
   project         = module.project-services.project_id
   dataset_id      = google_bigquery_dataset.ds_edw.dataset_id
@@ -127,21 +260,7 @@ resource "google_bigquery_routine" "sp_bigqueryml_model" {
   definition_body = templatefile("${path.module}/src/sql/sp_bigqueryml_model.sql", { project_id = module.project-services.project_id })
 
   depends_on = [
-    google_bigquery_table.tbl_edw_taxi,
-  ]
-}
-
-# # Add Translation Scripts
-resource "google_bigquery_routine" "sp_sample_translation_queries" {
-  project         = module.project-services.project_id
-  dataset_id      = google_bigquery_dataset.ds_edw.dataset_id
-  routine_id      = "sp_sample_translation_queries"
-  routine_type    = "PROCEDURE"
-  language        = "SQL"
-  definition_body = templatefile("${path.module}/src/sql/sp_sample_translation_queries.sql", { project_id = module.project-services.project_id })
-
-  depends_on = [
-    google_bigquery_table.tbl_edw_taxi,
+    google_bigquery_table.tbl_edw_inventory_items,
   ]
 }
 
@@ -151,6 +270,8 @@ resource "google_project_service_identity" "bigquery_data_transfer_sa" {
   provider = google-beta
   project  = module.project-services.project_id
   service  = "bigquerydatatransfer.googleapis.com"
+
+  depends_on = [time_sleep.wait_after_apis]
 }
 
 # # Grant the DTS service account access
@@ -162,6 +283,8 @@ resource "google_project_iam_member" "dts_service_account_roles" {
   project = module.project-services.project_id
   role    = each.key
   member  = "serviceAccount:${google_project_service_identity.bigquery_data_transfer_sa.email}"
+
+  depends_on = [time_sleep.wait_after_apis]
 }
 
 # Create specific service account for DTS Run
@@ -182,6 +305,8 @@ resource "google_project_iam_member" "dts_roles" {
   project = module.project-services.project_id
   role    = each.key
   member  = "serviceAccount:${google_service_account.dts.email}"
+
+  depends_on = [google_service_account.dts]
 }
 
 # # Grant the DTS specific service account Token Creator to the DTS Service Identity
@@ -194,6 +319,8 @@ resource "google_service_account_iam_binding" "dts_token_creator" {
 
   depends_on = [
     google_project_iam_member.dts_service_account_roles,
+    google_service_account.dts,
+    google_project_service_identity.bigquery_data_transfer_sa
   ]
 }