[COST-5773] Handle multiple resource for OCP on GCP managed flow

Makefile

@@ -188,7 +188,7 @@ delete-test-sources:
 delete-cost-models:
 	$(PYTHON) $(SCRIPTDIR)/delete_cost_models.py
 
-delete-test-customer-data: delete-test-sources delete-cost-models
+delete-test-customer-data: delete-test-sources delete-cost-models delete-testing
 
 test_source=all
 load-test-customer-data:

dev/scripts/load_test_customer_data.sh

@@ -213,6 +213,32 @@ trigger_ocp_ingest() {
       fi
     done
 
+    check_has_data() {
+          local source_name=$1
+          response=$(curl -s "${KOKU_URL_PREFIX}/v1/sources/")
+          has_data=$(echo "$response" | jq -r --arg source_name "$source_name" '.data[] | select(.name == $source_name) | .has_data')
+      }
+
+    local max_retries=50
+    local retries=0
+
+    while [ "$retries" -lt "$max_retries" ]; do
+      check_has_data "$1"
+
+      if [ "$has_data" == "true" ]; then
+        log-info "has_data is true for source_name $1, proceeding."
+        break
+      else
+        retries=$((retries + 1))
+        local wait_time=$((RANDOM % 10 + 1))
+        log-info "has_data is false for source_name $1, retrying in $wait_time seconds... (Attempt $retries/$max_retries)"
+        sleep "$wait_time"
+      fi
+    done
+    if [ "$has_data" != "true" ]; then
+        log-err "Failed to find has_data=true for source_name $1 after $max_retries retries."
+    fi
+
   else
       log-info "SKIPPED - ocp ingest, source_name: $1"
   fi
@@ -272,14 +298,15 @@ build_aws_data() {
   local _download_types=("Test AWS Source")
   local _ocp_ingest_name="Test OCP on AWS"
   local _ocp_payload
-  _ocp_payload="$(uuidgen | awk '{print tolower($0)}' | tr -d '-')"
+  log-info "Building OpenShift on ${_source_name} report data..."
 
   log-info "Rendering ${_source_name} YAML files..."
   render_yaml_files "${_yaml_files[@]}"
 
-  log-info "Building OpenShift on ${_source_name} report data..."
+  local _ocp_payload
+  _ocp_payload="$(uuidgen | awk '{print tolower($0)}' | tr -d '-')"
   nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload"
-  # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload http://localhost:9000 --payload-name "$_ocp_payload"
+  trigger_ocp_ingest "$_ocp_ingest_name" "$_ocp_payload"
   nise_report aws --static-report-file "$YAML_PATH/ocp_on_aws/rendered_aws_static_data.yml" --aws-s3-report-name None --aws-s3-bucket-name "$NISE_DATA_PATH/local_providers/aws_local"
 
   log-info "Cleanup ${_source_name} rendered YAML files..."
@@ -291,7 +318,6 @@ build_aws_data() {
 
   log-info "Trigger downloads..."
   trigger_download "${_download_types[@]}"
-  trigger_ocp_ingest "$_ocp_ingest_name" "$_ocp_payload"
 }
 
 # Azure customer data
@@ -307,14 +333,15 @@ build_azure_data() {
 
   local _download_types=("Test Azure Source" "Test Azure v2 Source")
   local _ocp_ingest_name="Test OCP on Azure"
-  local _ocp_payload
-  _ocp_payload="$(uuidgen | awk '{print tolower($0)}' | tr -d '-')"
 
   log-info "Rendering ${_source_name} YAML files..."
   render_yaml_files "${_yaml_files[@]}"
 
   log-info "Building OpenShift on ${_source_name} report data..."
+  local _ocp_payload
+  _ocp_payload="$(uuidgen | awk '{print tolower($0)}' | tr -d '-')"
   nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload"
+  trigger_ocp_ingest "$_ocp_ingest_name" "$_ocp_payload"
   # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload http://localhost:9000 --payload-name "$_ocp_payload"
   nise_report azure --static-report-file "$YAML_PATH/ocp_on_azure/rendered_azure_static_data.yml" --azure-container-name "$NISE_DATA_PATH/local_providers/azure_local" --azure-report-name azure-report
   nise_report azure --static-report-file "$YAML_PATH/rendered_azure_v2.yml" --azure-container-name "$NISE_DATA_PATH/local_providers/azure_local" --azure-report-name azure-report-v2 --resource-group
@@ -327,31 +354,45 @@ build_azure_data() {
 
   log-info "Trigger downloads..."
   trigger_download "${_download_types[@]}"
-  trigger_ocp_ingest "$_ocp_ingest_name" "$_ocp_payload"
 }
 
 # GCP customer data
 build_gcp_data() {
+  log-info "Building OpenShift on ${_source_name} report data..."
   local _source_name="GCP"
   local _yaml_files=("gcp/gcp_static_data.yml"
                      "ocp_on_gcp/ocp_static_data.yml"
+                     "ocp_on_gcp/ocp_static_replicate_pvc.yml"
                      "ocp_on_gcp/gcp_static_data.yml")
 
   local _rendered_yaml_files=("$YAML_PATH/gcp/rendered_gcp_static_data.yml"
                               "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml"
+                              "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_replicate_pvc.yml"
                               "$YAML_PATH/ocp_on_gcp/rendered_gcp_static_data.yml")
 
   local _download_types=("Test GCP Source" "Test OCPGCP Source")
-  local _ocp_ingest_name="Test OCP on GCP"
+  local _ocp_ingest_names=("Test OCP on GCP duplicate" "Test OCP on GCP")
   local _ocp_payload
-  _ocp_payload="$(uuidgen | awk '{print tolower($0)}' | tr -d '-')"
 
   log-info "Rendering ${_source_name} YAML files..."
   render_yaml_files "${_yaml_files[@]}"
 
-  log-info "Building OpenShift on ${_source_name} report data..."
-  nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload"
-  # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload http://localhost:9000 --payload-name "$_ocp_payload"
+  for i in "${!_ocp_ingest_names[@]}"; do
+      _ocp_ingest_name="${_ocp_ingest_names[$i]}"
+
+      # Generate a new unique payload for each source
+      _ocp_payload="$(uuidgen | awk '{print tolower($0)}' | tr -d '-')"
+      log-info "Triggering OCP ingest for $_ocp_ingest_name with new payload $_ocp_payload"
+
+      if [[ "$i" -eq 0 ]]; then
+          nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_replicate_pvc.yml" --ocp-cluster-id test-ocp-gcp-cluster-duplicate --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload"
+      elif [[ "$i" -eq 1 ]]; then
+          nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload"
+      fi
+      trigger_ocp_ingest "$_ocp_ingest_name" "$_ocp_payload"
+  done
+
+
   nise_report gcp --static-report-file "$YAML_PATH/gcp/rendered_gcp_static_data.yml" --gcp-bucket-name "$NISE_DATA_PATH/local_providers/gcp_local"
   nise_report gcp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_gcp_static_data.yml" --gcp-bucket-name "$NISE_DATA_PATH/local_providers/gcp_local_0" -r
 
@@ -363,7 +404,7 @@ build_gcp_data() {
 
   log-info "Trigger downloads..."
   trigger_download "${_download_types[@]}"
-  trigger_ocp_ingest "$_ocp_ingest_name" "$_ocp_payload"
+
 }
 
 # ONPREM customer data

dev/scripts/nise_ymls/ocp_on_gcp/gcp_static_data.yml

@@ -13,6 +13,18 @@ generators:
       resource.name: projects/nise-populator/instances/gcp_compute1
       resource.global_name: //compute.googleapis.com/projects/nise-populator/zones/australia-southeast1-a/instances/3447398860992947181
       labels: [{"environment": "clyde", "app":"winter", "version":"green", "kubernetes-io-cluster-c32se93c-73z3-3s3d-cs23-d3245sj45349": "owned"}]
+  - ComputeEngineGenerator:
+      start_date: {{start_date}}
+      end_date: {{end_date}}
+      price: 2
+      sku_id: CF4E-A0C7-E3BF
+      usage.amount_in_pricing_units: 1
+      usage.pricing_unit: hour
+      currency: USD
+      instance_type: m2-megamem-416
+      location.region: australia-southeast1-a
+      resource.name: projects/nise-populator/instances/sushi_compute1
+      resource.global_name: //compute.googleapis.com/projects/nise-populator/zones/australia-southeast1-a/instances/3447398860992947181
   - ComputeEngineGenerator:
       start_date: {{start_date}}
       end_date: {{end_date}}

dev/scripts/nise_ymls/ocp_on_gcp/ocp_static_replicate_pvc.yml

@@ -0,0 +1,37 @@
+---
+generators:
+  - OCPGenerator:
+      start_date: {{start_date}}
+      end_date: {{end_date}}
+      nodes:
+        - node:
+          node_name: sushi_compute1
+          cpu_cores: 4
+          memory_gig: 16
+          namespaces:
+            fall:
+              pods:
+                - pod:
+                  pod_name: nilla
+                  cpu_request: 1
+                  mem_request_gig: 2
+                  cpu_limit: 1
+                  mem_limit_gig: 4
+                  pod_seconds: 3600
+                - pod:
+                  pod_name: pod_name1b
+                  cpu_request: 1
+                  mem_request_gig: 2
+                  cpu_limit: 1
+                  mem_limit_gig: 4
+                  pod_seconds: 3600
+              volumes:
+                - volume:
+                  volume_name: pvc-volume_1
+                  storage_class: pd-standard
+                  volume_request_gig: 20
+                  volume_claims:
+                  - volume_claim:
+                    volume_claim_name: nilla_data
+                    pod_name: nilla
+                    capacity_gig: 20

dev/scripts/test_customer.yaml

@@ -92,6 +92,12 @@ customer:
       authentication:
         credentials:
           cluster_id: test-ocp-gcp-cluster
+    - ocp_on_gcp_source_replicate_pvc:
+      source_name: Test OCP on GCP duplicate
+      source_type: 'OCP'
+      authentication:
+        credentials:
+          cluster_id: test-ocp-gcp-cluster-duplicate
     - gcp_source:
       source_name: Test OCPGCP Source
       source_type: 'GCP-local'

koku/masu/database/aws_report_db_accessor.py

@@ -7,6 +7,8 @@
 import logging
 import pkgutil
 import uuid
+from typing import Any
+from typing import List
 
 from dateutil.parser import parse
 from django.db import connection
@@ -22,6 +24,7 @@
 from masu.database import OCP_REPORT_TABLE_MAP
 from masu.database.report_db_accessor_base import ReportDBAccessorBase
 from masu.processor import is_feature_unattributed_storage_enabled_aws
+from masu.processor.parquet.managed_flow_params import ManagedSqlMetadata
 from reporting.models import OCP_ON_ALL_PERSPECTIVES
 from reporting.models import OCP_ON_AWS_PERSPECTIVES
 from reporting.models import OCP_ON_AWS_TEMP_MANAGED_TABLES
@@ -477,53 +480,48 @@ def populate_ec2_compute_summary_table_trino(self, source_uuid, start_date, bill
 
         self._execute_trino_raw_sql_query(sql, sql_params=sql_params, log_ref=f"{table_name}.sql")
 
-    def verify_populate_ocp_on_cloud_daily_trino(self, verification_params):
+    def verify_populate_ocp_on_cloud_daily_trino(self, verification_tags: List[str], sql_metadata: ManagedSqlMetadata):
         """
         Verify the managed trino table population went successfully.
         """
+        params = sql_metadata.build_params(["schema", "cloud_source_uuid", "year", "month"])
+        params["matched_tag_array"] = verification_tags
         verification_sql = pkgutil.get_data("masu.database", "trino_sql/verify/managed_ocp_on_aws_verification.sql")
         verification_sql = verification_sql.decode("utf-8")
-        LOG.info(log_json(msg="running verification for managed OCP on AWS daily SQL", **verification_params))
-        result = self._execute_trino_multipart_sql_query(verification_sql, bind_params=verification_params)
+        LOG.info(log_json(msg="running verification for managed OCP on AWS daily SQL", **params))
+        result = self._execute_trino_multipart_sql_query(verification_sql, bind_params=params)
         if False in result[0]:
-            LOG.error(log_json(msg="Verification failed", **verification_params))
+            LOG.error(log_json(msg="Verification failed", **params))
         else:
-            LOG.info(log_json(msg="Verification successful", **verification_params))
+            LOG.info(log_json(msg="Verification successful", **params))
 
-    def populate_ocp_on_cloud_daily_trino(
-        self, aws_provider_uuid, openshift_provider_uuid, start_date, end_date, matched_tags
-    ):
+    def populate_ocp_on_cloud_daily_trino(self, sql_metadata: ManagedSqlMetadata) -> Any:
         """Populate the managed_aws_openshift_daily trino table for OCP on AWS.
         Args:
-            aws_provider_uuid (UUID) AWS source UUID.
-            ocp_provider_uuid (UUID) OCP source UUID.
-            start_date (datetime.date) The date to start populating the table.
-            end_date (datetime.date) The date to end on.
-            matched_tag_strs (str) matching tags.
+            sql_metadata: object of ManagedSqlMetadata class
         Returns
             (None)
         """
-        year = start_date.strftime("%Y")
-        month = start_date.strftime("%m")
-        table = TRINO_MANAGED_OCP_AWS_DAILY_TABLE
-        days = self.date_helper.list_days(start_date, end_date)
-        days_tup = tuple(str(day.day) for day in days)
-        self.delete_ocp_on_aws_hive_partition_by_day(
-            days_tup, aws_provider_uuid, openshift_provider_uuid, year, month, table
-        )
-
-        summary_sql = pkgutil.get_data("masu.database", "trino_sql/aws/openshift/managed_aws_openshift_daily.sql")
-        summary_sql = summary_sql.decode("utf-8")
-        summary_sql_params = {
-            "schema": self.schema,
-            "start_date": start_date,
-            "year": year,
-            "month": month,
-            "days": days_tup,
-            "end_date": end_date,
-            "aws_source_uuid": aws_provider_uuid,
-            "ocp_source_uuid": openshift_provider_uuid,
-            "matched_tag_array": matched_tags,
-        }
-        LOG.info(log_json(msg="running managed OCP on AWS daily SQL", **summary_sql_params))
-        self._execute_trino_multipart_sql_query(summary_sql, bind_params=summary_sql_params)
+        verification_tags = []
+        for ocp_provider_uuid in sql_metadata.ocp_source_uuids:
+            matched_tags_result = self.find_openshift_keys_expected_values(ocp_provider_uuid, sql_metadata)
+            verification_tags.extend(matched_tags_result)
+            self.delete_ocp_on_aws_hive_partition_by_day(
+                sql_metadata.days_tup,
+                sql_metadata.cloud_provider_uuid,
+                ocp_provider_uuid,
+                sql_metadata.year,
+                sql_metadata.month,
+                TRINO_MANAGED_OCP_AWS_DAILY_TABLE,
+            )
+            summary_sql_params = sql_metadata.build_params(
+                ["schema", "start_date", "year", "month", "days", "end_date", "cloud_provider_uuid"]
+            )
+            summary_sql_params["ocp_source_uuid"] = ocp_provider_uuid
+            summary_sql_params["matched_tag_array"] = matched_tags_result
+            LOG.info(log_json(msg="running managed OCP on AZURE daily SQL", **summary_sql_params))
+            summary_sql = pkgutil.get_data("masu.database", "trino_sql/aws/openshift/managed_aws_openshift_daily.sql")
+            summary_sql = summary_sql.decode("utf-8")
+            self._execute_trino_multipart_sql_query(summary_sql, bind_params=summary_sql_params)
+        verification_tags = list(dict.fromkeys(verification_tags))
+        self.verify_populate_ocp_on_cloud_daily_trino(verification_tags, sql_metadata)