Fix datalake_recipe parameter type, expose tag variable and update CD…

…P provider (#55)

Signed-off-by: Jim Enright <[email protected]>

modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf

@@ -38,7 +38,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.4.1"
+      version = "0.4.2"
     }
   }
 }

modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf

@@ -51,7 +51,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.4.1"
+      version = "0.4.2"
     }
   }
 }

modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf

@@ -58,7 +58,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.4.1"
+      version = "0.4.2"
     }
   }
 }

modules/terraform-cdp-deploy/README.md

@@ -20,7 +20,7 @@ In each directory an example `terraform.tfvars.sample` values file is included t
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3.0 |
-| <a name="requirement_cdp"></a> [cdp](#requirement\_cdp) | 0.4.1 |
+| <a name="requirement_cdp"></a> [cdp](#requirement\_cdp) | 0.4.2 |
 
 ## Providers
 
@@ -49,6 +49,7 @@ No resources.
 | <a name="input_infra_type"></a> [infra\_type](#input\_infra\_type) | Cloud Provider to deploy CDP. | `string` | n/a | yes |
 | <a name="input_log_storage_location"></a> [log\_storage\_location](#input\_log\_storage\_location) | Log storage location. The location has to be in uri format for the cloud provider - i.e. s3a:// for AWS, abfs:// for Azure,  gs:// | `string` | n/a | yes |
 | <a name="input_region"></a> [region](#input\_region) | Region which cloud resources will be created | `string` | n/a | yes |
+| <a name="input_agent_source_tag"></a> [agent\_source\_tag](#input\_agent\_source\_tag) | Tag to identify deployment source | `map(any)` | <pre>{<br>  "agent_source": "tf-cdp-module"<br>}</pre> | no |
 | <a name="input_aws_datalake_admin_role_arn"></a> [aws\_datalake\_admin\_role\_arn](#input\_aws\_datalake\_admin\_role\_arn) | Datalake Admin Role ARN. Required for CDP deployment on AWS. | `string` | `null` | no |
 | <a name="input_aws_idbroker_instance_profile_arn"></a> [aws\_idbroker\_instance\_profile\_arn](#input\_aws\_idbroker\_instance\_profile\_arn) | IDBroker Instance Profile ARN. Required for CDP deployment on AWS. | `string` | `null` | no |
 | <a name="input_aws_log_instance_profile_arn"></a> [aws\_log\_instance\_profile\_arn](#input\_aws\_log\_instance\_profile\_arn) | Log Instance Profile ARN. Required for CDP deployment on AWS. | `string` | `null` | no |
@@ -86,7 +87,7 @@ No resources.
 | <a name="input_datalake_java_version"></a> [datalake\_java\_version](#input\_datalake\_java\_version) | The Java major version to use on the datalake cluster. | `number` | `null` | no |
 | <a name="input_datalake_name"></a> [datalake\_name](#input\_datalake\_name) | Name of the CDP datalake. Defaults to '<env\_prefix>-<aw\|az\|gc\|>-dl' if not specified. | `string` | `null` | no |
 | <a name="input_datalake_polling_timeout"></a> [datalake\_polling\_timeout](#input\_datalake\_polling\_timeout) | Timeout value in minutes for how long to poll for CDP datalake resource creation/deletion | `number` | `90` | no |
-| <a name="input_datalake_recipes"></a> [datalake\_recipes](#input\_datalake\_recipes) | Additional recipes that will be attached on the datalake instances | <pre>set(<br>    object({<br>      instance_group_name = string,<br>      recipe_names        = set(object({}))<br>    })<br>  )</pre> | `null` | no |
+| <a name="input_datalake_recipes"></a> [datalake\_recipes](#input\_datalake\_recipes) | Additional recipes that will be attached on the datalake instances | <pre>set(<br>    object({<br>      instance_group_name = string,<br>      recipe_names        = set(string)<br>    })<br>  )</pre> | `null` | no |
 | <a name="input_datalake_scale"></a> [datalake\_scale](#input\_datalake\_scale) | The scale of the datalake. Valid values are LIGHT\_DUTY, ENTERPRISE. | `string` | `null` | no |
 | <a name="input_datalake_version"></a> [datalake\_version](#input\_datalake\_version) | The Datalake Runtime version. Valid values are latest or a semantic version, e.g. 7.2.17 | `string` | `"latest"` | no |
 | <a name="input_enable_ccm_tunnel"></a> [enable\_ccm\_tunnel](#input\_enable\_ccm\_tunnel) | Flag to enable Cluster Connectivity Manager tunnel. If false then access from Cloud to CDP Control Plane CIDRs is required from via SG ingress | `bool` | `true` | no |
@@ -96,6 +97,7 @@ No resources.
 | <a name="input_encryption_key_resource_group_name"></a> [encryption\_key\_resource\_group\_name](#input\_encryption\_key\_resource\_group\_name) | Name of the existing Azure resource group hosting the Azure Key Vault containing customer managed key which will be used to encrypt the Azure Managed Disk. Only applicable for CDP deployment on Azure. | `string` | `null` | no |
 | <a name="input_encryption_key_url"></a> [encryption\_key\_url](#input\_encryption\_key\_url) | URL of the key which will be used to encrypt the Azure Managed Disks. Only applicable for CDP deployment on Azure. | `string` | `null` | no |
 | <a name="input_endpoint_access_scheme"></a> [endpoint\_access\_scheme](#input\_endpoint\_access\_scheme) | The scheme for the workload endpoint gateway. PUBLIC creates an external endpoint that can be accessed over the Internet. PRIVATE which restricts the traffic to be internal to the VPC / Vnet. Relevant in Private Networks. | `string` | `null` | no |
+| <a name="input_env_tags"></a> [env\_tags](#input\_env\_tags) | Tags applied to provisioned resources | `map(any)` | `null` | no |
 | <a name="input_environment_name"></a> [environment\_name](#input\_environment\_name) | Name of the CDP environment. Defaults to '<env\_prefix>-cdp-env' if not specified. | `string` | `null` | no |
 | <a name="input_environment_polling_timeout"></a> [environment\_polling\_timeout](#input\_environment\_polling\_timeout) | Timeout value in minutes for how long to poll for CDP Environment resource creation/deletion | `number` | `60` | no |
 | <a name="input_freeipa_catalog"></a> [freeipa\_catalog](#input\_freeipa\_catalog) | Image catalog to use for FreeIPA image selection | `string` | `null` | no |

modules/terraform-cdp-deploy/defaults.tf

@@ -14,12 +14,11 @@
 
 locals {
   # ------- Global settings -------
-  # NOTE: Waiting on provider fix
-  # env_tags = merge(var.agent_source_tag, (
-  #   coalesce(var.env_tags,
-  #     { env_prefix = var.env_prefix }
-  #   ))
-  # )
+  env_tags = merge(var.agent_source_tag, (
+    coalesce(var.env_tags,
+      { env_prefix = var.env_prefix }
+    ))
+  )
 
   # ------- CDP Environment Deployment -------
   environment_name = coalesce(var.environment_name,

modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf

@@ -79,7 +79,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.4.1"
+      version = "0.4.2"
     }
   }
 }

modules/terraform-cdp-deploy/main.tf

@@ -18,7 +18,7 @@ module "cdp_on_aws" {
 
   source = "./modules/aws"
 
-  # tags = local.env_tags # NOTE: Waiting on provider fix
+  tags = local.env_tags
 
   environment_name              = local.environment_name
   datalake_name                 = local.datalake_name
@@ -42,9 +42,6 @@ module "cdp_on_aws" {
   environment_polling_timeout = var.environment_polling_timeout
   datalake_polling_timeout    = var.datalake_polling_timeout
 
-  # TODO: Will be re-introducted once provider supports other regions
-  # cdp_control_plane_region = var.cdp_control_plane_region
-
   region            = var.region
   vpc_id            = var.aws_vpc_id
   public_subnet_ids = var.aws_public_subnet_ids
@@ -88,7 +85,7 @@ module "cdp_on_azure" {
 
   source = "./modules/azure"
 
-  # tags = local.env_tags # NOTE: Waiting on provider fix
+  tags = local.env_tags
 
   environment_name              = local.environment_name
   datalake_name                 = local.datalake_name
@@ -110,9 +107,6 @@ module "cdp_on_azure" {
   environment_polling_timeout = var.environment_polling_timeout
   datalake_polling_timeout    = var.datalake_polling_timeout
 
-  # TODO: Will be re-introducted once provider supports other regions
-  # cdp_control_plane_region = var.cdp_control_plane_region
-
   use_single_resource_group = var.use_single_resource_group
   use_public_ips            = local.use_public_ips
 
@@ -167,7 +161,7 @@ module "cdp_on_gcp" {
 
   source = "./modules/gcp"
 
-  # tags = local.env_tags # NOTE: Waiting on provider fix
+  tags = local.env_tags
 
   environment_name              = local.environment_name
   datalake_name                 = local.datalake_name

modules/terraform-cdp-deploy/modules/aws/main.tf

@@ -67,7 +67,7 @@ resource "cdp_environments_aws_environment" "cdp_env" {
     polling_timeout = var.environment_polling_timeout
   }
 
-  # tags               = var.tags # NOTE: Waiting on provider fix
+  tags = var.tags
 
   depends_on = [
     cdp_environments_aws_credential.cdp_cred
@@ -142,7 +142,7 @@ resource "cdp_datalake_aws_datalake" "cdp_datalake" {
     polling_timeout = var.datalake_polling_timeout
   }
 
-  # tags = var.tags # NOTE: Waiting on provider fix
+  tags = var.tags
 
   depends_on = [
     cdp_environments_aws_credential.cdp_cred,

modules/terraform-cdp-deploy/modules/aws/provider.tf

@@ -16,7 +16,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.4.1"
+      version = "0.4.2"
     }
   }
 

modules/terraform-cdp-deploy/modules/aws/variables.tf

@@ -13,12 +13,11 @@
 # limitations under the License.
 
 # ------- Global settings -------
-# NOTE: Waiting on provider fix
-# variable "tags" {
-#   type        = map(any)
-#   description = "Tags applied to provisioned resources"
+variable "tags" {
+  type        = map(any)
+  description = "Tags applied to provisioned resources"
 
-# }
+}
 
 # ------- CDP Environment Deployment -------
 variable "environment_name" {
@@ -51,13 +50,6 @@ variable "cdp_user_group_name" {
 
 }
 
-# TODO: Will be re-introducted once provider supports other regions
-# variable "cdp_control_plane_region" {
-#   type        = string
-#   description = "CDP Control Plane Region"
-
-# }
-
 variable "enable_ccm_tunnel" {
   type = bool
 
@@ -201,7 +193,7 @@ variable "datalake_recipes" {
   type = set(
     object({
       instance_group_name = string,
-      recipe_names        = set(object({}))
+      recipe_names        = set(string)
     })
   )
 

modules/terraform-cdp-deploy/modules/azure/main.tf

@@ -79,7 +79,7 @@ resource "cdp_environments_azure_environment" "cdp_env" {
     polling_timeout = var.environment_polling_timeout
   }
 
-  # tags               = var.tags # NOTE: Waiting on provider fix
+  tags = var.tags
 
   depends_on = [
     cdp_environments_azure_credential.cdp_cred
@@ -153,7 +153,7 @@ resource "cdp_datalake_azure_datalake" "cdp_datalake" {
     polling_timeout = var.datalake_polling_timeout
   }
 
-  # tags = var.tags # NOTE: Waiting on provider fix
+  tags = var.tags
 
   depends_on = [
     cdp_environments_azure_credential.cdp_cred,

modules/terraform-cdp-deploy/modules/azure/provider.tf

@@ -16,7 +16,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.4.1"
+      version = "0.4.2"
     }
   }
 

modules/terraform-cdp-deploy/modules/azure/variables.tf

@@ -13,12 +13,11 @@
 # limitations under the License.
 
 # ------- Global settings -------
-# NOTE: Waiting on provider fix
-# variable "tags" {
-#   type        = map(any)
-#   description = "Tags applied to provisioned resources"
+variable "tags" {
+  type        = map(any)
+  description = "Tags applied to provisioned resources"
 
-# }
+}
 
 # ------- CDP Environment Deployment -------
 variable "environment_name" {
@@ -51,13 +50,6 @@ variable "cdp_user_group_name" {
 
 }
 
-# TODO: Will be re-introducted once provider supports other regions
-# variable "cdp_control_plane_region" {
-#   type        = string
-#   description = "CDP Control Plane Region"
-
-# }
-
 variable "enable_ccm_tunnel" {
   type = bool
 
@@ -197,7 +189,7 @@ variable "datalake_recipes" {
   type = set(
     object({
       instance_group_name = string,
-      recipe_names        = set(object({}))
+      recipe_names        = set(string)
     })
   )
 

modules/terraform-cdp-deploy/modules/gcp/main.tf

@@ -47,8 +47,6 @@ resource "cdp_environments_gcp_environment" "cdp_env" {
 
   endpoint_access_gateway_scheme = var.endpoint_access_scheme
 
-  # TODO: Other resource variables
-  # availability_zones
   encryption_key         = var.encryption_key
   proxy_config_name      = var.proxy_config_name
   report_deployment_logs = var.report_deployment_logs
@@ -67,7 +65,7 @@ resource "cdp_environments_gcp_environment" "cdp_env" {
     polling_timeout = var.environment_polling_timeout
   }
 
-  # tags               = var.tags # NOTE: Waiting on provider fix
+  tags = var.tags
 
   depends_on = [
     cdp_environments_gcp_credential.cdp_cred
@@ -131,19 +129,17 @@ resource "cdp_datalake_gcp_datalake" "cdp_datalake" {
 
   runtime = var.datalake_version == "latest" ? null : var.datalake_version
   scale   = var.datalake_scale
-  # enable_ranger_raz = var.enable_raz # TODO: Test this
-  # multi_az          = var.multiaz # TODO: Test this
 
   custom_instance_groups = var.datalake_custom_instance_groups
   image                  = var.datalake_image
   java_version           = var.datalake_java_version
-  recipes                = var.datalake_recipes
+  recipes                = null
 
   polling_options = {
     polling_timeout = var.datalake_polling_timeout
   }
 
-  # tags = var.tags # NOTE: Waiting on provider fix
+  tags = var.tags
 
   depends_on = [
     cdp_environments_gcp_credential.cdp_cred,

modules/terraform-cdp-deploy/modules/gcp/provider.tf

@@ -16,7 +16,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.4.1"
+      version = "0.4.2"
     }
   }
 

modules/terraform-cdp-deploy/modules/gcp/variables.tf

@@ -13,12 +13,11 @@
 # limitations under the License.
 
 # ------- Global settings -------
-# NOTE: Waiting on provider fix
-# variable "tags" {
-#   type        = map(any)
-#   description = "Tags applied to provisioned resources"
+variable "tags" {
+  type        = map(any)
+  description = "Tags applied to provisioned resources"
 
-# }
+}
 
 # ------- CDP Environment Deployment -------
 variable "environment_name" {
@@ -165,7 +164,7 @@ variable "datalake_recipes" {
   type = set(
     object({
       instance_group_name = string,
-      recipe_names        = set(object({}))
+      recipe_names        = set(string)
     })
   )
 

modules/terraform-cdp-deploy/provider.tf

@@ -16,7 +16,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.4.1"
+      version = "0.4.2"
     }
   }