From 2bef64e541fc45909f4a2ab7ccc8cfda3287492d Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Fri, 9 Feb 2024 17:12:51 +0000 Subject: [PATCH] Fix datalake_recipe parameter type, expose tag variable and update CDP provider (#55) Signed-off-by: Jim Enright --- .../examples/ex01-minimal_inputs/main.tf | 2 +- .../examples/ex02-existing-vpc/main.tf | 2 +- .../examples/ex03-create-keypair/main.tf | 2 +- modules/terraform-cdp-deploy/README.md | 6 ++-- modules/terraform-cdp-deploy/defaults.tf | 11 +++---- .../examples/ex01-aws-basic/main.tf | 2 +- modules/terraform-cdp-deploy/main.tf | 12 ++----- .../terraform-cdp-deploy/modules/aws/main.tf | 4 +-- .../modules/aws/provider.tf | 2 +- .../modules/aws/variables.tf | 18 +++------- .../modules/azure/main.tf | 4 +-- .../modules/azure/provider.tf | 2 +- .../modules/azure/variables.tf | 18 +++------- .../terraform-cdp-deploy/modules/gcp/main.tf | 10 ++---- .../modules/gcp/provider.tf | 2 +- .../modules/gcp/variables.tf | 11 +++---- modules/terraform-cdp-deploy/provider.tf | 2 +- modules/terraform-cdp-deploy/variables.tf | 33 +++++++------------ modules/terraform-cdp-gcp-pre-reqs/README.md | 3 -- 19 files changed, 53 insertions(+), 93 deletions(-) diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf index e051efa..c3e8c7c 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf @@ -38,7 +38,7 @@ terraform { required_providers { cdp = { source = "cloudera/cdp" - version = "0.4.1" + version = "0.4.2" } } } diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf index bf210c9..a6cc886 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf @@ -51,7 +51,7 @@ terraform { required_providers { cdp = { source = "cloudera/cdp" - version = "0.4.1" + version = "0.4.2" } } } diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf index 93496af..a198939 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf @@ -58,7 +58,7 @@ terraform { required_providers { cdp = { source = "cloudera/cdp" - version = "0.4.1" + version = "0.4.2" } } } diff --git a/modules/terraform-cdp-deploy/README.md b/modules/terraform-cdp-deploy/README.md index 99d72ab..9934b51 100644 --- a/modules/terraform-cdp-deploy/README.md +++ b/modules/terraform-cdp-deploy/README.md @@ -20,7 +20,7 @@ In each directory an example `terraform.tfvars.sample` values file is included t | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.0 | -| [cdp](#requirement\_cdp) | 0.4.1 | +| [cdp](#requirement\_cdp) | 0.4.2 | ## Providers @@ -49,6 +49,7 @@ No resources. | [infra\_type](#input\_infra\_type) | Cloud Provider to deploy CDP. | `string` | n/a | yes | | [log\_storage\_location](#input\_log\_storage\_location) | Log storage location. The location has to be in uri format for the cloud provider - i.e. s3a:// for AWS, abfs:// for Azure, gs:// | `string` | n/a | yes | | [region](#input\_region) | Region which cloud resources will be created | `string` | n/a | yes | +| [agent\_source\_tag](#input\_agent\_source\_tag) | Tag to identify deployment source | `map(any)` | 
{
  "agent_source": "tf-cdp-module"
}
| no | | [aws\_datalake\_admin\_role\_arn](#input\_aws\_datalake\_admin\_role\_arn) | Datalake Admin Role ARN. Required for CDP deployment on AWS. | `string` | `null` | no | | [aws\_idbroker\_instance\_profile\_arn](#input\_aws\_idbroker\_instance\_profile\_arn) | IDBroker Instance Profile ARN. Required for CDP deployment on AWS. | `string` | `null` | no | | [aws\_log\_instance\_profile\_arn](#input\_aws\_log\_instance\_profile\_arn) | Log Instance Profile ARN. Required for CDP deployment on AWS. | `string` | `null` | no | @@ -86,7 +87,7 @@ No resources. | [datalake\_java\_version](#input\_datalake\_java\_version) | The Java major version to use on the datalake cluster. | `number` | `null` | no | | [datalake\_name](#input\_datalake\_name) | Name of the CDP datalake. Defaults to '--dl' if not specified. | `string` | `null` | no | | [datalake\_polling\_timeout](#input\_datalake\_polling\_timeout) | Timeout value in minutes for how long to poll for CDP datalake resource creation/deletion | `number` | `90` | no | -| [datalake\_recipes](#input\_datalake\_recipes) | Additional recipes that will be attached on the datalake instances | 
set(
    object({
      instance_group_name = string,
      recipe_names        = set(object({}))
    })
  )
| `null` | no | +| [datalake\_recipes](#input\_datalake\_recipes) | Additional recipes that will be attached on the datalake instances | 
set(
    object({
      instance_group_name = string,
      recipe_names        = set(string)
    })
  )
| `null` | no | | [datalake\_scale](#input\_datalake\_scale) | The scale of the datalake. Valid values are LIGHT\_DUTY, ENTERPRISE. | `string` | `null` | no | | [datalake\_version](#input\_datalake\_version) | The Datalake Runtime version. Valid values are latest or a semantic version, e.g. 7.2.17 | `string` | `"latest"` | no | | [enable\_ccm\_tunnel](#input\_enable\_ccm\_tunnel) | Flag to enable Cluster Connectivity Manager tunnel. If false then access from Cloud to CDP Control Plane CIDRs is required from via SG ingress | `bool` | `true` | no | @@ -96,6 +97,7 @@ No resources. | [encryption\_key\_resource\_group\_name](#input\_encryption\_key\_resource\_group\_name) | Name of the existing Azure resource group hosting the Azure Key Vault containing customer managed key which will be used to encrypt the Azure Managed Disk. Only applicable for CDP deployment on Azure. | `string` | `null` | no | | [encryption\_key\_url](#input\_encryption\_key\_url) | URL of the key which will be used to encrypt the Azure Managed Disks. Only applicable for CDP deployment on Azure. | `string` | `null` | no | | [endpoint\_access\_scheme](#input\_endpoint\_access\_scheme) | The scheme for the workload endpoint gateway. PUBLIC creates an external endpoint that can be accessed over the Internet. PRIVATE which restricts the traffic to be internal to the VPC / Vnet. Relevant in Private Networks. | `string` | `null` | no | +| [env\_tags](#input\_env\_tags) | Tags applied to provisioned resources | `map(any)` | `null` | no | | [environment\_name](#input\_environment\_name) | Name of the CDP environment. Defaults to '-cdp-env' if not specified. | `string` | `null` | no | | [environment\_polling\_timeout](#input\_environment\_polling\_timeout) | Timeout value in minutes for how long to poll for CDP Environment resource creation/deletion | `number` | `60` | no | | [freeipa\_catalog](#input\_freeipa\_catalog) | Image catalog to use for FreeIPA image selection | `string` | `null` | no | diff --git a/modules/terraform-cdp-deploy/defaults.tf b/modules/terraform-cdp-deploy/defaults.tf index eb75204..9b4513a 100644 --- a/modules/terraform-cdp-deploy/defaults.tf +++ b/modules/terraform-cdp-deploy/defaults.tf @@ -14,12 +14,11 @@ locals { # ------- Global settings ------- - # NOTE: Waiting on provider fix - # env_tags = merge(var.agent_source_tag, ( - # coalesce(var.env_tags, - # { env_prefix = var.env_prefix } - # )) - # ) + env_tags = merge(var.agent_source_tag, ( + coalesce(var.env_tags, + { env_prefix = var.env_prefix } + )) + ) # ------- CDP Environment Deployment ------- environment_name = coalesce(var.environment_name, diff --git a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf index cfccf84..c6e2080 100644 --- a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf +++ b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf @@ -79,7 +79,7 @@ terraform { required_providers { cdp = { source = "cloudera/cdp" - version = "0.4.1" + version = "0.4.2" } } } diff --git a/modules/terraform-cdp-deploy/main.tf b/modules/terraform-cdp-deploy/main.tf index 6d91e2f..c0311e5 100644 --- a/modules/terraform-cdp-deploy/main.tf +++ b/modules/terraform-cdp-deploy/main.tf @@ -18,7 +18,7 @@ module "cdp_on_aws" { source = "./modules/aws" - # tags = local.env_tags # NOTE: Waiting on provider fix + tags = local.env_tags environment_name = local.environment_name datalake_name = local.datalake_name @@ -42,9 +42,6 @@ module "cdp_on_aws" { environment_polling_timeout = var.environment_polling_timeout datalake_polling_timeout = var.datalake_polling_timeout - # TODO: Will be re-introducted once provider supports other regions - # cdp_control_plane_region = var.cdp_control_plane_region - region = var.region vpc_id = var.aws_vpc_id public_subnet_ids = var.aws_public_subnet_ids @@ -88,7 +85,7 @@ module "cdp_on_azure" { source = "./modules/azure" - # tags = local.env_tags # NOTE: Waiting on provider fix + tags = local.env_tags environment_name = local.environment_name datalake_name = local.datalake_name @@ -110,9 +107,6 @@ module "cdp_on_azure" { environment_polling_timeout = var.environment_polling_timeout datalake_polling_timeout = var.datalake_polling_timeout - # TODO: Will be re-introducted once provider supports other regions - # cdp_control_plane_region = var.cdp_control_plane_region - use_single_resource_group = var.use_single_resource_group use_public_ips = local.use_public_ips @@ -167,7 +161,7 @@ module "cdp_on_gcp" { source = "./modules/gcp" - # tags = local.env_tags # NOTE: Waiting on provider fix + tags = local.env_tags environment_name = local.environment_name datalake_name = local.datalake_name diff --git a/modules/terraform-cdp-deploy/modules/aws/main.tf b/modules/terraform-cdp-deploy/modules/aws/main.tf index 96344e0..f61c85f 100644 --- a/modules/terraform-cdp-deploy/modules/aws/main.tf +++ b/modules/terraform-cdp-deploy/modules/aws/main.tf @@ -67,7 +67,7 @@ resource "cdp_environments_aws_environment" "cdp_env" { polling_timeout = var.environment_polling_timeout } - # tags = var.tags # NOTE: Waiting on provider fix + tags = var.tags depends_on = [ cdp_environments_aws_credential.cdp_cred @@ -142,7 +142,7 @@ resource "cdp_datalake_aws_datalake" "cdp_datalake" { polling_timeout = var.datalake_polling_timeout } - # tags = var.tags # NOTE: Waiting on provider fix + tags = var.tags depends_on = [ cdp_environments_aws_credential.cdp_cred, diff --git a/modules/terraform-cdp-deploy/modules/aws/provider.tf b/modules/terraform-cdp-deploy/modules/aws/provider.tf index 6fb6576..1f0a70b 100644 --- a/modules/terraform-cdp-deploy/modules/aws/provider.tf +++ b/modules/terraform-cdp-deploy/modules/aws/provider.tf @@ -16,7 +16,7 @@ terraform { required_providers { cdp = { source = "cloudera/cdp" - version = "0.4.1" + version = "0.4.2" } } diff --git a/modules/terraform-cdp-deploy/modules/aws/variables.tf b/modules/terraform-cdp-deploy/modules/aws/variables.tf index 9817e8b..c956875 100644 --- a/modules/terraform-cdp-deploy/modules/aws/variables.tf +++ b/modules/terraform-cdp-deploy/modules/aws/variables.tf @@ -13,12 +13,11 @@ # limitations under the License. # ------- Global settings ------- -# NOTE: Waiting on provider fix -# variable "tags" { -# type = map(any) -# description = "Tags applied to provisioned resources" +variable "tags" { + type = map(any) + description = "Tags applied to provisioned resources" -# } +} # ------- CDP Environment Deployment ------- variable "environment_name" { @@ -51,13 +50,6 @@ variable "cdp_user_group_name" { } -# TODO: Will be re-introducted once provider supports other regions -# variable "cdp_control_plane_region" { -# type = string -# description = "CDP Control Plane Region" - -# } - variable "enable_ccm_tunnel" { type = bool @@ -201,7 +193,7 @@ variable "datalake_recipes" { type = set( object({ instance_group_name = string, - recipe_names = set(object({})) + recipe_names = set(string) }) ) diff --git a/modules/terraform-cdp-deploy/modules/azure/main.tf b/modules/terraform-cdp-deploy/modules/azure/main.tf index eb7629a..bb44de7 100644 --- a/modules/terraform-cdp-deploy/modules/azure/main.tf +++ b/modules/terraform-cdp-deploy/modules/azure/main.tf @@ -79,7 +79,7 @@ resource "cdp_environments_azure_environment" "cdp_env" { polling_timeout = var.environment_polling_timeout } - # tags = var.tags # NOTE: Waiting on provider fix + tags = var.tags depends_on = [ cdp_environments_azure_credential.cdp_cred @@ -153,7 +153,7 @@ resource "cdp_datalake_azure_datalake" "cdp_datalake" { polling_timeout = var.datalake_polling_timeout } - # tags = var.tags # NOTE: Waiting on provider fix + tags = var.tags depends_on = [ cdp_environments_azure_credential.cdp_cred, diff --git a/modules/terraform-cdp-deploy/modules/azure/provider.tf b/modules/terraform-cdp-deploy/modules/azure/provider.tf index 6fb6576..1f0a70b 100644 --- a/modules/terraform-cdp-deploy/modules/azure/provider.tf +++ b/modules/terraform-cdp-deploy/modules/azure/provider.tf @@ -16,7 +16,7 @@ terraform { required_providers { cdp = { source = "cloudera/cdp" - version = "0.4.1" + version = "0.4.2" } } diff --git a/modules/terraform-cdp-deploy/modules/azure/variables.tf b/modules/terraform-cdp-deploy/modules/azure/variables.tf index ba9804d..a99f91a 100644 --- a/modules/terraform-cdp-deploy/modules/azure/variables.tf +++ b/modules/terraform-cdp-deploy/modules/azure/variables.tf @@ -13,12 +13,11 @@ # limitations under the License. # ------- Global settings ------- -# NOTE: Waiting on provider fix -# variable "tags" { -# type = map(any) -# description = "Tags applied to provisioned resources" +variable "tags" { + type = map(any) + description = "Tags applied to provisioned resources" -# } +} # ------- CDP Environment Deployment ------- variable "environment_name" { @@ -51,13 +50,6 @@ variable "cdp_user_group_name" { } -# TODO: Will be re-introducted once provider supports other regions -# variable "cdp_control_plane_region" { -# type = string -# description = "CDP Control Plane Region" - -# } - variable "enable_ccm_tunnel" { type = bool @@ -197,7 +189,7 @@ variable "datalake_recipes" { type = set( object({ instance_group_name = string, - recipe_names = set(object({})) + recipe_names = set(string) }) ) diff --git a/modules/terraform-cdp-deploy/modules/gcp/main.tf b/modules/terraform-cdp-deploy/modules/gcp/main.tf index 9a711cc..c6663fa 100644 --- a/modules/terraform-cdp-deploy/modules/gcp/main.tf +++ b/modules/terraform-cdp-deploy/modules/gcp/main.tf @@ -47,8 +47,6 @@ resource "cdp_environments_gcp_environment" "cdp_env" { endpoint_access_gateway_scheme = var.endpoint_access_scheme - # TODO: Other resource variables - # availability_zones encryption_key = var.encryption_key proxy_config_name = var.proxy_config_name report_deployment_logs = var.report_deployment_logs @@ -67,7 +65,7 @@ resource "cdp_environments_gcp_environment" "cdp_env" { polling_timeout = var.environment_polling_timeout } - # tags = var.tags # NOTE: Waiting on provider fix + tags = var.tags depends_on = [ cdp_environments_gcp_credential.cdp_cred @@ -131,19 +129,17 @@ resource "cdp_datalake_gcp_datalake" "cdp_datalake" { runtime = var.datalake_version == "latest" ? null : var.datalake_version scale = var.datalake_scale - # enable_ranger_raz = var.enable_raz # TODO: Test this - # multi_az = var.multiaz # TODO: Test this custom_instance_groups = var.datalake_custom_instance_groups image = var.datalake_image java_version = var.datalake_java_version - recipes = var.datalake_recipes + recipes = null polling_options = { polling_timeout = var.datalake_polling_timeout } - # tags = var.tags # NOTE: Waiting on provider fix + tags = var.tags depends_on = [ cdp_environments_gcp_credential.cdp_cred, diff --git a/modules/terraform-cdp-deploy/modules/gcp/provider.tf b/modules/terraform-cdp-deploy/modules/gcp/provider.tf index 6fb6576..1f0a70b 100644 --- a/modules/terraform-cdp-deploy/modules/gcp/provider.tf +++ b/modules/terraform-cdp-deploy/modules/gcp/provider.tf @@ -16,7 +16,7 @@ terraform { required_providers { cdp = { source = "cloudera/cdp" - version = "0.4.1" + version = "0.4.2" } } diff --git a/modules/terraform-cdp-deploy/modules/gcp/variables.tf b/modules/terraform-cdp-deploy/modules/gcp/variables.tf index e1651f9..b240262 100644 --- a/modules/terraform-cdp-deploy/modules/gcp/variables.tf +++ b/modules/terraform-cdp-deploy/modules/gcp/variables.tf @@ -13,12 +13,11 @@ # limitations under the License. # ------- Global settings ------- -# NOTE: Waiting on provider fix -# variable "tags" { -# type = map(any) -# description = "Tags applied to provisioned resources" +variable "tags" { + type = map(any) + description = "Tags applied to provisioned resources" -# } +} # ------- CDP Environment Deployment ------- variable "environment_name" { @@ -165,7 +164,7 @@ variable "datalake_recipes" { type = set( object({ instance_group_name = string, - recipe_names = set(object({})) + recipe_names = set(string) }) ) diff --git a/modules/terraform-cdp-deploy/provider.tf b/modules/terraform-cdp-deploy/provider.tf index 6fb6576..1f0a70b 100644 --- a/modules/terraform-cdp-deploy/provider.tf +++ b/modules/terraform-cdp-deploy/provider.tf @@ -16,7 +16,7 @@ terraform { required_providers { cdp = { source = "cloudera/cdp" - version = "0.4.1" + version = "0.4.2" } } diff --git a/modules/terraform-cdp-deploy/variables.tf b/modules/terraform-cdp-deploy/variables.tf index 4f4dc8c..a657533 100644 --- a/modules/terraform-cdp-deploy/variables.tf +++ b/modules/terraform-cdp-deploy/variables.tf @@ -23,21 +23,19 @@ variable "infra_type" { } } -# NOTE: Waiting on provider fix -# variable "env_tags" { -# type = map(any) -# description = "Tags applied to provisioned resources" +variable "env_tags" { + type = map(any) + description = "Tags applied to provisioned resources" -# default = null -# } + default = null +} -# NOTE: Waiting on provider fix -# variable "agent_source_tag" { -# type = map(any) -# description = "Tag to identify deployment source" +variable "agent_source_tag" { + type = map(any) + description = "Tag to identify deployment source" -# default = { agent_source = "tf-cdp-module" } -# } + default = { agent_source = "tf-cdp-module" } +} variable "env_prefix" { type = string @@ -81,15 +79,6 @@ variable "cdp_user_group_name" { default = null } -# TODO: Will be re-introducted once provider supports other regions -# variable "cdp_control_plane_region" { -# type = string -# description = "CDP Control Plane Region" - -# # Region is us-west-1 unless explicitly specified -# default = "us-west-1" -# } - variable "deployment_template" { type = string @@ -253,7 +242,7 @@ variable "datalake_recipes" { type = set( object({ instance_group_name = string, - recipe_names = set(object({})) + recipe_names = set(string) }) ) diff --git a/modules/terraform-cdp-gcp-pre-reqs/README.md b/modules/terraform-cdp-gcp-pre-reqs/README.md index 1adaad2..88bb4b9 100644 --- a/modules/terraform-cdp-gcp-pre-reqs/README.md +++ b/modules/terraform-cdp-gcp-pre-reqs/README.md @@ -72,7 +72,6 @@ In each directory an example `terraform.tfvars.sample` values file is included t |------|-------------|------|---------|:--------:| | [deployment\_template](#input\_deployment\_template) | Deployment Pattern to use for Cloud resources and CDP | `string` | n/a | yes | | [env\_prefix](#input\_env\_prefix) | Shorthand name for the environment. Used in resource descriptions | `string` | n/a | yes | -| [agent\_source\_tag](#input\_agent\_source\_tag) | Tag to identify deployment source | `map(any)` | 
{
  "agent_source": "tf-cdp-module"
}
| no | | [backup\_storage\_bucket](#input\_backup\_storage\_bucket) | Optional Backup location for CDP environment. | `string` | `null` | no | | [bucket\_storage\_class](#input\_bucket\_storage\_class) | The GCS storage class to use for the data, log and backup storage | `string` | `"NEARLINE"` | no | | [bucket\_storage\_region](#input\_bucket\_storage\_region) | The location of the Google Cloud Storage buckets for data, backups and logs. By default this follows the gcp\_region variable. | `string` | `null` | no | @@ -88,8 +87,6 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [datalake\_admin\_custom\_role\_name](#input\_datalake\_admin\_custom\_role\_name) | Name of Ranger Audit and Datalake Admin Custom Role | `string` | `null` | no | | [datalake\_admin\_role\_permissions](#input\_datalake\_admin\_role\_permissions) | List of Permission Assignments to the Ranger Audit and Datalake Admin Custom Role | `list(string)` | 
[
  "storage.buckets.get",
  "storage.objects.create",
  "storage.objects.delete",
  "storage.objects.get",
  "storage.objects.list"
]
| no | | [datalake\_admin\_service\_account\_name](#input\_datalake\_admin\_service\_account\_name) | Datalake Admin service account name | `string` | `null` | no | -| [enable\_raz](#input\_enable\_raz) | Flag to enable Ranger Authorization Service (RAZ) | `bool` | `true` | no | -| [env\_tags](#input\_env\_tags) | Tags applied to provisioned resources | `map(any)` | `null` | no | | [firewall\_default\_name](#input\_firewall\_default\_name) | Name of Default Firewall for CDP environment | `string` | `null` | no | | [firewall\_internal\_name](#input\_firewall\_internal\_name) | Name of Firewall for Internal Virtual Network communication | `string` | `null` | no | | [firewall\_knox\_name](#input\_firewall\_knox\_name) | Name of Knox Firewall for CDP environment | `string` | `null` | no |