From 37fc2b194fba6f791d72482f803025185a1ed538 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Mon, 18 Dec 2023 16:51:49 +0000 Subject: [PATCH] Update AWS pre-reqs IAM resources for multi-bucket support (#44) Signed-off-by: Jim Enright --- modules/terraform-cdp-aws-pre-reqs/README.md | 16 ++- .../terraform-cdp-aws-pre-reqs/defaults.tf | 20 ++- modules/terraform-cdp-aws-pre-reqs/main.tf | 120 +++++++++++++++--- .../terraform-cdp-aws-pre-reqs/variables.tf | 33 ++++- 4 files changed, 162 insertions(+), 27 deletions(-) diff --git a/modules/terraform-cdp-aws-pre-reqs/README.md b/modules/terraform-cdp-aws-pre-reqs/README.md index ba61c28..59fe3b9 100644 --- a/modules/terraform-cdp-aws-pre-reqs/README.md +++ b/modules/terraform-cdp-aws-pre-reqs/README.md @@ -48,11 +48,13 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [aws_iam_instance_profile.cdp_idbroker_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_instance_profile) | resource | | [aws_iam_instance_profile.cdp_log_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_instance_profile) | resource | | [aws_iam_instance_profile.cdp_ranger_audit_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_instance_profile) | resource | -| [aws_iam_policy.cdp_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_backup_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_data_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | | [aws_iam_policy.cdp_datalake_admin_s3_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | | [aws_iam_policy.cdp_datalake_backup_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | | [aws_iam_policy.cdp_datalake_restore_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | | [aws_iam_policy.cdp_idbroker_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_log_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | | [aws_iam_policy.cdp_log_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | | [aws_iam_policy.cdp_ranger_audit_s3_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | | [aws_iam_policy.cdp_xaccount_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | @@ -65,6 +67,8 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach4](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach5](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach6](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cdp_idbroker_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cdp_idbroker_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cdp_log_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | @@ -74,6 +78,8 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach4](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach5](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach6](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.cdp_xaccount_role_attach](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | | [aws_kms_alias.cdp_kms_alias](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/kms_alias) | resource | | [aws_kms_key.cdp_kms_key](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/kms_key) | resource | @@ -125,9 +131,9 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [xaccount\_external\_id](#input\_xaccount\_external\_id) | External ID of the cross account | `string` | n/a | yes | | [agent\_source\_tag](#input\_agent\_source\_tag) | Tag to identify deployment source | `map(any)` | 
{
  "agent_source": "tf-cdp-module"
}
| no | | [aws\_region](#input\_aws\_region) | Region which Cloud resources will be created | `string` | `null` | no | +| [backup\_bucket\_access\_policy\_doc](#input\_backup\_bucket\_access\_policy\_doc) | Backup Bucket Access Data Access Policy | `string` | `null` | no | +| [backup\_bucket\_access\_policy\_name](#input\_backup\_bucket\_access\_policy\_name) | Backup Bucket Access Data Access Policy Name | `string` | `null` | no | | [backup\_storage](#input\_backup\_storage) | Optional Backup location for CDP environment. If not provided follow the data\_storage variable | 
object({
    backup_storage_bucket = string
    backup_storage_object = string
  })
| `null` | no | -| [bucket\_access\_policy\_doc](#input\_bucket\_access\_policy\_doc) | Bucket Access Data Access Policy | `string` | `null` | no | -| [bucket\_access\_policy\_name](#input\_bucket\_access\_policy\_name) | Bucket Access Data Access Policy Name | `string` | `null` | no | | [cdp\_default\_sg\_egress\_cidrs](#input\_cdp\_default\_sg\_egress\_cidrs) | List of egress CIDR blocks for CDP Default Security Group Egress rule | `list(string)` | 
[
  "0.0.0.0/0"
]
| no | | [cdp\_endpoint\_sg\_egress\_cidrs](#input\_cdp\_endpoint\_sg\_egress\_cidrs) | List of egress CIDR blocks for VPC Endpoint Security Group Egress rule | `list(string)` | 
[
  "0.0.0.0/0"
]
| no | | [cdp\_knox\_sg\_egress\_cidrs](#input\_cdp\_knox\_sg\_egress\_cidrs) | List of egress CIDR blocks for CDP Knox Security Group Egress rule | `list(string)` | 
[
  "0.0.0.0/0"
]
| no | @@ -136,6 +142,8 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [cdp\_vpc\_id](#input\_cdp\_vpc\_id) | VPC ID for CDP environment. Required if create\_vpc is false. | `string` | `null` | no | | [create\_vpc](#input\_create\_vpc) | Flag to specify if the VPC should be created | `bool` | `true` | no | | [create\_vpc\_endpoints](#input\_create\_vpc\_endpoints) | Flag to specify if VPC Endpoints should be created | `bool` | `true` | no | +| [data\_bucket\_access\_policy\_doc](#input\_data\_bucket\_access\_policy\_doc) | Data Bucket Access Data Access Policy | `string` | `null` | no | +| [data\_bucket\_access\_policy\_name](#input\_data\_bucket\_access\_policy\_name) | Data Bucket Access Data Access Policy Name | `string` | `null` | no | | [data\_storage](#input\_data\_storage) | Data storage locations for CDP environment | 
object({
    data_storage_bucket = string
    data_storage_object = string
  })
| `null` | no | | [datalake\_admin\_role\_name](#input\_datalake\_admin\_role\_name) | Datalake Admin role Name | `string` | `null` | no | | [datalake\_admin\_s3\_policy\_doc](#input\_datalake\_admin\_s3\_policy\_doc) | Location or Contents of Datalake Admin S3 Data Access Policy | `string` | `null` | no | @@ -149,6 +157,8 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [idbroker\_policy\_name](#input\_idbroker\_policy\_name) | IDBroker Policy name | `string` | `null` | no | | [idbroker\_role\_name](#input\_idbroker\_role\_name) | IDBroker service role Name | `string` | `null` | no | | [ingress\_extra\_cidrs\_and\_ports](#input\_ingress\_extra\_cidrs\_and\_ports) | List of extra CIDR blocks and ports to include in Security Group Ingress rules | 
object({
    cidrs = list(string)
    ports = list(number)
  })
| 
{
  "cidrs": [],
  "ports": []
}
| no | +| [log\_bucket\_access\_policy\_doc](#input\_log\_bucket\_access\_policy\_doc) | Log Bucket Access Data Access Policy | `string` | `null` | no | +| [log\_bucket\_access\_policy\_name](#input\_log\_bucket\_access\_policy\_name) | Log Bucket Access Data Access Policy Name | `string` | `null` | no | | [log\_data\_access\_policy\_doc](#input\_log\_data\_access\_policy\_doc) | Location or Contents of Log Data Access Policy | `string` | `null` | no | | [log\_data\_access\_policy\_name](#input\_log\_data\_access\_policy\_name) | Log Data Access Policy Name | `string` | `null` | no | | [log\_role\_name](#input\_log\_role\_name) | Log service role Name | `string` | `null` | no | diff --git a/modules/terraform-cdp-aws-pre-reqs/defaults.tf b/modules/terraform-cdp-aws-pre-reqs/defaults.tf index 54f043c..646dca0 100644 --- a/modules/terraform-cdp-aws-pre-reqs/defaults.tf +++ b/modules/terraform-cdp-aws-pre-reqs/defaults.tf @@ -121,17 +121,29 @@ locals { datalake_admin_s3_policy_doc = coalesce(var.datalake_admin_s3_policy_doc, local.datalake_admin_s3_policy_doc_processed) # CDP Data Access Policies - bucket_access - bucket_access_policy_name = coalesce(var.bucket_access_policy_name, "${var.env_prefix}-storage-policy") + # Note - separate policies for data, log and backup buckets + data_bucket_access_policy_name = coalesce(var.data_bucket_access_policy_name, "${var.env_prefix}-data-bucket-access-policy") + log_bucket_access_policy_name = coalesce(var.log_bucket_access_policy_name, "${var.env_prefix}-log-bucket-access-policy") + backup_bucket_access_policy_name = coalesce(var.backup_bucket_access_policy_name, "${var.env_prefix}-backup-bucket-access-policy") - # bucket_access_policy_doc # ...first process placeholders in the downloaded policy doc - bucket_access_policy_doc_processed = replace( + data_bucket_access_policy_doc_processed = replace( replace( data.http.bucket_access_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), "$${DATALAKE_BUCKET}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}") + log_bucket_access_policy_doc_processed = replace( + replace( + data.http.bucket_access_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), + "$${DATALAKE_BUCKET}", "${local.log_storage.log_storage_bucket}${local.storage_suffix}") + backup_bucket_access_policy_doc_processed = replace( + replace( + data.http.bucket_access_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), + "$${DATALAKE_BUCKET}", "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}") # ...then assign either input or downloaded policy doc to var used in resource - bucket_access_policy_doc = coalesce(var.bucket_access_policy_doc, local.bucket_access_policy_doc_processed) + data_bucket_access_policy_doc = coalesce(var.data_bucket_access_policy_doc, local.data_bucket_access_policy_doc_processed) + log_bucket_access_policy_doc = coalesce(var.log_bucket_access_policy_doc, local.log_bucket_access_policy_doc_processed) + backup_bucket_access_policy_doc = coalesce(var.backup_bucket_access_policy_doc, local.backup_bucket_access_policy_doc_processed) # CDP Datalake backup Policy datalake_backup_policy_name = coalesce(var.datalake_backup_policy_name, "${var.env_prefix}-datalake-backup-policy") diff --git a/modules/terraform-cdp-aws-pre-reqs/main.tf b/modules/terraform-cdp-aws-pre-reqs/main.tf index c5803a1..cd7865c 100644 --- a/modules/terraform-cdp-aws-pre-reqs/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/main.tf @@ -390,13 +390,47 @@ resource "aws_iam_policy" "cdp_datalake_admin_s3_data_access_policy" { } # ------- CDP Data Access Policies - bucket_access ------- -resource "aws_iam_policy" "cdp_bucket_data_access_policy" { - name = local.bucket_access_policy_name - description = "CDP Bucket S3 Access policy for ${var.env_prefix}" +# Policy for Data bucket +resource "aws_iam_policy" "cdp_data_bucket_data_access_policy" { + name = local.data_bucket_access_policy_name + description = "CDP Data Bucket S3 Access policy for ${var.env_prefix}" - tags = merge(local.env_tags, { Name = local.bucket_access_policy_name }) + tags = merge(local.env_tags, { Name = local.data_bucket_access_policy_name }) - policy = local.bucket_access_policy_doc + policy = local.data_bucket_access_policy_doc +} + +# Policy for Log bucket...Only required if log bucket different from data bucket +resource "aws_iam_policy" "cdp_log_bucket_data_access_policy" { + + count = (( + local.log_storage.log_storage_bucket != local.data_storage.data_storage_bucket) ? + 1 : 0 + ) + + name = local.log_bucket_access_policy_name + description = "CDP Log Bucket S3 Access policy for ${var.env_prefix}" + + tags = merge(local.env_tags, { Name = local.log_bucket_access_policy_name }) + + policy = local.log_bucket_access_policy_doc +} +# Policy for backup bucket...requied only if different from backup and log bucket +resource "aws_iam_policy" "cdp_backup_bucket_data_access_policy" { + + count = (( + local.backup_storage.backup_storage_bucket != local.data_storage.data_storage_bucket + || + local.backup_storage.backup_storage_bucket != local.log_storage.log_storage_bucket) ? + 1 : 0 + ) + + name = local.backup_bucket_access_policy_name + description = "CDP Backup Bucket S3 Access policy for ${var.env_prefix}" + + tags = merge(local.env_tags, { Name = local.backup_bucket_access_policy_name }) + + policy = local.backup_bucket_access_policy_doc } # ------- CDP Data Access Policies - datalake_backup_policy ------- @@ -549,18 +583,18 @@ resource "aws_iam_role_policy_attachment" "cdp_log_role_attach1" { policy_arn = aws_iam_policy.cdp_log_data_access_policy.arn } -# Attach AWS Bucket Access Policy to the Role +# Attach AWS Datalake Restore Policy to the Role resource "aws_iam_role_policy_attachment" "cdp_log_role_attach2" { role = aws_iam_role.cdp_log_role.name - policy_arn = aws_iam_policy.cdp_bucket_data_access_policy.arn + policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn } -# Attach AWS Datalake Restore Policy to the Role +# Attach AWS Datalake Backup Policy to the Role resource "aws_iam_role_policy_attachment" "cdp_log_role_attach3" { role = aws_iam_role.cdp_log_role.name - policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn + policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn } # ------- AWS Data Access Roles - CDP Datalake Admin ------- @@ -603,21 +637,48 @@ resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach1" { } # Attach AWS Bucket Access Policy to the Role +# ..data bucket policy resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach2" { role = aws_iam_role.cdp_datalake_admin_role.name - policy_arn = aws_iam_policy.cdp_bucket_data_access_policy.arn + policy_arn = aws_iam_policy.cdp_data_bucket_data_access_policy.arn } -# Attach AWS Datalake Backup Policy to the Role +# ..log bucket policy, if required resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach3" { + count = (( + local.log_storage.log_storage_bucket != local.data_storage.data_storage_bucket) ? + 1 : 0 + ) + + role = aws_iam_role.cdp_datalake_admin_role.name + policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy[0].arn +} + +# ..backup bucket policy, if required +resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach4" { + + count = (( + local.backup_storage.backup_storage_bucket != local.data_storage.data_storage_bucket + || + local.backup_storage.backup_storage_bucket != local.log_storage.log_storage_bucket) ? + 1 : 0 + ) + + role = aws_iam_role.cdp_datalake_admin_role.name + policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy[0].arn +} + +# Attach AWS Datalake Backup Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach5" { + role = aws_iam_role.cdp_datalake_admin_role.name policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn } # Attach AWS Datalake Restore Policy to the Role -resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach4" { +resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach6" { role = aws_iam_role.cdp_datalake_admin_role.name policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn @@ -662,22 +723,49 @@ resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach1" { policy_arn = aws_iam_policy.cdp_ranger_audit_s3_data_access_policy.arn } -# Attach AWS Bucket Access Policy to the Role +# Attach AWS Bucket Access Policies to the Role +# ..data bucket policy resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach2" { role = aws_iam_role.cdp_ranger_audit_role.name - policy_arn = aws_iam_policy.cdp_bucket_data_access_policy.arn + policy_arn = aws_iam_policy.cdp_data_bucket_data_access_policy.arn } -# Attach AWS Datalake Backup Policy to the Role +# ..log bucket policy, if required resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach3" { + count = (( + local.log_storage.log_storage_bucket != local.data_storage.data_storage_bucket) ? + 1 : 0 + ) + + role = aws_iam_role.cdp_ranger_audit_role.name + policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy[0].arn +} + +# ..backup bucket policy, if required +resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach4" { + + count = (( + local.backup_storage.backup_storage_bucket != local.data_storage.data_storage_bucket + || + local.backup_storage.backup_storage_bucket != local.log_storage.log_storage_bucket) ? + 1 : 0 + ) + + role = aws_iam_role.cdp_ranger_audit_role.name + policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy[0].arn +} + +# Attach AWS Datalake Backup Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach5" { + role = aws_iam_role.cdp_ranger_audit_role.name policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn } # Attach AWS Datalake Restore Policy to the Role -resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach4" { +resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach6" { role = aws_iam_role.cdp_ranger_audit_role.name policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn diff --git a/modules/terraform-cdp-aws-pre-reqs/variables.tf b/modules/terraform-cdp-aws-pre-reqs/variables.tf index b431fab..8e54286 100644 --- a/modules/terraform-cdp-aws-pre-reqs/variables.tf +++ b/modules/terraform-cdp-aws-pre-reqs/variables.tf @@ -441,9 +441,22 @@ variable "datalake_restore_policy_doc" { } # CDP Data Access Policies - bucket_access -variable "bucket_access_policy_name" { +variable "data_bucket_access_policy_name" { type = string - description = "Bucket Access Data Access Policy Name" + description = "Data Bucket Access Data Access Policy Name" + + default = null +} +variable "log_bucket_access_policy_name" { + type = string + description = "Log Bucket Access Data Access Policy Name" + + default = null +} + +variable "backup_bucket_access_policy_name" { + type = string + description = "Backup Bucket Access Data Access Policy Name" default = null } @@ -464,9 +477,21 @@ variable "datalake_backup_policy_name" { default = null } -variable "bucket_access_policy_doc" { +variable "data_bucket_access_policy_doc" { + type = string + description = "Data Bucket Access Data Access Policy" + + default = null +} +variable "log_bucket_access_policy_doc" { + type = string + description = "Log Bucket Access Data Access Policy" + + default = null +} +variable "backup_bucket_access_policy_doc" { type = string - description = "Bucket Access Data Access Policy" + description = "Backup Bucket Access Data Access Policy" default = null }