Update AWS pre-reqs IAM resources for multi-bucket support (#44)

Signed-off-by: Jim Enright <[email protected]>

modules/terraform-cdp-aws-pre-reqs/README.md

@@ -48,11 +48,13 @@ In each directory an example `terraform.tfvars.sample` values file is included t
 | [aws_iam_instance_profile.cdp_idbroker_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_instance_profile) | resource |
 | [aws_iam_instance_profile.cdp_log_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_instance_profile) | resource |
 | [aws_iam_instance_profile.cdp_ranger_audit_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_instance_profile) | resource |
-| [aws_iam_policy.cdp_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource |
+| [aws_iam_policy.cdp_backup_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource |
+| [aws_iam_policy.cdp_data_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.cdp_datalake_admin_s3_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.cdp_datalake_backup_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.cdp_datalake_restore_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.cdp_idbroker_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource |
+| [aws_iam_policy.cdp_log_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.cdp_log_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.cdp_ranger_audit_s3_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.cdp_xaccount_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource |
@@ -65,6 +67,8 @@ In each directory an example `terraform.tfvars.sample` values file is included t
 | [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach4](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach5](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach6](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.cdp_idbroker_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.cdp_idbroker_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.cdp_log_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
@@ -74,6 +78,8 @@ In each directory an example `terraform.tfvars.sample` values file is included t
 | [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach4](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach5](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach6](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.cdp_xaccount_role_attach](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_kms_alias.cdp_kms_alias](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/kms_alias) | resource |
 | [aws_kms_key.cdp_kms_key](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/kms_key) | resource |
@@ -125,9 +131,9 @@ In each directory an example `terraform.tfvars.sample` values file is included t
 | <a name="input_xaccount_external_id"></a> [xaccount\_external\_id](#input\_xaccount\_external\_id) | External ID of the cross account | `string` | n/a | yes |
 | <a name="input_agent_source_tag"></a> [agent\_source\_tag](#input\_agent\_source\_tag) | Tag to identify deployment source | `map(any)` | <pre>{<br>  "agent_source": "tf-cdp-module"<br>}</pre> | no |
 | <a name="input_aws_region"></a> [aws\_region](#input\_aws\_region) | Region which Cloud resources will be created | `string` | `null` | no |
+| <a name="input_backup_bucket_access_policy_doc"></a> [backup\_bucket\_access\_policy\_doc](#input\_backup\_bucket\_access\_policy\_doc) | Backup Bucket Access Data Access Policy | `string` | `null` | no |
+| <a name="input_backup_bucket_access_policy_name"></a> [backup\_bucket\_access\_policy\_name](#input\_backup\_bucket\_access\_policy\_name) | Backup Bucket Access Data Access Policy Name | `string` | `null` | no |
 | <a name="input_backup_storage"></a> [backup\_storage](#input\_backup\_storage) | Optional Backup location for CDP environment. If not provided follow the data\_storage variable | <pre>object({<br>    backup_storage_bucket = string<br>    backup_storage_object = string<br>  })</pre> | `null` | no |
-| <a name="input_bucket_access_policy_doc"></a> [bucket\_access\_policy\_doc](#input\_bucket\_access\_policy\_doc) | Bucket Access Data Access Policy | `string` | `null` | no |
-| <a name="input_bucket_access_policy_name"></a> [bucket\_access\_policy\_name](#input\_bucket\_access\_policy\_name) | Bucket Access Data Access Policy Name | `string` | `null` | no |
 | <a name="input_cdp_default_sg_egress_cidrs"></a> [cdp\_default\_sg\_egress\_cidrs](#input\_cdp\_default\_sg\_egress\_cidrs) | List of egress CIDR blocks for CDP Default Security Group Egress rule | `list(string)` | <pre>[<br>  "0.0.0.0/0"<br>]</pre> | no |
 | <a name="input_cdp_endpoint_sg_egress_cidrs"></a> [cdp\_endpoint\_sg\_egress\_cidrs](#input\_cdp\_endpoint\_sg\_egress\_cidrs) | List of egress CIDR blocks for VPC Endpoint Security Group Egress rule | `list(string)` | <pre>[<br>  "0.0.0.0/0"<br>]</pre> | no |
 | <a name="input_cdp_knox_sg_egress_cidrs"></a> [cdp\_knox\_sg\_egress\_cidrs](#input\_cdp\_knox\_sg\_egress\_cidrs) | List of egress CIDR blocks for CDP Knox Security Group Egress rule | `list(string)` | <pre>[<br>  "0.0.0.0/0"<br>]</pre> | no |
@@ -136,6 +142,8 @@ In each directory an example `terraform.tfvars.sample` values file is included t
 | <a name="input_cdp_vpc_id"></a> [cdp\_vpc\_id](#input\_cdp\_vpc\_id) | VPC ID for CDP environment. Required if create\_vpc is false. | `string` | `null` | no |
 | <a name="input_create_vpc"></a> [create\_vpc](#input\_create\_vpc) | Flag to specify if the VPC should be created | `bool` | `true` | no |
 | <a name="input_create_vpc_endpoints"></a> [create\_vpc\_endpoints](#input\_create\_vpc\_endpoints) | Flag to specify if VPC Endpoints should be created | `bool` | `true` | no |
+| <a name="input_data_bucket_access_policy_doc"></a> [data\_bucket\_access\_policy\_doc](#input\_data\_bucket\_access\_policy\_doc) | Data Bucket Access Data Access Policy | `string` | `null` | no |
+| <a name="input_data_bucket_access_policy_name"></a> [data\_bucket\_access\_policy\_name](#input\_data\_bucket\_access\_policy\_name) | Data Bucket Access Data Access Policy Name | `string` | `null` | no |
 | <a name="input_data_storage"></a> [data\_storage](#input\_data\_storage) | Data storage locations for CDP environment | <pre>object({<br>    data_storage_bucket = string<br>    data_storage_object = string<br>  })</pre> | `null` | no |
 | <a name="input_datalake_admin_role_name"></a> [datalake\_admin\_role\_name](#input\_datalake\_admin\_role\_name) | Datalake Admin role Name | `string` | `null` | no |
 | <a name="input_datalake_admin_s3_policy_doc"></a> [datalake\_admin\_s3\_policy\_doc](#input\_datalake\_admin\_s3\_policy\_doc) | Location or Contents of Datalake Admin S3 Data Access Policy | `string` | `null` | no |
@@ -149,6 +157,8 @@ In each directory an example `terraform.tfvars.sample` values file is included t
 | <a name="input_idbroker_policy_name"></a> [idbroker\_policy\_name](#input\_idbroker\_policy\_name) | IDBroker Policy name | `string` | `null` | no |
 | <a name="input_idbroker_role_name"></a> [idbroker\_role\_name](#input\_idbroker\_role\_name) | IDBroker service role Name | `string` | `null` | no |
 | <a name="input_ingress_extra_cidrs_and_ports"></a> [ingress\_extra\_cidrs\_and\_ports](#input\_ingress\_extra\_cidrs\_and\_ports) | List of extra CIDR blocks and ports to include in Security Group Ingress rules | <pre>object({<br>    cidrs = list(string)<br>    ports = list(number)<br>  })</pre> | <pre>{<br>  "cidrs": [],<br>  "ports": []<br>}</pre> | no |
+| <a name="input_log_bucket_access_policy_doc"></a> [log\_bucket\_access\_policy\_doc](#input\_log\_bucket\_access\_policy\_doc) | Log Bucket Access Data Access Policy | `string` | `null` | no |
+| <a name="input_log_bucket_access_policy_name"></a> [log\_bucket\_access\_policy\_name](#input\_log\_bucket\_access\_policy\_name) | Log Bucket Access Data Access Policy Name | `string` | `null` | no |
 | <a name="input_log_data_access_policy_doc"></a> [log\_data\_access\_policy\_doc](#input\_log\_data\_access\_policy\_doc) | Location or Contents of Log Data Access Policy | `string` | `null` | no |
 | <a name="input_log_data_access_policy_name"></a> [log\_data\_access\_policy\_name](#input\_log\_data\_access\_policy\_name) | Log Data Access Policy Name | `string` | `null` | no |
 | <a name="input_log_role_name"></a> [log\_role\_name](#input\_log\_role\_name) | Log service role Name | `string` | `null` | no |

modules/terraform-cdp-aws-pre-reqs/defaults.tf

@@ -121,17 +121,29 @@ locals {
   datalake_admin_s3_policy_doc = coalesce(var.datalake_admin_s3_policy_doc, local.datalake_admin_s3_policy_doc_processed)
 
   # CDP Data Access Policies - bucket_access
-  bucket_access_policy_name = coalesce(var.bucket_access_policy_name, "${var.env_prefix}-storage-policy")
+  # Note - separate policies for data, log and backup buckets
+  data_bucket_access_policy_name   = coalesce(var.data_bucket_access_policy_name, "${var.env_prefix}-data-bucket-access-policy")
+  log_bucket_access_policy_name    = coalesce(var.log_bucket_access_policy_name, "${var.env_prefix}-log-bucket-access-policy")
+  backup_bucket_access_policy_name = coalesce(var.backup_bucket_access_policy_name, "${var.env_prefix}-backup-bucket-access-policy")
 
-  # bucket_access_policy_doc
   # ...first process placeholders in the downloaded policy doc
-  bucket_access_policy_doc_processed = replace(
+  data_bucket_access_policy_doc_processed = replace(
     replace(
     data.http.bucket_access_policy_doc.response_body, "$${ARN_PARTITION}", "aws"),
   "$${DATALAKE_BUCKET}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}")
+  log_bucket_access_policy_doc_processed = replace(
+    replace(
+    data.http.bucket_access_policy_doc.response_body, "$${ARN_PARTITION}", "aws"),
+  "$${DATALAKE_BUCKET}", "${local.log_storage.log_storage_bucket}${local.storage_suffix}")
+  backup_bucket_access_policy_doc_processed = replace(
+    replace(
+    data.http.bucket_access_policy_doc.response_body, "$${ARN_PARTITION}", "aws"),
+  "$${DATALAKE_BUCKET}", "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}")
 
   # ...then assign either input or downloaded policy doc to var used in resource
-  bucket_access_policy_doc = coalesce(var.bucket_access_policy_doc, local.bucket_access_policy_doc_processed)
+  data_bucket_access_policy_doc   = coalesce(var.data_bucket_access_policy_doc, local.data_bucket_access_policy_doc_processed)
+  log_bucket_access_policy_doc    = coalesce(var.log_bucket_access_policy_doc, local.log_bucket_access_policy_doc_processed)
+  backup_bucket_access_policy_doc = coalesce(var.backup_bucket_access_policy_doc, local.backup_bucket_access_policy_doc_processed)
 
   # CDP Datalake backup Policy
   datalake_backup_policy_name = coalesce(var.datalake_backup_policy_name, "${var.env_prefix}-datalake-backup-policy")

modules/terraform-cdp-aws-pre-reqs/main.tf

@@ -390,13 +390,47 @@ resource "aws_iam_policy" "cdp_datalake_admin_s3_data_access_policy" {
 }
 
 # ------- CDP Data Access Policies - bucket_access -------
-resource "aws_iam_policy" "cdp_bucket_data_access_policy" {
-  name        = local.bucket_access_policy_name
-  description = "CDP Bucket S3 Access policy for ${var.env_prefix}"
+# Policy for Data bucket
+resource "aws_iam_policy" "cdp_data_bucket_data_access_policy" {
+  name        = local.data_bucket_access_policy_name
+  description = "CDP Data Bucket S3 Access policy for ${var.env_prefix}"
 
-  tags = merge(local.env_tags, { Name = local.bucket_access_policy_name })
+  tags = merge(local.env_tags, { Name = local.data_bucket_access_policy_name })
 
-  policy = local.bucket_access_policy_doc
+  policy = local.data_bucket_access_policy_doc
+}
+
+# Policy for Log bucket...Only required if log bucket different from data bucket
+resource "aws_iam_policy" "cdp_log_bucket_data_access_policy" {
+
+  count = ((
+    local.log_storage.log_storage_bucket != local.data_storage.data_storage_bucket) ?
+    1 : 0
+  )
+
+  name        = local.log_bucket_access_policy_name
+  description = "CDP Log Bucket S3 Access policy for ${var.env_prefix}"
+
+  tags = merge(local.env_tags, { Name = local.log_bucket_access_policy_name })
+
+  policy = local.log_bucket_access_policy_doc
+}
+# Policy for backup bucket...requied only if different from backup and log bucket
+resource "aws_iam_policy" "cdp_backup_bucket_data_access_policy" {
+
+  count = ((
+    local.backup_storage.backup_storage_bucket != local.data_storage.data_storage_bucket
+    ||
+    local.backup_storage.backup_storage_bucket != local.log_storage.log_storage_bucket) ?
+    1 : 0
+  )
+
+  name        = local.backup_bucket_access_policy_name
+  description = "CDP Backup Bucket S3 Access policy for ${var.env_prefix}"
+
+  tags = merge(local.env_tags, { Name = local.backup_bucket_access_policy_name })
+
+  policy = local.backup_bucket_access_policy_doc
 }
 
 # ------- CDP Data Access Policies - datalake_backup_policy -------
@@ -549,18 +583,18 @@ resource "aws_iam_role_policy_attachment" "cdp_log_role_attach1" {
   policy_arn = aws_iam_policy.cdp_log_data_access_policy.arn
 }
 
-# Attach AWS Bucket Access Policy to the Role
+# Attach AWS Datalake Restore Policy to the Role
 resource "aws_iam_role_policy_attachment" "cdp_log_role_attach2" {
 
   role       = aws_iam_role.cdp_log_role.name
-  policy_arn = aws_iam_policy.cdp_bucket_data_access_policy.arn
+  policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn
 }
 
-# Attach AWS Datalake Restore Policy to the Role
+# Attach AWS Datalake Backup Policy to the Role
 resource "aws_iam_role_policy_attachment" "cdp_log_role_attach3" {
 
   role       = aws_iam_role.cdp_log_role.name
-  policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn
+  policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn
 }
 
 # ------- AWS Data Access Roles - CDP Datalake Admin -------
@@ -603,21 +637,48 @@ resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach1" {
 }
 
 # Attach AWS Bucket Access Policy to the Role
+# ..data bucket policy
 resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach2" {
 
   role       = aws_iam_role.cdp_datalake_admin_role.name
-  policy_arn = aws_iam_policy.cdp_bucket_data_access_policy.arn
+  policy_arn = aws_iam_policy.cdp_data_bucket_data_access_policy.arn
 }
 
-# Attach AWS Datalake Backup Policy to the Role
+# ..log bucket policy, if required
 resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach3" {
 
+  count = ((
+    local.log_storage.log_storage_bucket != local.data_storage.data_storage_bucket) ?
+    1 : 0
+  )
+
+  role       = aws_iam_role.cdp_datalake_admin_role.name
+  policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy[0].arn
+}
+
+# ..backup bucket policy, if required
+resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach4" {
+
+  count = ((
+    local.backup_storage.backup_storage_bucket != local.data_storage.data_storage_bucket
+    ||
+    local.backup_storage.backup_storage_bucket != local.log_storage.log_storage_bucket) ?
+    1 : 0
+  )
+
+  role       = aws_iam_role.cdp_datalake_admin_role.name
+  policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy[0].arn
+}
+
+# Attach AWS Datalake Backup Policy to the Role
+resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach5" {
+
   role       = aws_iam_role.cdp_datalake_admin_role.name
   policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn
 }
 
 # Attach AWS Datalake Restore Policy to the Role
-resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach4" {
+resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach6" {
 
   role       = aws_iam_role.cdp_datalake_admin_role.name
   policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn
@@ -662,22 +723,49 @@ resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach1" {
   policy_arn = aws_iam_policy.cdp_ranger_audit_s3_data_access_policy.arn
 }
 
-# Attach AWS Bucket Access Policy to the Role
+# Attach AWS Bucket Access Policies to the Role
+# ..data bucket policy
 resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach2" {
 
   role       = aws_iam_role.cdp_ranger_audit_role.name
-  policy_arn = aws_iam_policy.cdp_bucket_data_access_policy.arn
+  policy_arn = aws_iam_policy.cdp_data_bucket_data_access_policy.arn
 }
 
-# Attach AWS Datalake Backup Policy to the Role
+# ..log bucket policy, if required
 resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach3" {
 
+  count = ((
+    local.log_storage.log_storage_bucket != local.data_storage.data_storage_bucket) ?
+    1 : 0
+  )
+
+  role       = aws_iam_role.cdp_ranger_audit_role.name
+  policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy[0].arn
+}
+
+# ..backup bucket policy, if required
+resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach4" {
+
+  count = ((
+    local.backup_storage.backup_storage_bucket != local.data_storage.data_storage_bucket
+    ||
+    local.backup_storage.backup_storage_bucket != local.log_storage.log_storage_bucket) ?
+    1 : 0
+  )
+
+  role       = aws_iam_role.cdp_ranger_audit_role.name
+  policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy[0].arn
+}
+
+# Attach AWS Datalake Backup Policy to the Role
+resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach5" {
+
   role       = aws_iam_role.cdp_ranger_audit_role.name
   policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn
 }
 
 # Attach AWS Datalake Restore Policy to the Role
-resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach4" {
+resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach6" {
 
   role       = aws_iam_role.cdp_ranger_audit_role.name
   policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn