BFD-2668: BCDA Insights Dev Access (#1904)

CMSgov · Oct 5, 2023 · 20873b0 · 20873b0
1 parent d5f8b0d
commit 20873b0
Show file tree

Hide file tree

Showing 16 changed files with 565 additions and 156 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,6 +6,7 @@
 !.gitattributes
 !.github
 !.sensitive-files
+!.terraform-docs.yml
 
 # OSX system files
 .DS_Store

diff --git a/insights/terraform/modules/bucket/main.tf b/insights/terraform/modules/bucket/main.tf
@@ -1,15 +1,15 @@
 data "aws_caller_identity" "current" {}
 
 locals {
-  full_name = "bfd-insights-${var.name}-${data.aws_caller_identity.current.account_id}"
-  key_name  = "bfd-insights-${var.name}-cmk"
+  full_name  = "bfd-insights-${var.name}-${data.aws_caller_identity.current.account_id}"
+  key_name   = "bfd-insights-${var.name}-cmk"
   account_id = data.aws_caller_identity.current.account_id
 }
 
+# Main S3 bucket
 resource "aws_s3_bucket" "main" {
-  bucket    = local.full_name
-  acl       = "private"
-  tags      = merge({sensitivity = var.sensitivity}, var.tags)
+  bucket = local.full_name
+  tags   = merge({ sensitivity = var.sensitivity }, var.tags) # TODO: is sensitivity needed?
 
   server_side_encryption_configuration {
     rule {
@@ -26,32 +26,38 @@ resource "aws_s3_bucket" "main" {
   }
 }
 
+# Public access block configuration for the main S3 bucket
 resource "aws_s3_bucket_public_access_block" "main" {
   bucket = aws_s3_bucket.main.id
 
-  block_public_acls   = true
-  block_public_policy = true
-  ignore_public_acls  = true
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
   restrict_public_buckets = true
 }
 
-## Folders in bucket
-
+## Create any top level folders
 resource "aws_s3_bucket_object" "top" {
-  for_each      = toset(var.folders)
-  bucket        = aws_s3_bucket.main.id
-  content_type  = "application/x-directory"
-  key           = "${each.value}/"
+  for_each     = toset(var.folders)
+  bucket       = aws_s3_bucket.main.id
+  content_type = "application/x-directory"
+  key          = "${each.value}/"
 }
 
-# Every bucket has its own CMK which allows cross-acount 
+# KMS key configuration (CMK)
 resource "aws_kms_key" "main" {
-  description   = "CMK for the ${local.full_name} bucket"
-  tags          = var.tags
-  key_usage     = "ENCRYPT_DECRYPT"
-  is_enabled    = true
-  policy        = length(var.cross_accounts) > 0 ? data.aws_iam_policy_document.cmk_policy.json : null
+  description         = "CMK for the ${local.full_name} bucket"
+  key_usage           = "ENCRYPT_DECRYPT"
+  is_enabled          = true
+  policy              = length(var.cross_accounts) > 0 ? data.aws_iam_policy_document.cmk_policy.json : null
   enable_key_rotation = true
+  tags                = var.tags
+
+  # When/if we enable bucket keys, it will only apply to new objects, so we need to ensure we don't delete the cmk until
+  # all objects encrypted with the cmk are either deleted or re-encrypted with the bucket key.
+  lifecycle {
+    prevent_destroy = true
+  }
 }
 
 resource "aws_kms_alias" "main" {

diff --git a/insights/terraform/modules/bucket/policies.tf b/insights/terraform/modules/bucket/policies.tf
@@ -112,13 +112,13 @@ resource "aws_iam_policy" "full" {
         }
     ]
 }
-POLICY  
+POLICY
 }
 
 resource "aws_iam_group_policy_attachment" "full_attach" {
-  count       = length(var.full_groups)
-  group       = var.full_groups[count.index]
-  policy_arn  = aws_iam_policy.full.arn
+  count      = length(var.full_groups)
+  group      = var.full_groups[count.index]
+  policy_arn = aws_iam_policy.full.arn
 }
 
 # Allows writes to outputs
@@ -160,20 +160,20 @@ resource "aws_iam_policy" "athena_query" {
       ],
       "Resource": "${aws_kms_key.main.arn}"
     }]
-  } 
-  POLICY  
+  }
+  POLICY
 }
 
 resource "aws_iam_group_policy_attachment" "athena_attach" {
-  count       = length(var.athena_groups)
-  group       = var.athena_groups[count.index]
-  policy_arn  = aws_iam_policy.full.arn
+  count      = length(var.athena_groups)
+  group      = var.athena_groups[count.index]
+  policy_arn = aws_iam_policy.full.arn
 }
 
 resource "aws_s3_bucket_policy" "cross_account" {
-    count       = length(var.cross_accounts) > 0 ? 1 : 0
-    bucket      = aws_s3_bucket.main.id
-    policy      = <<-POLICY
+  count  = length(var.cross_accounts) > 0 ? 1 : 0
+  bucket = aws_s3_bucket.main.id
+  policy = <<-POLICY
     {
       "Version": "2012-10-17",
       "Id": "AccessToDB",
@@ -218,29 +218,29 @@ resource "aws_s3_bucket_policy" "cross_account" {
     POLICY
 }
 
-## KMS CMK 
+## KMS CMK
 
 data "aws_iam_policy_document" "cmk_policy" {
   statement {
     sid     = "AllowAdmin"
     effect  = "Allow"
     actions = ["kms:*"]
     principals {
-      type = "AWS"
+      type        = "AWS"
       identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
     }
     resources = ["*"]
   }
-  
+
   statement {
-    sid = "AllowUse"
+    sid    = "AllowUse"
     effect = "Allow"
     actions = [
-          "kms:Encrypt",
-          "kms:Decrypt",
-          "kms:ReEncrypt*",
-          "kms:GenerateDataKey*",
-          "kms:DescribeKey"
+      "kms:Encrypt",
+      "kms:Decrypt",
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:DescribeKey"
     ]
     resources = ["*"]
     principals {

diff --git a/insights/terraform/modules/bucket/variables.tf b/insights/terraform/modules/bucket/variables.tf
@@ -12,6 +12,7 @@ variable "sensitivity" {
 variable "tags" {
   description = "tags"
   type        = map(string)
+  default     = {}
 }
 
 variable "full_groups" {

diff --git a/insights/terraform/modules/database/variables.tf b/insights/terraform/modules/database/variables.tf
@@ -16,5 +16,6 @@ variable "bucket_cmk" {
 variable "tags" {
   description = "tags"
   type        = map(string)
+  default     = {}
 }
 
diff --git a/insights/terraform/modules/firehose/variables.tf b/insights/terraform/modules/firehose/variables.tf
@@ -21,6 +21,7 @@ variable "bucket_cmk" {
 variable "tags" {
   description = "tags"
   type        = map(string)
+  default     = {}
 }
 
 variable "buffer_size" {

diff --git a/insights/terraform/modules/jobs/variables.tf b/insights/terraform/modules/jobs/variables.tf
@@ -6,6 +6,7 @@ variable "project" {
 variable "tags" {
   description = "tags"
   type        = map(string)
+  default     = {}
 }
 
 variable "buckets" {

diff --git a/insights/terraform/modules/table/main.tf b/insights/terraform/modules/table/main.tf
@@ -64,7 +64,7 @@ resource "aws_glue_catalog_table" "aws_glue_catalog_table" {
   database_name = var.database
   description   = var.description
   table_type    = "EXTERNAL_TABLE"
-  owner         = "owner"
+  owner         = var.owner
 
   parameters = local.table_parameters[var.storage_format]
 
@@ -82,7 +82,7 @@ resource "aws_glue_catalog_table" "aws_glue_catalog_table" {
     location      = "s3://${data.aws_s3_bucket.main.id}/databases/${var.database}/${var.table}"
     input_format  = local.storage_options[var.storage_format].input_format
     output_format = local.storage_options[var.storage_format].output_format
-    compressed    = true
+    compressed    = var.storage_compressed
 
     dynamic "columns" {
       for_each = var.columns

diff --git a/insights/terraform/modules/table/variables.tf b/insights/terraform/modules/table/variables.tf
@@ -1,3 +1,8 @@
+variable "owner" {
+  description = "Owner of the table (bfd, bcda, etc.)"
+  type        = string
+}
+
 variable "table" {
   description = "Name of the table"
   type        = string
@@ -27,6 +32,7 @@ variable "bucket_cmk" {
 variable "tags" {
   description = "tags"
   type        = map(string)
+  default     = {}
 }
 
 variable "partitions" {
@@ -68,3 +74,9 @@ variable "serde_parameters" {
   type        = map
   default     = {}
 }
+
+variable "storage_compressed" {
+  description = "whether or not the data is compressed (defaults to true)"
+  type        = bool
+  default     = true
+}
diff --git a/insights/terraform/modules/workgroup/variables.tf b/insights/terraform/modules/workgroup/variables.tf
@@ -16,4 +16,5 @@ variable "bucket_cmk" {
 variable "tags" {
   description = "tags"
   type        = map(string)
+  default     = {}
 }
diff --git a/insights/terraform/projects/bcda/.terraform-docs.yml b/insights/terraform/projects/bcda/.terraform-docs.yml
@@ -0,0 +1,69 @@
+formatter: markdown table
+sections:
+  show:
+    - requirements
+    - inputs
+    - outputs
+    - resources
+    - data-sources
+
+# this `content` string is implemented with as a golang template https://pkg.go.dev/text/template
+# updates here correspond to `{{ .Content }}` in `output.template` setting below
+content: |-
+    {{ .Requirements }}
+
+    <!-- GENERATED WITH `terraform-docs .`
+         Manually updating the README.md will be overwritten.
+         For more details, see the file '.terraform-docs.yml' or
+         https://terraform-docs.io/user-guide/configuration/
+    -->
+
+    {{ .Inputs }}
+
+    <!-- GENERATED WITH `terraform-docs .`
+         Manually updating the README.md will be overwritten.
+         For more details, see the file '.terraform-docs.yml' or
+         https://terraform-docs.io/user-guide/configuration/
+    -->
+
+    {{ .Modules }}
+
+    <!-- GENERATED WITH `terraform-docs .`
+         Manually updating the README.md will be overwritten.
+         For more details, see the file '.terraform-docs.yml' or
+         https://terraform-docs.io/user-guide/configuration/
+    -->
+
+    {{ .Outputs }}
+
+    <!-- GENERATED WITH `terraform-docs .`
+         Manually updating the README.md will be overwritten.
+         For more details, see the file '.terraform-docs.yml' or
+         https://terraform-docs.io/user-guide/configuration/
+    -->
+
+    {{ .Resources }}
+
+output:
+  file: README.md
+  mode: inject
+  template: |-
+    <!-- BEGIN_TF_DOCS -->
+    <!-- GENERATED WITH `terraform-docs .`
+         Manually updating the README.md will be overwritten.
+         For more details, see the file '.terraform-docs.yml' or
+         https://terraform-docs.io/user-guide/configuration/
+    -->
+    {{ .Content }}
+    <!-- END_TF_DOCS -->
+
+sort:
+  enabled: true
+  by: required
+
+settings:
+  indent: 2
+  espace: false
+  default: true
+  required: true
+  type: true