NASA-IMPACT · jsrikish · Nov 1, 2023 · Oct 17, 2023
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -25,7 +25,7 @@
         "marvhen.reflow-markdown",
         "ms-azuretools.vscode-docker",
         "rvest.vs-code-prettier-eslint",
-        "tabnine.tabnine-vscode",
+        "shopify.ruby-lsp",
         "timonwong.shellcheck",
         "wholroyd.hcl",
         "yzhang.markdown-all-in-one"

diff --git a/app/stacks/cumulus/iam.tf b/app/stacks/cumulus/iam.tf
@@ -1,3 +1,55 @@
+# <% if !in_sandbox? then %>
+data "aws_iam_policy_document" "allow_s3_access_logging" {
+  statement {
+    sid    = "AllowS3AccessLogging"
+    effect = "Allow"
+    principals {
+      type        = "Service"
+      identifiers = ["logging.s3.amazonaws.com"]
+    }
+    actions = [
+      "s3:PutObject",
+      "s3:PutObjectAcl"
+    ]
+    resources = ["arn:aws:s3:::${var.system_bucket}/*"]
+  }
+}
+# <% end %>
+
+# Additional permissions required in order to allow Step Functions to include
+# Distributed Map states.
+# See https://docs.aws.amazon.com/step-functions/latest/dg/use-dist-map-orchestrate-large-scale-parallel-workloads.html#dist-map-permissions
+data "aws_iam_policy_document" "allow_sfn_distributed_maps" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "states:DescribeExecution",
+      "states:StartExecution",
+      "states:StopExecution",
+    ]
+    resources = ["*"]
+  }
+}
+
+# Associate permissions above with a policy
+resource "aws_iam_policy" "allow_sfn_distributed_maps" {
+  name        = "${var.prefix}-additional-step-policy"
+  description = "Allows Step Functions to include Distributed Map states"
+  policy      = data.aws_iam_policy_document.allow_sfn_distributed_maps.json
+}
+
+# Attach policy above to the role that Cumulus assigns to Step Functions, so we can
+# add Distributed Map states to Step Functions.
+resource "aws_iam_role_policy_attachment" "allow_sfn_distributed_maps" {
+  # Ideally, the role would be referenced via the Terraform resource address, but I
+  # cannot tell if it is not accessible to us here, or if I simply cannot determine
+  # the correct way to address it.  The address module.cumulus.step.id doesn't work,
+  # so I simply grabbed the value "${var.prefix}-steprole" from the Cumulus source file.
+  # See https://github.com/nasa/cumulus/blob/v13.4.0/tf-modules/ingest/iam.tf#L56
+  role       = "${var.prefix}-steprole"
+  policy_arn = aws_iam_policy.allow_sfn_distributed_maps.arn
+}
+
 # temporary workaround for dashboard permissions issue
 data "aws_iam_role" "api_gateway_role" {
   depends_on = [module.cumulus]

diff --git a/app/stacks/cumulus/main.tf b/app/stacks/cumulus/main.tf
@@ -22,6 +22,8 @@ locals {
   elasticsearch_hostname          = jsondecode("<%= json_output('data-persistence.elasticsearch_hostname') %>")
   elasticsearch_security_group_id = jsondecode("<%= json_output('data-persistence.elasticsearch_security_group_id') %>")
 
+  lambda_runtime = "nodejs16.x"
+
   lambda_timeouts = {
     add_missing_file_checksums_task_timeout              = 900
     discover_granules_task_timeout                       = 900
@@ -82,24 +84,6 @@ data "archive_file" "lambda" {
   output_path = "${data.external.lambda_archive_exploded.result.dir}/../lambda.zip"
 }
 
-# <% if !in_sandbox? then %>
-data "aws_iam_policy_document" "allow_s3_access_logging" {
-  statement {
-    sid    = "AllowS3AccessLogging"
-    effect = "Allow"
-    principals {
-      type        = "Service"
-      identifiers = ["logging.s3.amazonaws.com"]
-    }
-    actions = [
-      "s3:PutObject",
-      "s3:PutObjectAcl"
-    ]
-    resources = ["arn:aws:s3:::${var.system_bucket}/*"]
-  }
-}
-# <% end %>
-
 #-------------------------------------------------------------------------------
 # RESOURCES
 #-------------------------------------------------------------------------------
@@ -228,13 +212,42 @@ resource "aws_lambda_permission" "background_job_queue_watcher" {
   source_arn    = aws_cloudwatch_event_rule.background_job_queue_watcher.arn
 }
 
-resource "aws_lambda_function" "prefix_granule_ids" {
-  function_name = "${var.prefix}-PrefixGranuleIds"
+resource "aws_lambda_function" "format_provider_paths" {
+  function_name = "${var.prefix}-FormatProviderPaths"
   filename      = data.archive_file.lambda.output_path
   role          = module.cumulus.lambda_processing_role_arn
-  handler       = "index.prefixGranuleIdsCMAHandler"
-  runtime       = "nodejs14.x"
-  timeout       = lookup(local.lambda_timeouts, "discover_granules_task_timeout", 300)
+  handler       = "index.formatProviderPathsHandler"
+  runtime       = local.lambda_runtime
+  timeout       = 60
+  memory_size   = 256
+
+  source_code_hash = data.archive_file.lambda.output_base64sha256
+  layers           = [module.cma.lambda_layer_version_arn]
+
+  tags = local.tags
+
+  dynamic "vpc_config" {
+    for_each = length(module.vpc.subnets.ids) == 0 ? [] : [1]
+    content {
+      subnet_ids         = module.vpc.subnets.ids
+      security_group_ids = [aws_security_group.egress_only.id]
+    }
+  }
+
+  environment {
+    variables = {
+      CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/"
+    }
+  }
+}
+
+resource "aws_lambda_function" "batch_granules" {
+  function_name = "${var.prefix}-BatchGranules"
+  filename      = data.archive_file.lambda.output_path
+  role          = module.cumulus.lambda_processing_role_arn
+  handler       = "index.batchGranulesCMAHandler"
+  runtime       = local.lambda_runtime
+  timeout       = 900
   memory_size   = 3008
 
   source_code_hash = data.archive_file.lambda.output_base64sha256
@@ -253,25 +266,24 @@ resource "aws_lambda_function" "prefix_granule_ids" {
   environment {
     variables = {
       stackName                   = var.prefix
-      GranulesTable               = local.dynamo_tables.granules.name
       CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/"
     }
   }
 }
 
-resource "aws_lambda_function" "format_provider_path" {
-  function_name = "${var.prefix}-FormatProviderPath"
+resource "aws_lambda_function" "unbatch_granules" {
+  function_name = "${var.prefix}-UnbatchGranules"
   filename      = data.archive_file.lambda.output_path
   role          = module.cumulus.lambda_processing_role_arn
-  handler       = "index.formatProviderPathCMAHandler"
-  runtime       = "nodejs14.x"
-  timeout       = 300
+  handler       = "index.unbatchGranulesCMAHandler"
+  runtime       = local.lambda_runtime
+  timeout       = 900
   memory_size   = 3008
 
   source_code_hash = data.archive_file.lambda.output_base64sha256
   layers           = [module.cma.lambda_layer_version_arn]
 
-  tags = local.tags
+  tags = var.tags
 
   dynamic "vpc_config" {
     for_each = length(module.vpc.subnets.ids) == 0 ? [] : [1]
@@ -283,24 +295,25 @@ resource "aws_lambda_function" "format_provider_path" {
 
   environment {
     variables = {
+      stackName                   = var.prefix
       CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/"
     }
   }
 }
 
-resource "aws_lambda_function" "advance_start_date" {
-  function_name = "${var.prefix}-AdvanceStartDate"
+resource "aws_lambda_function" "prefix_granule_ids" {
+  function_name = "${var.prefix}-PrefixGranuleIds"
   filename      = data.archive_file.lambda.output_path
   role          = module.cumulus.lambda_processing_role_arn
-  handler       = "index.advanceStartDateCMAHandler"
-  runtime       = "nodejs14.x"
-  timeout       = 300
+  handler       = "index.prefixGranuleIdsCMAHandler"
+  runtime       = local.lambda_runtime
+  timeout       = 900
   memory_size   = 3008
 
   source_code_hash = data.archive_file.lambda.output_base64sha256
   layers           = [module.cma.lambda_layer_version_arn]
 
-  tags = local.tags
+  tags = var.tags
 
   dynamic "vpc_config" {
     for_each = length(module.vpc.subnets.ids) == 0 ? [] : [1]
@@ -312,6 +325,7 @@ resource "aws_lambda_function" "advance_start_date" {
 
   environment {
     variables = {
+      stackName                   = var.prefix
       CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/"
     }
   }
@@ -322,7 +336,7 @@ resource "aws_lambda_function" "require_cmr_files" {
   filename      = data.archive_file.lambda.output_path
   role          = module.cumulus.lambda_processing_role_arn
   handler       = "index.requireCmrFilesCMAHandler"
-  runtime       = "nodejs14.x"
+  runtime       = local.lambda_runtime
   timeout       = 300
   memory_size   = 3008
 
@@ -351,7 +365,7 @@ resource "aws_lambda_function" "add_ummg_checksums" {
   filename      = data.archive_file.lambda.output_path
   role          = module.cumulus.lambda_processing_role_arn
   handler       = "index.addUmmgChecksumsCMAHandler"
-  runtime       = "nodejs14.x"
+  runtime       = local.lambda_runtime
   timeout       = 300
   memory_size   = 3008
 
@@ -375,85 +389,6 @@ resource "aws_lambda_function" "add_ummg_checksums" {
   }
 }
 
-
-# aws_sfn_activity means AWS Step Functions Activity
-resource "aws_sfn_activity" "prefix_granule_ids" {
-  name = "${var.prefix}-PrefixGranuleIds"
-}
-
-module "prefix_granule_ids_service" {
-  source = "https://github.com/nasa/cumulus/releases/download/<%= cumulus_version %>/terraform-aws-cumulus-ecs-service.zip"
-
-  prefix = var.prefix
-  name   = "PrefixGranuleIds"
-
-  cluster_arn   = module.cumulus.ecs_cluster_arn
-  desired_count = 1
-  image         = local.ecs_task_image
-
-  cpu                = local.ecs_task_cpu
-  memory_reservation = local.ecs_task_memory_reservation
-
-  environment = {
-    AWS_DEFAULT_REGION = data.aws_region.current.name
-  }
-  command = [
-    "cumulus-ecs-task",
-    "--activityArn",
-    aws_sfn_activity.prefix_granule_ids.id,
-    "--lambdaArn",
-    aws_lambda_function.prefix_granule_ids.arn
-  ]
-  alarms = {
-    MemoryUtilizationHigh = {
-      comparison_operator = "GreaterThanThreshold"
-      evaluation_periods  = 1
-      metric_name         = "MemoryUtilization"
-      statistic           = "SampleCount"
-      threshold           = 75
-    }
-  }
-}
-
-
-resource "aws_sfn_activity" "queue_granules" {
-  name = "${var.prefix}-QueueGranules"
-}
-
-module "queue_granules_service" {
-  source = "https://github.com/nasa/cumulus/releases/download/<%= cumulus_version %>/terraform-aws-cumulus-ecs-service.zip"
-
-  prefix = var.prefix
-  name   = "QueueGranules"
-
-  cluster_arn   = module.cumulus.ecs_cluster_arn
-  desired_count = 1
-  image         = local.ecs_task_image
-
-  cpu                = local.ecs_task_cpu
-  memory_reservation = local.ecs_task_memory_reservation
-
-  environment = {
-    AWS_DEFAULT_REGION = data.aws_region.current.name
-  }
-  command = [
-    "cumulus-ecs-task",
-    "--activityArn",
-    aws_sfn_activity.queue_granules.id,
-    "--lambdaArn",
-    module.cumulus.queue_granules_task.task_arn
-  ]
-  alarms = {
-    MemoryUtilizationHigh = {
-      comparison_operator = "GreaterThanThreshold"
-      evaluation_periods  = 1
-      metric_name         = "MemoryUtilization"
-      statistic           = "SampleCount"
-      threshold           = 75
-    }
-  }
-}
-
 #-------------------------------------------------------------------------------
 # MODULES
 #-------------------------------------------------------------------------------
@@ -520,11 +455,12 @@ module "discover_granules_workflow" {
 
   state_machine_definition = templatefile("${path.module}/templates/discover-granules-workflow.asl.json", {
     ingest_granule_workflow_name : module.ingest_and_publish_granule_workflow.name,
-    format_provider_path_task_arn : aws_lambda_function.format_provider_path.arn,
+    format_provider_paths_task_arn : aws_lambda_function.format_provider_paths.arn,
     discover_granules_task_arn : module.cumulus.discover_granules_task.task_arn,
-    prefix_granule_ids_task_arn : aws_sfn_activity.prefix_granule_ids.id,
-    queue_granules_task_arn : aws_sfn_activity.queue_granules.id,
-    advance_start_date_task_arn : aws_lambda_function.advance_start_date.arn,
+    batch_granules_task_arn : aws_lambda_function.batch_granules.arn,
+    unbatch_granules_task_arn : aws_lambda_function.unbatch_granules.arn,
+    prefix_granule_ids_task_arn : aws_lambda_function.prefix_granule_ids.arn,
+    queue_granules_task_arn : module.cumulus.queue_granules_task.task_arn,
     background_job_queue_url : aws_sqs_queue.background_job_queue.id
   })
 }

diff --git a/app/stacks/cumulus/resources/collections/PSScene3Band___1.json b/app/stacks/cumulus/resources/collections/PSScene3Band___1.json
@@ -15,7 +15,7 @@
     }
   ],
   "meta": {
-    "preferredQueueBatchSize": 5,
+    "preferredQueueBatchSize": 1,
     "prefixGranuleIds": true,
     "granuleRecoveryWorkflow": "OrcaRecoveryWorkflow"
   }

diff --git a/app/stacks/cumulus/resources/collections/WV02_MSI_L1B___1.json b/app/stacks/cumulus/resources/collections/WV02_MSI_L1B___1.json
@@ -7,7 +7,7 @@
   "sampleFileName": "WV02_20140824082814_10400100012AD900_14AUG24082814-M1BS-504548417070_01_P001-BROWSE.jpg",
   "url_path": "{cmrMetadata.CollectionReference.ShortName}___{cmrMetadata.CollectionReference.Version}/{dateFormat(cmrMetadata.TemporalExtent.SingleDateTime, YYYY)}/{dateFormat(cmrMetadata.TemporalExtent.SingleDateTime, DDDD)}/{cmrMetadata.GranuleUR}",
   "meta": {
-    "preferredQueueBatchSize": 5
+    "preferredQueueBatchSize": 1
   },
   "ignoreFilesConfigForDiscovery": false,
   "files": [

diff --git a/app/stacks/cumulus/resources/collections/WV03_MSI_L1B___1.json b/app/stacks/cumulus/resources/collections/WV03_MSI_L1B___1.json
@@ -7,7 +7,7 @@
   "sampleFileName": "WV03_20140824082814_10400100012AD900_14AUG24082814-M1BS-504548417070_01_P001-BROWSE.jpg",
   "url_path": "{cmrMetadata.CollectionReference.ShortName}___{cmrMetadata.CollectionReference.Version}/{dateFormat(cmrMetadata.TemporalExtent.SingleDateTime, YYYY)}/{dateFormat(cmrMetadata.TemporalExtent.SingleDateTime, DDDD)}/{cmrMetadata.GranuleUR}",
   "meta": {
-    "preferredQueueBatchSize": 5
+    "preferredQueueBatchSize": 1
   },
   "ignoreFilesConfigForDiscovery": false,
   "files": [

diff --git a/app/stacks/cumulus/resources/collections/WV03_Pan_L1B___1.json b/app/stacks/cumulus/resources/collections/WV03_Pan_L1B___1.json
@@ -7,7 +7,7 @@
   "sampleFileName": "WV03_20140824052550_104001000109D200_14AUG24052550-P1BS-506481065090_01_P001-BROWSE.jpg",
   "url_path": "{cmrMetadata.CollectionReference.ShortName}___{cmrMetadata.CollectionReference.Version}/{dateFormat(cmrMetadata.TemporalExtent.SingleDateTime, YYYY)}/{dateFormat(cmrMetadata.TemporalExtent.SingleDateTime, DDDD)}/{cmrMetadata.GranuleUR}",
   "meta": {
-    "preferredQueueBatchSize": 5
+    "preferredQueueBatchSize": 1
   },
   "ignoreFilesConfigForDiscovery": false,
   "files": [

diff --git a/app/stacks/cumulus/resources/collections/WV04_MSI_L1B___1.json b/app/stacks/cumulus/resources/collections/WV04_MSI_L1B___1.json
@@ -7,7 +7,7 @@
   "sampleFileName": "WV04_20170504052256_f9ca790e-2502-46a6-9918-205faa15cd4c-inv_17MAY04052256-M1BS-059097041070_01_P001-BROWSE.jpg",
   "url_path": "{cmrMetadata.CollectionReference.ShortName}___{cmrMetadata.CollectionReference.Version}/{dateFormat(cmrMetadata.TemporalExtent.SingleDateTime, YYYY)}/{dateFormat(cmrMetadata.TemporalExtent.SingleDateTime, DDDD)}/{cmrMetadata.GranuleUR}",
   "meta": {
-    "preferredQueueBatchSize": 5
+    "preferredQueueBatchSize": 1
   },
   "ignoreFilesConfigForDiscovery": false,
   "files": [

diff --git a/app/stacks/cumulus/resources/rules/PSScene3Band/v1/PSScene3Band___1_SmokeTest.json b/app/stacks/cumulus/resources/rules/PSScene3Band/v1/PSScene3Band___1_SmokeTest.json
@@ -17,6 +17,7 @@
     "endDate": "2017-12-01T05:00:00.000Z",
     "step": "PT1H",
     "discoverOnly": false,
+    "maxBatchSize": 2,
     "rule": {
       "state": "DISABLED"
     }