Write DiscoverGranulesMap output to S3

Fixes #300
NASA-IMPACT · Nov 28, 2023 · 44562a1 · 44562a1
1 parent 6af1af1
commit 44562a1
Show file tree

Hide file tree

Showing 4 changed files with 75 additions and 60 deletions.
diff --git a/app/stacks/cumulus/iam.tf b/app/stacks/cumulus/iam.tf
@@ -37,13 +37,16 @@ data "aws_iam_policy_document" "allow_sfn_distributed_maps" {
     resources = ["*"]
   }
 
-  # Allow StepFunctions to read input from S3, which is necessary when the size
-  # of the input array message might exceed the quota (256KiB).
+  # Allow StepFunctions to read input from S3, as well as write output to it,
+  # which is necessary when the size of the input array message might exceed the
+  # quota (256KiB).
   statement {
     effect = "Allow"
     actions = [
       "s3:Get*",
-      "s3:List*"
+      "s3:List*",
+      "s3:PutObject",
+      "s3:AbortMultipartUpload",
     ]
     resources = ["*"]
   }

diff --git a/app/stacks/cumulus/templates/discover-granules-workflow.asl.json b/app/stacks/cumulus/templates/discover-granules-workflow.asl.json
@@ -42,6 +42,13 @@
           "Key.$": "$.key"
         }
       },
+      "ResultWriter": {
+        "Resource": "arn:aws:states:::s3:putObject",
+        "Parameters": {
+          "Bucket.$": "$.bucket",
+          "Prefix": "mapRun"
+        }
+      },
       "ItemProcessor": {
         "ProcessorConfig": {
           "Mode": "DISTRIBUTED",
@@ -140,7 +147,18 @@
                     "MaxConcurrency": 10,
                     "ToleratedFailurePercentage": 0,
                     "ItemsPath": "$",
+                    "ResultWriter": {
+                      "Resource": "arn:aws:states:::s3:putObject",
+                      "Parameters": {
+                        "Bucket.$": "$.[0].cumulus_meta.system_bucket",
+                        "Prefix": "mapRun"
+                      }
+                    },
                     "ItemProcessor": {
+                      "ProcessorConfig": {
+                        "Mode": "DISTRIBUTED",
+                        "ExecutionType": "STANDARD"
+                      },
                       "StartAt": "UnbatchGranules",
                       "States": {
                         "UnbatchGranules": {