Upgrade to Cumulus 18.1.0, ORCA 8.1.0 (#283)

.terraform-version

@@ -1 +1 @@
-0.13.6
+1.5.7

Dockerfile

@@ -75,20 +75,6 @@ RUN : \
   && npm install -g [email protected] \
   && :
 
-# Install Cumulus CLI (see https://github.com/NASA-IMPACT/cumulus-cli)
-WORKDIR /usr/src
-RUN git clone --depth 1 https://github.com/NASA-IMPACT/cumulus-cli.git
-WORKDIR /usr/src/cumulus-cli
-# hadolint ignore=SC1091
-RUN : \
-  && source "${NVM_DIR}/nvm.sh" --install \
-  && nvm install \
-  && npm install \
-  && npm run build \
-  && npm install -g \
-  && ln -s "$(which cumulus)" /usr/local/bin/cumulus \
-  && :
-
 WORKDIR /work
 
 # Install node dependencies
@@ -107,6 +93,10 @@ RUN : \
 # changes in the wrong environment.
 # hadolint ignore=SC2016
 RUN : \
+  # Add yarn bin to path so that the `cumulus` command can be found.  It was
+  # installed by `yarn install` because it is in the `devDependencies` list in
+  # the package.json file.
+  && echo 'export PATH="$(yarn bin):${PATH}"' >> ~/.bashrc \
   && echo 'export PS1="(${AWS_PROFILE:-[ERROR: AWS_PROFILE is not defined]}:${TS_ENV:-[ERROR: TS_ENV is undefined]}):\w \$ "' >> ~/.bashrc \
   #-----------------------------------------------------------------------------
   # IMPORTANT

app/modules/cma/main.tf

@@ -16,7 +16,7 @@ resource "null_resource" "download_cma_zip_file" {
   }
 }
 
-resource "aws_s3_bucket_object" "cma_zip_file" {
+resource "aws_s3_object" "cma_zip_file" {
   depends_on = [null_resource.download_cma_zip_file]
   bucket     = var.bucket
   key        = local.cma_zip_name
@@ -30,7 +30,7 @@ resource "aws_s3_bucket_object" "cma_zip_file" {
 
 resource "aws_lambda_layer_version" "cma" {
   s3_bucket   = var.bucket
-  s3_key      = aws_s3_bucket_object.cma_zip_file.key
+  s3_key      = aws_s3_object.cma_zip_file.key
   layer_name  = "${var.prefix}-cumulus-message-adapter"
   description = "Lambda layer for Cumulus Message Adapter ${var.cma_version}"
 }

app/modules/cma/outputs.tf

@@ -1,5 +1,5 @@
 output "cma_zip_file_id" {
-  value = aws_s3_bucket_object.cma_zip_file.id
+  value = aws_s3_object.cma_zip_file.id
 }
 
 output "lambda_layer_version_arn" {

app/stacks/cumulus/config/hooks/terraform.rb

@@ -0,0 +1,40 @@
+class PatchCumulus18_1_0
+  def call(runner)
+    mod = runner.mod
+
+    # This is a hack to work around the fact that the Cumulus 18.1.0 S3
+    # Replicator module references an undeclared Terraform variable, due to a
+    # typo.  This simply patches the module to fix the typo before running
+    # `terraform apply`.
+
+    filepath = "#{mod.cache_dir}/.terraform/modules/s3-replicator/main.tf"
+
+    File.write(filepath, File.open(filepath) do |f|
+      f.read.gsub(/lambda_memory_size\b/, "lambda_memory_sizes")
+    end)
+
+    # Replace all references to deprecated `aws_s3_bucket_object` with `aws_s3_object`
+    # This isn't strictly necessary, but it eliminates noisy warnings from Terraform.
+    # This can be removed once Cumulus directly addresses the deprecations.
+
+    filepaths = [
+      "cumulus/tf-modules/archive/api.tf",
+      "cumulus/tf-modules/cumulus/ecs_cluster.tf",
+      "cumulus/tf-modules/cumulus/main.tf",
+      "cumulus/tf-modules/ingest/message_template.tf",
+      "discover_granules_workflow/tf-modules/workflow/main.tf",
+      "ingest_and_publish_granule_workflow/tf-modules/workflow/main.tf",
+    ]
+
+    filepaths.each do |filepath|
+      full_filepath = "#{mod.cache_dir}/.terraform/modules/#{filepath}"
+      File.write(full_filepath, File.open(full_filepath) do |f|
+        f.read.gsub(/aws_s3_bucket_object/, "aws_s3_object")
+      end)
+    end
+  end
+end
+
+before("plan", "apply",
+  execute: PatchCumulus18_1_0,
+)

app/stacks/cumulus/main.tf

@@ -163,7 +163,7 @@ resource "random_string" "token_secret" {
   special = true
 }
 
-resource "aws_s3_bucket_object" "bucket_map_yaml_distribution" {
+resource "aws_s3_object" "bucket_map_yaml_distribution" {
   bucket  = var.system_bucket
   key     = "${var.prefix}/cumulus_distribution/bucket_map.yaml"
   content = local.bucket_map_yaml
@@ -402,7 +402,7 @@ module "cma" {
 
   prefix      = var.prefix
   bucket      = var.system_bucket
-  cma_version = "2.0.2"
+  cma_version = "2.0.3"
 }
 
 module "s3-replicator" {
@@ -423,7 +423,7 @@ module "cumulus_distribution" {
 
   api_gateway_stage         = "dev"
   api_url                   = var.cumulus_distribution_url
-  bucket_map_file           = aws_s3_bucket_object.bucket_map_yaml_distribution.id
+  bucket_map_file           = aws_s3_object.bucket_map_yaml_distribution.id
   bucketname_prefix         = ""
   buckets                   = var.buckets
   cmr_acl_based_credentials = true

app/stacks/cumulus/orca.tf

@@ -15,18 +15,18 @@ data "aws_secretsmanager_secret_version" "rds_cluster_user_credentials_secret_ve
 }
 
 module "orca" {
-  source = "https://github.com/nasa/cumulus-orca/releases/download/v8.0.1/cumulus-orca-terraform.zip"
+  source = "https://github.com/nasa/cumulus-orca/releases/download/v8.1.0/cumulus-orca-terraform.zip"
   #--------------------------
   # Cumulus variables
   #--------------------------
   # REQUIRED
+  aws_region               = data.aws_region.current.name
   buckets                  = var.buckets
   lambda_subnet_ids        = module.vpc.subnets.ids
   permissions_boundary_arn = local.permissions_boundary_arn
   prefix                   = var.prefix
   system_bucket            = var.system_bucket
   vpc_id                   = module.vpc.vpc_id
-  workflow_config          = module.cumulus.workflow_config
 
   # OPTIONAL
   tags = var.tags

app/stacks/data-persistence/config/hooks/terraform.rb

@@ -13,7 +13,7 @@
 #
 # See https://nasa.github.io/cumulus/docs/v9.6.0/deployment/deployment-readme#vpc-subnets-and-security-group
 #
-# This hook simply check to see if this service linked role exists, and if not,
+# This hook simply checks to see if this service linked role exists, and if not,
 # it creates it so that we don't have to remember to do this manually before
 # deploying the first Cumulus deployment to a given AWS account.
 #

config/args/terraform.rb

@@ -1,5 +1,11 @@
 # See https://terraspace.cloud/docs/config/args/terraform/
 
 command("init",
-  args: ["-reconfigure"],
+  args: ["-no-color", "-reconfigure", "-upgrade"],
+)
+command("plan",
+  args: ["-no-color", "-refresh=false"],
+)
+command("apply",
+  args: ["-no-color"],
 )

config/helpers/cumulus_version_helper.rb

@@ -1,5 +1,5 @@
 module Terraspace::Project::CumulusVersionHelper
   def cumulus_version
-    "v16.1.1"
+    "v18.1.0"
   end
 end

config/terraform/terraform.tf

@@ -1,13 +1,13 @@
 terraform {
-  required_version = "0.13.6"
+  required_version = "1.5.7"
   required_providers {
     archive = {
       source  = "hashicorp/archive",
       version = "~> 2.2.0"
     }
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 3.14.1"
+      version = "~> 5.0"
     }
     external = {
       source  = "hashicorp/external"
@@ -19,7 +19,7 @@ terraform {
     }
     random = {
       source  = "hashicorp/random"
-      version = "~> 2.3"
+      version = "~> 3.1.0"
     }
   }
 }

package.json

@@ -60,9 +60,9 @@
     "@aws-sdk/lib-dynamodb": "^3.370.0",
     "@aws-sdk/lib-storage": "^3.370.0",
     "@aws-sdk/types": "^3.370.0",
-    "@cumulus/aws-client": "16.1.1",
-    "@cumulus/cmrjs": "16.1.1",
-    "@cumulus/common": "16.1.1",
+    "@cumulus/aws-client": "18.1.0",
+    "@cumulus/cmrjs": "18.1.0",
+    "@cumulus/common": "18.1.0",
     "@cumulus/cumulus-message-adapter-js": "2.0.4",
     "@smithy/util-stream": "^2.0.17",
     "date-fns": "^2.29.3",
@@ -78,7 +78,8 @@
   "devDependencies": {
     "@ava/typescript": "^3.0.1",
     "@aws-sdk/client-dynamodb": "^3.370.0",
-    "@cumulus/types": "16.1.1",
+    "@cumulus/api-client": "18.1.0",
+    "@cumulus/types": "18.1.0",
     "@istanbuljs/nyc-config-typescript": "^1.0.1",
     "@tsconfig/node16": "^16.1.1",
     "@types/aws-lambda": "^8.10.85",
@@ -88,6 +89,7 @@
     "@typescript-eslint/parser": "^5.1.0",
     "ava": "^4.3.3",
     "codecov": "^3.5.0",
+    "cumulus-cli": "github:NASA-IMPACT/cumulus-cli#2eb0c6b",
     "cz-conventional-changelog": "^3.3.0",
     "eslint": "^8.0.1",
     "eslint-config-prettier": "^8.3.0",