Update infrastructure to support package download

Updates Terraform infrastructure (and Lambda variables) to support
downloading ASUS packages. There are now two Lambda functions and SQS
queues for syncing products: one for regular use, and one with
additional memory and time for handling product downloads.

code/catalog_sync.py

@@ -109,6 +109,7 @@ def lambda_handler(event, context):
     # Environmental Variables
     S3_BUCKET = anejocommon.set_env_var('S3_BUCKET')
     PRODUCT_QUEUE_URL = anejocommon.set_env_var('PRODUCT_QUEUE_URL')
+    PRODUCT_DOWNLOAD_QUEUE_URL = anejocommon.set_env_var('PRODUCT_DOWNLOAD_QUEUE_URL')
     WRITE_CATALOG_QUEUE_URL = anejocommon.set_env_var('WRITE_CATALOG_QUEUE_URL')
     WRITE_CATALOG_DELAY = anejocommon.set_env_var('WRITE_CATALOG_DELAY', 300)
 
@@ -168,6 +169,12 @@ def lambda_handler(event, context):
             products = catalog_plist['Products']
             product_keys = list(products.keys())
 
+            # Choose correct queue for product_sync
+            if download_packages:
+                queue_url = PRODUCT_DOWNLOAD_QUEUE_URL
+            else:
+                queue_url = PRODUCT_QUEUE_URL
+
             # Send to product_sync queue
             for product_key in product_keys:
                 product_info = anejocommon.compress_dict(products[product_key], True)

terraform/iam.tf

@@ -139,8 +139,10 @@ resource "aws_iam_role_policy" "anejo_sqs_iam_policy" {
             ],
             "Resource": [
                 "${aws_sqs_queue.anejo_catalog_sync_queue.arn}",
-                "${aws_sqs_queue.anejo_write_local_catalog_queue.arn}",
-                "${aws_sqs_queue.anejo_product_sync_queue.arn}"
+                "${aws_sqs_queue.anejo_product_sync_queue.arn}",
+                "${aws_sqs_queue.anejo_product_sync_download_queue.arn}",
+                "${aws_sqs_queue.anejo_product_sync_failed_queue.arn}",
+                "${aws_sqs_queue.anejo_write_local_catalog_queue.arn}"
             ]
         },
         {

terraform/lambda.tf

@@ -1,14 +1,14 @@
 ### Anejo – Lambda Functions ###
 
-# Repo Sync
+# Repo Sync Function
 resource "aws_lambda_function" "anejo_repo_sync" {
   function_name = "anejo_repo_sync"
   description   = "Sync Anejo repo with Apple SUS"
   filename      = "${var.zip_file_path}"
   role          = "${aws_iam_role.anejo_iam_role.arn}"
   handler       = "repo_sync.lambda_handler"
   runtime       = "python3.7"
-  timeout       = 10
+  timeout       = 60
 
   environment {
     variables = {
@@ -19,7 +19,7 @@ resource "aws_lambda_function" "anejo_repo_sync" {
 }
 
 
-# Catalog Sync
+# Catalog Sync Function
 resource "aws_lambda_function" "anejo_catalog_sync" {
   function_name = "anejo_catalog_sync"
   description   = "Replicate Apple SUS catalog to Anejo repo"
@@ -31,16 +31,17 @@ resource "aws_lambda_function" "anejo_catalog_sync" {
 
   environment {
     variables = {
-      S3_BUCKET               = "${var.anejo_repo_bucket}",
-      PRODUCT_QUEUE_URL       = "${aws_sqs_queue.anejo_product_sync_queue.id}",
-      WRITE_CATALOG_QUEUE_URL = "${aws_sqs_queue.anejo_write_local_catalog_queue.id}",
-      WRITE_CATALOG_DELAY     = "${var.anejo_write_catalog_delay}"
+      S3_BUCKET                  = "${var.anejo_repo_bucket}",
+      PRODUCT_QUEUE_URL          = "${aws_sqs_queue.anejo_product_sync_queue.id}",
+      PRODUCT_DOWNLOAD_QUEUE_URL = "${aws_sqs_queue.anejo_product_sync_download_queue.id}",
+      WRITE_CATALOG_QUEUE_URL    = "${aws_sqs_queue.anejo_write_local_catalog_queue.id}",
+      WRITE_CATALOG_DELAY        = "${var.anejo_write_catalog_delay}"
     }
   }
 }
 
 
-# Product Sync
+# Product Sync Function
 resource "aws_lambda_function" "anejo_product_sync" {
   function_name = "anejo_product_sync"
   description   = "Replicate Apple SUS product to Anejo repo"
@@ -49,6 +50,27 @@ resource "aws_lambda_function" "anejo_product_sync" {
   handler       = "product_sync.lambda_handler"
   runtime       = "python3.7"
   timeout       = 300
+  memory_size   = 128
+
+  environment {
+    variables = {
+      PRODUCT_INFO_TABLE = "${aws_dynamodb_table.anejo_product_info_metadata.id}",
+      S3_BUCKET          = "${var.anejo_repo_bucket}"
+    }
+  }
+}
+
+
+# Product Sync/Download Function
+resource "aws_lambda_function" "anejo_product_sync_download" {
+  function_name = "anejo_product_sync_download"
+  description   = "Replicate Apple SUS product and packages to Anejo repo"
+  filename      = "${var.zip_file_path}"
+  role          = "${aws_iam_role.anejo_iam_role.arn}"
+  handler       = "product_sync.lambda_handler"
+  runtime       = "python3.7"
+  timeout       = 900
+  memory_size   = 512
 
   environment {
     variables = {
@@ -86,17 +108,25 @@ resource "aws_lambda_event_source_mapping" "anejo_catalog_sync_trigger" {
 }
 
 
-# Catalog Sync Trigger
-resource "aws_lambda_event_source_mapping" "anejo_write_local_catalog_trigger" {
-  event_source_arn = "${aws_sqs_queue.anejo_write_local_catalog_queue.arn}"
-  function_name    = "${aws_lambda_function.anejo_write_local_catalog.arn}"
-  batch_size       = 1
-}
-
-
 # Products Sync Trigger
 resource "aws_lambda_event_source_mapping" "anejo_product_sync_trigger" {
   event_source_arn = "${aws_sqs_queue.anejo_product_sync_queue.arn}"
   function_name    = "${aws_lambda_function.anejo_product_sync.arn}"
   batch_size       = 10
 }
+
+
+# Products Sync Download Trigger
+resource "aws_lambda_event_source_mapping" "anejo_product_sync_download_trigger" {
+  event_source_arn = "${aws_sqs_queue.anejo_product_sync_download_queue.arn}"
+  function_name    = "${aws_lambda_function.anejo_product_sync_download.arn}"
+  batch_size       = 1
+}
+
+
+# Write Local Catalog Trigger
+resource "aws_lambda_event_source_mapping" "anejo_write_local_catalog_trigger" {
+  event_source_arn = "${aws_sqs_queue.anejo_write_local_catalog_queue.arn}"
+  function_name    = "${aws_lambda_function.anejo_write_local_catalog.arn}"
+  batch_size       = 1
+}

terraform/sqs.tf

@@ -9,19 +9,39 @@ resource "aws_sqs_queue" "anejo_catalog_sync_queue" {
 }
 
 
-# Write Local Catalog Queue
-resource "aws_sqs_queue" "anejo_write_local_catalog_queue" {
-  name                       = "AnejoWriteLocalCatalogQueue"
+# Product Sync Queue
+resource "aws_sqs_queue" "anejo_product_sync_queue" {
+  name                       = "AnejoProductSyncQueue"
   visibility_timeout_seconds = 300
+  message_retention_seconds  = 600
+  receive_wait_time_seconds  = 0
+  redrive_policy             = "{\"deadLetterTargetArn\":\"${aws_sqs_queue.anejo_product_sync_failed_queue.arn}\",\"maxReceiveCount\":3}"
+}
+
+
+# Product Sync/Download Queue
+resource "aws_sqs_queue" "anejo_product_sync_download_queue" {
+  name                       = "AnejoProductSyncDownloadQueue"
+  visibility_timeout_seconds = 900
   message_retention_seconds  = 900
   receive_wait_time_seconds  = 0
+  redrive_policy             = "{\"deadLetterTargetArn\":\"${aws_sqs_queue.anejo_product_sync_failed_queue.arn}\",\"maxReceiveCount\":3}"
 }
 
 
-# Product Sync Queue
-resource "aws_sqs_queue" "anejo_product_sync_queue" {
-  name                       = "AnejoProductSyncQueue"
+# Product Sync Failed Queue (Dead Letter Queue)
+resource "aws_sqs_queue" "anejo_product_sync_failed_queue" {
+  name                       = "AnejoProductSyncFailedQueue"
+  visibility_timeout_seconds = 900
+  message_retention_seconds  = 604800
+  receive_wait_time_seconds  = 0
+}
+
+
+# Write Local Catalog Queue
+resource "aws_sqs_queue" "anejo_write_local_catalog_queue" {
+  name                       = "AnejoWriteLocalCatalogQueue"
   visibility_timeout_seconds = 300
-  message_retention_seconds  = 600
+  message_retention_seconds  = 900
   receive_wait_time_seconds  = 0
 }