Prepare for ingesting WV04_Pan_L1B into CBA Prod (#293)

Fixes #292

.github/workflows/terraspace.yml

@@ -47,10 +47,10 @@ jobs:
       TS_ENV: ${{ inputs.TS_ENV }}
     steps:
       - name: Checkout source
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME_ARN }}
           role-session-name: ${{ github.actor }}
@@ -89,7 +89,7 @@ jobs:
         run: sudo bundle install
 
       - name: Setup Node from .nvmrc
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version-file: .nvmrc
 
@@ -107,6 +107,7 @@ jobs:
         run: |
           echo "Node version: $(node --version)"
           bundle exec terraspace logs -f &
+          bundle exec terraspace all init
           bundle exec terraspace all plan
           # Kill background logs job
           kill $!
@@ -116,6 +117,7 @@ jobs:
         run: |
           echo "Node version: $(node --version)"
           bundle exec terraspace logs -f &
+          bundle exec terraspace all init
           bundle exec terraspace all up --yes
           # Kill background logs job
           kill $!

app/stacks/cumulus/resources/providers/cumulus.json

@@ -0,0 +1,5 @@
+{
+  "id": "cumulus",
+  "protocol": "s3",
+  "host": "csdap-cumulus-prod-protected"
+}

app/stacks/cumulus/resources/rules/WV04_Pan_L1B/v1/WV04_Pan_L1B___1.json

@@ -0,0 +1,20 @@
+{
+  "name": "WV04_Pan_L1B___1",
+  "state": "DISABLED",
+  "rule": {
+    "type": "onetime"
+  },
+  "provider": "cumulus",
+  "collection": {
+    "name": "WV04_Pan_L1B",
+    "version": "1"
+  },
+  "workflow": "DiscoverAndQueueGranules",
+  "meta": {
+    "discoverOnly": true,
+    "providerPathFormat": "'WV04_Pan_L1B___1/'yyyy/DDD",
+    "startDate": "2017-01-01T00:00:00Z",
+    "endDate": "2019-01-01T00:00:00Z",
+    "step": "P1D"
+  }
+}

app/stacks/cumulus/resources/rules/WV04_Pan_L1B/v1/WV04_Pan_L1B___1_SmokeTest.json

@@ -4,15 +4,15 @@
   "rule": {
     "type": "onetime"
   },
-  "provider": "maxar",
+  "provider": "cumulus",
   "collection": {
     "name": "WV04_Pan_L1B",
     "version": "1"
   },
   "workflow": "DiscoverAndQueueGranules",
   "meta": {
     "discoverOnly": false,
-    "providerPathFormat": "'css/nga/WV04/1B/'yyyy/DDD",
+    "providerPathFormat": "'WV04_Pan_L1B___1/'yyyy/DDD",
     "startDate": "2017-05-05T00:00:00Z",
     "endDate": "2017-05-06T00:00:00Z",
     "step": "P1D"

app/stacks/cumulus/templates/discover-granules-workflow.asl.json

@@ -105,6 +105,7 @@
                           "TargetPath": "$.payload"
                         },
                         "task_config": {
+                          "providerPath": "{$.meta.providerPath}",
                           "maxBatchSize": "{$.meta.maxBatchSize}"
                         }
                       }
@@ -143,6 +144,7 @@
                                 "TargetPath": "$.payload"
                               },
                               "task_config": {
+                                "providerPath": "{$.meta.providerPath}",
                                 "batchIndex": "{$.meta.batchIndex}"
                               }
                             }
@@ -180,7 +182,7 @@
                               "Next": "PrefixGranuleIds"
                             }
                           ],
-                          "Default": "QueueGranules"
+                          "Default": "QueueGranules?"
                         },
                         "PrefixGranuleIds": {
                           "Parameters": {
@@ -210,7 +212,30 @@
                               "BackoffRate": 2
                             }
                           ],
-                          "Next": "QueueGranules"
+                          "Next": "QueueGranules?"
+                        },
+                        "QueueGranules?": {
+                          "Type": "Choice",
+                          "Choices": [
+                            {
+                              "And": [
+                                {
+                                  "Variable": "$.meta.discoverOnly",
+                                  "IsPresent": true
+                                },
+                                {
+                                  "Variable": "$.meta.discoverOnly",
+                                  "BooleanEquals": true
+                                }
+                              ],
+                              "Next": "DontQueueGranules"
+                            }
+                          ],
+                          "Default": "QueueGranules"
+                        },
+                        "DontQueueGranules": {
+                          "Type": "Pass",
+                          "End": true
                         },
                         "QueueGranules": {
                           "Parameters": {

app/stacks/cumulus/tfvars/base.tfvars

@@ -14,7 +14,7 @@
 #<% depends_on("data-persistence") %>
 #<% depends_on("rds-cluster") %>
 
-cmr_environment = "UAT"
+cmr_environment             = "UAT"
 orca_dlq_subscription_email = "[email protected]"
 
 system_bucket = "<%= bucket('internal') %>"
@@ -49,7 +49,7 @@ buckets = {
     name = "<%= bucket('dashboard') %>"
     type = "dashboard"
   }
-  #----<% if in_sandbox? then %>
+  #-----<% if in_sandbox? then %>
   # Sandbox provider bucket
   provider = {
     name = "<%= bucket('provider') %>"
@@ -65,5 +65,9 @@ buckets = {
     name = "csdap-maxar-delivery"
     type = "provider"
   }
+  cumulus = {
+    name = "csdap-cumulus-prod-protected"
+    type = "provider"
+  }
   #-----<% end %>
 }

src/lib/discovery.spec.ts

@@ -565,7 +565,7 @@ test('prefixGranuleIds should prefix granule IDs with collection name', (t) => {
 
 test('batchGranules should output singleton array identical to input when there are no granules', (t) => {
   const actual = batchGranules({
-    config: { maxBatchSize: 0 },
+    config: { providerPath: 'foo', maxBatchSize: 0 },
     input: { granules: [] },
   });
   const expected = [{ granules: [] }];
@@ -575,7 +575,7 @@ test('batchGranules should output singleton array identical to input when there
 
 test('batchGranules should output a singleton array identical to input when there are no more than 1000 granules', (t) => {
   const actual = batchGranules({
-    config: { maxBatchSize: 0 },
+    config: { providerPath: 'foo', maxBatchSize: 0 },
     input: { granules: [{ granuleId: 'foo' }] },
   });
   const expected = [{ granules: [{ granuleId: 'foo' }] }];
@@ -588,7 +588,10 @@ test('batchGranules should output array with nearly equally sized batches when t
   const batch1 = granules.slice(0, 34);
   const batch2 = granules.slice(34, 67);
   const batch3 = granules.slice(67, 100);
-  const actual = batchGranules({ config: { maxBatchSize: 40 }, input: { granules } });
+  const actual = batchGranules({
+    config: { providerPath: 'foo', maxBatchSize: 40 },
+    input: { granules },
+  });
   const expected = [{ granules: batch1 }, { granules: batch2 }, { granules: batch3 }];
 
   t.deepEqual(actual, expected);

src/lib/discovery.ts

@@ -7,7 +7,7 @@ import * as fp from 'lodash/fp';
 
 import * as L from './aws/lambda';
 import * as $t from './io';
-import { batches } from './stdlib/arrays';
+import * as A from './stdlib/arrays';
 import { range as dateRange } from './stdlib/dates';
 
 const Granule = t.type({
@@ -53,6 +53,7 @@ export const FormatProviderPathsOutput = t.intersection([
 export const BatchGranulesInput = t.readonly(
   t.type({
     config: t.type({
+      providerPath: t.string,
       maxBatchSize: tt.fromNullable(t.number, 1000),
     }),
     input: DiscoverGranulesOutput,
@@ -62,6 +63,7 @@ export const BatchGranulesInput = t.readonly(
 export const UnbatchGranulesInput = t.readonly(
   t.type({
     config: t.type({
+      providerPath: t.string,
       batchIndex: t.Int,
     }),
     input: t.readonlyArray(DiscoverGranulesOutput),
@@ -187,6 +189,16 @@ export const formatProviderPaths = (args: FormatProviderPathsInput) => {
   );
   const encodedArgs = FormatProviderPathsInput.encode(args);
 
+  console.info(
+    JSON.stringify({
+      startDate,
+      endDate,
+      step: O.getOrElseW(() => null)(step),
+      numDates: providerPaths.length,
+      providerPaths: A.ellipsize(providerPaths),
+    })
+  );
+
   return providerPaths.map((providerPath) =>
     fp.set('meta.providerPath', providerPath, encodedArgs)
   );
@@ -248,9 +260,20 @@ export const batchGranules = (
 ): readonly DiscoverGranulesOutput[] => {
   const { input, config } = event;
   const { granules } = input;
-  const { maxBatchSize } = config;
+  const { providerPath, maxBatchSize } = config;
+  const batches = A.batches(granules, maxBatchSize).map((granules) => ({ granules }));
+
+  console.info(
+    JSON.stringify({
+      providerPath,
+      numGranules: granules.length,
+      maxBatchSize,
+      numBatches: batches.length,
+      batchSizes: batches.map((batch) => batch.granules.length),
+    })
+  );
 
-  return batches(granules, maxBatchSize).map((granules) => ({ granules }));
+  return batches;
 };
 
 /**
@@ -260,8 +283,21 @@ export const batchGranules = (
  * @param args.config.batchIndex - index of the batch of granules to select
  * @returns element at the 0-based batch index in the input array
  */
-export const unbatchGranules = (args: UnbatchGranulesInput): DiscoverGranulesOutput =>
-  args.input[args.config.batchIndex];
+export const unbatchGranules = (args: UnbatchGranulesInput): DiscoverGranulesOutput => {
+  const { providerPath, batchIndex } = args.config;
+  const batch = args.input[batchIndex];
+
+  console.info(
+    JSON.stringify({
+      providerPath,
+      numBatches: args.input.length,
+      batchIndex,
+      batchSize: batch.granules.length,
+    })
+  );
+
+  return batch;
+};
 
 /**
  * Prefixes the granule ID of each granule in an array of granules with the name of

src/lib/stdlib/arrays.spec.ts

@@ -2,40 +2,62 @@
 
 import test from 'ava';
 
-import { batchBounds } from './arrays';
+import * as A from './arrays';
 
 //------------------------------------------------------------------------------
-// slices
+// batchBounds
 //------------------------------------------------------------------------------
 
-test('slices should produce [[0, 0]] when totalLength == 0', (t) => {
-  t.deepEqual(batchBounds({ totalSize: 0, maxBatchSize: 0 }), [[0, 0]]);
+test('batchBounds should produce [[0, 0]] when totalLength == 0', (t) => {
+  t.deepEqual(A.batchBounds({ totalSize: 0, maxBatchSize: 0 }), [[0, 0]]);
 });
 
-test('slices should produce [[0, 0]] when totalLength < 0', (t) => {
-  t.deepEqual(batchBounds({ totalSize: -10, maxBatchSize: 10 }), [[0, 0]]);
+test('batchBounds should produce [[0, 0]] when totalLength < 0', (t) => {
+  t.deepEqual(A.batchBounds({ totalSize: -10, maxBatchSize: 10 }), [[0, 0]]);
 });
 
-test('slices should produce a singleton array when totalLength < maxSliceLength', (t) => {
-  t.deepEqual(batchBounds({ totalSize: 100, maxBatchSize: 200 }), [[0, 100]]);
+test('batchBounds should produce a singleton array when totalLength < maxSliceLength', (t) => {
+  t.deepEqual(A.batchBounds({ totalSize: 100, maxBatchSize: 200 }), [[0, 100]]);
 });
 
-test('slices should produce a singleton array when totalLength == maxSliceLength', (t) => {
-  t.deepEqual(batchBounds({ totalSize: 100, maxBatchSize: 100 }), [[0, 100]]);
+test('batchBounds should produce a singleton array when totalLength == maxSliceLength', (t) => {
+  t.deepEqual(A.batchBounds({ totalSize: 100, maxBatchSize: 100 }), [[0, 100]]);
 });
 
-test('slices should produce singleton slices when maxSliceLength <= 0', (t) => {
-  t.deepEqual(batchBounds({ totalSize: 3, maxBatchSize: 0 }), [
+test('batchBounds should produce single-step bounds when maxSliceLength <= 0', (t) => {
+  t.deepEqual(A.batchBounds({ totalSize: 3, maxBatchSize: 0 }), [
     [0, 1],
     [1, 2],
     [2, 3],
   ]);
 });
 
-test('slices should produce slices of nearly equal lengths', (t) => {
-  t.deepEqual(batchBounds({ totalSize: 100, maxBatchSize: 40 }), [
+test('batchBounds should produce bounds of nearly equal range sizes', (t) => {
+  t.deepEqual(A.batchBounds({ totalSize: 100, maxBatchSize: 40 }), [
     [0, 34],
     [34, 67],
     [67, 100],
   ]);
 });
+
+//------------------------------------------------------------------------------
+// ellipsize
+//------------------------------------------------------------------------------
+
+test('ellipsize should produce empty string for empty array', (t) => {
+  t.deepEqual(A.ellipsize([]), '');
+});
+
+test('ellipsize should not include ellipsis with short array and default params', (t) => {
+  t.deepEqual(
+    A.ellipsize([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
+    '1, 2, 3, 4, 5, 6, 7, 8, 9, 10'
+  );
+});
+
+test('ellipsize should include ellipsis with explicit params', (t) => {
+  t.deepEqual(
+    A.ellipsize([1, 2, 3, 4, 5], { head: 2, sep: '|', tail: 2 }),
+    '1|2|...|4|5'
+  );
+});