ADES stage-out fix (partial) for cwl workflow outputs of type Directo…

…ry[]

deploy/eoepca/zoo-files/stageout.py

@@ -8,15 +8,19 @@
 from urllib.parse import urlparse
 from datetime import datetime
 
-cat_url = sys.argv[1]
-bucket = sys.argv[2]
-subfolder = sys.argv[3]
-collection_id = sys.argv[4]
-print(f"cat_url: {cat_url}", file=sys.stderr)
+bucket = sys.argv[1]
+subfolder = sys.argv[2]
+collection_id = sys.argv[3]
 print(f"bucket: {bucket}", file=sys.stderr)
 print(f"subfolder: {subfolder}", file=sys.stderr)
 print(f"collection_id: {collection_id}", file=sys.stderr)
 
+# cat_url
+# Should really be one or more, for cwl outputs of type Directory[]
+# - but as a quick fix we just take the first one for now.
+cat_url = sys.argv[4]
+print(f"cat_url: {cat_url}", file=sys.stderr)
+
 aws_access_key_id = os.environ["AWS_ACCESS_KEY_ID"]
 aws_secret_access_key = os.environ["AWS_SECRET_ACCESS_KEY"]
 region_name = os.environ["AWS_REGION"]

deploy/eoepca/zoo-files/stageout.yaml

@@ -26,10 +26,22 @@ baseCommand:
   - python
   - stageout.py
 arguments:
-  - $( inputs.wf_outputs.path )
   - $( inputs.STAGEOUT_OUTPUT )
   - $( inputs.process )
   - $( inputs.collection_id )
+  - valueFrom: |
+      ${
+        if( !Array.isArray(inputs.wf_outputs) ) 
+        {
+          return inputs.wf_outputs.path;
+        }
+        var args=[];
+        for (var i = 0; i < inputs.wf_outputs.length; i++) 
+        {
+          args.push(inputs.wf_outputs[i].path);
+        }
+        return args;
+      }
 requirements:
   DockerRequirement:
     dockerPull: ghcr.io/terradue/ogc-eo-application-package-hands-on/stage:1.3.2

deploy/samples/requests/processing/snuggs-app.cwl

@@ -1,78 +1,71 @@
-$graph:
-
-- class: Workflow
-  doc: Applies s expressions to EO acquisitions
-  id: snuggs
-  requirements:
-  - class: ScatterFeatureRequirement
-  inputs:
-    input_reference:
-      doc: Input product reference
-      label: Input product reference
-      type: string[]
-    s_expression:
-      doc: s expression
-      label: s expression
-      type: string[]
-  label: s expressions
-  outputs:
-  - id: wf_outputs
-    outputSource:
-    - step_1/results
-    type:
-      Directory[]
-
-  steps:
-    step_1:
-      in:
-        input_reference: input_reference
-        s_expression: s_expression
-      out:
-      - results
-      run: '#clt'
-      scatter: [input_reference, s_expression]
-      scatterMethod: flat_crossproduct
-
-
-- baseCommand: s-expression
-  class: CommandLineTool
-
-  id: clt
-
-  arguments:
-  - --input_reference
-  - valueFrom: $( inputs.input_reference )
-  - --s-expression
-  - valueFrom: ${ return inputs.s_expression.split(":")[1]; }
-  - --cbn
-  - valueFrom: ${ return inputs.s_expression.split(":")[0]; }
-
-  inputs:
-    input_reference:
-      type: string
-    s_expression:
-      type: string
-
-  outputs:
-    results:
-      outputBinding:
-        glob: .
-      type: Directory
-  requirements:
-    EnvVarRequirement:
-      envDef:
-        PATH: /srv/conda/envs/env_app_snuggs/bin:/srv/conda/bin:/srv/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-    ResourceRequirement: {}
-    InlineJavascriptRequirement: {}
-    DockerRequirement:
-      dockerPull: eoepca/snuggs:0.3.0
-  #stderr: std.err
-  #stdout: std.out
-
 cwlVersion: v1.0
-
 $namespaces:
   s: https://schema.org/
 s:softwareVersion: 0.3.0
 schemas:
-- http://schema.org/version/9.0/schemaorg-current-http.rdf
+  - http://schema.org/version/9.0/schemaorg-current-http.rdf
+$graph:
+  - class: Workflow
+    id: snuggs
+    label: s expressions
+    doc: Applies s expressions to EO acquisitions
+    requirements:
+      ScatterFeatureRequirement: {}
+      ResourceRequirement:
+        coresMax: 1
+        ramMax: 1024
+    inputs:
+      input_reference:
+        doc: Input product reference
+        label: Input product reference
+        type: string[]
+      s_expression:
+        doc: s expression
+        label: s expression
+        type: string[]
+    outputs:
+      - id: snuggs_outputs
+        type: Directory[]
+        outputSource:
+          - step_1/results
+    steps:
+      step_1:
+        in:
+          input_reference: input_reference
+          s_expression: s_expression
+        out:
+          - results
+        run: "#clt"
+        scatter: [input_reference, s_expression]
+        scatterMethod: flat_crossproduct
+
+  - class: CommandLineTool
+    id: clt
+    baseCommand: s-expression
+    arguments:
+      - --input_reference
+      - valueFrom: $( inputs.input_reference )
+      - --s-expression
+      - valueFrom: ${ return inputs.s_expression.split(":")[1]; }
+      - --cbn
+      - valueFrom: ${ return inputs.s_expression.split(":")[0]; }
+    inputs:
+      input_reference:
+        type: string
+      s_expression:
+        type: string
+    outputs:
+      results:
+        outputBinding:
+          glob: .
+        type: Directory
+    requirements:
+      EnvVarRequirement:
+        envDef:
+          PATH: /srv/conda/envs/env_app_snuggs/bin:/srv/conda/bin:/srv/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+      ResourceRequirement: {}
+      InlineJavascriptRequirement: {}
+      DockerRequirement:
+        dockerPull: eoepca/snuggs:0.3.0
+    #stderr: std.err
+    #stdout: std.out

docs/index.md

@@ -10,6 +10,7 @@
       _Namespace can now be set via chart environment variable `APP_HUB_NAMESPACE`_
     * **[FIX]** Update Application Hub to chart version `2.0.59` to add support for path-prefix (`BASE_URL`)
     * **[FIX]** Correct default value of `PROCESSING_MAX_RAM` to the integer value in Mi `1024` (was string `8Gi`)
+    * **[FIX]** ADES stage-out fix (partial) for cwl workflow outputs of type Directory[] - e.g. `snuggs` sample app
 
 The Deployment Guide captures each release of the EOEPCA Reference Implementation, by providing for each version...