fix empty scatter bug

Author: mr-c

cwltool/workflow_job.py

@@ -172,7 +172,7 @@ def nested_crossproduct_scatter(
     runtimeContext: RuntimeContext,
 ) -> JobsGeneratorType:
     scatter_key = scatter_keys[0]
-    jobl = len(cast(Sized, joborder[scatter_key]))
+    jobl = len(cast(Sized, joborder[scatter_key])) if joborder[scatter_key] else 0
     output = {}  # type: ScatterDestinationsType
     for i in process.tool["outputs"]:
         output[i["id"]] = [None] * jobl
@@ -219,7 +219,9 @@ def crossproduct_size(
         ssum = len(cast(Sized, joborder[scatter_key]))
     else:
         ssum = 0
-        for _ in range(0, len(cast(Sized, joborder[scatter_key]))):
+        for _ in range(
+            0, len(cast(Sized, joborder[scatter_key])) if joborder[scatter_key] else 0
+        ):
             ssum += crossproduct_size(joborder, scatter_keys[1:])
     return ssum
 
@@ -252,7 +254,7 @@ def _flat_crossproduct_scatter(
 ) -> Tuple[List[Optional[JobsGeneratorType]], int,]:
     """Inner loop."""
     scatter_key = scatter_keys[0]
-    jobl = len(cast(Sized, joborder[scatter_key]))
+    jobl = len(cast(Sized, joborder[scatter_key])) if joborder[scatter_key] else 0
     steps = []  # type: List[Optional[JobsGeneratorType]]
     put = startindex
     for index in range(0, jobl):
@@ -292,8 +294,8 @@ def dotproduct_scatter(
     jobl = None  # type: Optional[int]
     for key in scatter_keys:
         if jobl is None:
-            jobl = len(cast(Sized, joborder[key]))
-        elif jobl != len(cast(Sized, joborder[key])):
+            jobl = len(cast(Sized, joborder[key])) if joborder[key] else 0
+        elif jobl != len(cast(Sized, joborder[key])) if joborder[key] else 0:
             raise WorkflowException(
                 "Length of input arrays must be equal when performing "
                 "dotproduct scatter."
@@ -729,7 +731,9 @@ def valueFromFunc(
                 runtimeContext.postScatterEval = postScatterEval
 
                 emptyscatter = [
-                    shortname(s) for s in scatter if len(cast(Sized, inputobj[s])) == 0
+                    shortname(s)
+                    for s in scatter
+                    if not inputobj[s] or len(cast(Sized, inputobj[s])) == 0
                 ]
                 if emptyscatter:
                     _logger.warning(

tests/test_conditionals.py

@@ -17,3 +17,158 @@ def test_conditional_step_no_inputs() -> None:
     stderr = re.sub(r"\s\s+", " ", stderr)
     assert err_code == 0, stderr
     assert result is None
+
+
+def test_conditional_scatter_missing_input() -> None:
+    """Test that scattering a missing array skips execution."""
+    err_code, stdout, stderr = get_main_output(
+        [
+            get_data("tests/wf/scatter_before_when.cwl"),
+        ]
+    )
+    result = json.loads(stdout)["optional_echoed_messages"]
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert err_code == 0, stderr
+    assert result == []
+
+
+def test_scatter_empty_array() -> None:
+    """Test that scattering an empty array skips execution."""
+    err_code, stdout, stderr = get_main_output(
+        [
+            get_data("tests/wf/scatter_before_when.cwl"),
+            get_data("tests/wf/scatter_before_when-inp_empty.yaml"),
+        ]
+    )
+    result = json.loads(stdout)["optional_echoed_messages"]
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert err_code == 0, stderr
+    assert result == []
+
+
+def test_scatter_and_conditional() -> None:
+    """Test scattering a partially empty array with a conditional."""
+    err_code, stdout, stderr = get_main_output(
+        [
+            get_data("tests/wf/scatter_before_when.cwl"),
+            get_data("tests/wf/scatter_before_when-inp.yaml"),
+        ]
+    )
+    result = json.loads(stdout)["optional_echoed_messages"]
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert err_code == 0, stderr
+    assert result == ["We\n", "come\n", "in\n", "peace\n"]
+
+
+def test_scatter_dotproduct_empty_arrays() -> None:
+    """Test that dotproduct scattering empty arrays skips execution."""
+    err_code, stdout, stderr = get_main_output(
+        [
+            get_data("tests/wf/scatter_before_when_dotproduct.cwl"),
+        ]
+    )
+    result = json.loads(stdout)["optional_echoed_messages"]
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert err_code == 0, stderr
+    assert result == []
+
+
+def test_scatter_dotproduct_and_conditional() -> None:
+    """Test dotproduct scattering with partially empty arrays."""
+    err_code, stdout, stderr = get_main_output(
+        [
+            get_data("tests/wf/scatter_before_when_dotproduct.cwl"),
+            get_data("tests/wf/scatter_before_when_dotproduct-inp.yaml"),
+        ]
+    )
+    result = json.loads(stdout)["optional_echoed_messages"]
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert err_code == 0, stderr
+    assert result == [
+        "We Never\n",
+        "Come Out\n",
+        "In Anything But\n",
+        "Peace -- The Aliens\n",
+    ]
+
+
+def test_scatter_nested_crossproduct_empty_arrays() -> None:
+    """Test that nested_dotproduct scattering empty arrays skips execution."""
+    err_code, stdout, stderr = get_main_output(
+        [
+            get_data("tests/wf/scatter_before_when-nested_crossproduct.cwl"),
+        ]
+    )
+    result = json.loads(stdout)["optional_echoed_messages"]
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert err_code == 0, stderr
+    assert result == []
+
+
+def test_scatter_nested_crossproduct_and_conditional() -> None:
+    """Test nested_crossproduct scattering with partially empty arrays."""
+    err_code, stdout, stderr = get_main_output(
+        [
+            get_data("tests/wf/scatter_before_when-nested_crossproduct.cwl"),
+            get_data("tests/wf/scatter_before_when_dotproduct-inp.yaml"),
+        ]
+    )
+    result = json.loads(stdout)["optional_echoed_messages"]
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert err_code == 0, stderr
+    assert result == [
+        ["We Never\n", "We Out\n", "We Anything But\n", None, "We -- The Aliens\n"],
+        [
+            "Come Never\n",
+            "Come Out\n",
+            "Come Anything But\n",
+            None,
+            "Come -- The Aliens\n",
+        ],
+        ["In Never\n", "In Out\n", "In Anything But\n", None, "In -- The Aliens\n"],
+        [None, None, None, None, None],
+    ]
+
+
+def test_scatter_flat_crossproduct_empty_arrays() -> None:
+    """Test that flat_dotproduct scattering empty arrays skips execution."""
+    err_code, stdout, stderr = get_main_output(
+        [
+            get_data("tests/wf/scatter_before_when-flat_crossproduct.cwl"),
+        ]
+    )
+    result = json.loads(stdout)["optional_echoed_messages"]
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert err_code == 0, stderr
+    assert result == []
+
+
+def test_scatter_flat_crossproduct_and_conditional() -> None:
+    """Test flat_crossproduct scattering with partially empty arrays."""
+    err_code, stdout, stderr = get_main_output(
+        [
+            get_data("tests/wf/scatter_before_when-flat_crossproduct.cwl"),
+            get_data("tests/wf/scatter_before_when_dotproduct-inp.yaml"),
+        ]
+    )
+    result = json.loads(stdout)["optional_echoed_messages"]
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert err_code == 0, stderr
+    assert result == [
+        "We Never\n",
+        "We Out\n",
+        "We Anything But\n",
+        "We -- The Aliens\n",
+        "Come Never\n",
+        "Come Out\n",
+        "Come Anything But\n",
+        "Come -- The Aliens\n",
+        "In Never\n",
+        "In Out\n",
+        "In Anything But\n",
+        "In -- The Aliens\n",
+        "Peace Never\n",
+        "Peace Out\n",
+        "Peace Anything But\n",
+        "Peace -- The Aliens\n",
+    ]

tests/wf/scatter_before_when-flat_crossproduct.cwl

@@ -0,0 +1,38 @@
+cwlVersion: v1.2
+class: Workflow
+
+requirements:
+  ScatterFeatureRequirement: {}
+  InlineJavascriptRequirement: {}
+  StepInputExpressionRequirement: {}
+
+inputs:
+  messages:
+    type:
+      - "null"
+      - type: array
+        items: [string, "null"]
+  extras:
+    type:
+      - "null"
+      - type: array
+        items: [string, "null"]
+
+steps:
+  optional_echo_scatter:
+    when: $(inputs.messages !== null && inputs.extra !== null)
+    run: ../echo.cwl
+    scatter: [messages, extra]
+    scatterMethod: flat_crossproduct
+    in:
+      extra: extras
+      messages: messages
+      inp:
+        valueFrom: $(inputs.messages) $(inputs.extra)
+    out: [out]
+
+outputs:
+  optional_echoed_messages:
+    type: string[]?
+    pickValue: all_non_null
+    outputSource: optional_echo_scatter/out

tests/wf/scatter_before_when-inp.yaml

@@ -0,0 +1,6 @@
+messages:
+  - We
+  - come
+  - in
+  - null
+  - peace

tests/wf/scatter_before_when-inp_empty.yaml

@@ -0,0 +1,2 @@
+messages: []
+

tests/wf/scatter_before_when-nested_crossproduct.cwl

@@ -0,0 +1,38 @@
+cwlVersion: v1.2
+class: Workflow
+
+requirements:
+  ScatterFeatureRequirement: {}
+  InlineJavascriptRequirement: {}
+  StepInputExpressionRequirement: {}
+
+inputs:
+  messages:
+    type:
+      - "null"
+      - type: array
+        items: [string, "null"]
+  extras:
+    type:
+      - "null"
+      - type: array
+        items: [string, "null"]
+
+steps:
+  optional_echo_scatter:
+    when: $(inputs.messages !== null && inputs.extra !== null)
+    run: ../echo.cwl
+    scatter: [messages, extra]
+    scatterMethod: nested_crossproduct
+    in:
+      extra: extras
+      messages: messages
+      inp:
+        valueFrom: $(inputs.messages) $(inputs.extra)
+    out: [out]
+
+outputs:
+  optional_echoed_messages:
+    type: string[]?
+    pickValue: all_non_null
+    outputSource: optional_echo_scatter/out

tests/wf/scatter_before_when.cwl

@@ -0,0 +1,28 @@
+cwlVersion: v1.2
+class: Workflow
+
+requirements:
+  ScatterFeatureRequirement: {}
+  InlineJavascriptRequirement: {}
+
+inputs:
+  messages:
+    type:
+      - "null"
+      - type: array
+        items: [string, "null"]
+
+steps:
+  optional_echo_scatter:
+    when: $(inputs.inp !== null)
+    run: ../echo.cwl
+    scatter: inp
+    in:
+      inp: messages
+    out: [out]
+
+outputs:
+  optional_echoed_messages:
+    type: string[]?
+    pickValue: all_non_null
+    outputSource: optional_echo_scatter/out

tests/wf/scatter_before_when_dotproduct-inp.yaml

@@ -0,0 +1,12 @@
+messages:
+  - We
+  - Come
+  - In
+  - null
+  - Peace
+extras:
+  - Never
+  - Out
+  - Anything But
+  - null
+  - "-- The Aliens"

tests/wf/scatter_before_when_dotproduct.cwl

@@ -0,0 +1,38 @@
+cwlVersion: v1.2
+class: Workflow
+
+requirements:
+  ScatterFeatureRequirement: {}
+  InlineJavascriptRequirement: {}
+  StepInputExpressionRequirement: {}
+
+inputs:
+  messages:
+    type:
+      - "null"
+      - type: array
+        items: [string, "null"]
+  extras:
+    type:
+      - "null"
+      - type: array
+        items: [string, "null"]
+
+steps:
+  optional_echo_scatter:
+    when: $(inputs.messages !== null && inputs.extras !== null)
+    run: ../echo.cwl
+    scatter: [messages, extra]
+    scatterMethod: dotproduct
+    in:
+      extra: extras
+      messages: messages
+      inp:
+        valueFrom: $(inputs.messages) $(inputs.extra)
+    out: [out]
+
+outputs:
+  optional_echoed_messages:
+    type: string[]?
+    pickValue: all_non_null
+    outputSource: optional_echo_scatter/out