Keep directory parameters in job parameters

and set LoadListingRequirement as supported requirement, since cwltool
handles this for us.

Author: mvdbeek

lib/galaxy/tool_util/cwl/parser.py

@@ -70,6 +70,7 @@
     "EnvVarRequirement",
     "InitialWorkDirRequirement",
     "InlineJavascriptRequirement",
+    "LoadListingRequirement",
     "ResourceRequirement",
     "ShellCommandRequirement",
     "ScatterFeatureRequirement",
@@ -346,15 +347,16 @@ def _ensure_cwl_job_initialized(self):
             self._is_command_line_job = hasattr(self._cwl_job, "command_line")
 
     def _normalize_job(self):
+        runtime_context = RuntimeContext({})
+        make_fs_access = getdefault(runtime_context.make_fs_access, StdFsAccess)
+        fs_access = make_fs_access(runtime_context.basedir)
+
         # Somehow reuse whatever causes validate in cwltool... maybe?
         def pathToLoc(p):
             if "location" not in p and "path" in p:
                 p["location"] = p["path"]
                 del p["path"]
 
-        runtime_context = RuntimeContext({})
-        make_fs_access = getdefault(runtime_context.make_fs_access, StdFsAccess)
-        fs_access = make_fs_access(runtime_context.basedir)
         process.fill_in_defaults(self._tool_proxy._tool.tool["inputs"], self._input_dict, fs_access)
         visit_class(self._input_dict, ("File", "Directory"), pathToLoc)
         # TODO: Why doesn't fillInDefault fill in locations instead of paths?

lib/galaxy/tool_util/cwl/representation.py

@@ -7,6 +7,7 @@
 from enum import Enum
 from typing import (
     Any,
+    Dict,
     NamedTuple,
     Optional,
 )
@@ -206,6 +207,9 @@ def dataset_wrapper_to_directory_json(inputs_dir, dataset_wrapper):
     except Exception:
         archive_location = None
 
+    extra_params = getattr(dataset_wrapper.unsanitized, "extra_params", {})
+    # We need to resolve path to location if there is a listing
+
     directory_json = {
         "location": dataset_wrapper.extra_files_path,
         "class": "Directory",
@@ -214,8 +218,20 @@ def dataset_wrapper_to_directory_json(inputs_dir, dataset_wrapper):
         "archive_nameext": nameext,
         "archive_nameroot": nameroot,
     }
-
-    return directory_json
+    extra_params.update(directory_json)
+    entry_to_location(extra_params, extra_params["location"])
+    return extra_params
+
+
+def entry_to_location(entry: Dict[str, Any], parent_location: str):
+    # TODO unit test
+    if entry["class"] == "File" and "path" in entry and "location" not in entry:
+        entry["location"] = os.path.join(parent_location, entry.pop("path"))
+    elif entry["class"] == "Directory" and "listing" in entry:
+        if "location" not in entry and "path" in entry:
+            entry["location"] = os.path.join(parent_location, entry.pop("path"))
+        for listing_entry in entry["listing"]:
+            entry_to_location(listing_entry, parent_location=entry["location"])
 
 
 def collection_wrapper_to_array(inputs_dir, wrapped_value):

lib/galaxy/tool_util/cwl/util.py

@@ -312,7 +312,9 @@ def replacement_directory(value):
         finally:
             if temp_dir:
                 shutil.rmtree(temp_dir)
-        return upload_tar(tmp.name)
+        upload_response = upload_tar(tmp.name)
+        upload_response.update(value)
+        return upload_response
 
     def replacement_list(value) -> Dict[str, str]:
         collection_element_identifiers = []

lib/galaxy/tools/parameters/basic.py

@@ -1899,7 +1899,10 @@ def single_to_json(value):
                 src = "hda"
             if src is not None:
                 object_id = cached_id(value)
-                return {"id": app.security.encode_id(object_id) if use_security else object_id, "src": src}
+                new_val = getattr(value, "extra_params", {})
+                new_val["id"] = app.security.encode_id(object_id) if use_security else object_id
+                new_val["src"] = src
+                return new_val
 
         if value not in [None, "", "None"]:
             if isinstance(value, list) and len(value) > 0:
@@ -1912,15 +1915,9 @@ def single_to_json(value):
     def to_python(self, value, app):
         def single_to_python(value):
             if isinstance(value, dict) and "src" in value:
-                id = value["id"] if isinstance(value["id"], int) else app.security.decode_id(value["id"])
-                if value["src"] == "dce":
-                    return app.model.context.query(DatasetCollectionElement).get(id)
-                elif value["src"] == "hdca":
-                    return app.model.context.query(HistoryDatasetCollectionAssociation).get(id)
-                elif value["src"] == "ldda":
-                    return app.model.context.query(LibraryDatasetDatasetAssociation).get(id)
-                else:
-                    return app.model.context.query(HistoryDatasetAssociation).get(id)
+                if not value["src"] in ("hda", "dce", "ldda", "hdca"):
+                    raise ParameterValueError(f"Invalid value {value}", self.name)
+                return src_id_to_item(sa_session=app.model.context, security=app.security, value=value)
 
         if isinstance(value, dict) and "values" in value:
             if hasattr(self, "multiple") and self.multiple is True:
@@ -1996,6 +1993,23 @@ def do_validate(v):
                 raise ValueError("At most %d datasets are required for %s" % (self.max, self.name))
 
 
+def src_id_to_item(sa_session, value, security):
+    src_to_class = {
+        "hda": HistoryDatasetAssociation,
+        "ldda": LibraryDatasetDatasetAssociation,
+        "dce": DatasetCollectionElement,
+        "hdca": HistoryDatasetCollectionAssociation,
+    }
+    id_value = value["id"]
+    decoded_id = id_value if isinstance(id_value, int) else security.decode_id(id_value)
+    try:
+        item = sa_session.query(src_to_class[value["src"]]).get(decoded_id)
+    except KeyError:
+        raise ValueError(f"Unknown input source {value['src']} passed to job submission API.")
+    item.extra_params = {k: v for k, v in value.items() if k not in ("src", "id")}
+    return item
+
+
 class DataToolParameter(BaseDataToolParameter):
     # TODO, Nate: Make sure the following unit tests appropriately test the dataset security
     # components.  Add as many additional tests as necessary.
@@ -2063,21 +2077,13 @@ def from_json(self, value, trans, other_values=None):
             value = [int(value_part) for value_part in value.split(",")]
         rval = []
         if isinstance(value, list):
-            found_hdca = False
+            found_srcs = set()
             for single_value in value:
                 if isinstance(single_value, dict) and "src" in single_value and "id" in single_value:
-                    if single_value["src"] == "hda":
-                        decoded_id = trans.security.decode_id(single_value["id"])
-                        rval.append(trans.sa_session.query(HistoryDatasetAssociation).get(decoded_id))
-                    elif single_value["src"] == "hdca":
-                        found_hdca = True
-                        decoded_id = trans.security.decode_id(single_value["id"])
-                        rval.append(trans.sa_session.query(HistoryDatasetCollectionAssociation).get(decoded_id))
-                    elif single_value["src"] == "ldda":
-                        decoded_id = trans.security.decode_id(single_value["id"])
-                        rval.append(trans.sa_session.query(LibraryDatasetDatasetAssociation).get(decoded_id))
-                    else:
-                        raise ValueError(f"Unknown input source {single_value['src']} passed to job submission API.")
+                    found_srcs.add(single_value["src"])
+                    rval.append(
+                        src_id_to_item(sa_session=trans.sa_session, value=single_value, security=trans.security)
+                    )
                 elif isinstance(
                     single_value,
                     (
@@ -2095,24 +2101,15 @@ def from_json(self, value, trans, other_values=None):
                         log.warning("Encoded ID where unencoded ID expected.")
                         single_value = trans.security.decode_id(single_value)
                     rval.append(trans.sa_session.query(HistoryDatasetAssociation).get(single_value))
-            if found_hdca:
-                for val in rval:
-                    if not isinstance(val, HistoryDatasetCollectionAssociation):
-                        raise ParameterValueError(
-                            "if collections are supplied to multiple data input parameter, only collections may be used",
-                            self.name,
-                        )
+                if len(found_srcs) > 1 and "hdca" in found_srcs:
+                    raise ParameterValueError(
+                        "if collections are supplied to multiple data input parameter, only collections may be used",
+                        self.name,
+                    )
         elif isinstance(value, (HistoryDatasetAssociation, LibraryDatasetDatasetAssociation)):
             rval.append(value)
         elif isinstance(value, dict) and "src" in value and "id" in value:
-            if value["src"] == "hda":
-                decoded_id = trans.security.decode_id(value["id"])
-                rval.append(trans.sa_session.query(HistoryDatasetAssociation).get(decoded_id))
-            elif value["src"] == "hdca":
-                decoded_id = trans.security.decode_id(value["id"])
-                rval.append(trans.sa_session.query(HistoryDatasetCollectionAssociation).get(decoded_id))
-            else:
-                raise ValueError(f"Unknown input source {value['src']} passed to job submission API.")
+            rval.append(src_id_to_item(sa_session=trans.sa_session, value=value, security=trans.security))
         elif str(value).startswith("__collection_reduce__|"):
             encoded_ids = [v[len("__collection_reduce__|") :] for v in str(value).split(",")]
             decoded_ids = map(trans.security.decode_id, encoded_ids)