From 00fce36adcf503587954417db7a9887bcc9e8b14 Mon Sep 17 00:00:00 2001
From: Peter Amstutz <tetron@interreality.org>
Date: Mon, 19 Sep 2016 11:23:21 -0400
Subject: [PATCH] Scandeps only picks up file objects when "location" in
urlfileds. (#194)
* Scandeps only picks up file objects when "location" in urlfileds.
* Make test for scandeps actually test something. Also expand pack() test to
include external (non-packable) files.
---
cwltool/main.py | 20 ++++++++++--------
cwltool/process.py | 2 +-
tests/test_examples.py | 43 ++++++++++++++++++++++++++++++++------
tests/test_pack.py | 9 ++++++++
tests/wf/expect_packed.cwl | 6 +++++-
tests/wf/hello.txt | 6 ++++++
tests/wf/revsort.cwl | 3 +++
7 files changed, 72 insertions(+), 17 deletions(-)
create mode 100644 tests/wf/hello.txt
diff --git a/cwltool/main.py b/cwltool/main.py
index fadb57466..49b8f3807 100755
--- a/cwltool/main.py
+++ b/cwltool/main.py
@@ -461,6 +461,14 @@ def pathToLoc(p):
return (job_order_object, input_basedir)
+def makeRelative(base, ob):
+ u = ob.get("location", ob.get("path"))
+ if ":" in u.split("/")[0] and not u.startswith("file://"):
+ pass
+ else:
+ if u.startswith("file://"):
+ u = u[7:]
+ ob["location"] = os.path.relpath(u, base)
def printdeps(obj, document_loader, stdout, relative_deps, uri, basedir=None):
# type: (Dict[Text, Any], Loader, IO[Any], bool, Text, Text) -> None
@@ -472,7 +480,7 @@ def loadref(b, u):
sf = scandeps(
basedir if basedir else uri, obj, set(("$import", "run")),
- set(("$include", "$schemas")), loadref)
+ set(("$include", "$schemas", "location")), loadref)
if sf:
deps["secondaryFiles"] = sf
@@ -484,14 +492,8 @@ def loadref(b, u):
else:
raise Exception(u"Unknown relative_deps %s" % relative_deps)
- def makeRelative(ob):
- u = ob.get("location", ob.get("path"))
- if ":" in u.split("/")[0] and not u.startswith("file://"):
- pass
- else:
- ob["location"] = os.path.relpath(u, base)
- adjustFileObjs(deps, makeRelative)
- adjustDirObjs(deps, makeRelative)
+ adjustFileObjs(deps, functools.partial(makeRelative, base))
+ adjustDirObjs(deps, functools.partial(makeRelative, base))
stdout.write(json.dumps(deps, indent=4))
diff --git a/cwltool/process.py b/cwltool/process.py
index 1316f7cb2..2b07affa9 100644
--- a/cwltool/process.py
+++ b/cwltool/process.py
@@ -645,7 +645,7 @@ def scandeps(base, doc, reffields, urlfields, loadref):
})
base = df
- if doc.get("class") in ("File", "Directory"):
+ if doc.get("class") in ("File", "Directory") and "location" in urlfields:
u = doc.get("location", doc.get("path"))
if u:
deps = {
diff --git a/tests/test_examples.py b/tests/test_examples.py
index ef0f4f139..8d7778d68 100644
--- a/tests/test_examples.py
+++ b/tests/test_examples.py
@@ -122,7 +122,7 @@ def test_scandeps(self):
"id": "file:///example/foo.cwl#input1",
"default": {
"class": "File",
- "path": "file:///example/data.txt"
+ "location": "file:///example/data.txt"
}
}],
"run": {
@@ -130,8 +130,8 @@ def test_scandeps(self):
"inputs": [{
"id": "file:///example/bar.cwl#input2",
"default": {
- "class": "File",
- "path": "file:///example/data2.txt"
+ "class": "Directory",
+ "location": "file:///example/data2"
}
}]
}
@@ -145,10 +145,41 @@ def loadref(base, p):
else:
raise Exception("test case can't load things")
- print json.dumps(cwltool.process.scandeps(obj["id"], obj,
+ sc = cwltool.process.scandeps(obj["id"], obj,
set(("$import", "run")),
- set(("$include", "$schemas", "path")),
- loadref), indent=4)
+ set(("$include", "$schemas", "location")),
+ loadref)
+
+ sc.sort(key=lambda k: k["basename"])
+
+ self.assertEquals([{
+ "basename": "bar.cwl",
+ "class": "File",
+ "location": "file:///example/bar.cwl"
+ },
+ {
+ "basename": "data.txt",
+ "class": "File",
+ "location": "file:///example/data.txt"
+ },
+ {
+ "basename": "data2",
+ "class": "Directory",
+ "location": "file:///example/data2"
+ }], sc)
+
+ sc = cwltool.process.scandeps(obj["id"], obj,
+ set(("run"),),
+ set(), loadref)
+
+ sc.sort(key=lambda k: k["basename"])
+
+ self.assertEquals([{
+ "basename": "bar.cwl",
+ "class": "File",
+ "location": "file:///example/bar.cwl"
+ }], sc)
+
class TestTypeCompare(unittest.TestCase):
def test_typecompare(self):
diff --git a/tests/test_pack.py b/tests/test_pack.py
index 1c4eed63c..d0ff4457f 100644
--- a/tests/test_pack.py
+++ b/tests/test_pack.py
@@ -1,8 +1,14 @@
import unittest
import json
+import os
+from functools import partial
+
from cwltool.load_tool import fetch_document, validate_document
import cwltool.pack
import cwltool.workflow
+from cwltool.main import makeRelative
+from cwltool.process import adjustFileObjs, adjustDirObjs
+
class TestPack(unittest.TestCase):
def test_pack(self):
@@ -14,4 +20,7 @@ def test_pack(self):
packed = cwltool.pack.pack(document_loader, processobj, uri, metadata)
with open("tests/wf/expect_packed.cwl") as f:
expect_packed = json.load(f)
+ adjustFileObjs(packed, partial(makeRelative, os.path.abspath("tests/wf")))
+ adjustDirObjs(packed, partial(makeRelative, os.path.abspath("tests/wf")))
+
self.assertEqual(expect_packed, packed)
diff --git a/tests/wf/expect_packed.cwl b/tests/wf/expect_packed.cwl
index b1e0879cc..65cbc8b4c 100644
--- a/tests/wf/expect_packed.cwl
+++ b/tests/wf/expect_packed.cwl
@@ -6,7 +6,11 @@
{
"doc": "The input file to be processed.",
"type": "File",
- "id": "#main/input"
+ "id": "#main/input",
+ "default": {
+ "class": "File",
+ "location": "hello.txt"
+ }
},
{
"default": true,
diff --git a/tests/wf/hello.txt b/tests/wf/hello.txt
new file mode 100644
index 000000000..bbd22bd35
--- /dev/null
+++ b/tests/wf/hello.txt
@@ -0,0 +1,6 @@
+Hello
+world
+testing
+one
+two
+three.
diff --git a/tests/wf/revsort.cwl b/tests/wf/revsort.cwl
index a6b2774ad..d4d5a473b 100644
--- a/tests/wf/revsort.cwl
+++ b/tests/wf/revsort.cwl
@@ -23,6 +23,9 @@ inputs:
input:
type: File
doc: "The input file to be processed."
+ default:
+ class: File
+ location: hello.txt
reverse_sort:
type: boolean
default: true