Merge pull request #294 from common-workflow-language/ship-tests

ship tests, add release tester

MANIFEST.in

@@ -1 +1,19 @@
 include gittaggers.py ez_setup.py Makefile cwltool.py
+include tests/*
+include tests/wf/*
+include cwltool/schemas/v1.0/*.yml
+include cwltool/schemas/draft-2/*.yml
+include cwltool/schemas/draft-3/*.yml
+include cwltool/schemas/draft-3/*.md
+include cwltool/schemas/draft-3/salad/schema_salad/metaschema/*.yml
+include cwltool/schemas/draft-3/salad/schema_salad/metaschema/*.md
+include cwltool/schemas/v1.0/*.yml
+include cwltool/schemas/v1.0/*.md
+include cwltool/schemas/v1.0/salad/schema_salad/metaschema/*.yml
+include cwltool/schemas/v1.0/salad/schema_salad/metaschema/*.md
+include cwltool/schemas/v1.1.0-dev1/*.yml
+include cwltool/schemas/v1.1.0-dev1/*.md
+include cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.yml
+include cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.md
+include cwltool/cwlNodeEngine.js
+global-exclude *.pyc

Makefile

@@ -26,7 +26,7 @@ MODULE=cwltool
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
-DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8
+DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8 pytest
 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pep257 sloccount python-flake8
 VERSION=1.0.$(shell date +%Y%m%d%H%M%S --date=`git log --first-parent \
 	--max-count=1 --format=format:%cI`)
@@ -50,7 +50,7 @@ install-deb-dep:
 
 ## install     : install the ${MODULE} module and schema-salad-tool
 install: FORCE
-	./setup.py build install
+	pip install .
 
 ## dist        : create a module package for distribution
 dist: dist/${MODULE}-$(VERSION).tar.gz

release-test.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+set -e
+set -x
+
+package=cwltool
+module=cwltool
+repo=https://github.com/common-workflow-language/cwltool.git
+run_tests="py.test --ignore ${module}/schemas/ --pyarg cwltool"
+pipver=7.0.2 # minimum required version of pip
+
+rm -Rf testenv? || /bin/true
+
+export HEAD=`git rev-parse HEAD`
+virtualenv testenv1
+virtualenv testenv2
+virtualenv testenv3
+virtualenv testenv4
+
+# First we test the head
+source testenv1/bin/activate
+rm testenv1/lib/python-wheels/setuptools* \
+	&& pip install --force-reinstall -U pip==${pipver} \
+        && pip install setuptools==20.10.1 wheel
+make install-dep
+make test
+pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
+mkdir testenv1/not-${module}
+# if there is a subdir named '${module}' py.test will execute tests
+# there instead of the installed module's tests
+pushd testenv1/not-${module}; ../bin/${run_tests}; popd
+
+
+# Secondly we test via pip
+
+cd testenv2
+source bin/activate
+rm lib/python-wheels/setuptools* \
+	&& pip install --force-reinstall -U pip==${pipver} \
+        && pip install setuptools==20.10.1 wheel
+pip install -e git+${repo}@${HEAD}#egg=${package}
+cd src/${package}
+make install-dep
+make dist
+make test
+cp dist/${package}*tar.gz ../../../testenv3/
+pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
+cd ../.. # no subdir named ${proj} here, safe for py.testing the installed module
+bin/${run_tests}
+
+# Is the distribution in testenv2 complete enough to build another
+# functional distribution?
+
+cd ../testenv3/
+source bin/activate
+rm lib/python-wheels/setuptools* \
+	&& pip install --force-reinstall -U pip==${pipver} \
+        && pip install setuptools==20.10.1 wheel
+pip install ${package}*tar.gz
+pip install pytest
+mkdir out
+tar --extract --directory=out -z -f ${package}*.tar.gz
+cd out/${package}*
+make dist
+make test
+pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
+mkdir ../not-${module}
+pushd ../not-${module} ; ../../bin/${run_tests}; popd

setup.cfg

@@ -2,5 +2,9 @@
 ignore = E124,E128,E129,E201,E202,E225,E226,E231,E265,E271,E302,E303,F401,E402,E501,W503,E731,F811,F821,F841
 exclude = cwltool/schemas
 
-[easy_install]
+[aliases]
+test=pytest
 
+[tool:pytest]
+addopts=--ignore cwltool/schemas
+testpaths = tests

setup.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 import os
+import sys
 
 import setuptools.command.egg_info as egg_info_cmd
 from setuptools import setup
@@ -14,6 +15,9 @@
 except ImportError:
     tagger = egg_info_cmd.egg_info
 
+needs_pytest = {'pytest', 'test', 'ptr'}.intersection(sys.argv)
+pytest_runner = ['pytest-runner'] if needs_pytest else []
+
 setup(name='cwltool',
       version='1.0',
       description='Common workflow language reference implementation',
@@ -23,7 +27,8 @@
       url="https://github.com/common-workflow-language/cwltool",
       download_url="https://github.com/common-workflow-language/cwltool",
       license='Apache 2.0',
-      packages=["cwltool"],
+      packages=["cwltool", 'cwltool.tests'],
+      package_dir={'cwltool.tests': 'tests'},
       package_data={'cwltool': ['schemas/draft-2/*.yml',
                                 'schemas/draft-3/*.yml',
                                 'schemas/draft-3/*.md',
@@ -38,6 +43,7 @@
                                 'schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.yml',
                                 'schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.md',
                                 'cwlNodeEngine.js']},
+      include_package_data=True,
       install_requires=[
           'setuptools',
           'requests >= 1.0',
@@ -47,8 +53,9 @@
           'schema-salad >= 2.2.20170216125639, < 3',
           'typing >= 3.5.2, < 3.6'
       ],
+      setup_requires=[] + pytest_runner,
       test_suite='tests',
-      tests_require=[],
+      tests_require=['pytest'],
       entry_points={
           'console_scripts': ["cwltool=cwltool.main:main"]
       },

tests/test_examples.py

@@ -5,7 +5,7 @@
 import cwltool.pathmapper
 import cwltool.process
 import cwltool.workflow
-
+from .util import get_data
 
 class TestParamMatching(unittest.TestCase):
     def test_params(self):
@@ -112,12 +112,12 @@ def test_params(self):
 class TestFactory(unittest.TestCase):
     def test_factory(self):
         f = cwltool.factory.Factory()
-        echo = f.make("tests/echo.cwl")
+        echo = f.make(get_data("tests/echo.cwl"))
         self.assertEqual(echo(inp="foo"), {"out": "foo\n"})
 
     def test_partial_scatter(self):
         f = cwltool.factory.Factory(on_error="continue")
-        fail = f.make("tests/wf/scatterfail.cwl")
+        fail = f.make(get_data("tests/wf/scatterfail.cwl"))
         try:
             fail()
         except cwltool.factory.WorkflowStatus as e:
@@ -129,7 +129,7 @@ def test_partial_scatter(self):
 
     def test_partial_output(self):
         f = cwltool.factory.Factory(on_error="continue")
-        fail = f.make("tests/wf/wffail.cwl")
+        fail = f.make(get_data("tests/wf/wffail.cwl"))
         try:
             fail()
         except cwltool.factory.WorkflowStatus as e:
@@ -333,7 +333,7 @@ def test_lifting(self):
         # fails if the step 'out' doesn't match.
         with self.assertRaises(cwltool.workflow.WorkflowException):
             f = cwltool.factory.Factory()
-            echo = f.make("tests/test_bad_outputs_wf.cwl")
+            echo = f.make(get_data("tests/test_bad_outputs_wf.cwl"))
             self.assertEqual(echo(inp="foo"), {"out": "foo\n"})
 
 

tests/test_pack.py

@@ -8,20 +8,23 @@
 from cwltool.load_tool import fetch_document, validate_document
 from cwltool.main import makeRelative
 from cwltool.process import adjustFileObjs, adjustDirObjs
-
+from .util import get_data
 
 class TestPack(unittest.TestCase):
     def test_pack(self):
         self.maxDiff = None
 
-        document_loader, workflowobj, uri = fetch_document("tests/wf/revsort.cwl")
+        document_loader, workflowobj, uri = fetch_document(
+            get_data("tests/wf/revsort.cwl"))
         document_loader, avsc_names, processobj, metadata, uri = validate_document(
             document_loader, workflowobj, uri)
         packed = cwltool.pack.pack(document_loader, processobj, uri, metadata)
-        with open("tests/wf/expect_packed.cwl") as f:
+        with open(get_data("tests/wf/expect_packed.cwl")) as f:
             expect_packed = json.load(f)
-        adjustFileObjs(packed, partial(makeRelative, os.path.abspath("tests/wf")))
-        adjustDirObjs(packed, partial(makeRelative, os.path.abspath("tests/wf")))
+        adjustFileObjs(packed, partial(makeRelative,
+            os.path.abspath(get_data("tests/wf"))))
+        adjustDirObjs(packed, partial(makeRelative,
+            os.path.abspath(get_data("tests/wf"))))
         self.assertIn("$schemas", packed)
         del packed["$schemas"]
         del expect_packed["$schemas"]

tests/test_toolargparse.py

@@ -2,7 +2,7 @@
 from tempfile import NamedTemporaryFile
 
 from cwltool.main import main
-
+from .util import get_data
 
 class ToolArgparse(unittest.TestCase):
     script = '''
@@ -66,8 +66,10 @@ def test_help(self):
         with NamedTemporaryFile() as f:
             f.write(self.script)
             f.flush()
-            self.assertEquals(main(["--debug", f.name, '--input', 'README.rst']), 0)
-            self.assertEquals(main(["--debug", f.name, '--input', 'README.rst']), 0)
+            self.assertEquals(main(["--debug", f.name, '--input',
+                get_data('tests/echo.cwl')]), 0)
+            self.assertEquals(main(["--debug", f.name, '--input',
+                get_data('tests/echo.cwl')]), 0)
 
     def test_bool(self):
         with NamedTemporaryFile() as f:
@@ -92,8 +94,8 @@ def test_record(self):
             f.write(self.script3)
             f.flush()
             try:
-                self.assertEquals(main([f.name, '--foo.one', 'README.rst',
-                                        '--foo.two', 'test']), 0)
+                self.assertEquals(main([f.name, '--foo.one',
+                    get_data('tests/echo.cwl'), '--foo.two', 'test']), 0)
             except SystemExit as e:
                 self.assertEquals(e.code, 0)
 

tests/util.py

@@ -0,0 +1,13 @@
+from pkg_resources import Requirement, resource_filename, ResolutionError  # type: ignore
+import os
+
+def get_data(filename):
+    filepath = None
+    try:
+        filepath = resource_filename(
+            Requirement.parse("cwltool"), filename)
+    except ResolutionError:
+        pass
+    if not filepath or not os.path.isfile(filepath):
+        filepath = os.path.join(os.path.dirname(__file__), os.pardir, filename)
+    return filepath