From e9fdbceca9ba46e644d6d9dc8b157ea0b26c6db7 Mon Sep 17 00:00:00 2001 From: Julio Date: Wed, 15 Apr 2020 04:56:25 +0200 Subject: [PATCH 01/15] Prevent autover from failing when used alongside async frameworks --- .travis.yml | 2 +- autover/version.py | 2 +- conda.recipe/meta.yaml | 1 + examples/PkgBundle/version.py | 2 +- examples/pkg_bundle/version.py | 2 +- tests/test_asgi_server.py | 6 ++++++ tox.ini | 8 ++++++-- 7 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 tests/test_asgi_server.py diff --git a/.travis.yml b/.travis.yml index 63f5c3fe..f38cdfb3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ jobs: - stage: test python: 3.6 - install: pip install tox + install: pip install tox gunicorn uvicorn starlette script: tox -e py${TRAVIS_PYTHON_VERSION//./} - stage: test diff --git a/autover/version.py b/autover/version.py index 76fc1d02..a2b43002 100644 --- a/autover/version.py +++ b/autover/version.py @@ -117,7 +117,7 @@ def run_cmd(args, cwd=None): cwd=cwd) output, error = (str(s.decode()).strip() for s in proc.communicate()) - if proc.returncode != 0: + if proc.returncode != 0 or len(error) > 0: raise Exception(proc.returncode, error) return output diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 9ceba99b..853fa2e0 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -26,6 +26,7 @@ test: source_files: - tests commands: + - conda install -y -c conda-forge gunicorn uvicorn starlette - nosetests -vv --nologcapture about: diff --git a/examples/PkgBundle/version.py b/examples/PkgBundle/version.py index 76fc1d02..a2b43002 100644 --- a/examples/PkgBundle/version.py +++ b/examples/PkgBundle/version.py @@ -117,7 +117,7 @@ def run_cmd(args, cwd=None): cwd=cwd) output, error = (str(s.decode()).strip() for s in proc.communicate()) - if proc.returncode != 0: + if proc.returncode != 0 or len(error) > 0: raise Exception(proc.returncode, error) return output diff --git a/examples/pkg_bundle/version.py b/examples/pkg_bundle/version.py index 76fc1d02..a2b43002 100644 --- a/examples/pkg_bundle/version.py +++ b/examples/pkg_bundle/version.py @@ -117,7 +117,7 @@ def run_cmd(args, cwd=None): cwd=cwd) output, error = (str(s.decode()).strip() for s in proc.communicate()) - if proc.returncode != 0: + if proc.returncode != 0 or len(error) > 0: raise Exception(proc.returncode, error) return output diff --git a/tests/test_asgi_server.py b/tests/test_asgi_server.py new file mode 100644 index 00000000..dbc313f5 --- /dev/null +++ b/tests/test_asgi_server.py @@ -0,0 +1,6 @@ +import autover +from starlette.applications import Starlette + +print(autover.__version__) + +app = Starlette() diff --git a/tox.ini b/tox.ini index 1d98793e..641c62b4 100644 --- a/tox.ini +++ b/tox.ini @@ -3,8 +3,12 @@ envlist = py36,py27 [testenv] passenv = GIT_VERSION -deps = nose -commands = nosetests -vv --nologcapture --with-doctest --with-coverage --cover-package=autover +deps = + nose + gunicorn;python_version>"3.4" + uvicorn;python_version>"3.4" + starlette;python_version>"3.4" +commands = nosetests -vv --nologcapture --with-doctest --with-coverage --cover-package=autover --ignore-files=test_asgi_server.py [testenv:lint_checks] deps = flake8 From 6f56e3bd8f0372ce6b231199f64a734f36af44c0 Mon Sep 17 00:00:00 2001 From: Julio Date: Wed, 15 Apr 2020 04:58:17 +0200 Subject: [PATCH 02/15] Added test for async server --- tests/testversion.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/testversion.py b/tests/testversion.py index 8af07e68..2f6a46c9 100644 --- a/tests/testversion.py +++ b/tests/testversion.py @@ -1,7 +1,11 @@ """ Unit test for autover.Version """ +import os import unittest +import subprocess +import sys +import time import pkg_resources._vendor.packaging.version as packaging_version from autover import Version @@ -75,6 +79,7 @@ def pep440(version): class TestVersion(unittest.TestCase): + maxDiff=None def git_describe_check(self, describe_tests, index): @@ -160,6 +165,31 @@ def test_git_describe_4(self): def test_git_describe_5(self): self.git_describe_check(describe_tests, 5) + #=======================================# + # Compatibility with ASGI applications # + #=======================================# + + def test_run_asgi_server(self): + if sys.version[0] == "3": + serverfile_dir = os.path.abspath(os.path.join(__file__, os.pardir)) + logfile_path = os.path.abspath(os.path.join(__file__, os.pardir, "stderr_gunicorn.log")) + subprocess.Popen(["gunicorn", "-D", "--log-level", "ERROR", + "--chdir", serverfile_dir, + "-k", "uvicorn.workers.UvicornWorker", + "--error-logfile", logfile_path, + "--bind", "unix:/tmp/gunicorn.sock", + "test_asgi_server:app"]) + time.sleep(5) + with open(logfile_path, "r") as gunicorn_log_file: + stderr_log = gunicorn_log_file.read() + + subprocess.Popen(["pkill", "-f", "gunicorn"]) + time.sleep(5) + os.remove(logfile_path) + else: + stderr_log = "" + self.assertEqual(stderr_log, "") + if __name__ == "__main__": import nose nose.runmodule() From d7ecba87d7be90d4d28b11bc7643b00bc2e91f0c Mon Sep 17 00:00:00 2001 From: Julio Date: Wed, 15 Apr 2020 13:12:20 +0200 Subject: [PATCH 03/15] Removed dependencies installation from .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f38cdfb3..63f5c3fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ jobs: - stage: test python: 3.6 - install: pip install tox gunicorn uvicorn starlette + install: pip install tox script: tox -e py${TRAVIS_PYTHON_VERSION//./} - stage: test From fc94f70c7b0ed69b859f2f508a6c132c2725f57d Mon Sep 17 00:00:00 2001 From: Julio Date: Wed, 15 Apr 2020 18:17:20 +0200 Subject: [PATCH 04/15] Better documentation for new test --- tests/test_asgi_server.py | 20 +++++++++++++++++++- tests/testversion.py | 1 + 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/test_asgi_server.py b/tests/test_asgi_server.py index dbc313f5..0c8599fa 100644 --- a/tests/test_asgi_server.py +++ b/tests/test_asgi_server.py @@ -1,6 +1,24 @@ import autover from starlette.applications import Starlette -print(autover.__version__) +# A simple ASGI server application +# that tries to make use of autover. +# +# Given that autover makes use of +# subprocess.Popen and the program +# workflow depends on that, there +# were some corner cases where +# autover would just crash if we +# tried to get its __version__, +# therefore crashing the whole +# ASGI application. +# +# This test application is used +# with Gunicorn in the unit tests +# (testversion.py), just to check +# if the ASGI application crashes +# or not. + +_ = autover.__version__ app = Starlette() diff --git a/tests/testversion.py b/tests/testversion.py index 2f6a46c9..5118eb6a 100644 --- a/tests/testversion.py +++ b/tests/testversion.py @@ -170,6 +170,7 @@ def test_git_describe_5(self): #=======================================# def test_run_asgi_server(self): + """test_run_asgi_server (An ASGI app that uses autover and is launched with Gunicorn should not crash)""" if sys.version[0] == "3": serverfile_dir = os.path.abspath(os.path.join(__file__, os.pardir)) logfile_path = os.path.abspath(os.path.join(__file__, os.pardir, "stderr_gunicorn.log")) From 83bb8feca9341e89e8b8523b4e4d970cbeab7554 Mon Sep 17 00:00:00 2001 From: jlstevens Date: Fri, 5 Feb 2021 11:09:08 +0000 Subject: [PATCH 05/15] Adding uvloop dependency --- conda.recipe/meta.yaml | 2 +- tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 853fa2e0..8323f8cf 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -26,7 +26,7 @@ test: source_files: - tests commands: - - conda install -y -c conda-forge gunicorn uvicorn starlette + - conda install -y -c conda-forge gunicorn uvicorn starlette uvloop - nosetests -vv --nologcapture about: diff --git a/tox.ini b/tox.ini index 641c62b4..9e8a2301 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ deps = nose gunicorn;python_version>"3.4" uvicorn;python_version>"3.4" + uvloop;python_version>"3.4" starlette;python_version>"3.4" commands = nosetests -vv --nologcapture --with-doctest --with-coverage --cover-package=autover --ignore-files=test_asgi_server.py From 96c29627d82943aad5e460b641be6a062461eb88 Mon Sep 17 00:00:00 2001 From: jlstevens Date: Fri, 5 Feb 2021 12:37:35 +0000 Subject: [PATCH 06/15] Printing error log --- tests/testversion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/testversion.py b/tests/testversion.py index 5118eb6a..243ca123 100644 --- a/tests/testversion.py +++ b/tests/testversion.py @@ -189,6 +189,7 @@ def test_run_asgi_server(self): os.remove(logfile_path) else: stderr_log = "" + print(stderr_log) self.assertEqual(stderr_log, "") if __name__ == "__main__": From 23c75b830b8035c2e4bfbae96d25d08d9c8b6f4e Mon Sep 17 00:00:00 2001 From: jlstevens Date: Fri, 5 Feb 2021 13:33:19 +0000 Subject: [PATCH 07/15] Added httptools dependency --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 9e8a2301..912d7fa7 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ deps = uvicorn;python_version>"3.4" uvloop;python_version>"3.4" starlette;python_version>"3.4" + httptools>"3.4" commands = nosetests -vv --nologcapture --with-doctest --with-coverage --cover-package=autover --ignore-files=test_asgi_server.py [testenv:lint_checks] From d247a616dde45c76ffc77f612d961632289f211f Mon Sep 17 00:00:00 2001 From: jlstevens Date: Fri, 5 Feb 2021 14:17:21 +0000 Subject: [PATCH 08/15] Fixed error in tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 912d7fa7..91afa017 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ deps = uvicorn;python_version>"3.4" uvloop;python_version>"3.4" starlette;python_version>"3.4" - httptools>"3.4" + httptools;python_version>"3.4" commands = nosetests -vv --nologcapture --with-doctest --with-coverage --cover-package=autover --ignore-files=test_asgi_server.py [testenv:lint_checks] From b077c1405370593830af4baea7b356b2cdc6366a Mon Sep 17 00:00:00 2001 From: jlstevens Date: Fri, 5 Feb 2021 15:00:03 +0000 Subject: [PATCH 09/15] Specifying setuptools in tox.ini --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 91afa017..24bc7d80 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ deps = uvloop;python_version>"3.4" starlette;python_version>"3.4" httptools;python_version>"3.4" + setuptools>=40.8.0 commands = nosetests -vv --nologcapture --with-doctest --with-coverage --cover-package=autover --ignore-files=test_asgi_server.py [testenv:lint_checks] From c40fb5a414e9ad0ebe0af6af5600e10e3216d275 Mon Sep 17 00:00:00 2001 From: jlstevens Date: Fri, 5 Feb 2021 18:00:18 +0000 Subject: [PATCH 10/15] Removed setuptools listing --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 24bc7d80..91afa017 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,6 @@ deps = uvloop;python_version>"3.4" starlette;python_version>"3.4" httptools;python_version>"3.4" - setuptools>=40.8.0 commands = nosetests -vv --nologcapture --with-doctest --with-coverage --cover-package=autover --ignore-files=test_asgi_server.py [testenv:lint_checks] From faef79dcdb294a2d180e45184b2bb636b3ea8c25 Mon Sep 17 00:00:00 2001 From: jlstevens Date: Tue, 9 Feb 2021 10:32:47 +0000 Subject: [PATCH 11/15] Added explanatory comment --- examples/pkg_bundle/version.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/pkg_bundle/version.py b/examples/pkg_bundle/version.py index a2b43002..2fcf20dc 100644 --- a/examples/pkg_bundle/version.py +++ b/examples/pkg_bundle/version.py @@ -117,6 +117,10 @@ def run_cmd(args, cwd=None): cwd=cwd) output, error = (str(s.decode()).strip() for s in proc.communicate()) + # Checking length of stderr as subprocess.Popen() is not always + # async-friendly when spawned from an os fork instruction. This + # means it can misreport the return code as zero when there is an + # error. if proc.returncode != 0 or len(error) > 0: raise Exception(proc.returncode, error) return output From 66f15d26c46dbe183f7713a6ab88cb9e8099e7b8 Mon Sep 17 00:00:00 2001 From: Jean-Luc Stevens Date: Tue, 9 Feb 2021 07:42:52 -0800 Subject: [PATCH 12/15] Updated explanatory comment Co-authored-by: James A. Bednar --- examples/pkg_bundle/version.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/pkg_bundle/version.py b/examples/pkg_bundle/version.py index 2fcf20dc..934a9a0e 100644 --- a/examples/pkg_bundle/version.py +++ b/examples/pkg_bundle/version.py @@ -117,10 +117,9 @@ def run_cmd(args, cwd=None): cwd=cwd) output, error = (str(s.decode()).strip() for s in proc.communicate()) - # Checking length of stderr as subprocess.Popen() is not always - # async-friendly when spawned from an os fork instruction. This - # means it can misreport the return code as zero when there is an - # error. + # Detects errors as _either_ a non-zero return code _or_ messages + # printed to stderr, because the return code is erroneously fixed at + # zero in some cases (see https://github.com/holoviz/param/pull/389). if proc.returncode != 0 or len(error) > 0: raise Exception(proc.returncode, error) return output From 3456c74ab2d3ae8f74f8c9bdf06057e72386996b Mon Sep 17 00:00:00 2001 From: jlstevens Date: Tue, 9 Feb 2021 18:08:13 +0000 Subject: [PATCH 13/15] Removed tests --- conda.recipe/meta.yaml | 1 - tests/test_asgi_server.py | 24 ------------------------ tests/testversion.py | 34 +--------------------------------- tox.ini | 12 +++--------- 4 files changed, 4 insertions(+), 67 deletions(-) delete mode 100644 tests/test_asgi_server.py diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 8323f8cf..9ceba99b 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -26,7 +26,6 @@ test: source_files: - tests commands: - - conda install -y -c conda-forge gunicorn uvicorn starlette uvloop - nosetests -vv --nologcapture about: diff --git a/tests/test_asgi_server.py b/tests/test_asgi_server.py deleted file mode 100644 index 0c8599fa..00000000 --- a/tests/test_asgi_server.py +++ /dev/null @@ -1,24 +0,0 @@ -import autover -from starlette.applications import Starlette - -# A simple ASGI server application -# that tries to make use of autover. -# -# Given that autover makes use of -# subprocess.Popen and the program -# workflow depends on that, there -# were some corner cases where -# autover would just crash if we -# tried to get its __version__, -# therefore crashing the whole -# ASGI application. -# -# This test application is used -# with Gunicorn in the unit tests -# (testversion.py), just to check -# if the ASGI application crashes -# or not. - -_ = autover.__version__ - -app = Starlette() diff --git a/tests/testversion.py b/tests/testversion.py index 243ca123..20375dd3 100644 --- a/tests/testversion.py +++ b/tests/testversion.py @@ -1,11 +1,7 @@ """ Unit test for autover.Version """ -import os import unittest -import subprocess -import sys -import time import pkg_resources._vendor.packaging.version as packaging_version from autover import Version @@ -79,7 +75,6 @@ def pep440(version): class TestVersion(unittest.TestCase): - maxDiff=None def git_describe_check(self, describe_tests, index): @@ -165,33 +160,6 @@ def test_git_describe_4(self): def test_git_describe_5(self): self.git_describe_check(describe_tests, 5) - #=======================================# - # Compatibility with ASGI applications # - #=======================================# - - def test_run_asgi_server(self): - """test_run_asgi_server (An ASGI app that uses autover and is launched with Gunicorn should not crash)""" - if sys.version[0] == "3": - serverfile_dir = os.path.abspath(os.path.join(__file__, os.pardir)) - logfile_path = os.path.abspath(os.path.join(__file__, os.pardir, "stderr_gunicorn.log")) - subprocess.Popen(["gunicorn", "-D", "--log-level", "ERROR", - "--chdir", serverfile_dir, - "-k", "uvicorn.workers.UvicornWorker", - "--error-logfile", logfile_path, - "--bind", "unix:/tmp/gunicorn.sock", - "test_asgi_server:app"]) - time.sleep(5) - with open(logfile_path, "r") as gunicorn_log_file: - stderr_log = gunicorn_log_file.read() - - subprocess.Popen(["pkill", "-f", "gunicorn"]) - time.sleep(5) - os.remove(logfile_path) - else: - stderr_log = "" - print(stderr_log) - self.assertEqual(stderr_log, "") - if __name__ == "__main__": import nose - nose.runmodule() + nose.runmodule() \ No newline at end of file diff --git a/tox.ini b/tox.ini index 91afa017..4113b05e 100644 --- a/tox.ini +++ b/tox.ini @@ -3,16 +3,10 @@ envlist = py36,py27 [testenv] passenv = GIT_VERSION -deps = - nose - gunicorn;python_version>"3.4" - uvicorn;python_version>"3.4" - uvloop;python_version>"3.4" - starlette;python_version>"3.4" - httptools;python_version>"3.4" -commands = nosetests -vv --nologcapture --with-doctest --with-coverage --cover-package=autover --ignore-files=test_asgi_server.py +deps = nose +commands = nosetests -vv --nologcapture --with-doctest --with-coverage --cover-package=autover [testenv:lint_checks] deps = flake8 skip_install = true -commands = flake8 setup.py autover tests examples +commands = flake8 setup.py autover tests examples \ No newline at end of file From 40cd887eb045e8cc33b579c83785805b637f58fe Mon Sep 17 00:00:00 2001 From: jlstevens Date: Tue, 9 Feb 2021 18:09:46 +0000 Subject: [PATCH 14/15] Whitespace fix --- tests/testversion.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testversion.py b/tests/testversion.py index 20375dd3..8af07e68 100644 --- a/tests/testversion.py +++ b/tests/testversion.py @@ -162,4 +162,4 @@ def test_git_describe_5(self): if __name__ == "__main__": import nose - nose.runmodule() \ No newline at end of file + nose.runmodule() diff --git a/tox.ini b/tox.ini index 4113b05e..1d98793e 100644 --- a/tox.ini +++ b/tox.ini @@ -9,4 +9,4 @@ commands = nosetests -vv --nologcapture --with-doctest --with-coverage --cover-p [testenv:lint_checks] deps = flake8 skip_install = true -commands = flake8 setup.py autover tests examples \ No newline at end of file +commands = flake8 setup.py autover tests examples From 5b8a1cf30a631ebc64614f3089e23557a5e3a451 Mon Sep 17 00:00:00 2001 From: jlstevens Date: Tue, 9 Feb 2021 18:11:38 +0000 Subject: [PATCH 15/15] Synchronized the different version.py files --- autover/version.py | 3 +++ examples/PkgBundle/version.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/autover/version.py b/autover/version.py index a2b43002..934a9a0e 100644 --- a/autover/version.py +++ b/autover/version.py @@ -117,6 +117,9 @@ def run_cmd(args, cwd=None): cwd=cwd) output, error = (str(s.decode()).strip() for s in proc.communicate()) + # Detects errors as _either_ a non-zero return code _or_ messages + # printed to stderr, because the return code is erroneously fixed at + # zero in some cases (see https://github.com/holoviz/param/pull/389). if proc.returncode != 0 or len(error) > 0: raise Exception(proc.returncode, error) return output diff --git a/examples/PkgBundle/version.py b/examples/PkgBundle/version.py index a2b43002..934a9a0e 100644 --- a/examples/PkgBundle/version.py +++ b/examples/PkgBundle/version.py @@ -117,6 +117,9 @@ def run_cmd(args, cwd=None): cwd=cwd) output, error = (str(s.decode()).strip() for s in proc.communicate()) + # Detects errors as _either_ a non-zero return code _or_ messages + # printed to stderr, because the return code is erroneously fixed at + # zero in some cases (see https://github.com/holoviz/param/pull/389). if proc.returncode != 0 or len(error) > 0: raise Exception(proc.returncode, error) return output