#6118: Bump minimum python version to 3.8 and Upgrade pantsbuild to 2.17

StackStorm · Jan 29, 2024 · f744a26 · f744a26
2 parents de8948b + 1df2823
commit f744a26
Show file tree

Hide file tree

Showing 51 changed files with 2,803 additions and 4,578 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -54,14 +54,6 @@ jobs:
         # NOTE: We need to use full Python version as part of Python deps cache key otherwise
         # setup virtualenv step will fail.
         include:
-          - name: 'Lint Checks (black, flake8, etc.)'
-            task: 'ci-checks'
-            python-version-short: '3.6'
-            python-version: '3.6.13'
-          - name: 'Compile (pip deps, pylint, etc.)'
-            task: 'ci-compile'
-            python-version-short: '3.6'
-            python-version: '3.6.13'
           - name: 'Lint Checks (black, flake8, etc.)'
             task: 'ci-checks'
             python-version-short: '3.8'
@@ -152,10 +144,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-         # TODO: Check if we want to fix the errors on Py 3.6 to have it tested as well
-         #- name: 'Self-check on Python 3.6'
-         #   python-version-short: '3.6'
-         #   python-version: '3.6.13'
           - name: 'Self-check on Python 3.8'
             python-version-short: '3.8'
             python-version: '3.8.14'
@@ -312,18 +300,6 @@ jobs:
         # NOTE: To speed the CI run, we split unit and integration tests into multiple jobs where
         # each job runs subset of tests.
         include:
-          - name: 'Unit Tests (chunk 1)'
-            task: 'ci-unit'
-            nosetests_node_total: 2
-            nosetests_node_index: 0
-            python-version-short: '3.6'
-            python-version: '3.6.13'
-          - name: 'Unit Tests (chunk 2)'
-            task: 'ci-unit'
-            nosetests_node_total: 2
-            nosetests_node_index: 1
-            python-version-short: '3.6'
-            python-version: '3.6.13'
           - name: 'Unit Tests (chunk 1)'
             task: 'ci-unit'
             nosetests_node_total: 2
@@ -499,24 +475,6 @@ jobs:
         include:
           # We run pack tests here since they rely on some integration tests set
           # up (aka stanley user being present, etc.)
-          - name: 'Pack Tests'
-            task: 'ci-packs-tests'
-            nosetests_node_total: 1
-            nosetests_node_index: 0
-            python-version-short: '3.6'
-            python-version: '3.6.13'
-          - name: 'Integration Tests (chunk 1)'
-            task: 'ci-integration'
-            nosetests_node_total: 2
-            nosetests_node_index: 0
-            python-version-short: '3.6'
-            python-version: '3.6.13'
-          - name: 'Integration Tests (chunk 2)'
-            task: 'ci-integration'
-            nosetests_node_total: 2
-            nosetests_node_index: 1
-            python-version-short: '3.6'
-            python-version: '3.6.13'
           - name: 'Pack Tests'
             task: 'ci-packs-tests'
             nosetests_node_total: 1

diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -57,7 +57,7 @@ jobs:
           ./scripts/github/install-apt-packages-use-cache.sh
 
       - name: Initialize Pants and its GHA caches
-        uses: pantsbuild/actions/init-pants@v2
+        uses: pantsbuild/actions/init-pants@v6-scie-pants
         # This action adds an env var to make pants use both pants.ci.toml & pants.toml.
         # This action also creates 3 GHA caches (1 is optional).
         # - `pants-setup` has the bootsrapped pants install
@@ -75,10 +75,12 @@ jobs:
           named-caches-hash: ${{ hashFiles('requirements.txt') }}
           # enable the optional lmdb_store cache since we're not using remote caching.
           cache-lmdb-store: 'true'
+          # install whatever version of python we need for our in-repo pants-plugins
+          setup-python-for-plugins: 'true'
 
       - name: Lint
         run: |
-          ./pants lint ::
+          pants lint ::
 
       - name: Upload pants log
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/microbenchmarks.yaml b/.github/workflows/microbenchmarks.yaml
@@ -34,12 +34,6 @@ jobs:
         # NOTE: We need to use full Python version as part of Python deps cache key otherwise
         # setup virtualenv step will fail.
         include:
-          - name: 'Microbenchmarks'
-            task: 'micro-benchmarks'
-            nosetests_node_total: 1
-            nosetests_node_index: 0
-            python-version-short: '3.6'
-            python-version: '3.6.13'
           - name: 'Microbenchmarks'
             task: 'micro-benchmarks'
             nosetests_node_total: 1

diff --git a/.github/workflows/orquesta-integration-tests.yaml b/.github/workflows/orquesta-integration-tests.yaml
@@ -55,12 +55,6 @@ jobs:
         # NOTE: We need to use full Python version as part of Python deps cache key otherwise
         # setup virtualenv step will fail.
         include:
-          - name: 'Integration Tests (Orquesta)'
-            task: 'ci-orquesta'
-            nosetests_node_total: 1
-            nosetests_node_index: 0
-            python-version: '3.6.13'
-            python-version-short: '3.6'
           - name: 'Integration Tests (Orquesta)'
             task: 'ci-orquesta'
             nosetests_node_total: 1

diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml
@@ -30,7 +30,7 @@ jobs:
           submodules: 'true'
 
       - name: Initialize Pants and its GHA caches
-        uses: pantsbuild/actions/init-pants@v2
+        uses: pantsbuild/actions/init-pants@v6-scie-pants
         # This action adds an env var to make pants use both pants.ci.toml & pants.toml.
         # This action also creates 3 GHA caches (1 is optional).
         # - `pants-setup` has the bootsrapped pants install
@@ -48,10 +48,12 @@ jobs:
           named-caches-hash: ${{ hashFiles('requirements.txt') }}
           # enable the optional lmdb_store cache since we're not using remote caching.
           cache-lmdb-store: 'true'
+          # install whatever version of python we need for our in-repo pants-plugins
+          setup-python-for-plugins: 'true'
 
       - name: Check BUILD files
         run: |
-          ./pants tailor --check update-build-files --check ::
+          pants tailor --check update-build-files --check ::
 
       - name: Upload pants log
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -32,9 +32,6 @@ jobs:
         # NOTE: We need to use full Python version as part of Python deps cache key otherwise
         # setup virtualenv step will fail.
         include:
-          - name: 'Test (pants runs: pytest)'
-            python-version-short: '3.6'
-            python-version: '3.6.13'
           - name: 'Test (pants runs: pytest)'
             python-version-short: '3.8'
             python-version: '3.8.10'
@@ -105,7 +102,7 @@ jobs:
           ./scripts/github/install-apt-packages-use-cache.sh
 
       - name: Initialize Pants and its GHA caches
-        uses: pantsbuild/actions/init-pants@v2
+        uses: pantsbuild/actions/init-pants@v6-scie-pants
         # This action adds an env var to make pants use both pants.ci.toml & pants.toml.
         # This action also creates 3 GHA caches (1 is optional).
         # - `pants-setup` has the bootsrapped pants install
@@ -123,13 +120,15 @@ jobs:
           named-caches-hash: ${{ hashFiles('requirements.txt') }}
           # enable the optional lmdb_store cache since we're not using remote caching.
           cache-lmdb-store: 'true'
+          # install whatever version of python we need for our in-repo pants-plugins
+          setup-python-for-plugins: 'true'
 
       - name: Test
         # We do not support running pytest everywhere yet. When we do it will be simply:
-        #   ./pants test ::
+        #   pants test ::
         # Until then, we need to manually adjust this command line to test what we can.
         run: |
-          ./pants test pylint_plugins/:: pants-plugins/::
+          pants test pylint_plugins/:: pants-plugins/::
 
       - name: Upload pants log
         uses: actions/upload-artifact@v2

diff --git a/BUILD b/BUILD
@@ -47,3 +47,7 @@ file(
     name="license",
     source="LICENSE",
 )
+
+shell_sources(
+    name="root",
+)
diff --git a/BUILD.tools b/BUILD.tools
@@ -7,9 +7,6 @@ python_requirement(
         "bandit==1.7.0",
         "setuptools",
         "GitPython==3.1.18",
-        # bandit needs stevedore which needs importlib-metadata<5
-        # see: https://github.com/PyCQA/bandit/pull/952
-        "importlib-metadata<5;python_version<'3.8'",
     ],
 )
 
@@ -40,14 +37,18 @@ python_requirement(
     resolve="pytest",
     requirements=[
         "pytest==7.0.1",  # copied from https://www.pantsbuild.org/v2.14/docs/reference-pytest#version
-        "pytest-benchmark[histogram]==3.4.1",
-        # "pytest-timer[colorama]",
-        "pytest-icdiff",
-        "pygments",
-        # "pytest-timeout",
-        # "pytest-mock",
-        "pytest-cov>=2.12,!=2.12.1,<3.1",
-        "pytest-xdist>=2.5,<3",
+        "pytest-benchmark[histogram]==3.4.1",  # used for st2common/benchmarks
+        # "pytest-timer[colorama]",  # report test timing (--with-timer ala nose-timer)
+        "pytest-icdiff",  # make diff output easier to read
+        "pygments",  # highlight code in tracebacks
+        #
+        # other possible plugins
+        # "pytest-timeout",  # time limit on tests
+        # "pytest-mock",  # more convenient mocking
+        #
+        # needed by pants
+        "pytest-cov>=2.12,!=2.12.1,<3.1",  # coverage
+        "pytest-xdist>=2.5,<3",  # parallel test runs (pants uses this if [pytest].xdist_enabled)
     ],
 )
 

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -3,9 +3,24 @@ Changelog
 
 in development
 --------------
+
+Python 3.6 is no longer supported; Stackstorm requires at least Python 3.8.
+
 Fixed
 ~~~~~
 
+Changed
+~~~~~~~
+* Bumped `jsonschema` 2.6.0 -> 3.2.0 now that python3.6 is not supported. #6118
+
+Added
+~~~~~
+* Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)
+  working on StackStorm, improve our security posture, and improve CI reliability thanks in part
+  to pants' use of PEX lockfiles. This is not a user-facing addition.
+  #6118
+  Contributed by @cognifloyd
+
 3.8.1 - December 13, 2023
 -------------------------
 Fixed

diff --git a/Makefile b/Makefile
@@ -37,12 +37,11 @@ COMPONENTS_TEST_DIRS := $(wildcard st2*/tests) $(wildcard contrib/runners/*/test
 COMPONENT_SPECIFIC_TESTS := st2tests st2client.egg-info
 
 # nasty hack to get a space into a variable
+space_char := $(subst ,, )
 colon := :
 comma := ,
 dot := .
 slash := /
-space_char :=
-space_char +=
 COMPONENT_PYTHONPATH = $(subst $(space_char),:,$(realpath $(COMPONENTS_WITH_RUNNERS)))
 COMPONENTS_TEST := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS_WITH_RUNNERS)),$(component))
 COMPONENTS_TEST_COMMA := $(subst $(slash),$(dot),$(subst $(space_char),$(comma),$(COMPONENTS_TEST)))

diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample
@@ -84,7 +84,7 @@ token_ttl = 86400
 # Standalone mode options - options below only apply when auth service is running in the standalone
 # mode.
 
-# Authentication backend to use in a standalone mode. Available backends: ldap, flat_file.
+# Authentication backend to use in a standalone mode. Available backends: flat_file, ldap.
 backend = flat_file
 # JSON serialized arguments which are passed to the authentication backend in a standalone mode.
 backend_kwargs = None

diff --git a/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py b/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py
@@ -18,7 +18,6 @@
 from __future__ import absolute_import
 
 import mock
-import six
 
 from orquesta import statuses as wf_statuses
 
@@ -215,7 +214,7 @@ def assert_data_flow(self, data):
         self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED)
 
         # Check workflow output.
-        expected_value = wf_input["a1"] if six.PY3 else wf_input["a1"].decode("utf-8")
+        expected_value = wf_input["a1"]
 
         expected_output = {
             "a6": expected_value,

diff --git a/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py b/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py
@@ -16,7 +16,6 @@
 from __future__ import absolute_import
 
 import mock
-import six
 
 from orquesta import statuses as wf_statuses
 
@@ -333,10 +332,7 @@ def test_fail_start_task_input_expr_eval(self):
         self.assertDictEqual(ac_ex_db.result, expected_result)
 
     def test_fail_start_task_input_value_type(self):
-        if six.PY3:
-            msg = "Value \"{'x': 'foobar'}\" must either be a string or None. Got \"dict\"."
-        else:
-            msg = "Value \"{u'x': u'foobar'}\" must either be a string or None. Got \"dict\"."
+        msg = "Value \"{'x': 'foobar'}\" must either be a string or None. Got \"dict\"."
 
         msg = "ValueError: " + msg
 
@@ -488,10 +484,7 @@ def test_fail_next_task_input_expr_eval(self):
         self.assertDictEqual(ac_ex_db.result, expected_result)
 
     def test_fail_next_task_input_value_type(self):
-        if six.PY3:
-            msg = "Value \"{'x': 'foobar'}\" must either be a string or None. Got \"dict\"."
-        else:
-            msg = "Value \"{u'x': u'foobar'}\" must either be a string or None. Got \"dict\"."
+        msg = "Value \"{'x': 'foobar'}\" must either be a string or None. Got \"dict\"."
 
         msg = "ValueError: " + msg
 

diff --git a/fixed-requirements.txt b/fixed-requirements.txt
@@ -21,7 +21,7 @@ gitdb==4.0.2
 greenlet==1.0.0
 gunicorn==21.2.0
 jsonpath-rw==1.4.0
-jsonschema==2.6.0
+jsonschema==3.2.0
 kombu==5.0.2
 lockfile==0.12.2
 # Fix MarkupSafe to < 2.1.0 as 2.1.0 removes soft_unicode