#6118: Bump minimum python version to 3.8 and Upgrade pantsbuild to 2.17

Author: cognifloyd

.github/workflows/ci.yaml

@@ -54,14 +54,6 @@ jobs:
         # NOTE: We need to use full Python version as part of Python deps cache key otherwise
         # setup virtualenv step will fail.
         include:
-          - name: 'Lint Checks (black, flake8, etc.)'
-            task: 'ci-checks'
-            python-version-short: '3.6'
-            python-version: '3.6.13'
-          - name: 'Compile (pip deps, pylint, etc.)'
-            task: 'ci-compile'
-            python-version-short: '3.6'
-            python-version: '3.6.13'
           - name: 'Lint Checks (black, flake8, etc.)'
             task: 'ci-checks'
             python-version-short: '3.8'
@@ -152,10 +144,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-         # TODO: Check if we want to fix the errors on Py 3.6 to have it tested as well
-         #- name: 'Self-check on Python 3.6'
-         #   python-version-short: '3.6'
-         #   python-version: '3.6.13'
           - name: 'Self-check on Python 3.8'
             python-version-short: '3.8'
             python-version: '3.8.14'
@@ -312,18 +300,6 @@ jobs:
         # NOTE: To speed the CI run, we split unit and integration tests into multiple jobs where
         # each job runs subset of tests.
         include:
-          - name: 'Unit Tests (chunk 1)'
-            task: 'ci-unit'
-            nosetests_node_total: 2
-            nosetests_node_index: 0
-            python-version-short: '3.6'
-            python-version: '3.6.13'
-          - name: 'Unit Tests (chunk 2)'
-            task: 'ci-unit'
-            nosetests_node_total: 2
-            nosetests_node_index: 1
-            python-version-short: '3.6'
-            python-version: '3.6.13'
           - name: 'Unit Tests (chunk 1)'
             task: 'ci-unit'
             nosetests_node_total: 2
@@ -499,24 +475,6 @@ jobs:
         include:
           # We run pack tests here since they rely on some integration tests set
           # up (aka stanley user being present, etc.)
-          - name: 'Pack Tests'
-            task: 'ci-packs-tests'
-            nosetests_node_total: 1
-            nosetests_node_index: 0
-            python-version-short: '3.6'
-            python-version: '3.6.13'
-          - name: 'Integration Tests (chunk 1)'
-            task: 'ci-integration'
-            nosetests_node_total: 2
-            nosetests_node_index: 0
-            python-version-short: '3.6'
-            python-version: '3.6.13'
-          - name: 'Integration Tests (chunk 2)'
-            task: 'ci-integration'
-            nosetests_node_total: 2
-            nosetests_node_index: 1
-            python-version-short: '3.6'
-            python-version: '3.6.13'
           - name: 'Pack Tests'
             task: 'ci-packs-tests'
             nosetests_node_total: 1

.github/workflows/lint.yaml

@@ -57,7 +57,7 @@ jobs:
           ./scripts/github/install-apt-packages-use-cache.sh
 
       - name: Initialize Pants and its GHA caches
-        uses: pantsbuild/actions/init-pants@v2
+        uses: pantsbuild/actions/init-pants@v6-scie-pants
         # This action adds an env var to make pants use both pants.ci.toml & pants.toml.
         # This action also creates 3 GHA caches (1 is optional).
         # - `pants-setup` has the bootsrapped pants install
@@ -75,10 +75,12 @@ jobs:
           named-caches-hash: ${{ hashFiles('requirements.txt') }}
           # enable the optional lmdb_store cache since we're not using remote caching.
           cache-lmdb-store: 'true'
+          # install whatever version of python we need for our in-repo pants-plugins
+          setup-python-for-plugins: 'true'
 
       - name: Lint
         run: |
-          ./pants lint ::
+          pants lint ::
 
       - name: Upload pants log
         uses: actions/upload-artifact@v2

.github/workflows/microbenchmarks.yaml

@@ -34,12 +34,6 @@ jobs:
         # NOTE: We need to use full Python version as part of Python deps cache key otherwise
         # setup virtualenv step will fail.
         include:
-          - name: 'Microbenchmarks'
-            task: 'micro-benchmarks'
-            nosetests_node_total: 1
-            nosetests_node_index: 0
-            python-version-short: '3.6'
-            python-version: '3.6.13'
           - name: 'Microbenchmarks'
             task: 'micro-benchmarks'
             nosetests_node_total: 1

.github/workflows/orquesta-integration-tests.yaml

@@ -55,12 +55,6 @@ jobs:
         # NOTE: We need to use full Python version as part of Python deps cache key otherwise
         # setup virtualenv step will fail.
         include:
-          - name: 'Integration Tests (Orquesta)'
-            task: 'ci-orquesta'
-            nosetests_node_total: 1
-            nosetests_node_index: 0
-            python-version: '3.6.13'
-            python-version-short: '3.6'
           - name: 'Integration Tests (Orquesta)'
             task: 'ci-orquesta'
             nosetests_node_total: 1

.github/workflows/pants.yaml

@@ -30,7 +30,7 @@ jobs:
           submodules: 'true'
 
       - name: Initialize Pants and its GHA caches
-        uses: pantsbuild/actions/init-pants@v2
+        uses: pantsbuild/actions/init-pants@v6-scie-pants
         # This action adds an env var to make pants use both pants.ci.toml & pants.toml.
         # This action also creates 3 GHA caches (1 is optional).
         # - `pants-setup` has the bootsrapped pants install
@@ -48,10 +48,12 @@ jobs:
           named-caches-hash: ${{ hashFiles('requirements.txt') }}
           # enable the optional lmdb_store cache since we're not using remote caching.
           cache-lmdb-store: 'true'
+          # install whatever version of python we need for our in-repo pants-plugins
+          setup-python-for-plugins: 'true'
 
       - name: Check BUILD files
         run: |
-          ./pants tailor --check update-build-files --check ::
+          pants tailor --check update-build-files --check ::
 
       - name: Upload pants log
         uses: actions/upload-artifact@v2

.github/workflows/test.yaml

@@ -32,9 +32,6 @@ jobs:
         # NOTE: We need to use full Python version as part of Python deps cache key otherwise
         # setup virtualenv step will fail.
         include:
-          - name: 'Test (pants runs: pytest)'
-            python-version-short: '3.6'
-            python-version: '3.6.13'
           - name: 'Test (pants runs: pytest)'
             python-version-short: '3.8'
             python-version: '3.8.10'
@@ -105,7 +102,7 @@ jobs:
           ./scripts/github/install-apt-packages-use-cache.sh
 
       - name: Initialize Pants and its GHA caches
-        uses: pantsbuild/actions/init-pants@v2
+        uses: pantsbuild/actions/init-pants@v6-scie-pants
         # This action adds an env var to make pants use both pants.ci.toml & pants.toml.
         # This action also creates 3 GHA caches (1 is optional).
         # - `pants-setup` has the bootsrapped pants install
@@ -123,13 +120,15 @@ jobs:
           named-caches-hash: ${{ hashFiles('requirements.txt') }}
           # enable the optional lmdb_store cache since we're not using remote caching.
           cache-lmdb-store: 'true'
+          # install whatever version of python we need for our in-repo pants-plugins
+          setup-python-for-plugins: 'true'
 
       - name: Test
         # We do not support running pytest everywhere yet. When we do it will be simply:
-        #   ./pants test ::
+        #   pants test ::
         # Until then, we need to manually adjust this command line to test what we can.
         run: |
-          ./pants test pylint_plugins/:: pants-plugins/::
+          pants test pylint_plugins/:: pants-plugins/::
 
       - name: Upload pants log
         uses: actions/upload-artifact@v2

BUILD

@@ -47,3 +47,7 @@ file(
     name="license",
     source="LICENSE",
 )
+
+shell_sources(
+    name="root",
+)

BUILD.tools

@@ -7,9 +7,6 @@ python_requirement(
         "bandit==1.7.0",
         "setuptools",
         "GitPython==3.1.18",
-        # bandit needs stevedore which needs importlib-metadata<5
-        # see: https://github.com/PyCQA/bandit/pull/952
-        "importlib-metadata<5;python_version<'3.8'",
     ],
 )
 
@@ -40,14 +37,18 @@ python_requirement(
     resolve="pytest",
     requirements=[
         "pytest==7.0.1",  # copied from https://www.pantsbuild.org/v2.14/docs/reference-pytest#version
-        "pytest-benchmark[histogram]==3.4.1",
-        # "pytest-timer[colorama]",
-        "pytest-icdiff",
-        "pygments",
-        # "pytest-timeout",
-        # "pytest-mock",
-        "pytest-cov>=2.12,!=2.12.1,<3.1",
-        "pytest-xdist>=2.5,<3",
+        "pytest-benchmark[histogram]==3.4.1",  # used for st2common/benchmarks
+        # "pytest-timer[colorama]",  # report test timing (--with-timer ala nose-timer)
+        "pytest-icdiff",  # make diff output easier to read
+        "pygments",  # highlight code in tracebacks
+        #
+        # other possible plugins
+        # "pytest-timeout",  # time limit on tests
+        # "pytest-mock",  # more convenient mocking
+        #
+        # needed by pants
+        "pytest-cov>=2.12,!=2.12.1,<3.1",  # coverage
+        "pytest-xdist>=2.5,<3",  # parallel test runs (pants uses this if [pytest].xdist_enabled)
     ],
 )
 

CHANGELOG.rst

@@ -3,9 +3,24 @@ Changelog
 
 in development
 --------------
+
+Python 3.6 is no longer supported; Stackstorm requires at least Python 3.8.
+
 Fixed
 ~~~~~
 
+Changed
+~~~~~~~
+* Bumped `jsonschema` 2.6.0 -> 3.2.0 now that python3.6 is not supported. #6118
+
+Added
+~~~~~
+* Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)
+  working on StackStorm, improve our security posture, and improve CI reliability thanks in part
+  to pants' use of PEX lockfiles. This is not a user-facing addition.
+  #6118
+  Contributed by @cognifloyd
+
 3.8.1 - December 13, 2023
 -------------------------
 Fixed

Makefile

@@ -37,12 +37,11 @@ COMPONENTS_TEST_DIRS := $(wildcard st2*/tests) $(wildcard contrib/runners/*/test
 COMPONENT_SPECIFIC_TESTS := st2tests st2client.egg-info
 
 # nasty hack to get a space into a variable
+space_char := $(subst ,, )
 colon := :
 comma := ,
 dot := .
 slash := /
-space_char :=
-space_char +=
 COMPONENT_PYTHONPATH = $(subst $(space_char),:,$(realpath $(COMPONENTS_WITH_RUNNERS)))
 COMPONENTS_TEST := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS_WITH_RUNNERS)),$(component))
 COMPONENTS_TEST_COMMA := $(subst $(slash),$(dot),$(subst $(space_char),$(comma),$(COMPONENTS_TEST)))