From c8193e901676a9a42f22709c281e9e13924b65df Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Wed, 14 May 2025 09:19:10 -0700 Subject: [PATCH 01/16] Upgrade paasta to python 3.10 --- Makefile | 2 +- mypy.ini | 2 +- requirements-bootstrap.txt | 2 +- requirements-dev.txt | 2 +- requirements-minimal.txt | 12 +++++++---- requirements.txt | 27 +++++++++++++----------- tests/test_task_processing.py | 21 ------------------ tox.ini | 20 +++++++++--------- yelp_package/extra_requirements_yelp.txt | 2 +- 9 files changed, 38 insertions(+), 52 deletions(-) delete mode 100644 tests/test_task_processing.py diff --git a/Makefile b/Makefile index 02fc1c437b..ed6fda6561 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ dev-api: .tox/py38-linux .paasta/bin/tox -e dev-api .paasta/bin/activate: requirements.txt requirements-dev.txt - test -d .paasta/bin/activate || virtualenv -p python3.8 .paasta + test -d .paasta/bin/activate || virtualenv -p python3.10 .paasta .paasta/bin/pip install -r requirements-bootstrap.txt .paasta/bin/pip install -U tox==3.28.0 touch .paasta/bin/activate diff --git a/mypy.ini b/mypy.ini index a39d3f6434..17210c509b 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,5 +1,5 @@ [mypy] -python_version = 3.8 +python_version = 3.10 check_untyped_defs = False warn_incomplete_stub = True follow_imports = silent diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt index 757a8c7af7..bccb878c3c 100644 --- a/requirements-bootstrap.txt +++ b/requirements-bootstrap.txt @@ -1,3 +1,3 @@ pip==24.0 -setuptools==68.0.0 +setuptools==69.0.3 wheel==0.42.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index dbe193a3be..164867afaf 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -28,7 +28,7 @@ pep8==1.5.7 pluggy==1.2.0 pre-commit==2.17.0 pycodestyle==2.3.1 -pyfakefs==4.1.0 +pyfakefs==5.3.5 pyflakes==1.6.0 Pygments==2.7.4 pylint==2.6.0 diff --git a/requirements-minimal.txt b/requirements-minimal.txt index 43353c9469..639163ec87 100644 --- a/requirements-minimal.txt +++ b/requirements-minimal.txt @@ -37,11 +37,12 @@ nats-py nulltype objgraph ply -progressbar2 >= 3.10.0 +progressbar2>=4.3.2 prometheus-client pymesos >= 0.2.0 -pyramid >= 1.8 pyramid-swagger >= 2.3.0 +pyramid>=2.0.2 +pyrsistent>=0.18.1 pysensu-yelp >= 0.3.4 PyStaticConfiguration python-crontab>=2.1.1 @@ -49,23 +50,26 @@ python-dateutil >= 2.4.0 python-iptables pytimeparse >= 1.1.0 pytz >= 2014.10 +repoze.lru>=0.7 # the upper-bound is required for older docker-py (<=7.0.0) # to function: https://github.com/psf/requests/issues/6707 requests >= 2.18.4, <2.32.0 requests-cache >= 0.4.10 retry ruamel.yaml +ruamel.yaml.clib>=0.2.8 sensu-plugin service-configuration-lib >= 3.2.0 signalfx slackclient >= 1.2.1 sticht >= 1.1.0 syslogmp -task-processing -thriftpy2 +thriftpy2>=0.5.2 transitions typing-extensions tzlocal urllib3 utaw >= 0.2.0 +WebOb>=1.8.7 wsgicors +zope.interface>=7.2 diff --git a/requirements.txt b/requirements.txt index ac0aa9bd41..dddcaaba28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,8 @@ click==6.6 containerd==1.5.3 cookiecutter==1.4.0 croniter==1.3.4 + +cython==3.1.0 decorator==4.1.2 docker==5.0.3 dulwich==0.17.3 @@ -33,7 +35,7 @@ gunicorn==19.8.1 http-parser==0.9.0 humanfriendly==4.18 humanize==0.5.1 -hupper==1.0 +hupper==1.12.1 idna==2.6 inotify==0.2.8 ipaddress==1.0.22 @@ -58,9 +60,11 @@ nulltype==2.3.1 oauthlib==3.1.0 objgraph==3.4.0 PasteDeploy==1.5.2 +plaster==1.1.2 +plaster-pastedeploy==0.5 ply==3.4 poyo==0.4.0 -progressbar2==3.10.1 +progressbar2==4.3.2 prometheus-client==0.7.1 protobuf==3.15.0 py==1.5.2 @@ -69,17 +73,17 @@ pyasn1-modules==0.2.1 pyformance==0.4 pyinotify==0.9.6 pymesos==0.3.9 -pyramid==1.8.4 +pyramid==2.0.2 pyramid-swagger==2.4.1 -pyrsistent==0.13.0 +pyrsistent==0.18.1 pysensu-yelp==0.4.1 PyStaticConfiguration==0.10.3 python-crontab==2.1.1 python-dateutil==2.8.1 python-iptables==1.0.1 -python-utils==2.0.1 +python-utils==3.8.2 pytimeparse==1.1.5 -pytz==2016.10 +pytz==2024.1 pyyaml==6.0.1 repoze.lru==0.7 requests==2.25.0 @@ -93,7 +97,7 @@ ruamel.yaml.clib==0.2.8 s3transfer==0.10.0 sensu-plugin==0.3.1 service-configuration-lib==3.2.0 -setuptools==39.0.1 +setuptools==69.0.3 signalfx==1.0.17 simplejson==3.10.0 six==1.15.0 @@ -101,12 +105,11 @@ slackclient==1.2.1 splunk-sdk==1.7.0 sseclient-py==1.7 # make sure you bump sticht in extra_requirements_yelp.txt as well -sticht==1.2.0 +sticht==1.2.1 strict-rfc3339==0.7 swagger-spec-validator==2.1.0 syslogmp==0.2.2 -task-processing==0.1.3 -thriftpy2==0.4.11 +thriftpy2==0.5.2 transitions==0.6.9 translationstring==1.3 typing-extensions==4.3.0 @@ -116,11 +119,11 @@ urllib3==1.26.18 utaw==0.2.0 venusian==1.1.0 webcolors==1.7 -WebOb==1.7.4 +WebOb==1.8.7 websocket-client==0.44.0 whichcraft==0.4.0 ws4py==0.5.1 wsgicors==0.7.0 yarl==1.1.1 zope.deprecation==4.2.0 -zope.interface==4.7.2 +zope.interface==7.2 diff --git a/tests/test_task_processing.py b/tests/test_task_processing.py deleted file mode 100644 index d3b8bde476..0000000000 --- a/tests/test_task_processing.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2015-2017 Yelp Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# We just want to test that task_processing is available in the virtualenv - - -def test_import(): - from task_processing.task_processor import TaskProcessor - - tp = TaskProcessor() - tp.load_plugin("task_processing.plugins.mesos") diff --git a/tox.ini b/tox.ini index 52399ba10d..cf9495836f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,13 @@ [tox] skipsdist=True -envlist=py38-linux +envlist=py310-linux docker_compose_version = 1.26.2 requires = tox==3.28.0 [testenv] -basepython = python3.8 +basepython = python3.10 passenv = SSH_AUTH_SOCK PAASTA_ENV DOCKER_HOST CI deps = --only-binary=grpcio @@ -23,7 +23,7 @@ commands = -pip install -r yelp_package/extra_requirements_yelp.txt [testenv:dev-api] -envdir = .tox/py38-linux/ +envdir = .tox/py310-linux/ passenv = PAASTA_TEST_CLUSTER KUBECONFIG PAASTA_SYSTEM_CONFIG_DIR KUBECONTEXT AWS_PROFILE deps = --only-binary=grpcio @@ -35,7 +35,7 @@ commands = python -m paasta_tools.run-paasta-api-in-dev-mode [testenv:playground-api] -envdir = .tox/py38-linux/ +envdir = .tox/py310-linux/ passenv = PAASTA_TEST_CLUSTER KUBECONFIG PAASTA_SYSTEM_CONFIG_DIR PAASTA_API_SOA_DIR USER setenv = KUBECONFIG = ./k8s_itests/kubeconfig @@ -52,7 +52,7 @@ commands = python -m paasta_tools.run-paasta-api-playground [testenv:tests] -envdir = .tox/py38-linux/ +envdir = .tox/py310-linux/ commands = check-requirements -vv pre-commit install -f --install-hooks @@ -62,7 +62,7 @@ commands = coverage report -m [testenv:tests-yelpy] -envdir = .tox/py38-linux/ +envdir = .tox/py310-linux/ setenv = PIP_INDEX_URL = http://169.254.255.254:20641/simple/ deps = @@ -88,7 +88,7 @@ commands = sphinx-build -j auto -b html -d docs/build/doctrees docs/source docs/build/html [testenv:k8s_itests] -basepython = python3.8 +basepython = python3.10 whitelist_externals = bash # one day we'll use a fully pinned venv here... deps = @@ -142,7 +142,7 @@ commands = docker-compose rm --force [testenv:general_itests] -basepython = python3.8 +basepython = python3.10 setenv = PAASTA_SYSTEM_CONFIG_DIR = {toxinidir}/general_itests/fake_etc_paasta changedir=general_itests/ @@ -157,7 +157,7 @@ commands = behave {posargs} [testenv:mypy] -basepython = python3.8 +basepython = python3.10 deps = {[testenv]deps} mypy==0.720 @@ -181,7 +181,7 @@ commands = mypy {posargs:{[testenv:mypy]mypy_paths}} [testenv:install-hooks] -basepython = python3.8 +basepython = python3.10 deps = pre-commit commands = pre-commit install -f --install-hooks diff --git a/yelp_package/extra_requirements_yelp.txt b/yelp_package/extra_requirements_yelp.txt index 05cdbdbbf9..3a6899ea8c 100644 --- a/yelp_package/extra_requirements_yelp.txt +++ b/yelp_package/extra_requirements_yelp.txt @@ -31,7 +31,7 @@ slo-transcoder==3.3.0 smmap2==2.0.3 # slo-transcoder, vault-tools dependency splunk-sdk==1.7.0 # sticht dependency srv-configs==1.1.0 # yelp-profiling dependency -sticht[yelp_internal]==1.2.0 +sticht[yelp_internal]==1.2.1 tenacity==8.3.0 # yelp-clog dependency vault-tools==0.9.2 yelp-cgeom==1.3.1 # scribereader dependency From 2db9b0057defc1aab179785584fbd0bd39772927 Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Wed, 14 May 2025 09:56:17 -0700 Subject: [PATCH 02/16] upgrade precommit config file to 3.10 --- .pre-commit-config.yaml | 4 ++-- requirements-dev-minimal.txt | 1 + requirements-dev.txt | 10 ++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8923dc8192..fba5ba0dde 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ --- default_language_version: - python: python3.8 + python: python3.10 repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.1.0 @@ -42,7 +42,7 @@ repos: rev: 22.3.0 hooks: - id: black - args: ['--target-version', 'py38'] + args: ['--target-version', 'py310'] exclude: ^paasta_tools/paastaapi - repo: https://github.com/PyCQA/flake8 rev: 3.7.7 diff --git a/requirements-dev-minimal.txt b/requirements-dev-minimal.txt index 1c46473aa1..25dce143ad 100644 --- a/requirements-dev-minimal.txt +++ b/requirements-dev-minimal.txt @@ -1,3 +1,4 @@ +appdirs astroid asynctest coverage diff --git a/requirements-dev.txt b/requirements-dev.txt index 164867afaf..bccbc11efa 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,10 +6,10 @@ Babel==2.9.1 cfgv==2.0.1 coverage==6.5.0 debugpy==1.8.0 -distlib==0.3.4 +distlib==0.3.9 docutils==0.12 exceptiongroup==1.1.2 -filelock==3.0.12 +filelock==3.13.1 flake8==3.5.0 freezegun==0.3.7 hypothesis==6.79.4 @@ -25,8 +25,10 @@ packaging==20.9 path.py==8.1 pbr==3.1.1 pep8==1.5.7 + +platformdirs==4.3.8 pluggy==1.2.0 -pre-commit==2.17.0 +pre-commit==3.7.0 pycodestyle==2.3.1 pyfakefs==5.3.5 pyflakes==1.6.0 @@ -48,6 +50,6 @@ sphinxcontrib-qthelp==1.0.2 sphinxcontrib-serializinghtml==1.1.3 toml==0.9.4 tomli==2.0.1 -virtualenv==20.4.3 +virtualenv==20.24.7 wrapt==1.12.1 zake==0.2.2 From 3677284fffb005628e73345a77e2beb7af122e40 Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Wed, 14 May 2025 10:00:19 -0700 Subject: [PATCH 03/16] Upgrade github workflows to 3.10 --- .github/workflows/ci.yml | 8 ++++---- .github/workflows/pypi.yml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7ab8e0f9b..140fe797fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: toxenv: - - py38-linux + - py310-linux - docs - mypy - tests @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.10 - run: python -m pip install --upgrade pip - run: pip install -r requirements-gha.txt - run: tox -e ${{ matrix.toxenv }} @@ -40,7 +40,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.10 - run: python -m pip install --upgrade pip virtualenv - run: curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ - run: python -m pip install --upgrade pip @@ -56,7 +56,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.10 - run: sudo apt-get update - run: sudo apt-get install -yq devscripts - run: make itest_${{ matrix.dist }} diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 8f718516a3..b4ed76fd87 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: toxenv: - - py38-linux,docs,mypy,tests + - py310-linux,docs,mypy,tests - general_itests env: DOCKER_REGISTRY: "" @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.10 - run: python -m pip install --upgrade pip - run: pip install -r requirements-gha.txt - run: tox -e ${{ matrix.toxenv }} @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.10 - run: pip install build # this will create both a source distribution and a wheel - run: python -m build From 7b094da4038c0a39cb7a1fab0a3f0821311d8419 Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Wed, 14 May 2025 10:06:56 -0700 Subject: [PATCH 04/16] references to python3.8 in the codebase has been upgraded to 3.10 --- .readthedocs.yaml | 2 +- .../contrib/bounce_log_latency_parser.py | 2 +- paasta_tools/contrib/ide_helper.py | 28 +++++++++---------- paasta_tools/contrib/mock_patch_checker.py | 2 +- paasta_tools/contrib/render_template.py | 2 +- paasta_tools/contrib/shared_ip_check.py | 2 +- paasta_tools/run-paasta-api-in-dev-mode.py | 4 +-- paasta_tools/run-paasta-api-playground.py | 4 +-- paasta_tools/yaml_tools.py | 2 +- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index dc3c22889d..b57419dbbd 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,7 +8,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.8" + python: "3.10" # You can also specify other tool versions: # nodejs: "20" # rust: "1.70" diff --git a/paasta_tools/contrib/bounce_log_latency_parser.py b/paasta_tools/contrib/bounce_log_latency_parser.py index 2b2383f230..c05850449b 100755 --- a/paasta_tools/contrib/bounce_log_latency_parser.py +++ b/paasta_tools/contrib/bounce_log_latency_parser.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.8 +#!/usr/bin/env python3.10 import itertools import json import sys diff --git a/paasta_tools/contrib/ide_helper.py b/paasta_tools/contrib/ide_helper.py index ef1f911ef7..376f9e0026 100644 --- a/paasta_tools/contrib/ide_helper.py +++ b/paasta_tools/contrib/ide_helper.py @@ -53,12 +53,12 @@ def install_vscode_support() -> None: "python": "${workspaceFolder}/.paasta/bin/python", "program": "${workspaceFolder}/.paasta/bin/tox", "subProcess": True, - "args": ["-e", "py38-linux,docs,mypy,tests"], + "args": ["-e", "py310-linux,docs,mypy,tests"], }, { "name": "paasta cli", "cwd": "${workspaceFolder}", - "python": "${workspaceFolder}/.tox/py38-linux/bin/python", + "python": "${workspaceFolder}/.tox/py310-linux/bin/python", "type": "python", "request": "launch", "module": "paasta_tools.cli.cli", @@ -66,7 +66,7 @@ def install_vscode_support() -> None: { "name": "paasta rollback", "cwd": "${workspaceFolder}", - "python": "${workspaceFolder}/.tox/py38-linux/bin/python", + "python": "${workspaceFolder}/.tox/py310-linux/bin/python", "type": "python", "request": "launch", "module": "paasta_tools.cli.cli", @@ -83,7 +83,7 @@ def install_vscode_support() -> None: { "name": "paasta mark-for-deployment", "cwd": "${workspaceFolder}", - "python": "${workspaceFolder}/.tox/py38-linux/bin/python", + "python": "${workspaceFolder}/.tox/py310-linux/bin/python", "type": "python", "request": "launch", "module": "paasta_tools.cli.cli", @@ -101,7 +101,7 @@ def install_vscode_support() -> None: { "name": "paasta status", "cwd": "${workspaceFolder}", - "python": "${workspaceFolder}/.tox/py38-linux/bin/python", + "python": "${workspaceFolder}/.tox/py310-linux/bin/python", "type": "python", "request": "launch", "module": "paasta_tools.cli.cli", @@ -118,7 +118,7 @@ def install_vscode_support() -> None: { "name": "paasta playground", "cwd": "${workspaceFolder}", - "python": "${workspaceFolder}/.tox/py38-linux/bin/python", + "python": "${workspaceFolder}/.tox/py310-linux/bin/python", "type": "python", "request": "launch", "module": "paasta_tools.cli.cli", @@ -138,7 +138,7 @@ def install_vscode_support() -> None: { "name": "paasta status playground", "cwd": "${workspaceFolder}", - "python": "${workspaceFolder}/.tox/py38-linux/bin/python", + "python": "${workspaceFolder}/.tox/py310-linux/bin/python", "type": "python", "request": "launch", "module": "paasta_tools.cli.cli", @@ -157,7 +157,7 @@ def install_vscode_support() -> None: { "name": "paasta logs", "cwd": "${workspaceFolder}", - "python": "${workspaceFolder}/.tox/py38-linux/bin/python", + "python": "${workspaceFolder}/.tox/py310-linux/bin/python", "type": "python", "request": "launch", "module": "paasta_tools.cli.cli", @@ -175,7 +175,7 @@ def install_vscode_support() -> None: "name": "paasta validate", # This command has to be ran from inside the service repo in yelpsoa-configs "cwd": "${userHome}/pg/yelpsoa-configs/", - "python": "${workspaceFolder}/.tox/py38-linux/bin/python", + "python": "${workspaceFolder}/.tox/py310-linux/bin/python", "type": "python", "request": "launch", "module": "paasta_tools.cli.cli", @@ -184,10 +184,10 @@ def install_vscode_support() -> None: { # 1) Follow step 1 in "Running the PaaSTA HTTP API Locally" wiki # 2) Run this "paasta API" test to debug paasta API - # 3) Run client command, e.g. PAASTA_SYSTEM_CONFIG_DIR=./etc_paasta_for_development/ .tox/py38-linux/bin/python paasta_tools/cli/cli.py status --clusters norcal-devc --service katamari_test_service + # 3) Run client command, e.g. PAASTA_SYSTEM_CONFIG_DIR=./etc_paasta_for_development/ .tox/py310-linux/bin/python paasta_tools/cli/cli.py status --clusters norcal-devc --service katamari_test_service "name": "paasta API", "cwd": "${workspaceFolder}", - "python": "${workspaceFolder}/.tox/py38-linux/bin/python", + "python": "${workspaceFolder}/.tox/py310-linux/bin/python", "type": "python", "request": "launch", "module": "paasta_tools.run-paasta-api-in-dev-mode", @@ -203,7 +203,7 @@ def install_vscode_support() -> None: { "name": "paasta API playground", "cwd": "${workspaceFolder}", - "python": "${workspaceFolder}/.tox/py38-linux/bin/python", + "python": "${workspaceFolder}/.tox/py310-linux/bin/python", "type": "python", "request": "launch", "module": "paasta_tools.run-paasta-api-playground", @@ -221,7 +221,7 @@ def install_vscode_support() -> None: { "name": "Run setup k8s job in playground", "cwd": "${workspaceFolder}", - "python": "${workspaceFolder}/.tox/py38-linux/bin/python", + "python": "${workspaceFolder}/.tox/py310-linux/bin/python", "type": "python", "request": "launch", "module": "paasta_tools.setup_kubernetes_job", @@ -244,7 +244,7 @@ def install_vscode_support() -> None: { "name": "Generate deployments.json in playground", "cwd": "${workspaceFolder}", - "python": "${workspaceFolder}/.tox/py38-linux/bin/python", + "python": "${workspaceFolder}/.tox/py310-linux/bin/python", "type": "python", "request": "launch", "module": "paasta_tools.generate_deployments_for_service", diff --git a/paasta_tools/contrib/mock_patch_checker.py b/paasta_tools/contrib/mock_patch_checker.py index 28e20ce835..91eee24d16 100755 --- a/paasta_tools/contrib/mock_patch_checker.py +++ b/paasta_tools/contrib/mock_patch_checker.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.8 +#!/usr/bin/env python3.10 import ast import sys diff --git a/paasta_tools/contrib/render_template.py b/paasta_tools/contrib/render_template.py index 419b3bf9ad..e2b53ef397 100755 --- a/paasta_tools/contrib/render_template.py +++ b/paasta_tools/contrib/render_template.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.8 +#!/usr/bin/env python3.10 import argparse import os import re diff --git a/paasta_tools/contrib/shared_ip_check.py b/paasta_tools/contrib/shared_ip_check.py index 06eb2820b1..f33ec8619e 100755 --- a/paasta_tools/contrib/shared_ip_check.py +++ b/paasta_tools/contrib/shared_ip_check.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.8 +#!/usr/bin/env python3.10 import sys from collections import defaultdict diff --git a/paasta_tools/run-paasta-api-in-dev-mode.py b/paasta_tools/run-paasta-api-in-dev-mode.py index 6d931d14e2..a21c580ce8 100644 --- a/paasta_tools/run-paasta-api-in-dev-mode.py +++ b/paasta_tools/run-paasta-api-in-dev-mode.py @@ -45,8 +45,8 @@ def main(): api.main("dev-mode") else: os.execl( - ".tox/py38-linux/bin/python", - ".tox/py38-linux/bin/python", + ".tox/py310-linux/bin/python", + ".tox/py310-linux/bin/python", "-m", "paasta_tools.api.api", *["-D", "-c", cluster, str(port)], diff --git a/paasta_tools/run-paasta-api-playground.py b/paasta_tools/run-paasta-api-playground.py index 408c6a66dc..c0abcae4b3 100644 --- a/paasta_tools/run-paasta-api-playground.py +++ b/paasta_tools/run-paasta-api-playground.py @@ -39,8 +39,8 @@ def main(): api.main("dev-mode") else: os.execl( - ".tox/py38-linux/bin/python", - ".tox/py38-linux/bin/python", + ".tox/py310-linux/bin/python", + ".tox/py310-linux/bin/python", "-m", "paasta_tools.api.api", *["-D", "-c", cluster, str(port)], diff --git a/paasta_tools/yaml_tools.py b/paasta_tools/yaml_tools.py index 86109bbbd2..b2e028291e 100644 --- a/paasta_tools/yaml_tools.py +++ b/paasta_tools/yaml_tools.py @@ -2,7 +2,7 @@ import yaml -# try and catch both /opt/venvs/paasta-tools and ~/pg/paasta/.tox/py38-linux as if we're being run as an application, +# try and catch both /opt/venvs/paasta-tools and ~/pg/paasta/.tox/py310-linux as if we're being run as an application, # we likely want to fail on a potential slowdown rather than experience a performance regression if "paasta" in sys.prefix: from yaml import CSafeLoader as Loader From 993b2d81a94cdafa14357ca62e40c9e872d174c7 Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Wed, 14 May 2025 10:13:18 -0700 Subject: [PATCH 05/16] Upgrade debian files --- debian/control | 4 ++-- debian/rules | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/debian/control b/debian/control index a1a5c7e3e7..659124b4d5 100644 --- a/debian/control +++ b/debian/control @@ -2,11 +2,11 @@ Source: paasta-tools Section: python Priority: optional Maintainer: Compute Infrastructure -Build-Depends: debhelper (>= 7), python3.8, dh-virtualenv +Build-Depends: debhelper (>= 7), python3.10, dh-virtualenv Standards-Version: 3.8.3 Package: paasta-tools Architecture: any -Depends: python3.8, ${shlibs:Depends}, ${misc:Depends} +Depends: python3.10, ${shlibs:Depends}, ${misc:Depends} Description: CLI tools for PaaSTA Conflicts: service-deployment-tools diff --git a/debian/rules b/debian/rules index 190d3f72d5..8c1fb52516 100755 --- a/debian/rules +++ b/debian/rules @@ -20,12 +20,12 @@ DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs DH_VENV_DIR=debian/$(PACKAGE)$(DH_VIRTUALENV_INSTALL_ROOT)/$(PACKAGE) override_dh_virtualenv: dh_virtualenv \ - --python=/usr/bin/python3.8 \ + --python=/usr/bin/python3.10 \ --preinstall no-manylinux1 \ --preinstall=-rrequirements-bootstrap.txt cp yelp_package/gopath/paasta_go $(DH_VENV_DIR)/bin/paasta_go @echo patching k8s client lib - patch $(DH_VENV_DIR)/lib/python3.8/site-packages/kubernetes/client/api_client.py contrib/python-k8s-client.diff + patch $(DH_VENV_DIR)/lib/python3.10/site-packages/kubernetes/client/api_client.py contrib/python-k8s-client.diff override_dh_shlibdeps: # pylibmc manylinux bundle libraries fail unless this is passed. From 3e661080b8a14984f0696989c69184ed3a467692 Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Wed, 14 May 2025 10:19:29 -0700 Subject: [PATCH 06/16] fix github workflow --- .github/workflows/ci.yml | 6 +++--- .github/workflows/pypi.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 140fe797fd..82ee817f64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: '3.10' - run: python -m pip install --upgrade pip - run: pip install -r requirements-gha.txt - run: tox -e ${{ matrix.toxenv }} @@ -40,7 +40,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: '3.10' - run: python -m pip install --upgrade pip virtualenv - run: curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ - run: python -m pip install --upgrade pip @@ -56,7 +56,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: '3.10' - run: sudo apt-get update - run: sudo apt-get install -yq devscripts - run: make itest_${{ matrix.dist }} diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index b4ed76fd87..26685cd4df 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: '3.10' - run: python -m pip install --upgrade pip - run: pip install -r requirements-gha.txt - run: tox -e ${{ matrix.toxenv }} @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: '3.10' - run: pip install build # this will create both a source distribution and a wheel - run: python -m build From a051163673facac76908f59537e63d5bed3f6836 Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Wed, 14 May 2025 12:29:09 -0700 Subject: [PATCH 07/16] lets see if this fixes k8s_itests --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index cf9495836f..567da263c4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] skipsdist=True envlist=py310-linux -docker_compose_version = 1.26.2 +docker_compose_version = 1.29.2 requires = tox==3.28.0 From 3047a1d7d9f1d96f16df5b6cea2c7e622c7896c4 Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Wed, 14 May 2025 12:37:02 -0700 Subject: [PATCH 08/16] use docker compose v2 --- tox.ini | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index 567da263c4..8931910b07 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] skipsdist=True envlist=py310-linux -docker_compose_version = 1.29.2 +docker_compose_version = 1.26.2 requires = tox==3.28.0 @@ -95,7 +95,7 @@ deps = urllib3<2.0 requests<2.32.0 # required for older docker-py (<=7.0.0) to function: https://github.com/psf/requests/issues/6707 cryptography<42 - docker-compose=={[tox]docker_compose_version} + # Requires system Docker Compose V2 (`docker compose` CLI) to be installed and available in PATH. setenv = passenv = KIND_CLUSTER @@ -113,12 +113,14 @@ passenv = INDEX_URL_BUILD_ARG changedir=k8s_itests/ commands = - # Build /etc/paasta used by docker-compose - {toxinidir}/k8s_itests/scripts/setup.sh + # Fail fast if Docker Compose V2 is not available + docker compose version + # Build /etc/paasta used by docker compose + {toxinidir}/k8s_itests/scripts/setup.sh # Run paasta-tools k8s_itests in docker - docker-compose down - docker-compose --verbose build --parallel --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg {env:INDEX_URL_BUILD_ARG:UNUSED}=https://pypi.org/simple - docker-compose up \ + docker compose down + docker compose --verbose build --parallel --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg {env:INDEX_URL_BUILD_ARG:UNUSED}=https://pypi.org/simple + docker compose up \ --abort-on-container-exit [testenv:example_cluster] From 1792ce932d3adb01f91db89937a8020270f39cfe Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Wed, 14 May 2025 13:48:39 -0700 Subject: [PATCH 09/16] upgrade dockerfile to py310 and fix the jammy test --- requirements-dev.txt | 1 - requirements-minimal.txt | 4 ++++ requirements.txt | 11 ++++++++--- yelp_package/dockerfiles/jammy/Dockerfile | 8 ++++---- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index bccbc11efa..e97365a9ec 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -32,7 +32,6 @@ pre-commit==3.7.0 pycodestyle==2.3.1 pyfakefs==5.3.5 pyflakes==1.6.0 -Pygments==2.7.4 pylint==2.6.0 pyparsing==2.4.7 pytest==7.4.0 diff --git a/requirements-minimal.txt b/requirements-minimal.txt index 639163ec87..f73a9a053b 100644 --- a/requirements-minimal.txt +++ b/requirements-minimal.txt @@ -14,6 +14,7 @@ croniter docker dulwich >= 0.17.3 ephemeral-port-reserve >= 1.0.1 +future>=0.18.3 graphviz grpcio gunicorn @@ -22,6 +23,7 @@ humanize >= 0.5.1 inotify >= 0.2.8 ipaddress >= 1.0.22 isodate >= 0.7.2 +jinja2-time>=0.1.0 jsonschema[format] kazoo >= 2.0.0 # the upper-bound here is mainly for things that use paasta-tools as a library and don't benefit @@ -37,6 +39,7 @@ nats-py nulltype objgraph ply +poyo>=0.4.0 progressbar2>=4.3.2 prometheus-client pymesos >= 0.2.0 @@ -71,5 +74,6 @@ tzlocal urllib3 utaw >= 0.2.0 WebOb>=1.8.7 +whichcraft>=0.4.0 wsgicors zope.interface>=7.2 diff --git a/requirements.txt b/requirements.txt index dddcaaba28..0a6dccc34b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,9 +17,10 @@ cachetools==5.5.0 certifi==2017.11.5 chardet==3.0.4 choice==0.1 -click==6.6 +click==8.1.7 +commonmark==0.9.1 containerd==1.5.3 -cookiecutter==1.4.0 +cookiecutter==2.6.0 croniter==1.3.4 cython==3.1.0 @@ -27,7 +28,7 @@ decorator==4.1.2 docker==5.0.3 dulwich==0.17.3 ephemeral-port-reserve==1.1.0 -future==0.16.0 +future==0.18.3 google-auth==1.2.0 graphviz==0.8.2 grpcio==1.62.2 @@ -71,6 +72,7 @@ py==1.5.2 pyasn1==0.4.2 pyasn1-modules==0.2.1 pyformance==0.4 +pygments==2.7.4 pyinotify==0.9.6 pymesos==0.3.9 pyramid==2.0.2 @@ -81,6 +83,7 @@ PyStaticConfiguration==0.10.3 python-crontab==2.1.1 python-dateutil==2.8.1 python-iptables==1.0.1 +python-slugify==4.0.1 python-utils==3.8.2 pytimeparse==1.1.5 pytz==2024.1 @@ -91,6 +94,7 @@ requests-cache==0.6.3 requests-oauthlib==1.2.0 retry==0.9.2 rfc3987==1.3.7 +rich==13.1.0 rsa==4.7.2 ruamel.yaml==0.16.12 ruamel.yaml.clib==0.2.8 @@ -109,6 +113,7 @@ sticht==1.2.1 strict-rfc3339==0.7 swagger-spec-validator==2.1.0 syslogmp==0.2.2 +text-unidecode==1.3 thriftpy2==0.5.2 transitions==0.6.9 translationstring==1.3 diff --git a/yelp_package/dockerfiles/jammy/Dockerfile b/yelp_package/dockerfiles/jammy/Dockerfile index 760a95fb63..45ecde8f1e 100644 --- a/yelp_package/dockerfiles/jammy/Dockerfile +++ b/yelp_package/dockerfiles/jammy/Dockerfile @@ -36,17 +36,17 @@ RUN apt-get update > /dev/null && \ libgpgme-dev \ libssl-dev \ libyaml-dev \ - python3.8-dev \ + python3.10-dev \ python3-pip \ - python3.8-distutils \ + python3.10-distutils \ tox \ wget \ sphinx-rtd-theme-common \ zsh > /dev/null \ && rm -rf /var/lib/apt/lists/* -RUN python3.8 -m pip install --upgrade pip==20.0.2 -RUN pip3.8 install virtualenv==16.0.0 +RUN python3.10 -m pip install --upgrade pip==20.0.2 +RUN pip3.10 install virtualenv==20.29.3 ADD mesos-slave-secret /etc/mesos-slave-secret From aaa598e490d88fc5036a9c6dc67b2938fc969d5d Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Thu, 15 May 2025 09:22:26 -0700 Subject: [PATCH 10/16] Update noble dockerfile and fix the test --- yelp_package/dockerfiles/noble/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yelp_package/dockerfiles/noble/Dockerfile b/yelp_package/dockerfiles/noble/Dockerfile index 1d7120564d..a19e8f7766 100644 --- a/yelp_package/dockerfiles/noble/Dockerfile +++ b/yelp_package/dockerfiles/noble/Dockerfile @@ -36,9 +36,9 @@ RUN apt-get update > /dev/null && \ libgpgme-dev \ libssl-dev \ libyaml-dev \ - python3.8-dev \ + python3.10-dev \ python3-pip \ - python3.8-distutils \ + python3.10-distutils \ tox \ wget \ sphinx-rtd-theme-common \ From 3e039c90c72a90513d01806dd4c1836736620503 Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Thu, 15 May 2025 09:24:32 -0700 Subject: [PATCH 11/16] Stop building for bionic as python3.10 is not supported on it --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82ee817f64..e115eda38b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: strategy: fail-fast: false matrix: - dist: [bionic, jammy, noble] + dist: [jammy, noble] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 From 23bdd0dd71b86f9997d301b359a085f2ad637ac2 Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Thu, 15 May 2025 10:36:38 -0700 Subject: [PATCH 12/16] upgrade mypy & fix the issues, upgrade api client patch --- contrib/python-k8s-client.diff | 4 ++-- paasta_tools/api/tweens/auth.py | 5 +++-- paasta_tools/cli/cmds/local_run.py | 4 ++-- paasta_tools/instance/kubernetes.py | 20 ++++++++++---------- paasta_tools/mesos/master.py | 2 +- paasta_tools/metrics/metastatus_lib.py | 2 +- requirements.txt | 2 +- tests/test_kubernetes_tools.py | 11 +++++++---- tests/test_setup_kubernetes_job.py | 4 ++-- tox.ini | 2 +- 10 files changed, 30 insertions(+), 26 deletions(-) diff --git a/contrib/python-k8s-client.diff b/contrib/python-k8s-client.diff index 1ee45ccca7..87aba8da1b 100644 --- a/contrib/python-k8s-client.diff +++ b/contrib/python-k8s-client.diff @@ -1,5 +1,5 @@ ---- a/debian/paasta/opt/venvs/paasta/lib/python3.8/site-packages/kubernetes/client/api_client.py -+++ b/debian/paasta/opt/venvs/paasta/lib/python3.8/site-packages/kubernetes/client/api_client.py +--- a/debian/paasta/opt/venvs/paasta/lib/python3.10/site-packages/kubernetes/client/api_client.py ++++ b/debian/paasta/opt/venvs/paasta/lib/python3.10/site-packages/kubernetes/client/api_client.py @@ -629,7 +629,7 @@ 'get_real_child_model'): return data diff --git a/paasta_tools/api/tweens/auth.py b/paasta_tools/api/tweens/auth.py index db2c7e76f8..af00531b56 100644 --- a/paasta_tools/api/tweens/auth.py +++ b/paasta_tools/api/tweens/auth.py @@ -69,7 +69,7 @@ def __call__(self, request: Request) -> Response: ) return self.handler(request) - @cachetools.func.ttl_cache(maxsize=AUTH_CACHE_SIZE, ttl=AUTH_CACHE_TTL) + @cachetools.func.ttl_cache(maxsize=AUTH_CACHE_SIZE, ttl=AUTH_CACHE_TTL) # type: ignore def is_request_authorized( self, path: str, @@ -77,7 +77,8 @@ def is_request_authorized( method: str, service: Optional[str], ) -> AuthorizationOutcome: - """Check if API request is authorized + """ + Check if API request is authorized :param str path: API path :param str token: authentication token diff --git a/paasta_tools/cli/cmds/local_run.py b/paasta_tools/cli/cmds/local_run.py index 9e1c0da721..927ad4757a 100755 --- a/paasta_tools/cli/cmds/local_run.py +++ b/paasta_tools/cli/cmds/local_run.py @@ -977,12 +977,12 @@ def run_docker_container( # First try to write the file as a string # This is for text like config files with open(temp_secret_filename, "w") as f: - f.write(secret_content) + f.write(secret_content) # type: ignore except TypeError: # If that fails, try to write it as bytes # This is for binary files like TLS keys with open(temp_secret_filename, "wb") as fb: - fb.write(secret_content) + fb.write(secret_content) # type: ignore # Append this to the list of volumes passed to docker run volumes.append(f"{temp_secret_filename}:{container_mount_path}:ro") diff --git a/paasta_tools/instance/kubernetes.py b/paasta_tools/instance/kubernetes.py index 7055dd2720..ee89f72117 100644 --- a/paasta_tools/instance/kubernetes.py +++ b/paasta_tools/instance/kubernetes.py @@ -641,7 +641,7 @@ async def kubernetes_status_v2( kube_client, job_config, job_config.get_kubernetes_namespace() ) ) - tasks.append(autoscaling_task) + tasks.append(autoscaling_task) # type: ignore else: autoscaling_task = None @@ -653,7 +653,7 @@ async def kubernetes_status_v2( namespaces=relevant_namespaces, ) ) - tasks.append(pods_task) + tasks.append(pods_task) # type: ignore service_namespace_config = kubernetes_tools.load_service_namespace_config( service=service, @@ -674,9 +674,9 @@ async def kubernetes_status_v2( ) ) backends_task = asyncio.create_task( - get_backends_from_mesh_status(mesh_status_task) + get_backends_from_mesh_status(mesh_status_task) # type: ignore ) - tasks.extend([mesh_status_task, backends_task]) + tasks.extend([mesh_status_task, backends_task]) # type: ignore else: mesh_status_task = None backends_task = None @@ -685,7 +685,7 @@ async def kubernetes_status_v2( pod_status_by_sha_and_readiness_task = asyncio.create_task( get_pod_status_tasks_by_sha_and_readiness( pods_task, - backends_task, + backends_task, # type: ignore kube_client, verbose, ) @@ -696,15 +696,15 @@ async def kubernetes_status_v2( service=service, instance=instance, namespaces=relevant_namespaces, - pod_status_by_sha_and_readiness_task=pod_status_by_sha_and_readiness_task, + pod_status_by_sha_and_readiness_task=pod_status_by_sha_and_readiness_task, # type: ignore ) ) - tasks.extend([pod_status_by_sha_and_readiness_task, versions_task]) + tasks.extend([pod_status_by_sha_and_readiness_task, versions_task]) # type: ignore else: pod_status_by_replicaset_task = asyncio.create_task( get_pod_status_tasks_by_replicaset( pods_task, - backends_task, + backends_task, # type: ignore kube_client, verbose, ) @@ -715,10 +715,10 @@ async def kubernetes_status_v2( service=service, instance=instance, namespaces=relevant_namespaces, - pod_status_by_replicaset_task=pod_status_by_replicaset_task, + pod_status_by_replicaset_task=pod_status_by_replicaset_task, # type: ignore ) ) - tasks.extend([pod_status_by_replicaset_task, versions_task]) + tasks.extend([pod_status_by_replicaset_task, versions_task]) # type: ignore await asyncio.gather(*tasks, return_exceptions=True) diff --git a/paasta_tools/mesos/master.py b/paasta_tools/mesos/master.py index 890a4842f1..13c3d66c52 100644 --- a/paasta_tools/mesos/master.py +++ b/paasta_tools/mesos/master.py @@ -147,7 +147,7 @@ async def post(self, url, **kwargs): def _file_resolver(self, cfg): return self.resolve(open(cfg[6:], "r+").read().strip()) - @retry(KazooTimeoutError, tries=5, delay=0.5, logger=logger) + @retry(KazooTimeoutError, tries=5, delay=0.5, logger=logger) # type: ignore def _zookeeper_resolver(self, cfg): hosts, path = cfg[5:].split("/", 1) path = "/" + path diff --git a/paasta_tools/metrics/metastatus_lib.py b/paasta_tools/metrics/metastatus_lib.py index 0a2dc96efe..9ab734f5a9 100755 --- a/paasta_tools/metrics/metastatus_lib.py +++ b/paasta_tools/metrics/metastatus_lib.py @@ -614,7 +614,7 @@ def group_slaves_by_key_func( """ sorted_slaves: Sequence[_GenericNodeT] if sort_func is None: - sorted_slaves = sorted(slaves, key=key_func) + sorted_slaves = sorted(slaves, key=key_func) # type: ignore else: sorted_slaves = sort_func(slaves) diff --git a/requirements.txt b/requirements.txt index 0a6dccc34b..74296db44c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,7 +55,7 @@ MarkupSafe==1.1.1 monotonic==1.4 msgpack-python==0.5.6 multidict==4.7.6 -mypy-extensions==0.4.1 +mypy-extensions==0.4.3 nats-py==2.8.0 nulltype==2.3.1 oauthlib==3.1.0 diff --git a/tests/test_kubernetes_tools.py b/tests/test_kubernetes_tools.py index 7b3691b756..bd918e6cc4 100644 --- a/tests/test_kubernetes_tools.py +++ b/tests/test_kubernetes_tools.py @@ -1933,9 +1933,7 @@ def mock_should_use_metrics_provider_fn(p: str) -> bool: return False mock_get_prometheus_port.return_value = prometheus_port - self.deployment.should_use_metrics_provider = ( - mock_should_use_metrics_provider_fn - ) + self.deployment.should_use_metrics_provider = mock_should_use_metrics_provider_fn # type: ignore mock_service_namespace_config = mock.Mock() mock_service_namespace_config.is_in_smartstack.return_value = in_smtstk mock_system_paasta_config = mock.Mock() @@ -2925,7 +2923,12 @@ def test_get_volume_claim_templates(self): assert len(ret) == 2 def test_get_storage_class_name_default(self): - pv = kubernetes_tools.PersistentVolume() + pv = kubernetes_tools.PersistentVolume( + mode="rw", + size=1000, + container_path="/tmp", + storage_class_name="ebs", + ) assert self.deployment.get_storage_class_name(pv) == "ebs" def test_get_storage_class_name_wrong(self): diff --git a/tests/test_setup_kubernetes_job.py b/tests/test_setup_kubernetes_job.py index a6f4c4188d..cfe3ffede4 100644 --- a/tests/test_setup_kubernetes_job.py +++ b/tests/test_setup_kubernetes_job.py @@ -460,7 +460,7 @@ def simple_create_application_object( branch_dict=None, soa_dir=soa_dir, ) - fake_app.__str__ = lambda app: "fake_app" + fake_app.__str__ = lambda app: "fake_app" # type: ignore return True, fake_app with mock.patch( @@ -910,7 +910,7 @@ def test_setup_kube_deployments_skip_malformed_apps( fake_app.create = mock.Mock( side_effect=[Exception("Kaboom!"), mock.Mock(create=mock.Mock())] ) - fake_app.__str__ = mock.Mock(return_value="fake_app") + fake_app.__str__ = mock.Mock(return_value="fake_app") # type: ignore mock_create_application_object.return_value = (True, fake_app) setup_kube_deployments( diff --git a/tox.ini b/tox.ini index 8931910b07..bf40d4d7bd 100644 --- a/tox.ini +++ b/tox.ini @@ -162,7 +162,7 @@ commands = basepython = python3.10 deps = {[testenv]deps} - mypy==0.720 + mypy==0.790 setenv = MYPYPATH = {toxinidir} ignore_errors = True From fd78847919bbc82169e4677af5835b57f88a6f7d Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Thu, 15 May 2025 13:53:25 -0700 Subject: [PATCH 13/16] attempt to fix general_itests --- requirements-bootstrap.txt | 2 +- requirements.txt | 2 +- tox.ini | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt index bccb878c3c..8aca0b0d20 100644 --- a/requirements-bootstrap.txt +++ b/requirements-bootstrap.txt @@ -1,3 +1,3 @@ pip==24.0 -setuptools==69.0.3 +setuptools==57.5.0 wheel==0.42.0 diff --git a/requirements.txt b/requirements.txt index 74296db44c..aa639a4843 100644 --- a/requirements.txt +++ b/requirements.txt @@ -101,7 +101,7 @@ ruamel.yaml.clib==0.2.8 s3transfer==0.10.0 sensu-plugin==0.3.1 service-configuration-lib==3.2.0 -setuptools==69.0.3 +setuptools==57.5.0 signalfx==1.0.17 simplejson==3.10.0 six==1.15.0 diff --git a/tox.ini b/tox.ini index bf40d4d7bd..f2671f9660 100644 --- a/tox.ini +++ b/tox.ini @@ -154,6 +154,8 @@ deps = behave==1.2.5 commands = # TODO: upgrade behave if they ever take this reasonable PR + # starting from setuptools >= 58.0.0, use_2to3 wont be supported + # as behave includes use_2to3 in setup.py, it will fail to install pip install git+https://github.com/Yelp/behave@1.2.5-issue_533-fork pylint -E {toxinidir}/paasta_tools/mesos/ --ignore master.py,task.py behave {posargs} From c511cd5e52244ab3f5463d5e7323be1fcfa18fa2 Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Thu, 15 May 2025 14:13:49 -0700 Subject: [PATCH 14/16] update dulwich --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aa639a4843..b41d9ba823 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ croniter==1.3.4 cython==3.1.0 decorator==4.1.2 docker==5.0.3 -dulwich==0.17.3 +dulwich==0.20.46 ephemeral-port-reserve==1.1.0 future==0.18.3 google-auth==1.2.0 From 210680542501097e5c16bd3f6b18ba2a95163524 Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Fri, 16 May 2025 13:25:34 -0700 Subject: [PATCH 15/16] our attempt to fix dulwich issue --- paasta_tools/remote_git.py | 4 ++-- requirements-bootstrap.txt | 2 +- requirements-dev.txt | 2 +- requirements.txt | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/paasta_tools/remote_git.py b/paasta_tools/remote_git.py index a27ee9ab2c..e8ebb89ee0 100644 --- a/paasta_tools/remote_git.py +++ b/paasta_tools/remote_git.py @@ -72,8 +72,8 @@ def create_remote_refs(git_url, ref_mutator, force=False): determine_wants = ref_mutator # We know we don't need to push any objects. - def generate_pack_contents(have, want): - return [] + def generate_pack_contents(have, want, ofs_delta=None): + return 0, [] return client.send_pack(path, determine_wants, generate_pack_contents) diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt index 8aca0b0d20..757a8c7af7 100644 --- a/requirements-bootstrap.txt +++ b/requirements-bootstrap.txt @@ -1,3 +1,3 @@ pip==24.0 -setuptools==57.5.0 +setuptools==68.0.0 wheel==0.42.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index e97365a9ec..14ec48ddc3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -33,7 +33,7 @@ pycodestyle==2.3.1 pyfakefs==5.3.5 pyflakes==1.6.0 pylint==2.6.0 -pyparsing==2.4.7 +pyparsing==3.1.2 pytest==7.4.0 pytest-asyncio==0.21.1 requirements-tools==1.1.2 diff --git a/requirements.txt b/requirements.txt index b41d9ba823..192cbf124b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ croniter==1.3.4 cython==3.1.0 decorator==4.1.2 docker==5.0.3 -dulwich==0.20.46 +dulwich==0.19.11 ephemeral-port-reserve==1.1.0 future==0.18.3 google-auth==1.2.0 @@ -101,7 +101,7 @@ ruamel.yaml.clib==0.2.8 s3transfer==0.10.0 sensu-plugin==0.3.1 service-configuration-lib==3.2.0 -setuptools==57.5.0 +setuptools==58.0.0 signalfx==1.0.17 simplejson==3.10.0 six==1.15.0 From 20b44ad3e607257a8585ad5f4e38b29db15f2f66 Mon Sep 17 00:00:00 2001 From: Eman Elsabban Date: Fri, 16 May 2025 13:32:33 -0700 Subject: [PATCH 16/16] Use luisp's behave branch --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f2671f9660..71121c5a51 100644 --- a/tox.ini +++ b/tox.ini @@ -156,7 +156,7 @@ commands = # TODO: upgrade behave if they ever take this reasonable PR # starting from setuptools >= 58.0.0, use_2to3 wont be supported # as behave includes use_2to3 in setup.py, it will fail to install - pip install git+https://github.com/Yelp/behave@1.2.5-issue_533-fork + pip install git+https://github.com/Yelp/behave@luisp/1.2.5-issue_533-fork-updates pylint -E {toxinidir}/paasta_tools/mesos/ --ignore master.py,task.py behave {posargs}