From c57d9219cd70e189dbd7ce79c0b1c38ae8ee73d2 Mon Sep 17 00:00:00 2001 From: Ken Brewer Date: Fri, 27 Oct 2023 18:44:39 +0000 Subject: [PATCH] feat(ci): refactor actions and add linting - refactor: revise testing into unified integration pipeline - refactor: create re-usable setup-env action - feat: add pipeline support for poetry dynamic versioning - feat: add actionlint to pre-commit hooks. --- .github/actions/setup-env/action.yaml | 81 +++++++++++++++++++ .github/workflows/codecov.yml | 66 --------------- .github/workflows/integration-test.yml | 106 +++++++++++++++++++++++++ .github/workflows/pre-commit-ci.yml | 22 ----- .github/workflows/publish-to-pypi.yml | 20 ----- .github/workflows/pypi-release.yml | 45 +++++++++++ .github/workflows/pytest.yml | 65 --------------- .pre-commit-config.yaml | 4 + pycytominer/consensus.py | 2 +- 9 files changed, 237 insertions(+), 174 deletions(-) create mode 100644 .github/actions/setup-env/action.yaml delete mode 100644 .github/workflows/codecov.yml create mode 100644 .github/workflows/integration-test.yml delete mode 100644 .github/workflows/pre-commit-ci.yml delete mode 100644 .github/workflows/publish-to-pypi.yml create mode 100644 .github/workflows/pypi-release.yml delete mode 100644 .github/workflows/pytest.yml diff --git a/.github/actions/setup-env/action.yaml b/.github/actions/setup-env/action.yaml new file mode 100644 index 00000000..a79d8db7 --- /dev/null +++ b/.github/actions/setup-env/action.yaml @@ -0,0 +1,81 @@ +name: Setup Environment and Cache +description: | + A reusable composite action to setup the environment for a job. + This action will: + - Setup Python + - Cache the pre-commit installation + - Cache the virtual environment + - Setup poetry + - Install dependencies +inputs: + python-version: + description: The version of Python to use (passed to actions/setup-python) + required: true + cache-pre-commit: + description: Whether to cache the pre-commit installation + required: false + default: "true" + cache-venv: + description: Whether to cache the virtual environment + required: false + default: "true" + setup-poetry: + description: Whether to setup poetry + required: false + default: "true" + install-deps: + description: Whether to install dependencies + required: false + default: "true" +runs: + using: "composite" + steps: + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + - name: Generate Cache Key PY + # The cache key PY is a hash of the Python version and the + # output of `pip freeze` (the installed packages). + # This ensures that the cache is invalidated when the Python + # version or the default packages change. Both of the remaining + # cache keys depend on PY. + shell: bash + run: echo "PY=$((python -VV; pip freeze) | sha256sum | cut -d' ' -f1)" >> + $GITHUB_ENV + - name: Cache pre-commit installation + # The cache key for the pre-commit installation is a hash of the + # operating system, PY (see above), and the pre-commit config. + # This allows for caching the pre-commit hook installation across + # jobs and workflows. + if: ${{ inputs.cache-pre-commit == 'true' }} + uses: actions/cache@v3 + with: + path: | + .cache + pre-commit + key: cache|${{ runner.os }}|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} + - name: Cache venv + # The cache key for the virtual environment is a hash of the + # operating system, PY (see above), the pyproject.toml and + # the poetry.lock file. This allows for caching the virtual + # environment with all the dependencies. + if: ${{ inputs.cache-venv == 'true' }} + uses: actions/cache@v3 + with: + path: | + .cache + .venv + key: + cache|${{ runner.os }}|${{ env.PY }}|${{ hashFiles('pyproject.toml') }}|${{ + hashFiles('poetry.lock') }} + - name: Setup poetry and poetry-dynamic-versioning + shell: bash + if: ${{ inputs.setup-poetry == 'true' }} + run: | + python -m pip install poetry poetry-dynamic-versioning + - name: Install dependencies with poetry + shell: bash + if: ${{ inputs.install-deps == 'true' }} + run: | + poetry install --with dev,docs --all-extras -v --no-interaction diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml deleted file mode 100644 index 39abd87c..00000000 --- a/.github/workflows/codecov.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Code coverage - -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - run: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.11 - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - - name: Install library - run: poetry install --no-interaction - #---------------------------------------------- - # run test suite and output coverage file - #---------------------------------------------- - - name: Test with pytest - run: poetry run pytest --cov=./ --cov-report=xml - #---------------------------------------------- - # upload coverage stats - # (requires CODECOV_TOKEN in repository secrets) - #---------------------------------------------- - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 - with: - file: ./coverage.xml - files: ./coverage1.xml,./coverage2.xml - directory: ./coverage/reports/ - flags: unittests - name: codecov-umbrella - fail_ci_if_error: false - path_to_write_report: ./coverage/codecov_report.gz diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 00000000..27f4a367 --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,106 @@ +name: Integration Workflow + +on: + pull_request: + push: + branches: [main] + tags: ["*"] + workflow_dispatch: + inputs: + pytest_addopts: + description: Extra options for pytest; use -vv for full details; see + https://docs.pytest.org/en/latest/example/simple.html#how-to-change-command-line-options-defaults + required: false + default: "" + +env: + LANG: "en_US.utf-8" + LC_ALL: "en_US.utf-8" + PIP_CACHE_DIR: ${{ github.workspace }}/.cache/pip + POETRY_CACHE_DIR: ${{ github.workspace }}/.cache/pypoetry + POETRY_VIRTUALENVS_IN_PROJECT: "true" + PRE_COMMIT_HOME: ${{ github.workspace }}/.cache/pre-commit + PYTEST_ADDOPTS: ${{ github.event.inputs.pytest_addopts }} + PYTHONIOENCODING: "UTF-8" + TARGET_PYTHON_VERSION: "3.9" + +jobs: + quality-test: + # This job is used to run pre-commit checks to ensure that all files are + # are formatted correctly. + name: Pre-commit checks + # Runs pre-commit checks on all files + # This job doesn't fail fast to ensure that feedback on function is still provided + strategy: + fail-fast: false + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + # Full history likely required for planned commitizen checks + # See #345 + fetch-depth: 0 + - name: Setup python, and check pre-commit cache + uses: ./.github/actions/setup-env + with: + python-version: ${{ env.TARGET_PYTHON_VERSION }} + cache-pre-commit: true + cache-venv: false + setup-poetry: false + - name: Run pre-commit checks on all files + uses: pre-commit/action@v3.0.0 + with: + extra_args: --all-files + integration-test: + name: Pytest (Python ${{ matrix.python-version }} on ${{ matrix.os }}) + # Runs pytest on all tested versions of python and OSes + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - ubuntu-latest + python-version: ["3.8", "3.9", "3.10", "3.11"] + runs-on: ${{ matrix.os }} + env: + OS: ${{ matrix.os }} + # This is needed to avoid a warning from SQLAlchemy + # https://sqlalche.me/e/b8d9 + # We can remove this once we upgrade to SQLAlchemy >= 2.0 + SQLALCHEMY_SILENCE_UBER_WARNING: "1" + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup python, and load cache + uses: ./.github/actions/setup-env + with: + python-version: ${{ matrix.python-version }} + cache-pre-commit: false + cache-venv: true + setup-poetry: true + install-deps: true + - name: Run pytest and generate coverage report + # For the target version and ubuntu, run pytest and generate coverage report + if: (matrix.os == 'ubuntu-latest') && (matrix.python-version == env.TARGET_PYTHON_VERSION) + run: poetry run pytest --cov=./ --cov-report=xml ${{ github.event.inputs.pytest_addopts }} + - name: Upload coverage to Codecov + # For the target version and ubuntu, upload coverage to Codecov + continue-on-error: true + if: (matrix.os == 'ubuntu-latest') && (matrix.python-version == env.TARGET_PYTHON_VERSION ) + uses: codecov/codecov-action@v3 + env: + OS: ${{ matrix.os }} + PYTHON: ${{ matrix.python-version }} + with: + file: ./coverage.xml + files: ./coverage1.xml,./coverage2.xml + directory: ./coverage/reports/ + env_vars: OS,PYTHON + fail_ci_if_error: true + flags: unittests + name: pycytominer + - name: Run pytest + # For every other version and/or OS, run pytest without coverage + if: (matrix.os != 'ubuntu-latest') || (matrix.python-version != env.TARGET_PYTHON_VERSION ) + run: poetry run pytest ${{ github.event.inputs.pytest_addopts }} diff --git a/.github/workflows/pre-commit-ci.yml b/.github/workflows/pre-commit-ci.yml deleted file mode 100644 index 085db314..00000000 --- a/.github/workflows/pre-commit-ci.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: pre-commit-ci -on: - pull_request: - branches: - - main -jobs: - pre-commit-ci: - name: Run pre-commit checks on changed files - runs-on: ubuntu-latest - steps: - - name: Checkout repo with all branches and tags - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Setup python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Run pre-commit checks on any files changed between base_ref and HEAD - uses: pre-commit/action@v3.0.0 - with: - extra_args: --from-ref origin/${{github.base_ref}} --to-ref HEAD diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml deleted file mode 100644 index 189236d8..00000000 --- a/.github/workflows/publish-to-pypi.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI -on: [release] - -jobs: - build-n-publish: - name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Publish any distribution 📦 to Test PyPI - uses: JRubics/poetry-publish@v1.17 - with: - pypi_token: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository_url: https://test.pypi.org/legacy/ - ignore_dev_requirements: "yes" - - name: Publish tagged distribution 📦 to PyPI - uses: JRubics/poetry-publish@v1.17 - if: startsWith(github.ref, 'refs/tags') - with: - pypi_token: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml new file mode 100644 index 00000000..1a6f5b29 --- /dev/null +++ b/.github/workflows/pypi-release.yml @@ -0,0 +1,45 @@ +name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI +on: [release] + +env: + LANG: "en_US.utf-8" + LC_ALL: "en_US.utf-8" + PYTHONIOENCODING: "UTF-8" + TARGET_PYTHON_VERSION: "3.9" + +jobs: + build-and-publish: + name: Build and publish Python 🐍 distributions + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + # We do not need to fetch all the history of the repository + # as commit being released is guaranteed to have a tag. + # poetry-dynamic-versioning will use that tag use to set the + # build version of the package. + fetch-depth: 1 + - name: Setup base environment + uses: ./.github/actions/setup-env + with: + python-version: ${{ env.TARGET_PYTHON_VERSION }} + cache-pre-commit: false + cache-venv: false + setup-poetry: true + install-deps: false + - name: Build 📦 distributions + run: | + poetry build + - name: Publish any distribution 📦 to Test PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + repository-url: https://test.pypi.org/legacy/ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + - name: Publish tagged distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + if: startsWith(github.ref, 'refs/tags') + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml deleted file mode 100644 index a8c343f4..00000000 --- a/.github/workflows/pytest.yml +++ /dev/null @@ -1,65 +0,0 @@ -# This workflow will install Python dependencies, run tests and black with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python build - -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - os: [ubuntu-latest, macos-latest] - env: - OS: ${{ matrix.os }} - # This is needed to avoid a warning from SQLAlchemy - # https://sqlalche.me/e/b8d9 - # We can remove this once we upgrade to SQLAlchemy >= 2.0 - SQLALCHEMY_SILENCE_UBER_WARNING: "1" - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - run: | - python -m pip install poetry - poetry config virtualenvs.in-project true --local - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root --all-extras --without docs - #---------------------------------------------- - # install pycytominer - #---------------------------------------------- - - name: Install library - run: poetry install --no-interaction --only-root - #---------------------------------------------- - # run test suite and output coverage file - #---------------------------------------------- - - name: Test with pytest - run: | - poetry run pytest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6907bd56..210a5c30 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,3 +13,7 @@ repos: rev: "1.5.1" hooks: - id: poetry-check + - repo: https://github.com/rhysd/actionlint + rev: v1.6.26 + hooks: + - id: actionlint diff --git a/pycytominer/consensus.py b/pycytominer/consensus.py index 561fcafa..44357c74 100644 --- a/pycytominer/consensus.py +++ b/pycytominer/consensus.py @@ -109,7 +109,7 @@ def consensus( population_df=profiles, replicate_columns=replicate_columns, features=features, - **modz_args + **modz_args, ) else: consensus_df = aggregate(