From 790dffbb571909f29a553cf2d1e007c1f91ff7f1 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 21 Aug 2023 14:13:43 +0200 Subject: [PATCH 01/22] Use windows for pytest --- .github/workflows/_local_ci_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml index f65ab62e..d99560ae 100644 --- a/.github/workflows/_local_ci_tests.yml +++ b/.github/workflows/_local_ci_tests.yml @@ -35,12 +35,13 @@ jobs: pytest: name: pytest - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] + os: ["ubuntu-latest", "windows-latest"] steps: - name: Checkout repository From 69047ddca39588ba42bd578c8c342493874a1254 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 21 Aug 2023 14:41:43 +0200 Subject: [PATCH 02/22] Go through Path before replacing / with . --- ci_cd/tasks/api_reference_docs.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ci_cd/tasks/api_reference_docs.py b/ci_cd/tasks/api_reference_docs.py index c904b013..fbe1aa9c 100644 --- a/ci_cd/tasks/api_reference_docs.py +++ b/ci_cd/tasks/api_reference_docs.py @@ -295,13 +295,15 @@ def write_file(full_path: Path, content: str) -> None: package.relative_to(root_repo_path) if relative else package.name ) py_path = ( - f"{py_path_root}/{filename.stem}".replace("/", ".") + f"{py_path_root}/{filename.stem}" if str(relpath) == "." or (str(relpath) == package.name and not single_package) - else f"{py_path_root}/{relpath if single_package else relpath.relative_to(package.name)}/{filename.stem}".replace( - "/", "." - ) + else f"{py_path_root}/{relpath if single_package else relpath.relative_to(package.name)}/{filename.stem}" ) + + # Cast py_path to Path to ensure correct formatting (forward slashes) + py_path = str(Path(py_path)).replace("/", ".") + LOGGER.debug("filename: %s\npy_path: %s", filename, py_path) if debug: print("filename:", filename, flush=True) From b25acc1660ab72f5d9f37ed37cbbd17563caa911 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 21 Aug 2023 14:47:00 +0200 Subject: [PATCH 03/22] Use os.sep as a pre-replacer --- ci_cd/tasks/api_reference_docs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci_cd/tasks/api_reference_docs.py b/ci_cd/tasks/api_reference_docs.py index fbe1aa9c..971f71f4 100644 --- a/ci_cd/tasks/api_reference_docs.py +++ b/ci_cd/tasks/api_reference_docs.py @@ -301,8 +301,9 @@ def write_file(full_path: Path, content: str) -> None: else f"{py_path_root}/{relpath if single_package else relpath.relative_to(package.name)}/{filename.stem}" ) - # Cast py_path to Path to ensure correct formatting (forward slashes) - py_path = str(Path(py_path)).replace("/", ".") + # Replace OS specific path separators with forward slashes before + # replacing that with dots (for Python import paths). + py_path = py_path.replace(os.sep, "/").replace("/", ".") LOGGER.debug("filename: %s\npy_path: %s", filename, py_path) if debug: From 2b39074fa1fb4fa86cf463f2a3c0f8e38a8b69f8 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 21 Aug 2023 14:58:21 +0200 Subject: [PATCH 04/22] Also use os.sep in tests --- tests/tasks/test_api_reference_docs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tasks/test_api_reference_docs.py b/tests/tasks/test_api_reference_docs.py index 471cf49d..9a6487cb 100644 --- a/tests/tasks/test_api_reference_docs.py +++ b/tests/tasks/test_api_reference_docs.py @@ -479,7 +479,7 @@ def test_larger_package(tmp_path: "Path") -> None: ]: py_path = f"{package_dir.name}." + str( module_dir.relative_to(api_reference_folder) - ).replace("/", ".") + ).replace(os.sep, "/").replace("/", ".") assert (module_dir / ".pages").read_text( encoding="utf8" ) == f'title: "{module_dir.name}"\n', ( @@ -637,7 +637,7 @@ def test_larger_multi_packages(tmp_path: "Path") -> None: for module_dir in [package_dir / _ for _ in new_submodules]: py_path = f"{package_dir.name}." + str( module_dir.relative_to(package_dir) - ).replace("/", ".") + ).replace(os.sep, "/").replace("/", ".") assert (module_dir / ".pages").read_text( encoding="utf8" ) == f'title: "{module_dir.name}"\n', ( From 16b6d281f4d2814a7363287e2d038f512542fdf8 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 21 Aug 2023 15:02:21 +0200 Subject: [PATCH 05/22] Add in test for create_api_reference_docs pre-commit --- tests/tasks/test_api_reference_docs.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/tasks/test_api_reference_docs.py b/tests/tasks/test_api_reference_docs.py index 9a6487cb..f3271b7f 100644 --- a/tests/tasks/test_api_reference_docs.py +++ b/tests/tasks/test_api_reference_docs.py @@ -2,11 +2,14 @@ # pylint: disable=too-many-locals from typing import TYPE_CHECKING +import pytest + if TYPE_CHECKING: from pathlib import Path -def test_default_run(tmp_path: "Path") -> None: +@pytest.mark.parametrize("pre_commit", [True, False]) +def test_default_run(tmp_path: "Path", pre_commit: bool) -> None: """Check create_api_reference_docs runs with defaults.""" import os import shutil @@ -29,6 +32,7 @@ def test_default_run(tmp_path: "Path") -> None: MockContext(), [str(package_dir.relative_to(tmp_path))], root_repo_path=str(tmp_path), + pre_commit=pre_commit, ) api_reference_folder = docs_folder / "api_reference" From 641085e410ecf55243a8d73bc11b79be93365a0c Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 10:09:20 +0200 Subject: [PATCH 06/22] Revert "Add in test for create_api_reference_docs pre-commit" This reverts commit 16b6d281f4d2814a7363287e2d038f512542fdf8. --- tests/tasks/test_api_reference_docs.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/tasks/test_api_reference_docs.py b/tests/tasks/test_api_reference_docs.py index f3271b7f..9a6487cb 100644 --- a/tests/tasks/test_api_reference_docs.py +++ b/tests/tasks/test_api_reference_docs.py @@ -2,14 +2,11 @@ # pylint: disable=too-many-locals from typing import TYPE_CHECKING -import pytest - if TYPE_CHECKING: from pathlib import Path -@pytest.mark.parametrize("pre_commit", [True, False]) -def test_default_run(tmp_path: "Path", pre_commit: bool) -> None: +def test_default_run(tmp_path: "Path") -> None: """Check create_api_reference_docs runs with defaults.""" import os import shutil @@ -32,7 +29,6 @@ def test_default_run(tmp_path: "Path", pre_commit: bool) -> None: MockContext(), [str(package_dir.relative_to(tmp_path))], root_repo_path=str(tmp_path), - pre_commit=pre_commit, ) api_reference_folder = docs_folder / "api_reference" From b82e1685f732978f57e942c5c0deed130a781456 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 10:09:59 +0200 Subject: [PATCH 07/22] Run all invoke tasks as pre-commit hooks in CI --- .github/utils/.pre-commit-config_testing.yaml | 34 +++++++++++++++++ .github/workflows/_local_ci_tests.yml | 38 +++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 .github/utils/.pre-commit-config_testing.yaml diff --git a/.github/utils/.pre-commit-config_testing.yaml b/.github/utils/.pre-commit-config_testing.yaml new file mode 100644 index 00000000..8116c7bc --- /dev/null +++ b/.github/utils/.pre-commit-config_testing.yaml @@ -0,0 +1,34 @@ +repos: + - repo: . + rev: HEAD + hooks: + - id: docs-api-reference + args: + - "--package-dir=ci_cd" + - "--debug" + - id: docs-landing-page + - id: update-pyproject + + - repo: local + hooks: + - id: set-version + name: Set package version + entry: "ci-cd setver" + language: python + files: "" + exclude: ^$ + types: [] + types_or: [] + exclude_types: [] + always_run: false + fail_fast: false + verbose: false + pass_filenames: false + require_serial: false + description: "Sets the specified version of specified Python package." + language_version: default + minimum_pre_commit_version: "2.16.0" + args: + - "--package-dir=ci_cd" + - "--version=9.8.7" + - "--test" diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml index d99560ae..a6b91fb3 100644 --- a/.github/workflows/_local_ci_tests.yml +++ b/.github/workflows/_local_ci_tests.yml @@ -67,3 +67,41 @@ jobs: with: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} + + # These jobs are mainly to test a default run of the hooks including `--pre-commit` + run_hooks: + name: Run custom pre-commit hooks + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + python-version: ["3.7", "3.10"] + os: ["ubuntu-latest", "windows-latest"] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version}} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version}} + + - name: Install Python dependencies + run: | + python -m pip install -U pip + pip install -U setuptools wheel flit + pip install -U pre-commit + + - name: Run docs-api-reference + run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose docs-api-reference + + - name: Run docs-api-reference + run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose docs-landing-page + + - name: Run docs-api-reference + run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose update-pyproject + + - name: Run docs-api-reference + run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose set-version From a348749d02d9b8e41d8333df4c563d99cf6c84c6 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 10:14:34 +0200 Subject: [PATCH 08/22] Take into account changes to landing page --- .github/workflows/_local_ci_tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml index a6b91fb3..ab882aa0 100644 --- a/.github/workflows/_local_ci_tests.yml +++ b/.github/workflows/_local_ci_tests.yml @@ -94,14 +94,14 @@ jobs: pip install -U setuptools wheel flit pip install -U pre-commit - - name: Run docs-api-reference + - name: Run docs-api-reference ('ci-cd create-api-reference-docs') run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose docs-api-reference - - name: Run docs-api-reference - run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose docs-landing-page + - name: Run docs-landing-page ('ci-cd create-docs-index') + run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose docs-landing-page | exit 0 || grep "The landing page has been updated." | exit 0 || exit 1 - - name: Run docs-api-reference + - name: Run update-pyproject ('ci-cd update-deps') run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose update-pyproject - - name: Run docs-api-reference + - name: Run 'ci-cd setver' run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose set-version From b8399db3ae0f0a9e4d68edea9dd91fe9f6cda3d0 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 10:20:02 +0200 Subject: [PATCH 09/22] Catch changes in tested hooks --- .github/workflows/_local_ci_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml index ab882aa0..3b460e69 100644 --- a/.github/workflows/_local_ci_tests.yml +++ b/.github/workflows/_local_ci_tests.yml @@ -98,10 +98,10 @@ jobs: run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose docs-api-reference - name: Run docs-landing-page ('ci-cd create-docs-index') - run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose docs-landing-page | exit 0 || grep "The landing page has been updated." | exit 0 || exit 1 + run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose docs-landing-page || grep "The landing page has been updated." | exit 0 || exit 1 - name: Run update-pyproject ('ci-cd update-deps') - run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose update-pyproject + run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose update-pyproject || grep "Successfully updated the following dependencies:" | exit 0 || exit 1 - name: Run 'ci-cd setver' run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose set-version From 3285c88cd0ee696b2a16a4e35fbd7293c0406958 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 10:21:32 +0200 Subject: [PATCH 10/22] Install ci-cd for testing hooks --- .github/workflows/_local_ci_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml index 3b460e69..8588a9b4 100644 --- a/.github/workflows/_local_ci_tests.yml +++ b/.github/workflows/_local_ci_tests.yml @@ -92,6 +92,7 @@ jobs: run: | python -m pip install -U pip pip install -U setuptools wheel flit + pip install -e . pip install -U pre-commit - name: Run docs-api-reference ('ci-cd create-api-reference-docs') From a283378f6505d2f3980711ebca744384d5faad9b Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 10:24:29 +0200 Subject: [PATCH 11/22] Ensure changes from setver hooks is OK Change target version to 0.0.0, which should never be an actual version used, meaning there is no single point this test job should suddenly not incur a "change" in the base files. --- .github/utils/.pre-commit-config_testing.yaml | 2 +- .github/workflows/_local_ci_tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/utils/.pre-commit-config_testing.yaml b/.github/utils/.pre-commit-config_testing.yaml index 8116c7bc..c02fd9c0 100644 --- a/.github/utils/.pre-commit-config_testing.yaml +++ b/.github/utils/.pre-commit-config_testing.yaml @@ -30,5 +30,5 @@ repos: minimum_pre_commit_version: "2.16.0" args: - "--package-dir=ci_cd" - - "--version=9.8.7" + - "--version=0.0.0" - "--test" diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml index 8588a9b4..a4d79f63 100644 --- a/.github/workflows/_local_ci_tests.yml +++ b/.github/workflows/_local_ci_tests.yml @@ -105,4 +105,4 @@ jobs: run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose update-pyproject || grep "Successfully updated the following dependencies:" | exit 0 || exit 1 - name: Run 'ci-cd setver' - run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose set-version + run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose set-version || grep "Bumped version for ci_cd to 0.0.0." | exit 0 || exit 1 From 5accc31da39a789492bb50ebe4db6ffe9b966ae8 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 10:55:01 +0200 Subject: [PATCH 12/22] Escape unicode characters for Windows terminals --- ci_cd/tasks/api_reference_docs.py | 33 ++++++++++++++++++++----------- ci_cd/utils.py | 11 +++++++++++ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/ci_cd/tasks/api_reference_docs.py b/ci_cd/tasks/api_reference_docs.py index 971f71f4..d419fd8f 100644 --- a/ci_cd/tasks/api_reference_docs.py +++ b/ci_cd/tasks/api_reference_docs.py @@ -10,7 +10,7 @@ import shutil import sys from collections import defaultdict -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING from invoke import task @@ -131,6 +131,12 @@ def create_api_reference_docs( # pylint: disable=too-many-locals,too-many-branc if not special_option: special_option: list[str] = [] # type: ignore[no-redef] + # Initialize user-given paths as pure POSIX paths + package_dir: list[PurePosixPath] = [PurePosixPath(_) for _ in package_dir] + root_repo_path: PurePosixPath = PurePosixPath(root_repo_path) # type: ignore[no-redef] + docs_folder: PurePosixPath = PurePosixPath(docs_folder) # type: ignore[no-redef] + full_docs_folder: list[PurePosixPath] = [PurePosixPath(_) for _ in full_docs_folder] # type: ignore[no-redef] + def write_file(full_path: Path, content: str) -> None: """Write file with `content` to `full_path`""" if full_path.exists(): @@ -141,14 +147,22 @@ def write_file(full_path: Path, content: str) -> None: del cached_content full_path.write_text(content, encoding="utf8") + if pre_commit: + # Ensure git is installed + result: "Result" = context.run("git --version", hide=True) + if result.exited != 0: + sys.exit( + "Git is not installed. Please install it before running this task." + ) + if pre_commit and root_repo_path == ".": # Use git to determine repo root - result: "Result" = context.run("git rev-parse --show-toplevel", hide=True) - root_repo_path = result.stdout.strip("\n") + result = context.run("git rev-parse --show-toplevel", hide=True) + root_repo_path = result.stdout.strip("\n") # type: ignore[no-redef] root_repo_path: Path = Path(root_repo_path).resolve() # type: ignore[no-redef] - package_dirs: list[Path] = [root_repo_path / _ for _ in package_dir] - docs_api_ref_dir = root_repo_path / docs_folder / "api_reference" + package_dirs: list[Path] = [Path(root_repo_path / _) for _ in package_dir] + docs_api_ref_dir = Path(root_repo_path / docs_folder / "api_reference") LOGGER.debug( """package_dirs: %s @@ -197,11 +211,8 @@ def write_file(full_path: Path, content: str) -> None: if debug: print("special_options_files:", special_options_files, flush=True) - if any("/" in _ for _ in unwanted_folder + unwanted_file): - sys.exit( - "Unwanted folders and files may NOT be paths. A forward slash (/) was " - "found in some of them." - ) + if any(os.sep in _ or "/" in _ for _ in unwanted_folder + unwanted_file): + sys.exit("Unwanted folders and files may NOT be paths.") pages_template = 'title: "{name}"\n' md_template = "# {name}\n\n::: {py_path}\n" @@ -359,7 +370,7 @@ def write_file(full_path: Path, content: str) -> None: # (which will be good in this case). # Concerning the weird last grep command see: # http://manpages.ubuntu.com/manpages/precise/en/man1/git-status.1.html - result: "Result" = context.run( # type: ignore[no-redef] + result = context.run( f'git -C "{root_repo_path}" status --porcelain ' f"{docs_api_ref_dir.relative_to(root_repo_path)} | " "grep -E '^[? MARC][?MD]' || exit 0", diff --git a/ci_cd/utils.py b/ci_cd/utils.py index 70d1e534..b88d5982 100644 --- a/ci_cd/utils.py +++ b/ci_cd/utils.py @@ -2,6 +2,7 @@ More information on `invoke` can be found at [pyinvoke.org](http://www.pyinvoke.org/). """ import logging +import platform import re from enum import Enum from pathlib import Path @@ -18,6 +19,16 @@ class Emoji(str, Enum): """Unicode strings for certain emojis.""" + def __new__(cls, value: str) -> "Emoji": + obj = str.__new__(cls, value) + if platform.system() == "Windows": + # Windows does not support unicode emojis, so we replace them with + # their corresponding unicode escape sequences + obj._value_ = value.encode("unicode_escape").decode("utf-8") + else: + obj._value_ = value + return obj + PARTY_POPPER = "\U0001f389" CHECK_MARK = "\u2714" CROSS_MARK = "\u274c" From 7a2e6fbd2a88db522fd1fe3dbe0140f5dda41026 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 11:31:20 +0200 Subject: [PATCH 13/22] Try to improve testing hooks in different OS' --- .github/workflows/_local_ci_tests.yml | 133 ++++++++++++++++++++++++-- 1 file changed, 126 insertions(+), 7 deletions(-) diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml index a4d79f63..36dfc29c 100644 --- a/.github/workflows/_local_ci_tests.yml +++ b/.github/workflows/_local_ci_tests.yml @@ -95,14 +95,133 @@ jobs: pip install -e . pip install -U pre-commit + # docs-api-reference - name: Run docs-api-reference ('ci-cd create-api-reference-docs') - run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose docs-api-reference - + continue-on-error: true + id: docs_api_reference + run: | + { + echo 'docs_api_reference_output<> "$GITHUB_ENV" + + - name: Check if failure is expected/acceptible + if: steps.docs_api_reference.outcome == 'failure' + run: printf '%s\n' "$docs_api_reference_output" | grep "The following files have been changed/added/removed:" | exit 0 || exit 1 + shell: bash + + - name: Run docs-api-reference ('ci-cd create-api-reference-docs') (cmd) + if: runner.os == 'Windows' + continue-on-error: true + id: docs_api_reference_cmd + run: | + { + echo 'docs_api_reference_output<> "$GITHUB_ENV" + shell: cmd + + - name: Check if failure is expected/acceptible + if: runner.os == 'Windows' && steps.docs_api_reference_cmd.outcome == 'failure' + run: printf '%s\n' "$docs_api_reference_output" | grep "The following files have been changed/added/removed:" | exit 0 || exit 1 + shell: bash + + # docs-landing-page - name: Run docs-landing-page ('ci-cd create-docs-index') - run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose docs-landing-page || grep "The landing page has been updated." | exit 0 || exit 1 - + continue-on-error: true + id: docs_landing_page + run: | + { + echo 'docs_landing_page_output<> "$GITHUB_ENV" + + - name: Check if failure is expected/acceptible + if: steps.docs_landing_page.outcome == 'failure' + run: printf '%s\n' "$docs_landing_page_output" | grep "The landing page has been updated." | exit 0 || exit 1 + shell: bash + + - name: Run docs-landing-page ('ci-cd create-docs-index') (cmd) + if: runner.os == 'Windows' + continue-on-error: true + id: docs_landing_page_cmd + run: | + { + echo 'docs_landing_page_output<> "$GITHUB_ENV" + shell: cmd + + - name: Check if failure is expected/acceptible + if: runner.os == 'Windows' && steps.docs_landing_page_cmd.outcome == 'failure' + run: printf '%s\n' "$docs_landing_page_output" | grep "The landing page has been updated." | exit 0 || exit 1 + shell: bash + + # update-pyproject - name: Run update-pyproject ('ci-cd update-deps') - run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose update-pyproject || grep "Successfully updated the following dependencies:" | exit 0 || exit 1 - + continue-on-error: true + id: update_pyproject + run: | + { + echo 'update_pyproject_output<> "$GITHUB_ENV" + + - name: Check if failure is expected/acceptible + if: steps.update_pyproject.outcome == 'failure' + run: printf '%s\n' "$update_pyproject_output" | grep "Successfully updated the following dependencies:" | exit 0 || exit 1 + shell: bash + + - name: Run update-pyproject ('ci-cd update-deps') (cmd) + if: runner.os == 'Windows' + continue-on-error: true + id: update_pyproject_cmd + run: | + { + echo 'update_pyproject_output<> "$GITHUB_ENV" + shell: cmd + + - name: Check if failure is expected/acceptible + if: runner.os == 'Windows' && steps.update_pyproject_cmd.outcome == 'failure' + run: printf '%s\n' "$update_pyproject_output" | grep "Successfully updated the following dependencies:" | exit 0 || exit 1 + shell: bash + + # set-version - name: Run 'ci-cd setver' - run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose set-version || grep "Bumped version for ci_cd to 0.0.0." | exit 0 || exit 1 + continue-on-error: true + id: set_version + run: | + { + echo 'set_version_output<> "$GITHUB_ENV" + + - name: Check if failure is expected/acceptible + if: steps.set_version.outcome == 'failure' + run: printf '%s\n' "$set_version_output" | grep "Bumped version for ci_cd to 0.0.0." | exit 0 || exit 1 + shell: bash + + - name: Run 'ci-cd setver' (cmd) + if: runner.os == 'Windows' + continue-on-error: true + id: set_version_cmd + run: | + { + echo 'set_version_output<> "$GITHUB_ENV" + + - name: Check if failure is expected/acceptible + if: runner.os == 'Windows' && steps.set_version_cmd.outcome == 'failure' + run: printf '%s\n' "$set_version_output" | grep "Bumped version for ci_cd to 0.0.0." | exit 0 || exit 1 + shell: bash From af58faff62fd009bbe1cea030185a07fe9c1f8d5 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 13:37:12 +0200 Subject: [PATCH 14/22] Use a Py script to run hooks in CI --- .github/utils/run_hooks.py | 75 +++++++++++++++++ .github/workflows/_local_ci_tests.yml | 113 ++------------------------ 2 files changed, 84 insertions(+), 104 deletions(-) create mode 100755 .github/utils/run_hooks.py diff --git a/.github/utils/run_hooks.py b/.github/utils/run_hooks.py new file mode 100755 index 00000000..aaa50e96 --- /dev/null +++ b/.github/utils/run_hooks.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Run pre-commit hooks on all files in the repository. + +File used to test running the hooks in the CI/CD pipeline independently of the shell. +""" +import platform +import subprocess # nosec +import sys + +SUCCESSFUL_FAILURES_MAPPING = { + "docs-api-reference": "The following files have been changed/added/removed:", + "docs-landing-page": "The landing page has been updated.", + "update-pyproject": "Successfully updated the following dependencies:", + "set-version": "Bumped version for ci_cd to 0.0.0.", +} + + +def main(hook: str, options: list[str]) -> None: + """Run pre-commit hooks on all files in the repository.""" + run_pre_commit = [ + "pre-commit", + "run", + "-c", + ".github/utils/.pre-commit-config_testing.yaml", + "--all-files", + "--verbose", + ] + + if platform.system() == "Windows": + result = subprocess.run( + ["py", "-m"] + run_pre_commit + options + [hook], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, # nosec + ) + else: + result = subprocess.run( + run_pre_commit + options + [hook], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, # nosec + ) + + if result.returncode != 0: + if SUCCESSFUL_FAILURES_MAPPING[hook] in result.stdout.decode(): + print(f"Successfully failed {hook} hook.\n\n", flush=True) + print(result.stdout.decode(), flush=True) + else: + sys.exit(result.stdout.decode()) + print(f"Successfully ran {hook} hook.\n\n", flush=True) + print(result.stdout.decode(), flush=True) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + raise sys.exit("Missing arguments") + + # "Parse" arguments + # The first argument should be the hook name + if sys.argv[1] not in SUCCESSFUL_FAILURES_MAPPING: + raise sys.exit( + f"Invalid hook name: {sys.argv[1]}\n" + "The hook name should be the first argument. Any number of hook options " + "can then follow." + ) + + try: + main( + hook=sys.argv[1], + options=sys.argv[2:] if len(sys.argv) > 2 else [], + ) + except Exception as exc: # pylint: disable=broad-except + sys.exit(str(exc)) diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml index 36dfc29c..0bdd8a0e 100644 --- a/.github/workflows/_local_ci_tests.yml +++ b/.github/workflows/_local_ci_tests.yml @@ -97,131 +97,36 @@ jobs: # docs-api-reference - name: Run docs-api-reference ('ci-cd create-api-reference-docs') - continue-on-error: true - id: docs_api_reference - run: | - { - echo 'docs_api_reference_output<> "$GITHUB_ENV" - - - name: Check if failure is expected/acceptible - if: steps.docs_api_reference.outcome == 'failure' - run: printf '%s\n' "$docs_api_reference_output" | grep "The following files have been changed/added/removed:" | exit 0 || exit 1 - shell: bash + run: python .github/utils/run_hooks.py docs-api-reference - name: Run docs-api-reference ('ci-cd create-api-reference-docs') (cmd) if: runner.os == 'Windows' - continue-on-error: true - id: docs_api_reference_cmd - run: | - { - echo 'docs_api_reference_output<> "$GITHUB_ENV" + run: python .github/utils/run_hooks.py docs-api-reference shell: cmd - - name: Check if failure is expected/acceptible - if: runner.os == 'Windows' && steps.docs_api_reference_cmd.outcome == 'failure' - run: printf '%s\n' "$docs_api_reference_output" | grep "The following files have been changed/added/removed:" | exit 0 || exit 1 - shell: bash - # docs-landing-page - name: Run docs-landing-page ('ci-cd create-docs-index') - continue-on-error: true - id: docs_landing_page - run: | - { - echo 'docs_landing_page_output<> "$GITHUB_ENV" - - - name: Check if failure is expected/acceptible - if: steps.docs_landing_page.outcome == 'failure' - run: printf '%s\n' "$docs_landing_page_output" | grep "The landing page has been updated." | exit 0 || exit 1 - shell: bash + run: python .github/utils/run_hooks.py docs-landing-page - name: Run docs-landing-page ('ci-cd create-docs-index') (cmd) if: runner.os == 'Windows' - continue-on-error: true - id: docs_landing_page_cmd - run: | - { - echo 'docs_landing_page_output<> "$GITHUB_ENV" + run: python .github/utils/run_hooks.py docs-landing-page shell: cmd - - name: Check if failure is expected/acceptible - if: runner.os == 'Windows' && steps.docs_landing_page_cmd.outcome == 'failure' - run: printf '%s\n' "$docs_landing_page_output" | grep "The landing page has been updated." | exit 0 || exit 1 - shell: bash - # update-pyproject - name: Run update-pyproject ('ci-cd update-deps') - continue-on-error: true - id: update_pyproject - run: | - { - echo 'update_pyproject_output<> "$GITHUB_ENV" - - - name: Check if failure is expected/acceptible - if: steps.update_pyproject.outcome == 'failure' - run: printf '%s\n' "$update_pyproject_output" | grep "Successfully updated the following dependencies:" | exit 0 || exit 1 - shell: bash + run: python .github/utils/run_hooks.py update-pyproject - name: Run update-pyproject ('ci-cd update-deps') (cmd) if: runner.os == 'Windows' - continue-on-error: true - id: update_pyproject_cmd - run: | - { - echo 'update_pyproject_output<> "$GITHUB_ENV" + run: python .github/utils/run_hooks.py update-pyproject shell: cmd - - name: Check if failure is expected/acceptible - if: runner.os == 'Windows' && steps.update_pyproject_cmd.outcome == 'failure' - run: printf '%s\n' "$update_pyproject_output" | grep "Successfully updated the following dependencies:" | exit 0 || exit 1 - shell: bash - # set-version - name: Run 'ci-cd setver' - continue-on-error: true - id: set_version - run: | - { - echo 'set_version_output<> "$GITHUB_ENV" - - - name: Check if failure is expected/acceptible - if: steps.set_version.outcome == 'failure' - run: printf '%s\n' "$set_version_output" | grep "Bumped version for ci_cd to 0.0.0." | exit 0 || exit 1 - shell: bash + run: python .github/utils/run_hooks.py set-version - name: Run 'ci-cd setver' (cmd) if: runner.os == 'Windows' - continue-on-error: true - id: set_version_cmd - run: | - { - echo 'set_version_output<> "$GITHUB_ENV" - - - name: Check if failure is expected/acceptible - if: runner.os == 'Windows' && steps.set_version_cmd.outcome == 'failure' - run: printf '%s\n' "$set_version_output" | grep "Bumped version for ci_cd to 0.0.0." | exit 0 || exit 1 - shell: bash + run: python .github/utils/run_hooks.py set-version + shell: cmd From 501986eedda09f2ad7d19af4bfb0d6ff2d504a14 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 13:56:28 +0200 Subject: [PATCH 15/22] Use a string for args instead of a list --- .github/utils/run_hooks.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/utils/run_hooks.py b/.github/utils/run_hooks.py index aaa50e96..06477841 100755 --- a/.github/utils/run_hooks.py +++ b/.github/utils/run_hooks.py @@ -17,18 +17,14 @@ def main(hook: str, options: list[str]) -> None: """Run pre-commit hooks on all files in the repository.""" - run_pre_commit = [ - "pre-commit", - "run", - "-c", - ".github/utils/.pre-commit-config_testing.yaml", - "--all-files", - "--verbose", - ] + run_pre_commit = ( + "pre-commit run -c .github/utils/.pre-commit-config_testing.yaml " + "--all-files --verbose" + ) if platform.system() == "Windows": result = subprocess.run( - ["py", "-m"] + run_pre_commit + options + [hook], + f"py -m {run_pre_commit} {options} {hook}", check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -36,7 +32,7 @@ def main(hook: str, options: list[str]) -> None: ) else: result = subprocess.run( - run_pre_commit + options + [hook], + f"{run_pre_commit} {options} {hook}", check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, From 758c7cb0c508381a17f293dcb164f0a968884484 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 13:57:20 +0200 Subject: [PATCH 16/22] Import annotations for the sake of Py3.7 --- .github/utils/run_hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/utils/run_hooks.py b/.github/utils/run_hooks.py index 06477841..71c037d0 100755 --- a/.github/utils/run_hooks.py +++ b/.github/utils/run_hooks.py @@ -3,6 +3,8 @@ File used to test running the hooks in the CI/CD pipeline independently of the shell. """ +from __future__ import annotations + import platform import subprocess # nosec import sys From 698b5a67e84c90e3af885a3f09bdf36ff33e5a69 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 13:59:13 +0200 Subject: [PATCH 17/22] Don't use pre-commit as module in Windows --- .github/utils/run_hooks.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/.github/utils/run_hooks.py b/.github/utils/run_hooks.py index 71c037d0..b598cb3a 100755 --- a/.github/utils/run_hooks.py +++ b/.github/utils/run_hooks.py @@ -5,7 +5,7 @@ """ from __future__ import annotations -import platform +# import platform import subprocess # nosec import sys @@ -24,22 +24,13 @@ def main(hook: str, options: list[str]) -> None: "--all-files --verbose" ) - if platform.system() == "Windows": - result = subprocess.run( - f"py -m {run_pre_commit} {options} {hook}", - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - shell=True, # nosec - ) - else: - result = subprocess.run( - f"{run_pre_commit} {options} {hook}", - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - shell=True, # nosec - ) + result = subprocess.run( + f"{run_pre_commit} {options} {hook}", + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, # nosec + ) if result.returncode != 0: if SUCCESSFUL_FAILURES_MAPPING[hook] in result.stdout.decode(): From 5fb190df1c4a011415161ba8bd2d2c7eb797d0c8 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 14:02:19 +0200 Subject: [PATCH 18/22] Unpack options for subprocess cmd --- .github/utils/run_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/utils/run_hooks.py b/.github/utils/run_hooks.py index b598cb3a..ab0fc690 100755 --- a/.github/utils/run_hooks.py +++ b/.github/utils/run_hooks.py @@ -25,7 +25,7 @@ def main(hook: str, options: list[str]) -> None: ) result = subprocess.run( - f"{run_pre_commit} {options} {hook}", + f"{run_pre_commit} {' '.join(_ for _ in options)} {hook}", check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, From 4cd2220d69d8905f2ef6031477d21e9c7f2954a7 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 14:15:16 +0200 Subject: [PATCH 19/22] Add bare run of pre-commit with cmd shell --- .github/workflows/_local_ci_tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml index 0bdd8a0e..66ac9c19 100644 --- a/.github/workflows/_local_ci_tests.yml +++ b/.github/workflows/_local_ci_tests.yml @@ -130,3 +130,8 @@ jobs: if: runner.os == 'Windows' run: python .github/utils/run_hooks.py set-version shell: cmd + + - name: Bare run + if: runner.os == 'Windows' + run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose docs-api-reference + shell: cmd From 2b1a95606d8f3ac169d10b7ea8336829c73c8d79 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 14:34:16 +0200 Subject: [PATCH 20/22] Revert addition of "bare run" CI job --- .github/workflows/_local_ci_tests.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml index 66ac9c19..0bdd8a0e 100644 --- a/.github/workflows/_local_ci_tests.yml +++ b/.github/workflows/_local_ci_tests.yml @@ -130,8 +130,3 @@ jobs: if: runner.os == 'Windows' run: python .github/utils/run_hooks.py set-version shell: cmd - - - name: Bare run - if: runner.os == 'Windows' - run: pre-commit run -c .github/utils/.pre-commit-config_testing.yaml --all-files --verbose docs-api-reference - shell: cmd From d601906cc9a29205190e47d3ef517bad279dc6cb Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 14:41:48 +0200 Subject: [PATCH 21/22] Cast relative_file_path to Path, convert to posix path --- ci_cd/tasks/api_reference_docs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci_cd/tasks/api_reference_docs.py b/ci_cd/tasks/api_reference_docs.py index d419fd8f..f2e5f875 100644 --- a/ci_cd/tasks/api_reference_docs.py +++ b/ci_cd/tasks/api_reference_docs.py @@ -321,9 +321,9 @@ def write_file(full_path: Path, content: str) -> None: print("filename:", filename, flush=True) print("py_path:", py_path, flush=True) - relative_file_path = ( + relative_file_path = Path( str(filename) if str(relpath) == "." else str(relpath / filename) - ) + ).as_posix() # For special files we want to include EVERYTHING, even if it doesn't # have a doc-string From f502549309e911416b0efb065939b225a7fe4739 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 22 Aug 2023 15:01:40 +0200 Subject: [PATCH 22/22] Remove usage of grep and pipes in hooks --- ci_cd/tasks/api_reference_docs.py | 20 ++++++++++---------- ci_cd/tasks/docs_index.py | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/ci_cd/tasks/api_reference_docs.py b/ci_cd/tasks/api_reference_docs.py index f2e5f875..6273f998 100644 --- a/ci_cd/tasks/api_reference_docs.py +++ b/ci_cd/tasks/api_reference_docs.py @@ -366,22 +366,22 @@ def write_file(full_path: Path, content: str) -> None: # Check if there have been any changes. # List changes if yes. - # NOTE: grep returns an exit code of 1 if it doesn't find anything - # (which will be good in this case). - # Concerning the weird last grep command see: + # NOTE: Concerning the weird regular expression, see: # http://manpages.ubuntu.com/manpages/precise/en/man1/git-status.1.html result = context.run( f'git -C "{root_repo_path}" status --porcelain ' - f"{docs_api_ref_dir.relative_to(root_repo_path)} | " - "grep -E '^[? MARC][?MD]' || exit 0", + f"{docs_api_ref_dir.relative_to(root_repo_path)}", hide=True, ) if result.stdout: - sys.exit( - f"{Emoji.CURLY_LOOP.value} The following files have been " - f"changed/added/removed:\n\n{result.stdout}\nPlease stage them:\n\n" - f" git add {docs_api_ref_dir.relative_to(root_repo_path)}" - ) + for line in result.stdout.splitlines(): + if re.match(r"^[? MARC][?MD]", line): + sys.exit( + f"{Emoji.CURLY_LOOP.value} The following files have been " + f"changed/added/removed:\n\n{result.stdout}\n" + "Please stage them:\n\n" + f" git add {docs_api_ref_dir.relative_to(root_repo_path)}" + ) print( f"{Emoji.CHECK_MARK.value} No changes - your API reference documentation " "is up-to-date !" diff --git a/ci_cd/tasks/docs_index.py b/ci_cd/tasks/docs_index.py index 83b57802..df8ae09f 100644 --- a/ci_cd/tasks/docs_index.py +++ b/ci_cd/tasks/docs_index.py @@ -2,6 +2,7 @@ Create the documentation index (home) page from `README.md`. """ +import re import sys from pathlib import Path from typing import TYPE_CHECKING @@ -87,22 +88,21 @@ def create_docs_index( # pylint: disable=too-many-locals # Check if there have been any changes. # List changes if yes. - # NOTE: grep returns an exit code of 1 if it doesn't find anything - # (which will be good in this case). - # Concerning the weird last grep command see: + # NOTE: Concerning the weird regular expression, see: # http://manpages.ubuntu.com/manpages/precise/en/man1/git-status.1.html result: "Result" = context.run( # type: ignore[no-redef] f'git -C "{root_repo_path}" status --porcelain ' - f"{docs_index.relative_to(root_repo_path)} | " - "grep -E '^[? MARC][?MD]' || exit 0", + f"{docs_index.relative_to(root_repo_path)}", hide=True, ) if result.stdout: - sys.exit( - f"{Emoji.CURLY_LOOP.value} The landing page has been updated.\n\n" - "Please stage it:\n\n" - f" git add {docs_index.relative_to(root_repo_path)}" - ) + for line in result.stdout.splitlines(): + if re.match(r"^[? MARC][?MD]", line): + sys.exit( + f"{Emoji.CURLY_LOOP.value} The landing page has been updated." + "\n\nPlease stage it:\n\n" + f" git add {docs_index.relative_to(root_repo_path)}" + ) print( f"{Emoji.CHECK_MARK.value} No changes - your landing page is up-to-date !" )