diff --git a/.github/utils/.pre-commit-config_testing.yaml b/.github/utils/.pre-commit-config_testing.yaml new file mode 100644 index 00000000..c02fd9c0 --- /dev/null +++ b/.github/utils/.pre-commit-config_testing.yaml @@ -0,0 +1,34 @@ +repos: + - repo: . + rev: HEAD + hooks: + - id: docs-api-reference + args: + - "--package-dir=ci_cd" + - "--debug" + - id: docs-landing-page + - id: update-pyproject + + - repo: local + hooks: + - id: set-version + name: Set package version + entry: "ci-cd setver" + language: python + files: "" + exclude: ^$ + types: [] + types_or: [] + exclude_types: [] + always_run: false + fail_fast: false + verbose: false + pass_filenames: false + require_serial: false + description: "Sets the specified version of specified Python package." + language_version: default + minimum_pre_commit_version: "2.16.0" + args: + - "--package-dir=ci_cd" + - "--version=0.0.0" + - "--test" diff --git a/.github/utils/run_hooks.py b/.github/utils/run_hooks.py new file mode 100755 index 00000000..ab0fc690 --- /dev/null +++ b/.github/utils/run_hooks.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Run pre-commit hooks on all files in the repository. + +File used to test running the hooks in the CI/CD pipeline independently of the shell. +""" +from __future__ import annotations + +# import platform +import subprocess # nosec +import sys + +SUCCESSFUL_FAILURES_MAPPING = { + "docs-api-reference": "The following files have been changed/added/removed:", + "docs-landing-page": "The landing page has been updated.", + "update-pyproject": "Successfully updated the following dependencies:", + "set-version": "Bumped version for ci_cd to 0.0.0.", +} + + +def main(hook: str, options: list[str]) -> None: + """Run pre-commit hooks on all files in the repository.""" + run_pre_commit = ( + "pre-commit run -c .github/utils/.pre-commit-config_testing.yaml " + "--all-files --verbose" + ) + + result = subprocess.run( + f"{run_pre_commit} {' '.join(_ for _ in options)} {hook}", + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, # nosec + ) + + if result.returncode != 0: + if SUCCESSFUL_FAILURES_MAPPING[hook] in result.stdout.decode(): + print(f"Successfully failed {hook} hook.\n\n", flush=True) + print(result.stdout.decode(), flush=True) + else: + sys.exit(result.stdout.decode()) + print(f"Successfully ran {hook} hook.\n\n", flush=True) + print(result.stdout.decode(), flush=True) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + raise sys.exit("Missing arguments") + + # "Parse" arguments + # The first argument should be the hook name + if sys.argv[1] not in SUCCESSFUL_FAILURES_MAPPING: + raise sys.exit( + f"Invalid hook name: {sys.argv[1]}\n" + "The hook name should be the first argument. Any number of hook options " + "can then follow." + ) + + try: + main( + hook=sys.argv[1], + options=sys.argv[2:] if len(sys.argv) > 2 else [], + ) + except Exception as exc: # pylint: disable=broad-except + sys.exit(str(exc)) diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml index f65ab62e..0bdd8a0e 100644 --- a/.github/workflows/_local_ci_tests.yml +++ b/.github/workflows/_local_ci_tests.yml @@ -35,12 +35,13 @@ jobs: pytest: name: pytest - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] + os: ["ubuntu-latest", "windows-latest"] steps: - name: Checkout repository @@ -66,3 +67,66 @@ jobs: with: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} + + # These jobs are mainly to test a default run of the hooks including `--pre-commit` + run_hooks: + name: Run custom pre-commit hooks + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + python-version: ["3.7", "3.10"] + os: ["ubuntu-latest", "windows-latest"] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version}} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version}} + + - name: Install Python dependencies + run: | + python -m pip install -U pip + pip install -U setuptools wheel flit + pip install -e . + pip install -U pre-commit + + # docs-api-reference + - name: Run docs-api-reference ('ci-cd create-api-reference-docs') + run: python .github/utils/run_hooks.py docs-api-reference + + - name: Run docs-api-reference ('ci-cd create-api-reference-docs') (cmd) + if: runner.os == 'Windows' + run: python .github/utils/run_hooks.py docs-api-reference + shell: cmd + + # docs-landing-page + - name: Run docs-landing-page ('ci-cd create-docs-index') + run: python .github/utils/run_hooks.py docs-landing-page + + - name: Run docs-landing-page ('ci-cd create-docs-index') (cmd) + if: runner.os == 'Windows' + run: python .github/utils/run_hooks.py docs-landing-page + shell: cmd + + # update-pyproject + - name: Run update-pyproject ('ci-cd update-deps') + run: python .github/utils/run_hooks.py update-pyproject + + - name: Run update-pyproject ('ci-cd update-deps') (cmd) + if: runner.os == 'Windows' + run: python .github/utils/run_hooks.py update-pyproject + shell: cmd + + # set-version + - name: Run 'ci-cd setver' + run: python .github/utils/run_hooks.py set-version + + - name: Run 'ci-cd setver' (cmd) + if: runner.os == 'Windows' + run: python .github/utils/run_hooks.py set-version + shell: cmd diff --git a/ci_cd/tasks/api_reference_docs.py b/ci_cd/tasks/api_reference_docs.py index c904b013..6273f998 100644 --- a/ci_cd/tasks/api_reference_docs.py +++ b/ci_cd/tasks/api_reference_docs.py @@ -10,7 +10,7 @@ import shutil import sys from collections import defaultdict -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING from invoke import task @@ -131,6 +131,12 @@ def create_api_reference_docs( # pylint: disable=too-many-locals,too-many-branc if not special_option: special_option: list[str] = [] # type: ignore[no-redef] + # Initialize user-given paths as pure POSIX paths + package_dir: list[PurePosixPath] = [PurePosixPath(_) for _ in package_dir] + root_repo_path: PurePosixPath = PurePosixPath(root_repo_path) # type: ignore[no-redef] + docs_folder: PurePosixPath = PurePosixPath(docs_folder) # type: ignore[no-redef] + full_docs_folder: list[PurePosixPath] = [PurePosixPath(_) for _ in full_docs_folder] # type: ignore[no-redef] + def write_file(full_path: Path, content: str) -> None: """Write file with `content` to `full_path`""" if full_path.exists(): @@ -141,14 +147,22 @@ def write_file(full_path: Path, content: str) -> None: del cached_content full_path.write_text(content, encoding="utf8") + if pre_commit: + # Ensure git is installed + result: "Result" = context.run("git --version", hide=True) + if result.exited != 0: + sys.exit( + "Git is not installed. Please install it before running this task." + ) + if pre_commit and root_repo_path == ".": # Use git to determine repo root - result: "Result" = context.run("git rev-parse --show-toplevel", hide=True) - root_repo_path = result.stdout.strip("\n") + result = context.run("git rev-parse --show-toplevel", hide=True) + root_repo_path = result.stdout.strip("\n") # type: ignore[no-redef] root_repo_path: Path = Path(root_repo_path).resolve() # type: ignore[no-redef] - package_dirs: list[Path] = [root_repo_path / _ for _ in package_dir] - docs_api_ref_dir = root_repo_path / docs_folder / "api_reference" + package_dirs: list[Path] = [Path(root_repo_path / _) for _ in package_dir] + docs_api_ref_dir = Path(root_repo_path / docs_folder / "api_reference") LOGGER.debug( """package_dirs: %s @@ -197,11 +211,8 @@ def write_file(full_path: Path, content: str) -> None: if debug: print("special_options_files:", special_options_files, flush=True) - if any("/" in _ for _ in unwanted_folder + unwanted_file): - sys.exit( - "Unwanted folders and files may NOT be paths. A forward slash (/) was " - "found in some of them." - ) + if any(os.sep in _ or "/" in _ for _ in unwanted_folder + unwanted_file): + sys.exit("Unwanted folders and files may NOT be paths.") pages_template = 'title: "{name}"\n' md_template = "# {name}\n\n::: {py_path}\n" @@ -295,21 +306,24 @@ def write_file(full_path: Path, content: str) -> None: package.relative_to(root_repo_path) if relative else package.name ) py_path = ( - f"{py_path_root}/{filename.stem}".replace("/", ".") + f"{py_path_root}/{filename.stem}" if str(relpath) == "." or (str(relpath) == package.name and not single_package) - else f"{py_path_root}/{relpath if single_package else relpath.relative_to(package.name)}/{filename.stem}".replace( - "/", "." - ) + else f"{py_path_root}/{relpath if single_package else relpath.relative_to(package.name)}/{filename.stem}" ) + + # Replace OS specific path separators with forward slashes before + # replacing that with dots (for Python import paths). + py_path = py_path.replace(os.sep, "/").replace("/", ".") + LOGGER.debug("filename: %s\npy_path: %s", filename, py_path) if debug: print("filename:", filename, flush=True) print("py_path:", py_path, flush=True) - relative_file_path = ( + relative_file_path = Path( str(filename) if str(relpath) == "." else str(relpath / filename) - ) + ).as_posix() # For special files we want to include EVERYTHING, even if it doesn't # have a doc-string @@ -352,22 +366,22 @@ def write_file(full_path: Path, content: str) -> None: # Check if there have been any changes. # List changes if yes. - # NOTE: grep returns an exit code of 1 if it doesn't find anything - # (which will be good in this case). - # Concerning the weird last grep command see: + # NOTE: Concerning the weird regular expression, see: # http://manpages.ubuntu.com/manpages/precise/en/man1/git-status.1.html - result: "Result" = context.run( # type: ignore[no-redef] + result = context.run( f'git -C "{root_repo_path}" status --porcelain ' - f"{docs_api_ref_dir.relative_to(root_repo_path)} | " - "grep -E '^[? MARC][?MD]' || exit 0", + f"{docs_api_ref_dir.relative_to(root_repo_path)}", hide=True, ) if result.stdout: - sys.exit( - f"{Emoji.CURLY_LOOP.value} The following files have been " - f"changed/added/removed:\n\n{result.stdout}\nPlease stage them:\n\n" - f" git add {docs_api_ref_dir.relative_to(root_repo_path)}" - ) + for line in result.stdout.splitlines(): + if re.match(r"^[? MARC][?MD]", line): + sys.exit( + f"{Emoji.CURLY_LOOP.value} The following files have been " + f"changed/added/removed:\n\n{result.stdout}\n" + "Please stage them:\n\n" + f" git add {docs_api_ref_dir.relative_to(root_repo_path)}" + ) print( f"{Emoji.CHECK_MARK.value} No changes - your API reference documentation " "is up-to-date !" diff --git a/ci_cd/tasks/docs_index.py b/ci_cd/tasks/docs_index.py index 83b57802..df8ae09f 100644 --- a/ci_cd/tasks/docs_index.py +++ b/ci_cd/tasks/docs_index.py @@ -2,6 +2,7 @@ Create the documentation index (home) page from `README.md`. """ +import re import sys from pathlib import Path from typing import TYPE_CHECKING @@ -87,22 +88,21 @@ def create_docs_index( # pylint: disable=too-many-locals # Check if there have been any changes. # List changes if yes. - # NOTE: grep returns an exit code of 1 if it doesn't find anything - # (which will be good in this case). - # Concerning the weird last grep command see: + # NOTE: Concerning the weird regular expression, see: # http://manpages.ubuntu.com/manpages/precise/en/man1/git-status.1.html result: "Result" = context.run( # type: ignore[no-redef] f'git -C "{root_repo_path}" status --porcelain ' - f"{docs_index.relative_to(root_repo_path)} | " - "grep -E '^[? MARC][?MD]' || exit 0", + f"{docs_index.relative_to(root_repo_path)}", hide=True, ) if result.stdout: - sys.exit( - f"{Emoji.CURLY_LOOP.value} The landing page has been updated.\n\n" - "Please stage it:\n\n" - f" git add {docs_index.relative_to(root_repo_path)}" - ) + for line in result.stdout.splitlines(): + if re.match(r"^[? MARC][?MD]", line): + sys.exit( + f"{Emoji.CURLY_LOOP.value} The landing page has been updated." + "\n\nPlease stage it:\n\n" + f" git add {docs_index.relative_to(root_repo_path)}" + ) print( f"{Emoji.CHECK_MARK.value} No changes - your landing page is up-to-date !" ) diff --git a/ci_cd/utils.py b/ci_cd/utils.py index 70d1e534..b88d5982 100644 --- a/ci_cd/utils.py +++ b/ci_cd/utils.py @@ -2,6 +2,7 @@ More information on `invoke` can be found at [pyinvoke.org](http://www.pyinvoke.org/). """ import logging +import platform import re from enum import Enum from pathlib import Path @@ -18,6 +19,16 @@ class Emoji(str, Enum): """Unicode strings for certain emojis.""" + def __new__(cls, value: str) -> "Emoji": + obj = str.__new__(cls, value) + if platform.system() == "Windows": + # Windows does not support unicode emojis, so we replace them with + # their corresponding unicode escape sequences + obj._value_ = value.encode("unicode_escape").decode("utf-8") + else: + obj._value_ = value + return obj + PARTY_POPPER = "\U0001f389" CHECK_MARK = "\u2714" CROSS_MARK = "\u274c" diff --git a/tests/tasks/test_api_reference_docs.py b/tests/tasks/test_api_reference_docs.py index 471cf49d..9a6487cb 100644 --- a/tests/tasks/test_api_reference_docs.py +++ b/tests/tasks/test_api_reference_docs.py @@ -479,7 +479,7 @@ def test_larger_package(tmp_path: "Path") -> None: ]: py_path = f"{package_dir.name}." + str( module_dir.relative_to(api_reference_folder) - ).replace("/", ".") + ).replace(os.sep, "/").replace("/", ".") assert (module_dir / ".pages").read_text( encoding="utf8" ) == f'title: "{module_dir.name}"\n', ( @@ -637,7 +637,7 @@ def test_larger_multi_packages(tmp_path: "Path") -> None: for module_dir in [package_dir / _ for _ in new_submodules]: py_path = f"{package_dir.name}." + str( module_dir.relative_to(package_dir) - ).replace("/", ".") + ).replace(os.sep, "/").replace("/", ".") assert (module_dir / ".pages").read_text( encoding="utf8" ) == f'title: "{module_dir.name}"\n', (