From 10a33747bc8d12be866e4098896b31848bb2de28 Mon Sep 17 00:00:00 2001 From: Johan Herland Date: Mon, 16 Sep 2024 15:15:02 +0200 Subject: [PATCH] Add environment.yml support to the rest of FawltyDeps Now that we can parse dependency declarations from environment.yml files, we need to expose this functionality in our docs + CLI, as well as automatically find environment.yml files while traversing projects. --- README.md | 5 +++-- fawltydeps/extract_deps.py | 4 ++++ fawltydeps/types.py | 1 + tests/conftest.py | 13 ++++++++++++- tests/test_cmdline.py | 2 ++ tests/test_cmdline_options.py | 1 + tests/test_deps_parser_determination.py | 5 +++++ tests/test_extract_deps_success.py | 15 +++++++++++++++ 8 files changed, 43 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ff766658..7c2088f6 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ dependencies. A number of file formats are supported: `extras_require` arguments) - `setup.cfg` - `pixi.toml` +- `environment.yml` The `--deps` option accepts a space-separated list of files or directories. Each file will be parsed for declared dependencies; each directory will @@ -437,8 +438,8 @@ Here is a complete list of configuration directives we support: in the repository. - `deps_parser_choice`: Manually select which format to use for parsing declared dependencies. Must be one of `"requirements.txt"`, `"setup.py"`, - `"setup.cfg"`, `"pyproject.toml"`, `"pixi.toml"`, or leave it unset - (i.e. the default) for auto-detection (based on filename). + `"setup.cfg"`, `"pyproject.toml"`, `"pixi.toml"`, `"environment.yml"`, or + leave it unset (i.e. the default) for auto-detection (based on filename). - `install-deps`: Automatically install Python dependencies gathered with FawltyDeps into a temporary virtual environment. This will use `uv` or `pip` to download and install packages from PyPI by default. diff --git a/fawltydeps/extract_deps.py b/fawltydeps/extract_deps.py index ef2d3a8a..4b96f668 100644 --- a/fawltydeps/extract_deps.py +++ b/fawltydeps/extract_deps.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Callable, Iterable, Iterator, NamedTuple, Optional +from fawltydeps.extract_deps_environment_yml import parse_environment_yml from fawltydeps.extract_deps_pixi import parse_pixi_toml from fawltydeps.extract_deps_pyproject import parse_pyproject_toml from fawltydeps.extract_deps_requirements import parse_requirements_txt @@ -53,6 +54,9 @@ def first_applicable_parser(path: Path) -> Optional[ParserChoice]: ParserChoice.PIXI_TOML: ParsingStrategy( lambda path: path.name == "pixi.toml", parse_pixi_toml ), + ParserChoice.ENVIRONMENT_YML: ParsingStrategy( + lambda path: path.name == "environment.yml", parse_environment_yml + ), } diff --git a/fawltydeps/types.py b/fawltydeps/types.py index da086b17..4763960b 100644 --- a/fawltydeps/types.py +++ b/fawltydeps/types.py @@ -45,6 +45,7 @@ class ParserChoice(Enum): SETUP_CFG = "setup.cfg" PYPROJECT_TOML = "pyproject.toml" PIXI_TOML = "pixi.toml" + ENVIRONMENT_YML = "environment.yml" def __str__(self) -> str: return self.value diff --git a/tests/conftest.py b/tests/conftest.py index 18f7bbf5..7ee936c4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -164,7 +164,7 @@ def fake_project(write_tmp_files, fake_venv): # noqa: C901 lists of strings (extras/optional deps). The dependencies will be written into associated files, formatted according to the filenames (must be one of requirements.txt, setup.py, - setup.cfg, pyproject.toml, or pixi.toml). + setup.cfg, pyproject.toml, pixi.toml, or environment.yml). - extra_file_contents: a dict with extra files and their associated contents to be forwarded directly to write_tmp_files(). @@ -231,6 +231,16 @@ def format_pixi_toml(deps: Deps, extras: ExtraDeps) -> str: ret += "\n".join(f'{dep} = "*"' for dep in deps) return ret + def format_environment_yml(deps: Deps, no_extras: ExtraDeps) -> str: + assert not no_extras # not supported + return dedent( + """\ + name: MyLib + + dependencies: + """ + ) + "".join(f" - {dep}\n" for dep in deps) + def format_deps( filename: str, all_deps: Union[Deps, Tuple[Deps, ExtraDeps]] ) -> str: @@ -244,6 +254,7 @@ def format_deps( "setup.cfg": format_setup_cfg, "pyproject.toml": format_pyproject_toml, "pixi.toml": format_pixi_toml, + "environment.yml": format_environment_yml, } formatter = formatters.get(Path(filename).name, format_requirements_txt) return formatter(deps, extras) diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py index ea8c67ce..fb9a2e41 100644 --- a/tests/test_cmdline.py +++ b/tests/test_cmdline.py @@ -495,6 +495,7 @@ def test_list_sources__in_varied_project__lists_all_files(fake_project): "pyproject.toml": ["foo"], "setup.py": ["foo"], "setup.cfg": ["foo"], + "environment.yml": ["foo"], }, fake_venvs={"my_venv": {}}, ) @@ -512,6 +513,7 @@ def test_list_sources__in_varied_project__lists_all_files(fake_project): "pyproject.toml", "setup.py", "setup.cfg", + "environment.yml", str(_site_packages), ] ] diff --git a/tests/test_cmdline_options.py b/tests/test_cmdline_options.py index cc0c50d8..76c6389a 100644 --- a/tests/test_cmdline_options.py +++ b/tests/test_cmdline_options.py @@ -56,6 +56,7 @@ "setup.cfg", "pyproject.toml", "pixi.toml", + "environment.yml", } example_python_stdin = dedent( """\ diff --git a/tests/test_deps_parser_determination.py b/tests/test_deps_parser_determination.py index f62e594e..5a1321c9 100644 --- a/tests/test_deps_parser_determination.py +++ b/tests/test_deps_parser_determination.py @@ -30,6 +30,7 @@ ("setup.cfg", ParserChoice.SETUP_CFG), ("pyproject.toml", ParserChoice.PYPROJECT_TOML), ("pixi.toml", ParserChoice.PIXI_TOML), + ("environment.yml", ParserChoice.ENVIRONMENT_YML), ("anything_else", None), # in subdir: (str(Path("sub", "requirements.txt")), ParserChoice.REQUIREMENTS_TXT), @@ -37,6 +38,7 @@ (str(Path("sub", "setup.cfg")), ParserChoice.SETUP_CFG), (str(Path("sub", "pyproject.toml")), ParserChoice.PYPROJECT_TOML), (str(Path("sub", "pixi.toml")), ParserChoice.PIXI_TOML), + (str(Path("sub", "environment.yml")), ParserChoice.ENVIRONMENT_YML), (str(Path("sub", "anything_else")), None), # TODO: Make these absolute paths? (str(Path("abs", "requirements.txt")), ParserChoice.REQUIREMENTS_TXT), @@ -44,6 +46,7 @@ (str(Path("abs", "setup.cfg")), ParserChoice.SETUP_CFG), (str(Path("abs", "pyproject.toml")), ParserChoice.PYPROJECT_TOML), (str(Path("abs", "pixi.toml")), ParserChoice.PIXI_TOML), + (str(Path("abs", "environment.yml")), ParserChoice.ENVIRONMENT_YML), (str(Path("abs", "anything_else")), None), # using dep file name as a directory name is not supported: (str(Path("requirements.txt", "wat")), None), @@ -51,6 +54,7 @@ (str(Path("setup.cfg", "wat")), None), (str(Path("pyproject.toml", "wat")), None), (str(Path("pixi.toml", "wat")), None), + (str(Path("environment.yml", "wat")), None), # variations that all map to requirements.txt parser; ("requirements-dev.txt", ParserChoice.REQUIREMENTS_TXT), ("test-requirements.txt", ParserChoice.REQUIREMENTS_TXT), @@ -72,6 +76,7 @@ def test_first_applicable_parser(path, expect_choice): ParserChoice.SETUP_CFG: "setup.cfg", ParserChoice.PYPROJECT_TOML: "pyproject.toml", ParserChoice.PIXI_TOML: "pixi.toml", + ParserChoice.ENVIRONMENT_YML: "environment.yml", } PARSER_CHOICE_FILE_NAME_MISMATCH_GRID = { pc: [fn for _pc, fn in PARSER_CHOICE_FILE_NAME_MATCH_GRID.items() if pc != _pc] diff --git a/tests/test_extract_deps_success.py b/tests/test_extract_deps_success.py index fc10fa7c..63bf4ccd 100644 --- a/tests/test_extract_deps_success.py +++ b/tests/test_extract_deps_success.py @@ -721,6 +721,21 @@ def test_find_and_parse_sources__project_with_pixi_toml__returns_list(fake_proje assert_unordered_equivalence(actual, expect) +def test_find_and_parse_sources__project_with_environment_yml__returns_list( + fake_project, +): + tmp_path = fake_project( + files_with_declared_deps={ + "environment.yml": ["numpy", "pandas"], # dependencies + }, + ) + expect = ["numpy", "pandas"] + settings = Settings(code=set(), deps={tmp_path}) + deps_sources = list(find_sources(settings, {DepsSource})) + actual = collect_dep_names(parse_sources(deps_sources)) + assert_unordered_equivalence(actual, expect) + + def test_find_and_parse_sources__project_with_setup_cfg__returns_list(fake_project): tmp_path = fake_project( files_with_declared_deps={