test_sample_projects: Add exclude_from tests

Add support for specifying Settings.exclude_from in a sample_projects
test.

Add a new mixed_project_with_exclude_files sample project that allows us
to selectively exclude parts of an example project.

tests/sample_projects/mixed_project_with_exclude_files/.ignore

@@ -0,0 +1 @@
+/subdir1/*

tests/sample_projects/mixed_project_with_exclude_files/custom_ignore_subdir2

@@ -0,0 +1 @@
+subdir2/*

tests/sample_projects/mixed_project_with_exclude_files/expected.toml

@@ -0,0 +1,157 @@
+[project]
+# General information about a simplified project: Its name, why we test it,
+# its relation to real world projects
+name = "mixed_project_with_exclude_files"
+description = """
+    A project with multiple exclude files:
+    - ./.ignore excludes "/subdir1/*"
+    - ./custom_ignore_subdir2 excludes "subdir2/*"
+"""
+
+[experiments.default]
+description = "FawltyDeps does not obey .ignore by default"
+imports = [
+    # ./main.py:
+    "click",
+    # ./subdir1/notebook.ipynb:
+    "pandas",
+    "pytorch",
+    # ./subdir1/script.py
+    "requests",
+    # ./subdir2/notebook.ipynb:
+    "pandas",
+    "numpy",
+    # ./subdir2/script.py
+    "requests",
+    "tomli",
+    # ./subdir2/setup.py
+    "setuptools",
+]
+declared_deps = [
+    # ./pyproject.toml:
+    "numpy",
+    "setuptools",
+    "black",
+    # ./subdir1/setup.cfg:
+    "pandas",
+    "tox",
+    "pytorch",
+    # ./subdir2/setup.py:
+    "pandas",
+    "click",
+    "requests",
+    "jieba",
+]
+undeclared_deps = ["tomli"]
+unused_deps = [
+    # "black" and "tox" are in the default_ignored_unused list.
+    "jieba",
+]
+
+[experiments.exclude_from_ignore_file]
+description = "FawltyDeps obeys .ignore when asked directly"
+exclude_from = [".ignore"]
+imports = [
+    # ./main.py:
+    "click",
+    # ./subdir2/notebook.ipynb:
+    "pandas",
+    "numpy",
+    # ./subdir2/script.py
+    "requests",
+    "tomli",
+    # ./subdir2/setup.py
+    "setuptools",
+]
+declared_deps = [
+    # ./pyproject.toml:
+    "numpy",
+    "setuptools",
+    "black",
+    # ./subdir2/setup.py:
+    "pandas",
+    "click",
+    "requests",
+    "jieba",
+]
+
+# Import names in the code that do not have a matching dependency declared:
+undeclared_deps = ["tomli"]
+
+# Declared dependencies which were never `import`ed from the code:
+unused_deps = [
+    # "black" is in the default_ignored_unused list.
+    "jieba",
+]
+
+[experiments.exclude_from_other_ignore_file]
+description = "FawltyDeps obeys only custom_ignore_subdir2 when told so"
+exclude_from = ["custom_ignore_subdir2"]
+imports = [
+    # ./main.py:
+    "click",
+    # ./subdir1/notebook.ipynb:
+    "pandas",
+    "pytorch",
+    # ./subdir1/script.py
+    "requests",
+]
+declared_deps = [
+    # ./pyproject.toml:
+    "numpy",
+    "setuptools",
+    "black",
+    # ./subdir1/setup.cfg:
+    "pandas",
+    "tox",
+    "pytorch",
+]
+undeclared_deps = ["click", "requests"]
+unused_deps = [
+    # "black" and "tox" are in the default_ignored_unused list.
+    "numpy",
+    "setuptools",
+]
+
+[experiments.multiple_exclude_from]
+description = "FawltyDeps obeys both exclude_from files"
+exclude_from = ["custom_ignore_subdir2", ".ignore"]
+imports = [
+    # ./main.py:
+    "click",
+]
+declared_deps = [
+    # ./pyproject.toml:
+    "numpy",
+    "setuptools",
+    "black",
+]
+undeclared_deps = ["click"]
+unused_deps = [
+    # "black" is in the default_ignored_unused list.
+    "numpy",
+    "setuptools",
+]
+
+[experiments.combine_exclude_and_exclude_from]
+description = "Combine --exclude and --exclude-from in one run"
+exclude = [
+    "/main.*",
+    "/*.toml",
+    "!subdir1/setup.cfg",
+    "!/subdir2/notebook.ipynb"
+]
+exclude_from = ["custom_ignore_subdir2", ".ignore"]
+imports = [
+    # ./subdir2/notebook.ipynb:
+    "pandas",
+    "numpy",
+]
+declared_deps = [
+    # ./subdir1/setup.cfg:
+    "pandas",
+    "tox",
+    "pytorch",
+]
+undeclared_deps = ["numpy"]
+unused_deps = ["pytorch"]  # "tox" is in the default_ignored_unused list

tests/sample_projects/mixed_project_with_exclude_files/main.py

@@ -0,0 +1 @@
+import click

tests/sample_projects/mixed_project_with_exclude_files/pyproject.toml

@@ -0,0 +1,7 @@
+[project]
+name = "mixed"
+
+dependencies = ["numpy", "setuptools", "black"]
+
+[tool.black]
+target-version = ["py37"]

tests/sample_projects/mixed_project_with_exclude_files/subdir1/notebook.ipynb

@@ -0,0 +1,42 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": ""
+      },
+      "outputs": [],
+      "source": [
+        "import pandas"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": ""
+      },
+      "outputs": [],
+      "source": [
+        "import pytorch"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python",
+      "version": "3.10.9"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

tests/sample_projects/mixed_project_with_exclude_files/subdir1/script.py

@@ -0,0 +1,11 @@
+from requests import Request, Session
+
+pyproject_url = "https://raw.githubusercontent.com/tweag/nickel/master/Cargo.toml"
+s = Session()
+req = Request("GET", pyproject_url)
+prepped = req.prepare()
+timeout = 10
+resp = s.send(prepped, timeout=timeout)
+
+print(f"GET {pyproject_url} => {resp.status_code}")
+print(resp.text)

tests/sample_projects/mixed_project_with_exclude_files/subdir1/setup.cfg

@@ -0,0 +1,4 @@
+[options]
+install_requires = pandas
+tests_require = tox
+extras_require = pytorch

tests/sample_projects/mixed_project_with_exclude_files/subdir2/notebook.ipynb

@@ -0,0 +1,42 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": ""
+      },
+      "outputs": [],
+      "source": [
+        "import pandas"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": ""
+      },
+      "outputs": [],
+      "source": [
+        "import numpy"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python",
+      "version": "3.10.9"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

tests/sample_projects/mixed_project_with_exclude_files/subdir2/script.py

@@ -0,0 +1,20 @@
+import sys
+
+from requests import Request, Session
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+pyproject_url = "https://raw.githubusercontent.com/tweag/nickel/master/Cargo.toml"
+s = Session()
+req = Request("GET", pyproject_url)
+prepped = req.prepare()
+timeout = 10
+resp = s.send(prepped, timeout=timeout)
+
+print(f"GET {pyproject_url} => {resp.status_code}")
+cargo = tomllib.loads(resp.text)
+print()
+print(f"{cargo['package']['name']} v{cargo['package']['version']}")

tests/sample_projects/mixed_project_with_exclude_files/subdir2/setup.py

@@ -0,0 +1,7 @@
+from setuptools import setup
+
+setup(
+    name="MyLib",
+    install_requires=["pandas", "click>=1.2"],
+    extras_require={"http": ["requests"], "chinese": ["jieba"]},
+)

tests/test_sample_projects.py

@@ -61,6 +61,7 @@ class Experiment(BaseExperiment):
     pyenvs: Optional[List[str]]
     install_deps: bool
     exclude: List[str]
+    exclude_from: Optional[List[str]]
 
     @classmethod
     def from_toml(cls, name: str, data: TomlData) -> Experiment:
@@ -70,6 +71,7 @@ def from_toml(cls, name: str, data: TomlData) -> Experiment:
             pyenvs=data.get("pyenvs", None),
             install_deps=data.get("install_deps", False),
             exclude=data.get("exclude", None),
+            exclude_from=data.get("exclude_from", None),
             **cls._init_args_from_toml(name, data),
         )
 
@@ -86,6 +88,7 @@ def build_settings(self, cache: pytest.Cache) -> Settings:
             pyenvs=pyenvs,
             install_deps=self.install_deps,
             exclude=Settings().exclude if self.exclude is None else set(self.exclude),
+            exclude_from={Path(path) for path in (self.exclude_from or [])},
         )