tests: add more cache tests and add instance_caches fixture

Author: ap--

fsspec/conftest.py

@@ -3,11 +3,12 @@
 import subprocess
 import sys
 import time
+from collections import deque
+from collections.abc import Generator, Sequence
 
 import pytest
 
 import fsspec
-from fsspec.implementations.cached import CachingFileSystem
 
 
 @pytest.fixture()
@@ -27,16 +28,142 @@ def m():
         m.pseudo_dirs.append("")
 
 
-@pytest.fixture
+class InstanceCacheInspector:
+    """
+    Helper class to inspect instance caches of filesystem classes in tests.
+    """
+
+    @staticmethod
+    def classes_from_refs(
+        cls_reference: tuple[str | type[fsspec.AbstractFileSystem], ...],
+        /,
+        *,
+        empty_is_all: bool = True,
+    ) -> deque[type[fsspec.AbstractFileSystem]]:
+        """
+        Convert class references (strings or types) to a deque of filesystem classes.
+
+        Parameters
+        ----------
+        cls_reference:
+            Tuple of class references as strings or types.
+            Supports fqns, protocol names, or the class types themselves.
+        empty_is_all:
+            If True and no classes are specified, include all imported filesystem classes.
+
+        Returns
+        -------
+        fs_classes:
+            Deque of filesystem classes corresponding to the provided references.
+        """
+        classes: deque[type[fsspec.AbstractFileSystem]] = deque()
+
+        for ref in cls_reference:
+            if isinstance(ref, str):
+                try:
+                    cls = fsspec.get_filesystem_class(ref)
+                except ValueError:
+                    module_name, _, class_name = ref.rpartition(".")
+                    module = __import__(module_name, fromlist=[class_name])
+                    cls = getattr(module, class_name)
+                classes.append(cls)
+            else:
+                classes.append(ref)
+        if empty_is_all and not classes:
+            classes.append(fsspec.spec.AbstractFileSystem)
+        return classes
+
+    def clear(
+        self,
+        *cls_reference: str | type[fsspec.AbstractFileSystem],
+        recursive: bool = True,
+    ) -> None:
+        """
+        Clear instance caches of specified filesystem classes.
+        """
+        classes = self.classes_from_refs(cls_reference)
+        # Clear specified classes and optionally their subclasses
+        while classes:
+            cls = classes.popleft()
+            cls.clear_instance_cache()
+            if recursive:
+                subclasses = cls.__subclasses__()
+                classes.extend(subclasses)
+
+    def gather_counts(
+        self,
+        *cls_reference: str | type[fsspec.AbstractFileSystem],
+        omit_zero: bool = True,
+        recursive: bool = True,
+    ) -> dict[str, int]:
+        """
+        Gather counts of filesystem instances in the instance caches of all loaded classes.
+
+        Parameters
+        ----------
+        cls_reference:
+            class references as strings or types.
+        omit_zero:
+            Whether to omit instance types with no cached instances.
+        recursive:
+            Whether to include subclasses of the specified classes.
+        """
+        out: dict[str, int] = {}
+        classes = self.classes_from_refs(cls_reference)
+        while classes:
+            cls = classes.popleft()
+            count = len(cls._cache)
+            # note: skip intermediate AbstractFileSystem subclasses
+            #   if they proxy the protocol attribute via a property.
+            if isinstance(cls.protocol, (Sequence, str)):
+                key = cls.protocol if isinstance(cls.protocol, str) else cls.protocol[0]
+                if count or not omit_zero:
+                    out[key] = count
+            if recursive:
+                subclasses = cls.__subclasses__()
+                classes.extend(subclasses)
+        return out
+
+
+@pytest.fixture(scope="function", autouse=True)
+def instance_caches() -> Generator[InstanceCacheInspector, None, None]:
+    """
+    Fixture to ensure empty filesystem instance caches before and after a test.
+
+    Used by default for all tests.
+    Clears caches of all imported filesystem classes.
+    Can be used to write test assertions about instance caches.
+
+    Usage:
+
+        def test_something(instance_caches):
+            # Test code here
+            fsspec.open("file://abc")
+            fsspec.open("memory://foo/bar")
+
+            # Test assertion
+            assert instance_caches.gather_counts() == {"file": 1, "memory": 1}
+
+    Returns
+    -------
+    instance_caches: An instance cache inspector for clearing and inspecting caches.
+    """
+    ic = InstanceCacheInspector()
+
+    ic.clear()
+    try:
+        yield ic
+    finally:
+        ic.clear()
+
+
+@pytest.fixture(scope="function")
 def ftp_writable(tmpdir):
     """
     Fixture providing a writable FTP filesystem.
     """
     pytest.importorskip("pyftpdlib")
-    from fsspec.implementations.ftp import FTPFileSystem
 
-    FTPFileSystem.clear_instance_cache()  # remove lingering connections
-    CachingFileSystem.clear_instance_cache()
     d = str(tmpdir)
     with open(os.path.join(d, "out"), "wb") as f:
         f.write(b"hello" * 10000)

fsspec/implementations/tests/test_cached.py

@@ -1361,8 +1361,30 @@ def test_local_temp_file_put_by_list2(protocol, mocker, tmp_path) -> None:
 
 
 def test_simplecache_tokenization_independent_of_path():
+    # check that the tokenization is independent of the path
     of0 = fsspec.open("simplecache::memory://foo/bar.txt")
     of1 = fsspec.open("simplecache::memory://baz/qux.txt")
     assert of0.path != of1.path
     assert of0.fs._fs_token_ == of1.fs._fs_token_
     assert of0.fs is of1.fs
+
+
+def test_simplecache_instance_cache(instance_caches):
+    # check that the simplecache instance cache does not grow with every unique path
+
+    assert instance_caches.gather_counts() == {}
+
+    # check that the cache does not grow with multiple paths
+    fsspec.open("simplecache::memory://foo/bar.txt")
+    fsspec.open("simplecache::memory://bar/baz.txt")
+    fsspec.open("simplecache::memory://baz/qux.txt")
+    fsspec.open("simplecache::file:///foo/bar.txt")
+    fsspec.open("simplecache::memory://bar/baz.txt")
+    fsspec.open("simplecache::https://example.com/")
+
+    assert instance_caches.gather_counts() == {
+        "simplecache": 3,
+        "memory": 1,
+        "file": 1,
+        "https": 1,
+    }

fsspec/tests/test_core.py

@@ -20,19 +20,6 @@
 )
 
 
-@pytest.fixture(scope="function", autouse=True)
-def clear_cachingfilesystem_instance_caches():
-    # prevent test cross-contamination due to cached fs instances
-    from fsspec.implementations.cached import CachingFileSystem
-
-    classes = [CachingFileSystem]
-    while classes:
-        cls = classes.pop()
-        cls.clear_instance_cache()
-        classes.extend(cls.__subclasses__())
-    yield
-
-
 @contextmanager
 def tempzip(data=None):
     data = data or {}