tests: add more cache tests and add clear_instance_cache fixture

Author: ap--

fsspec/conftest.py

@@ -27,16 +27,55 @@ def m():
         m.pseudo_dirs.append("")
 
 
-@pytest.fixture
+def _recursive_clear_instance_caches(*classes):
+    """
+    Clear instance caches for the given filesystem classes and their subclasses.
+    """
+    for cls in classes:
+        cls.clear_instance_cache()
+        _recursive_clear_instance_caches(*cls.__subclasses__())
+
+
+@pytest.fixture(scope="function")
+def clear_instance_cache(request):
+    """
+    Fixture to ensure empty filesystem instance caches before and after a test.
+
+    Can be used with and without indirect parametrization.
+    Defaults to clearing caches of all imported filesystem classes.
+
+    Usage:
+        @pytest.mark.parametrize(
+            "clear_instance_cache",
+            ["s3,simplecache", "ftp", "file,memory,simplecache"],
+            indirect=True,
+        )
+    """
+    from fsspec import get_filesystem_class
+    from fsspec.spec import AbstractFileSystem
+
+    try:
+        protocols = request.param.split(",")
+    except AttributeError:
+        classes = [AbstractFileSystem]
+    else:
+        classes = [get_filesystem_class(p) for p in protocols]
+
+    try:
+        _recursive_clear_instance_caches(*classes)
+        yield
+    finally:
+        _recursive_clear_instance_caches(*classes)
+
+
+@pytest.fixture(scope="function")
 def ftp_writable(tmpdir):
     """
     Fixture providing a writable FTP filesystem.
     """
     pytest.importorskip("pyftpdlib")
     from fsspec.implementations.ftp import FTPFileSystem
 
-    FTPFileSystem.clear_instance_cache()  # remove lingering connections
-    CachingFileSystem.clear_instance_cache()
     d = str(tmpdir)
     with open(os.path.join(d, "out"), "wb") as f:
         f.write(b"hello" * 10000)
@@ -53,3 +92,6 @@ def ftp_writable(tmpdir):
             shutil.rmtree(tmpdir)
         except Exception:
             pass
+        _recursive_clear_instance_caches(
+            FTPFileSystem, CachingFileSystem
+        )  # remove lingering connections

fsspec/implementations/tests/test_cached.py

@@ -1361,8 +1361,44 @@ def test_local_temp_file_put_by_list2(protocol, mocker, tmp_path) -> None:
 
 
 def test_simplecache_tokenization_independent_of_path():
+    # check that the tokenization is independent of the path
     of0 = fsspec.open("simplecache::memory://foo/bar.txt")
     of1 = fsspec.open("simplecache::memory://baz/qux.txt")
     assert of0.path != of1.path
     assert of0.fs._fs_token_ == of1.fs._fs_token_
     assert of0.fs is of1.fs
+
+
+@pytest.mark.parametrize(
+    "clear_instance_cache",
+    ["simplecache"],
+    indirect=True,
+)
+def test_simplecache_instance_cache_(clear_instance_cache):
+    cache = fsspec.get_filesystem_class("simplecache")._cache
+
+    assert len(cache) == 0
+
+    # check that the cache does not grow with multiple paths
+    fsspec.open("simplecache::memory://foo/bar.txt")
+    fsspec.open("simplecache::memory://bar/baz.txt")
+    fsspec.open("simplecache::memory://baz/qux.txt")
+    fsspec.open("simplecache::file:///foo/bar.txt")
+    fsspec.open("simplecache::memory://bar/baz.txt")
+    fsspec.open("simplecache::https://example.com/")
+
+    assert len(cache) == 3
+    assert [obj.storage_options for obj in cache.values()] == [
+        {
+            "target_protocol": "memory",
+            "target_options": {},
+        },
+        {
+            "target_protocol": "file",
+            "target_options": {},
+        },
+        {
+            "target_protocol": "https",
+            "target_options": {},
+        },
+    ]

fsspec/tests/test_core.py

@@ -20,19 +20,6 @@
 )
 
 
-@pytest.fixture(scope="function", autouse=True)
-def clear_cachingfilesystem_instance_caches():
-    # prevent test cross-contamination due to cached fs instances
-    from fsspec.implementations.cached import CachingFileSystem
-
-    classes = [CachingFileSystem]
-    while classes:
-        cls = classes.pop()
-        cls.clear_instance_cache()
-        classes.extend(cls.__subclasses__())
-    yield
-
-
 @contextmanager
 def tempzip(data=None):
     data = data or {}