Make work again for arrow

Author: martindurant

fsspec/caching.py

@@ -628,7 +628,7 @@ def __init__(
         fetcher: Fetcher,
         size: int,
         data: dict[tuple[int, int], bytes] | None = None,
-        strict: bool = True,
+        strict: bool = False,
         **_: Any,
     ):
         super().__init__(blocksize, fetcher, size)
@@ -659,20 +659,30 @@ def _fetch(self, start: int | None, stop: int | None) -> bytes:
             stop = self.size
 
         out = b""
+        started = False
+        loc_old = 0
         for loc0, loc1 in sorted(self.data):
             if (loc0 <= start < loc1) and (loc0 <= stop <= loc1):
                 # entirely within the block
                 off = start - loc0
                 return self.data[(loc0, loc1)][off : off + stop - start]
+            if started and loc0 > loc_old:
+                # a gap where we need data
+                if self.strict:
+                    raise ValueError
+                out += b"\x00" * (loc0 - loc_old)
             if loc0 <= start < loc1:
                 # found the start
                 off = start - loc0
                 out = self.data[(loc0, loc1)][off : off + stop - start]
-            if start < loc0 and stop > loc1:
+                started = True
+            elif start < loc0 and stop > loc1:
                 # the whole block
                 out += self.data[(loc0, loc1)]
-            if loc0 <= stop <= loc1:
+            elif loc0 <= stop <= loc1:
+                # end block
                 return out + self.data[(loc0, loc1)][: stop - loc0]
+            loc_old = loc1
         raise ValueError
 
 

fsspec/parquet.py

@@ -19,9 +19,6 @@ class AlreadyBufferedFile(AbstractBufferedFile):
     def _fetch_range(self, start, end):
         raise NotImplementedError
 
-    def close(self):
-        pass
-
 
 def open_parquet_file(
     path,

fsspec/tests/test_parquet.py

@@ -21,7 +21,7 @@
 
 @pytest.fixture(
     params=[
-        pytest.param("fastparquet", marks=FASTPARQUET_MARK),
+        # pytest.param("fastparquet", marks=FASTPARQUET_MARK),
         pytest.param("pyarrow", marks=PYARROW_MARK),
     ]
 )
@@ -40,8 +40,6 @@ def test_open_parquet_file(
 ):
     # Pandas required for this test
     pd = pytest.importorskip("pandas")
-    if engine != "fastparquet":
-        return
     if columns == ["z"] and engine == "fastparquet":
         columns = ["z.a"]  # fastparquet is more specific
 
@@ -135,10 +133,9 @@ def test_open_parquet_file(
             result = pd.read_parquet(f, columns=columns, engine=engine)
         pd.testing.assert_frame_equal(expect, result)
     elif engine == "pyarrow":
-        # Should raise ValueError for "pyarrow"
         import pyarrow
 
-        with pytest.raises((ValueError, pyarrow.ArrowError)):
+        with pytest.raises((ValueError, pyarrow.ArrowException)):
             open_parquet_file(
                 path,
                 metadata=["Not-None"],