Improve usability. Test.

Author: danielballan

tiled/_tests/test_writing.py

@@ -121,37 +121,33 @@ def test_write_array_chunked(tree):
         assert result.specs == specs
 
 
-def test_write_array_append(tree):
+def test_extend_array(tree):
+    "Extend an array with additional data, expanding its shape."
     with Context.from_app(
         build_app(tree, validation_registry=validation_registry)
     ) as context:
         client = from_context(context)
 
-        a = numpy.ones((1, 5, 5))
-        new_chunk = numpy.ones((1, 5, 5)) * 2
-        full_array = numpy.concatenate((a, new_chunk), axis=0)
-        metadata = {"scan_id": 1, "method": "A"}
-        specs = [Spec("SomeSpec")]
-
-        with record_history() as history:
-            new_arr = client.write_array(a, metadata=metadata, specs=specs)
-            new_arr.append_block(new_chunk, 0)
-        # one request for metadata, one for data
-        assert len(history.requests) == 1 + 1 + 1
-
-        results = client.search(Key("scan_id") == 1)
-        result = results.values().first()
-        result_array = result.read()
-        assert result.shape == (2, 5, 5)  # does the database have the right shape?
-        assert result_array.shape == (
-            2,
-            5,
-            5,
-        )  # does the array over the wire have the right shape?
-
-        numpy.testing.assert_equal(result_array, full_array)
-        assert result.metadata == metadata
-        assert result.specs == specs
+        a = numpy.ones((3, 2, 2))
+        new_data = numpy.ones((1, 2, 2)) * 2
+        full_array = numpy.concatenate((a, new_data), axis=0)
+
+        # Upload a (3, 2, 2) array.
+        ac = client.write_array(a)
+        assert ac.shape == a.shape
+
+        # Patching data into a region beyond the current extent of the array
+        # raises a ValueError (catching a 409 from the server).
+        with pytest.raises(ValueError):
+            ac.patch(new_data, slice=slice(3, 4))
+        # With extend=True, the array is expanded.
+        ac.patch(new_data, slice=slice(3, 4), extend=True)
+        # The local cache of the structure is updated.
+        assert ac.shape == full_array.shape
+        actual = ac.read()
+        # The array has the expected shape and data.
+        assert actual.shape == full_array.shape
+        numpy.testing.assert_equal(actual, full_array)
 
 
 def test_write_dataframe_full(tree):

tiled/adapters/zarr.py

@@ -15,7 +15,7 @@
 from ..structures.array import ArrayStructure
 from ..structures.core import Spec, StructureFamily
 from ..type_aliases import JSON, NDSlice
-from ..utils import node_repr, path_from_uri
+from ..utils import Conflicts, node_repr, path_from_uri
 from .array import ArrayAdapter, slice_and_shape_from_block_and_chunks
 from .protocols import AccessPolicy
 
@@ -221,7 +221,10 @@ async def patch(
                 # Resize the Zarr array to accommodate new data
                 self._array.resize(new_shape_tuple)
             else:
-                raise ValueError(f"Slice does not fit into array shape {current_shape}")
+                raise Conflicts(
+                    f"Slice {slice} does not fit into array shape {current_shape}. "
+                    f"Use ?extend=true to extend array dimension to fit."
+                )
         self._array[slice] = data
         new_chunks = []
         # Zarr has regularly-sized chunks, so no user input is required to

tiled/catalog/adapter.py

@@ -1070,6 +1070,7 @@ async def patch(self, *args, **kwargs):
             data_source.structure_id = new_structure_id
             db.add(data_source)
             await db.commit()
+            return structure_dict
 
 
 class CatalogAwkwardAdapter(CatalogNodeAdapter):

tiled/client/array.py

@@ -2,9 +2,11 @@
 
 import dask
 import dask.array
+import httpx
 import numpy
 from numpy.typing import NDArray
 
+from ..structures.core import STRUCTURE_TYPES
 from ..type_aliases import NDSlice
 from .base import BaseClient
 from .utils import export_util, handle_error, params_from_slice
@@ -196,14 +198,23 @@ def patch(self, array: NDArray, slice: NDSlice, extend=False):
         params = params_from_slice(slice)
         params["shape"] = ",".join(map(str, array_.shape))
         params["extend"] = bool(extend)
-        handle_error(
-            self.context.http_client.patch(
-                self.item["links"]["full"],
-                content=array_.tobytes(),
-                headers={"Content-Type": "application/octet-stream"},
-                params=params,
-            )
+        response = self.context.http_client.patch(
+            self.item["links"]["full"],
+            content=array_.tobytes(),
+            headers={"Content-Type": "application/octet-stream"},
+            params=params,
         )
+        if response.status_code == httpx.codes.CONFLICT:
+            raise ValueError(
+                f"Slice {slice} does not fit within current array shape. "
+                "Pass keyword argument extend=True to extend the array "
+                "dimensions to fit."
+            )
+        handle_error(response)
+        # Update cached structure.
+        new_structure = response.json()
+        structure_type = STRUCTURE_TYPES[self.structure_family]
+        self._structure = structure_type.from_json(new_structure)
 
     def __getitem__(self, slice):
         return self.read(slice)

tiled/client/base.py

@@ -184,9 +184,10 @@ def refresh(self):
             )
         ).json()
         self._item = content["data"]
-        attributes = self._item["attributes"]
-        structure_type = STRUCTURE_TYPES[attributes["structure_family"]]
-        self._structure = structure_type.from_json(attributes["structure"])
+        structure_type = STRUCTURE_TYPES[self.structure_family]
+        self._structure = structure_type.from_json(
+            self._item["attributes"]["structure"]
+        )
         return self
 
     @property

tiled/server/router.py

@@ -1316,8 +1316,8 @@ async def patch_array_full(
     media_type = request.headers["content-type"]
     deserializer = deserialization_registry.dispatch("array", media_type)
     data = await ensure_awaitable(deserializer, body, dtype, shape)
-    await ensure_awaitable(entry.patch, data, slice, extend)
-    return json_or_msgpack(request, None)
+    structure = await ensure_awaitable(entry.patch, data, slice, extend)
+    return json_or_msgpack(request, structure)
 
 
 @router.put("/table/full/{path:path}")