Support optional params in mimetypes with serializers (#991)

* Enabled passing mimetype format to serializers in the server

* Added mimetype when passing content from client

* Added mimetype to the serializers and to their registrations

* Formatting after rebase

* Added header check for export of csv tables. Includes test cases

* Added formatting

* whitespace

* Fixed mimetype placeholdder for xarrays

* Fixed error with deserialize_arrow

* Fixed deserializer error for awkward arrays

* Fixed error message for xdi serializer

* Revert partial changes for serialize_csv

* Fixed error message for serializing arrays with as_buffer

* Formatting

* Added parse_mimetype method forserver and serialization tasks

* Revert unwanted local pre-commit change

* Whitespace

* Avoid repetition between conditional branches.

* TST: parse_mimetype

---------

Co-authored-by: Dan Allan <[email protected]>
Co-authored-by: Dan Allan <[email protected]>

Author: jmaruland

tiled/_tests/test_array.py

@@ -202,5 +202,5 @@ def test_unparsable_nested_array_stringified(kind, context):
 
 @pytest.mark.parametrize("kind", list(array_cases))
 def test_as_buffer(kind):
-    output = as_buffer(array_cases[kind], {})
+    output = as_buffer("application/octet-stream", array_cases[kind], {})
     assert len(output) == len(bytes(output))

tiled/_tests/test_export.py

@@ -1,3 +1,4 @@
+import csv
 import json
 from pathlib import Path
 
@@ -78,6 +79,14 @@ def client():
 # can be very cloud on cloud CI VMs.
 
 
+def has_csv_header(filepath):
+    with open(filepath, "r") as csv_f:
+        sniffer = csv.Sniffer()
+        has_header = sniffer.has_header(csv_f.read(2048))
+        csv_f.seek(0)
+    return has_header
+
+
 @pytest.mark.parametrize("filename", ["numbers.csv", "image.png", "image.tiff"])
 def test_export_2d_array(client, filename, tmpdir):
     client["A"].export(Path(tmpdir, filename))
@@ -88,6 +97,14 @@ def test_export_table(client, filename, tmpdir):
     client["C"].export(Path(tmpdir, filename))
 
 
+@pytest.mark.parametrize("filename", ["numbers.csv"])
+def test_csv_mimetype_opt_params(client, filename, tmpdir):
+    client["C"].export(Path(tmpdir, filename), format="text/csv;header=absent")
+    assert not has_csv_header(Path(tmpdir, filename))
+    client["C"].export(Path(tmpdir, filename), format="text/csv;header=present")
+    assert has_csv_header(Path(tmpdir, filename))
+
+
 def test_streaming_export(client, buffer):
     "The application/json-seq format is streamed via a generator."
     client["C"].export(buffer, format="application/json-seq")

tiled/_tests/test_utils.py

@@ -8,6 +8,7 @@
     ListView,
     OneShotCachedMap,
     ensure_specified_sql_driver,
+    parse_mimetype,
     parse_time_string,
     sanitize_uri,
     walk,
@@ -339,3 +340,28 @@ def test_sanitize_uri(uri, expected_clean_uri, expected_username, expected_passw
     assert clean_uri == expected_clean_uri
     assert username == expected_username
     assert password == expected_password
+
+
+@pytest.mark.parametrize(
+    "mimetype, expected",
+    [
+        ("text/csv", ("text/csv", {})),
+        ("text/csv;header=absent", ("text/csv", {"header": "absent"})),
+        (
+            "text/csv;header=absent; charset=utf-8",
+            ("text/csv", {"header": "absent", "charset": "utf-8"}),
+        ),
+        (
+            "text/csv; header=absent; charset=utf-8",
+            ("text/csv", {"header": "absent", "charset": "utf-8"}),
+        ),
+    ],
+)
+def test_parse_valid_mimetype(mimetype, expected):
+    assert parse_mimetype(mimetype) == expected
+
+
+def test_parse_invalid_mimetype():
+    with pytest.raises(ValueError):
+        # Parameter does not have form 'key=value'
+        assert parse_mimetype("text/csv;oops")

tiled/_tests/test_xarray.py

@@ -12,6 +12,7 @@
 from ..serialization.xarray import serialize_json
 from ..server.app import build_app
 from ..structures.core import Spec
+from ..utils import APACHE_ARROW_FILE_MIME_TYPE
 from .utils import URL_LIMITS
 
 image = numpy.random.random((3, 5))
@@ -190,7 +191,9 @@ async def test_serialize_json(ds_node: DatasetAdapter):
     """
     metadata = None  # Not used
     filter_for_access = None  # Not used
-    result = await serialize_json(ds_node, metadata, filter_for_access)
+    result = await serialize_json(
+        APACHE_ARROW_FILE_MIME_TYPE, ds_node, metadata, filter_for_access
+    )
 
     result_data_keys = orjson.loads(result).keys()
     ds_coords_and_vars = set(ds_node)

tiled/client/awkward.py

@@ -43,7 +43,9 @@ def write(self, container):
                 handle_error(
                     self.context.http_client.put(
                         self.item["links"]["full"],
-                        content=bytes(to_zipped_buffers(components, {})),
+                        content=bytes(
+                            to_zipped_buffers("application/zip", components, {})
+                        ),
                         headers={"Content-Type": "application/zip"},
                     )
                 )

tiled/client/dataframe.py

@@ -222,7 +222,9 @@ def write(self, dataframe):
                 handle_error(
                     self.context.http_client.put(
                         self.item["links"]["full"],
-                        content=bytes(serialize_arrow(dataframe, {})),
+                        content=bytes(
+                            serialize_arrow(APACHE_ARROW_FILE_MIME_TYPE, dataframe, {})
+                        ),
                         headers={"Content-Type": APACHE_ARROW_FILE_MIME_TYPE},
                     )
                 )
@@ -233,7 +235,9 @@ def write_partition(self, dataframe, partition):
                 handle_error(
                     self.context.http_client.put(
                         self.item["links"]["partition"].format(index=partition),
-                        content=bytes(serialize_arrow(dataframe, {})),
+                        content=bytes(
+                            serialize_arrow(APACHE_ARROW_FILE_MIME_TYPE, dataframe, {})
+                        ),
                         headers={"Content-Type": APACHE_ARROW_FILE_MIME_TYPE},
                     )
                 )
@@ -246,7 +250,9 @@ def append_partition(self, dataframe, partition):
                 handle_error(
                     self.context.http_client.patch(
                         self.item["links"]["partition"].format(index=partition),
-                        content=bytes(serialize_arrow(dataframe, {})),
+                        content=bytes(
+                            serialize_arrow(APACHE_ARROW_FILE_MIME_TYPE, dataframe, {})
+                        ),
                         headers={"Content-Type": APACHE_ARROW_FILE_MIME_TYPE},
                     )
                 )

tiled/client/sparse.py

@@ -115,7 +115,9 @@ def write(self, coords, data):
                 handle_error(
                     self.context.http_client.put(
                         self.item["links"]["full"],
-                        content=bytes(serialize_arrow(df, {})),
+                        content=bytes(
+                            serialize_arrow(APACHE_ARROW_FILE_MIME_TYPE, df, {})
+                        ),
                         headers={"Content-Type": APACHE_ARROW_FILE_MIME_TYPE},
                     )
                 )
@@ -131,7 +133,9 @@ def write_block(self, coords, data, block):
                 handle_error(
                     self.context.http_client.put(
                         self.item["links"]["block"].format(*block),
-                        content=bytes(serialize_arrow(df, {})),
+                        content=bytes(
+                            serialize_arrow(APACHE_ARROW_FILE_MIME_TYPE, df, {})
+                        ),
                         headers={"Content-Type": APACHE_ARROW_FILE_MIME_TYPE},
                     )
                 )

tiled/examples/xdi.py

@@ -203,7 +203,7 @@ def read_xdi(data_uri, structure=None, metadata=None, specs=None, access_policy=
     )
 
 
-def write_xdi(df, metadata):
+def write_xdi(mimetype, df, metadata):
     output = io.StringIO()
 
     xdi_version = metadata.get("xdi_version")

tiled/serialization/array.py

@@ -15,7 +15,7 @@
 )
 
 
-def as_buffer(array, metadata):
+def as_buffer(mimetype, array, metadata):
     "Give back a zero-copy memoryview of the array if possible. Otherwise, copy to bytes."
     # The memoryview path fails for datetime type (and possibly some others?)
     # but it generally works for standard types like int, float, bool, str.
@@ -34,11 +34,11 @@ def as_buffer(array, metadata):
     default_serialization_registry.register(
         "array",
         "application/json",
-        lambda array, metadata: safe_json_dump(array),
+        lambda mimetype, array, metadata: safe_json_dump(array),
     )
 
 
-def serialize_csv(array, metadata):
+def serialize_csv(mimetype, array, metadata):
     if array.ndim > 2:
         raise UnsupportedShape(array.shape)
     file = io.StringIO()
@@ -61,7 +61,7 @@ def serialize_csv(array, metadata):
 )
 if modules_available("PIL"):
 
-    def save_to_buffer_PIL(array, format):
+    def save_to_buffer_PIL(mimetype, array, format):
         # The logic of which shapes are support is subtle, and we'll leave the details
         # PIL ("beg forgiveness rather than ask permission"). But we can rule out
         # anything above 3 dimensions as definitely not supported.
@@ -99,21 +99,25 @@ def array_from_buffer_PIL(buffer, format, dtype, shape):
         return numpy.asarray(image).asdtype(dtype).reshape(shape)
 
     default_serialization_registry.register(
-        "array", "image/png", lambda array, metadata: save_to_buffer_PIL(array, "png")
+        "array",
+        "image/png",
+        lambda mimetype, array, metadata: save_to_buffer_PIL("image/png", array, "png"),
     )
     default_deserialization_registry.register(
         "array",
         "image/png",
-        lambda buffer, dtype, shape: array_from_buffer_PIL(buffer, "png", dtype, shape),
+        lambda mimetype, buffer, dtype, shape: array_from_buffer_PIL(
+            buffer, "png", dtype, shape
+        ),
     )
 if modules_available("tifffile"):
 
-    def array_from_buffer_tifffile(buffer, dtype, shape):
+    def array_from_buffer_tifffile(mimetype, buffer, dtype, shape):
         from tifffile import imread
 
         return imread(buffer).astype(dtype).reshape(shape)
 
-    def save_to_buffer_tifffile(array, metadata):
+    def save_to_buffer_tifffile(mimetype, array, metadata):
         from tifffile import imwrite
 
         # Handle too *few* dimensions here, and let tifffile raise if there are too
@@ -136,7 +140,7 @@ def save_to_buffer_tifffile(array, metadata):
     )
 
 
-def serialize_html(array, metadata):
+def serialize_html(mimetype, array, metadata):
     "Try to display as image. Fall back to CSV."
     try:
         png_data = default_serialization_registry.dispatch("array", "image/png")(

tiled/serialization/awkward.py

@@ -12,7 +12,7 @@
 
 
 @default_serialization_registry.register(StructureFamily.awkward, "application/zip")
-def to_zipped_buffers(components, metadata):
+def to_zipped_buffers(mimetype, components, metadata):
     (form, length, container) = components
     file = io.BytesIO()
     # Pack multiple buffers into a zipfile, uncompressed. This enables
@@ -37,7 +37,7 @@ def from_zipped_buffers(buffer, form, length):
 
 
 @default_serialization_registry.register(StructureFamily.awkward, "application/json")
-def to_json(components, metadata):
+def to_json(mimetype, components, metadata):
     (form, length, container) = components
     file = io.StringIO()
     array = awkward.from_buffers(form, length, container)
@@ -50,7 +50,7 @@ def to_json(components, metadata):
     @default_serialization_registry.register(
         StructureFamily.awkward, APACHE_ARROW_FILE_MIME_TYPE
     )
-    def to_arrow(components, metadata):
+    def to_arrow(mimetype, components, metadata):
         import pyarrow
 
         (form, length, container) = components
@@ -66,7 +66,7 @@ def to_arrow(components, metadata):
     @default_serialization_registry.register(
         StructureFamily.awkward, "application/x-parquet"
     )
-    def to_parquet(components, metadata):
+    def to_parquet(mimetype, components, metadata):
         import pyarrow.parquet
 
         (form, length, container) = components