Merge branch 'main' into issue799

Author: danielballan

.github/workflows/ci.yml

@@ -27,7 +27,6 @@ jobs:
     strategy:
       matrix:
         python-version:
-        - "3.8"
         - "3.9"
         - "3.10"
         - "3.11"

CHANGELOG.md

@@ -5,9 +5,18 @@ Write the date in place of the "Unreleased" in the case a new version is release
 
 ## Unreleased
 
+### Added
+
 - Add adapters for reading back assets with the image/jpeg and
   multipart/related;type=image/jpeg mimetypes.
-- Add a check for the `openpyxcl` module when importing excel serializer.
+- Automatic reshaping of tiff data by the adapter to account for
+  extra/missing singleton dimension
+ - Add a check for the `openpyxcl` module when importing excel serializer.
+
+### Changed
+
+- Drop support for Python 3.8, which is reached end of life
+  upstream on 7 October 2024.
 
 ## v0.1.0b10 (2024-10-11)
 

pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 maintainers = [
   { name = "Brookhaven National Laboratory", email = "[email protected]" },
 ]
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 
 # All dependencies are optional; it depends on whether you are running
 # a client or server (or both) and what data structures you care about.
@@ -23,8 +23,6 @@ classifiers = [
   "Development Status :: 4 - Beta",
   "License :: OSI Approved :: BSD License",
   "Programming Language :: Python :: 3 :: Only",
-  "Programming Language :: Python :: 3.7",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",

tiled/_tests/test_tiff.py

@@ -10,9 +10,11 @@
 from ..client import Context, from_context
 from ..client.register import IMG_SEQUENCE_EMPTY_NAME_ROOT, register
 from ..server.app import build_app
+from ..structures.array import ArrayStructure, BuiltinDtype
 from ..utils import ensure_uri
 
 COLOR_SHAPE = (11, 17, 3)
+rng = numpy.random.default_rng(12345)
 
 
 @pytest.fixture(scope="module")
@@ -21,20 +23,27 @@ def client(tmpdir_module):
     sequence_directory.mkdir()
     filepaths = []
     for i in range(3):
-        data = numpy.random.random((5, 7, 4))
+        data = rng.integers(0, 255, size=(5, 7, 4), dtype="uint8")
         filepath = sequence_directory / f"temp{i:05}.tif"
         tf.imwrite(filepath, data)
         filepaths.append(filepath)
-    color_data = numpy.random.randint(0, 255, COLOR_SHAPE, dtype="uint8")
+    color_data = rng.integers(0, 255, size=COLOR_SHAPE, dtype="uint8")
     path = Path(tmpdir_module, "color.tif")
     tf.imwrite(path, color_data)
-
     tree = MapAdapter(
         {
             "color": TiffAdapter(ensure_uri(path)),
             "sequence": TiffSequenceAdapter.from_uris(
                 [ensure_uri(filepath) for filepath in filepaths]
             ),
+            "5d_sequence": TiffSequenceAdapter.from_uris(
+                [ensure_uri(filepath) for filepath in filepaths],
+                structure=ArrayStructure(
+                    shape=(3, 1, 5, 7, 4),
+                    chunks=((1, 1, 1), (1,), (5,), (7,), (4,)),
+                    data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("uint8")),
+                ),
+            ),
         }
     )
     app = build_app(tree)
@@ -62,6 +71,27 @@ def test_tiff_sequence(client, slice_input, correct_shape):
     assert arr.shape == correct_shape
 
 
+@pytest.mark.parametrize(
+    "slice_input, correct_shape",
+    [
+        (None, (3, 1, 5, 7, 4)),
+        (..., (3, 1, 5, 7, 4)),
+        ((), (3, 1, 5, 7, 4)),
+        (0, (1, 5, 7, 4)),
+        (slice(0, 3, 2), (2, 1, 5, 7, 4)),
+        ((1, slice(0, 10), slice(0, 3), slice(0, 3)), (1, 3, 3, 4)),
+        ((slice(0, 3), 0, slice(0, 3), slice(0, 3)), (3, 3, 3, 4)),
+        ((..., 0, 0, 0, 0), (3,)),
+        ((0, slice(0, 1), slice(0, 1), slice(0, 2), ...), (1, 1, 2, 4)),
+        ((0, ..., slice(0, 2)), (1, 5, 7, 2)),
+        ((..., slice(0, 1)), (3, 1, 5, 7, 1)),
+    ],
+)
+def test_forced_reshaping(client, slice_input, correct_shape):
+    arr = client["5d_sequence"].read(slice=slice_input)
+    assert arr.shape == correct_shape
+
+
 @pytest.mark.parametrize("block_input, correct_shape", [((0, 0, 0, 0), (1, 5, 7, 4))])
 def test_tiff_sequence_block(client, block_input, correct_shape):
     arr = client["sequence"].read_block(block_input)

tiled/adapters/sequence.py

@@ -1,9 +1,12 @@
 import builtins
+import math
+import warnings
 from abc import abstractmethod
 from pathlib import Path
 from typing import Any, List, Optional, Tuple, Union
 
 import numpy as np
+from ndindex import ndindex
 from numpy._typing import NDArray
 
 from ..structures.array import ArrayStructure, BuiltinDtype
@@ -13,6 +16,47 @@
 from .type_alliases import JSON, NDSlice
 
 
+def force_reshape(arr: np.array, desired_shape: Tuple[int, ...]) -> np.array:
+    """Reshape a numpy array to match the desited shape, if possible.
+
+    Parameters
+    ----------
+
+    arr : np.array
+        The original ND array to be reshaped
+    desired_shape : Tuple[int, ...]
+        The desired shape of the resulting array
+
+    Returns
+    -------
+
+    A view of the original array
+    """
+
+    if arr.shape == desired_shape:
+        # Nothing to do here
+        return arr
+
+    if arr.size == math.prod(desired_shape):
+        if len(arr.shape) != len(desired_shape):
+            # Missing or extra singleton dimensions
+            warnings.warn(
+                f"Forcefully reshaping {arr.shape} to {desired_shape}",
+                category=RuntimeWarning,
+            )
+            return arr.reshape(desired_shape)
+        else:
+            # Some dimensions might be swapped or completely wrong
+            # TODO: needs to be treated more carefully
+            pass
+
+    warnings.warn(
+        f"Can not reshape array of {arr.shape} to match {desired_shape}; proceeding without changes",
+        category=RuntimeWarning,
+    )
+    return arr
+
+
 class FileSequenceAdapter:
     """Base adapter class for image (and other file) sequences
 
@@ -124,12 +168,12 @@ def metadata(self) -> JSON:
     def read(self, slice: Optional[NDSlice] = ...) -> NDArray[Any]:
         """Return a numpy array
 
-        Receives a sequence of values to select from a collection of image files
-        that were saved in a folder The input order is defined as: files -->
+        Receives a sequence of values to select from a collection of data files
+        that were saved in a folder. The input order is defined as: files -->
         vertical slice --> horizontal slice --> color slice --> ... read() can
         receive one value or one slice to select all the data from one file or
         a sequence of files; or it can receive a tuple (int or slice) to select
-        a more specific sequence of pixels of a group of images.
+        a more specific sequence of pixels of a group of images, for example.
 
         Parameters
         ----------
@@ -165,11 +209,15 @@ def read(self, slice: Optional[NDSlice] = ...) -> NDArray[Any]:
                     the_rest.insert(0, Ellipsis)  # Include any leading dimensions
                 elif isinstance(left_axis, builtins.slice):
                     arr = self.read(slice=left_axis)
+
+                sliced_shape = ndindex(left_axis).newshape(self.structure().shape)
+                arr = force_reshape(arr, sliced_shape)
                 arr = np.atleast_1d(arr[tuple(the_rest)])
         else:
             raise RuntimeError(f"Unsupported slice type, {type(slice)} in {slice}")
 
-        return arr
+        sliced_shape = ndindex(slice).newshape(self.structure().shape)
+        return force_reshape(arr, sliced_shape)
 
     def read_block(
         self, block: Tuple[int, ...], slice: Optional[NDSlice] = ...

tiled/client/auth.py

@@ -79,11 +79,7 @@ def sync_clear_token(self, key):
         with self._sync_lock:
             self.tokens.pop(key, None)
             filepath = self.token_directory / key
-            # filepath.unlink(missing_ok=False)  # Python 3.8+
-            try:
-                filepath.unlink()
-            except FileNotFoundError:
-                pass
+            filepath.unlink(missing_ok=False)
             self.tokens.pop(key, None)  # Clear cached value.
 
     def sync_auth_flow(self, request, attempt=0):