feature/use-in-memory-data-for-non-dask-calls (#148)

* Change get_image_data to use in memory / read_immediate

* Black formatting

* Minor update to README quick start notes

* Update 2019 -> 2020

* Change wording on chunked reading docstring

* Black formatting

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2019 Allen Institute for Cell Science
+Copyright 2020 Allen Institute for Cell Science
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 

README.md

@@ -37,12 +37,13 @@ img = AICSImage("my_file.tiff")
 img.data  # returns 6D STCZYX numpy array
 img.dims  # returns string "STCZYX"
 img.shape  # returns tuple of dimension sizes in STCZYX order
+img.get_image_data("CZYX", S=0, T=0)  # returns 4D CZYX numpy array
 
 # Get 6D STCZYX numpy array
 data = imread("my_file.tiff")
 ```
 
-### Delayed Image Slice Reading
+### Delayed Image Reading
 ```python
 from aicsimageio import AICSImage, imread_dask
 
@@ -52,7 +53,6 @@ img.dask_data  # returns 6D STCZYX dask array
 img.dims  # returns string "STCZYX"
 img.shape  # returns tuple of dimension sizes in STCZYX order
 img.size("STC")  # returns tuple of dimensions sizes for just STC
-img.get_image_data("CZYX", S=0, T=0)  # returns 4D CZYX numpy array
 img.get_image_dask_data("CZYX", S=0, T=0)  # returns 4D CZYX dask array
 
 # Read specified portion of dask array
@@ -65,6 +65,14 @@ lazy_s0t0 = lazy_data[0, 0, :]
 s0t0 = lazy_s0t0.compute()
 ```
 
+#### Quick Start Notes
+In short, if the word "dask" appears in the function or property name, the function
+utilizes delayed reading, if not, the underlying operation is backed by the image fully
+read into memory. I.E. `AICSImage.data` and `AICSImage.get_image_data` load the entire
+image into memory before performing their operation, and `AICSImage.dask_data` and
+`AICSImage.get_image_dask_data` do not load any image data until the user calls
+`compute` on the `dask.Array` object and only the requested chunk will be loaded into
+memory instead of the entire image.
 
 ### Speed up IO and Processing with Dask Clients and Clusters
 If you have already spun up a `distributed.Client` object in your Python process or

aicsimageio/aics_image.py

@@ -76,13 +76,14 @@ def __init__(
 
         Examples
         --------
-        Initialize an image and read the slices specified as a numpy array.
+        Initialize an image then read the file and return specified slices as a numpy
+        array.
 
         >>> img = AICSImage("my_file.tiff")
         ... zstack_t8 = img.get_image_data("ZYX", S=0, T=8, C=0)
 
         Initialize an image, construct a delayed dask array for certain slices, then
-        read the data.
+        read only that data.
 
         >>> img = AICSImage("my_file.czi")
         ... zstack_t8 = img.get_image_dask_data("ZYX", S=0, T=8, C=0)
@@ -405,7 +406,7 @@ def get_image_data(
         self, out_orientation: Optional[str] = None, **kwargs
     ) -> np.ndarray:
         """
-        Get specific dimension image data out of an image as a numpy array.
+        Read the image as a numpy array then return specific dimension image data.
 
         Parameters
         ----------
@@ -464,12 +465,21 @@ def get_image_data(
         -----
         * If a requested dimension is not present in the data the dimension is
           added with a depth of 1.
+        * This will preload the entire image before returning the requested data.
 
         See `aicsimageio.transforms.reshape_data` for more details.
         """
-        return self.get_image_dask_data(
-            out_orientation=out_orientation, **kwargs
-        ).compute()
+        # If no out orientation, simply return current data as dask array
+        if out_orientation is None:
+            return self.data
+
+        # Transform and return
+        return transforms.reshape_data(
+            data=self.data,
+            given_dims=self.dims,
+            return_dims=out_orientation,
+            **kwargs,
+        )
 
     def view_napari(self, rgb: bool = False, **kwargs):
         """

aicsimageio/readers/lif_reader.py

@@ -771,7 +771,10 @@ def _read_immediate(self) -> np.ndarray:
             # have the same dimensions
             # Read all data in the image
             data, _ = LifReader._get_array_from_offset(
-                self._file, self._chunk_offsets, self._chunk_lengths, self.metadata,
+                self._file,
+                self._chunk_offsets,
+                self._chunk_lengths,
+                self.metadata,
             )
 
         return data

aicsimageio/readers/reader.py

@@ -269,7 +269,7 @@ def get_image_data(
         self, out_orientation: Optional[str] = None, **kwargs
     ) -> np.ndarray:
         """
-        Get specific dimension image data out of an image as a numpy array.
+        Read the image as a numpy array then return specific dimension image data.
 
         Parameters
         ----------
@@ -328,12 +328,21 @@ def get_image_data(
         -----
         * If a requested dimension is not present in the data the dimension is
           added with a depth of 1.
+        * This will preload the entire image before returning the requested data.
 
         See `aicsimageio.transforms.reshape_data` for more details.
         """
-        return self.get_image_dask_data(
-            out_orientation=out_orientation, **kwargs
-        ).compute()
+        # If no out orientation, simply return current data as dask array
+        if out_orientation is None:
+            return self.data
+
+        # Transform and return
+        return transforms.reshape_data(
+            data=self.data,
+            given_dims=self.dims,
+            return_dims=out_orientation,
+            **kwargs,
+        )
 
     @property
     @abstractmethod

aicsimageio/tests/readers/test_arraylike_reader.py

@@ -19,7 +19,10 @@
         (da.ones((1, 1, 1)), (1, 1, 1), "ZYX"),
         (da.ones((1, 1, 1, 1)), (1, 1, 1, 1), "CZYX"),
         pytest.param(
-            "hello_word", None, None, marks=pytest.mark.raises(exceptions=TypeError),
+            "hello_word",
+            None,
+            None,
+            marks=pytest.mark.raises(exceptions=TypeError),
         ),
     ],
 )

aicsimageio/tests/readers/test_czi_reader.py

@@ -60,7 +60,14 @@
             ("S", "Y", "X"),
         ),
         # Check that Spatial Y and Spatial X dims are always added to chunk dims
-        ("s_3_t_1_c_3_z_5.czi", (1, 3, 3, 5, 325, 475), "BSCZYX", np.uint16, 0, ("S"),),
+        (
+            "s_3_t_1_c_3_z_5.czi",
+            (1, 3, 3, 5, 325, 475),
+            "BSCZYX",
+            np.uint16,
+            0,
+            ("S"),
+        ),
         (
             "variable_per_scene_dims.czi",
             (1, 1, 2, 1, 2, 1248, 1848),

aicsimageio/tests/readers/test_default_reader.py

@@ -25,7 +25,10 @@
     ],
 )
 def test_default_reader(
-    resources_dir, filename, expected_shape, expected_dims,
+    resources_dir,
+    filename,
+    expected_shape,
+    expected_dims,
 ):
     # Get file
     f = resources_dir / filename

aicsimageio/tests/readers/test_ome_tiff_reader.py

@@ -14,8 +14,18 @@
     "filename, " "expected_shape, " "expected_dims, " "select_scene",
     [
         ("s_1_t_1_c_1_z_1.ome.tiff", (325, 475), "YX", 0),
-        ("s_1_t_1_c_10_z_1.ome.tiff", (10, 1736, 1776), "CYX", 0,),
-        ("s_3_t_1_c_3_z_5.ome.tiff", (3, 5, 3, 325, 475), "SZCYX", 0,),
+        (
+            "s_1_t_1_c_10_z_1.ome.tiff",
+            (10, 1736, 1776),
+            "CYX",
+            0,
+        ),
+        (
+            "s_3_t_1_c_3_z_5.ome.tiff",
+            (3, 5, 3, 325, 475),
+            "SZCYX",
+            0,
+        ),
         pytest.param(
             "example.txt",
             None,
@@ -33,7 +43,11 @@
     ],
 )
 def test_ome_tiff_reader(
-    resources_dir, filename, expected_shape, expected_dims, select_scene,
+    resources_dir,
+    filename,
+    expected_shape,
+    expected_dims,
+    select_scene,
 ):
     # Get file
     f = resources_dir / filename

aicsimageio/tests/readers/test_tiff_reader.py

@@ -12,11 +12,41 @@
 @pytest.mark.parametrize(
     "filename, " "expected_shape, " "expected_dims, " "expected_dtype, " "select_scene",
     [
-        ("s_1_t_1_c_1_z_1.ome.tiff", (325, 475), "YX", np.uint16, 0,),
-        ("s_1_t_1_c_1_z_1.tiff", (325, 475), "YX", np.uint16, 0,),
-        ("s_1_t_1_c_10_z_1.ome.tiff", (10, 1736, 1776), "CYX", np.uint16, 0,),
-        ("s_1_t_10_c_3_z_1.tiff", (10, 3, 325, 475), "TCYX", np.uint16, 0,),
-        ("s_3_t_1_c_3_z_5.ome.tiff", (3, 5, 3, 325, 475), "SZCYX", np.uint16, 0,),
+        (
+            "s_1_t_1_c_1_z_1.ome.tiff",
+            (325, 475),
+            "YX",
+            np.uint16,
+            0,
+        ),
+        (
+            "s_1_t_1_c_1_z_1.tiff",
+            (325, 475),
+            "YX",
+            np.uint16,
+            0,
+        ),
+        (
+            "s_1_t_1_c_10_z_1.ome.tiff",
+            (10, 1736, 1776),
+            "CYX",
+            np.uint16,
+            0,
+        ),
+        (
+            "s_1_t_10_c_3_z_1.tiff",
+            (10, 3, 325, 475),
+            "TCYX",
+            np.uint16,
+            0,
+        ),
+        (
+            "s_3_t_1_c_3_z_5.ome.tiff",
+            (3, 5, 3, 325, 475),
+            "SZCYX",
+            np.uint16,
+            0,
+        ),
         pytest.param(
             "example.txt",
             None,

aicsimageio/tests/test_aics_image.py

@@ -423,14 +423,22 @@ def test_view_napari(
         (TIF_FILE, (1, 1, 1, 1, 325, 475), str),
         (OME_FILE, (1, 1, 1, 1, 325, 475), omexml.OMEXML),
         (CZI_FILE, (1, 1, 1, 1, 325, 475), _Element),
-        (LIF_FILE, (1, 1, 2, 1, 2048, 2048), Element,),
+        (
+            LIF_FILE,
+            (1, 1, 2, 1, 2048, 2048),
+            Element,
+        ),
         (MED_TIF_FILE, (1, 10, 3, 1, 325, 475), str),
         (BIG_OME_FILE, (3, 1, 3, 5, 325, 475), omexml.OMEXML),
         (BIG_CZI_FILE, (3, 1, 3, 5, 325, 475), _Element),
     ],
 )
 def test_aicsimage_serialize(
-    resources_dir, tmpdir, filename, expected_shape, expected_metadata_type,
+    resources_dir,
+    tmpdir,
+    filename,
+    expected_shape,
+    expected_metadata_type,
 ):
     """
     Test that the entire AICSImage object can be serialized - a requirement to

aicsimageio/tests/test_transforms.py

@@ -13,13 +13,37 @@
 @pytest.mark.parametrize(
     "data_shape, given_dims, return_dims, other_args, expected_shape",
     [
-        ((10, 1, 5, 6, 200, 400), "STCZYX", "CSZYX", {}, (5, 10, 6, 200, 400),),
+        (
+            (10, 1, 5, 6, 200, 400),
+            "STCZYX",
+            "CSZYX",
+            {},
+            (5, 10, 6, 200, 400),
+        ),
         ((6, 200, 400), "ZYX", "STCZYX", {}, (1, 1, 1, 6, 200, 400)),
         ((6, 200, 400), "ZYX", "ZCYSXT", {}, (6, 1, 200, 1, 400, 1)),
         ((6, 200, 400), "ZYX", "CYSXT", {"Z": 2}, (1, 200, 1, 400, 1)),
-        ((6, 200, 400), "ZYX", "ZCYSXT", {"Z": [0, 1]}, (2, 1, 200, 1, 400, 1),),
-        ((6, 200, 400), "ZYX", "ZCYSXT", {"Z": (0, 1)}, (2, 1, 200, 1, 400, 1),),
-        ((6, 200, 400), "ZYX", "ZCYSXT", {"Z": range(2)}, (2, 1, 200, 1, 400, 1),),
+        (
+            (6, 200, 400),
+            "ZYX",
+            "ZCYSXT",
+            {"Z": [0, 1]},
+            (2, 1, 200, 1, 400, 1),
+        ),
+        (
+            (6, 200, 400),
+            "ZYX",
+            "ZCYSXT",
+            {"Z": (0, 1)},
+            (2, 1, 200, 1, 400, 1),
+        ),
+        (
+            (6, 200, 400),
+            "ZYX",
+            "ZCYSXT",
+            {"Z": range(2)},
+            (2, 1, 200, 1, 400, 1),
+        ),
         (
             (6, 200, 400),
             "ZYX",
@@ -127,7 +151,12 @@
     ],
 )
 def test_reshape_data_shape(
-    array_maker, data_shape, given_dims, return_dims, other_args, expected_shape,
+    array_maker,
+    data_shape,
+    given_dims,
+    return_dims,
+    other_args,
+    expected_shape,
 ):
     data = array_maker(data_shape)
 
@@ -223,15 +252,35 @@ def test_reshape_data_values(data, given_dims, return_dims, idx_in, idx_out):
         ("ZYX", "ZYX", {"Z": slice(6, 3, -1)}, [6, 5, 4], None),
         ("ZYX", "ZYX", {"Z": slice(-1, 3, -1)}, [6, 5, 4], None),
         # Dimension selection and order swap
-        ("ZYX", "YXZ", {"Z": (0, -1)}, [0, -1], (1, 2, 0),),
-        ("ZYX", "YXZ", {"Z": range(0, 6, 2)}, [0, 2, 4], (1, 2, 0),),
+        (
+            "ZYX",
+            "YXZ",
+            {"Z": (0, -1)},
+            [0, -1],
+            (1, 2, 0),
+        ),
+        (
+            "ZYX",
+            "YXZ",
+            {"Z": range(0, 6, 2)},
+            [0, 2, 4],
+            (1, 2, 0),
+        ),
     ],
 )
 def test_reshape_data_kwargs_values(
-    data, given_dims, return_dims, other_args, getitem_ops_for_expected, transposer,
+    data,
+    given_dims,
+    return_dims,
+    other_args,
+    getitem_ops_for_expected,
+    transposer,
 ):
     actual = reshape_data(
-        data=data, given_dims=given_dims, return_dims=return_dims, **other_args,
+        data=data,
+        given_dims=given_dims,
+        return_dims=return_dims,
+        **other_args,
     )
 
     expected = data[getitem_ops_for_expected]

aicsimageio/transforms.py

@@ -215,7 +215,9 @@ def reshape_data(
 
 
 def transpose_to_dims(
-    data: types.ArrayLike, given_dims: str, return_dims: str,
+    data: types.ArrayLike,
+    given_dims: str,
+    return_dims: str,
 ) -> types.ArrayLike:
     """
     This shuffles the data dimensions from given_dims to return_dims. Each dimension