diff --git a/LICENSE b/LICENSE index 737cae8da..4fe6ea85a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2019 Allen Institute for Cell Science +Copyright 2020 Allen Institute for Cell Science Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.md b/README.md index 1dbb8561a..ff5c937a8 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,13 @@ img = AICSImage("my_file.tiff") img.data # returns 6D STCZYX numpy array img.dims # returns string "STCZYX" img.shape # returns tuple of dimension sizes in STCZYX order +img.get_image_data("CZYX", S=0, T=0) # returns 4D CZYX numpy array # Get 6D STCZYX numpy array data = imread("my_file.tiff") ``` -### Delayed Image Slice Reading +### Delayed Image Reading ```python from aicsimageio import AICSImage, imread_dask @@ -52,7 +53,6 @@ img.dask_data # returns 6D STCZYX dask array img.dims # returns string "STCZYX" img.shape # returns tuple of dimension sizes in STCZYX order img.size("STC") # returns tuple of dimensions sizes for just STC -img.get_image_data("CZYX", S=0, T=0) # returns 4D CZYX numpy array img.get_image_dask_data("CZYX", S=0, T=0) # returns 4D CZYX dask array # Read specified portion of dask array @@ -65,6 +65,14 @@ lazy_s0t0 = lazy_data[0, 0, :] s0t0 = lazy_s0t0.compute() ``` +#### Quick Start Notes +In short, if the word "dask" appears in the function or property name, the function +utilizes delayed reading, if not, the underlying operation is backed by the image fully +read into memory. I.E. `AICSImage.data` and `AICSImage.get_image_data` load the entire +image into memory before performing their operation, and `AICSImage.dask_data` and +`AICSImage.get_image_dask_data` do not load any image data until the user calls +`compute` on the `dask.Array` object and only the requested chunk will be loaded into +memory instead of the entire image. ### Speed up IO and Processing with Dask Clients and Clusters If you have already spun up a `distributed.Client` object in your Python process or diff --git a/aicsimageio/aics_image.py b/aicsimageio/aics_image.py index 959cca6f0..221b094d2 100644 --- a/aicsimageio/aics_image.py +++ b/aicsimageio/aics_image.py @@ -76,13 +76,14 @@ def __init__( Examples -------- - Initialize an image and read the slices specified as a numpy array. + Initialize an image then read the file and return specified slices as a numpy + array. >>> img = AICSImage("my_file.tiff") ... zstack_t8 = img.get_image_data("ZYX", S=0, T=8, C=0) Initialize an image, construct a delayed dask array for certain slices, then - read the data. + read only that data. >>> img = AICSImage("my_file.czi") ... zstack_t8 = img.get_image_dask_data("ZYX", S=0, T=8, C=0) @@ -405,7 +406,7 @@ def get_image_data( self, out_orientation: Optional[str] = None, **kwargs ) -> np.ndarray: """ - Get specific dimension image data out of an image as a numpy array. + Read the image as a numpy array then return specific dimension image data. Parameters ---------- @@ -464,12 +465,21 @@ def get_image_data( ----- * If a requested dimension is not present in the data the dimension is added with a depth of 1. + * This will preload the entire image before returning the requested data. See `aicsimageio.transforms.reshape_data` for more details. """ - return self.get_image_dask_data( - out_orientation=out_orientation, **kwargs - ).compute() + # If no out orientation, simply return current data as dask array + if out_orientation is None: + return self.data + + # Transform and return + return transforms.reshape_data( + data=self.data, + given_dims=self.dims, + return_dims=out_orientation, + **kwargs, + ) def view_napari(self, rgb: bool = False, **kwargs): """ diff --git a/aicsimageio/readers/lif_reader.py b/aicsimageio/readers/lif_reader.py index 4334a441e..5976e55cd 100644 --- a/aicsimageio/readers/lif_reader.py +++ b/aicsimageio/readers/lif_reader.py @@ -771,7 +771,10 @@ def _read_immediate(self) -> np.ndarray: # have the same dimensions # Read all data in the image data, _ = LifReader._get_array_from_offset( - self._file, self._chunk_offsets, self._chunk_lengths, self.metadata, + self._file, + self._chunk_offsets, + self._chunk_lengths, + self.metadata, ) return data diff --git a/aicsimageio/readers/reader.py b/aicsimageio/readers/reader.py index d7adcc90f..e966b7a01 100644 --- a/aicsimageio/readers/reader.py +++ b/aicsimageio/readers/reader.py @@ -269,7 +269,7 @@ def get_image_data( self, out_orientation: Optional[str] = None, **kwargs ) -> np.ndarray: """ - Get specific dimension image data out of an image as a numpy array. + Read the image as a numpy array then return specific dimension image data. Parameters ---------- @@ -328,12 +328,21 @@ def get_image_data( ----- * If a requested dimension is not present in the data the dimension is added with a depth of 1. + * This will preload the entire image before returning the requested data. See `aicsimageio.transforms.reshape_data` for more details. """ - return self.get_image_dask_data( - out_orientation=out_orientation, **kwargs - ).compute() + # If no out orientation, simply return current data as dask array + if out_orientation is None: + return self.data + + # Transform and return + return transforms.reshape_data( + data=self.data, + given_dims=self.dims, + return_dims=out_orientation, + **kwargs, + ) @property @abstractmethod diff --git a/aicsimageio/tests/readers/test_arraylike_reader.py b/aicsimageio/tests/readers/test_arraylike_reader.py index 0fcc95da1..3a0249a45 100644 --- a/aicsimageio/tests/readers/test_arraylike_reader.py +++ b/aicsimageio/tests/readers/test_arraylike_reader.py @@ -19,7 +19,10 @@ (da.ones((1, 1, 1)), (1, 1, 1), "ZYX"), (da.ones((1, 1, 1, 1)), (1, 1, 1, 1), "CZYX"), pytest.param( - "hello_word", None, None, marks=pytest.mark.raises(exceptions=TypeError), + "hello_word", + None, + None, + marks=pytest.mark.raises(exceptions=TypeError), ), ], ) diff --git a/aicsimageio/tests/readers/test_czi_reader.py b/aicsimageio/tests/readers/test_czi_reader.py index 5f359619e..cdd2ebce4 100644 --- a/aicsimageio/tests/readers/test_czi_reader.py +++ b/aicsimageio/tests/readers/test_czi_reader.py @@ -60,7 +60,14 @@ ("S", "Y", "X"), ), # Check that Spatial Y and Spatial X dims are always added to chunk dims - ("s_3_t_1_c_3_z_5.czi", (1, 3, 3, 5, 325, 475), "BSCZYX", np.uint16, 0, ("S"),), + ( + "s_3_t_1_c_3_z_5.czi", + (1, 3, 3, 5, 325, 475), + "BSCZYX", + np.uint16, + 0, + ("S"), + ), ( "variable_per_scene_dims.czi", (1, 1, 2, 1, 2, 1248, 1848), diff --git a/aicsimageio/tests/readers/test_default_reader.py b/aicsimageio/tests/readers/test_default_reader.py index 7fa095b96..1f50d41aa 100644 --- a/aicsimageio/tests/readers/test_default_reader.py +++ b/aicsimageio/tests/readers/test_default_reader.py @@ -25,7 +25,10 @@ ], ) def test_default_reader( - resources_dir, filename, expected_shape, expected_dims, + resources_dir, + filename, + expected_shape, + expected_dims, ): # Get file f = resources_dir / filename diff --git a/aicsimageio/tests/readers/test_ome_tiff_reader.py b/aicsimageio/tests/readers/test_ome_tiff_reader.py index 4e58b38f9..942636dd1 100644 --- a/aicsimageio/tests/readers/test_ome_tiff_reader.py +++ b/aicsimageio/tests/readers/test_ome_tiff_reader.py @@ -14,8 +14,18 @@ "filename, " "expected_shape, " "expected_dims, " "select_scene", [ ("s_1_t_1_c_1_z_1.ome.tiff", (325, 475), "YX", 0), - ("s_1_t_1_c_10_z_1.ome.tiff", (10, 1736, 1776), "CYX", 0,), - ("s_3_t_1_c_3_z_5.ome.tiff", (3, 5, 3, 325, 475), "SZCYX", 0,), + ( + "s_1_t_1_c_10_z_1.ome.tiff", + (10, 1736, 1776), + "CYX", + 0, + ), + ( + "s_3_t_1_c_3_z_5.ome.tiff", + (3, 5, 3, 325, 475), + "SZCYX", + 0, + ), pytest.param( "example.txt", None, @@ -33,7 +43,11 @@ ], ) def test_ome_tiff_reader( - resources_dir, filename, expected_shape, expected_dims, select_scene, + resources_dir, + filename, + expected_shape, + expected_dims, + select_scene, ): # Get file f = resources_dir / filename diff --git a/aicsimageio/tests/readers/test_tiff_reader.py b/aicsimageio/tests/readers/test_tiff_reader.py index 8274c5d21..7df8c73bf 100644 --- a/aicsimageio/tests/readers/test_tiff_reader.py +++ b/aicsimageio/tests/readers/test_tiff_reader.py @@ -12,11 +12,41 @@ @pytest.mark.parametrize( "filename, " "expected_shape, " "expected_dims, " "expected_dtype, " "select_scene", [ - ("s_1_t_1_c_1_z_1.ome.tiff", (325, 475), "YX", np.uint16, 0,), - ("s_1_t_1_c_1_z_1.tiff", (325, 475), "YX", np.uint16, 0,), - ("s_1_t_1_c_10_z_1.ome.tiff", (10, 1736, 1776), "CYX", np.uint16, 0,), - ("s_1_t_10_c_3_z_1.tiff", (10, 3, 325, 475), "TCYX", np.uint16, 0,), - ("s_3_t_1_c_3_z_5.ome.tiff", (3, 5, 3, 325, 475), "SZCYX", np.uint16, 0,), + ( + "s_1_t_1_c_1_z_1.ome.tiff", + (325, 475), + "YX", + np.uint16, + 0, + ), + ( + "s_1_t_1_c_1_z_1.tiff", + (325, 475), + "YX", + np.uint16, + 0, + ), + ( + "s_1_t_1_c_10_z_1.ome.tiff", + (10, 1736, 1776), + "CYX", + np.uint16, + 0, + ), + ( + "s_1_t_10_c_3_z_1.tiff", + (10, 3, 325, 475), + "TCYX", + np.uint16, + 0, + ), + ( + "s_3_t_1_c_3_z_5.ome.tiff", + (3, 5, 3, 325, 475), + "SZCYX", + np.uint16, + 0, + ), pytest.param( "example.txt", None, diff --git a/aicsimageio/tests/test_aics_image.py b/aicsimageio/tests/test_aics_image.py index f16fd4a91..f0f237012 100644 --- a/aicsimageio/tests/test_aics_image.py +++ b/aicsimageio/tests/test_aics_image.py @@ -423,14 +423,22 @@ def test_view_napari( (TIF_FILE, (1, 1, 1, 1, 325, 475), str), (OME_FILE, (1, 1, 1, 1, 325, 475), omexml.OMEXML), (CZI_FILE, (1, 1, 1, 1, 325, 475), _Element), - (LIF_FILE, (1, 1, 2, 1, 2048, 2048), Element,), + ( + LIF_FILE, + (1, 1, 2, 1, 2048, 2048), + Element, + ), (MED_TIF_FILE, (1, 10, 3, 1, 325, 475), str), (BIG_OME_FILE, (3, 1, 3, 5, 325, 475), omexml.OMEXML), (BIG_CZI_FILE, (3, 1, 3, 5, 325, 475), _Element), ], ) def test_aicsimage_serialize( - resources_dir, tmpdir, filename, expected_shape, expected_metadata_type, + resources_dir, + tmpdir, + filename, + expected_shape, + expected_metadata_type, ): """ Test that the entire AICSImage object can be serialized - a requirement to diff --git a/aicsimageio/tests/test_transforms.py b/aicsimageio/tests/test_transforms.py index 7c975ea2a..e14de53cb 100644 --- a/aicsimageio/tests/test_transforms.py +++ b/aicsimageio/tests/test_transforms.py @@ -13,13 +13,37 @@ @pytest.mark.parametrize( "data_shape, given_dims, return_dims, other_args, expected_shape", [ - ((10, 1, 5, 6, 200, 400), "STCZYX", "CSZYX", {}, (5, 10, 6, 200, 400),), + ( + (10, 1, 5, 6, 200, 400), + "STCZYX", + "CSZYX", + {}, + (5, 10, 6, 200, 400), + ), ((6, 200, 400), "ZYX", "STCZYX", {}, (1, 1, 1, 6, 200, 400)), ((6, 200, 400), "ZYX", "ZCYSXT", {}, (6, 1, 200, 1, 400, 1)), ((6, 200, 400), "ZYX", "CYSXT", {"Z": 2}, (1, 200, 1, 400, 1)), - ((6, 200, 400), "ZYX", "ZCYSXT", {"Z": [0, 1]}, (2, 1, 200, 1, 400, 1),), - ((6, 200, 400), "ZYX", "ZCYSXT", {"Z": (0, 1)}, (2, 1, 200, 1, 400, 1),), - ((6, 200, 400), "ZYX", "ZCYSXT", {"Z": range(2)}, (2, 1, 200, 1, 400, 1),), + ( + (6, 200, 400), + "ZYX", + "ZCYSXT", + {"Z": [0, 1]}, + (2, 1, 200, 1, 400, 1), + ), + ( + (6, 200, 400), + "ZYX", + "ZCYSXT", + {"Z": (0, 1)}, + (2, 1, 200, 1, 400, 1), + ), + ( + (6, 200, 400), + "ZYX", + "ZCYSXT", + {"Z": range(2)}, + (2, 1, 200, 1, 400, 1), + ), ( (6, 200, 400), "ZYX", @@ -127,7 +151,12 @@ ], ) def test_reshape_data_shape( - array_maker, data_shape, given_dims, return_dims, other_args, expected_shape, + array_maker, + data_shape, + given_dims, + return_dims, + other_args, + expected_shape, ): data = array_maker(data_shape) @@ -223,15 +252,35 @@ def test_reshape_data_values(data, given_dims, return_dims, idx_in, idx_out): ("ZYX", "ZYX", {"Z": slice(6, 3, -1)}, [6, 5, 4], None), ("ZYX", "ZYX", {"Z": slice(-1, 3, -1)}, [6, 5, 4], None), # Dimension selection and order swap - ("ZYX", "YXZ", {"Z": (0, -1)}, [0, -1], (1, 2, 0),), - ("ZYX", "YXZ", {"Z": range(0, 6, 2)}, [0, 2, 4], (1, 2, 0),), + ( + "ZYX", + "YXZ", + {"Z": (0, -1)}, + [0, -1], + (1, 2, 0), + ), + ( + "ZYX", + "YXZ", + {"Z": range(0, 6, 2)}, + [0, 2, 4], + (1, 2, 0), + ), ], ) def test_reshape_data_kwargs_values( - data, given_dims, return_dims, other_args, getitem_ops_for_expected, transposer, + data, + given_dims, + return_dims, + other_args, + getitem_ops_for_expected, + transposer, ): actual = reshape_data( - data=data, given_dims=given_dims, return_dims=return_dims, **other_args, + data=data, + given_dims=given_dims, + return_dims=return_dims, + **other_args, ) expected = data[getitem_ops_for_expected] diff --git a/aicsimageio/transforms.py b/aicsimageio/transforms.py index 4e9c7e87d..31388da41 100644 --- a/aicsimageio/transforms.py +++ b/aicsimageio/transforms.py @@ -215,7 +215,9 @@ def reshape_data( def transpose_to_dims( - data: types.ArrayLike, given_dims: str, return_dims: str, + data: types.ArrayLike, + given_dims: str, + return_dims: str, ) -> types.ArrayLike: """ This shuffles the data dimensions from given_dims to return_dims. Each dimension diff --git a/aicsimageio/writers/ome_tiff_writer.py b/aicsimageio/writers/ome_tiff_writer.py index 8d0f0f438..59d320edf 100644 --- a/aicsimageio/writers/ome_tiff_writer.py +++ b/aicsimageio/writers/ome_tiff_writer.py @@ -191,7 +191,7 @@ def save( tif.close() def save_slice(self, data, z=0, c=0, t=0): - """ this doesn't do the necessary functionality at this point + """this doesn't do the necessary functionality at this point TODO: * make this insert a YX slice in between two other slices inside a full diff --git a/docs/conf.py b/docs/conf.py index 3bba5e3e2..c2089ee2e 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,9 +21,10 @@ import os import sys -import aicsimageio import sphinx_rtd_theme +import aicsimageio + sys.path.insert(0, os.path.abspath("..")) @@ -59,9 +60,9 @@ # You can specify multiple suffix as a list of string: # source_suffix = { - ".rst": "restructuredtext", - ".txt": "markdown", - ".md": "markdown", + ".rst": "restructuredtext", + ".txt": "markdown", + ".md": "markdown", } # The master toctree document. @@ -69,7 +70,7 @@ # General information about the project. project = u"aicsimageio" -copyright = u'2019, Allen Institute for Cell Science' +copyright = u"2020, Allen Institute for Cell Science" author = u"Allen Institute for Cell Science" # The version info for the project you"re documenting, acts as replacement @@ -134,15 +135,12 @@ # The paper size ("letterpaper" or "a4paper"). # # "papersize": "letterpaper", - # The font size ("10pt", "11pt" or "12pt"). # # "pointsize": "10pt", - # Additional stuff for the LaTeX preamble. # # "preamble": "", - # Latex figure (float) alignment # # "figure_align": "htbp", @@ -152,9 +150,13 @@ # (source start file, target name, title, author, documentclass # [howto, manual, or own class]). latex_documents = [ - (master_doc, "aicsimageio.tex", - u"aicsimageio Documentation", - u"Allen Institute for Cell Science", "manual"), + ( + master_doc, + "aicsimageio.tex", + u"aicsimageio Documentation", + u"Allen Institute for Cell Science", + "manual", + ), ] @@ -162,11 +164,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, "aicsimageio", - u"aicsimageio Documentation", - [author], 1) -] +man_pages = [(master_doc, "aicsimageio", u"aicsimageio Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------- @@ -175,10 +173,13 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, "aicsimageio", - u"aicsimageio Documentation", - author, - "aicsimageio", - "A Python library for reading and writing image data with specific support for handling bio-formats.", - "Image Processing, Computational Biology"), + ( + master_doc, + "aicsimageio", + u"aicsimageio Documentation", + author, + "aicsimageio", + "A Python library for reading and writing image data with specific support for handling bio-formats.", + "Image Processing, Computational Biology", + ), ] diff --git a/scripts/chart_benchmarks.py b/scripts/chart_benchmarks.py index b7c68fb05..1b0894a53 100644 --- a/scripts/chart_benchmarks.py +++ b/scripts/chart_benchmarks.py @@ -78,7 +78,12 @@ def _generate_chart(results: pd.DataFrame, sorted: bool = False): return ( alt.Chart(results) .mark_circle() - .encode(x="yx_planes:Q", y="read_duration:Q", color="reader:N", column=column,) + .encode( + x="yx_planes:Q", + y="read_duration:Q", + color="reader:N", + column=column, + ) ) diff --git a/setup.py b/setup.py index dae078621..27abd8e03 100644 --- a/setup.py +++ b/setup.py @@ -90,16 +90,13 @@ *test_requirements, *setup_requirements, *dev_requirements, - *interactive_requirements - ] + *interactive_requirements, + ], } setup( author="Allen Institute for Cell Science", - author_email=( - "jacksonb@alleninstitute.org, " - "bowdenm@spu.edu" - ), + author_email=("jacksonb@alleninstitute.org, " "bowdenm@spu.edu"), classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research",