From 6d88e4a9ee7dce5e6c524c88e401113b082de446 Mon Sep 17 00:00:00 2001 From: BrianWhitneyAI <94479316+BrianWhitneyAI@users.noreply.github.com> Date: Mon, 1 May 2023 10:39:03 -0700 Subject: [PATCH] Adjusting tiffile versioning (#487) * Adjusting tiffile versioning * Adding pinned version of tifffile for bfio * new tiff.series logic to mmstack in tifffile 2023.3.15 * removing codecov from setup.py * applying reshape for mmstack tiff images * removing dim overwrite from tiff_reader * override mmstack parser with is_mmstack=False in Tifffile * set default parser for tiff_glob_reader to is_mmstack=False * reverting tifffile versioning for bfio * changing range of tifffile version * removing non-bfio readers from bfio reader tests * pillow 9.5.0 breaks tests * comment on mmstack reading * Adjusting tiffile versioning * Adding pinned version of tifffile for bfio * new tiff.series logic to mmstack in tifffile 2023.3.15 * applying reshape for mmstack tiff images * removing dim overwrite from tiff_reader * override mmstack parser with is_mmstack=False in Tifffile * set default parser for tiff_glob_reader to is_mmstack=False * reverting tifffile versioning for bfio * changing range of tifffile version * removing non-bfio readers from bfio reader tests * pillow 9.5.0 breaks tests * comment on mmstack reading * pinning tifffile verison 9.4.0 * unpinning pillow, pinning imageio to 2.27.0 --- aicsimageio/readers/ome_tiff_reader.py | 18 ++++---------- aicsimageio/readers/tiff_glob_reader.py | 16 +++++++++---- aicsimageio/readers/tiff_reader.py | 24 ++++++++----------- .../test_ome_tiled_tiff_reader.py | 12 ---------- setup.py | 6 ++--- 5 files changed, 29 insertions(+), 47 deletions(-) diff --git a/aicsimageio/readers/ome_tiff_reader.py b/aicsimageio/readers/ome_tiff_reader.py index 7789c9e46..d6b6dc5a0 100644 --- a/aicsimageio/readers/ome_tiff_reader.py +++ b/aicsimageio/readers/ome_tiff_reader.py @@ -38,7 +38,6 @@ class OmeTiffReader(TiffReader): """ Wraps the tifffile and ome-types APIs to provide the same aicsimageio Reader API but for volumetric OME-TIFF images. - Parameters ---------- image: types.PathLike @@ -56,12 +55,10 @@ class OmeTiffReader(TiffReader): fs_kwargs: Dict[str, Any] Any specific keyword arguments to pass down to the fsspec created filesystem. Default: {} - Notes ----- If the OME metadata in your file isn't OME schema compilant or does not validate this will fail to read your file and raise an exception. - If the OME metadata in your file doesn't use the latest OME schema (2016-06), this reader will make a request to the referenced remote OME schema to validate. """ @@ -82,6 +79,8 @@ def _is_supported_image( with fs.open(path) as open_resource: with TiffFile(open_resource) as tiff: # Get first page description (aka the description tag in general) + # after Tifffile version 2023.3.15 mmstack images read all scenes + # into tiff.pages[0] xml = tiff.pages[0].description ome = OmeTiffReader._get_ome(xml, clean_metadata) @@ -126,7 +125,6 @@ def _is_supported_image( def _guess_ome_dim_order(tiff: TiffFile, ome: OME, scene_index: int) -> List[str]: """ Guess the dimension order based on OME metadata and actual TIFF data. - Parameters ------- tiff: TiffFile @@ -135,7 +133,6 @@ def _guess_ome_dim_order(tiff: TiffFile, ome: OME, scene_index: int) -> List[str A constructed OME object to retrieve data from. scene_index: int The current operating scene index to pull metadata from. - Returns ------- dims: List[str] @@ -188,7 +185,7 @@ def __init__( # Get ome-types object and warn of other behaviors with self._fs.open(self._path) as open_resource: - with TiffFile(open_resource) as tiff: + with TiffFile(open_resource, is_mmstack=False) as tiff: # Get and store OME self._ome = self._get_ome( tiff.pages[0].description, self.clean_metadata @@ -299,21 +296,19 @@ def _general_data_array_constructor( def _read_delayed(self) -> xr.DataArray: """ Construct the delayed xarray DataArray object for the image. - Returns ------- image: xr.DataArray The fully constructed and fully delayed image as a DataArray object. Metadata is attached in some cases as coords, dims, and attrs contains unprocessed tags and processed OME object. - Raises ------ exceptions.UnsupportedFileFormatError The file could not be read or is not supported. """ with self._fs.open(self._path) as open_resource: - with TiffFile(open_resource) as tiff: + with TiffFile(open_resource, is_mmstack=False) as tiff: # Get unprocessed metadata from tags tiff_tags = self._get_tiff_tags(tiff) @@ -346,21 +341,19 @@ def _read_delayed(self) -> xr.DataArray: def _read_immediate(self) -> xr.DataArray: """ Construct the in-memory xarray DataArray object for the image. - Returns ------- image: xr.DataArray The fully constructed and fully read into memory image as a DataArray object. Metadata is attached in some cases as coords, dims, and attrs contains unprocessed tags and processed OME object. - Raises ------ exceptions.UnsupportedFileFormatError The file could not be read or is not supported. """ with self._fs.open(self._path) as open_resource: - with TiffFile(open_resource) as tiff: + with TiffFile(open_resource, is_mmstack=False) as tiff: # Get unprocessed metadata from tags tiff_tags = self._get_tiff_tags(tiff) @@ -397,7 +390,6 @@ def physical_pixel_sizes(self) -> PhysicalPixelSizes: sizes: PhysicalPixelSizes Using available metadata, the floats representing physical pixel sizes for dimensions Z, Y, and X. - Notes ----- We currently do not handle unit attachment to these values. Please see the file diff --git a/aicsimageio/readers/tiff_glob_reader.py b/aicsimageio/readers/tiff_glob_reader.py index 249706ab3..3386f46a5 100644 --- a/aicsimageio/readers/tiff_glob_reader.py +++ b/aicsimageio/readers/tiff_glob_reader.py @@ -117,7 +117,7 @@ def _is_supported_image( ) -> bool: try: with fs.open(path) as open_resource: - with TiffFile(open_resource): + with TiffFile(open_resource, is_mmstack=False): return True except (TiffFileError, TypeError): @@ -256,7 +256,7 @@ def indexer(x: str) -> pd.Series: if single_file_shape is None: with self._fs.open(self._path) as open_resource: - with TiffFile(open_resource) as tiff: + with TiffFile(open_resource, is_mmstack=False) as tiff: self._single_file_shape = tiff.series[0].shape else: @@ -299,7 +299,9 @@ def _read_delayed(self) -> xr.DataArray: scene_files = scene_files.drop(self.scene_glob_character, axis=1) scene_nunique = scene_files.nunique() - tiff_tags = self._get_tiff_tags(TiffFile(scene_files.filename.iloc[0])) + tiff_tags = self._get_tiff_tags( + TiffFile(scene_files.filename.iloc[0], is_mmstack=False) + ) group_dims = [ x for x in scene_files.columns if x not in ["filename", *self.chunk_dims] @@ -364,7 +366,9 @@ def _read_delayed(self) -> xr.DataArray: dims = list(expanded_blocks_sizes.keys()) else: # assemble array in a single chunk - zarr_im = imread(scene_files.filename.tolist(), aszarr=True, level=0) + zarr_im = imread( + scene_files.filename.tolist(), aszarr=True, level=0, is_mmstack=False + ) darr = da.from_zarr(zarr_im).rechunk(-1) darr = darr.reshape(reshape_sizes) darr = darr.transpose(axes_order) @@ -403,7 +407,9 @@ def _read_immediate(self) -> xr.DataArray: scene_files = scene_files.drop(self.scene_glob_character, axis=1) scene_nunique = scene_files.nunique() - tiff_tags = self._get_tiff_tags(TiffFile(scene_files.filename.iloc[0])) + tiff_tags = self._get_tiff_tags( + TiffFile(scene_files.filename.iloc[0], is_mmstack=False) + ) chunk_sizes = self._get_chunk_sizes(scene_nunique) diff --git a/aicsimageio/readers/tiff_reader.py b/aicsimageio/readers/tiff_reader.py index 7a63a7b05..44a6fefd8 100644 --- a/aicsimageio/readers/tiff_reader.py +++ b/aicsimageio/readers/tiff_reader.py @@ -33,7 +33,6 @@ class TiffReader(Reader): """ Wraps the tifffile API to provide the same aicsimageio Reader API but for volumetric Tiff (and other tifffile supported) images. - Parameters ---------- image: types.PathLike @@ -127,7 +126,7 @@ def __init__( def scenes(self) -> Tuple[str, ...]: if self._scenes is None: with self._fs.open(self._path) as open_resource: - with TiffFile(open_resource) as tiff: + with TiffFile(open_resource, is_mmstack=False) as tiff: # This is non-metadata tiff, just use available series indices self._scenes = tuple( metadata_utils.generate_ome_image_id(i) @@ -163,7 +162,6 @@ def _get_image_data( """ Open a file for reading, construct a Zarr store, select data, and compute to numpy. - Parameters ---------- fs: AbstractFileSystem @@ -176,7 +174,6 @@ def _get_image_data( The image indices to retrieve. transpose_indices: List[int] The indices to transpose to prior to requesting data. - Returns ------- chunk: np.ndarray @@ -184,7 +181,12 @@ def _get_image_data( """ with fs.open(path) as open_resource: with imread( - open_resource, aszarr=True, series=scene, level=0, chunkmode="page" + open_resource, + aszarr=True, + series=scene, + level=0, + chunkmode="page", + is_mmstack=False, ) as store: arr = da.from_zarr(store) arr = arr.transpose(transpose_indices) @@ -335,7 +337,6 @@ def _create_dask_array( ) -> da.Array: """ Creates a delayed dask array for the file. - Parameters ---------- tiff: TiffFile @@ -343,7 +344,6 @@ def _create_dask_array( selected_scene_dims_list: List[str] The dimensions to use for constructing the array with. Required for managing chunked vs non-chunked dimensions. - Returns ------- image_data: da.Array @@ -445,20 +445,18 @@ def _create_dask_array( def _read_delayed(self) -> xr.DataArray: """ Construct the delayed xarray DataArray object for the image. - Returns ------- image: xr.DataArray The fully constructed and fully delayed image as a DataArray object. Metadata is attached in some cases as coords, dims, and attrs. - Raises ------ exceptions.UnsupportedFileFormatError The file could not be read or is not supported. """ with self._fs.open(self._path) as open_resource: - with TiffFile(open_resource) as tiff: + with TiffFile(open_resource, is_mmstack=False) as tiff: # Get dims from provided or guess dims = self._get_dims_for_scene(tiff) @@ -500,20 +498,18 @@ def _read_delayed(self) -> xr.DataArray: def _read_immediate(self) -> xr.DataArray: """ Construct the in-memory xarray DataArray object for the image. - Returns ------- image: xr.DataArray The fully constructed and fully read into memory image as a DataArray object. Metadata is attached in some cases as coords, dims, and attrs. - Raises ------ exceptions.UnsupportedFileFormatError The file could not be read or is not supported. """ with self._fs.open(self._path) as open_resource: - with TiffFile(open_resource) as tiff: + with TiffFile(open_resource, is_mmstack=False) as tiff: # Get dims from provided or guess dims = self._get_dims_for_scene(tiff) @@ -581,7 +577,7 @@ def _get_pixel_size( ) -> Tuple[Optional[float], Optional[float], Optional[float]]: """Return the pixel size in microns (z,y,x) for the given series in a tiff path.""" - with TiffFile(path_or_file) as tiff: + with TiffFile(path_or_file, is_mmstack=False) as tiff: tags = tiff.series[series_index].pages[0].tags if tiff.is_imagej: diff --git a/aicsimageio/tests/readers/extra_readers/test_ome_tiled_tiff_reader.py b/aicsimageio/tests/readers/extra_readers/test_ome_tiled_tiff_reader.py index 6b215dc58..7585b9c18 100644 --- a/aicsimageio/tests/readers/extra_readers/test_ome_tiled_tiff_reader.py +++ b/aicsimageio/tests/readers/extra_readers/test_ome_tiled_tiff_reader.py @@ -214,22 +214,10 @@ def test_ome_tiff_reader_large_files( @pytest.mark.parametrize( "filename, expected_reader", [ - ( - "s_1_t_1_c_1_z_1.tiff", - readers.TiffReader, - ), - ( - "s_1_t_1_c_1_z_1.ome.tiff", - readers.OmeTiffReader, - ), ( "s_1_t_1_c_1_z_1_ome_tiff_tiles.ome.tif", readers.OmeTiledTiffReader, ), - ( # Multiscene tiff - "variable_scene_shape_first_scene_pyramid.ome.tiff", - readers.OmeTiffReader, - ), ], ) def test_selected_tiff_reader( diff --git a/setup.py b/setup.py index 18aa8e21d..4a95239ec 100644 --- a/setup.py +++ b/setup.py @@ -31,8 +31,8 @@ def run(self): # "READER_TO_INSTALL" lookup table from aicsimageio/formats.py. format_libs: Dict[str, List[str]] = { "base-imageio": [ - "imageio[ffmpeg]>=2.11.0", - "Pillow>=9.3", + "imageio[ffmpeg]>=2.11.0,<2.28.0", + "Pillow>=9.3.0", ], "nd2": ["nd2[legacy]>=0.2.0"], "dv": ["mrc>=0.2.0"], @@ -108,7 +108,7 @@ def run(self): "ome-zarr>=0.6.1", "wrapt>=1.12", "resource-backed-dask-array>=0.1.0", - "tifffile>=2021.8.30,<2023.3.15", + "tifffile>=2021.8.30", "xarray>=0.16.1,<2023.02.0", "xmlschema", # no pin because it's pulled in from OME types "zarr>=2.6,<3",