Reapply GS-specific modifications

Author: DavidStirling

cellprofiler_core/analysis/_runner.py

@@ -672,7 +672,8 @@ def start_workers(cls, num=None):
             # closed, the subprocess exits.
             if hasattr(sys, "frozen"):
                 if sys.platform == "darwin":
-                    executable = os.path.join(os.path.dirname(sys.executable), "cp")
+                    executable = os.path.join(os.path.dirname(sys.executable),
+                                              "cellprofilerapp")
                     args = [executable] + aw_args
                 elif sys.platform.startswith("linux"):
                     aw_path = os.path.join(os.path.dirname(__file__), "__init__.py")

cellprofiler_core/constants/image.py

@@ -103,4 +103,4 @@
 SUB_ALL = "All"
 SUB_SOME = "Some"
 FILE_SCHEME = "file:"
-PASSTHROUGH_SCHEMES = ("http", "https", "ftp", "omero", "s3")
+PASSTHROUGH_SCHEMES = ("http", "https", "ftp", "omero", "s3", "omero-3d")

cellprofiler_core/image/abstract_image/file/_file_image.py

@@ -9,16 +9,23 @@
 import numpy
 import skimage.io
 
+import requests
+from PIL import Image as PilImage
+from io import BytesIO
+from posixpath import join as urljoin
+
 import cellprofiler_core.preferences
 from .._abstract_image import AbstractImage
 from ..._image import Image
 from ....utilities.image import is_numpy_file
 from ....utilities.image import is_matlab_file
+from ....utilities.image import is_omero3d_path
 from ....utilities.image import loadmat
 from ....utilities.image import load_data_file
 from ....utilities.image import generate_presigned_url
 from ....constants.image import FILE_SCHEME, PASSTHROUGH_SCHEMES
 from ....utilities.pathname import pathname2url, url2pathname
+from ....utilities.zarr import get_zarr_reader
 
 
 class FileImage(AbstractImage):
@@ -35,6 +42,8 @@ def __init__(
         channel=None,
         volume=False,
         spacing=None,
+        z=None,
+        t=None,
     ):
         """
         :param name: Name of image to be provided
@@ -84,6 +93,8 @@ def __init__(
         self.__index = index
         self.__volume = volume
         self.__spacing = spacing
+        self.z_index = z if z is not None else 0
+        self.t_index = t if t is not None else 0
         self.scale = None
 
     @property
@@ -176,6 +187,8 @@ def cache_file(self):
                 )
             finally:
                 os.close(tempfd)
+        elif url.lower().endswith('.zarr'):
+            self.__cached_file = url
         else:
             from bioformats.formatreader import get_image_reader
 
@@ -185,6 +198,8 @@ def cache_file(self):
         return True
 
     def get_full_name(self):
+        if is_omero3d_path(self.__url):
+            return self.get_url()
         self.cache_file()
         if self.__is_cached:
             return self.__cached_file
@@ -203,8 +218,12 @@ def get_md5_hash(self, measurements):
         #
         # Cache the MD5 hash on the image reader
         #
-        if is_matlab_file(self.__filename) or is_numpy_file(self.__filename):
+        if (is_matlab_file(self.__filename) or
+                is_numpy_file(self.__filename) or
+                is_omero3d_path(self.get_url())):
             rdr = None
+        elif self.get_url().endswith('.zarr'):
+            rdr = get_zarr_reader(None, url=self.get_url())
         else:
             from bioformats.formatreader import get_image_reader
 
@@ -273,6 +292,8 @@ def __set_image(self):
             url = self.get_url()
             if url.lower().startswith("omero:"):
                 rdr = get_image_reader(self.get_name(), url=url)
+            elif url.lower().endswith('.zarr'):
+                rdr = get_zarr_reader(self.get_name(), url=url)
             else:
                 rdr = get_image_reader(self.get_name(), url=self.get_url())
             if numpy.isscalar(self.index) or self.index is None:
@@ -283,6 +304,8 @@ def __set_image(self):
                     rescale=self.rescale if isinstance(self.rescale, bool) else False,
                     wants_max_intensity=True,
                     channel_names=channel_names,
+                    z=self.z_index,
+                    t=self.t_index,
                 )
             else:
                 # It's a stack
@@ -305,6 +328,8 @@ def __set_image(self):
                         rescale=self.rescale if isinstance(self.rescale, bool) else False,
                         wants_max_intensity=True,
                         channel_names=channel_names,
+                        z=self.z_index,
+                        t=self.t_index,
                     )
                     stack.append(img)
                 img = numpy.dstack(stack)
@@ -330,10 +355,61 @@ def provide_image(self, image_set):
 
     def __set_image_volume(self):
         pathname = url2pathname(self.get_url())
-
+        print("Using pathname: {} for url {}".format(pathname, self.get_url()))
         # Volume loading is currently limited to tiffs/numpy files only
         if is_numpy_file(self.__filename):
             data = numpy.load(pathname)
+        elif is_omero3d_path(self.__url):
+            scheme = 'omero-3d:'
+            url = self.__url.split(scheme)[1]
+            parsed_url = urllib.parse.urlparse(url)
+            query_params = urllib.parse.parse_qs(parsed_url.query)
+            zmin = int(query_params['zmin'][0])
+            zmax = int(query_params['zmax'][0])
+            width = int(query_params['width'][0])
+            height = int(query_params['height'][0])
+            image_id = query_params['imageid'][0]
+            channel = query_params['c'][0]
+            stack = numpy.ndarray((zmax - zmin + 1, height, width))
+            for i in range(zmin, zmax + 1):
+                path = urljoin('/tile', image_id, str(i), channel, '0')
+                url = urllib.parse.urlunparse((
+                    parsed_url.scheme,
+                    parsed_url.netloc,
+                    path,
+                    '',
+                    parsed_url.query,
+                    ''
+                ))
+                print("Requesting URL: {}".format(url))
+                timeout = 2
+                response = None
+                while timeout < 500:
+                    try:
+                        response = requests.get(url, timeout=timeout)
+                    except Exception:
+                        print('Get %s with timeout %s sec failed' % (
+                            url, timeout))
+                        timeout = timeout**2
+                    else:
+                        break
+                if response is None:
+                    raise Exception('Failed to retrieve data from URL')
+                image_bytes = BytesIO(response.content)
+                image = PilImage.open(image_bytes)
+                stack[i - zmin, :, :] = image
+            data = stack
+        elif pathname.endswith('.zarr'):
+            rdr = get_zarr_reader(self.get_name(), url=self.get_url())
+            data = rdr.read(
+                c=self.channel,
+                series=self.series,
+                index=None,
+                rescale=False,
+                wants_max_intensity=False,
+                z=None,
+                t=self.t_index,
+            )
         else:
             data = imageio.volread(pathname)
 

cellprofiler_core/image/abstract_image/file/url/_color_image.py

@@ -7,7 +7,7 @@ class ColorImage(URLImage):
     """Provide a color image, tripling a monochrome plane if needed"""
 
     def __init__(
-        self, name, url, series, index, rescale=True, volume=False, spacing=None
+        self, name, url, series, index, rescale=True, volume=False, spacing=None, z=None, t=None,
     ):
         URLImage.__init__(
             self,
@@ -18,6 +18,8 @@ def __init__(
             index=index,
             volume=volume,
             spacing=spacing,
+            z=z,
+            t=t,
         )
 
     def provide_image(self, image_set):

cellprofiler_core/image/abstract_image/file/url/_mask_image.py

@@ -4,7 +4,7 @@
 class MaskImage(MonochromeImage):
     """Provide a boolean image, converting nonzero to True, zero to False if needed"""
 
-    def __init__(self, name, url, series, index, channel, volume=False, spacing=None):
+    def __init__(self, name, url, series, index, channel, volume=False, spacing=None, z=None, t=None):
         MonochromeImage.__init__(
             self,
             name,
@@ -15,6 +15,8 @@ def __init__(self, name, url, series, index, channel, volume=False, spacing=None
             channel=channel,
             volume=volume,
             spacing=spacing,
+            z=z,
+            t=t,
         )
 
     def provide_image(self, image_set):

cellprofiler_core/image/abstract_image/file/url/_monochrome_image.py

@@ -16,6 +16,8 @@ def __init__(
         rescale=True,
         volume=False,
         spacing=None,
+        z=None,
+        t=None,
     ):
         URLImage.__init__(
             self,
@@ -27,6 +29,8 @@ def __init__(
             channel=channel,
             volume=volume,
             spacing=spacing,
+            z=z,
+            t=t,
         )
 
     def provide_image(self, image_set):

cellprofiler_core/image/abstract_image/file/url/_url_image.py

@@ -1,6 +1,8 @@
 import os
 from .....utilities.pathname import url2pathname
 
+from .....utilities.image import is_omero3d_path
+
 from .._file_image import FileImage
 
 
@@ -17,6 +19,8 @@ def __init__(
         channel=None,
         volume=False,
         spacing=None,
+        z=None,
+        t=None,
     ):
         if url.lower().startswith("file:"):
             path = url2pathname(url)
@@ -25,11 +29,17 @@ def __init__(
             pathname = ""
             filename = url
         super(URLImage, self).__init__(
-            name, pathname, filename, rescale, series, index, channel, volume, spacing
+            name, pathname, filename, rescale, series, index, channel, volume, spacing, z=z, t=t,
         )
         self.url = url
 
     def get_url(self):
+        if is_omero3d_path(self.url):
+            print("OMERO-3D URL: {}".format(self.url))
+            url = self.url.split("omero-3d:")[1]
+            if url is not None:
+                return url
+            return self.url
         if self.cache_file():
             return super(URLImage, self).get_url()
         return self.url

cellprofiler_core/modules/loaddata.py

@@ -1060,6 +1060,33 @@ def fetch_provider(self, name, measurements, is_image_name=True):
             frame = measurements["Image", frame_feature]
         else:
             frame = None
+        if url.endswith('.zarr'):
+            # Zarrs need czt indexing rather than just index.
+            c, z, t = None, None, None
+
+            if measurements.has_feature("Image", f"Channel_{name}"):
+                c = measurements["Image", f"Channel_{name}"]
+            elif measurements.has_feature("Image", "Metadata_C"):
+                c = measurements["Image", "Metadata_C"]
+            if measurements.has_feature("Image", f"Z_{name}"):
+                z = measurements["Image", f"Z_{name}"]
+            elif measurements.has_feature("Image", "Metadata_Z"):
+                z = measurements["Image", "Metadata_Z"]
+            if measurements.has_feature("Image", f"T_{name}"):
+                t = measurements["Image", f"T_{name}"]
+            elif measurements.has_feature("Image", "Metadata_T"):
+                t = measurements["Image", "Metadata_T"]
+            return FileImage(
+                name,
+                path,
+                filename,
+                rescale=self.rescale.value and is_image_name,
+                series=series,
+                index=frame,
+                z=z,
+                channel=c,
+                t=t,
+            )
         return FileImage(
             name,
             path,

cellprofiler_core/modules/metadata.py

@@ -1133,6 +1133,7 @@ def msg(url):
         from bioformats.formatreader import get_omexml_metadata
         from typing import Optional, Any, Callable
         from dataclasses import dataclass
+        from cellprofiler_core.utilities.zarr import get_zarr_metadata
 
         @dataclass
         class ui_context:
@@ -1173,6 +1174,7 @@ def update_all_urls():
                 for i, url in enumerate(urls):
                     try:
                         if not pbar_context.update_callback(pbar_context.dlg, i, url):
+
                             break
                         if group.filter_choice == F_FILTERED_IMAGES:
                             match = group.filter.evaluate(
@@ -1186,7 +1188,10 @@ def update_all_urls():
                                 continue
                         metadata = filelist.get_metadata(url)
                         if metadata is None:
-                            metadata = get_omexml_metadata(url=url)
+                            if url.lower().endswith('.zarr'):
+                                metadata = get_zarr_metadata(url=url)
+                            else:
+                                metadata = get_omexml_metadata(url=url)
                             filelist.add_metadata(url, metadata)
                     except Exception as e:
                         import logging
@@ -1199,7 +1204,6 @@ def update_all_urls():
                         pbar_context.err_callback(errmsg)
 
         update_all_urls()
-
         group.metadata_autoextracted.value = True
 
     def on_activated(self, workspace):