Update 4.2.x branch

.github/workflows/test.yml

@@ -1,23 +1,67 @@
+name: run-core-tests
 jobs:
-  build:
-    runs-on: ubuntu-latest
+  test:
+    name: Install and test Core
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.8]
+        os: [macos-11]
+        python-version: [ 3.8, 3.9 ]
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+    if: github.event.pull_request.draft == false
+    env:
+      CP_MYSQL_TEST_HOST: "127.0.0.1"
+      CP_MYSQL_TEST_DB: "cellprofiler_test"
+      CP_MYSQL_TEST_USER: "root"
+      CP_MYSQL_TEST_PASSWORD: "None"
     steps:
-    - uses: actions/checkout@v1
-    - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Installation
-      run:
-        |
-        python -m pip install --upgrade pip
-        pip install numpy
-        pip install -e .[test]
-    - name: Test
-      run: pytest
-name: cellprofiler-core
-on: [push]
+      - uses: actions/checkout@v2
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          architecture: x64
+          python-version: ${{ matrix.python-version }}
+      - name: Get pip cache dir
+        id: pip-cache-dir
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+      - name: Cache pip
+        uses: actions/cache@v3
+        id: cache-pip
+        with:
+          path: ${{ steps.pip-cache-dir.outputs.dir }}
+          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/setup.py') }}
+      - name: Install java
+        uses: actions/setup-java@v1
+        with:
+          java-version: '14.0.1' # The JDK version to make available on the path.
+          java-package: jdk
+          architecture: x64
+      - name: Mac - Install and setup mysql
+        if: startsWith(matrix.os, 'macos')
+        env:
+          LC_ALL: "en_US.UTF-8"
+        run: |
+          brew install mysql
+          mysql.server start
+          mysql --host=$CP_MYSQL_TEST_HOST --user=$CP_MYSQL_TEST_USER --execute="CREATE DATABASE $CP_MYSQL_TEST_DB;" --skip-password
+      - name: Installation
+        run:
+          |
+          pip install pyinstaller
+          pip install --upgrade pip setuptools wheel
+          pip install numpy>=1.20.1
+          # git clone https://github.com/CellProfiler/CellProfiler.git ~/cellprofiler
+          pip install -e .[test]
+          # pip install -e ~/cellprofiler
+      - name: Test
+        run: pytest -k 'not test_load_objects and not test_load_single_object'
+on:
+  pull_request:
+    types: [opened, reopened, synchronize, ready_for_review, review_requested]
+  push:
+    branches:
+      - 'master'
+  workflow_dispatch:
+

cellprofiler_core/__init__.py

@@ -1 +1 @@
-__version__ = "4.2.1"
+__version__ = "4.2.6"

cellprofiler_core/analysis/_runner.py

@@ -27,6 +27,7 @@
 from ..preferences import get_plugin_directory
 from ..preferences import get_conserve_memory
 from ..preferences import get_temporary_directory
+from ..preferences import get_always_continue
 from ..preferences import preferences_as_dict
 from ..utilities.analysis import close_all_on_exec, start_daemon_thread
 from ..utilities.analysis import find_analysis_worker_source
@@ -664,6 +665,8 @@ def start_workers(cls, num=None):
                 get_plugin_directory(),
                 "--conserve-memory",
                 str(get_conserve_memory()),
+                "--always-continue",
+                str(get_always_continue())
             ]
             # stdin for the subprocesses serves as a deadman's switch.  When
             # closed, the subprocess exits.

cellprofiler_core/constants/measurement.py

@@ -5,6 +5,7 @@
 IMAGE = "Image"
 EXPERIMENT = "Experiment"
 RELATIONSHIP = "Relationship"
+DB_TEMP = "ExportToDb"
 NEIGHBORS = "Neighbors"
 OBJECT = "Object"
 disallowed_object_names = [IMAGE, EXPERIMENT, RELATIONSHIP]
@@ -33,6 +34,7 @@
 OBJECT_NUMBER = "ObjectNumber"
 GROUP_NUMBER = "Group_Number"  # 1-based group index
 GROUP_INDEX = "Group_Index"  # 1-based index within group
+GROUP_LENGTH = "Group_Length"
 R_FIRST_IMAGE_NUMBER = IMAGE_NUMBER + "_" + "First"
 R_FIRST_OBJECT_NUMBER = OBJECT_NUMBER + "_" + "First"
 R_SECOND_IMAGE_NUMBER = IMAGE_NUMBER + "_" + "Second"

cellprofiler_core/constants/pipeline.py

@@ -3,6 +3,7 @@
 IMAGE_NUMBER = "ImageNumber"
 GROUP_NUMBER = "Group_Number"
 GROUP_INDEX = "Group_Index"
+GROUP_LENGTH = "Group_Length"
 CURRENT = "Current"
 NUMBER_OF_IMAGE_SETS = "NumberOfImageSets"
 NUMBER_OF_MODULES = "NumberOfModules"

cellprofiler_core/image/abstract_image/file/_file_image.py

@@ -301,7 +301,7 @@ def __set_image(self):
                     c=self.channel,
                     series=self.series,
                     index=self.index,
-                    rescale=self.rescale,
+                    rescale=self.rescale if isinstance(self.rescale, bool) else False,
                     wants_max_intensity=True,
                     channel_names=channel_names,
                     z=self.z_index,
@@ -325,7 +325,7 @@ def __set_image(self):
                         c=channel,
                         series=series,
                         index=index,
-                        rescale=self.rescale,
+                        rescale=self.rescale if isinstance(self.rescale, bool) else False,
                         wants_max_intensity=True,
                         channel_names=channel_names,
                         z=self.z_index,
@@ -341,6 +341,7 @@ def __set_image(self):
             path_name=self.get_pathname(),
             file_name=self.get_filename(),
             scale=self.scale,
+            channelstack=img.ndim == 3 and img.shape[-1]>3
         )
         if img.ndim == 3 and len(channel_names) == img.shape[2]:
             self.__image.channel_names = list(channel_names)

cellprofiler_core/image/abstract_image/file/url/_color_image.py

@@ -26,6 +26,6 @@ def provide_image(self, image_set):
         image = URLImage.provide_image(self, image_set)
 
         if image.pixel_data.ndim == image.dimensions:
-            image.pixel_data = skimage.color.gray2rgb(image.pixel_data, alpha=False)
+            image.pixel_data = skimage.color.gray2rgb(image.pixel_data)
 
         return image

cellprofiler_core/image/abstract_image/file/url/_objects_image.py

@@ -86,9 +86,9 @@ def get_indexes(self, url):
 
     def get_image_volume(self):
         imdata = self.__data
-        planes = []  # newplanes = numpy.zeros_like(test2)
+        planes = []
         for planeid in range(imdata.shape[0]):
-            planes.append(convert_image_to_objects(imdata[planeid]).astype(numpy.int32))
+            planes.append(imdata[planeid].astype(numpy.int32))
         imdata = numpy.stack(planes)
 
         image = Image(

cellprofiler_core/measurement/_measurements.py

@@ -33,9 +33,11 @@
 from ..constants.measurement import C_PATH_NAME
 from ..constants.measurement import C_SERIES
 from ..constants.measurement import C_URL
+from ..constants.measurement import DB_TEMP
 from ..constants.measurement import EXPERIMENT
 from ..constants.measurement import GROUP_INDEX
 from ..constants.measurement import GROUP_NUMBER
+from ..constants.measurement import GROUP_LENGTH
 from ..constants.measurement import IMAGE
 from ..constants.measurement import IMAGE_NUMBER
 from ..constants.measurement import K_CASE_SENSITIVE
@@ -364,6 +366,15 @@ def set_group_index(self, group_index):
 
     group_index = property(get_group_index, set_group_index)
 
+    def get_group_length(self):
+        """The group length of the current image group"""
+        return self.get_current_image_measurement(GROUP_LENGTH)
+
+    def set_group_length(self, group_length):
+        self.add_image_measurement(GROUP_LENGTH, group_length)
+
+    group_length = property(get_group_length, set_group_length)
+
     def get_groupings(self, features):
         """Return groupings of image sets based on feature values
 
@@ -741,7 +752,7 @@ def clear(self):
     def get_object_names(self):
         """The list of object names (including Image) that have measurements
         """
-        return [x for x in self.hdf5_dict.top_level_names() if x != RELATIONSHIP]
+        return [x for x in self.hdf5_dict.top_level_names() if x not in (DB_TEMP, RELATIONSHIP)]
 
     object_names = property(get_object_names)
 
@@ -1295,6 +1306,7 @@ def write_image_sets(self, fd_or_file, start=None, stop=None):
         to_save = [
             GROUP_NUMBER,
             GROUP_INDEX,
+            GROUP_LENGTH
         ]
         to_save_prefixes = [
             C_URL,
@@ -1733,11 +1745,20 @@ def set_grouping_tags(self, grouping_tags):
         data = json.dumps(grouping_tags)
         self.add_experiment_measurement(M_GROUPING_TAGS, data)
 
-    def get_grouping_tags(self):
+    def get_grouping_tags_or_metadata(self):
         """Get the metadata tags that were used to group the image set
 
         """
         if not self.has_feature(EXPERIMENT, M_GROUPING_TAGS,):
             return self.get_metadata_tags()
 
         return json.loads(self.get_experiment_measurement(M_GROUPING_TAGS))
+
+    def get_grouping_tags_only(self):
+        """Get the metadata tags that were used to group the image set,
+        and only those, not metadata instead
+        """
+        if not self.has_feature(EXPERIMENT, M_GROUPING_TAGS,):
+            return []
+
+        return json.loads(self.get_experiment_measurement(M_GROUPING_TAGS))

cellprofiler_core/module/_module.py

@@ -1,4 +1,5 @@
 import abc
+import logging
 import sys
 import uuid
 
@@ -106,12 +107,15 @@ def from_dict(self, settings: list, attributes: dict):
         self.wants_pause = attributes["wants_pause"]
         self.svn_version = attributes["svn_version"]
         self.enabled = attributes["enabled"]
-        self.variable_revision_number = attributes["variable_revision_number"]
         self.module_path = attributes["module_path"]
         self.module_name = attributes["module_name"]
         setting_values = [setting["value"] for setting in settings]
+        if attributes["variable_revision_number"] > self.variable_revision_number:
+            logging.warning(f"Loaded module '{self.module_name}' is from a newer version of CellProfiler - "
+                            f"v{attributes['variable_revision_number']}, current version is "
+                            f"v{self.variable_revision_number}. Settings may load incorrectly.")
         self.set_settings_from_values(
-            setting_values, self.variable_revision_number, self.module_path
+            setting_values, attributes['variable_revision_number'], self.module_path
         )
 
     @abc.abstractmethod

cellprofiler_core/modules/groups.py

@@ -1,6 +1,6 @@
 import numpy
 
-from ..constants.measurement import COLTYPE_VARCHAR
+from ..constants.measurement import COLTYPE_INTEGER, COLTYPE_VARCHAR, IMAGE
 from ..constants.measurement import C_FILE_NAME
 from ..constants.measurement import C_METADATA
 from ..constants.measurement import C_OBJECTS_FILE_NAME
@@ -9,6 +9,7 @@
 from ..constants.measurement import EXPERIMENT
 from ..constants.measurement import GROUP_INDEX
 from ..constants.measurement import GROUP_NUMBER
+from ..constants.measurement import GROUP_LENGTH
 from ..constants.measurement import M_GROUPING_TAGS
 from ..measurement import Measurements
 from ..module import Module
@@ -559,7 +560,7 @@ def get_groupings(self, workspace):
         """
         if not self.wants_groups:
             return
-        key_list = self.get_grouping_tags()
+        key_list = self.get_grouping_tags_or_metadata()
         m = workspace.measurements
         for key in key_list:
             if key not in m.get_feature_names("Image"):
@@ -577,7 +578,7 @@ def get_groupings(self, workspace):
                 return None
         return key_list, m.get_groupings(key_list)
 
-    def get_grouping_tags(self):
+    def get_grouping_tags_or_metadata(self):
         """Return the metadata keys used for grouping"""
         if not self.wants_groups:
             return None
@@ -642,12 +643,19 @@ def prepare_run(self, workspace, changed_setting = False):
         group_indexes = numpy.hstack(
             [numpy.arange(len(image_numbers)) + 1 for keys, image_numbers in groupings]
         )
+        group_lens = numpy.hstack(
+            [
+                numpy.ones(len(image_numbers), int) * (len(image_numbers))
+                for i, (keys, image_numbers) in enumerate(groupings)
+            ]
+        )
         image_numbers = numpy.hstack(
             [image_numbers for keys, image_numbers in groupings]
         )
         order = numpy.lexsort((group_indexes, group_numbers))
         group_numbers = group_numbers[order]
         group_indexes = group_indexes[order]
+        group_lens = group_lens[order]
 
         m = workspace.measurements
         assert isinstance(m, Measurements)
@@ -664,7 +672,10 @@ def prepare_run(self, workspace, changed_setting = False):
         m.add_all_measurements(
             "Image", GROUP_INDEX, group_indexes,
         )
-        m.set_grouping_tags(self.get_grouping_tags())
+        m.add_all_measurements(
+            "Image", GROUP_LENGTH, group_lens,
+        )
+        m.set_grouping_tags(self.get_grouping_tags_or_metadata())
         return True
 
     def run(self, workspace):
@@ -678,12 +689,13 @@ def get_measurement_columns(self, pipeline):
         result = []
         if self.wants_groups:
             result.append((EXPERIMENT, M_GROUPING_TAGS, COLTYPE_VARCHAR,))
+            result.append((IMAGE, GROUP_LENGTH, COLTYPE_INTEGER))
             #
             # These are bound to be produced elsewhere, but it is quite
             # computationally expensive to find that out. If they are
             # duplicated by another module, no big deal.
             #
-            for ftr in self.get_grouping_tags():
+            for ftr in self.get_grouping_tags_or_metadata():
                 result.append(("Image", ftr, COLTYPE_VARCHAR,))
         return result
 

cellprofiler_core/modules/loaddata.py

@@ -29,6 +29,7 @@
 from ..constants.measurement import FTR_WELL
 from ..constants.measurement import M_WELL
 from ..constants.measurement import PATH_NAME_LENGTH
+from ..constants.measurement import GROUP_LENGTH
 from ..constants.module import IO_FOLDER_CHOICE_HELP_TEXT
 from ..constants.modules.load_data import DIR_ALL
 from ..constants.modules.load_data import IMAGE_CATEGORIES
@@ -953,12 +954,12 @@ def prepare_run(self, workspace):
             for eachval in groupvals:
                 group_lengths += [len(eachval[1])] * len(eachval[1])
             m.add_all_measurements(
-                "Image", "Group_Length", group_lengths,
+                "Image", GROUP_LENGTH, group_lengths,
             )
         else:
             group_lengths = [len(rows)] * len(rows)
             m.add_all_measurements(
-                "Image", "Group_Length", group_lengths,
+                "Image", GROUP_LENGTH, group_lengths,
             )
 
         return True