diff --git a/cellprofiler_core/__init__.py b/cellprofiler_core/__init__.py index 1eecfc49..213df64a 100644 --- a/cellprofiler_core/__init__.py +++ b/cellprofiler_core/__init__.py @@ -1 +1 @@ -__version__ = "4.2.6" +__version__ = "4.2.8" diff --git a/cellprofiler_core/__main__.py b/cellprofiler_core/__main__.py index e2d6fe32..e69de29b 100644 --- a/cellprofiler_core/__main__.py +++ b/cellprofiler_core/__main__.py @@ -1,55 +0,0 @@ -import os -import pathlib - -import click - -import cellprofiler_core.commands - -CONTEXT_SETTINGS = dict(auto_envvar_prefix="COMPLEX") - - -class Command(click.MultiCommand): - def get_command(self, context, name): - try: - name = f"cellprofiler_core.commands._{name}_command" - - imported_module = __import__(name, None, None, ["command"]) - except ImportError: - return - - return imported_module.command - - def list_commands(self, context): - command_names = [] - - commands_pathname = cellprofiler_core.commands.__file__ - - commands_directory = pathlib.Path(commands_pathname).parent - - for filename in os.listdir(commands_directory): - if filename.endswith("_command.py") and filename.startswith("_"): - command_name = filename[1:-11] - - command_names += [command_name] - - command_names.sort() - - return command_names - - -class Environment: - def __init__(self): - pass - - -pass_environment = click.make_pass_decorator(Environment, ensure=True) - - -@click.command(cls=Command, context_settings=CONTEXT_SETTINGS) -@pass_environment -def main(context): - pass - - -if __name__ == "__main__": - main({}) diff --git a/cellprofiler_core/analysis/_runner.py b/cellprofiler_core/analysis/_runner.py index f0ed2ed7..712bf80b 100644 --- a/cellprofiler_core/analysis/_runner.py +++ b/cellprofiler_core/analysis/_runner.py @@ -102,7 +102,7 @@ def __init__( # should have jobserver() call load_measurements_from_buffer() rather # than interface() doing so. Currently, passing measurements in this # way seems like it might be buggy: - # http://code.google.com/p/h5py/issues/detail?id=244 + # https://github.com/h5py/h5py/issues/244 self.received_measurements_queue = queue.Queue(maxsize=10) self.shared_dicts = None diff --git a/cellprofiler_core/commands/__init__.py b/cellprofiler_core/commands/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cellprofiler_core/commands/_pipeline_command.py b/cellprofiler_core/commands/_pipeline_command.py deleted file mode 100644 index 4a5d3d78..00000000 --- a/cellprofiler_core/commands/_pipeline_command.py +++ /dev/null @@ -1,40 +0,0 @@ -import click - -from ..__main__ import pass_environment - - -@click.group("pipeline") -@pass_environment -def command(context): - pass - - -@command.command("measurements", help="returns measurements extracted by the pipeline") -@pass_environment -def measurements(context): - pass - - -@command.command("run", help="executes the pipeline") -@click.argument("pipeline", type=click.File("r")) -@click.option("--batch-size", type=int) -@click.option("--data", type=click.Path()) -@click.option("--default-images-directory", type=click.Path()) -@click.option("--default-output-directory", type=click.Path()) -@click.option("--images", multiple=True, type=click.Path()) -@click.option("--beginning", default=1, type=int) -@click.option("--end", type=int) -@click.option("--group", type=str) -@pass_environment -def run( - context, - batch_size, - data, - images_directory, - output_directory, - images, - beginning, - end, - grouping, -): - pass diff --git a/cellprofiler_core/commands/_worker_command.py b/cellprofiler_core/commands/_worker_command.py deleted file mode 100644 index b2298f77..00000000 --- a/cellprofiler_core/commands/_worker_command.py +++ /dev/null @@ -1,15 +0,0 @@ -import click - -from ..__main__ import pass_environment - - -@click.group("worker") -@pass_environment -def command(context): - pass - - -@command.command("start") -@pass_environment -def start(context): - pass diff --git a/cellprofiler_core/module/_module.py b/cellprofiler_core/module/_module.py index a9316c95..8fdbb6bd 100644 --- a/cellprofiler_core/module/_module.py +++ b/cellprofiler_core/module/_module.py @@ -545,7 +545,7 @@ def set_notes(self, notes): """Give the module new user-entered notes """ - sanitization_dict = {"“":"\"","”":"\""} + sanitization_dict = {"“":"\"","”":"\"","—":"-","’":"'","`":"'"} self.__notes = [''.join(sanitization_dict.get(x,x) for x in note) for note in notes] notes = property(get_notes, set_notes) diff --git a/cellprofiler_core/utilities/zmq/__init__.py b/cellprofiler_core/utilities/zmq/__init__.py index 7bd69099..01faab0e 100644 --- a/cellprofiler_core/utilities/zmq/__init__.py +++ b/cellprofiler_core/utilities/zmq/__init__.py @@ -17,6 +17,7 @@ LockStatusRequest, Request, ) +from ._event import PollTimeoutException NOTIFY_SOCKET_ADDR = "inproc://BoundaryNotifications" SD_KEY_DICT = "__keydict__" diff --git a/cellprofiler_core/utilities/zmq/_event.py b/cellprofiler_core/utilities/zmq/_event.py new file mode 100644 index 00000000..ca125c73 --- /dev/null +++ b/cellprofiler_core/utilities/zmq/_event.py @@ -0,0 +1,4 @@ +class PollTimeoutException(Exception): + """Exception issued by a timeout from polling""" + + pass diff --git a/cellprofiler_core/worker/_worker.py b/cellprofiler_core/worker/_worker.py index 8c100020..e17f7dff 100644 --- a/cellprofiler_core/worker/_worker.py +++ b/cellprofiler_core/worker/_worker.py @@ -29,12 +29,15 @@ from ..constants.worker import the_zmq_context from ..measurement import Measurements from ..utilities.measurement import load_measurements_from_buffer +from ..utilities.zmq import PollTimeoutException from ..pipeline import CancelledException from ..preferences import get_awt_headless from ..preferences import set_preferences_from_dict from ..utilities.zmq.communicable.reply.upstream_exit import UpstreamExit from ..workspace import Workspace +LOGGER = logging.getLogger(__name__) + class Worker: """An analysis worker processing work at a given address @@ -124,6 +127,7 @@ def run(self): ) t0 = time.time() self.work_socket = the_zmq_context.socket(zmq.REQ) + self.work_socket.set_hwm(2000) self.work_socket.connect(self.work_request_address) # fetch a job the_request = Work(self.current_analysis_id) @@ -304,18 +308,21 @@ def do_job(self, job): return if worker_runs_post_group: - last_workspace.interaction_handler = self.interaction_handler - last_workspace.cancel_handler = self.cancel_handler - last_workspace.post_group_display_handler = ( - self.post_group_display_handler - ) - # There might be an exception in this call, but it will be - # handled elsewhere, and there's nothing we can do for it - # here. - current_pipeline.post_group( - last_workspace, current_measurements.get_grouping_keys() - ) - del last_workspace + if not last_workspace is None: + last_workspace.interaction_handler = self.interaction_handler + last_workspace.cancel_handler = self.cancel_handler + last_workspace.post_group_display_handler = ( + self.post_group_display_handler + ) + # There might be an exception in this call, but it will be + # handled elsewhere, and there's nothing we can do for it + # here. + current_pipeline.post_group( + last_workspace, current_measurements.get_grouping_keys() + ) + del last_workspace + else: + LOGGER.error("No workspace from last image set, cannot run post group") # send measurements back to server req = MeasurementsReport( @@ -323,7 +330,18 @@ def do_job(self, job): buf=current_measurements.file_contents(), image_set_numbers=image_set_numbers, ) - rep = self.send(req) + + while True: + try: + rep = self.send(req, timeout=4000) + break + except PollTimeoutException: + LOGGER.info(f"Worker sending MeasurementsReport halted, retrying for job {str(job.image_set_numbers)}") + self.work_socket.close(linger=0) + self.work_socket = the_zmq_context.socket(zmq.REQ) + self.work_socket.set_hwm(2000) + self.work_socket.connect(self.work_request_address) + continue except CancelledException: # Main thread received shutdown signal @@ -389,7 +407,7 @@ def omero_login_handler(self): rep = self.send(req) use_omero_credentials(rep.credentials) - def send(self, req, work_socket=None): + def send(self, req, work_socket=None, timeout=None): """Send a request and receive a reply req - request to send @@ -410,7 +428,10 @@ def send(self, req, work_socket=None): req.send_only(work_socket) response = None while response is None: - for socket, state in poller.poll(): + poll_res = poller.poll(timeout) + if len(poll_res) == 0: + raise PollTimeoutException + for socket, state in poll_res: if socket == self.notify_socket and state == zmq.POLLIN: notify_msg = self.notify_socket.recv() if notify_msg == NOTIFY_STOP: diff --git a/docs/conf.py b/docs/conf.py index 43001d65..aff4ad8f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,6 @@ project = "CellProfiler-core" -release = "4.2.6" +release = "4.2.8" templates_path = ["_templates"] diff --git a/setup.cfg b/setup.cfg index 7a10dda6..4441e861 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,6 +3,6 @@ filterwarnings = ignore::DeprecationWarning ignore::FutureWarning minversion = - 4.2.6 + 4.2.8 testpaths = ./tests/ diff --git a/setup.py b/setup.py index 80e545c2..45181574 100644 --- a/setup.py +++ b/setup.py @@ -8,21 +8,19 @@ "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ], extras_require={ "dev": [ - "black==19.10b0", - "click>=7.1.2", - "pre-commit==2.2.0", - "sphinx==3.1.2", + "sphinx>=3.1.2", "twine==3.1.1", ], - "test": ["pytest==5.4.1"], + "test": ["pytest~=7.4.1"], "wx": ["wxPython==4.1.0"], }, install_requires=[ "boto3==1.14.23", - "centrosome==1.2.2", + "centrosome==1.2.3", "docutils==0.15.2", "fsspec==2022.2.0", "h5py==3.6.0", # Consider 3.2.1 if problems happen @@ -30,8 +28,8 @@ "numpy==1.23.1", "prokaryote==2.4.4", "psutil==5.7.0", - "python-bioformats==4.0.7", - "python-javabridge==4.0.3", + "python-bioformats==4.1.0", + "python-javabridge==4.0.4", "pyzmq==22.3.0", "Pillow==8.3.2", "scikit-image==0.18.3", @@ -44,6 +42,6 @@ packages=setuptools.find_packages(exclude=["tests"]), python_requires=">=3.8, <4", url="https://github.com/CellProfiler/core", - version="4.2.6", + version="4.2.8", zip_safe=False, )