diff --git a/pyopia/__init__.py b/pyopia/__init__.py index 80e22f7..964a32a 100644 --- a/pyopia/__init__.py +++ b/pyopia/__init__.py @@ -1 +1 @@ -__version__ = '2.8.1' +__version__ = '2.8.2' diff --git a/pyopia/background.py b/pyopia/background.py index d5048be..8621513 100644 --- a/pyopia/background.py +++ b/pyopia/background.py @@ -56,8 +56,14 @@ def shift_bgstack_accurate(bgstack, imbg, imnew): updated actual background image ''' bgstack.pop(0) # pop the oldest image from the stack, - bgstack.append(imnew) # append the new image to the stack - imbg = np.mean(bgstack, axis=0) + + # Append the new image to the stack. Convert to uint8 from float [0-1], + # to save memory for large bgstacks, and speed up mean calculation of mean. + bgstack.append((255 * imnew).astype(np.uint8)) + + # Calculate mean image (float [0 - 1]) + imbg = np.mean(bgstack, axis=0) / 255 + return bgstack, imbg @@ -86,12 +92,20 @@ def shift_bgstack_fast(bgstack, imbg, imnew): updated actual background image ''' stacklength = len(bgstack) - imold = bgstack.pop(0) # pop the oldest image from the stack, - # subtract the old image from the average (scaled by the average window) + + # Pop oldest image from stack, convert to float [0 - 1] + imold = bgstack.pop(0).astype(np.float64) / 255 + + # Subtract the old image from the average (scaled by the average window) imbg -= (imold / stacklength) - # add the new image to the average (scaled by the average window) + + # Add the new image to the average (scaled by the average window) imbg += (imnew / stacklength) - bgstack.append(imnew) # append the new image to the stack + + # Append the new image to the stack. Convert to uint8 from float [0-1], + # to save memory for large bgstacks, and speed up mean calculation of mean. + bgstack.append((255 * imnew).astype(np.uint8)) # append the new image to the stack + return bgstack, imbg @@ -284,8 +298,13 @@ def _build_background_step(self, data): init_complete = True if len(data['bgstack']) < self.average_window: - data['bgstack'].append(data[self.image_source]) - data['imbg'] = np.mean(data['bgstack'], axis=0) + # Append the new image to the stack. Convert to uint8 from float [0-1], + # to save memory for large bgstacks, and speed up mean calculation of mean. + img_uint8 = (255 * data[self.image_source]).astype(np.uint8) + data['bgstack'].append(img_uint8) + + # Calculate background as mean image (float [0 - 1]) + data['imbg'] = np.mean(data['bgstack'], axis=0) / 255 init_complete = False return init_complete diff --git a/pyopia/cli.py b/pyopia/cli.py index 0604646..5eff364 100644 --- a/pyopia/cli.py +++ b/pyopia/cli.py @@ -14,7 +14,7 @@ import rich.progress import pandas as pd import multiprocessing - +import psutil import pyopia import pyopia.background import pyopia.instrument.silcam @@ -30,6 +30,15 @@ app = typer.Typer() +def log_memory_usage(): + logger = logging.getLogger() + process = psutil.Process(os.getpid()) + total = psutil.virtual_memory().total / 1e9 + used = process.memory_info().rss / 1e9 + used_percent = used / total * 100 + logger.info(f'Current memory usage is: {used:.1f} / {total:.1f} GB ({used_percent:3.0f} %)') + + @app.command() def docs(): '''Open browser at PyOPIA's readthedocs page @@ -186,9 +195,12 @@ def process_file_list(file_list, c): f'(chunk {c})') logger.error(e) logger.debug(''.join(traceback.format_tb(e.__traceback__))) + finally: + log_memory_usage() # With one chunk we keep the non-multiprocess functionality to ensure backwards compatibility job_list = [] + multiprocessing.set_start_method('fork') if num_chunks == 1: process_file_list(raw_files, 0) else: