From f5e55176efbd4191b8dd182c9dac9a5705594661 Mon Sep 17 00:00:00 2001 From: Raymond Nepstad Date: Fri, 11 Oct 2024 16:00:30 +0200 Subject: [PATCH 1/4] 296 Add memory usage to PyOPIA log output --- pyopia/cli.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pyopia/cli.py b/pyopia/cli.py index 0604646..5eff364 100644 --- a/pyopia/cli.py +++ b/pyopia/cli.py @@ -14,7 +14,7 @@ import rich.progress import pandas as pd import multiprocessing - +import psutil import pyopia import pyopia.background import pyopia.instrument.silcam @@ -30,6 +30,15 @@ app = typer.Typer() +def log_memory_usage(): + logger = logging.getLogger() + process = psutil.Process(os.getpid()) + total = psutil.virtual_memory().total / 1e9 + used = process.memory_info().rss / 1e9 + used_percent = used / total * 100 + logger.info(f'Current memory usage is: {used:.1f} / {total:.1f} GB ({used_percent:3.0f} %)') + + @app.command() def docs(): '''Open browser at PyOPIA's readthedocs page @@ -186,9 +195,12 @@ def process_file_list(file_list, c): f'(chunk {c})') logger.error(e) logger.debug(''.join(traceback.format_tb(e.__traceback__))) + finally: + log_memory_usage() # With one chunk we keep the non-multiprocess functionality to ensure backwards compatibility job_list = [] + multiprocessing.set_start_method('fork') if num_chunks == 1: process_file_list(raw_files, 0) else: From 9fab8f84e40605bda2b3b0183216ed1fbf665544 Mon Sep 17 00:00:00 2001 From: Raymond Nepstad Date: Mon, 14 Oct 2024 09:41:04 +0200 Subject: [PATCH 2/4] 296 Bgstack stored as uint8 to save memory and speed up mean calc In background.py, the bgstack can become memory intensive when large average window sizes are used. It is sufficient to store these as uint8, as long as the resulting mean image is stored as float64. Averaging over uint8 is also faster than over float64. --- pyopia/background.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pyopia/background.py b/pyopia/background.py index d5048be..44422db 100644 --- a/pyopia/background.py +++ b/pyopia/background.py @@ -56,8 +56,14 @@ def shift_bgstack_accurate(bgstack, imbg, imnew): updated actual background image ''' bgstack.pop(0) # pop the oldest image from the stack, - bgstack.append(imnew) # append the new image to the stack - imbg = np.mean(bgstack, axis=0) + + # Append the new image to the stack. Convert to uint8 from float [0-1], + # to save memory for large bgstacks, and speed up mean calculation of mean. + bgstack.append((255 * imnew).astype(np.uint8)) + + # Calculate mean image (float [0 - 1]) + imbg = np.mean(bgstack, axis=0) / 255 + return bgstack, imbg From f532ecff92cf1358cf3429c5f18ece7cb0add82f Mon Sep 17 00:00:00 2001 From: Raymond Nepstad Date: Mon, 14 Oct 2024 11:24:19 +0200 Subject: [PATCH 3/4] 296 Update remaining bgstack appends to use uint8 --- pyopia/background.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/pyopia/background.py b/pyopia/background.py index 44422db..8621513 100644 --- a/pyopia/background.py +++ b/pyopia/background.py @@ -92,12 +92,20 @@ def shift_bgstack_fast(bgstack, imbg, imnew): updated actual background image ''' stacklength = len(bgstack) - imold = bgstack.pop(0) # pop the oldest image from the stack, - # subtract the old image from the average (scaled by the average window) + + # Pop oldest image from stack, convert to float [0 - 1] + imold = bgstack.pop(0).astype(np.float64) / 255 + + # Subtract the old image from the average (scaled by the average window) imbg -= (imold / stacklength) - # add the new image to the average (scaled by the average window) + + # Add the new image to the average (scaled by the average window) imbg += (imnew / stacklength) - bgstack.append(imnew) # append the new image to the stack + + # Append the new image to the stack. Convert to uint8 from float [0-1], + # to save memory for large bgstacks, and speed up mean calculation of mean. + bgstack.append((255 * imnew).astype(np.uint8)) # append the new image to the stack + return bgstack, imbg @@ -290,8 +298,13 @@ def _build_background_step(self, data): init_complete = True if len(data['bgstack']) < self.average_window: - data['bgstack'].append(data[self.image_source]) - data['imbg'] = np.mean(data['bgstack'], axis=0) + # Append the new image to the stack. Convert to uint8 from float [0-1], + # to save memory for large bgstacks, and speed up mean calculation of mean. + img_uint8 = (255 * data[self.image_source]).astype(np.uint8) + data['bgstack'].append(img_uint8) + + # Calculate background as mean image (float [0 - 1]) + data['imbg'] = np.mean(data['bgstack'], axis=0) / 255 init_complete = False return init_complete From c13470397cf021f4c2a149d05f5e9d2b249e8130 Mon Sep 17 00:00:00 2001 From: Raymond Nepstad Date: Mon, 21 Oct 2024 09:57:06 +0200 Subject: [PATCH 4/4] 296 Increment PATCH version --- pyopia/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyopia/__init__.py b/pyopia/__init__.py index 80e22f7..964a32a 100644 --- a/pyopia/__init__.py +++ b/pyopia/__init__.py @@ -1 +1 @@ -__version__ = '2.8.1' +__version__ = '2.8.2'