From 854c4270a16ba3c483b8bcf3d7b2b436ec57c910 Mon Sep 17 00:00:00 2001 From: Mumin Ozturk Date: Sun, 25 Oct 2015 06:28:39 +0200 Subject: [PATCH] register atexit callback and import multiprocessing module only when process_pool is initialized. 1. Process pool is initialized by thread safe way. 2. atexit callback only registered at the same time with process_pool initiliazing. So both of them are singleton in threaded env. 3. "multiprocessing import" delayed as much as possible. this helps google app engine import issues. --- firebase/__init__.py | 14 -------------- firebase/async.py | 33 ++++++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/firebase/__init__.py b/firebase/__init__.py index 8c84e53..8821e56 100644 --- a/firebase/__init__.py +++ b/firebase/__init__.py @@ -1,15 +1 @@ -import atexit - -from .async import process_pool from firebase import * - - -@atexit.register -def close_process_pool(): - """ - Clean up function that closes and terminates the process pool - defined in the ``async`` file. - """ - process_pool.close() - process_pool.join() - process_pool.terminate() diff --git a/firebase/async.py b/firebase/async.py index b343d38..28e38fb 100644 --- a/firebase/async.py +++ b/firebase/async.py @@ -1,14 +1,37 @@ -import multiprocessing - +import threading from .lazy import LazyLoadProxy __all__ = ['process_pool'] _process_pool = None +_singleton_lock = threading.Lock() + + def get_process_pool(size=5): global _process_pool - if _process_pool is None: - _process_pool = multiprocessing.Pool(processes=size) + global _singleton_lock + + if _process_pool is not None: + return _process_pool + + # initialize process_pool thread safe way + with _singleton_lock: + if _process_pool is None: + import atexit + import multiprocessing + _process_pool = multiprocessing.Pool(processes=size) + + # atexit will work only if multiprocessing pool is initialized. + @atexit.register + def close_process_pool(): + """ + Clean up function that closes and terminates the process pool + """ + _process_pool.close() + _process_pool.join() + _process_pool.terminate() + return _process_pool -process_pool = LazyLoadProxy(get_process_pool) + +process_pool = LazyLoadProxy(get_process_pool)