From 9e89ef5ae208e765fd2ebcf2761edf28559a36a8 Mon Sep 17 00:00:00 2001 From: monchin Date: Sun, 21 Jan 2024 01:57:39 +0800 Subject: [PATCH 1/3] Add "reinstall()" method to make it easier in spawn multiprocessing (#818 #912 #1064) --- loguru/_logger.py | 42 +++++++++++++++++++++++++++++++++++ tests/test_multiprocessing.py | 14 ++++++++++-- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/loguru/_logger.py b/loguru/_logger.py index 2ef78c41f..9c9487665 100644 --- a/loguru/_logger.py +++ b/loguru/_logger.py @@ -86,6 +86,7 @@ import contextlib import functools import logging +import os import re import sys import warnings @@ -230,6 +231,7 @@ class Logger: def __init__(self, core, exception, depth, record, lazy, colors, raw, capture, patchers, extra): self._core = core self._options = (exception, depth, record, lazy, colors, raw, capture, patchers, extra) + self._own_pid = os.getpid() def __repr__(self): return "" % list(self._core.handlers.values()) @@ -1752,6 +1754,46 @@ def configure(self, *, handlers=None, levels=None, extra=None, patcher=None, act return [self.add(**params) for params in handlers] + def _replace_core(self, core: Core): + self._core = core + + def reinstall(self): + """Reinstall the core of logger. + + When using multiprocessing, you can pass logger as a parameter to the target of + ``multiprocessing.Process``, and run this method once, thus you don't need to pass logger to every + function you called in the same process with spawn multiprocessing. + + Examples + -------- + >>> def subworker(logger_): + ... logger_.reinstall() + ... logger.info("Child") + ... deeper_subworker() + + >>> def deeper_subworker(): + ... logger.info("Grandchild") + + >>> def test_process_spawn(): + ... spawn_context = multiprocessing.get_context("spawn") + ... logger.add("file.log", context=spawn_context, enqueue=True, catch=False) + ... + ... process = spawn_context.Process(target=subworker, args=(logger,)) + ... process.start() + ... process.join() + + ... assert process.exitcode == 0 + + ... logger.info("Main") + ... logger.remove() + """ + if self._own_pid == os.getpid(): # same process + return + from loguru import logger + + logger._replace_core(self._core) + + def _change_activation(self, name, status): if not (name is None or isinstance(name, str)): raise TypeError( diff --git a/tests/test_multiprocessing.py b/tests/test_multiprocessing.py index 4ee687474..1287a122e 100644 --- a/tests/test_multiprocessing.py +++ b/tests/test_multiprocessing.py @@ -36,6 +36,16 @@ def subworker(logger_): logger_.info("Child") +def subworker_spawn(logger_): + logger_.reinstall() + logger.info("Child") + deeper_subworker() + + +def deeper_subworker(): + logger.info("Grandchild") + + def subworker_inheritance(): logger.info("Child") @@ -209,7 +219,7 @@ def test_process_spawn(spawn_context): logger.add(writer, context=spawn_context, format="{message}", enqueue=True, catch=False) - process = spawn_context.Process(target=subworker, args=(logger,)) + process = spawn_context.Process(target=subworker_spawn, args=(logger,)) process.start() process.join() @@ -218,7 +228,7 @@ def test_process_spawn(spawn_context): logger.info("Main") logger.remove() - assert writer.read() == "Child\nMain\n" + assert writer.read() == "Child\nGrandchild\nMain\n" @pytest.mark.skipif(os.name == "nt", reason="Windows does not support forking") From 67af92d81e693a64a6c92e0923caea6ef7339ea4 Mon Sep 17 00:00:00 2001 From: monchin Date: Mon, 22 Jan 2024 12:28:24 +0800 Subject: [PATCH 2/3] Solve problems in code review --- loguru/_logger.py | 18 +++------ tests/test_multiprocessing.py | 14 +------ tests/test_reinstall.py | 71 +++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 25 deletions(-) create mode 100644 tests/test_reinstall.py diff --git a/loguru/_logger.py b/loguru/_logger.py index 9c9487665..fa04ef5d1 100644 --- a/loguru/_logger.py +++ b/loguru/_logger.py @@ -86,7 +86,6 @@ import contextlib import functools import logging -import os import re import sys import warnings @@ -231,7 +230,6 @@ class Logger: def __init__(self, core, exception, depth, record, lazy, colors, raw, capture, patchers, extra): self._core = core self._options = (exception, depth, record, lazy, colors, raw, capture, patchers, extra) - self._own_pid = os.getpid() def __repr__(self): return "" % list(self._core.handlers.values()) @@ -1754,20 +1752,17 @@ def configure(self, *, handlers=None, levels=None, extra=None, patcher=None, act return [self.add(**params) for params in handlers] - def _replace_core(self, core: Core): - self._core = core - def reinstall(self): """Reinstall the core of logger. When using multiprocessing, you can pass logger as a parameter to the target of - ``multiprocessing.Process``, and run this method once, thus you don't need to pass logger to every - function you called in the same process with spawn multiprocessing. + ``multiprocessing.Process``, and run this method once, thus you don't need to pass + logger to every function you called in the same process with spawn multiprocessing. Examples -------- - >>> def subworker(logger_): - ... logger_.reinstall() + >>> def subworker(logger): + ... logger.reinstall() ... logger.info("Child") ... deeper_subworker() @@ -1787,12 +1782,9 @@ def reinstall(self): ... logger.info("Main") ... logger.remove() """ - if self._own_pid == os.getpid(): # same process - return from loguru import logger - logger._replace_core(self._core) - + logger._core = self._core def _change_activation(self, name, status): if not (name is None or isinstance(name, str)): diff --git a/tests/test_multiprocessing.py b/tests/test_multiprocessing.py index 1287a122e..4ee687474 100644 --- a/tests/test_multiprocessing.py +++ b/tests/test_multiprocessing.py @@ -36,16 +36,6 @@ def subworker(logger_): logger_.info("Child") -def subworker_spawn(logger_): - logger_.reinstall() - logger.info("Child") - deeper_subworker() - - -def deeper_subworker(): - logger.info("Grandchild") - - def subworker_inheritance(): logger.info("Child") @@ -219,7 +209,7 @@ def test_process_spawn(spawn_context): logger.add(writer, context=spawn_context, format="{message}", enqueue=True, catch=False) - process = spawn_context.Process(target=subworker_spawn, args=(logger,)) + process = spawn_context.Process(target=subworker, args=(logger,)) process.start() process.join() @@ -228,7 +218,7 @@ def test_process_spawn(spawn_context): logger.info("Main") logger.remove() - assert writer.read() == "Child\nGrandchild\nMain\n" + assert writer.read() == "Child\nMain\n" @pytest.mark.skipif(os.name == "nt", reason="Windows does not support forking") diff --git a/tests/test_reinstall.py b/tests/test_reinstall.py new file mode 100644 index 000000000..50151a54b --- /dev/null +++ b/tests/test_reinstall.py @@ -0,0 +1,71 @@ +import multiprocessing +import os + +import pytest +from loguru import logger + + +@pytest.fixture +def fork_context(): + yield multiprocessing.get_context("fork") + + +@pytest.fixture +def spawn_context(): + yield multiprocessing.get_context("spawn") + + +class Writer: + def __init__(self): + self._output = "" + + def write(self, message): + self._output += message + + def read(self): + return self._output + + +def subworker(logger): + logger.reinstall() + logger.info("Child") + deeper_subworker() + + +def deeper_subworker(): + logger.info("Grandchild") + + +@pytest.mark.skipif(os.name == "nt", reason="Windows does not support forking") +def test_process_fork(fork_context): + writer = Writer() + + logger.add(writer, context=fork_context, format="{message}", enqueue=True, catch=False) + + process = fork_context.Process(target=subworker, args=(logger,)) + process.start() + process.join() + + assert process.exitcode == 0 + + logger.info("Main") + logger.remove() + + assert writer.read() == "Child\nGrandchild\nMain\n" + + +def test_process_spawn(spawn_context): + writer = Writer() + + logger.add(writer, context=spawn_context, format="{message}", enqueue=True, catch=False) + + process = spawn_context.Process(target=subworker, args=(logger,)) + process.start() + process.join() + + assert process.exitcode == 0 + + logger.info("Main") + logger.remove() + + assert writer.read() == "Child\nGrandchild\nMain\n" From b2056df38a94655c0283a4799872e8246d91853e Mon Sep 17 00:00:00 2001 From: monchin Date: Tue, 23 Jan 2024 14:10:45 +0800 Subject: [PATCH 3/3] Solve lint problems checked by ruff --- tests/test_reinstall.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_reinstall.py b/tests/test_reinstall.py index 50151a54b..188879e90 100644 --- a/tests/test_reinstall.py +++ b/tests/test_reinstall.py @@ -2,6 +2,7 @@ import os import pytest + from loguru import logger