Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 20 additions & 3 deletions fancylog/fancylog.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
import contextlib
import json
import logging
import multiprocessing
import os
import platform
import shutil
import subprocess
import sys
Expand Down Expand Up @@ -65,6 +65,8 @@ def start_logging(
Header for the log file, if the args are written.
multiprocessing_aware
Log from multiple processes. Default: True
Requires the 'fork' start method; otherwise warns and uses
ordinary logging.
write_header
Write a header for the log file. Default: True
write_git
Expand Down Expand Up @@ -578,6 +580,19 @@ def initialise_logger(
return logger


def _start_method():
"""Return the multiprocessing start method without fixing it.

``multiprocessing.get_start_method()`` without ``allow_none`` sets the
default context as a side effect, so a later ``set_start_method()`` call
by the user would raise. The first supported method is the default.
"""
return (
multiprocessing.get_start_method(allow_none=True)
or multiprocessing.get_all_start_methods()[0]
)


def setup_logging(
filename,
print_level="INFO",
Expand All @@ -600,6 +615,8 @@ def setup_logging(
Logging level for file output. Default is 'DEBUG'.
multiprocessing_aware
If True, enables multiprocessing-safe logging. Default is True.
Requires the 'fork' start method; otherwise warns and uses
ordinary logging.
log_to_console
If True, logs will also be printed to the console. Default is True.
logger_name
Expand All @@ -618,9 +635,9 @@ def setup_logging(
"must be performed with the root logger."
)

if multiprocessing_aware and platform.system() == "Windows":
if multiprocessing_aware and _start_method() != "fork":
warnings.warn(
"Multiprocessing logging is not supported on Windows. "
"Multiprocessing logging requires the 'fork' start method. "
"It has been disabled.",
UserWarning,
stacklevel=2,
Expand Down
66 changes: 60 additions & 6 deletions tests/tests/test_general.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import shutil
import subprocess
import sys
import warnings
from unittest.mock import MagicMock, patch

import pytest
Expand Down Expand Up @@ -488,18 +489,71 @@ def test_mock_no_environment(tmp_path):
assert f"{'pytest':20} {'1.1.1'}"


def test_multiprocessing_warning_on_windows(tmp_path):
"""A warning is raised and multiprocessing logging
is disabled on Windows.
"""
@pytest.mark.parametrize("start_method", ["spawn", "forkserver"])
def test_multiprocessing_warning_without_fork(tmp_path, start_method):
"""Unsupported start methods fall back to ordinary logging."""
mp_logging = MagicMock()
with (
patch("platform.system", return_value="Windows"),
patch("multiprocessing.get_start_method", return_value=start_method),
patch.dict(sys.modules, {"multiprocessing_logging": mp_logging}),
pytest.warns(
UserWarning, match="Multiprocessing logging is not supported"
UserWarning, match="Multiprocessing logging requires.*fork"
),
):
fancylog.start_logging(
tmp_path,
fancylog,
multiprocessing_aware=True,
)
mp_logging.install_mp_handler.assert_not_called()
log_file = next(tmp_path.glob("*.log"))
assert "Not logging multiple processes" in log_file.read_text()


def test_multiprocessing_with_fork(tmp_path):
"""The supported start method still installs the handler."""
mp_logging = MagicMock()
with (
patch("multiprocessing.get_start_method", return_value="fork"),
patch.dict(sys.modules, {"multiprocessing_logging": mp_logging}),
):
fancylog.start_logging(tmp_path, fancylog, multiprocessing_aware=True)
mp_logging.install_mp_handler.assert_called_once_with()


def test_multiprocessing_opt_out(tmp_path):
"""Opting out does not inspect the context or install a handler."""
mp_logging = MagicMock()
with (
patch("multiprocessing.get_start_method") as get_start_method,
patch.dict(sys.modules, {"multiprocessing_logging": mp_logging}),
):
fancylog.start_logging(tmp_path, fancylog, multiprocessing_aware=False)
get_start_method.assert_not_called()
mp_logging.install_mp_handler.assert_not_called()


@pytest.mark.parametrize(
"default_methods, installed",
[(["fork", "spawn"], True), (["spawn", "fork"], False)],
)
def test_multiprocessing_unset_start_method_uses_default(
tmp_path, default_methods, installed
):
"""An unset start method is read without fixing the global context."""
mp_logging = MagicMock()
with (
patch(
"multiprocessing.get_start_method", return_value=None
) as get_start_method,
patch(
"multiprocessing.get_all_start_methods",
return_value=default_methods,
),
patch.dict(sys.modules, {"multiprocessing_logging": mp_logging}),
warnings.catch_warnings(),
):
warnings.simplefilter("ignore", UserWarning)
fancylog.start_logging(tmp_path, fancylog, multiprocessing_aware=True)
get_start_method.assert_called_once_with(allow_none=True)
assert mp_logging.install_mp_handler.called == installed