Move logger handlers to opentelemetry-instrumentation-logging (#4210)

* Move logging integrations into auto-instrumentation

* opentelemetry-instrumentation-logging: move the sdk logging handler here

And hook it up via entry point

* Add header and future annotations import

* Rename entry point group to opentelemetry_logging_integrations

* Consider setting up the LoggingHandler as a normal instrumentation

* Fix typo

* Add missing import

* Copy handler tests from core

* More work towards green tests

* Cleanup properly after loggingHandler tests

* Quite hard to expect a mock to setup the handler

* Call removehandler also on local loggers

No change in practice

* Fix wrong noop test

* Move to our own env var for controlling autoinstrumentation

* Copy handler benchmark from sdk

* Document the new environment variables

* Add changelog

* Please pylint

* Added warning about coexistence with sdk code

* Reword a bit

* Assert that the LoggingHandler has not been setup in uninstrumented test

* Add manual handling of auto instrumentation and code attributes logging

* Update instrumentation/opentelemetry-instrumentation-logging/src/opentelemetry/instrumentation/logging/constants.py

* Apply suggestions from code review

Co-authored-by: Pablo Collins <pablo.collins@gmail.com>

* Apply more Pablo feedback

---------

Co-authored-by: Pablo Collins <pablo.collins@gmail.com>

Author: xrmx

CHANGELOG.md

@@ -60,6 +60,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   ([#4141](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4141))
 - `opentelemetry-instrumentation-pyramid`: pass request attributes at span creation
   ([#4139](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4139))
+- `opentelemetry-instrumentation-logging`: Move there the SDK LoggingHandler
+  ([#4210](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4210))
 
 ### Fixed
 

instrumentation/opentelemetry-instrumentation-logging/benchmark-requirements.txt

@@ -0,0 +1 @@
+pytest-benchmark==4.0.0

instrumentation/opentelemetry-instrumentation-logging/benchmarks/test_benchmark_logging_handler.py

@@ -0,0 +1,53 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+import pytest
+
+from opentelemetry.instrumentation.logging.handler import LoggingHandler
+from opentelemetry.sdk._logs import LoggerProvider
+from opentelemetry.sdk._logs.export import (
+    InMemoryLogRecordExporter,
+    SimpleLogRecordProcessor,
+)
+
+
+def _set_up_logging_handler(level):
+    logger_provider = LoggerProvider()
+    exporter = InMemoryLogRecordExporter()
+    processor = SimpleLogRecordProcessor(exporter=exporter)
+    logger_provider.add_log_record_processor(processor)
+    handler = LoggingHandler(level=level, logger_provider=logger_provider)
+    return handler
+
+
+def _create_logger(handler, name):
+    logger = logging.getLogger(name)
+    logger.addHandler(handler)
+    return logger
+
+
+@pytest.mark.parametrize("num_loggers", [1, 10, 100, 1000])
+def test_simple_get_logger_different_names(benchmark, num_loggers):
+    handler = _set_up_logging_handler(level=logging.DEBUG)
+    loggers = [
+        _create_logger(handler, str(f"logger_{i}")) for i in range(num_loggers)
+    ]
+
+    def benchmark_get_logger():
+        for index in range(1000):
+            loggers[index % num_loggers].warning("test message")
+
+    benchmark(benchmark_get_logger)

instrumentation/opentelemetry-instrumentation-logging/src/opentelemetry/instrumentation/logging/__init__.py

@@ -15,7 +15,11 @@
 # pylint: disable=empty-docstring,no-value-for-parameter,no-member,no-name-in-module
 
 """
-The OpenTelemetry `logging` integration automatically injects tracing context into
+The OpenTelemetry `logging` instrumentation automatically instruments Python logging
+system with an handler to convert log messages into OpenTelemetry logs.
+You can disable this setting `OTEL_PYTHON_LOG_AUTO_INSTRUMENTATION` to `false`.
+
+The OpenTelemetry `logging` integration can inject tracing context into
 log statements, though it is opt-in and must be enabled explicitly by setting the
 environment variable `OTEL_PYTHON_LOG_CORRELATION` to `true`.
 
@@ -59,16 +63,22 @@
 from os import environ
 from typing import Collection
 
+from opentelemetry._logs import get_logger_provider
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
 from opentelemetry.instrumentation.logging.constants import (
     _MODULE_DOC,
     DEFAULT_LOGGING_FORMAT,
 )
 from opentelemetry.instrumentation.logging.environment_variables import (
+    OTEL_PYTHON_LOG_AUTO_INSTRUMENTATION,
+    OTEL_PYTHON_LOG_CODE_ATTRIBUTES,
     OTEL_PYTHON_LOG_CORRELATION,
     OTEL_PYTHON_LOG_FORMAT,
     OTEL_PYTHON_LOG_LEVEL,
 )
+from opentelemetry.instrumentation.logging.handler import (
+    _setup_logging_handler,
+)
 from opentelemetry.instrumentation.logging.package import _instruments
 from opentelemetry.trace import (
     INVALID_SPAN,
@@ -86,6 +96,8 @@
     "error": logging.ERROR,
 }
 
+_logger = logging.getLogger(__name__)
+
 
 class LoggingInstrumentor(BaseInstrumentor):  # pylint: disable=empty-docstring
     __doc__ = f"""An instrumentor for stdlib logging module.
@@ -120,6 +132,7 @@ def log_hook(span: Span, record: LogRecord):
 
     _old_factory = None
     _log_hook = None
+    _logging_handler = None
 
     def instrumentation_dependencies(self) -> Collection[str]:
         return _instruments
@@ -199,7 +212,53 @@ def record_factory(*args, **kwargs):
 
         logging.setLogRecordFactory(record_factory)
 
+        # Here we need to handle 3 scenarios:
+        # - the sdk logging handler is enabled and we should do no nothing
+        # - the sdk logging handler is not enabled and we should setup the handler by default
+        # - the sdk logging handler is not enabled and the user do not want we setup the handler
+        sdk_autoinstrumentation_env_var = (
+            environ.get(
+                "OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED", "notset"
+            )
+            .strip()
+            .lower()
+        )
+        if sdk_autoinstrumentation_env_var == "true":
+            _logger.warning(
+                "Skipping installation of LoggingHandler from "
+                "`opentelemetry-instrumentation-logging` to avoid duplicate logs. "
+                "The SDK's deprecated LoggingHandler is already active "
+                "(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true). To migrate, unset "
+                "this environment variable. The SDK's handler will be removed in a future release."
+            )
+        elif kwargs.get(
+            "enable_log_auto_instrumentation",
+            environ.get(OTEL_PYTHON_LOG_AUTO_INSTRUMENTATION, "true")
+            .strip()
+            .lower()
+            == "true",
+        ):
+            log_code_attributes = kwargs.get(
+                "log_code_attributes",
+                environ.get(OTEL_PYTHON_LOG_CODE_ATTRIBUTES, "false")
+                .strip()
+                .lower()
+                == "true",
+            )
+            logger_provider = get_logger_provider()
+            handler = _setup_logging_handler(
+                logger_provider=logger_provider,
+                log_code_attributes=log_code_attributes,
+            )
+            LoggingInstrumentor._logging_handler = handler
+
     def _uninstrument(self, **kwargs):
         if LoggingInstrumentor._old_factory:
             logging.setLogRecordFactory(LoggingInstrumentor._old_factory)
             LoggingInstrumentor._old_factory = None
+
+        if LoggingInstrumentor._logging_handler:
+            logging.getLogger().removeHandler(
+                LoggingInstrumentor._logging_handler
+            )
+            LoggingInstrumentor._logging_handler = None

instrumentation/opentelemetry-instrumentation-logging/src/opentelemetry/instrumentation/logging/constants.py

@@ -16,7 +16,22 @@
 
 
 _MODULE_DOC = """
-The OpenTelemetry ``logging`` integration automatically injects tracing context into log statements.
+The OpenTelemetry ``logging`` instrumentation automatically instruments Python logging
+with a handler to convert Python log messages into OpenTelemetry logs and export them.
+You can disable this by setting ``OTEL_PYTHON_LOG_AUTO_INSTRUMENTATION`` to ``false``.
+
+.. warning::
+
+    This package provides a logging handler to replace the deprecated one in ``opentelemetry-sdk``.
+    Therefore if you have ``opentelemetry-instrumentation-logging`` installed, you don't need to set the
+    ``OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED`` environment variable to ``true``.
+    By default, this instrumentation does not add ``code`` namespace attributes as the SDK's logger does, but adding them can be enabled by using the
+    ``OTEL_PYTHON_LOG_CODE_ATTRIBUTES`` environment variable.
+
+Enable trace context injection
+------------------------------
+
+The OpenTelemetry ``logging`` integration can also be configured to inject tracing context into log statements.
 
 The integration registers a custom log record factory with the the standard library logging module that automatically inject
 tracing context into log record objects. Optionally, the integration can also call ``logging.basicConfig()`` to set a logging
@@ -35,19 +50,25 @@
 
     {default_logging_format}
 
-Enable trace context injection
-------------------------------
-
 The integration is opt-in and must be enabled explicitly by setting the environment variable ``OTEL_PYTHON_LOG_CORRELATION`` to ``true``.
-
-The integration always registers the custom factory that injects the tracing context into the log record objects. Setting
-``OTEL_PYTHON_LOG_CORRELATION`` to ``true`` calls ``logging.basicConfig()`` to set a logging format that actually makes
+Setting ``OTEL_PYTHON_LOG_CORRELATION`` to ``true`` calls ``logging.basicConfig()`` to set a logging format that actually makes
 use of the injected variables.
 
-
 Environment variables
 ---------------------
 
+.. envvar:: OTEL_PYTHON_LOG_AUTO_INSTRUMENTATION
+
+Set this env var to ``false`` to skip installing the logging handler provided by this package.
+
+The default value is ``true``.
+
+.. envvar:: OTEL_PYTHON_CODE_ATTRIBUTES
+
+Set this env var to ``true`` to add ``code`` attributes (``code.file.path``, ``code.function.name``, ``code.line.number``) to OpenTelemetry logs, referencing the Python source location that emitted each log message.
+
+The default value is ``false``.
+
 .. envvar:: OTEL_PYTHON_LOG_CORRELATION
 
 This env var must be set to ``true`` in order to enable trace context injection into logs by calling ``logging.basicConfig()`` and

instrumentation/opentelemetry-instrumentation-logging/src/opentelemetry/instrumentation/logging/environment_variables.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+OTEL_PYTHON_LOG_AUTO_INSTRUMENTATION = "OTEL_PYTHON_LOG_AUTO_INSTRUMENTATION"
+OTEL_PYTHON_LOG_CODE_ATTRIBUTES = "OTEL_PYTHON_LOG_CODE_ATTRIBUTES"
 OTEL_PYTHON_LOG_CORRELATION = "OTEL_PYTHON_LOG_CORRELATION"
 OTEL_PYTHON_LOG_FORMAT = "OTEL_PYTHON_LOG_FORMAT"
 OTEL_PYTHON_LOG_LEVEL = "OTEL_PYTHON_LOG_LEVEL"