use sanitary instead of clean_data

Author: berislavlopac

.readthedocs.yaml

@@ -9,7 +9,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.9"
+    python: "3.10"
 
 mkdocs:
   configuration: mkdocs.yml

docs/index.md

@@ -146,3 +146,30 @@ The [`context_bind`](reference.md#unclogger.context_bind) function will set valu
     }
     >>>
     ```
+
+## Custom Processors
+
+It is possible to add other `structlog` processors into the logger configuration. For example, to hide sensitive information that might be present in the logged data (using the [Sanitary](https://sanitary.readthedocs.io) library as an example):
+
+!!! Example
+
+    ```python
+    >>> from sanitary import StructlogSanitizer
+    >>> from unclogger import add_processors, get_logger
+    >>>
+    >>> add_processors(StructlogSanitizer(keys={"password", "email"}))
+    >>>
+    >>> logger = get_logger("test logger")
+    >>> logger.info("test test", foo="abc", email="[email protected]", password="myPa55w0rd")
+    {
+        "foo": "abc", 
+        "email": "********",
+        "password": "********",
+        "event": "test test", 
+        "logger": "test logger", 
+        "level": "info", 
+        "timestamp": "2021-02-12T22:40:07.600385Z"
+    }
+    >>>
+    ```
+

docs/reference.md

@@ -36,8 +36,6 @@
 
 ::: unclogger.set_level
 
-## Processors
+## Custom Processors
 
-### Sensitive Data
-
-::: unclogger.processors.clean_data.clean_sensitive_data
+::: unclogger.processors.add_processors

justfile

@@ -1,5 +1,3 @@
-dotenv-filename := ".env"
-
 # List available recipes.
 help:
     @just --list --unsorted

pyproject.toml

@@ -20,6 +20,12 @@ docs = [
     "pymdown-extensions>=10.7",
     "jinja2>=3.1.3",
 ]
+clean = [
+    "sanitary>=0.1.0",
+]
+sanitary = [
+    "sanitary>=0.1.0",
+]
 
 [project.urls]
 Homepage = "https://unclogger.readthedocs.io"
@@ -50,7 +56,7 @@ dev = [
 default-groups = "all"
 
 [tool.pytest.ini_options]
-minversion = "6.0"
+minversion = "7.0"
 
 [tool.coverage.run]
 source = [ "unclogger/", ]
@@ -63,9 +69,6 @@ fail_under = 90
 exclude_lines = [ "pragma: no cover", "@abstract",]
 
 [tool.ruff]
-extend-exclude = [
-    "migrations/*",
-]
 line-length = 96
 output-format = "grouped"
 
@@ -128,3 +131,13 @@ ignore_missing_imports = true
 add-ignore = "D105, D107, D212, D401"
 convention = "google"
 match-dir = "(?!tests).*"
+
+[tool.deptry.per_rule_ignores]
+DEP002 = [
+    "mkdocs",
+    "mkapi",
+    "mkdocs-material",
+    "pymdown-extensions",
+    "jinja2",
+    "sanitary",
+]

tests/logger/processors/__init__.py tests/logger/clean_data/__init__.pytests/logger/processors/__init__.py renamed to tests/logger/clean_data/__init__.py

@@ -0,0 +1,57 @@
+import re
+
+import pytest
+from sanitary import StructlogSanitizer
+
+SENSITIVE_KEYS = [
+    "password",
+    "email",
+    "email_1",
+    "firstname",
+    "lastname",
+    "currentpassword",
+    "newpassword",
+    "tmppassword",
+    "authentication",
+    "refresh",
+    "auth",
+    "http_refresh",
+    "http_x_forwarded_authorization",
+    "http_x_endpoint_api_userinfo",
+    "http_authorization",
+    "idtoken",
+    "oauthidtoken",
+    "publickey",
+    "privatekey",
+]
+SENSITIVE_PATTERNS = [
+    """'Authentication':""",
+    """"Authentication":""",
+    """'Refresh':""",
+    """"Refresh":""",
+    """'Bearer """,
+    """"Bearer """,
+    "Bearer ",
+]
+REPLACEMENT_MESSAGE = "#### WARNING: Log message replaced due to sensitive keyword: "
+
+
+@pytest.fixture
+def sanitary_replacement():
+    return StructlogSanitizer(
+        keys=SENSITIVE_KEYS,
+        patterns=map(re.compile, SENSITIVE_PATTERNS),
+        message=REPLACEMENT_MESSAGE,
+    )
+
+
+@pytest.fixture
+def sanitary_hash():
+    def get_processor(hash_function):
+        return StructlogSanitizer(
+            keys=SENSITIVE_KEYS,
+            patterns=map(re.compile, SENSITIVE_PATTERNS),
+            replacement=hash_function,
+        )
+
+    return get_processor

tests/logger/clean_data/conftest.py

@@ -0,0 +1,110 @@
+import json
+
+from unclogger import add_processors, get_logger
+
+message_replacement_part = "Log message replaced"
+
+
+def test_message_with_password_and_email_is_cleaned_correctly(caplog, sanitary_replacement):
+    add_processors(sanitary_replacement)
+    field_replacement = sanitary_replacement.replacement
+
+    caplog.set_level("INFO")
+
+    message = "Test with request password"
+    logger_name = "test logger"
+
+    logger = get_logger(logger_name)
+    request = {"Email": "[email protected]", "password": "this is a sensitive value"}
+
+    logger.info(message, request=request)
+
+    record = json.loads(caplog.messages[0])
+    assert len(record) == 5
+    assert record["event"] == "Test with request password"
+    assert record["level"] == "info"
+    assert record["logger"] == logger_name
+    assert record["request"]["password"] == field_replacement
+    assert record["request"]["Email"] == field_replacement
+
+
+def test_message_with_password_and_email_in_event_is_cleaned_correctly(
+    caplog, sanitary_replacement
+):
+    add_processors(sanitary_replacement)
+
+    caplog.set_level("INFO")
+
+    logger_name = "test logger"
+    message = {"email": "[email protected]", "password": "this is a sensitive value"}
+
+    logger = get_logger(logger_name)
+    logger.info(message)
+
+    record = json.loads(caplog.messages[0])
+    assert len(record) == 4
+    assert record["level"] == "info"
+    assert record["logger"] == logger_name
+    assert record["event"]["password"] == sanitary_replacement.replacement
+    assert record["event"]["email"] == sanitary_replacement.replacement
+
+
+def _test_adding_custom_sensitive_keywords_on_runtime_cleans_output_correctly(
+    caplog, sanitary_processor
+):
+    add_processors(sanitary_processor)
+
+    logger = get_logger("test logger")
+    logger.config.sensitive_keys = {"illegalkey", "foo"}
+
+    logger.info(
+        "clean sensitive keys with dict",
+        context_id="cxt_fe76c000000000000000000_0000000000000",
+        illegalKey="blabla",
+    )
+
+    record = json.loads(caplog.messages[0])
+    assert record["illegalKey"] == message_replacement_part
+    assert record["context_id"] == "cxt_fe76c000000000000000000_0000000000000"
+
+    logger.config.sensitive_keys = {}
+
+
+def _test_setting_custom_sensitive_keywords_on_runtime_cleans_output_correctly(caplog):
+    logger = get_logger("test logger")
+    logger.config.sensitive_keys = {"illegalkey", "foo"}
+
+    logger.info(
+        "clean sensitive keys with dict",
+        context_id="cxt_fe76c000000000000000000_0000000000000",
+        illegalKey="blabla",
+    )
+
+    record = json.loads(caplog.messages[0])
+    assert message_replacement_part in record["illegalKey"]
+    assert record["context_id"] == "cxt_fe76c000000000000000000_0000000000000"
+
+    logger.config.sensitive_keys = {}
+
+
+def _test_changing_custom_sensitive_keywords_on_runtime_cleans_output_correctly(caplog):
+    logger = get_logger("test logger")
+    payload = {"foo": "1234", "bar": "5678", "fooBar": "9876"}
+
+    logger.config.sensitive_keys = {"FOO", "bar"}
+    logger.info("clean sensitive keys with dict", payload=payload)
+    record = json.loads(caplog.messages[0])
+
+    assert message_replacement_part in record["payload"]["foo"]
+    assert message_replacement_part in record["payload"]["bar"]
+    assert message_replacement_part in record["payload"]["fooBar"]
+
+    logger.config.sensitive_keys.add("fooBar")
+    logger.info("clean sensitive keys with dict", payload=payload)
+    fully_cleaned_record = json.loads(caplog.messages[1])
+
+    assert message_replacement_part in fully_cleaned_record["payload"]["foo"]
+    assert message_replacement_part in fully_cleaned_record["payload"]["bar"]
+    assert message_replacement_part in fully_cleaned_record["payload"]["fooBar"]
+
+    logger.config.sensitive_keys = {}

tests/logger/clean_data/test_clean_fields.py

@@ -3,11 +3,10 @@
 
 import pytest
 
-from unclogger import get_logger
+from unclogger import add_processors, get_logger, processors
 
 
-def _expected(value, hash_algo):
-    hash_function = getattr(hashlib, hash_algo)
+def _expected(value, hash_function):
     hashed = hash_function(value.encode())
     try:
         return hashed.hexdigest()
@@ -16,14 +15,17 @@ def _expected(value, hash_algo):
 
 
 @pytest.mark.parametrize("hash_algo", hashlib.algorithms_guaranteed)
-def test_message_with_password_and_email_is_hashed_correctly(caplog, monkeypatch, hash_algo):
+def test_message_with_password_and_email_is_hashed_correctly(caplog, hash_algo, sanitary_hash):
+    processors.CUSTOM_PROCESSORS = []
+    hash_function = getattr(hashlib, hash_algo)
+
+    add_processors(sanitary_hash(hash_function))
     caplog.set_level("INFO")
 
     message = "Test with request password"
-    logger_name = "test logger"
+    logger_name = __name__
 
     logger = get_logger(logger_name)
-    logger.config.replacement = getattr(hashlib, hash_algo)
 
     request = {
         "email": "[email protected]",
@@ -38,5 +40,5 @@ def test_message_with_password_and_email_is_hashed_correctly(caplog, monkeypatch
     assert record["level"] == "info"
     assert record["logger"] == logger_name
     assert record["request"]["safe_value"] == request["safe_value"]
-    assert record["request"]["password"] == _expected(request["password"], hash_algo)
-    assert record["request"]["email"] == _expected(request["email"], hash_algo)
+    assert record["request"]["password"] == _expected(request["password"], hash_function)
+    assert record["request"]["email"] == _expected(request["email"], hash_function)