diff --git a/.gitignore b/.gitignore index 41306b7..2911127 100644 --- a/.gitignore +++ b/.gitignore @@ -1,157 +1,43 @@ # Created by .ignore support plugin (hsz.mobi) ### Python template # Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - # C extensions -*.so - # Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - # Installer logs -pip-log.txt -pip-delete-this-directory.txt - # Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ - # Translations -*.mo -*.pot - # Django stuff: -*.log -.static_storage/ -.media/ -local_settings.py - # Flask stuff: -instance/ -.webassets-cache - # Scrapy stuff: -.scrapy - # Sphinx documentation -docs/_build/ - # PyBuilder -target/ - # Jupyter Notebook -.ipynb_checkpoints - # pyenv -.python-version - # celery beat schedule file -celerybeat-schedule - # SageMath parsed files -*.sage.py - # Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - # Spyder project settings -.spyderproject -.spyproject - # Rope project settings -.ropeproject - # mkdocs documentation -/site - # mypy -.mypy_cache/ ### JetBrains template # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff: -.idea/**/workspace.xml -.idea/**/tasks.xml -.idea/dictionaries - # Sensitive or high-churn files: -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.xml -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml - # Gradle: -.idea/**/gradle.xml -.idea/**/libraries - # CMake -cmake-build-debug/ - # Mongo Explorer plugin: -.idea/**/mongoSettings.xml - ## File-based project format: -*.iws - ## Plugin-specific files: # IntelliJ -out/ - # mpeltonen/sbt-idea plugin -.idea_modules/ - # JIRA plugin -atlassian-ide-plugin.xml - # Cursive Clojure plugin -.idea/replstate.xml - # Crashlytics plugin (for Android Studio and IntelliJ) -com_crashlytics_export_strings.xml -crashlytics.properties -crashlytics-build.properties -fabric.properties .idea/ diff --git a/README.md b/README.md index fd6e170..bc9511f 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ pip3 install . ``` # 7. References ## [0] Full logging format references -2 types of logging statement will be emmited by this library: +2 types of logging statement will be emitted by this library: - Application log: normal logging statement e.g.: ``` @@ -301,7 +301,7 @@ pip3 install json_logging --index-url https://test.pypi.org/simple/ pypi ``` python setup.py sdist upload -r pypi -python setup.py bdist_wheel --universal upload -r pypi +python3 setup.py bdist_wheel --universal upload -r pypi pip3 install json_logging ``` bdist_wheel --universal diff --git a/README.rst b/README.rst index e0220ee..bccf229 100644 --- a/README.rst +++ b/README.rst @@ -317,7 +317,7 @@ you can install Sanic on windows by running these commands: [0] Full logging format references ---------------------------------- -2 types of logging statement will be emmited by this library: +2 types of logging statement will be emitted by this library: - Application log: normal logging statement e.g.: :: @@ -372,52 +372,52 @@ See following tables for detail format explanation: - Common field -+-----------------+-----------------+-----------------+-----------------+ -| Field | Description | Format | Example | -+=================+=================+=================+=================+ -| written_at | The date when | ISO 8601 | 2017-12-23T15:1 | -| | this log | YYYY-MM-DDTHH:M | 4:02.208Z | -| | message was | M:SS.milliZ | | -| | written. | | | -+-----------------+-----------------+-----------------+-----------------+ -| written_ts | The timestamp | long number | 145682055381684 | -| | in nano-second | | 9408 | -| | precision when | | | -| | this request | | | -| | metric message | | | -| | was written. | | | -+-----------------+-----------------+-----------------+-----------------+ -| correlation_id | The timestamp | string | db2d002e-2702-4 | -| | in nano-second | | 1ec-66f5-c002a8 | -| | precision when | | 0a3d3f | -| | this request | | | -| | metric message | | | -| | was written. | | | -+-----------------+-----------------+-----------------+-----------------+ -| type | Type of | string | | -| | logging. “logs” | | | -| | or “request” | | | -+-----------------+-----------------+-----------------+-----------------+ -| component_id | Uniquely | string | 9e6f3ecf-def0-4 | -| | identifies the | | baf-8fac-9339e6 | -| | software | | 1d5645 | -| | component that | | | -| | has processed | | | -| | the current | | | -| | request | | | -+-----------------+-----------------+-----------------+-----------------+ -| component_name | A | string | my-fancy-compon | -| | human-friendly | | ent | -| | name | | | -| | representing | | | -| | the software | | | -| | component | | | -+-----------------+-----------------+-----------------+-----------------+ -| component_insta | Instance’s | string | 0 | -| nce | index of | | | -| | horizontally | | | -| | scaled service | | | -+-----------------+-----------------+-----------------+-----------------+ ++-------------------+-----------------+-----------------+-----------------+ +| Field | Description | Format | Example | ++===================+=================+=================+=================+ +| written_at | The date when | ISO 8601 | 2017-12-23T15:1 | +| | this log | YYYY-MM-DDTHH:M | 4:02.208Z | +| | message was | M:SS.milliZ | | +| | written. | | | ++-------------------+-----------------+-----------------+-----------------+ +| written_ts | The timestamp | long number | 145682055381684 | +| | in nano-second | | 9408 | +| | precision when | | | +| | this request | | | +| | metric message | | | +| | was written. | | | ++-------------------+-----------------+-----------------+-----------------+ +| correlation_id | The timestamp | string | db2d002e-2702-4 | +| | in nano-second | | 1ec-66f5-c002a8 | +| | precision when | | 0a3d3f | +| | this request | | | +| | metric message | | | +| | was written. | | | ++-------------------+-----------------+-----------------+-----------------+ +| type | Type of | string | | +| | logging. “logs” | | | +| | or “request” | | | ++-------------------+-----------------+-----------------+-----------------+ +| component_id | Uniquely | string | 9e6f3ecf-def0-4 | +| | identifies the | | baf-8fac-9339e6 | +| | software | | 1d5645 | +| | component that | | | +| | has processed | | | +| | the current | | | +| | request | | | ++-------------------+-----------------+-----------------+-----------------+ +| component_name | A | string | my-component | +| | human-friendly | | | +| | name | | | +| | representing | | | +| | the software | | | +| | component | | | ++-------------------+-----------------+-----------------+-----------------+ +| component_instance| Instance’s | string | 0 | +| | index of | | | +| | horizontally | | | +| | scaled service | | | ++-------------------+-----------------+-----------------+-----------------+ - application logs @@ -549,8 +549,8 @@ See following tables for detail format explanation: | t_type | associated with | | n | | | the entity of | | | | | the response if | | | -| | available/speci | | | -| | fied | | | +| | available or | | | +| | specified | | | +-----------------+-----------------+-----------------+-----------------+ | referer | For HTTP | string | /index.html | | | requests, | | | diff --git a/example/non_web.py b/example/non_web.py index 088755b..8569f2b 100644 --- a/example/non_web.py +++ b/example/non_web.py @@ -1,4 +1,7 @@ -import json_logging, logging, sys +import logging +import sys + +import json_logging # log is initialized without a web framework name json_logging.ENABLE_JSON_LOGGING = True diff --git a/example/sanic_sample_app.py b/example/sanic_sample_app.py index 764e38d..9bb82d1 100644 --- a/example/sanic_sample_app.py +++ b/example/sanic_sample_app.py @@ -1,5 +1,6 @@ import json_logging import logging +# noinspection PyPackageRequirements import sanic import sys @@ -14,10 +15,12 @@ logger.addHandler(logging.StreamHandler(sys.stdout)) +# noinspection PyUnusedLocal @app.route("/") async def test(request): logger.info("test log statement") logger.info("test log statement", extra={'props': {"extra_property": 'extra_value'}}) + # noinspection PyUnresolvedReferences return sanic.response.text("hello world") diff --git a/json_logging/__init__.py b/json_logging/__init__.py index 56eb973..eb2c88c 100644 --- a/json_logging/__init__.py +++ b/json_logging/__init__.py @@ -31,8 +31,8 @@ def get_correlation_id(): """ - Get current request correlation-id. - In request context, if one is not present, a new one might be generated depends on CREATE_CORRELATION_ID_IF_NOT_EXISTS setting value. + Get current request correlation-id. If one is not present, a new one might be generated + depends on CREATE_CORRELATION_ID_IF_NOT_EXISTS setting value. :return: correlation-id string """ @@ -140,7 +140,7 @@ def init(framework_name=None): def init_request_instrument(app=None): """ - Sets up the current web application instance' request instrumentation logging configuration. Must be called after init method + Configure the request instrumentation logging configuration for given web app. Must be called after init method :param app: current web application instance """ @@ -166,6 +166,7 @@ def __init__(self, request, **kwargs): self.request = request self.request_received_at = util.iso_time_format(utcnow) + # noinspection PyAttributeOutsideInit def update_response_status(self, response): """ update response information into this object, must be called before invoke request logging statement @@ -279,6 +280,7 @@ def format(self, record): flask_support.FlaskResponseAdapter) # register flask support +# noinspection PyPep8 from json_logging.framework.sanic import SanicAppConfigurator, SanicAppRequestInstrumentationConfigurator, \ SanicRequestAdapter, SanicResponseAdapter diff --git a/json_logging/framework/flask/__init__.py b/json_logging/framework/flask/__init__.py index ea7ee97..01bf168 100644 --- a/json_logging/framework/flask/__init__.py +++ b/json_logging/framework/flask/__init__.py @@ -9,6 +9,7 @@ def is_flask_present(): + # noinspection PyPep8,PyBroadException try: import flask return True @@ -37,6 +38,7 @@ def config(self, app): json_logging.util.use_cf_logging_formatter([logging.getLogger('werkzeug')], JSONLogWebFormatter) + # noinspection PyAttributeOutsideInit self.request_logger = logging.getLogger('flask-request-logger') self.request_logger.setLevel(logging.DEBUG) self.request_logger.addHandler(logging.StreamHandler(sys.stdout)) diff --git a/json_logging/framework/sanic/__init__.py b/json_logging/framework/sanic/__init__.py index 38c7ac7..a72ea2e 100644 --- a/json_logging/framework/sanic/__init__.py +++ b/json_logging/framework/sanic/__init__.py @@ -9,6 +9,7 @@ def is_sanic_present(): + # noinspection PyBroadException try: # noinspection PyPackageRequirements from sanic import Sanic @@ -44,6 +45,7 @@ def config(self, app): if not isinstance(app, Sanic): raise RuntimeError("app is not a valid Sanic.app.Sanic app instance") + # noinspection PyAttributeOutsideInit self.request_logger = logging.getLogger('sanic-request') self.request_logger.setLevel(logging.DEBUG) self.request_logger.addHandler(logging.StreamHandler(sys.stdout)) diff --git a/json_logging/framework_base.py b/json_logging/framework_base.py index 34604c4..ba16dec 100644 --- a/json_logging/framework_base.py +++ b/json_logging/framework_base.py @@ -1,7 +1,7 @@ # coding=utf-8 class RequestAdapter: """ - Helper class help to extract logging-relevant information from HTTP request object + Helper class help to extract logging-relevant information from HTTP request object """ def __new__(cls, *arg, **kwargs): @@ -34,7 +34,7 @@ def get_http_header(self, request, header_name, default=None): """ get HTTP header value given it value name - :param request: + :param request: request object :param header_name: name of header :param default: default value if header value is not present :return: @@ -44,40 +44,40 @@ def get_http_header(self, request, header_name, default=None): def get_remote_user(self, request): """ - :param request: + :param request: request object """ raise NotImplementedError def is_in_request_context(self, request): """ - :param request: + :param request: request object """ raise NotImplementedError def set_correlation_id(self, request, value): """ We can safely assume that request is valid request object.\n - Set correlation to request context - Made sure that we can access it later from get_correlation_id_in_request_context given the same request. + Set correlation to request context. e.g Flask.g in Flask + Made sure that we can access it later from get_correlation_id_in_request_context given the same request. - :param value: - :param request: + :param value: correlation id string + :param request: request object """ raise NotImplementedError def get_correlation_id_in_request_context(self, request): """ We can safely assume that request is valid request object. - - :param request: + + :param request: request object """ raise NotImplementedError def get_protocol(self, request): """ We can safely assume that request is valid request object.\n - Gets the request protocol (e.g. HTTP/1.1). + Gets the request protocol (e.g. HTTP/1.1). :return: The request protocol or '-' if it cannot be determined """ @@ -96,7 +96,7 @@ def get_content_length(self, request): """ We can safely assume that request is valid request object.\n The content length of the request. - + :return: the content length of the request or '-' if it cannot be determined """ raise NotImplementedError @@ -105,7 +105,7 @@ def get_method(self, request): """ We can safely assume that request is valid request object.\n Gets the request method (e.g. GET, POST, etc.). - + :return: The request method or '-' if it cannot be determined """ raise NotImplementedError @@ -114,7 +114,7 @@ def get_remote_ip(self, request): """ We can safely assume that request is valid request object.\n Gets the remote ip of the request initiator. - + :return: An ip address or '-' if it cannot be determined """ raise NotImplementedError @@ -123,7 +123,7 @@ def get_remote_port(self, request): """ We can safely assume that request is valid request object.\n Gets the remote port of the request initiator. - + :return: A port or '-' if it cannot be determined """ raise NotImplementedError @@ -131,7 +131,7 @@ def get_remote_port(self, request): class ResponseAdapter: """ - Helper class help to extract logging-relevant information from HTTP response object + Helper class help to extract logging-relevant information from HTTP response object """ def __new__(cls, *arg, **kwargs): @@ -142,24 +142,24 @@ def __new__(cls, *arg, **kwargs): def get_status_code(self, response): """ get response's integer status code - - :param response: + + :param response: response object """ raise NotImplementedError def get_response_size(self, response): """ get response's size in bytes - - :param response: + + :param response: response object """ raise NotImplementedError def get_content_type(self, response): """ - get response's MIME/media type - - :param response: + get response's MIME/media type + + :param response: response object """ raise NotImplementedError @@ -184,14 +184,10 @@ def config(self): class AppRequestInstrumentationConfigurator: """ Class to perform request instrumentation logging configuration. Should at least contains: - 1- register before-request hook and create a RequestInfo object, store it to request context - 2- register after-request hook and update response to stored RequestInfo object - 3 - re-configure framework loggers. - - NOTE: logger's reference that is used to emit request instrumentation logging will need to assign to **self.request_logger** + NOTE: logger that is used to emit request instrumentation logs will need to assign to **self.request_logger** """ def __new__(cls, *args, **kw): diff --git a/json_logging/util.py b/json_logging/util.py index 26500e2..dd4452a 100644 --- a/json_logging/util.py +++ b/json_logging/util.py @@ -16,9 +16,9 @@ def is_env_var_toggle(var_name): def get_library_logger(logger_name): """ - - :param logger_name: - :return: + + :param logger_name: name + :return: logger """ # noinspection PyUnresolvedReferences if logger_name in logging.Logger.manager.loggerDict: @@ -47,7 +47,9 @@ def use_cf_logging_formatter(loggers_iter, formatter): handler.formatter = formatter() +# noinspection PyPep8 def parse_int(input_int, default): + # noinspection PyBroadException try: integer = int(input_int) except: @@ -58,9 +60,9 @@ def parse_int(input_int, default): def validate_subclass(subclass, superclass): """ - :param subclass: - :param superclass: - :return: + :param subclass + :param superclass + :return: bool """ if not issubclass(subclass, superclass): raise RuntimeError(str(subclass) + ' is not a subclass of ' + str(superclass)) @@ -83,8 +85,10 @@ def iso_time_format(datetime_): _no_of_go_up_level = 11 if hasattr(sys, '_getframe'): + # noinspection PyProtectedMember,PyPep8 currentframe = lambda: sys._getframe(_no_of_go_up_level) else: # pragma: no cover + # noinspection PyBroadException def currentframe(): """Return the frame object for the caller's stack frame.""" try: @@ -151,20 +155,20 @@ def get_correlation_id(self, request=None): def get_request_from_call_stack(self): """ - :return: get request object from call stack + :return: get request object from call stack """ """ python 3 call stack frame - 00 get_request_from_call_stack [util.py:225] + 00 get_request_from_call_stack [util.py:225] 01 get_correlation_id [util.py:177] 02 format [__init__.py:333] 03 format [__init__.py:830] - 04 emit [__init__.py:980] - 05 handle [__init__.py:855] - 06 callHandlers [__init__.py:1487] + 04 emit [__init__.py:980] + 05 handle [__init__.py:855] + 06 callHandlers [__init__.py:1487] 07 handle [__init__.py:1425] - 08 _log [__init__.py:1415] - 09 info [__init__.py:1279] + 08 _log [__init__.py:1415] + 09 info [__init__.py:1279] 10 logging statement """ # FIXME: find out the depth of logging call stack in Python 2.7 diff --git a/setup.py b/setup.py index 3e8edcd..212b3c4 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,22 @@ +import io +import sys + from setuptools import setup, find_packages +version_info_major = sys.version_info[0] +if version_info_major == 3: + long_description = open('README.rst', encoding="utf8").read() +else: + io_open = io.open('README.rst', encoding="utf8") + long_description = io_open.read() + setup( name="json-logging", - version='0.0.1', + version='0.0.7', packages=find_packages(exclude=['contrib', 'docs', 'tests*', 'example', 'dist', 'build']), license='Apache License 2.0', description="JSON Python Logging", - long_description=open('README.rst').read(), + long_description=long_description, author="Bui Nguyen Thang (Bob)", author_email="bob.bui@outlook.com", keywords=["json", "elastic", "python", "python3", "python2", "logging", "logging-library", "json", "elasticsearch", diff --git a/tests-performance/benmark_micro.py b/tests-performance/benmark_micro.py index 87b1b88..99ec8a9 100644 --- a/tests-performance/benmark_micro.py +++ b/tests-performance/benmark_micro.py @@ -1,9 +1,8 @@ # coding=utf-8 +import time import timeit from datetime import datetime -import time - utcnow = datetime.utcnow() numbers = 1000000 @@ -46,4 +45,4 @@ print(timeit3) # 1456820553816849408 # 1496635859 -# 1496637019171670000 \ No newline at end of file +# 1496637019171670000 diff --git a/tests-performance/flask_performance_test_pure_python.py b/tests-performance/flask_performance_test_pure_python.py index 1225a6b..dd39e71 100644 --- a/tests-performance/flask_performance_test_pure_python.py +++ b/tests-performance/flask_performance_test_pure_python.py @@ -19,7 +19,7 @@ @app.route('/') def home________________(): - count_ = 10; + count_ = 10 if 'count' in request.args: count_ = int(request.args['count']) simple_logging = Timer(lambda: logger.info(MESSAGE)).timeit(number=count_) diff --git a/tests-performance/flask_performance_test_v2.py b/tests-performance/flask_performance_test_v2.py index bb0f437..30ba879 100644 --- a/tests-performance/flask_performance_test_v2.py +++ b/tests-performance/flask_performance_test_v2.py @@ -1,4 +1,3 @@ -import json_logging import logging import os import sys @@ -6,12 +5,14 @@ from timeit import Timer from flask import Flask +from flask import request as request + +import json_logging logger = logging.getLogger("test logger") logger.setLevel(logging.DEBUG) logger.addHandler(StreamHandler(sys.stdout)) app = Flask(__name__) -from flask import request as request json_logging.init(framework_name='flask') json_logging.init_request_instrument(app)