From 729f8c9fe58c773df76154854e9947e4408e2153 Mon Sep 17 00:00:00 2001 From: eliasgranderubio Date: Thu, 2 Aug 2018 17:35:58 +0200 Subject: [PATCH] Issue #32: Enhanced debug logging --- dagda/api/dagda_server.py | 6 ++++- dagda/api/internal/internal_server.py | 10 ++++++++ dagda/cli/command/start_cli_parser.py | 8 ++++++- dagda/cli/dagda_cli.py | 3 ++- dagda/log/dagda_logger.py | 4 +++- dagda/vulnDB/db_composer.py | 34 +++++++++++++++++++++++++++ 6 files changed, 61 insertions(+), 4 deletions(-) diff --git a/dagda/api/dagda_server.py b/dagda/api/dagda_server.py index e0d8f36..a38e0c4 100644 --- a/dagda/api/dagda_server.py +++ b/dagda/api/dagda_server.py @@ -20,6 +20,7 @@ import os import json import datetime +import traceback from flask import Flask from flask_cors import CORS, cross_origin from api.internal.internal_server import InternalServer @@ -55,10 +56,11 @@ class DagdaServer: # DagdaServer Constructor def __init__(self, dagda_server_host='127.0.0.1', dagda_server_port=5000, mongodb_host='127.0.0.1', mongodb_port=27017, mongodb_ssl=False, mongodb_user=None, mongodb_pass=None, - falco_rules_filename=None, external_falco_output_filename=None): + falco_rules_filename=None, external_falco_output_filename=None, debug_logging=False): super(DagdaServer, self).__init__() self.dagda_server_host = dagda_server_host self.dagda_server_port = dagda_server_port + InternalServer.set_debug_logging_enabled(debug_logging) InternalServer.set_mongodb_driver(mongodb_host, mongodb_port, mongodb_ssl, mongodb_user, mongodb_pass) self.sysdig_falco_monitor = SysdigFalcoMonitor(InternalServer.get_docker_driver(), InternalServer.get_mongodb_driver(), @@ -150,6 +152,8 @@ def _init_or_update_db(): except Exception as ex: message = "Unexpected exception of type {0} occured: {1!r}".format(type(ex).__name__, ex.args) DagdaLogger.get_logger().error(message) + if InternalServer.is_debug_logging_enabled(): + traceback.print_exc() InternalServer.get_mongodb_driver().insert_init_db_process_status( {'status': message, 'timestamp': datetime.datetime.now().timestamp()}) diff --git a/dagda/api/internal/internal_server.py b/dagda/api/internal/internal_server.py index 3288cee..9536be6 100644 --- a/dagda/api/internal/internal_server.py +++ b/dagda/api/internal/internal_server.py @@ -32,6 +32,7 @@ class InternalServer: _mongodb_driver = MongoDbDriver() _docker_driver = DockerDriver() _external_falco = False + _debug_logging = False # -- Static methods @@ -82,3 +83,12 @@ def is_runtime_analysis_enabled(): return InternalServer._external_falco or \ len(InternalServer._docker_driver.get_docker_container_ids_by_image_name('sysdig/falco')) > 0 + # Sets if debug logging is enabled + @staticmethod + def set_debug_logging_enabled(debug_logging): + InternalServer._debug_logging = debug_logging + + # Is debug logging enabled + @staticmethod + def is_debug_logging_enabled(): + return InternalServer._debug_logging diff --git a/dagda/cli/command/start_cli_parser.py b/dagda/cli/command/start_cli_parser.py index 1e5571e..3ee59eb 100644 --- a/dagda/cli/command/start_cli_parser.py +++ b/dagda/cli/command/start_cli_parser.py @@ -31,6 +31,7 @@ class StartCLIParser: def __init__(self): super(StartCLIParser, self).__init__() self.parser = DagdaStartParser(prog='dagda.py start', usage=start_parser_text) + self.parser.add_argument('-d','--debug', action='store_true') self.parser.add_argument('-s', '--server_host', type=str) self.parser.add_argument('-p', '--server_port', type=int) self.parser.add_argument('-m', '--mongodb_host', type=str) @@ -48,6 +49,10 @@ def __init__(self): # -- Getters + # Gets if debug logging is required + def is_debug_logging_required(self): + return self.args.debug + # Gets server host def get_server_host(self): return self.args.server_host @@ -140,7 +145,7 @@ def format_help(self): # Custom text -start_parser_text = '''usage: dagda.py start [-h] [--server_host SERVER_HOST] [--server_port SERVER_PORT] +start_parser_text = '''usage: dagda.py start [-h] [-d] [--server_host SERVER_HOST] [--server_port SERVER_PORT] [--mongodb_host MONGODB_HOST] [--mongodb_port MONGODB_PORT] [--mongodb_ssl] [--mongodb_user MONGODB_USER] [--mongodb_pass MONGODB_PASS] [--falco_rules_file RULES_FILE] [--external_falco OUTPUT_FILE] @@ -149,6 +154,7 @@ def format_help(self): Optional Arguments: -h, --help show this help message and exit + -d, --debug enable debug logging -s SERVER_HOST, --server_host SERVER_HOST address/interface where the server binds itself. By diff --git a/dagda/cli/dagda_cli.py b/dagda/cli/dagda_cli.py index 0df4f00..d3755b6 100644 --- a/dagda/cli/dagda_cli.py +++ b/dagda/cli/dagda_cli.py @@ -38,7 +38,8 @@ def execute_dagda_cmd(cmd, args): mongodb_user=args.get_mongodb_user(), mongodb_pass=args.get_mongodb_pass(), falco_rules_filename=args.get_falco_rules_filename(), - external_falco_output_filename=args.get_external_falco_output_filename()) + external_falco_output_filename=args.get_external_falco_output_filename(), + debug_logging=args.is_debug_logging_required()) ds.run() # Executes agent sub-command diff --git a/dagda/log/dagda_logger.py b/dagda/log/dagda_logger.py index 85d9c53..53d5527 100644 --- a/dagda/log/dagda_logger.py +++ b/dagda/log/dagda_logger.py @@ -25,8 +25,10 @@ class DagdaLogger(logging.Logger): # -- Init - logging.basicConfig(format='<%(asctime)s> <%(levelname)s> <%(module)s> <%(message)s>') + logging.basicConfig(format='<%(asctime)s> <%(levelname)s> <%(module)s> <%(funcName)s:%(lineno)d> ' + + '<%(message)s>') _logger = logging.getLogger('DagdaLogger') + _logger.setLevel('DEBUG') # -- Static methods diff --git a/dagda/vulnDB/db_composer.py b/dagda/vulnDB/db_composer.py index 787cf71..58d4112 100644 --- a/dagda/vulnDB/db_composer.py +++ b/dagda/vulnDB/db_composer.py @@ -19,6 +19,7 @@ import io from datetime import date +from log.dagda_logger import DagdaLogger from api.internal.internal_server import InternalServer from vulnDB.ext_source_util import get_bug_traqs_lists_from_file from vulnDB.ext_source_util import get_bug_traqs_lists_from_online_mode @@ -47,10 +48,19 @@ def __init__(self): # Compose vuln DB def compose_vuln_db(self): + if InternalServer.is_debug_logging_enabled(): + DagdaLogger.get_logger().debug('ENTRY to the method for composing VulnDB') + # -- CVE # Adding or updating CVEs + if InternalServer.is_debug_logging_enabled(): + DagdaLogger.get_logger().debug('Updating CVE collection ...') + first_year = self.mongoDbDriver.remove_only_cve_for_update() for i in range(first_year, next_year): + if InternalServer.is_debug_logging_enabled(): + DagdaLogger.get_logger().debug('... Including CVEs - ' + str(i)) + compressed_content = get_http_resource_content( "https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.gz") cve_list = get_cve_list_from_file(compressed_content, i) @@ -68,8 +78,14 @@ def compose_vuln_db(self): if len(cve_ext_info_list) > 0: self.mongoDbDriver.bulk_insert_cves_info(cve_ext_info_list) + if InternalServer.is_debug_logging_enabled(): + DagdaLogger.get_logger().debug('CVE collection updated') + # -- Exploit DB # Adding or updating Exploit_db and Exploit_db info + if InternalServer.is_debug_logging_enabled(): + DagdaLogger.get_logger().debug('Updating Exploit DB collection ...') + self.mongoDbDriver.delete_exploit_db_collection() self.mongoDbDriver.delete_exploit_db_info_collection() csv_content = get_http_resource_content( @@ -78,8 +94,14 @@ def compose_vuln_db(self): self.mongoDbDriver.bulk_insert_exploit_db_ids(exploit_db_list) self.mongoDbDriver.bulk_insert_exploit_db_info(exploit_db_info_list) + if InternalServer.is_debug_logging_enabled(): + DagdaLogger.get_logger().debug('Exploit DB collection updated') + # -- BID # Adding BugTraqs from 20180328_sf_db.json.gz, where 103525 is the max bid in the gz file + if InternalServer.is_debug_logging_enabled(): + DagdaLogger.get_logger().debug('Updating BugTraqs Id collection ...') + max_bid = self.mongoDbDriver.get_max_bid_inserted() if max_bid < 103525: # Clean @@ -113,8 +135,14 @@ def compose_vuln_db(self): self.mongoDbDriver.bulk_insert_bid_info(bid_detail_array) bid_detail_array.clear() + if InternalServer.is_debug_logging_enabled(): + DagdaLogger.get_logger().debug('BugTraqs Id collection updated') + # -- RHSA (Red Hat Security Advisory) and RHBA (Red Hat Bug Advisory) # Adding or updating rhsa and rhba collections + if InternalServer.is_debug_logging_enabled(): + DagdaLogger.get_logger().debug('Updating RHSA & RHBA collections ...') + self.mongoDbDriver.delete_rhba_collection() self.mongoDbDriver.delete_rhba_info_collection() self.mongoDbDriver.delete_rhsa_collection() @@ -125,3 +153,9 @@ def compose_vuln_db(self): self.mongoDbDriver.bulk_insert_rhba(rhba_list) self.mongoDbDriver.bulk_insert_rhsa_info(rhsa_info_list) self.mongoDbDriver.bulk_insert_rhba_info(rhba_info_list) + + if InternalServer.is_debug_logging_enabled(): + DagdaLogger.get_logger().debug('RHSA & RHBA collections updated') + + if InternalServer.is_debug_logging_enabled(): + DagdaLogger.get_logger().debug('EXIT from the method for composing VulnDB') \ No newline at end of file