Logging enhancements

fakenet/configs/default.ini

@@ -112,11 +112,71 @@ BlackListPortsUDP: 67, 68, 137, 138, 443, 1900, 5355
 # Specify hosts to ignore when diverting traffic.
 # HostBlackList: 6.6.6.6
 
+###############################################################################
+# Multiple remote loggers can be specified.
+# Each remote logger section name must be unique and start with 'RemoteLogger'.
+#
+# [RemoteLogger:description]
+# Logger_Type - [syslog|splunk]     (required)
+# logger_host - [ip,name,fqdn,/dev/log]      (required)
+# Logger_Port - 0-65535             (optional)
+# Logger_Protocol - [TCP|UDP]       (optional)
+# https://docs.python.org/2/library/logging.html#logging-levels
+# Logger_Level - [DEBUG|INFO|WARNING|ERROR|CRITICAL]    (optional)
+#
+######
+# Syslog Example
+#
+# [RemoteLogger:syslogUDP]
+# Logger_Type : syslog
+# Logger_Host : logger.domain.tld
+# Logger_Port : 514
+# Logger_Level: INFO
+# Logger_Protocol : UDP
+# JSON_Only : 0
+#
+# [RemoteLogger:syslogSocket]
+# Logger_Type : syslog
+# Logger_Host : /dev/log
+# Logger_Level: INFO
+# JSON_Only : 0
+
+# Splunk specific configurations
+# Requires splunk_http_handler python module ('pip install splunk_http_handler')
+# If SSL capability is required, see https://github.com/vavarachen/splunk_http_handler
+# http://docs.splunk.com/Documentation/SplunkCloud/latest/Data/UsetheHTTPEventCollector
+# Splunk_HECToken - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx (required)
+#
+# Splunk_Source - Any valid splunk source name  (optional)
+#
+# Replace with custom sourcetype as needed
+# Splunk_Sourcetype - _json             (optional)
+#
+# Splunk_Cert_verify - [0|1]     (optional)
+# 0 - Don't very SSL cert
+# 1 - Verify SSL cert
+#
+# JSON_Only - [0|1]
+# 0 - Log all events
+# 1 - Only log events in JSON format
+######
+# Splunk Example
+#
+# [RemoteLogger:splunk]
+# Logger_Type: splunk
+# Logger_Host : localhost
+# Logger_Port: 8080
+# Logger_Level: INFO
+# Splunk_Cert_verify : 0
+# Splunk_HECToken : xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+# Splunk_Source : FakeNet
+# JSON_Only : 1
+
 ###############################################################################
 # Listener Configuration
 #
 # Listener configuration consists of generic settings used by the diverter which
-# are the same for all listeners and listener specific settings. 
+# are the same for all listeners and listener specific settings.
 #
 # NOTE: Listener section names will be used for logging.
 #

fakenet/diverters/diverterbase.py

@@ -16,7 +16,7 @@ class DiverterBase(fnconfig.Config):
 
 
     def init_base(self, diverter_config, listeners_config, ip_addrs,
-                  logging_level=logging.INFO):
+                  logger=None, logging_level=logging.DEBUG):
         # For fine-grained control of subclass debug output. Does not control
         # debug output from DiverterBase. To see DiverterBase debug output,
         # pass logging.DEBUG as the logging_level argument to init_base.
@@ -31,7 +31,10 @@ def init_base(self, diverter_config, listeners_config, ip_addrs,
         self.pcap_filename = ''
         self.pcap_lock = None
 
-        self.logger = logging.getLogger('Diverter')
+        if logger is not None:
+            self.logger = logger
+        else:
+            self.logger = logging.getLogger('Diverter')
         self.logger.setLevel(logging_level)
 
         portlists = ['BlackListPortsTCP', 'BlackListPortsUDP']
@@ -410,12 +413,12 @@ def parse_diverter_config(self):
             else:
                 self.default_listener['TCP'] = int(
                     self.listeners_config[self.getconfigval('defaulttcplistener').lower()]['port'])
-                self.logger.error('Using default listener %s on port %d', self.getconfigval(
+                self.logger.error('Using default listener %s on port %d.', self.getconfigval(
                     'defaulttcplistener').lower(), self.default_listener['TCP'])
 
                 self.default_listener['UDP'] = int(
                     self.listeners_config[self.getconfigval('defaultudplistener').lower()]['port'])
-                self.logger.error('Using default listener %s on port %d', self.getconfigval(
+                self.logger.error('Using default listener %s on port %d.', self.getconfigval(
                     'defaultudplistener').lower(), self.default_listener['UDP'])
 
             # Re-marshall these into a readily usable form...

fakenet/diverters/linux.py

@@ -157,7 +157,7 @@ def handle_pkt(self):
 
                     if ((not (self.diverter.pdebug_level & DGENPKTV)) and
                         pid and (pid != self.diverter.pid)):
-                        self.logger.info('  pid:  %d name: %s' %
+                        self.logger.debug('  pid:  %d name: %s' %
                                          (pid, comm if comm else 'Unknown'))
 
                     hdr_latest = self.hdr
@@ -200,9 +200,9 @@ class Diverter(DiverterBase, LinUtilMixin):
 
 
     def __init__(self, diverter_config, listeners_config, ip_addrs,
-                 logging_level=logging.INFO):
+                 logger=None, logging_level=logging.INFO):
         self.init_base(diverter_config, listeners_config, ip_addrs,
-                       logging_level)
+                       logger, logging_level)
 
         self.init_linux_mixin()
         self.init_diverter_linux()

fakenet/fakenet.py

@@ -18,6 +18,8 @@
 from optparse import OptionParser,OptionGroup
 from ConfigParser import ConfigParser
 
+from listeners import ListenerBase
+
 import platform
 
 from optparse import OptionParser
@@ -43,6 +45,9 @@ def __init__(self, logging_level = logging.INFO):
         # Diverter used to intercept and redirect traffic
         self.diverter = None
 
+        # Splunk logging options and parameters
+        self.splunk_config = dict()
+
         # FakeNet options and parameters
         self.fakenet_config = dict()
 
@@ -72,6 +77,8 @@ def parse_config(self, config_filename):
 
         config = ConfigParser()
         config.read(config_filename)
+        # For list of valid remote logger configs
+        self.remoteloggers = {}
 
         self.logger.info('Loaded configuration file: %s', config_filename)
 
@@ -84,6 +91,12 @@ def parse_config(self, config_filename):
             elif section == 'Diverter':
                 self.diverter_config = dict(config.items(section))
 
+            elif section.startswith('RemoteLogger'):
+                remotelogger_config = dict(config.items(section))
+                if ListenerBase.add_remote_logger(self.logger, remotelogger_config):
+                    # for adding to listeners when we initialize them
+                    self.remoteloggers[section] = remotelogger_config
+
             elif config.getboolean(section, 'enabled'):
                 self.listeners_config[section] = dict(config.items(section))
 
@@ -144,21 +157,21 @@ def start(self):
                 # Check Windows version
                 if platform.release() in ['2000', 'XP', '2003Server', 'post2003']:
                     self.logger.error('Error: FakeNet-NG only supports Windows Vista+.')
-                    self.logger.error('       Please use the original Fakenet for older versions of Windows.')
+                    self.logger.error('Please use the original Fakenet for older versions of Windows.')
                     sys.exit(1)
 
                 if self.diverter_config['networkmode'].lower() == 'auto':
                     self.diverter_config['networkmode'] = 'singlehost'
 
                 from diverters.windows import Diverter
-                self.diverter = Diverter(self.diverter_config, self.listeners_config, self.logging_level)
+                self.diverter = Diverter(self.diverter_config, self.listeners_config, self.logger)
 
             elif platform_name.lower().startswith('linux'):
                 if self.diverter_config['networkmode'].lower() == 'auto':
                     self.diverter_config['networkmode'] = 'multihost'
 
                 from diverters.linux import Diverter
-                self.diverter = Diverter(self.diverter_config, self.listeners_config, ip_addrs, self.logging_level)
+                self.diverter = Diverter(self.diverter_config, self.listeners_config, ip_addrs, self.logger)
 
             else:
                 self.logger.error('Error: Your system %s is currently not supported.', platform_name)
@@ -168,6 +181,8 @@ def start(self):
         for listener_name in self.listeners_config:
 
             listener_config = self.listeners_config[listener_name]
+            # Pass remote logger configs in case we want to enable it for listener
+            listener_config.update(self.remoteloggers)
 
             # Anonymous listener
             if not 'listener' in listener_config:

fakenet/listeners/BITSListener.py

@@ -1,6 +1,7 @@
 # Based on a simple BITS server by Dor Azouri <[email protected]>
 
 import logging
+import ListenerBase
 
 import os
 import sys
@@ -203,6 +204,7 @@ class SimpleBITSRequestHandler(SimpleHTTPRequestHandler):
     supported_protocols = ["{7df0354d-249b-430f-820d-3d2a9bef4931}"]  # The only existing protocol version to date
     fragment_size_limit = 100*1024*1024  # bytes
 
+
     def do_HEAD(self):
         self.server.logger.info('Received HEAD request')
 
@@ -434,6 +436,40 @@ def do_BITS_POST(self):
             repr(e.internal_exception))
         self.__send_response(headers, status_code = status_code)
 
+    def log_message(self, format, *args):
+        """Construct CIM compliant log message as a dict object which would be indexed in splunk as json"""
+
+        # http://docs.splunk.com/Documentation/CIM/4.9.1/User/Web
+        if 'user-agent' in self.headers.dict.keys():
+            self.headers.dict['http_user_agent'] = self.headers.dict.pop('user-agent')
+            self.headers.dict['http_user_agent_length'] = len(self.headers.dict['http_user_agent'])
+
+        if 'referrer' in self.headers.dict.keys():
+            self.headers.dict['http_referrer'] = self.headers.dict.pop('referrer')
+
+        if 'host' in self.headers.dict.keys():
+            self.headers.dict['site'] = self.headers.dict.pop('host')
+
+        try:
+            # Advertised fake web server signature
+            self.headers.dict['vendor'] = self.server.config.version
+        except:
+            pass
+
+        try:
+            self.headers.dict['protocol'] = self.server.config.protocol.lower()
+        except:
+            self.headers.dict['protocol'] = 'tcp'
+
+        logmsg = dict({'src': self.client_address[0], 'src_port':self.client_address[1], 'dest_port': self.server.server_port,
+                       'ssl':self.server.config['usessl'], 'http_method': self.command, 'http_header': self.headers.dict,
+                       'uri_query': self.path, 'http_protocol_version': self.protocol_version, 'listener': __name__})
+        if self.command == 'POST':
+            logmsg['post_body'] = self.post_body
+
+        self.server.logger.info(logmsg)
+        return
+
 class BITSListener():
 
     def taste(self, data, dport):
@@ -450,9 +486,8 @@ def taste(self, data, dport):
 
     def __init__(self, config={}, name='BITSListener', 
             logging_level=logging.DEBUG, running_listeners=None):
-        self.logger = logging.getLogger(name)
-        self.logger.setLevel(logging_level)
-
+
+        self.logger = ListenerBase.set_logger("%s:%s" % (self.__module__, name), config, logging_level)
         self.config = config
         self.name = name
         self.local_ip  = '0.0.0.0'
@@ -461,7 +496,8 @@ def __init__(self, config={}, name='BITSListener',
         self.NAME = 'BITS'
         self.PORT = self.config.get('port')
 
-        self.logger.info('Starting...')
+        ssl_str = 'HTTPS' if self.config.get('usessl') == 'Yes' else 'HTTP'
+        self.logger.info('Starting %s server on %s:%s' % (ssl_str, self.local_ip, self.config.get('port')))
 
         self.logger.debug('Initialized with config:')
         for key, value in config.iteritems():
@@ -470,8 +506,6 @@ def __init__(self, config={}, name='BITSListener',
         self.bits_file_prefix = self.config.get('bitsfileprefix', 'bits')
 
     def start(self):
-        self.logger.debug('Starting...')
-
         self.server = ThreadedHTTPServer((self.local_ip, int(self.config.get('port'))), SimpleBITSRequestHandler)
         self.server.logger = self.logger
         self.server.bits_file_prefix = self.bits_file_prefix
@@ -497,7 +531,8 @@ def start(self):
         self.server_thread.start()        
 
     def stop(self):
-        self.logger.info('Stopping...')
+        http_str = 'HTTPS' if self.config.get('usessl') == 'Yes' else 'HTTP'
+        self.logger.info('Stopping %s server on %s:%s' % (http_str, self.local_ip, self.config.get('port')))
         if self.server:
             self.server.shutdown()
             self.server.server_close()

fakenet/listeners/DNSListener.py

@@ -9,6 +9,7 @@
 import socket
 
 from . import *
+import ListenerBase
 
 class DNSListener():
 
@@ -30,16 +31,14 @@ def __init__(
             logging_level=logging.INFO, 
             ):
 
-        self.logger = logging.getLogger(name)
-        self.logger.setLevel(logging_level)
-
+        self.logger = ListenerBase.set_logger("%s:%s" % (self.__module__, name), config, logging_level)
         self.config = config
         self.local_ip = '0.0.0.0'
         self.server = None
         self.name = 'DNS'
         self.port = self.config.get('port', 53)
 
-        self.logger.info('Starting...')
+        self.logger.info('Starting %s %s server on %s:%s' % (self.config['protocol'], self.name, self.local_ip, self.config.get('port')))
 
         self.logger.debug('Initialized with config:')
         for key, value in config.iteritems():
@@ -49,12 +48,10 @@ def start(self):
 
         # Start UDP listener  
         if self.config['protocol'].lower() == 'udp':
-            self.logger.debug('Starting UDP ...')
             self.server = ThreadedUDPServer((self.local_ip, int(self.config.get('port', 53))), self.config, self.logger, UDPHandler)
 
         # Start TCP listener
         elif self.config['protocol'].lower() == 'tcp':
-            self.logger.debug('Starting TCP ...')
             self.server = ThreadedTCPServer((self.local_ip, int(self.config.get('port', 53))), self.config, self.logger, TCPHandler)
 
         self.server.nxdomains = int(self.config.get('nxdomains', 0))
@@ -64,8 +61,8 @@ def start(self):
         self.server_thread.start()
 
     def stop(self):
-        self.logger.debug('Stopping...')
-        
+        self.logger.info('Stopping %s %s server on %s:%s' % (self.config['protocol'], self.name, self.local_ip, self.config.get('port')))
+
         # Stop listener
         if self.server:
             self.server.shutdown()
@@ -177,7 +174,17 @@ def parse(self,data):
                     response.add_answer(RR(qname, getattr(QTYPE,qtype), rdata=RDMAP[qtype](fake_record)))
 
                 response = response.pack()
-
+
+                logmsg = dict({'src': self.client_address[0], 'src_port':self.client_address[1],
+                               'dest_port': self.server.config['port'], 'protocol': self.server.config['protocol'],
+                               'query_type': qtype, 'query': qname, 'listener': __name__})
+                if qtype in ['A', 'MX', 'TXT']:
+                    logmsg['answer'] = fake_record
+                else:
+                    logmsg['answer'] = "Not Implemented"
+
+                self.server.logger.info(logmsg)
+
         return response  
 
 class UDPHandler(DNSHandler, SocketServer.BaseRequestHandler):