diff --git a/Dockerfile b/Dockerfile index d3f5b6f..77e2295 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12.0-slim-bookworm +FROM python:3.12.2-slim-bookworm LABEL MAINTAINER="Mike Schiessl - mike.schiessl@akamai.com" LABEL APP_LONG="Akamai Universal Log Streamer" LABEL APP_SHORT="ULS" @@ -13,7 +13,7 @@ ARG EXT_DIR="$ULS_DIR/ext" ARG ETP_CLI_VERSION="0.4.5" ARG EAA_CLI_VERSION="0.6.3" ARG MFA_CLI_VERSION="0.1.1" -ARG GC_CLI_VERSION="v0.0.2(beta)" +ARG GC_CLI_VERSION="v0.0.3(beta)" ARG LINODE_CLI_VERSION="dev" # ENV VARS diff --git a/README.md b/README.md index 3d5be42..4db740a 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The Unified Log Streamer (ULS) is designed to simplify SIEM integrations for Akamai Secure Enterprise Access Products - [Enterprise Application Access (EAA)](https://www.akamai.com/us/en/products/security/enterprise-application-access.jsp) -- [Enterprise Threat Protector (ETP)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp) +- [Secure Internet Access Enterprise (SIA)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp) - [Akamai MFA (MFA)](https://www.akamai.com/us/en/products/security/akamai-mfa.jsp) - [Guardicore Micro Segmentation](https://www.akamai.com/lp/guardicore) @@ -39,7 +39,7 @@ It can be run directly as Python code, as a provided Docker container, through - [CONHEALTH](docs/LOG_OVERVIEW.md#connector-health-conhealth) - [DEVINV](docs/LOG_OVERVIEW.md#device-inventory-devinv) - [DIRHEALTH](docs/LOG_OVERVIEW.md#directory-health-dirhealth) - - [Enterprise Threat Protectors (ETP)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp) + - [Secure Internet Access Enterprise (SIA) (formerly Enterprise Threat Protector)](https://www.akamai.com/products/secure-internet-access-enterprise) - [THREAT](docs/LOG_OVERVIEW.md#threat-log-threat) - [AUP](docs/LOG_OVERVIEW.md#accceptable-use-policy-logs-aup) - [DNS](docs/LOG_OVERVIEW.md#dns) @@ -53,7 +53,7 @@ It can be run directly as Python code, as a provided Docker container, through - AGENT - SYSTEM - [Linode](https://www.linode.com/) (experimental) - - AUDIT + - [AUDIT](docs/LOG_OVERVIEW.md#audit-logs) - Supported data outputs @@ -93,6 +93,7 @@ Anyway, details for some specific SIEM solutions can be found in [this directory - AKAMAI .edgerc file ([see further documentation here](docs/AKAMAI_API_CREDENTIALS.md)) - Outbound Internet access (Akamai API, Dockerhub OR Python repositories) - Compute resource footprint is minimal: 1 vCPU, 1 GB RAM, 500 MB free disk. See also [High availability](docs/HIGH_AVAILABILITY.md) doc. +- NTP synchronized time on the OS ULS will be executed. ### Command Line Usage ![ULS command line usage](docs/images/uls_cli_help_example.png) diff --git a/bin/modules/UlsArgsParser.py b/bin/modules/UlsArgsParser.py index 6944c80..d0f907d 100644 --- a/bin/modules/UlsArgsParser.py +++ b/bin/modules/UlsArgsParser.py @@ -16,20 +16,28 @@ import os from distutils.util import strtobool -import config.global_config as uls_config +import uls_config.global_config as uls_config def init(): # Argument Parsing parser = argparse.ArgumentParser(description=f"{uls_config.__tool_name_long__}", formatter_class=argparse.RawTextHelpFormatter) - # Common params + # Loglevel parser.add_argument('-l', '--loglevel', action='store', type=str.upper, default=(os.environ.get('ULS_LOGLEVEL') or uls_config.log_level_default), choices=uls_config.log_levels_available, help=f'Adjust the loglevel Default: {uls_config.log_level_default}') + # put loglines into debug log + parser.add_argument('--debugloglines', + action='store', + type=bool, + nargs='?', + default=(os.environ.get('ULS_DEBUGLOGLINES') or uls_config.log_debugloglines_default), + const=True, + help=f"Should the log_lines appear in debug log? (Default: {uls_config.log_debugloglines_default})") # Version Information parser.add_argument('-v', '--version', diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py index 7536e20..a633584 100644 --- a/bin/modules/UlsInputCli.py +++ b/bin/modules/UlsInputCli.py @@ -21,7 +21,7 @@ import queue # ULS modules -import config.global_config as uls_config +import uls_config.global_config as uls_config import modules.aka_log as aka_log import modules.UlsTools as UlsTools @@ -210,7 +210,7 @@ def proc_create(self): shlex.split(self.rawcmd) # ETP config - elif self.product == "ETP": + elif self.product == "ETP" or self.product == "SIA": product_path = self.root_path + "/" + uls_config.bin_etp_cli product_feeds = uls_config.etp_cli_feeds @@ -229,9 +229,11 @@ def proc_create(self): # Append End and Starttime if self.endtime: # We need to remove "-f" from the end of the cli cmd if we work with endtime + aka_log.log.debug(f"Internally set my starttime to: {self.endtime}") cli_command = cli_command[:-1] cli_command.extend(self._prep_start_endtime('--end', self.endtime)) if self.starttime: + aka_log.log.debug(f"Internally set my starttime to: {self.starttime}") cli_command.extend(self._prep_start_endtime('--start', self.starttime)) else: @@ -328,6 +330,9 @@ def proc_create(self): cli_command = [self.bin_python, product_path] +\ self._uls_useragent(self.product, "rawcmd") +\ shlex.split(self.rawcmd) + # Mocked output + elif self.product == "MOCK": + print ("Not yet there") # Everything else (undefined) diff --git a/bin/modules/UlsMonitoring.py b/bin/modules/UlsMonitoring.py index 36e9cdc..a486c0c 100644 --- a/bin/modules/UlsMonitoring.py +++ b/bin/modules/UlsMonitoring.py @@ -19,7 +19,7 @@ import sys import modules.aka_log as aka_log -import config.global_config as uls_config +import uls_config.global_config as uls_config class UlsMonitoring: diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py index 21a78d1..10dde6f 100644 --- a/bin/modules/UlsOutput.py +++ b/bin/modules/UlsOutput.py @@ -25,7 +25,7 @@ import json # ULS specific modules -import config.global_config as uls_config +import uls_config.global_config as uls_config import modules.aka_log as aka_log diff --git a/bin/modules/UlsTools.py b/bin/modules/UlsTools.py index 584c0ae..2579b40 100644 --- a/bin/modules/UlsTools.py +++ b/bin/modules/UlsTools.py @@ -23,7 +23,7 @@ # ULS modules import modules.aka_log as aka_log -import config.global_config as uls_config +import uls_config.global_config as uls_config def uls_check_sys(root_path, uls_input=None): @@ -48,7 +48,7 @@ def _check_cli_installed(cli_bin): aka_log.log.critical(f"Error checking the cli'tools ") if uls_input == "EAA": _check_cli_installed(root_path + "/" + uls_config.bin_eaa_cli) - elif uls_input == "ETP": + elif uls_input == "ETP" or uls_input == "SIA": _check_cli_installed(root_path + "/" + uls_config.bin_etp_cli) elif uls_input == "MFA": _check_cli_installed(root_path + "/" + uls_config.bin_mfa_cli) @@ -97,7 +97,7 @@ def _get_cli_version(cli_bin, edgerc_mock_file): print(f"{uls_config.__tool_name_long__} Version information\n" f"ULS Version\t\t{uls_config.__version__}\n\n" f"EAA Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_eaa_cli, my_edgerc_mock_file)}\n" - f"ETP Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_etp_cli, my_edgerc_mock_file)}\n" + f"SIA/ETP Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_etp_cli, my_edgerc_mock_file)}\n" f"MFA Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_mfa_cli, my_edgerc_mock_file)}\n" f"GC Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_gc_cli, my_edgerc_mock_file)}\n" f"LINODE Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_linode_cli, my_edgerc_mock_file)}\n\n" @@ -187,7 +187,10 @@ def root_path(): def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoint_path): # Check if we're in a supported stream / feed - if input not in uls_config.autoresume_supported_inputs or feed == "CONHEALTH" or feed == "DEVINV" : + if (input not in uls_config.autoresume_supported_inputs or + feed == "CONHEALTH" or + feed == "DEVINV" or + feed == "DIRHEALTH"): aka_log.log.critical(f"Input {input} or feed {feed} currently not supported by AUTORESUME - Exiting.") sys.exit(1) @@ -216,13 +219,15 @@ def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoin aka_log.log.critical( f"Unhandeled input data in checkpointfile \'{checkpoint_full}\' --> {input} / {feed} - Exiting.") sys.exit(1) - checkpoint = int(datetime.datetime(year=int(mytime.split("T")[0].split("-")[0]), - month=int(mytime.split("T")[0].split("-")[1]), - day=int(mytime.split("T")[0].split("-")[2]), - hour=int(mytime.split("T")[1].split(":")[0]), - minute=int(mytime.split("T")[1].split(":")[1]), - second=int(mytime.split("T")[1].split(":")[2]), - ).timestamp()) + my_timestamp = datetime.datetime(year=int(mytime.split("T")[0].split("-")[0]), + month=int(mytime.split("T")[0].split("-")[1]), + day=int(mytime.split("T")[0].split("-")[2]), + hour=int(mytime.split("T")[1].split(":")[0]), + minute=int(mytime.split("T")[1].split(":")[1]), + second=int(mytime.split("T")[1].split(":")[2]), + ) + checkpoint = int(my_timestamp.replace(tzinfo=datetime.timezone.utc).timestamp()) + aka_log.log.debug(f"Checkpoint timestamp {data['checkpoint']} converted to epoch time {checkpoint}") else: aka_log.log.critical(f"Inconsitent data in checkpointfile \'{checkpoint_full}\' --> {data} - Exiting.") @@ -245,7 +250,7 @@ def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoin return {'filename': checkpoint_full, 'creation_time': creation_time, 'checkpoint': checkpoint} -def write_autoresume_ckpt(input, feed, autoresume_file, logline): +def write_autoresume_ckpt(input, feed, autoresume_file, logline, current_count): aka_log.log.info(f"AUTORESUME - IT's time to write a new checkpoint") # Adopt the field to the stream / feed @@ -256,6 +261,8 @@ def write_autoresume_ckpt(input, feed, autoresume_file, logline): checkpoint_timestamp = json.loads(checkpoint_line)['query']['time'] elif input == "EAA" and feed == "ACCESS": checkpoint_timestamp = json.loads(checkpoint_line)['datetime'] + elif input == "ETP" and feed == "NETCON": + checkpoint_timestamp = json.loads(checkpoint_line)['connStartTime'] else: aka_log.log.critical( f"AUTORESUME - Unhandled Input / Feed detected: '{input} / {feed}' (this should never happen !!)- Exiting") @@ -263,7 +270,7 @@ def write_autoresume_ckpt(input, feed, autoresume_file, logline): # Write out the file try: - autoresume_data = {'creation_time': str(datetime.datetime.now()), 'checkpoint': str(checkpoint_timestamp), 'input': input, 'feed': feed} + autoresume_data = {'creation_time': str(datetime.datetime.now()), 'checkpoint': str(checkpoint_timestamp), 'input': input, 'feed': feed, 'current_count': current_count} with open(autoresume_file, "w") as ckpt_fd: json.dump(autoresume_data, ckpt_fd) aka_log.log.debug(f"AUTORESUME - Wrote a new checkpoint to {autoresume_file}: {autoresume_data}") diff --git a/bin/modules/UlsTransformation.py b/bin/modules/UlsTransformation.py index 46dc3c2..d371c68 100644 --- a/bin/modules/UlsTransformation.py +++ b/bin/modules/UlsTransformation.py @@ -18,7 +18,7 @@ import time # ULS modules -import config.transformation_config as transform_config +import uls_config.transformation_config as transform_config import jmespath import modules.aka_log as aka_log diff --git a/bin/uls.py b/bin/uls.py index 824f874..c597f81 100755 --- a/bin/uls.py +++ b/bin/uls.py @@ -30,7 +30,7 @@ import modules.UlsMonitoring as UlsMonitoring import modules.UlsTransformation as UlsTransformation import modules.UlsTools as UlsTools -import config.global_config as uls_config +import uls_config.global_config as uls_config stopEvent = threading.Event() @@ -86,7 +86,6 @@ def main(): autoresume_data = UlsTools.check_autoresume(input=uls_args.input, feed=uls_args.feed, checkpoint_dir=uls_args.autoresumepath) uls_args.starttime = autoresume_data['checkpoint'] autoresume_file = autoresume_data['filename'] - # Check CLI Environment UlsTools.uls_check_sys(root_path=root_path, uls_input=uls_args.input) @@ -170,12 +169,19 @@ def main(): while not stopEvent.is_set(): try: input_data = event_q.get(block=True, timeout=0.05) - aka_log.log.debug(f" {input_data}") + if uls_args.debugloglines: + aka_log.log.debug(f" {input_data}") for log_line in input_data.splitlines(): # Write checkpoint to the checkpoint file (if autoresume is enabled) (not after transformation or filter) + if uls_args.autoresume and int(my_monitor.get_message_count()) >= autoresume_lastwrite + uls_args.autoresumewriteafter: - UlsTools.write_autoresume_ckpt(uls_args.input, uls_args.feed, autoresume_file, log_line) + aka_log.log.info(f"WRITING AUTORESUME CHECKPOINT - curr_message_count={int(my_monitor.get_message_count())} - last_write = {autoresume_lastwrite}") + UlsTools.write_autoresume_ckpt(uls_args.input, + uls_args.feed, + autoresume_file, + log_line, + current_count=int(my_monitor.get_message_count())) autoresume_lastwrite = int(my_monitor.get_message_count()) # Filter Enhancement @@ -206,7 +212,8 @@ def main(): # Send the data resend_status = my_output.send_data(log_line) my_monitor.increase_message_count(len(log_line)) - aka_log.log.debug(f" {log_line}") + if uls_args.debugloglines: + aka_log.log.debug(f" {log_line}") resend_counter = resend_counter + 1 if resend_counter == uls_config.main_resend_attempts and\ diff --git a/bin/config/global_config.py b/bin/uls_config/global_config.py similarity index 97% rename from bin/config/global_config.py rename to bin/uls_config/global_config.py index 2ef22bb..979ee62 100644 --- a/bin/config/global_config.py +++ b/bin/uls_config/global_config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Common global variables / constants -__version__ = "1.7.1" +__version__ = "1.7.2" __tool_name_long__ = "Akamai Unified Log Streamer" __tool_name_short__ = "ULS" @@ -40,7 +40,7 @@ linode_cli_feeds = ['AUDIT'] # Available LINODE CLI feeds # INPUT Choices -input_choices = ['EAA', 'ETP', 'MFA', 'GC', 'LINODE'] # Available input types +input_choices = ['EAA', 'ETP', 'SIA', 'MFA', 'GC', 'LINODE'] # Available input types input_format_choices = ['JSON', 'TEXT'] # Available input format choices (need to be supported by cli) # OUTPUT Choices @@ -53,6 +53,7 @@ # LogLevels log_levels_available = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] log_level_default = 'WARNING' +log_debugloglines_default = False # INPUT Configuration input_rerun_retries = 3 # Number of rerun attempts before giving up diff --git a/bin/config/transformation_config.py b/bin/uls_config/transformation_config.py similarity index 100% rename from bin/config/transformation_config.py rename to bin/uls_config/transformation_config.py diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md index 4b82b34..fab9783 100644 --- a/docs/ARGUMENTS_ENV_VARS.md +++ b/docs/ARGUMENTS_ENV_VARS.md @@ -1,17 +1,26 @@ # List of parameters / Environmental variables The following tables list all available command line parameters and their corresponding environmental variables (for advanced usage). +## Table of contents +- [Global](#global) +- [Input](#input) +- [Output](#output) +- [Special Arguments](#special-arguments) +- [Autoresume](#autoresume) + +--- ## Global + | Parameter | Env - Var | Options | Default | Description | |--------------------|--------------|-------------------------------------------------|---------|-----------------------------------------------------------| | -h
--help | n/a | n/a | None | Display help / usage information | | -l
--loglevel | ULS_LOGLEVEL | 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL' | WARNING | Adjust the overall loglevel | | -v
--version | n/a | n/a | None | Display ULS version information (incl. CLI & OS versions) | - - +| --debugloglines | ULS_DEBUGLOGLINES | 'True', 'False' | False | Should the debug log contain Loglines (useful to debug transformations) | ## INPUT + | Parameter | Env - Var | Options | Default | Description | |---------------------------|-----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | -i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA', 'GC', 'LINODE' | None | Specify the desired INPUT source | @@ -27,6 +36,7 @@ The following tables list all available command line parameters and their corres ## OUTPUT + | Parameter | Output Type | Env - Var | Options | Default | Description | |------------------|-------------|----------------------|----------------------------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | -o
--output | | ULS_OUTPUT | 'TCP', 'UDP', 'HTTP', 'RAW', 'FILE' | None | Specify the desired OUTPUT target | @@ -53,13 +63,16 @@ The following tables list all available command line parameters and their corres ## Special Arguments + | Parameter | Env - Var | Options | Default | Description | |-------------------------|----------------------------|------------------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | --filter | ULS_OUTPUT_FILTER | \ | None | Filter (regex) to reduce number of OUTPUT log lines
Only loglines **matching** the `--filter ` argument will bes sent to the output.
[Click here for more information](ADDITIONAL_FEATURES.md#filter---filter-feature) | | --transformation | ULS_TRANSFORMATION | 'MCAS', 'JMESPATH' | None | OPTIONAL: Specify an optional transformation to manipulate the output format
[Click here for more information](TRANSFORMATIONS.md) | | --transformationpattern | ULS_TRANSFORMATION_PATTERN | \ | None | Specifies the pattern used to transform the log event for the selected transformation. [Click here for more information](TRANSFORMATIONS.md) | + ## Autoresume + | Parameter | Env - Var | Options | Default | Description | |------------------------|---------------------------|-------------------------------|---------|---------------------------------------------------------------------------------------------------------------| | --autoresume | ULS_AUTORESUME | [True, False] | False | Enable automated resume on based on a checkpoint upon api failure or crash (do not use alongside --starttime) | diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f28862c..65f6110 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,10 +1,32 @@ # Version History +## v1.7.2 +||| +|---|---| +|Date|2024-02-08 +|Kind| MINOR release +|Author|mschiess@akamai.com +- **Minor improvements** + - Introduced **Secure Internet Access** (formerly ETP) as INPUT specification (as an alias to ETP) + - added "ETP NETCON" to the autoresume feature + - prevented "EAA DIRHEALTH" to be mistakenly autoresumed + - Imrpoved [log overview](LOG_OVERVIEW.md) readability + - Added `--debugloglines` to allow control of input loglines being sent to the debug log + - Added [Microsoft Sentinel SIA / ETP integration](SIEM/SENTINEL/Readme.md) documentation + - Added a FAQ entry [regarding time synchronization](FAQ.md#error-invalid-timestamp-on-api-call) + - [docker] bumped python version to "3.12.2" + - [docker] bumped gc_logs version to "0.0.3(beta)" + - Fixed a doc error (PR by @pizza0rodeo ) - thanks for your contribution +- **BUGFIX** + - Fixed a bug in the autoresume function that created a problem with timezones in certain circumstances +- **Housekeeping** + - improved local container testing + ## v1.7.1 ||| |---|---| |Date|2023-10-11 |Kind| BUGFIX release -|Author|mschiess@akamai.com, +|Author|mschiess@akamai.com - **BUGFIX** - Fixed a bug in the ETP & EAA CLI that prevented ULS to run properly in docker environment - [docker] bumped CLI-EAA to "0.6.3" diff --git a/docs/COMMAND_LINE_USAGE.md b/docs/COMMAND_LINE_USAGE.md index d7f3386..72ea121 100644 --- a/docs/COMMAND_LINE_USAGE.md +++ b/docs/COMMAND_LINE_USAGE.md @@ -13,7 +13,6 @@ All commands referenced in this document are run from the repositories root leve - [Clone ULS repository](#clone-uls-repository) - [Akamai Enterprise Access CLI's](#akamai-enterprise-access-clis) - [Setup the .EDGERC File](#setup-the-edgerc-file) - - [Setup the .EDGERC File](#setup-the-edgerc-file-1) - [Usage](#usage) - [Usage examples](#usage-examples) - [ULS as a service: systemd](#uls-as-a-service-systemd) @@ -83,12 +82,6 @@ pip3 install -q -r ext/cli-linode/bin/requirements.txt ### Setup the .EDGERC File -Copy the `.edgerc` file ([instructions for creation](AKAMAI_API_CREDENTIALS.md)) to your users home directory (~): -```bash -cp /path/to/your/.edgerc ~/.edgerc -``` -### Setup the .EDGERC File - Copy the `.edgerc` file ([instructions for creation](AKAMAI_API_CREDENTIALS.md)) to your users home directory (~): ```bash cp /path/to/your/.edgerc ~/.edgerc @@ -116,9 +109,9 @@ All log output will be directed to STDOUT by default. python3 bin/uls.py --input eaa --feed admin --output tcp --host 10.10.10.200 --port 9090 ``` -- ETP THREAT LOG ==> UDP LISTENER +- SIA THREAT LOG ==> UDP LISTENER ```bash - python3 bin/uls.py --input etp --feed threat --output udp --host 10.10.10.200 --port 9090 + python3 bin/uls.py --input sia --feed threat --output udp --host 10.10.10.200 --port 9090 ``` - MFA AUTH LOG ==> HTTP LISTENER (SPLUNK) disabled TLS verification @@ -128,7 +121,7 @@ All log output will be directed to STDOUT by default. - Logging to a file and sending process to the background ```bash - python3 bin/uls.py --input etp --feed threat --output udp --host 10.10.10.200 --port 9090 &> /path/to/my/logfile & + python3 bin/uls.py --input sia --feed threat --output udp --host 10.10.10.200 --port 9090 &> /path/to/my/logfile & ``` Rather consider [docker usage](./DOCKER_USAGE.md) instead of this @@ -139,14 +132,14 @@ If you are planning to use multiple Akamai feed with ULS, bear in mind you will We assume you have followed the instruction above to install ULS as command line. Before you install the service, make sure it works manually with the configured user. -Create a new file in systemd directory (e.g. `/etc/systemd/system`), use a name that is easy to remember, like `uls-etp-dns.service` +Create a new file in systemd directory (e.g. `/etc/systemd/system`), use a name that is easy to remember, like `uls-sia-dns.service` ```INI # ULS as systemd service example -# uls-etp-dns.service +# uls-sia-dns.service [Unit] -Description=Akamai ULS feed ETP/DNS # << Adjust the description with the feed name +Description=Akamai ULS feed SIA/DNS # << Adjust the description with the feed name After=network.target StartLimitIntervalSec=0 @@ -156,7 +149,7 @@ Restart=always RestartSec=1 User=uls # << Change this to reflect the user on your system WorkingDirectory=/usr/local/uls # << Change with ULS location -ExecStart=/usr/bin/python3 /usr/local/uls/bin/uls.py --input etp --feed dns --output tcp --host 127.0.0.1 --port 9090 # << Adjust python path and uls path +ExecStart=/usr/bin/python3 /usr/local/uls/bin/uls.py --input sia --feed dns --output tcp --host 127.0.0.1 --port 9090 # << Adjust python path and uls path [Install] WantedBy=multi-user.target @@ -164,25 +157,25 @@ WantedBy=multi-user.target Test the service with the following command: ``` -$ systemctl start uls-etp-dns +$ systemctl start uls-sia-dns ``` Make sure everything is working: ``` -$ systemctl status uls-etp-dns +$ systemctl status uls-sia-dns ``` Expected output: ``` -● uls-etp-dns.service - Akamai ULS feed ETP/DNS - Loaded: loaded (/etc/systemd/system/uls-etp-dns.service; disabled; vendor preset: disabled) +● uls-sia-dns.service - Akamai ULS feed SIA/DNS + Loaded: loaded (/etc/systemd/system/uls-sia-dns.service; disabled; vendor preset: disabled) Active: active (running) since Wed 2021-09-01 23:51:39 UTC; 4s ago Main PID: 9428 (python3) - CGroup: /system.slice/uls-etp-dns.service - ├─9428 /usr/bin/python3 /root/uls/bin/uls.py --input etp --feed dns --output tcp --host 127.0.0.1 --port 9090 - └─9430 python3 ext/cli-etp/bin/akamai-etp --edgerc /root/.edgerc --section default --user-agent-prefix ULS/1.1.0_ETP-DNS event dns -f + CGroup: /system.slice/uls-sia-dns.service + ├─9428 /usr/bin/python3 /root/uls/bin/uls.py --input sia --feed dns --output tcp --host 127.0.0.1 --port 9090 + └─9430 python3 ext/cli-etp/bin/akamai-etp --edgerc /root/.edgerc --section default --user-agent-prefix ULS/1.1.SIA-DNS event dns -f ``` To make sure the service will start if the machine restarts, use: diff --git a/docs/DEBUGGING.md b/docs/DEBUGGING.md index 73b6e83..52b6048 100644 --- a/docs/DEBUGGING.md +++ b/docs/DEBUGGING.md @@ -54,7 +54,7 @@ Akamai Unified Log Streamer Version information ULS Version 0.0.1 EAA Version 0.3.8 -ETP Version 0.3.4 +SIA/ETP Version 0.3.4 MFA Version 0.0.4 OS Plattform Linux-5.10.25-linuxkit-x86_64-with-glibc2.28 diff --git a/docs/FAQ.md b/docs/FAQ.md index d4aa14a..9c24982 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -13,7 +13,8 @@ - [ULS throws TLS an error when connecting towards Guardicore API (--input GC)](#uls-throws-tls-an-error-when-connecting-towards-guardicore-api---input-gc) - [WHY JMESPATH and not JSONPATH](#why-jmespath-and-not-jsonpath) - [What is HTTP FORMATTYPE](#what-is-http-formattype) -- +- [Error: "Capacity exceeded, too many incoming data vs. slow output"](#error-capacity-exceeded-too-many-incoming-data-vs-slow-output) +- [Error: "Invalid timestamp" on API call](#error-invalid-timestamp-on-api-call) ---- ## FAQ @@ -173,4 +174,15 @@ Within the `single-event` mode, you can freely amend line breake configuration l ### Error: "Capacity exceeded, too many incoming data vs. slow output" This error indicates, that more data is coming in to ULS than it can send towards the sepcified output. As this might be an indication for I/O problems either on the ULS output or the receiving system, it could also just be a specific race condition when the API operations with big pages or at a high speed (e.g. within local LAN). -If requried, the size can be adjusted by using the "--inputqueuesize" introduced in ULS 1.6.7. \ No newline at end of file +If requried, the size can be adjusted by using the "--inputqueuesize" introduced in ULS 1.6.7. + +--- +### Error: "Invalid timestamp" on API call +```bash +Oct 15 07:00:17 myhost python3[9751453]: 2023-10-24 07:00:17,315 ULS E UlsInputCli - CLI process [712679], sadly stderr has been disabled +Oct 15 07:00:19 myhost python3[9751453]: 2023-10-24 07:00:19,216 ULS E UlsInputCli - CLI process [712679] was found stale - Reason: "2023-10-24 07:00:19,175 cli-etp MainThread E API call failed with HTTP/400: b'{\n "type": "https://problems.luna.akamaiapis.net/-/pep-authn/request-error",\n "title": "Bad request",\n "status": 400,\n "detail": "Invalid timestamp",\n " +instance": "https://akab-1234567890ABCDEF-0987654321BAC.luna.akamaiapis.net/etp-report/v3/configs/12345/dns-activities/details",\n "method": "POST",\n "serverIp": "10.10.10.10",\n " +clientIp": "10.9.9.9",\n "requestId": "ALC1234",\n "requestTime": "2023-10-24T07:01:40Z"\n}\n' +``` +This error points towards a potential issue with the time configuration on the ULS host. The time of the host ULS runs on, should be synced with some NTP service(s). +As you can see in the above example, the host timestamp is `2023-10-24 07:00:17,315` but the request timestamp (returned from the API) is more than 1 minute ahaed `"2023-10-24T07:01:40Z"`. diff --git a/docs/LOG_OVERVIEW.md b/docs/LOG_OVERVIEW.md index 5f10e64..456eecf 100644 --- a/docs/LOG_OVERVIEW.md +++ b/docs/LOG_OVERVIEW.md @@ -29,13 +29,15 @@ Here are some examples (per product) and links to additional information. - [Linode](#linode) - [AUDIT Logs](#audit-logs) +--- ## Enterprise Application Access (EAA) - When configuring ULS to access EAA these feed, set `input` argument/variable to `EAA` and `feed` as indicated below in parathesis. - + ### Access Logs (ACCESS) Additional information regarding the log fields can be found on [here](https://techdocs.akamai.com/eaa/docs/data-feed-siem#access-logs) +
+ EAA access logs example (JSON) ```json { @@ -70,10 +72,13 @@ Additional information regarding the log fields can be found on [here](https://t "session_id": "1b1c728b-298e-4ebd-ce7c-0c1f99ad943f" } ``` +
### Admin Logs (ADMIN) - Additional information regarding the log fields can be found on [here](https://techdocs.akamai.com/eaa/docs/data-feed-siem#admin-logs). +
+ EAA admin logs example (JSON) + ```json { "datetime": "2021-07-23T05:54:40", @@ -84,10 +89,14 @@ Additional information regarding the log fields can be found on [here](https://t "event_type": "system" } ``` +
### Connector Health (CONHEALTH) Additional information regarding the log fields can be found on [here](https://techdocs.akamai.com/eaa/docs/data-feed-siem#connector-health) +
+ EAA connector health example (JSON) + ```json { "connector_uuid": "cht3_GEjQWyMW9LEk7KQfg", @@ -109,6 +118,7 @@ Additional information regarding the log fields can be found on [here](https://t "dialout_active": 1 } ``` +
### Device Posture Inventory (DEVINV) @@ -280,6 +290,7 @@ Schema is documented on the [EAA Directory List API doc](https://techdocs.akamai ``` +--- ## Secure Internet Access Enterprise (SIA-E) Formerly known as Enterprise Threat Protector (ETP). @@ -693,6 +704,7 @@ Additional information regarding the log fields can be found [here](https://tech
Acceptable Use Policy Event example (JSON) + ```json { "pageInfo": { @@ -1081,6 +1093,7 @@ Additional information regarding the log fields can be found [here](https://tech ] } ``` +
### DNS Activity @@ -1088,6 +1101,7 @@ Additional information regarding the log fields can be found [here](https://tech
DNS Activity Event example (JSON) + ```json { "pageInfo": { @@ -1312,6 +1326,7 @@ Additional information regarding the log fields can be found [here](https://tech ] } ``` +
### PROXY @@ -1320,6 +1335,7 @@ Additional information regarding the log fields can be found [here](https://tech
Proxy Activity Event example (JSON) + ```json { "pageInfo": { @@ -2142,6 +2158,7 @@ Additional information regarding the log fields can be found [here](https://tech ] } ``` +
### NETCON @@ -2149,6 +2166,7 @@ Additional information regarding the log fields can be found [here](https://tech
Network Connection Event example (JSON) + ```json { "id": "123", @@ -2191,11 +2209,14 @@ Additional information regarding the log fields can be found [here](https://tech ```
+--- ## Akamai MFA (MFA) Additional information regarding the MFA log fields can be found on [here](https://techdocs.akamai.com/mfa/docs/splunk-app). ### Authentication Logs (AUTH) -Authentication Events Example: +
+ Authentication Events Example (JSON) + ```json { "uuid": "aud_JfNqdl6zSByrU0ovrbJ6m", @@ -2227,11 +2248,16 @@ Authentication Events Example: "principal_uuid": null } ``` +
+--- ## Guardicore ### NETLOG -Guardicore netlog example + +
+ Guardicore netlog example (JSON) + ```json { "id": "123", @@ -2273,8 +2299,12 @@ Guardicore netlog example } ``` +
+ ### INCIDENT -Guardicore incident example +
+ Guardicore incident example (JSON) + ```json { "_cls": "Incident.NetworkVisibilityIncident", @@ -2621,13 +2651,19 @@ Guardicore incident example }, "source_vm_id": "74238291-b85a-42fb-bac9-80c402abee04", "start_time": 1504688829035, - "total_events_count": + "total_events_count": 3 } ``` +
+ +--- ## Linode ### AUDIT Logs Additional information regarding the log fields can be found on [here](https://www.linode.com/docs/api/account/#events-list) +
+ Linode audit example (JSON) + ```json { @@ -2657,4 +2693,4 @@ Additional information regarding the log fields can be found on [here](https://w "username": "exampleUser" } ``` - +
diff --git a/docs/SIEM/SENTINEL/Readme.md b/docs/SIEM/SENTINEL/Readme.md new file mode 100644 index 0000000..ad2160f --- /dev/null +++ b/docs/SIEM/SENTINEL/Readme.md @@ -0,0 +1,41 @@ +# Microsoft Sentinel +## Introduction +This document describes how to ingest data into [Microsoft Sentinel](https://learn.microsoft.com/en-us/azure/sentinel/overview) via the Akamai Unified Log Streamer (ULS). +Here's an overview of the "processing" workflow + + +## Details +The currently recommended way is to feed ULS data into a UDP/TCP SYSLOG Server. +The [Azure Monitor Agent](https://learn.microsoft.com/en-us/azure/azure-monitor/agents/agents-overview) collects the logs and ingests them into the Microsoft Sentinel platform. +ULS and SYSLOG daemon do not need to reside on the same host. + +### Steps to set up the conection +#### Prerequisites +- Install [ULS](https://github.com/akamai/uls/tree/main#documentation) +- Install a [SYSLOG server](https://www.syslog-ng.com/community/tags/install) (if not already comes with your system) +- Install the [Azure Monitor Agent](https://learn.microsoft.com/en-us/azure/azure-monitor/agents/agents-overview) ([MS ARC AGENT](https://learn.microsoft.com/en-us/azure/azure-arc/servers/deployment-options) might be required in order to install the Monitor Agent) + +#### Configuration +- Syslog Server + Configure the Syslog server to accept logs via UDP or TCP (tcp is recommended) + Config Example for `RSYSLOG` Server + ```editorconfig + # TCP PORT (rsyslogd) + module(load="imtcp") + input(type="imtcp" port="514") + ``` + +- ULS + Configure ULS to send the logs via OUTPUT TCP to the specified port (514 in that case). + Example (fetching SIA/ETP THREAT logs): + ```bash + bin/uls.py --input ETP --feed THREAT --output TCP --hostname 10.9.8.7 --port 514 + ``` + +- Azure + A [Data Collection Rule](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/data-collection-rule-overview) needs to be deployed with the following configuration: + - Syslog Facility (e.g. LOCAL1) pointing to the corresponding syslog facility the ULS streams the data into). + - Log Transformation (Log Parsing) + + As an alternative, here is a [working template](templates/AkamaiETPtableAndDCR.json) that can be [directly imported](https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/quickstart-create-templates-use-the-portal). + (A massive thanks to Joana from Microsoft who shared the template with us) \ No newline at end of file diff --git a/docs/SIEM/SENTINEL/templates/AkamaiETPtableAndDCR.json b/docs/SIEM/SENTINEL/templates/AkamaiETPtableAndDCR.json new file mode 100644 index 0000000..c19292b --- /dev/null +++ b/docs/SIEM/SENTINEL/templates/AkamaiETPtableAndDCR.json @@ -0,0 +1,394 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "workspaceName": { + "defaultValue": "loganalyticsworkspacename", + "type": "String", + "metadata": { + "description": "Workspace name for Log Analytics where Microsoft Sentinel is setup" + } + }, + "DataCollectionRuleName": { + "defaultValue": "Syslog-AkamaiETP-DCR", + "type": "String", + "metadata": { + "description": "Data Collection Rule Name" + } + } + }, + "variables": { + "logAnalyticsTableName": "akamaiETP_CL", + "workspaceResourceId": "[resourceId('microsoft.OperationalInsights/Workspaces', parameters('workspaceName'))]", + "location": "[resourceGroup().location]", + "subscriptionID": "[last(split(subscription().id, '/'))]" + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/tables", + "apiVersion": "2021-12-01-preview", + "name": "[format('{0}/{1}', parameters('workspaceName'), variables('logAnalyticsTableName'))]", + "location": "[variables('location')]", + "tags": {}, + "properties": { + "schema": { + "name": "[variables('logAnalyticsTableName')]", + "columns": [ + { + "name": "actionId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "actionName", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "alexaRanking", + "type": "int", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "applicationId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "applicationName", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "aupCategories", + "type": "dynamic", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "catalogId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "categoryId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "categoryName", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "cidr", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "clientAgents", + "type": "dynamic", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "clientIp", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "clientRequestId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "confidenceName", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "configId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "deepScanned", + "type": "boolean", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "deviceId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "deviceName", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "deviceOwnerId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "dnsIp", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "dohAttribution", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "domain", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "encryptedInternalClientIP", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "encryptedInternalClientName", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "hitCount", + "type": "int", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "etpId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "internalClientIP", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "listId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "listName", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "observedAupCategories", + "type": "dynamic", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "onRamp", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "onrampType", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "policyEvaluationSource", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "policyId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "policyName", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "queryType", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "reason", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "resolved", + "type": "dynamic", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "riskId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "riskName", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "scId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "scName", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "securityCategoryId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "securityCategoryName", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "siteId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "siteName", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "sublocationId", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "sublocationName", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "TimeGenerated", + "type": "datetime", + "isDefaultDisplay": false, + "isHidden": false + }, + { + "name": "transportType", + "type": "string", + "isDefaultDisplay": false, + "isHidden": false + } + ] + } + } + }, + { + "type": "Microsoft.Insights/dataCollectionRules", + "apiVersion": "2022-06-01", + "name": "[parameters('DataCollectionRuleName')]", + "location": "[variables('location')]", + "kind": "Linux", + "properties": { + "dataSources": { + "syslog": [ + { + "streams": [ + "Microsoft-Syslog" + ], + "facilityNames": [ + "user" + ], + "logLevels": [ + "Debug", + "Info", + "Notice", + "Warning", + "Error", + "Critical", + "Alert", + "Emergency" + ], + "name": "sysLogsDataSource-1688419672" + } + ] + }, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": "[variables('workspaceResourceId')]", + "name": "clv2ws1" + } + ] + }, + "dataFlows": [ + { + "streams": [ + "Microsoft-Syslog" + ], + "destinations": [ + "clv2ws1" + ], + "transformKql": "source | extend SyslogMessage = iff(SyslogMessage endswith_cs '#015', substring(SyslogMessage, 0, strlen(SyslogMessage)-4), SyslogMessage) | extend SyslogMessage = parse_json(SyslogMessage) | extend TimeGenerated = todatetime(SyslogMessage.query['time']) | extend etpId=tostring(SyslogMessage.id), configId=tostring(SyslogMessage.configId) | extend hitCount=toint(SyslogMessage.hitCount) | extend alexaRanking=toint(SyslogMessage.alexaRanking) | extend clientIp=tostring(SyslogMessage.query.clientIp), dnsIp=tostring(SyslogMessage.query.dnsIp) | extend domain=tostring(SyslogMessage.query.domain) | extend queryType=tostring(SyslogMessage.query.queryType) | extend deviceId=tostring(SyslogMessage.query.deviceId), deviceName=tostring(SyslogMessage.query.deviceName) | extend resolved=SyslogMessage.query.resolved | extend siteId=tostring(SyslogMessage.event.siteId), siteName=tostring(SyslogMessage.event.siteName) | extend policyId=tostring(SyslogMessage.event.policyId) | extend listId=tostring(SyslogMessage.event.listId), listName=tostring(SyslogMessage.event.listName) | extend categoryName=tostring(SyslogMessage.event.categoryName) | extend confidenceName=tostring(SyslogMessage.event.confidenceName) | extend actionId=tostring(SyslogMessage.event.actionId), actionName=tostring(SyslogMessage.event.actionName) | extend reason=tostring(SyslogMessage.event.reason) | extend onRamp=tostring(SyslogMessage.event.onRamp), onrampType=tostring(SyslogMessage.event.onrampType) | extend internalClientIP=tostring(SyslogMessage.event.internalClientIP) | extend clientRequestId=tostring(SyslogMessage.event.clientRequestId) | extend policyEvaluationSource=tostring(SyslogMessage.event.policyEvaluationSource) | extend encryptedInternalClientIP=tostring(SyslogMessage.event.encryptedInternalClientIP), encryptedInternalClientName=tostring(SyslogMessage.event.encryptedInternalClientName) | extend scId=tostring(SyslogMessage.event.scId) | extend applicationId=tostring(SyslogMessage.event.applicationId), applicationName=tostring(SyslogMessage.event.applicationName) | extend riskId=tostring(SyslogMessage.event.riskId), riskName=tostring(SyslogMessage.event.riskName) | extend catalogId=tostring(SyslogMessage.event.catalogId) | extend observedAupCategories=SyslogMessage.event.observedAupCategories | extend aupCategories=SyslogMessage.event.aupCategories | extend securityCategoryName=tostring(SyslogMessage.event.securityCategoryName) | extend sublocationId=tostring(SyslogMessage.event.sublocationId) | extend sublocationName=tostring(SyslogMessage.event.sublocationName) | extend clientAgents=SyslogMessage.event.clientAgents | extend transportType=tostring(SyslogMessage.event.transportType) | extend dohAttribution=tostring(SyslogMessage.event.dohAttribution) | extend securityCategoryId=tostring(SyslogMessage.event.securityCategoryId) | extend cidr=tostring(SyslogMessage.event.cidr) | extend deepScanned=tobool(SyslogMessage.event.deepScanned)", + "outputStream": "[concat('Custom-', variables('logAnalyticsTableName'))]" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/docs/SIEM/SENTINEL/uls_sentinel1.png b/docs/SIEM/SENTINEL/uls_sentinel1.png new file mode 100644 index 0000000..e772868 Binary files /dev/null and b/docs/SIEM/SENTINEL/uls_sentinel1.png differ diff --git a/docs/SIEM/SIEM_OVERVIEW.md b/docs/SIEM/SIEM_OVERVIEW.md index 5d3bbe5..b70e3bb 100644 --- a/docs/SIEM/SIEM_OVERVIEW.md +++ b/docs/SIEM/SIEM_OVERVIEW.md @@ -6,5 +6,8 @@ All contributions in terms of documentation a welcome. ## SIEM Integration guides (alphabetical) - [GRAYLOG](./GRAYLOG/README.md) +- [MICROSOFT SENTINEL](./SENTINEL/Readme.md) - [QRADAR](./QRADAR/README.md) - [SPLUNK](./SPLUNK/README.md) +- [SUMOLOGIC](./SUMOLOGIC/README.md) + diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml index 00334d2..3a3f671 100644 --- a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml +++ b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml @@ -4,4 +4,4 @@ description: Akamai Universal Log Streamer Helm installation type: application version: 2.0.0 -appVersion: "1.7.1" +appVersion: "1.7.2" diff --git a/test/README.md b/test/README.md index 691a75c..7993d2c 100644 --- a/test/README.md +++ b/test/README.md @@ -37,6 +37,7 @@ For better testing stability, we packed the required dependencies into the test brew install bats-core brew install helm brew install hadolint +brew install trivy git clone https://github.com/ztombol/bats-assert.git test/bats/bats-assert git clone https://github.com/ztombol/bats-support.git test/bats/bats-support diff --git a/test/basic_test.bats b/test/basic_test.bats index 6fa5572..7da9620 100644 --- a/test/basic_test.bats +++ b/test/basic_test.bats @@ -51,7 +51,7 @@ current_version=$(cat docs/CHANGELOG.md | grep "##" | head -n 1 | sed 's/.* v//' } @test "cat bin/config/global - Version output should be ($current_version) according to CHANGELOG" { - run echo $(cat bin/config/global_config.py | grep "__version__ =" | cut -d " " -f 3) + run echo $(cat bin/uls_config/global_config.py | grep "__version__ =" | cut -d " " -f 3) assert_output --partial "$current_version" [ "$status" -eq 0 ] } diff --git a/test/docker_test.bats b/test/docker_test.bats index f4858f5..ffd16ed 100644 --- a/test/docker_test.bats +++ b/test/docker_test.bats @@ -64,6 +64,20 @@ teardown () { [ "$status" -eq 0 ] } +## Test container security posture +@test "DOCKER SECURITY SCAN 1 (scout)" { + #run docker scout quickview uls:bats + run docker scout cves --only-fixed uls:bats + [ "$status" -eq 0 ] +} + +## Test container security posture +@test "DOCKER SECURITY SCAN 2 (trivy)" { + run trivy image akamai/uls --ignore-unfixed + [ "$status" -eq 0 ] +} + + ## RUN AN EAA TEST @test "DOCKER EAA TEST" { run timeout ${uls_timeout_params} docker run --rm --name "uls-bats-test" --mount type=bind,source="${uls_edgerc}",target="/opt/akamai-uls/.edgerc",readonly uls:bats --section ${uls_section} --input eaa --feed access --output raw --loglevel info @@ -97,7 +111,7 @@ teardown () { } ## REMOVE the local DOCKER IMAGE -@test "DOCKER IMAGE BUILD" { - docker image rm uls:bats +@test "DOCKER IMAGE DELETE" { + run docker image rm uls:bats [ "$status" -eq 0 ] } \ No newline at end of file diff --git a/test/positive_test.bats b/test/positive_test.bats index 49a9040..a9eecfc 100644 --- a/test/positive_test.bats +++ b/test/positive_test.bats @@ -97,6 +97,7 @@ load 'bats/bats-assert/load.bash' #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } + @test "EAA - DIRHEALTH" { run timeout ${uls_timeout_params} ${uls_bin} --input eaa --feed dirhealth --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info #assert_output --partial $eaa_devinv_assert @@ -159,6 +160,18 @@ load 'bats/bats-assert/load.bash' [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } + +## SIA (ALIAS TO ETP) +@test "SIA - DNS" { + run timeout ${uls_timeout_params} ${uls_bin} --input sia --feed dns --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output --partial $etp_assert + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout +} + ## MFA @test "MFA - EVENT" { run timeout ${uls_timeout_params} ${uls_bin} --input mfa --feed event --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info