From 919a05ee3a26f6aa330785c406394972fa25256e Mon Sep 17 00:00:00 2001 From: Mike Schiessl <77062930+MikeSchiessl@users.noreply.github.com> Date: Tue, 2 Apr 2024 11:10:41 +0200 Subject: [PATCH] Development (#61) v1.7.3 preparation --- Dockerfile | 4 +- README.md | 1 + bin/modules/UlsInputCli.py | 24 ++++--- bin/modules/UlsOutput.py | 39 ++++++++++- bin/modules/UlsTools.py | 29 +++++++- bin/uls.py | 11 +++ bin/uls_config/global_config.py | 6 +- docs/ARGUMENTS_ENV_VARS.md | 67 +++++++++++++++---- docs/CHANGELOG.md | 56 +++++++++++++--- docs/COMMAND_LINE_USAGE.md | 11 +-- .../kubernetes/helm/akamai-uls/Chart.yaml | 2 +- scripts/get-uls.sh | 47 +++++++++++-- test/positive_test.bats | 8 +++ 13 files changed, 254 insertions(+), 51 deletions(-) diff --git a/Dockerfile b/Dockerfile index 77e2295..f37924a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,10 +10,10 @@ ARG HOMEDIR="/opt/akamai-uls" ARG ULS_DIR="$HOMEDIR/uls" ARG EXT_DIR="$ULS_DIR/ext" -ARG ETP_CLI_VERSION="0.4.5" +ARG ETP_CLI_VERSION="0.4.7" ARG EAA_CLI_VERSION="0.6.3" ARG MFA_CLI_VERSION="0.1.1" -ARG GC_CLI_VERSION="v0.0.3(beta)" +ARG GC_CLI_VERSION="v0.0.4(beta)" ARG LINODE_CLI_VERSION="dev" # ENV VARS diff --git a/README.md b/README.md index 4db740a..4254867 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ It can be run directly as Python code, as a provided Docker container, through - [INCIDENT](docs/LOG_OVERVIEW.md#incident) - AGENT - SYSTEM + - AUDIT - [Linode](https://www.linode.com/) (experimental) - [AUDIT](docs/LOG_OVERVIEW.md#audit-logs) diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py index a633584..5faf067 100644 --- a/bin/modules/UlsInputCli.py +++ b/bin/modules/UlsInputCli.py @@ -86,6 +86,9 @@ def __init__(self, self.disable_stderr = uls_config.input_disable_stderr #Specify if STDERR should be disabled at all after $disable_stderr_after cycles self.disable_stderr_after = uls_config.input_disable_stderr_after # Disable StdErr Output after # cycles + # Generic vars + self.edgerc_hostname = None + def _feed_selector(self, feed, product_feeds): if feed in product_feeds: # feed matches the given list @@ -155,7 +158,7 @@ def proc_create(self): if not self.rawcmd: my_feed = self._feed_selector(self.feed, product_feeds) if my_feed == "CONHEALTH": - UlsTools.uls_check_edgerc(self.credentials_file, + self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section, uls_config.edgerc_openapi) cli_command = [self.bin_python, @@ -166,7 +169,7 @@ def proc_create(self): '--perf', '--tail'] elif my_feed == "DEVINV": - UlsTools.uls_check_edgerc(self.credentials_file, + self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section, uls_config.edgerc_openapi) cli_command = [self.bin_python, @@ -176,7 +179,7 @@ def proc_create(self): 'inventory', '--tail'] elif my_feed == "DIRHEALTH": - UlsTools.uls_check_edgerc(self.credentials_file, + self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section, uls_config.edgerc_openapi) cli_command = [self.bin_python, @@ -186,7 +189,7 @@ def proc_create(self): 'list', '--tail'] else: - UlsTools.uls_check_edgerc(self.credentials_file, + self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section, uls_config.edgerc_eaa_legacy) cli_command = [self.bin_python, '-u', product_path, 'log', my_feed.lower(), '-f'] @@ -218,7 +221,7 @@ def proc_create(self): aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) " f"not available for {product_path}, continuing with JSON.") if not self.rawcmd: - UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section, + self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section, uls_config.edgerc_openapi + ["etp_config_id"]) my_feed = self._feed_selector(self.feed, product_feeds) cli_command = [self.bin_python, '-u', product_path, 'event', my_feed.lower(), '-f'] @@ -249,7 +252,7 @@ def proc_create(self): aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) " f"not available for {product_path}, continuing with JSON.") if not self.rawcmd: - UlsTools.uls_check_edgerc(self.credentials_file, + self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section, uls_config.edgerc_mfa) my_feed = self._feed_selector(self.feed, product_feeds) @@ -279,7 +282,7 @@ def proc_create(self): aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) " f"not available for {product_path}, continuing with JSON.") if not self.rawcmd: - UlsTools.uls_check_edgerc(self.credentials_file, + self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section, uls_config.edgerc_gc) my_feed = self._feed_selector(self.feed, product_feeds) @@ -309,7 +312,7 @@ def proc_create(self): aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) " f"not available for {product_path}, continuing with JSON.") if not self.rawcmd: - UlsTools.uls_check_edgerc(self.credentials_file, + self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section, uls_config.edgerc_linode) my_feed = self._feed_selector(self.feed, product_feeds) @@ -476,4 +479,7 @@ def ingest_loop(self): except Exception: aka_log.log.exception("Error in ingest_loop") -# EOF + def get_edgerc_hostname(self): + return self.edgerc_hostname + + # EOF diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py index 10dde6f..2e52e54 100644 --- a/bin/modules/UlsOutput.py +++ b/bin/modules/UlsOutput.py @@ -97,7 +97,7 @@ def __init__(self, output_type: str, if self.output_type in ['TCP', 'UDP'] and host and port: self.host = host self.port = port - if "%s" in tcpudp_out_format: + if "%s" in str(tcpudp_out_format): self.tcpudp_out_format = tcpudp_out_format else: aka_log.log.critical( @@ -513,4 +513,41 @@ def tear_down(self): self.httpSession.close() self.connected = False +### FEATURE REQ 20240318 - https://github.com/akamai/uls/issues/57 + + def ingest_vars_into_output_format(self, placeholder: str = None, replacement: str = None): + if not placeholder or not replacement: + aka_log.log.debug(f"{self.name} Variable substition triggered but no value given (inline code issue)") + #print(f"PLACEHOLDER: {placeholder}") + #print(f"REPLACEMENT: {replacement}") + #return True + sys.exit(1) + + if placeholder in str(self.tcpudp_out_format) and self.tcpudp_out_format: + self.tcpudp_out_format = str(self.tcpudp_out_format).replace(placeholder, replacement) + aka_log.log.debug(f"{self.name} Replacing {placeholder} in TCPUDP string with: {replacement} ") + aka_log.log.debug(f"{self.name} SUBSTITUTION new TCPUDP output string: {self.tcpudp_out_format} ") + + if placeholder in str(self.http_out_format) and self.http_out_format: + self.http_out_format = str(self.http_out_format).replace(placeholder, replacement) + aka_log.log.debug(f"{self.name} Replacing {placeholder} in HTTP string with: {replacement} ") + aka_log.log.debug(f"{self.name} SUBSTITUTION new HTTP output string: {self.http_out_format} ") + + + return True + + def ingest_os_vars_into_output_format(self): + aka_log.log.debug(f"{self.name} Replacing ENV VARS in output FORMAT") + if self.tcpudp_out_format: + self.tcpudp_out_format = os.path.expandvars(self.tcpudp_out_format) + aka_log.log.debug(f"{self.name} OS_ENV_VARS new TCPUDP output string: {self.tcpudp_out_format} ") + if self.http_out_format: + self.http_out_format = os.path.expandvars(self.http_out_format) + aka_log.log.debug(f"{self.name} OS_ENV_VARS new HTTP output string: {self.http_out_format} ") + + + return True + + ### / FEATURE REQ 20240318 - https://github.com/akamai/uls/issues/57 + # EOF diff --git a/bin/modules/UlsTools.py b/bin/modules/UlsTools.py index 2579b40..92d190d 100644 --- a/bin/modules/UlsTools.py +++ b/bin/modules/UlsTools.py @@ -121,9 +121,11 @@ def uls_check_edgerc(configfile, configsection, configvalues): Verify the given "edgerc" file to contain all required variables (for the desired stream) within the given section see https://github.com/akamai/uls/blob/main/docs/AKAMAI_API_CREDENTIALS.md for more information :param configfile: The path to the config file + :param configsection: The section within the config file [default] :param configvalues: A list of desiresd config values ["val1", "val2", ...] :return: + api_hostname """ config = configparser.ConfigParser() # Load config file @@ -141,13 +143,23 @@ def uls_check_edgerc(configfile, configsection, configvalues): aka_log.log.debug(f"Section '{configsection}' found.") # check for specified values + my_hostname = None for configvalue in configvalues: if not configvalue in config[configsection]: aka_log.log.critical(f"Required configuration value '{configvalue}' not found in section / file. Please see: {uls_config.edgerc_documentation_url} - Exiting") sys.exit(1) else: + + ### FEATURE REQ 20240318 + if 'host' in configvalue: + my_hostname = config[configsection][configvalue] + #print(f"HOSTNAME{my_hostname}") + ### FEATURE REQ 20240318 + aka_log.log.debug(f"Required configuration value '{configvalue}' found.") - return 0 + if not my_hostname: + my_hostname = "no_hostname_available" + return my_hostname def uls_check_args(input, output): @@ -190,6 +202,7 @@ def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoin if (input not in uls_config.autoresume_supported_inputs or feed == "CONHEALTH" or feed == "DEVINV" or + feed == "AGENT" or feed == "DIRHEALTH"): aka_log.log.critical(f"Input {input} or feed {feed} currently not supported by AUTORESUME - Exiting.") sys.exit(1) @@ -205,7 +218,7 @@ def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoin checkpoint = None else: try: - with open (checkpoint_full, "r") as ckpt_f: + with open(checkpoint_full, "r") as ckpt_f: data = json.load(ckpt_f) if data['creation_time'] and data['checkpoint']: aka_log.log.debug(f"Autoresume Checkpoint successfully loaded. Checkpoint Time: {data['checkpoint']}, Creation_time: {data['creation_time']}") @@ -215,6 +228,8 @@ def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoin mytime = data['checkpoint'].split("Z")[0] elif data['input'] == "EAA": mytime = data['checkpoint'].split("+")[0] + elif data['input'] == "GC": + mytime = data['checkpoint'].split(".")[0] else: aka_log.log.critical( f"Unhandeled input data in checkpointfile \'{checkpoint_full}\' --> {input} / {feed} - Exiting.") @@ -263,6 +278,12 @@ def write_autoresume_ckpt(input, feed, autoresume_file, logline, current_count): checkpoint_timestamp = json.loads(checkpoint_line)['datetime'] elif input == "ETP" and feed == "NETCON": checkpoint_timestamp = json.loads(checkpoint_line)['connStartTime'] + elif input == "GC" and feed == "AUDIT": + checkpoint_timestamp = json.loads(checkpoint_line)['time'] + elif input == "GC" and feed == "INCIDENT": + checkpoint_timestamp = json.loads(checkpoint_line)['closed_time'] + elif input == "GC" and feed == "NETLOG": + checkpoint_timestamp = json.loads(checkpoint_line)['db_insert_time'] else: aka_log.log.critical( f"AUTORESUME - Unhandled Input / Feed detected: '{input} / {feed}' (this should never happen !!)- Exiting") @@ -311,4 +332,6 @@ def get_install_id(install_id_file=str(root_path()) + "/var/uls_install_id"): aka_log.log.debug(f"Not able to read install file - returning mocked data. Error: {error}") data = {'install_id': "ERROR-GETTING-INSTALLATION-ID"} #return install_id - return data \ No newline at end of file + return data + + diff --git a/bin/uls.py b/bin/uls.py index c597f81..d21dc6e 100755 --- a/bin/uls.py +++ b/bin/uls.py @@ -89,6 +89,7 @@ def main(): # Check CLI Environment UlsTools.uls_check_sys(root_path=root_path, uls_input=uls_args.input) + # Create & Start monitoring Instance my_monitor = UlsMonitoring.UlsMonitoring(stopEvent=stopEvent, product=uls_args.input, @@ -109,6 +110,7 @@ def main(): endtime=uls_args.endtime, root_path=root_path) + # Connect to the selected input UlsOutput my_output = UlsOutput.UlsOutput(output_type=uls_args.output, host=uls_args.host, @@ -151,11 +153,20 @@ def main(): else: filter_pattern = None + # Now let's handle the data and send input to output # Initiate the Input handler my_input.proc_create() + # Append extra vars to the output + #my_output.ingest_vars_into_output_format(api_hostname=my_input.get_edgerc_hostname()) + my_output.ingest_vars_into_output_format(placeholder='{api_hostname}', replacement=my_input.get_edgerc_hostname()) + my_output.ingest_vars_into_output_format(placeholder='{uls_input}', replacement=uls_args.input) + my_output.ingest_vars_into_output_format(placeholder='{uls_feed}', replacement=uls_args.feed) + my_output.ingest_os_vars_into_output_format() + + # Connect the output handler my_output.connect() diff --git a/bin/uls_config/global_config.py b/bin/uls_config/global_config.py index 979ee62..e1f0360 100644 --- a/bin/uls_config/global_config.py +++ b/bin/uls_config/global_config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Common global variables / constants -__version__ = "1.7.2" +__version__ = "1.7.3" __tool_name_long__ = "Akamai Unified Log Streamer" __tool_name_short__ = "ULS" @@ -33,7 +33,7 @@ # Guardicore bin_gc_cli = "ext/cli-gc/bin/akamai-gc" # Path to the GC CLI Executable -gc_cli_feeds = ['NETLOG', 'INCIDENT', 'AGENT', 'SYSTEM'] # Available GC CLI feeds +gc_cli_feeds = ['NETLOG', 'INCIDENT', 'AGENT', 'SYSTEM', 'AUDIT'] # Available GC CLI feeds # LINODE bin_linode_cli = "ext/cli-linode/bin/akamai-linode" # Path to the LINODE CLI Executable @@ -107,6 +107,6 @@ # Autoresume Configuration autoresume_checkpoint_path = "var/" # (Default) Path, where the checkpointfiles should be stored to -autoresume_supported_inputs = ['ETP', 'EAA'] # Internal Var only, to adjust supported inputs +autoresume_supported_inputs = ['ETP', 'EAA', 'GC'] # Internal Var only, to adjust supported inputs autoresume_write_after = 1000 # Write checkpoint only every ${autoresume_write_every} loglines diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md index fab9783..c55b5ce 100644 --- a/docs/ARGUMENTS_ENV_VARS.md +++ b/docs/ARGUMENTS_ENV_VARS.md @@ -21,18 +21,18 @@ The following tables list all available command line parameters and their corres ## INPUT -| Parameter | Env - Var | Options | Default | Description | -|---------------------------|-----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| -i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA', 'GC', 'LINODE' | None | Specify the desired INPUT source | -| --feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV'

ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'

MFA: 'EVENT'

GC: 'NETLOG', 'INCIDENT', 'AGENT', 'SYSTEM'

LINODE: 'AUDIT' | None | Specify the desired INPUT feed | -| --format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format | -| --inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT | None | Adjust proxy usage for INPUT data collection (cli)
If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) | -| --rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature) | -| --edgerc | ULS_EDGERC | /path/to/your/.edgerc | '~/.edgerc' | Specify the location of the .edgerc EDGE GRID AUTH file | -| --section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file | -| --starttime | ULS_STARTTIME | EPOCH timestamp (in seconds) | `cli_default` | Specify an EPOCH timestamp from where to start the log collection. | -| --endtime | ULS_ENDTIME | EPOCH timestamp (in seconds) | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! | -| --inputqueuesize | ULS_INPUT_QUEUESIZE | INPUT_QUEUE_SIZE(int) | 15000 | Maximum threshold of the input queue. When threshold is reached, ULS will stop operations and exit "Capacity exceeded, too many incoming data vs. slow output" | +| Parameter | Env - Var | Options | Default | Description | +|---------------------------|-----------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA', 'GC', 'LINODE' | None | Specify the desired INPUT source | +| --feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV'

ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'

MFA: 'EVENT'

GC: 'NETLOG', 'INCIDENT', 'AGENT', 'SYSTEM', 'AUDIT'

LINODE: 'AUDIT' | None | Specify the desired INPUT feed | +| --format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format | +| --inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT | None | Adjust proxy usage for INPUT data collection (cli)
If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) | +| --rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature) | +| --edgerc | ULS_EDGERC | /path/to/your/.edgerc | '~/.edgerc' | Specify the location of the .edgerc EDGE GRID AUTH file | +| --section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file | +| --starttime | ULS_STARTTIME | EPOCH timestamp (in seconds) | `cli_default` | Specify an EPOCH timestamp from where to start the log collection. | +| --endtime | ULS_ENDTIME | EPOCH timestamp (in seconds) | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! | +| --inputqueuesize | ULS_INPUT_QUEUESIZE | INPUT_QUEUE_SIZE(int) | 15000 | Maximum threshold of the input queue. When threshold is reached, ULS will stop operations and exit "Capacity exceeded, too many incoming data vs. slow output" | ## OUTPUT @@ -78,3 +78,46 @@ The following tables list all available command line parameters and their corres | --autoresume | ULS_AUTORESUME | [True, False] | False | Enable automated resume on based on a checkpoint upon api failure or crash (do not use alongside --starttime) | | --autoresumepath | ULS_AUTORESUME_PATH | '/path/to/store/checkpoints/' | var/ | Specify the path where checkpoint files should be written to. (Trailing /) | | --autoresumewriteafter | ULS_AUTORESUME_WRITEAFTER | | 1000 | Specify after how many loglines a checkpoint should be written. | + + +## Customizing HTTP & TCPUDP Formatting + +Applicable to argument `--tcpudpformat` or environment variable `ULS_TCPUDP_FORMAT`. + +By default ULS will write the exact payload received from the API to +the configured ULS output. Thus, `%s` is the default value. + +### Payload decoration (Variable Substition) + +If you need to surround the payload with extra information (i.e. metadata for your SIEM), +you can surround the `%s`. In the example below, didn't surround the `%s` by double-quote +since we want the output to remain a valid JSON: + +```json +{"event": %s} +``` + +### Output Variables + +While customizing you might want to use dynamic variables. Substitution happens when ULS +software is starting, if you change your configuration file API hostname, you'll need +to restart ULS so it can take effect. +Besides the below replacements, ULS will also interprete OS (or DOCKER) ENV Vars. + + +| Variable | Description | +|----------------|--------------------------------------------------------------| +| {api_hostname} | This variable will be replaced with the Akamai API Hostname | +| {uls_input} | This variable will be replaced with the ULS Input | +| {uls_feed} | This variable will be replaced with the ULS Feed | + + +### Example +```bash +# Substitution example +'{"api_host": "{api_hostname}", "input_feed": "{uls_input}-{uls_feed}", "event": %s}' + +# OS ENV Var example +export MY_ENV_VAR="THIS IS MY ENV VAR" +'{"api_host": "{api_hostname}", "input_feed": "{uls_input}-{uls_feed}", "additional_env": "$MY_ENV_VAR", "event": %s}' +``` \ No newline at end of file diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 65f6110..9489c04 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,4 +1,25 @@ # Version History +## v1.7.3 +||| +|---|---| +|Date|2024-04-02 +|Kind| MINOR release +|Author|mschiess@akamai.com, androcho@akamai.com + +- **Features** + - introduced "audit logs" for Guardicore + - introduced "AUTORESUME" functionality for GC: NETLOG, INCIDENTS and AUDIT + - TCPUDP/HTTP format string now support [varialbe substitution](ARGUMENTS_ENV_VARS.md#payload-decoration-variable-substition): + - Substitution: {api_hostname}, {uls_input}, {uls_feed} + - OS ENV VARS: $VAR_NAME + +- **Minor improvements** + - [docker] bumped CLI-ETP to "0.4.7" - thx to @Antoine for a couple of bugfixes + - [docker] bumped CLI-EAA to "0.6.3" + - [docker] bumped CLI-GC to "v0.0.4(beta)" + - [CLI] Fixed an auto installer [Issue #58](https://github.com/akamai/uls/issues/58) - thx @Antoine +--- + ## v1.7.2 ||| |---|---| @@ -20,6 +41,7 @@ - Fixed a bug in the autoresume function that created a problem with timezones in certain circumstances - **Housekeeping** - improved local container testing +--- ## v1.7.1 ||| @@ -33,6 +55,7 @@ - [docker] bumped CLI-ETP version to "0.4.5" - **Housekeeping** - Added additional automated testing to the docker release process +--- ## v1.7.0 ||| @@ -54,7 +77,7 @@ - **Housekeeping** - DocFix Readme.md (thx [@ihommani](https://github.com/akamai/uls/pull/47)) - Increased default input_queue_size from 10000 to 15000 to avoid race conditions when an API is answering very fast - +--- ## v1.6.6 ||| @@ -72,7 +95,8 @@ - **Housekeeping** - Updated the ETP Links from developer.akamai.com to techdocs.akamai.com - Added "docker file liniting" into test scripts - +--- + ## v1.6.5 ||| |---|---| @@ -82,6 +106,7 @@ - **Minor improvements** - Allow manipulation of the [TCP & UDP output format](ARGUMENTS_ENV_VARS.md#list-of-parameters--environmental-variables) (--tcpudpformat / ULS_TCPUDP_FORMAT). - [docker] bumped source image to 3.11.4-slim-bookworm (new debian release) +--- ## v1.6.4 ||| @@ -98,6 +123,7 @@ - **Bugfix** - `--endtime ` didn't cause ULS to eventually stop ops. This is now fixed. - improved container detection (only cosmetic improvement) +--- ## v1.6.3 @@ -116,6 +142,7 @@ - fixed a bug in the "get_uls.sh" script which stated the wrong error message when pip was not found - minor fix to properly detect "podman" as docker alternative - Amendend installation steps for Guardicore and Linode log-fetcher(s) +--- ## v1.6.2 @@ -132,6 +159,7 @@ - fixed a bug in the Dockerfile that left uls/var unusable - **Housekeeping** - fixed some bugs in testing (false negative) & speeded up testing process +--- ## v1.6.1 @@ -146,7 +174,7 @@ - **Housekeeping** - Added parallel testing processes to speed up testing (see [Testing Readme](../test/README.md)) - added randomization tokens for "mocked" edgerc file (to avoid race condition in prallel testing) - +--- ## v1.6.0 @@ -170,6 +198,7 @@ - bumped python container (docker) to version 3.10.7 - bumped ETP-CLI version to 0.3.9 which should massively reduce the fetch lag - Added [documentation](./HIGH_AVAILABILITY.md) to explain high availability options for ULS +--- ## v1.5.1 @@ -182,6 +211,7 @@ - **Bugfix** - Use cli-eaa version 0.5.0.2 fixing a timezone issue on the EAA ADMIN feed +--- ## v1.5.0 @@ -207,7 +237,7 @@ - `event_bytes_interval` # total size in bytes processed - **Housekeeping** - Documented missing dependencies in [test/README.md](../test/README.md) - +--- ## v1.4.0 ||| @@ -226,6 +256,7 @@ - **Bugfix** - Fixed a bug in the test scripts to support real `.edgerc` +--- ## v1.3.5 ||| @@ -236,6 +267,7 @@ - **Minor improvements** - More QRADAR log source type definitions (thx to bitonio) - Added docker-compose ETP - Tenant example +--- ## v1.3.4 ||| @@ -248,6 +280,7 @@ - Added SUMO Logic (thx to huskar20 for the contribution) - bumped CLI-MFA to v0.0.9 - added resources, nodeSelector, tolerations and affinity to the helm values.yaml / template +--- ## v1.3.3 @@ -260,7 +293,7 @@ - Adopted to new MFA CLI Version (only single feed "EVENT" available anymore) - Amended new dates to the file headers - Added volume to dockerfile as data storage for "autoresume" - +--- ## v1.3.2 ||| @@ -284,6 +317,7 @@ - added a sanity check for "HTTP_OUT_FORMAT" to avoid issues with the ´%s´ seclector - removed forced http authentication token "--httpauthheader" (allow None) - discovered a bug in configparser -> [see FAQ entry](FAQ.md#uls-does-not-start-due-to-missing-field-in-config) +--- ## v1.3.1 @@ -295,6 +329,7 @@ - **Bugfixes** - fixed a checkpoint issue when using ETP / THREAD - some doc fixes +--- ## v1.3.0 ||| @@ -320,7 +355,7 @@ - Fixed an output issue on "CLI failure", added configureable output handling to the config - replaced pip with pip3 in CLI usage docs - Fixed a windows bug (bypass blocking on windows) + added a [FAQ entry on how fix a installation specific bug](FAQ.md#uls-on-windows-error-winerror-2-the-system-cannot-find-the-file-specified) - +--- ## v1.2.0 ||| @@ -349,6 +384,7 @@ - bumped EAA CLI Version to 0.4.4 (docker only) - Introduced dedicated ["OUTPUT" documentation](OUTPUTS.md) - introduced uls own requirements.txt in the [bin directory](../bin/requirements.txt) - still trying to keep req's as low as possible. +--- ## v1.1.0 ||| @@ -366,6 +402,7 @@ - added better error guidance when basic stuff is unset (input / output) - moved docker-compose from root dir to /docs - added `read_only: true` to the docker-compose.yml files (security enhancement) +--- ## v1.0.0 ||| @@ -380,7 +417,7 @@ - simplified cli - command re-usage (visual parsing of subprocess cmd) - cleaned up the Dockerfile - added [Log_Overview](LOG_OVERVIEW.md) page to extend background on logged data - +--- ## v0.9.0 ||| @@ -399,7 +436,7 @@ - added FAQ documents - Feature: - [FILTER (--filter) feature](ADDITIONAL_FEATURES.md#filter---filter-feature) introduced to reduce number of sent log lines towards SIEM - +--- ## v0.0.4 ||| @@ -416,6 +453,7 @@ - Feature: - EAA CONNECTOR HEALTH (CONHEALTH) now available - Preflight (forced) check for available cli's +--- ## v0.0.3 ||| @@ -428,6 +466,7 @@ - fixed a bug that caused Popen PIPE to hang in certain circumstances - bumped Dockerfile to newer CLI versions - [introduced RAW output](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature) (send data to stdout) +--- ## v0.0.2 ||| @@ -438,6 +477,7 @@ - fixed monitoring output bug in docker-compose - fixed bug in Dockerfile that prevented development builds - fixed a bug in EAA CLI handler +--- ## v0.0.1 (Initial Commit) |version|v0.0.1| diff --git a/docs/COMMAND_LINE_USAGE.md b/docs/COMMAND_LINE_USAGE.md index 72ea121..3919e9c 100644 --- a/docs/COMMAND_LINE_USAGE.md +++ b/docs/COMMAND_LINE_USAGE.md @@ -6,7 +6,7 @@ All commands referenced in this document are run from the repositories root leve ## Table of contents - [ULS Command Line Usage](#uls-command-line-usage) - - [Requirements](#requirements) + - [Pre-requisites](#pre-requisites) - [Installation](#installation) - [Automated Installation](#automated-installation) - [Manual Installation](#manual-installation) @@ -20,18 +20,19 @@ All commands referenced in this document are run from the repositories root leve - [Automated Update](#automated-update) - [Manual Update](#manual-update) -## Requirements +## Pre-requisites + +To run the operations in this documentation, make sure you have the following tools installed: -To run the operations within the following documentation, you need to have the following tools installed: - git -- python >= 3.9 (including pip) +- python >= `3.9` (including pip >= `22.2`) - Akamai API credentials file - `.edgerc` (see [API Credentials](AKAMAI_API_CREDENTIALS.md) for creation instructions) - Understanding of available [ULS CLI PARAMETERS](ARGUMENTS_ENV_VARS.md) - Access to `github.com`, `pypi.org`, `pythonhosted.org` and `pypi.python.org` within your firewall ## Installation -To install ULS, you can choose 2 different ways: automated or manual +To install ULS, you can choose 2 different ways: automated or manual. ### Automated Installation The automated installation actually does everything, the described below in the manual installation but saves you from the copying the blocks. diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml index 3a3f671..577d52e 100644 --- a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml +++ b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml @@ -4,4 +4,4 @@ description: Akamai Universal Log Streamer Helm installation type: application version: 2.0.0 -appVersion: "1.7.2" +appVersion: "1.7.3" diff --git a/scripts/get-uls.sh b/scripts/get-uls.sh index 2b8ed58..93da53c 100755 --- a/scripts/get-uls.sh +++ b/scripts/get-uls.sh @@ -5,6 +5,21 @@ default_modules="eaa,etp,mfa,gc,ln" default_install_dir="$(pwd)/uls" +function min_version() { + # Test if a given version string matches the requirements + # Usage: + # min_version required_min_version version + # Function returns: + # 0 if the version is equal or greater than the requirement + # 1 if the version is lower + local required_ver="$1" + local current_ver="$2" + if [ "$(printf '%s\n' "$required_ver" "$current_ver" | sort -V | head -n1)" = "$required_ver" ]; then + return 0 + else + return 1 + fi +} echo -e " AKAMAI @@ -14,7 +29,17 @@ echo -e " _/ _/ _/ _/ _/_/ _/_/_/_/ _/_/_/ -Universal Log Stream - Installer" +Unified Log Streamer - Installer + +Available ULS modules: + eaa: Enterprise Application Access + etp: Secure Internet Access Enterprise + mfa: Akamai phishproof MFA + gc: Akamai Guardicore Segmentation + ln: Linode Audit log + +More about supported feed: +https://github.com/akamai/uls/blob/main/docs/LOG_OVERVIEW.md" echo -e "\n\n\n" # Preflight checks @@ -32,12 +57,12 @@ if [[ -z $(which python3) ]] ; then fi ### check python version is correct for ULS +py_min_version="3.9" py_version=$(python3 --version | cut -d " " -f 2) -if [[ $(echo ${py_version} | cut -d "." -f 1 ) -ne 3 ]] || [[ $(echo ${py_version} | cut -d "." -f 2 ) -lt 9 ]]; then - echo "Wrong Python version - exiting" +min_version "${py_min_version}" "${py_version}" || { + echo "Python version must be >= ${py_min_version}, found ${py_version} - exiting" exit 1 -fi - +} ## pip3 if [[ -z $(which pip3) ]] ; then @@ -45,6 +70,16 @@ if [[ -z $(which pip3) ]] ; then exit 1 fi +### Check PIP version +pip3_min_version="22.2" +pip3_version="$(pip3 --version|cut -d' ' -f2)" +min_version "${pip3_min_version}" "${pip3_version}" || { + echo "pip3 version must be >= ${pip3_min_version}, version detected: ${pip3_version}" + echo "Consider upgrading your PIP version with command:" + echo " $(which pip3) install --upgrade pip" + exit 1 +} + ### Show versions to verify the correct binaries for python and pip are being used echo "We will use the following python binaries to install ULS:" echo -ne "python3: \t $(ls $(which python3))\n" @@ -131,8 +166,6 @@ function py_reqs() { ;; esac fi - - } ## Grab EAA-CLI diff --git a/test/positive_test.bats b/test/positive_test.bats index a9eecfc..7e0b45c 100644 --- a/test/positive_test.bats +++ b/test/positive_test.bats @@ -256,6 +256,14 @@ load 'bats/bats-assert/load.bash' rm -f /tmp/uls_eaa_access.ckpt } +## OUTPUTFORMAT SUBSITUTION +@test "OUTPUTFORMAT SUBSITUTION" { + export BATS_VAR="BATSTEST123" + run timeout ${uls_timeout_params} ${uls_bin} --input eaa --feed access --output tcp --host 127.0.0.1 --port 1234 --edgerc $uls_edgerc --section $uls_section --loglevel debug --tcpudpformat '$BATS_VAR - {api_hostname} %s' + assert_output --partial "OS_ENV_VARS new TCPUDP output string: BATSTEST123 - manage.akamai-access.com %s" + unset BATS_VAR +} + ## HELM LINT @test "LINT the HELM CHART" { run helm lint docs/examples/kubernetes/helm/akamai-uls --strict