diff --git a/Dockerfile b/Dockerfile index 5234558..2dd6467 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.5-slim-bullseye +FROM python:3.10.7-slim-bullseye LABEL MAINTAINER="Mike Schiessl - mike.schiessl@akamai.com" LABEL APP_LONG="Akamai Universal Log Streamer" LABEL APP_SHORT="ULS" @@ -10,9 +10,11 @@ ARG HOMEDIR="/opt/akamai-uls" ARG ULS_DIR="$HOMEDIR/uls" ARG EXT_DIR="$ULS_DIR/ext" -ARG ETP_CLI_VERSION="0.3.8" +ARG ETP_CLI_VERSION="0.3.9" ARG EAA_CLI_VERSION="0.5.0.2" ARG MFA_CLI_VERSION="0.0.9" +ARG GC_CLI_VERSION="dev" +ARG LINODE_CLI_VERSION="dev" # ENV VARS ENV ULS_DIR=$ULS_DIR @@ -63,6 +65,16 @@ ENV MFA-CLI_VERSION=$MFA_CLI_VERSION RUN git clone --depth 1 -b "${MFA_CLI_VERSION}" --single-branch https://github.com/akamai/cli-mfa.git ${EXT_DIR}/cli-mfa && \ pip3 install -r ${EXT_DIR}/cli-mfa/requirements.txt +## GC CLI +ENV GC_CLI_VERSION=$GC_CLI_VERSION +RUN git clone --depth 1 -b "${GC_CLI_VERSION}" --single-branch https://github.com/MikeSchiessl/gc-logs.git ${EXT_DIR}/cli-gc && \ + pip3 install -r ${EXT_DIR}/cli-gc/bin/requirements.txt + +## LINODE CLI +ENV LINODE_CLI_VERSION=$LINODE_CLI_VERSION +RUN git clone --depth 1 -b "${LINODE_CLI_VERSION}" --single-branch https://github.com/MikeSchiessl/ln-logs.git ${EXT_DIR}/cli-linode && \ + pip3 install -r ${EXT_DIR}/cli-linode/bin/requirements.txt + # ENTRYPOINTS / CMD VOLUME ["${ULS_DIR}/var"] ENTRYPOINT ["/usr/local/bin/python3","-u","bin/uls.py"] diff --git a/README.md b/README.md index 130dc97..d7bacde 100644 --- a/README.md +++ b/README.md @@ -6,21 +6,18 @@ The Unified Log Streamer (ULS) is designed to simplify SIEM integrations for Aka - [Enterprise Application Access (EAA)](https://www.akamai.com/us/en/products/security/enterprise-application-access.jsp) - [Enterprise Threat Protector (ETP)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp) - [Akamai MFA (MFA)](https://www.akamai.com/us/en/products/security/akamai-mfa.jsp) +- [Guardicore Micro Segmentation](https://www.akamai.com/lp/guardicore) Thanks to its modular design, ULS allows the connection of many SIEM solutions out-of-the-box. ULS can send data into any SIEM that supports either file, TCP, UDP or HTTP ingestion. It can be run directly as Python code, as a provided Docker container, through `docker compose` scripts or through helm within kubernetes. - - ![ULS docker compose usage](docs/images/uls_docker-compose_complex_example.png) - -## Table of contents +## Table of contents - [Akamai Unified Log Streamer (ULS)](#akamai-unified-log-streamer-uls) - [Introduction](#introduction) - - [Table of contents](#table-of-contents) - [Key Features](#key-features) - [Documentation](#documentation) - [Generic Requirements](#generic-requirements) @@ -35,7 +32,7 @@ It can be run directly as Python code, as a provided Docker container, through ## Key Features -- Supported Inputs (Secure Enterprise Access Products) +- Supported Inputs - [Enterprise Application Access (EAA)](https://www.akamai.com/us/en/products/security/enterprise-application-access.jsp) - [ACCESS](docs/LOG_OVERVIEW.md#access-logs-access) - [ADMIN](docs/LOG_OVERVIEW.md#admin-logs-admin) @@ -48,6 +45,13 @@ It can be run directly as Python code, as a provided Docker container, through - [PROXY](docs/LOG_OVERVIEW.md#proxy) - [Akamai Phish-proof Multi Factor Authenticator (AKAMAI-MFA)](https://www.akamai.com/us/en/products/security/akamai-mfa.jsp) - [EVENT](docs/LOG_OVERVIEW.md#authentication-logs-auth) + - [Akamai Guardicore Segmentation](https://www.akamai.com/lp/guardicore) (experimental) + - [NETLOG](docs/LOG_OVERVIEW.md#netlog) + - [INCIDENT](docs/LOG_OVERVIEW.md#incident) + - AGENT + - SYSTEM + - [Linode](https://www.linode.com/) (experimental) + - AUDIT - Supported data outputs @@ -122,6 +126,7 @@ In parallel, all new versions within the "main" branch will also be available on Contributions to this software can be provided via [Pull Requests](https://docs.github.com/en/github/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests) and will get merged after successful review. ## Changelog + Find a full Changelog of all added Features and fixed Bugs here: [ULS - Changelog](./docs/CHANGELOG.md) @@ -130,10 +135,11 @@ Find a full Changelog of all added Features and fixed Bugs here: Akamai ULS is provided "as-is". It is not supported by Akamai Support. Akamai is neither liable for the function nor for any caused problems that come along with the usage or caused by this tool. Please refer to the [LICENSE](./LICENSE) document for more information. To report an issue, feature request or bug, please open a new issue into the [GitHub Issues page](https://github.com/akamai/uls/issues). -This software is released under the "Apache License". Please read the [frequently asked questions](docs/FAQ.md) and visit the [debugging instructions](./docs/DEBUGGING.md) before opening a bug request. +This software is released under the "Apache License". Please read the [frequently asked questions](docs/FAQ.md) and visit the [troubleshooting and debugging instructions](./docs/DEBUGGING.md) before opening a ticket. [Pull requests](#development) to improve the code or enhance the functionality are welcome. ## LINKS / REFERENCES + [The ULS Project on GitHub](https://github.com/akamai/uls) [The ULS Project on Dockerhub](https://hub.docker.com/r/akamai/uls) diff --git a/bin/config/global_config.py b/bin/config/global_config.py index 80add73..846e0c1 100644 --- a/bin/config/global_config.py +++ b/bin/config/global_config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Common global variables / constants -__version__ = "1.5.1" +__version__ = "1.6.0" __tool_name_long__ = "Akamai Unified Log Streamer" __tool_name_short__ = "ULS" @@ -31,8 +31,16 @@ bin_mfa_cli = "ext/cli-mfa/bin/akamai-mfa" # Path to the MFA CLI Executable mfa_cli_feeds = ['EVENT'] # Available MFA CLI feeds + # Guardicore +bin_gc_cli = "ext/cli-gc/bin/akamai-gc" # Path to the GC CLI Executable +gc_cli_feeds = ['NETLOG', 'INCIDENT', 'AGENT', 'SYSTEM'] # Available GC CLI feeds + + # LINODE +bin_linode_cli = "ext/cli-linode/bin/akamai-linode" # Path to the LINODE CLI Executable +linode_cli_feeds = ['AUDIT'] # Available LINODE CLI feeds + # INPUT Choices -input_choices = ['EAA', 'ETP', 'MFA'] # Available input types +input_choices = ['EAA', 'ETP', 'MFA', 'GC', 'LINODE'] # Available input types input_format_choices = ['JSON', 'TEXT'] # Available input format choices (need to be supported by cli) # OUTPUT Choices @@ -85,6 +93,8 @@ edgerc_openapi = ["host", "client_token", "client_secret", "access_token"] # required fields for OPENAPI edgerc_eaa_legacy = ["eaa_api_host", "eaa_api_key", "eaa_api_secret"] # required for EAA - Legacy edgerc_mfa = ["mfa_integration_id", "mfa_signing_key"] # Required for MFA +edgerc_gc = ["gc_username", "gc_password", "gc_hostname"] # Required for Guardicore +edgerc_linode = ["linode_hostname", "linode_token"] # Required for Linode edgerc_documentation_url = "https://github.com/akamai/uls/blob/main/docs/AKAMAI_API_CREDENTIALS.md" edgerc_mock_file = "ext/edgerc" # Required for display the version if no edgercfile was given diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py index 2665b7f..9d33662 100644 --- a/bin/modules/UlsInputCli.py +++ b/bin/modules/UlsInputCli.py @@ -256,6 +256,67 @@ def proc_create(self): self._uls_useragent(self.product, "rawcmd") +\ shlex.split(self.rawcmd) + # Guardicore config + elif self.product == "GC": + product_path = self.root_path + "/" + uls_config.bin_gc_cli + product_feeds = uls_config.gc_cli_feeds + if not self.cliformat == "JSON": + aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) " + f"not available for {product_path}, continuing with JSON.") + if not self.rawcmd: + UlsTools.uls_check_edgerc(self.credentials_file, + self.credentials_file_section, + uls_config.edgerc_gc) + my_feed = self._feed_selector(self.feed, product_feeds) + cli_command = [self.bin_python, '-u', product_path, 'events', my_feed.lower(), '-f'] + cli_command[3:3] = self._uls_useragent(self.product, my_feed) + cli_command[3:3] = edgegrid_auth + cli_command[3:3] = self._prep_proxy(self.inproxy) + + # Append End and Starttime + if self.endtime: + # We need to remove "-f" from the end of the cli cmd if we work with endtime + cli_command = cli_command[:-1] + cli_command.extend(self._prep_start_endtime('--end', self.endtime)) + if self.starttime: + cli_command.extend(self._prep_start_endtime('--start', self.starttime)) + + else: + cli_command = [self.bin_python, product_path] +\ + self._uls_useragent(self.product, "rawcmd") +\ + shlex.split(self.rawcmd) + + # LINODE config + elif self.product == "LINODE": + product_path = self.root_path + "/" + uls_config.bin_linode_cli + product_feeds = uls_config.linode_cli_feeds + if not self.cliformat == "JSON": + aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) " + f"not available for {product_path}, continuing with JSON.") + if not self.rawcmd: + UlsTools.uls_check_edgerc(self.credentials_file, + self.credentials_file_section, + uls_config.edgerc_linode) + my_feed = self._feed_selector(self.feed, product_feeds) + cli_command = [self.bin_python, '-u', product_path, 'events', my_feed.lower(), '-f'] + cli_command[3:3] = self._uls_useragent(self.product, my_feed) + cli_command[3:3] = edgegrid_auth + cli_command[3:3] = self._prep_proxy(self.inproxy) + + # Append End and Starttime + if self.endtime: + # We need to remove "-f" from the end of the cli cmd if we work with endtime + cli_command = cli_command[:-1] + cli_command.extend(self._prep_start_endtime('--end', self.endtime)) + if self.starttime: + cli_command.extend(self._prep_start_endtime('--start', self.starttime)) + + else: + cli_command = [self.bin_python, product_path] +\ + self._uls_useragent(self.product, "rawcmd") +\ + shlex.split(self.rawcmd) + + # Everything else (undefined) else: aka_log.log.critical(f" {self.name} - No valid product selected " diff --git a/bin/modules/UlsTools.py b/bin/modules/UlsTools.py index 31010d3..8b62ba8 100644 --- a/bin/modules/UlsTools.py +++ b/bin/modules/UlsTools.py @@ -50,6 +50,8 @@ def _check_cli_installed(cli_bin): _check_cli_installed(root_path + "/" + uls_config.bin_eaa_cli) _check_cli_installed(root_path + "/" + uls_config.bin_etp_cli) _check_cli_installed(root_path + "/" + uls_config.bin_mfa_cli) + _check_cli_installed(root_path + "/" + uls_config.bin_gc_cli) + _check_cli_installed(root_path + "/" + uls_config.bin_linode_cli) def uls_version(root_path): @@ -60,9 +62,14 @@ def uls_version(root_path): my_edgerc_mock_file = root_path + "/" + uls_config.edgerc_mock_file def _get_cli_version(cli_bin, edgerc_mock_file): try: - version_proc = subprocess.Popen([uls_config.bin_python, cli_bin, "--edgerc", edgerc_mock_file, "version"], + if "gc" in cli_bin or 'linode' in cli_bin: + version_proc = subprocess.Popen([uls_config.bin_python, cli_bin, "--edgerc", edgerc_mock_file, "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + else: + version_proc = subprocess.Popen([uls_config.bin_python, cli_bin, "--edgerc", edgerc_mock_file, "version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) my_cli_version = version_proc.communicate()[0].decode().strip('\n') version_proc.terminate() if my_cli_version: @@ -84,7 +91,9 @@ def _get_cli_version(cli_bin, edgerc_mock_file): f"ULS Version\t\t{uls_config.__version__}\n\n" f"EAA Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_eaa_cli, my_edgerc_mock_file)}\n" f"ETP Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_etp_cli, my_edgerc_mock_file)}\n" - f"MFA Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_mfa_cli, my_edgerc_mock_file)}\n\n" + f"MFA Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_mfa_cli, my_edgerc_mock_file)}\n" + f"GC Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_gc_cli, my_edgerc_mock_file)}\n" + f"LINODE Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_linode_cli, my_edgerc_mock_file)}\n\n" f"OS Plattform\t\t{platform.platform()}\n" f"OS Version\t\t{platform.release()}\n" f"Python Version\t\t{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\n" diff --git a/bin/requirements.txt b/bin/requirements.txt index 817cf8a..1a83dcb 100644 --- a/bin/requirements.txt +++ b/bin/requirements.txt @@ -1,2 +1,3 @@ jmespath>=0.10.0 -requests>=2.25.1 \ No newline at end of file +requests>=2.25.1 +pytz>=2021.1 diff --git a/docs/AKAMAI_API_CREDENTIALS.md b/docs/AKAMAI_API_CREDENTIALS.md index 949f68a..0cb3f5f 100644 --- a/docs/AKAMAI_API_CREDENTIALS.md +++ b/docs/AKAMAI_API_CREDENTIALS.md @@ -13,6 +13,10 @@ This document describes how to create Akamai API credentials and configure them - [ETP {OPEN} API Reporting](#etp-open-api-reporting) - [Akamai MFA](#akamai-mfa) - [MFA Integration for logging](#mfa-integration-for-logging) + - [Guardicore](#guardicore) + - [Guardicore API Integration](#guardicore-api-integration) + - [Linode](#linode) + - [Linode API Token](#linode-api-credentials) - [Advanced .edgerc usage](#advanced-edgerc-usage) - [Multiple customer contracts](#multiple-customer-contracts) - [Partner & employee enhancement](#partner--employee-enhancement) @@ -20,17 +24,14 @@ This document describes how to create Akamai API credentials and configure them ## Feeds / API overview -|Product long name|Acronym|Feed|API| -|---|---|---|---| -|Enterprise Application Access|EAA|ACCESS|[EAA Legacy API](#eaa-legacy-api-for-access-and-admin-audit-feeds)| -|Enterprise Application Access|EAA|ADMIN|[EAA Legacy API](#eaa-legacy-api-for-access-and-admin-audit-feeds)| -|Enterprise Application Access|EAA|HEALTH|[{OPEN} API / Enterprise Application Access](#eaa-open-api-for-connector-health-feed)| -|Enterprise Threat Protector|ETP|THREAT|[{OPEN} API / ETP Report](#etp-open-api-reporting)| -|Enterprise Threat Protector|ETP|AUP|[{OPEN} API / ETP Report](#etp-open-api-reporting)| -|Enterprise Threat Protector|ETP|DNS|[{OPEN} API / ETP Report](#etp-open-api-reporting)| -|Enterprise Threat Protector|ETP|PROXY|[{OPEN} API / ETP Report](#etp-open-api-reporting)| -|Akamai MFA|MFA|AUTH|[MFA Integration](#mfa-integration-for-logging)| -|Akamai MFA|MFA|POLICY|[MFA Integration](#mfa-integration-for-logging)| +|Product long name|Acronym| Feed(s) | API | +|---|---|---------------------------------|---------------------------------------------------------------------------------------| +|Enterprise Application Access|EAA| ACCESS, ADMIN | [EAA Legacy API](#eaa-legacy-api-for-access-and-admin-audit-feeds) | +|Enterprise Application Access|EAA| HEALTH | [{OPEN} API / Enterprise Application Access](#eaa-open-api-for-connector-health-feed) | +|Enterprise Threat Protector|ETP| THREAT, AUP, DNS, PROXY | [{OPEN} API / ETP Report](#etp-open-api-reporting) | +|Akamai MFA|MFA| EVENTS | [MFA Integration](#mfa-integration-for-logging) | +|Guardicore|GC| NETLOG, INCIDENT, AGENT, SYSTEM | [Guardicore API Integration](#guardicore-api-integration) | +|Linode|LN| AUDIT | [Linode API Credentials](#linode-api-credentials) | ## Setting up API credentials for ULS @@ -137,6 +138,44 @@ mfa_integration_id = app_xxxxxxxxxxxxxxxxxxxxx mfa_signing_key = xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx ``` +### Guardicore + +#### Guardicore API Integration +Guardicore is using the portal users for API access. Therefore it is recommended to create a "read only" (= GUEST role) user within Centra Administration. +- Go to Administration +- Select "Users" in the left navigation tree +- Click the "Create User" button +- Enter a username and a password, select "Guest" as permission scheme +- Confirm by clicking the SAVE button +- Now logout and login with the newly created user and follow tha password change procedure +- Note down your guardicore Adminsitration (=API) url without https +- Add/replace/amend the following section to your `.edgerc` file and replace the data accordingly: + +```INI +[default] +; Guardicore integration credentials +gc_hostname = your_hostname.guardicore.com # Do not prepend https:// +gc_username = XXXXXXXXXXXX +gc_password = XXXXXXXXXXXXX +``` + +### Linode +#### Linode API Credentials +- Login into your linode cloud console +- Click on your user name on the top right +- Select API Tokens +- Create a personal Access Token +- Enter a Label and select all privleges to "READ ONLY" +- Set expiry to your needs +- Confirm by clicking the CREATE TOKEN button +- Copy the token provided in the next field +- Add/replace/amend the following section to your `.edgerc` file and replace the data accordingly: +```INI +[default] +; Guardicore integration credentials +linode_hostname = your_hostname.guardicore.com # Do not prepend https:// +linode_token = XXXXXXXXXXXX +``` ## Advanced .edgerc usage ### Multiple customer contracts diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md index 3be24a5..022649f 100644 --- a/docs/ARGUMENTS_ENV_VARS.md +++ b/docs/ARGUMENTS_ENV_VARS.md @@ -12,17 +12,17 @@ The following tables list all available command line parameters and their corres ## INPUT -| Parameter | Env - Var | Options | Default | Description | -|---------------------------|-----------------|---------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| -i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA' | None | Specify the desired INPUT source | -| --feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV'
ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'
MFA: 'EVENT' | None | Specify the desired INPUT feed | -| --format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format | -| --inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT | None | Adjust proxy usage for INPUT data collection (cli)
If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) | -| --rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature) | -| --edgerc | ULS_EDGERC | /path/to/your/.edgerc | '~/.edgerc' | Specify the location of the .edgerc EDGE GRID AUTH file | -| --section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file | -| --starttime | ULS_STARTTIME | EPOCH timestamp | `cli_default` | Specify an EPOCH timestamp from where to start the log collection. | -| --endtime | ULS_ENDTIME | EPOCH timestamp | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! | +| Parameter | Env - Var | Options | Default | Description | +|---------------------------|-----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA', 'GC', 'LINODE' | None | Specify the desired INPUT source | +| --feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV'

ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'

MFA: 'EVENT'

GC: 'NETLOG', 'INCIDENT', 'AGENT', 'SYSTEM'

LINODE: 'AUDIT' | None | Specify the desired INPUT feed | +| --format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format | +| --inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT | None | Adjust proxy usage for INPUT data collection (cli)
If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) | +| --rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature) | +| --edgerc | ULS_EDGERC | /path/to/your/.edgerc | '~/.edgerc' | Specify the location of the .edgerc EDGE GRID AUTH file | +| --section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file | +| --starttime | ULS_STARTTIME | EPOCH timestamp | `cli_default` | Specify an EPOCH timestamp from where to start the log collection. | +| --endtime | ULS_ENDTIME | EPOCH timestamp | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! | ## OUTPUT diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e0d14a2..b451751 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,29 @@ # Version History +## v1.6.0 + +||| +|---|---| +|Date|2022-08 +|Kind| BUGFIX release +|Author|mschiess@akamai.com +- **Features** + - Support for Akamai Guardicore Segmentation (experimental) + - Available feeds: netlog, incident, agent, system + Please ensure to [update your .edgerc](./AKAMAI_API_CREDENTIALS.md#guardicore-api-integration) file for GC usage + - Support for Akamai Linode Cloud (experimental) + - Available feed: audit + Please ensure to [update your .edgerc](AKAMAI_API_CREDENTIALS.md#linode-api-credentials) file for GC usage + +- **Minor improvements** + - ULS [Install Script](COMMAND_LINE_USAGE.md#automated-installation) allows fully working ULS installation via a single script + - ULS [Updater](COMMAND_LINE_USAGE.md#automated-update) helps to maintain a proper updated version of ULS + Modules + - Amended [Command Line Usage](COMMAND_LINE_USAGE.md) documentation on how to use the installer / updater + - bumped python container (docker) to version 3.10.7 + - bumped ETP-CLI version to 0.3.9 which should massively reduce the fetch lag + - Added [documentation](./HIGH_AVAILABILITY.md) to explain high availability options for ULS + + ## v1.5.1 ||| diff --git a/docs/COMMAND_LINE_USAGE.md b/docs/COMMAND_LINE_USAGE.md index 48d0f6b..e0704da 100644 --- a/docs/COMMAND_LINE_USAGE.md +++ b/docs/COMMAND_LINE_USAGE.md @@ -3,21 +3,26 @@ This document describes the "command line usage" of the ULS software. All commands referenced in this document are run from the repositories root level. -### Table of contents +## Table of contents - [ULS Command Line Usage](#uls-command-line-usage) - - [Overview](#overview) - [Requirements](#requirements) - [Installation](#installation) - - [Clone ULS Repo](#clone-uls-repo) - - [Enterprise Access CLI's](#enterprise-access-clis) + - [Automated Installation](#automated-installation) + - [Manual Installation](#manual-installation) + - [Clone ULS repository](#clone-uls-repository) + - [Akamai Enterprise Access CLI's](#akamai-enterprise-access-clis) - [Setup the .EDGERC File](#setup-the-edgerc-file) - [Setup the .EDGERC File](#setup-the-edgerc-file-1) - [Usage](#usage) - [Usage examples](#usage-examples) - [ULS as a service: systemd](#uls-as-a-service-systemd) +- [Updating](#updating) + - [Automated Update](#automated-update) + - [Manual Update](#manual-update) ## Requirements + To run the operations within the following documentation, you need to have the following tools installed: - git - python >= 3.9 (including pip) @@ -26,7 +31,23 @@ To run the operations within the following documentation, you need to have the f - Access to `github.com`, `pypi.org`, `pythonhosted.org` and `pypi.python.org` within your firewall ## Installation -### Clone ULS Repo + +To install ULS, you can choose 2 different ways: automated or manual + +### Automated Installation +The automated installation actually does everything, the described below in the manual installation but saves you from the copying the blocks. + +```bash +# Got to your preferred installation folder (it will install to a subdirectory ./uls +# run the following two lines and follow the on - screen guidance +curl -O https://raw.githubusercontent.com/akamai/uls/main/scripts/get-uls.sh +bash get-uls.sh +``` + +### Manual Installation + +#### Clone ULS repository + Clone the ULS repository from github, change into the ULS repository directory afterwards and install requirements. ```bash git clone https://github.com/akamai/uls.git @@ -34,7 +55,8 @@ cd uls pip3 install -r bin/requirements.txt ``` -### Enterprise Access CLI's +#### Akamai Enterprise Access CLI's + The Secure Enterprise Access Products CLI Tools need to be installed into the `ext` directory within this repo. Please run the following commands to download the CLI tools and install the requirements. ```bash @@ -52,11 +74,13 @@ pip3 install -r ext/cli-mfa/requirements.txt ``` ### Setup the .EDGERC File + Copy the `.edgerc` file ([instructions for creation](AKAMAI_API_CREDENTIALS.md)) to your users home directory (~): ```bash cp /path/to/your/.edgerc ~/.edgerc ``` ### Setup the .EDGERC File + Copy the `.edgerc` file ([instructions for creation](AKAMAI_API_CREDENTIALS.md)) to your users home directory (~): ```bash cp /path/to/your/.edgerc ~/.edgerc @@ -165,4 +189,39 @@ You might need to adjust some settings until you get what you need. Keep in mind $ systemctl daemon-reload ``` -For more information on systemd, see man help `man systemd.exec` \ No newline at end of file +For more information on systemd, see man help `man systemd.exec` + +# Updating + +To update ULS and the required modules, you can choose 2 different ways: automated or manual + +## Automated Update + +The automated Script will do exactly the steps described in the manual update section below, but saves you from the copying the blocks. + +From the root of the ULS directory, run +```bash +bash scripts/update-uls.sh +``` +and follow the on - screen guidance + +## Manual Update + +```bash +# make sure you are at the root level of your ULS directory +# ULS +git pull -q +pip3 install -q -r bin/requirements.txt + +# EAA CLI (only if installed) +git -C ext/cli-eaa pull -q +pip3 install -q -r ext/cli-eaa/requirements.txt + +# ETP CLI (only if installed) +git -C ext/cli-etppull -q +pip3 install -q -r ext/cli-etp/requirements.txt + +# MFA CLI (only if installed) +git -C ext/cli-mfa pull -q +pip3 install -q -r ext/cli-mfa/requirements.txt +``` diff --git a/docs/DEBUGGING.md b/docs/DEBUGGING.md index 00f7bc9..73b6e83 100644 --- a/docs/DEBUGGING.md +++ b/docs/DEBUGGING.md @@ -1,26 +1,54 @@ -# ULS DEBUGGING -This document describes the debugging of the ULS software. -Please make sure you follow the steps before filing an issue on the [GitHub Issues Page](https://github.com/akamai/uls/issues). -Follow the steps to collect "supportive" data you should also provide when filing an issue. +# ULS Troubleshooting and debugging + +This document describes how to troubleshoot and debug Unified Log Streamer software. + +Please make sure you follow the steps before filing an issue on the [GitHub Issues Page](https://github.com/akamai/uls/issues), collecting these information increase our chance to understand the issue, and shorten the time to provide you support. + +## Make sure you're using the latest version + +We strive at fixing issues as they arise and add new features, with new version coming throughout the months. + +We strongly encourage to always get the latest version of ULS. You can find the latest version in the [release page](https://github.com/akamai/uls/releases). See also [ULS updater script documentation](COMMAND_LINE_USAGE.md#automated-update). ## Table of contents -- [Version Information](#version-information) -- [Debug Output from ULS](#debug-output-from-uls) -- [Debug Output from CLI](#debug-output-from-cli) + +- [ULS Troubleshooting and debugging](#uls-troubleshooting-and-debugging) + - [Make sure you're using the latest version](#make-sure-youre-using-the-latest-version) + - [Table of contents](#table-of-contents) + - [Version information](#version-information) + - [Commands to trigger version output](#commands-to-trigger-version-output) + - [Command line](#command-line) + - [Docker](#docker) + - [Example Output](#example-output) + - [Debug output from ULS](#debug-output-from-uls) + - [Commands to trigger debug output](#commands-to-trigger-debug-output) + - [Command Line](#command-line-1) + - [Docker](#docker-1) + - [Debug output from CLI's](#debug-output-from-clis) + - [Steps to get CLI Debug output](#steps-to-get-cli-debug-output) + - [CLI with ULS](#cli-with-uls) + - [DOCKER / DOCKER-COMPOSE](#docker--docker-compose) ## Version information + Providing information about relevant module & software versions can help identify issues. -###Commands to trigger version output -####Command Line: + +### Commands to trigger version output + +#### Command line + ```bash python3 bin/uls.py --version ``` -####Docker + +#### Docker + ```bash docker run -ti --mount type=bind,source="/path/to/your/.edgerc",target="/opt/akamai-uls/.edgerc",readonly --rm akamai/uls -v ``` -###Example Output +### Example Output + ```text Akamai Unified Log Streamer Version information ULS Version 0.0.1 @@ -32,18 +60,21 @@ MFA Version 0.0.4 OS Plattform Linux-5.10.25-linuxkit-x86_64-with-glibc2.28 OS Version 5.10.25-linuxkit Python Version 3.9.5 - ``` - ## Debug output from ULS + To debug problems into depth, ULS provides an extremely verbose output about every step processed within ULS. -###Commands to trigger debug output -####Command Line: + +### Commands to trigger debug output +#### Command Line + ```bash python3 bin/uls.py --loglevel debug ``` -####Docker + +#### Docker + ```bash docker run -ti \ --mount type=bind,source="/path/to/your/.edgerc",target="/opt/akamai-uls/.edgerc",readonly \ @@ -55,9 +86,13 @@ docker run -ti \ Instead of adding it to the command line, you can also set the `ULS_LOGLEVEL` ENV VAR to "DEBUG" ## Debug output from CLI's + This is helpful when a deeper debugging from within the CLI tool is required. -###Steps to get CLI Debug output: + +### Steps to get CLI Debug output + #### CLI with ULS + 1) Run the desired stream in [ULS DEBUG mode](#debug-output-from-uls) and grab the command output i.e. ```bash ULS D UlsInputCli - CLI Command: ['python3', 'ext/cli-eaa/bin/akamai-eaa', '--edgerc', '~/.edgerc', '--section', 'akamaidemo', '--user-agent-prefix', 'ULS/0.0.3_EAA-CONHEALTH', 'connector', 'list', '--perf', '--tail', '--json'] @@ -73,7 +108,8 @@ This is helpful when a deeper debugging from within the CLI tool is required. ``` Via this way, you can get the full CLI debugging output within ULS -#### DOCKER / DOCKER-COMPOSE +#### DOCKER / DOCKER-COMPOSE + 1) Start a debugging docker instance and connect to it's console ```bash docker run -ti \ diff --git a/docs/HIGH_AVAILABILITY.md b/docs/HIGH_AVAILABILITY.md new file mode 100644 index 0000000..09dae75 --- /dev/null +++ b/docs/HIGH_AVAILABILITY.md @@ -0,0 +1,12 @@ +# ULS High Availability +This document describes high availability options for the ULS software. +ULS itself does not bring HA capabilities out of the box, but it was built to support different HA techniques. + +## Docker +To use high availability alongside with the provided ULS docker container, a HA orchestration layer like [SWARM](https://docs.docker.com/engine/swarm/), [KUBERNETES](https://kubernetes.io/) or [others](https://devopscube.com/docker-container-clustering-tools/) is required. +It is highly recommended, to enable the ULS auto-resume feature and store the according "resume - data" on a persistent volume which is available accross different compute nodes (e.g. PVC, NFS, ...). +This setup allows the ULS container(s) to maintain HA across multiple compute nodes. + +## Daemon +Whenever container usage isn't an option, high availability can be achieved by using OS-based high availability systems like [pacemaker](https://clusterlabs.org/) / [corosync](https://de.wikipedia.org/wiki/Corosync_Cluster_Engine) or [heartbeat](http://www.linux-ha.org/wiki/Heartbeat). +To avoid duplicate events in the SIEM, it is highly recommended to enable the ULS auto-resume feature and store the according "resume - data" on a persistent volume that is available across the designated n odes (e.g. DRBD, NFS, ...). diff --git a/docs/LOG_OVERVIEW.md b/docs/LOG_OVERVIEW.md index da82635..ca92d18 100644 --- a/docs/LOG_OVERVIEW.md +++ b/docs/LOG_OVERVIEW.md @@ -13,7 +13,6 @@ Here are some examples (per product) and links to additional information. - [Admin Logs (ADMIN)](#admin-logs-admin) - [Connector Health (CONHEALTH)](#connector-health-conhealth) - [Device Posture Inventory (DEVINV)](#device-posture-inventory-devinv) - - [Enterprise Threat Protector (ETP)](#enterprise-threat-protector-etp) - [Threat Log (THREAT)](#threat-log-threat) - [Accceptable Use Policy Logs (AUP)](#accceptable-use-policy-logs-aup) @@ -21,6 +20,12 @@ Here are some examples (per product) and links to additional information. - [PROXY](#proxy) - [Akamai MFA (MFA)](#akamai-mfa-mfa) - [Authentication Logs (AUTH)](#authentication-logs-auth) + - [Guardicore](#guardicore) + - [NETLOG](#netlog) + - [INCIDENT](#incident) + - AGENT + - SYSTEM + - Linode ## Enterprise Application Access (EAA) @@ -2083,7 +2088,6 @@ Additional information regarding the log fields can be found on [here](https://d ## Akamai MFA (MFA) - Additional information regarding the MFA log fields can be found on [here](https://techdocs.akamai.com/mfa/docs/splunk-app). ### Authentication Logs (AUTH) @@ -2119,3 +2123,436 @@ Authentication Events Example: "principal_uuid": null } ``` + +## Guardicore + +### NETLOG +Guardicore netlog example +```json +{ +'flow_id':'XXXXX2045a3630852de54dcb99e86f6b06d3969050e153efaed1cb2c4', +'bucket_id':'XXXX-62fa-459a-80e1-c96c2eacdee9', +'source_node_id':'XXXX-3a21-4508-87bd-d0a6512442bc', +'destination_node_id':'UnknownAsset_Internal_192.168.0.0/16', +'source_node_type':'asset', +'destination_node_type':'subnet', +'source_process':'gc-channel', +'destination_process':'Unknown Server (443/TCP)', +'source_process_id':'XXXX491e858806fc17a722c7e93f780e867df4800e6a9bddcc396abf39250', +'destination_process_id':'7XXXX5aad6d81ebe5037013fded72223e12e2d8a0d4e4823c90232139b', +'source_process_name':'gc-channel', +'destination_process_name':'Unknown Server (443/TCP)', +'destination_port':443, +'count':2, +'slot_start_time':XXX72413000, +'incidents':False, +'connection_type':'UNKNOWN', +'source_ip':'192.168.2.76', +'destination_ip':'192.168.2.68', +'ip_protocol':'Tcp', +'source_asset_hash':317458, +'destination_asset_hash':349875, +'violates_policy':False, +'policy_rule':'default', +'policy_ruleset':None, +'policy_verdict':'allowed', +'db_insert_time':'XXXX-11-09T04:09:54.293504', +'id':'XXXXXX-7d27-48b0-ab66-cdedcbc444c3', +'source':{ +'vm':{ +'_id':'XXXXXX-3a21-4508-87bd-d0a6512442bc', +'name':'Gollum Lab Server' +} +}, +'has_mismatch_alert':False, +'original_policy_verdict':'allowed', +'source_process_full_path':'/var/lib/guardicore/sbin/gc-channel', +'destination_process_full_path':None +} +``` + +### INCIDENT +Guardicore incident example +```json +{ + "_cls": "Incident.NetworkVisibilityIncident", + "_id": "4506a1ba-15d1-4d10-8299-5c10f34975cb", + "affected_assets": [ + { + "country": "Israel", + "country_code": "IL", + "ip": "172.17.0.3", + "is_inner": true, + "labels": [ + "source" + ], + "vm": { + "full_name": "192.168.0.102/Attacker2", + "id": "b40db74f-5f2d-4cda-9c9b-c2cdd8158b1f", + "name": "Attacker2", + "nics": [ + { + "discovered_ip_addresses": [ + "192.168.0.1" + ], + "ip_addresses": [ + "192.168.0.1" + ], + "mac_address": "00:50:56:bb:2d:ab", + "network_id": "fe3ef6a8-858f-407d-bd6e-30fb9cc30522", + "network_name": "CommandsNet", + "network_orchestration_id": "dvportgroup-105", + "orchestration_details": [ + {} + ], + "switch_id": "dvs-102", + "vif_id": "0", + "vlan_id": 1001 + } + ], + "orchestration_details": [ + { + "orchestration_id": "7f43c9a2-e8b9-4ce7-a2d1-908bd5182d51", + "orchestration_obj_id": "vm-280588", + "orchestration_type": "vSphere", + "revision_id": 190709142948 + } + ], + "recent_domains": [ + "mydomain.com" + ], + "tenant_name": "192.168.0.102" + }, + "vm_id": "b40db74f-5f2d-4cda-9c9b-c2cdd8158b1f" + } + ], + "closed_time": 1510979377066, + "concatenated_tags": [ + { + "display_name": "Internal", + "tag_class": "ENRICHER" + } + ], + "destination_asset": { + "ip": "172.17.0.3", + "is_inner": true, + "labels": [ + "destination" + ], + "vm": { + "full_name": "192.168.0.102/Attacker2", + "id": "b40db74f-5f2d-4cda-9c9b-c2cdd8158b1f", + "name": "Attacker2", + "nics": [ + { + "discovered_ip_addresses": [ + "192.168.0.1" + ], + "ip_addresses": [ + "192.168.0.1" + ], + "mac_address": "00:50:56:bb:2d:ab", + "network_id": "fe3ef6a8-858f-407d-bd6e-30fb9cc30522", + "network_name": "CommandsNet", + "network_orchestration_id": "dvportgroup-105", + "orchestration_details": [ + { + "orchestration_id": "7f43c9a2-e8b9-4ce7-a2d1-908bd5182d51", + "orchestration_obj_id": "vm-280588", + "orchestration_type": "vSphere", + "revision_id": 190709142948 + } + ], + "switch_id": "dvs-102", + "vif_id": "0", + "vlan_id": 1001 + } + ], + "orchestration_details": [ + { + "orchestration_id": "7f43c9a2-e8b9-4ce7-a2d1-908bd5182d51", + "orchestration_obj_id": "vm-280588", + "orchestration_type": "vSphere", + "revision_id": 190709142948 + } + ], + "recent_domains": [ + "mydomain.com" + ], + "tenant_name": "192.168.0.102" + }, + "vm_id": "74238291-b85a-42fb-bac9-80c402abee04" + }, + "destination_net": "many", + "destinations": [ + { + "ip_int": "1684300813", + "ports": [ + "ARP" + ] + } + ], + "direction": "unidirectional", + "doc_version": 59, + "end_time": 1504689940953, + "ended": true, + "enriched": true, + "events": [ + { + "_id": "a120a1ba-15d1-4d10-8299-5c10f34975cb", + "description": "Scanner detected.", + "destinations": [ + [ + 443, + "10.0.0.1" + ] + ], + "doc_version": 57, + "event_source": "DP-422FB8A7-D525-D1A4-B2B8-1ABAD6137A64", + "event_type": "DatapathScanDetectionEvent", + "id": "a120a1ba-15d1-4d10-8299-5c10f34975cb", + "incident_id": "4506a1ba-15d1-4d10-8299-5c10f34975cb", + "is_experimental": false, + "processed_time": 1512375007928, + "received_time": 1512375006000, + "severity": 40, + "source_ip": "127.0.0.1", + "source_vm": { + "full_name": "192.168.0.102/Attacker2", + "id": "b40db74f-5f2d-4cda-9c9b-c2cdd8158b1f", + "name": "Attacker2", + "nics": [ + { + "discovered_ip_addresses": [ + "192.168.0.1" + ], + "ip_addresses": [ + "192.168.0.1" + ], + "mac_address": "00:50:56:bb:2d:ab", + "network_id": "fe3ef6a8-858f-407d-bd6e-30fb9cc30522", + "network_name": "CommandsNet", + "network_orchestration_id": "dvportgroup-105", + "orchestration_details": [ + {} + ], + "switch_id": "dvs-102", + "vif_id": "0", + "vlan_id": 1001 + } + ], + "orchestration_details": [ + { + "orchestration_id": "7f43c9a2-e8b9-4ce7-a2d1-908bd5182d51", + "orchestration_obj_id": "vm-280588", + "orchestration_type": "vSphere", + "revision_id": 190709142948 + } + ], + "recent_domains": [ + "mydomain.com" + ], + "tenant_name": "192.168.0.102" + }, + "tag_refs": [ + "aa96a1ba-15d1-4d10-8299-5c10f34975cb" + ], + "time": 1512374896401, + "uuid": "a32a528c-293b-4185-80ec-652b435c1297" + } + ], + "experimental_id": "925fd3c9-f933-482f-9eb1-61f61ba4bc3a", + "first_asset": { + "asset_id": "2.20.153.161", + "asset_type": "IP" + }, + "flow_ids": [ + "17bf0add897bbb7a1bd55c24b9cc7ea5fb92ad6f2dd0be7704734accef4226e6__bd8d287246b85de5334d115adc61a4232fd1d904e6f57cf16c0f7d8adde3eb51__Tcp__80" + ], + "has_export": true, + "has_policy_violations": true, + "id": "4506a1ba-15d1-4d10-8299-5c10f34975cb", + "incident_group": [ + { + "gid": "a7f870fa-85ab-47fe-8156-f5e45e7208eb", + "gname": "GRP-a7f870fa" + } + ], + "incident_type": "Reveal", + "iocs": [ + { + "initiating_tags": [ + "0736307a-8aef-4d42-b4dd-c89da42e9135" + ], + "ioc_id": "bfc2399e-9546-4d4a-8989-cf9bcf5426e2", + "related_tags": [ + "3d7a9595-f293-4c54-addf-4fa22c29725e" + ], + "source": "LoginsDetector.detect_successful_logins" + } + ], + "is_experimental": true, + "labels": [ + "source" + ], + "last_updated_time": 1504689940952, + "originl_id": "", + "policy_revision": 22, + "processed_eventS_count": 1, + "recommendations": [ + { + "handle_template": "Quarantine File", + "id": "2206a1ba-15d1-4d10-8299-5c10f34975cb", + "parts": [ + { + "type": "text", + "value": "Quarantine malicious file " + } + ], + "rule_id": "a106a1ba-15d1-4d10-8299-5c10f34975cb", + "rule_type": "", + "type": "FileQuarantineRecommendation" + } + ], + "reenrich_count": 0, + "remote_index": "incidents__2017_12_03_00_00_00", + "second_asset": { + "asset_id": "2.20.153.161", + "asset_type": "IP" + }, + "sensor_name": "DP-422F8CE0-C6A1-D633-52C4-2EDE049F6094", + "sensor_type": "VISIBILITY", + "severity": 40, + "similarity_calculated": false, + "source_asset": { + "ip": "172.17.0.1", + "is_inner": true, + "labels": [ + "source" + ], + "vm": { + "full_name": "192.168.0.102/Attacker2", + "id": "b40db74f-5f2d-4cda-9c9b-c2cdd8158b1f", + "name": "Attacker2", + "nics": [ + { + "discovered_ip_addresses": [ + "192.168.0.1" + ], + "ip_addresses": [ + "192.168.0.1" + ], + "mac_address": "00:50:56:bb:2d:ab", + "network_id": "fe3ef6a8-858f-407d-bd6e-30fb9cc30522", + "network_name": "CommandsNet", + "network_orchestration_id": "dvportgroup-105", + "orchestration_details": [ + { + "orchestration_id": "7f43c9a2-e8b9-4ce7-a2d1-908bd5182d51", + "orchestration_obj_id": "vm-280588", + "orchestration_type": "vSphere", + "revision_id": 190709142948 + } + ], + "switch_id": "dvs-102", + "vif_id": "0", + "vlan_id": 1001 + } + ], + "orchestration_details": [ + { + "orchestration_id": "7f43c9a2-e8b9-4ce7-a2d1-908bd5182d51", + "orchestration_obj_id": "vm-280588", + "orchestration_type": "vSphere", + "revision_id": 190709142948 + } + ], + "recent_domains": [ + "mydomain.com" + ], + "tenant_name": "192.168.0.102" + }, + "vm_id": "11338291-b85a-42fb-bac9-80c402abee04" + }, + "source_ip": "10.0.0.1", + "source_vm": { + "full_name": "192.168.0.102/Attacker2", + "id": "b40db74f-5f2d-4cda-9c9b-c2cdd8158b1f", + "name": "Attacker2", + "nics": [ + { + "discovered_ip_addresses": [ + "192.168.0.1" + ], + "ip_addresses": [ + "192.168.0.1" + ], + "mac_address": "00:50:56:bb:2d:ab", + "network_id": "fe3ef6a8-858f-407d-bd6e-30fb9cc30522", + "network_name": "CommandsNet", + "network_orchestration_id": "dvportgroup-105", + "orchestration_details": [ + { + "orchestration_id": "7f43c9a2-e8b9-4ce7-a2d1-908bd5182d51", + "orchestration_obj_id": "vm-280588", + "orchestration_type": "vSphere", + "revision_id": 190709142948 + } + ], + "switch_id": "dvs-102", + "vif_id": "0", + "vlan_id": 1001 + } + ], + "orchestration_details": [ + { + "orchestration_id": "7f43c9a2-e8b9-4ce7-a2d1-908bd5182d51", + "orchestration_obj_id": "vm-280588", + "orchestration_type": "vSphere", + "revision_id": 190709142948 + } + ], + "recent_domains": [ + "mydomain.com" + ], + "tenant_name": "192.168.0.102" + }, + "source_vm_id": "74238291-b85a-42fb-bac9-80c402abee04", + "start_time": 1504688829035, + "total_events_count": +} +``` + +## Linode +### AUDIT Logs +Additional information regarding the log fields can be found on [here](https://www.linode.com/docs/api/account/#events-list) + +```json +{ + "action": "ticket_create", + "created": "2018-01-01T00:01:01", + "duration": 300.56, + "entity": { + "id": 11111, + "label": "Problem booting my Linode", + "type": "ticket", + "url": "/v4/support/tickets/11111" + }, + "id": 123, + "message": "None", + "percent_complete": null, + "rate": null, + "read": true, + "secondary_entity": { + "id": "linode/debian9", + "label": "linode1234", + "type": "linode", + "url": "/v4/linode/instances/1234" + }, + "seen": true, + "status": null, + "time_remaining": null, + "username": "exampleUser" + } +``` + diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml index 9ba5aa8..25a7259 100644 --- a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml +++ b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml @@ -3,5 +3,5 @@ name: akamai-uls description: Akamai Universal Log Streamer Helm installation type: application -version: 1.5.1 -appVersion: "1.5.1" +version: 1.6.0 +appVersion: "1.6.0" diff --git a/docs/examples/kubernetes/helm/akamai-uls/values.yaml b/docs/examples/kubernetes/helm/akamai-uls/values.yaml index 02943ce..05d2b40 100644 --- a/docs/examples/kubernetes/helm/akamai-uls/values.yaml +++ b/docs/examples/kubernetes/helm/akamai-uls/values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 akamai_uls: edgerc_secret: name: "akamai-edgerc" - # Ourput config + # OuTput config environment: ULS_LOGLEVEL: "WARN" ULS_INPUT: "ETP" diff --git a/scripts/get-uls.sh b/scripts/get-uls.sh new file mode 100755 index 0000000..fb5fec3 --- /dev/null +++ b/scripts/get-uls.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# This file will install the latest ULS including all of its modules (latest version) into the current directory/uls +# bash $(curl https://) + +default_modules="eaa,etp,mfa,gc,ln" +default_install_dir="$(pwd)/uls" + + +echo -e " + AKAMAI + _/ _/ _/ _/_/_/ + _/ _/ _/ _/ + _/ _/ _/ _/_/ +_/ _/ _/ _/ + _/_/ _/_/_/_/ _/_/_/ + +Universal Log Stream - Installer" +echo -e "\n\n\n" + +# Preflight checks +## check git +if [[ -z $(which git) ]] ; then + echo "GIT binary was not found - exiting" + exit 1 +fi + +## check python version +if [[ -z $(which python3) ]] ; then + echo "Python3 binary was not found - exiting" + exit 1 +fi + +py_version=$(python3 --version | cut -d " " -f 2) +if [[ $(echo ${py_version} | cut -d "." -f 1 ) -ne 3 ]] || [[ $(echo ${py_version} | cut -d "." -f 2 ) -lt 9 ]]; then + echo "Wrong Python version - exiting" + exit 1 +fi + +## pip3 +if [[ -z $(which pip3) ]] ; then + echo "Python3 binary was not found - exiting" + exit 1 +fi + +# ASK for INSTALL DETAILS +## Install DIR +echo -n "Installation Dir [Default: \"$default_install_dir\"] (ENTER for default or new value): " +read install_dir +if [[ -z "${install_dir}" ]] ; then + install_dir=${default_install_dir} +fi + +## Install Modules +echo -n "Modules [$default_modules] (ENTER for all): " +read install_modules +if [[ -z "$install_modules" ]] ; then + install_modules=$default_modules +fi + +echo -e "\n\n\n\n\n" +echo "Installing ULS to: $install_dir" +echo "Modules to install: $install_modules" + +## Are you allright to continue ? +echo -ne "Are the above settings correct? [yN]" +read continue + +if [[ ! "$continue" == "y" ]] ; then + echo "Configuration not confirmed, exiting" + exit 1 +fi + +## Check if ULS is already installed ? +if [[ -f "${install_dir}/bin/uls.py" ]] ; then + echo -e "\n\nAttention:\n" + echo "ULS seems to be already installed." + echo "Proceeding with the installation could lead to some serious issue." + echo -n "Do you want to proceed [yN]" + read already_installed + if [[ ! ${already_installed} == "y" ]] ; then + echo "Stopping installation - exiting" + exit 1 + fi +fi + +## Continue anywaY ? + + +# Installation +## Grab ULS +git clone -q https://github.com/akamai/uls.git $install_dir/ +pip3 install -q -r ${install_dir}/bin/requirements.txt + + +## Grab EAA-CLI +if [[ "$install_modules" == *"eaa"* ]] ; then +echo "Installing EAA-CLI" + git clone -q --depth 1 --single-branch https://github.com/akamai/cli-eaa.git ${install_dir}/ext/cli-eaa + pip3 install -q -r ${install_dir}/ext/cli-eaa/requirements.txt +fi + +## GRAB ETP-CLI +if [[ "$install_modules" == *"etp"* ]] ; then +echo "Installing ETP-CLI" + git clone -q --depth 1 --single-branch https://github.com/akamai/cli-etp.git ${install_dir}/ext/cli-etp + pip3 install -q -r ${install_dir}/ext/cli-etp/requirements.txt +fi + +## GRAB MFA-CLI +if [[ "$install_modules" == *"mfa"* ]] ; then +echo "Installing MFA-CLI" + git clone -q --depth 1 --single-branch https://github.com/akamai/cli-mfa.git ${install_dir}/ext/cli-mfa + pip3 install -q -r ${install_dir}/ext/cli-mfa/requirements.txt +fi + +## GRAB GC-CLI +if [[ "$install_modules" == *"gc"* ]] ; then +echo "Installing GC-CLI" + git clone -q --depth 1 -b dev --single-branch https://github.com/MikeSchiessl/gc-logs.git ${install_dir}/ext/cli-gc + pip3 install -q -r ${install_dir}/ext/cli-gc/bin/requirements.txt +fi + + +## GRAB LINODE-CLI +if [[ "$install_modules" == *"ln"* ]] ; then +echo "Installing LINODE-CLI" + git clone -q --depth 1 -b dev --single-branch https://github.com/MikeSchiessl/ln-logs.git ${install_dir}/ext/cli-linode + pip3 install -q -r ${install_dir}/ext/cli-ln/bin/requirements.txt +fi + +# Finishing off +echo -e "\n\n\n" +echo "ULS has been successfully installed" +${install_dir}/bin/uls.py --version + + +exit 0 \ No newline at end of file diff --git a/scripts/update-uls.sh b/scripts/update-uls.sh new file mode 100755 index 0000000..c12415c --- /dev/null +++ b/scripts/update-uls.sh @@ -0,0 +1,113 @@ +#!/bin/bash +set -e + +default_uls_dir="$(pwd)" + + +echo -e " + AKAMAI + _/ _/ _/ _/_/_/ + _/ _/ _/ _/ + _/ _/ _/ _/_/ +_/ _/ _/ _/ + _/_/ _/_/_/_/ _/_/_/ + +Universal Log Stream - Updater" +echo -e "\n\n\n" + + + + + +# ASK for INSTALL DETAILS +## Install DIR +echo -n "Update (ULS) Dir [Default: \"$default_uls_dir\"] (ENTER for default or new value): " +read uls_dir +if [[ -z "${uls_dir}" ]] ; then + uls_dir=${default_uls_dir} +fi + +# Pre flight checks +if [[ -f "${uls_dir}/bin/uls.py" ]] ; then + echo "ULS found in ${uls_dir}" +elif [[ -f "${uls_dir}/../bin/uls.py" ]] ; then + uls_dir="${uls_dir}/../" + echo "ULS found in ${uls_dir}" +else + echo -e "\n\n\n" + echo "ULS not found in ${uls_dir} ." + echo "Please specify a the correct directory (root of ULS) - exiting" + exit 1 +fi + +echo "Current Versions (pre update)" +${uls_dir}/bin/uls.py --version + +echo -e "\n\n\nDo you want to continue ? [yN]" +read continue + +if [[ ! "${continue}" == "y" ]] ; then + echo "Aborting update - exiting" + exit 1 +fi + + +# Running the updates +## ULS +echo "Updating ULS" +git -C ${uls_dir} pull -q +pip3 install -q -r ${uls_dir}/bin/requirements.txt + +## EAA +if [[ -d "${uls_dir}/ext/cli-eaa" ]] ; then + echo "EAA-CLI detected, updating" + git -C ${uls_dir}/ext/cli-eaa pull -q + pip3 install -q -r ${uls_dir}/ext/cli-eaa/requirements.txt +else + echo "NO EAA-CLI detected - skipping" +fi + +## ETP +if [[ -d "${uls_dir}/ext/cli-etp" ]] ; then + echo "ETP-CLI detected, updating" + git -C ${uls_dir}/ext/cli-etp pull -q + pip3 install -q -r ${uls_dir}/ext/cli-etp/requirements.txt +else + echo "NO ETP-CLI detected - skipping" +fi + +## MFA +if [[ -d "${uls_dir}/ext/cli-mfa" ]] ; then + echo "MFA-CLI detected, updating" + git -C ${uls_dir}/ext/cli-mfa pull -q + pip3 install -q -r ${uls_dir}/ext/cli-mfa/requirements.txt +else + echo "NO MFA-CLI detected - skipping" +fi + +## GC +if [[ -d "${uls_dir}/ext/cli-gc" ]] ; then + echo "GC-CLI detected, updating" + git -C ${uls_dir}/ext/cli-gc pull -q + pip3 install -q -r ${uls_dir}/ext/cli-gc/bin/requirements.txt +else + echo "NO GC-CLI detected - skipping" +fi + + +## LINODE +if [[ -d "${uls_dir}/ext/cli-ln" ]] ; then + echo "LINODE-CLI detected, updating" + git -C ${uls_dir}/ext/cli-linode pull -q + pip3 install -q -r ${uls_dir}/ext/cli-linode/bin/requirements.txt +else + echo "NO LN-CLI detected - skipping" +fi + + +echo -e "\n\n\nUpdate is complete." +echo "Updated versions (post update)" +${uls_dir}/bin/uls.py --version + + +exit 0 \ No newline at end of file diff --git a/test/positive_test.bats b/test/positive_test.bats index 611db5e..18ae2c0 100644 --- a/test/positive_test.bats +++ b/test/positive_test.bats @@ -11,6 +11,7 @@ mocked_edgerc=FALSE # TIMEOUT uls_test_timeout=60 +uls_kill_timeout=90 ### Switch between mocked and real edgerc @@ -22,6 +23,8 @@ if [ "$mocked_edgerc"=="FALSE" ] ; then eaa_access_assert="username" eaa_devinv_assert="client_version" etp_assert="configId" + gc_assert="flow_id" + linode_assert="" jmespath_assert="['" else # TESTING EDGERC FILE & section @@ -31,6 +34,8 @@ else eaa_access_assert="" eaa_devinv_assert="" etp_assert="" + gc_assert="" + linode_assert="" jmespath_assert="" fi @@ -42,28 +47,28 @@ load 'bats/bats-assert/load.bash' # POSITIVE tests ## EAA @test "EAA - ACCESS" { - run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --edgerc $uls_edgerc --section $uls_section + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --edgerc $uls_edgerc --section $uls_section assert_output --partial $eaa_access_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout } @test "EAA - ADMIN" { - run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed admin --output raw --edgerc $uls_edgerc --section $uls_section + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input eaa --feed admin --output raw --edgerc $uls_edgerc --section $uls_section assert_output --partial "" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout } @test "EAA - CONHEALTH" { - run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed admin --output raw --edgerc $uls_edgerc --section $uls_section + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input eaa --feed admin --output raw --edgerc $uls_edgerc --section $uls_section assert_output "" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout } @test "EAA - DEVINV" { - run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed devinv --output raw --edgerc $uls_edgerc --section $uls_section + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input eaa --feed devinv --output raw --edgerc $uls_edgerc --section $uls_section assert_output --partial $eaa_devinv_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status @@ -72,7 +77,7 @@ load 'bats/bats-assert/load.bash' ## ETP @test "ETP - THREAT" { - run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed threat --output raw --edgerc $uls_edgerc --section $uls_section + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed threat --output raw --edgerc $uls_edgerc --section $uls_section assert_output --partial $etp_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status @@ -80,7 +85,7 @@ load 'bats/bats-assert/load.bash' } @test "ETP - AUP" { - run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed aup --output raw --edgerc $uls_edgerc --section $uls_section + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed aup --output raw --edgerc $uls_edgerc --section $uls_section assert_output --partial $etp_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status @@ -88,7 +93,7 @@ load 'bats/bats-assert/load.bash' } @test "ETP - DNS" { - run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed dns --output raw --edgerc $uls_edgerc --section $uls_section + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed dns --output raw --edgerc $uls_edgerc --section $uls_section assert_output --partial $etp_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status @@ -96,7 +101,7 @@ load 'bats/bats-assert/load.bash' } @test "ETP - PROXY" { - run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed proxy --output raw --edgerc $uls_edgerc --section $uls_section + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed proxy --output raw --edgerc $uls_edgerc --section $uls_section assert_output --partial $etp_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status @@ -105,7 +110,25 @@ load 'bats/bats-assert/load.bash' ## MFA @test "MFA - EVENT" { - run timeout --preserve-status $uls_test_timeout $uls_bin --input mfa --feed event --output raw --edgerc $uls_edgerc --section $uls_section + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input mfa --feed event --output raw --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} + +## GUARDICORE +@test "GC - NETLOG" { + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input gc --feed netlog --output raw --edgerc $uls_edgerc --section $uls_section + assert_output --partial $gc_assert + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} + +## LINODE +@test "LINODE - AUDIT" { + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input linode --feed audit --output raw --edgerc $uls_edgerc --section $uls_section assert_output "" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status @@ -114,7 +137,7 @@ load 'bats/bats-assert/load.bash' ## FILE OUTPUT @test "FILE: ETP - THREAT" { - run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed threat --output file --filename "/tmp/uls_tmplogfile.log" --edgerc $uls_edgerc --section $uls_section + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed threat --output file --filename "/tmp/uls_tmplogfile.log" --edgerc $uls_edgerc --section $uls_section assert_output "" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status @@ -123,7 +146,7 @@ load 'bats/bats-assert/load.bash' } @test "FILEACTION: ETP - THREAT" { - run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed threat --output file --filename "/tmp/uls_tmplogfile.log" --filebackup 1 --fileaction "/bin/zip '%s'" --edgerc $uls_edgerc --section $uls_section + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed threat --output file --filename "/tmp/uls_tmplogfile.log" --filebackup 1 --fileaction "/bin/zip '%s'" --edgerc $uls_edgerc --section $uls_section assert_output "" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status @@ -133,7 +156,7 @@ load 'bats/bats-assert/load.bash' ## Transformation @test "TRANSFORM - MCAS" { - run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed dns --output raw --transformation mcas --edgerc $uls_edgerc --section $uls_section + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed dns --output raw --transformation mcas --edgerc $uls_edgerc --section $uls_section assert_output --partial "detection_time" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status @@ -141,7 +164,7 @@ load 'bats/bats-assert/load.bash' } @test "TRANSFORM - JMESPATH" { - run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --transformation jmespath --transformationpattern '[geo_country, geo_state]' --edgerc $uls_edgerc --section $uls_section + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --transformation jmespath --transformationpattern '[geo_country, geo_state]' --edgerc $uls_edgerc --section $uls_section assert_output --partial $jmespath_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status @@ -151,7 +174,7 @@ load 'bats/bats-assert/load.bash' ## AUTORESUME @test "AUTORESUME - Create File" { rm -f /tmp/uls_eaa_access.ckpt - run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --edgerc $uls_edgerc --section $uls_section --autoresume --autoresumepath /tmp/ + run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --edgerc $uls_edgerc --section $uls_section --autoresume --autoresumepath /tmp/ assert_output --partial $eaa_access_assert #assert_output --partial " seems to be empty" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" @@ -159,9 +182,10 @@ load 'bats/bats-assert/load.bash' [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout rm -f /tmp/uls_eaa_access.ckpt } + ## EAA @test "LINT the HELM CHART" { run helm lint docs/examples/kubernetes/helm/akamai-uls --strict assert_output --partial "0 chart(s) failed" [ "$status" -eq 0 ] #return value for Chart Lint: 0 -} \ No newline at end of file +}