diff --git a/.github/workflows/build_publish_docker.yml b/.github/workflows/build_publish_docker.yml index df0a30fa..980e2e6a 100644 --- a/.github/workflows/build_publish_docker.yml +++ b/.github/workflows/build_publish_docker.yml @@ -25,7 +25,8 @@ jobs: "inspector", "logcollector", "logserver", - "prefilter" + "prefilter", + "monitoring" ] permissions: contents: read diff --git a/config.yaml b/config.yaml index 531365a8..f441d13d 100644 --- a/config.yaml +++ b/config.yaml @@ -32,8 +32,8 @@ pipeline: - [ "response_ip", IpAddress ] - [ "size", RegEx, '^\d+b$' ] batch_handler: - batch_size: 1000 - batch_timeout: 20.0 + batch_size: 10000 + batch_timeout: 30.0 subnet_id: ipv4_prefix_length: 24 ipv6_prefix_length: 64 @@ -65,25 +65,25 @@ pipeline: monitoring: clickhouse_connector: - batch_size: 10000 + batch_size: 50 # do not set higher batch_timeout: 2.0 environment: timestamp_format: "%Y-%m-%dT%H:%M:%S.%fZ" kafka_brokers: - - hostname: 172.27.0.3 + - hostname: kafka1 port: 8097 - - hostname: 172.27.0.4 + - hostname: kafka2 port: 8098 - - hostname: 172.27.0.5 + - hostname: kafka3 port: 8099 kafka_topics: pipeline: - logserver_in: "pipeline.logserver_in" - logserver_to_collector: "pipeline.logserver_to_collector" - batch_sender_to_prefilter: "pipeline.batch_sender_to_prefilter" - prefilter_to_inspector: "pipeline.prefilter_to_inspector" - inspector_to_detector: "pipeline.inspector_to_detector" + logserver_in: "pipeline-logserver_in" + logserver_to_collector: "pipeline-logserver_to_collector" + batch_sender_to_prefilter: "pipeline-batch_sender_to_prefilter" + prefilter_to_inspector: "pipeline-prefilter_to_inspector" + inspector_to_detector: "pipeline-inspector_to_detector" monitoring: clickhouse_server: - hostname: 172.27.0.11 + hostname: clickhouse-server diff --git a/docker/.env b/docker/.env index bdaaff39..a4441b26 100644 --- a/docker/.env +++ b/docker/.env @@ -1 +1 @@ -MOUNT_PATH=./default.txt +MOUNT_PATH=../../default.txt diff --git a/docker/benchmark_tests/Dockerfile.run_test b/docker/benchmark_tests/Dockerfile.run_test new file mode 100644 index 00000000..ccef495b --- /dev/null +++ b/docker/benchmark_tests/Dockerfile.run_test @@ -0,0 +1,17 @@ +FROM python:3.11-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 + +WORKDIR /usr/src/app + +RUN pip --disable-pip-version-check install --no-cache-dir --no-compile marshmallow_dataclass colorlog pyYAML confluent_kafka numpy polars scikit-learn torch + +COPY src/base ./src/base +COPY src/train ./src/train +COPY config.yaml . +COPY docker/benchmark_tests . +COPY data ./data + +RUN rm -rf /root/.cache + +CMD [ "python", "run_test.py"] diff --git a/docker/benchmark_tests/docker-compose.run_test.yml b/docker/benchmark_tests/docker-compose.run_test.yml new file mode 100644 index 00000000..78231964 --- /dev/null +++ b/docker/benchmark_tests/docker-compose.run_test.yml @@ -0,0 +1,20 @@ +services: + benchmark_test_run: + build: + context: ../.. + dockerfile: docker/benchmark_tests/Dockerfile.run_test + network: host + networks: + docker_heidgaf: + deploy: + resources: + limits: + cpus: '2' + memory: 512m + reservations: + cpus: '1' + memory: 256m + +networks: + docker_heidgaf: + external: true diff --git a/docker/benchmark_tests/run_test.py b/docker/benchmark_tests/run_test.py new file mode 100644 index 00000000..4c2f6960 --- /dev/null +++ b/docker/benchmark_tests/run_test.py @@ -0,0 +1,248 @@ +import datetime +import ipaddress +import os +import random +import sys +import time + +import polars as pl +from confluent_kafka import KafkaError + +sys.path.append(os.getcwd()) +from src.base.kafka_handler import SimpleKafkaProduceHandler +from src.train.dataset import Dataset, DatasetLoader +from src.base.log_config import get_logger +from src.base.utils import setup_config + +logger = get_logger() +config = setup_config() + +PRODUCE_TO_TOPIC = config["environment"]["kafka_topics"]["pipeline"]["logserver_in"] + + +class DatasetGenerator: + """Generates log lines and datasets.""" + + def __init__(self, data_base_path: str = "./data"): + datasets = DatasetLoader(base_path=data_base_path, max_rows=10000) + + dataset = Dataset( + data_path="", + data=pl.concat( + [ + datasets.dgta_dataset.data, + # datasets.cic_dataset.data, + # datasets.bambenek_dataset.data, + # datasets.dga_dataset.data, + # datasets.dgarchive_dataset.data, + ] + ), + max_rows=1000, + ) + + self.domains = dataset.data + + def generate_random_logline( + self, statuses: list[str] = None, record_types: list[str] = None + ): + """Generates a (mostly) random logline.""" + if record_types is None: + record_types = 6 * ["AAAA"] + 10 * ["A"] + ["PR", "CNAME"] + + if statuses is None: + statuses = ["NOERROR", "NXDOMAIN"] + + # choose timestamp + timestamp = ( + datetime.datetime.now() + datetime.timedelta(0, 0, random.randint(0, 900)) + ).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + # choose status code + status = random.choice(statuses) + + # choose client IP address + number_of_subnets = 50 + client_ip = ( + f"192.168.{random.randint(0, number_of_subnets)}.{random.randint(1, 255)}" + ) + + # choose server IP address + server_ip = f"10.10.0.{random.randint(1, 100)}" + + # choose random domain (can be malicious or benign) + domain = self.get_random_domain() + + # choose random record type + record_type = random.choice(record_types) + + # choose random response IP address + def _get_random_ipv4(): + max_ipv4 = ipaddress.IPv4Address._ALL_ONES # 2 ** 32 - 1 + return ipaddress.IPv4Address._string_from_ip_int( + random.randint(0, max_ipv4) + ) + + def _get_random_ipv6(): + max_ipv6 = ipaddress.IPv6Address._ALL_ONES # 2 ** 128 - 1 + return ipaddress.IPv6Address._string_from_ip_int( + random.randint(0, max_ipv6) + ) + + ip_address_choices = [_get_random_ipv4(), _get_random_ipv6()] + response_ip_address = random.choice(ip_address_choices) + + # choose random size + size = f"{random.randint(50, 255)}b" + + return f"{timestamp} {status} {client_ip} {server_ip} {domain} {record_type} {response_ip_address} {size}" + + def get_random_domain(self) -> str: + random_domain = self.domains.sample(n=1) + return random_domain["query"].item() + + def generate_dataset(self, number_of_elements: int) -> list[str]: + dataset = [] + + for _ in range(number_of_elements): + logline = self.generate_random_logline() + dataset.append(logline) + + return dataset + + +class ScalabilityTest: + """Base class for tests that focus on the scalability of the software.""" + + def __init__(self): + self.dataset_generator = DatasetGenerator() + self.kafka_producer = SimpleKafkaProduceHandler() + + self.interval_lengths = None + self.msg_per_sec_in_intervals = None + + def execute(self): + """Executes the test with the configured parameters.""" + logger.warning(f"Start at: {datetime.datetime.now()}") + + cur_index = 0 + for i in range(len(self.msg_per_sec_in_intervals)): + cur_index = self._execute_one_interval( + cur_index=cur_index, + msg_per_sec=self.msg_per_sec_in_intervals[i], + length_in_sec=self.interval_lengths[i], + ) + + logger.warning(f"Stop at: {datetime.datetime.now()}") + + def _execute_one_interval( + self, cur_index: int, msg_per_sec: float | int, length_in_sec: float | int + ) -> int: + start_of_interval_timestamp = datetime.datetime.now() + logger.warning( + f"Start interval with {msg_per_sec} msg/s at {start_of_interval_timestamp}" + ) + + while ( + datetime.datetime.now() - start_of_interval_timestamp + < datetime.timedelta(seconds=length_in_sec) + ): + try: + self.kafka_producer.produce( + PRODUCE_TO_TOPIC, + self.dataset_generator.generate_random_logline(), + ) + logger.info( + f"Sent message {cur_index + 1} at: {datetime.datetime.now()}" + ) + cur_index += 1 + except KafkaError: + logger.warning(KafkaError) + time.sleep(1.0 / msg_per_sec) + + logger.warning(f"Finish interval with {msg_per_sec} msg/s") + return cur_index + + +class RampUpTest(ScalabilityTest): + """Starts with a low rate and increases the rate in fixed intervals.""" + + def __init__( + self, + msg_per_sec_in_intervals: list[float | int], + interval_length_in_sec: int | float | list[int | float], + ): + super().__init__() + self.msg_per_sec_in_intervals = msg_per_sec_in_intervals + + if type(interval_length_in_sec) is list: + self.interval_lengths = interval_length_in_sec + else: + self.interval_lengths = [ + interval_length_in_sec for _ in range(len(msg_per_sec_in_intervals)) + ] + + if len(interval_length_in_sec) != len(msg_per_sec_in_intervals): + raise Exception("Different lengths of interval lists. Must be equal.") + + +class BurstTest(ScalabilityTest): + """Starts with a normal rate, sends a high rate for a short period, then returns to normal rate. Repeats the + process for a defined number of times.""" + + def __init__( + self, + normal_rate_msg_per_sec: float | int, + burst_rate_msg_per_sec: float | int, + normal_rate_interval_length: float | int, + burst_rate_interval_length: float | int, + number_of_intervals: int = 1, + ): + super().__init__() + + self.msg_per_sec_in_intervals = [normal_rate_msg_per_sec] + self.interval_lengths = [normal_rate_interval_length] + + for _ in range(number_of_intervals): + self.msg_per_sec_in_intervals.append(burst_rate_msg_per_sec) + self.msg_per_sec_in_intervals.append(normal_rate_msg_per_sec) + + self.interval_lengths.append(burst_rate_interval_length) + self.interval_lengths.append(normal_rate_interval_length) + + +class LongTermTest(ScalabilityTest): + """Starts with a low rate and increases the rate in fixed intervals.""" + + def __init__(self, full_length: float | int, msg_per_sec: float | int): + super().__init__() + + self.msg_per_sec_in_intervals = [msg_per_sec] + self.interval_lengths = [full_length] + + +def main(): + """Creates the test instance and executes the test.""" + # ramp_up_test = RampUpTest( + # msg_per_sec_in_intervals=[1, 10, 50, 100, 150], + # interval_length_in_sec=[10, 5, 4, 4, 2], + # ) + # ramp_up_test.execute() + + burst_test = BurstTest( + normal_rate_msg_per_sec=20, + burst_rate_msg_per_sec=10000, + normal_rate_interval_length=10, + burst_rate_interval_length=2, + number_of_intervals=3, + ) + burst_test.execute() + + # long_term_test = LongTermTest( + # full_length=10.4, + # msg_per_sec=15, + # ) + # long_term_test.execute() + + +if __name__ == "__main__": + main() diff --git a/docker/create_datatest_tables/dgta_dataset.sql b/docker/create_datatest_tables/dgta_dataset.sql new file mode 100644 index 00000000..723dfe70 --- /dev/null +++ b/docker/create_datatest_tables/dgta_dataset.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS dgta_dataset ( + query String, + class Int32, + labels Array(String), + tld String, + fqdn String, + secondleveldomain String, + thirdleveldomain String +) +ENGINE = MergeTree +PRIMARY KEY(query); diff --git a/src/monitoring/create_tables/alerts.sql b/docker/create_tables/alerts.sql similarity index 88% rename from src/monitoring/create_tables/alerts.sql rename to docker/create_tables/alerts.sql index 92e4ad14..1fe9c73b 100644 --- a/src/monitoring/create_tables/alerts.sql +++ b/docker/create_tables/alerts.sql @@ -3,6 +3,7 @@ CREATE TABLE IF NOT EXISTS alerts ( alert_timestamp DateTime64(6) NOT NULL, suspicious_batch_id UUID NOT NULL, overall_score Float32 NOT NULL, + domain_names String NOT NULL, result String, ) ENGINE = MergeTree diff --git a/src/monitoring/create_tables/batch_timestamps.sql b/docker/create_tables/batch_timestamps.sql similarity index 100% rename from src/monitoring/create_tables/batch_timestamps.sql rename to docker/create_tables/batch_timestamps.sql diff --git a/src/monitoring/create_tables/dns_loglines.sql b/docker/create_tables/dns_loglines.sql similarity index 87% rename from src/monitoring/create_tables/dns_loglines.sql rename to docker/create_tables/dns_loglines.sql index c3468f79..6f705932 100644 --- a/src/monitoring/create_tables/dns_loglines.sql +++ b/docker/create_tables/dns_loglines.sql @@ -5,7 +5,7 @@ CREATE TABLE IF NOT EXISTS dns_loglines ( status_code String NOT NULL, client_ip String NOT NULL, record_type String NOT NULL, - additional_fields Nullable(String) + additional_fields String ) ENGINE = MergeTree PRIMARY KEY (logline_id); diff --git a/src/monitoring/create_tables/failed_dns_loglines.sql b/docker/create_tables/failed_dns_loglines.sql similarity index 100% rename from src/monitoring/create_tables/failed_dns_loglines.sql rename to docker/create_tables/failed_dns_loglines.sql diff --git a/docker/create_tables/fill_levels.sql b/docker/create_tables/fill_levels.sql new file mode 100644 index 00000000..3a694b01 --- /dev/null +++ b/docker/create_tables/fill_levels.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS fill_levels ( + timestamp DateTime64(6) NOT NULL, + stage String NOT NULL, + entry_type String NOT NULL, + entry_count UInt32 DEFAULT 0 +) +ENGINE = MergeTree +PRIMARY KEY (timestamp, stage, entry_type); diff --git a/src/monitoring/create_tables/logline_timestamps.sql b/docker/create_tables/logline_timestamps.sql similarity index 100% rename from src/monitoring/create_tables/logline_timestamps.sql rename to docker/create_tables/logline_timestamps.sql diff --git a/src/monitoring/create_tables/logline_to_batches.sql b/docker/create_tables/logline_to_batches.sql similarity index 100% rename from src/monitoring/create_tables/logline_to_batches.sql rename to docker/create_tables/logline_to_batches.sql diff --git a/src/monitoring/create_tables/server_logs.sql b/docker/create_tables/server_logs.sql similarity index 100% rename from src/monitoring/create_tables/server_logs.sql rename to docker/create_tables/server_logs.sql diff --git a/src/monitoring/create_tables/server_logs_timestamps.sql b/docker/create_tables/server_logs_timestamps.sql similarity index 100% rename from src/monitoring/create_tables/server_logs_timestamps.sql rename to docker/create_tables/server_logs_timestamps.sql diff --git a/src/monitoring/create_tables/suspicious_batch_timestamps.sql b/docker/create_tables/suspicious_batch_timestamps.sql similarity index 100% rename from src/monitoring/create_tables/suspicious_batch_timestamps.sql rename to docker/create_tables/suspicious_batch_timestamps.sql diff --git a/src/monitoring/create_tables/suspicious_batches_to_batch.sql b/docker/create_tables/suspicious_batches_to_batch.sql similarity index 100% rename from src/monitoring/create_tables/suspicious_batches_to_batch.sql rename to docker/create_tables/suspicious_batches_to_batch.sql diff --git a/docker/datatests.json b/docker/datatests.json new file mode 100644 index 00000000..bb7c1c94 --- /dev/null +++ b/docker/datatests.json @@ -0,0 +1,1175 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": true, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT count(DISTINCT client_ip) AS malicious_detected\nFROM dns_loglines\nWHERE client_ip IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)\nAND client_ip IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)", + "refId": "malicious_detected" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": true, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT count(DISTINCT client_ip) AS benign_detected\nFROM dns_loglines\nWHERE client_ip IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)\nAND client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)", + "refId": "benign_detected" + }, + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$malicious_detected / ($benign_detected + $malicious_detected)", + "hide": false, + "refId": "A", + "type": "math" + } + ], + "title": "True Positive Rate", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": true, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT count(DISTINCT client_ip) AS malicious_detected\nFROM dns_loglines\nWHERE client_ip IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)\nAND client_ip IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)", + "refId": "malicious_detected" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": true, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT count(DISTINCT client_ip) AS benign_detected\nFROM dns_loglines\nWHERE client_ip IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)\nAND client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)", + "refId": "benign_detected" + }, + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$benign_detected / ($benign_detected + $malicious_detected)", + "hide": false, + "refId": "A", + "type": "math" + } + ], + "title": "False Positive Rate", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 9, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT count(DISTINCT client_ip) AS malicious_detected\nFROM dns_loglines\nWHERE client_ip IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)\nAND client_ip IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)", + "refId": "B" + } + ], + "title": "malicious/detected", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 13, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT count(DISTINCT client_ip) AS benign_detected\nFROM dns_loglines\nWHERE client_ip IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)\nAND client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)", + "refId": "B" + } + ], + "title": "benign/detected", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 27, + "options": { + "folderUID": "", + "includeVars": true, + "keepTime": true, + "maxItems": 10, + "query": "", + "showFolderNames": false, + "showHeadings": false, + "showRecentlyViewed": false, + "showSearch": true, + "showStarred": false, + "tags": [] + }, + "pluginVersion": "11.2.2+security-01", + "title": "Dashboards", + "transparent": true, + "type": "dashlist" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": true, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT count(DISTINCT client_ip) AS malicious_notdetected\nFROM dns_loglines\nWHERE client_ip IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)\nAND client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)", + "refId": "malicious_notdetected" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": true, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT count(DISTINCT client_ip) AS benign_notdetected\nFROM dns_loglines\nWHERE client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)\nAND client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)", + "refId": "benign_notdetected" + }, + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$malicious_notdetected / ($benign_notdetected + $malicious_notdetected)", + "hide": false, + "refId": "A", + "type": "math" + } + ], + "title": "False Negative Rate", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": true, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT count(DISTINCT client_ip) AS malicious_notdetected\nFROM dns_loglines\nWHERE client_ip IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)\nAND client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)", + "refId": "malicious_notdetected" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": true, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT count(DISTINCT client_ip) AS benign_notdetected\nFROM dns_loglines\nWHERE client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)\nAND client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)", + "refId": "benign_notdetected" + }, + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$benign_notdetected / ($benign_notdetected + $malicious_notdetected)", + "hide": false, + "refId": "A", + "type": "math" + } + ], + "title": "True Negative Rate", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 9, + "y": 3 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT count(DISTINCT client_ip) AS malicious_notdetected\nFROM dns_loglines\nWHERE client_ip IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)\nAND client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)", + "refId": "B" + } + ], + "title": "malicious/not detected", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 13, + "y": 3 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT count(DISTINCT client_ip) AS benign_notdetected\nFROM dns_loglines\nWHERE client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)\nAND client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)", + "refId": "B" + } + ], + "title": "benign/not detected", + "type": "stat" + }, + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 26, + "title": "Details", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 23, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT 'benign' AS type, count(DISTINCT client_ip) AS number\nFROM dns_loglines\nWHERE client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)\n\nUNION ALL\n\nSELECT 'malicious' AS type, count(DISTINCT client_ip) AS number\nFROM dns_loglines\nWHERE client_ip IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)\n\nUNION ALL\n\nSELECT 'undetected' AS type, count(DISTINCT client_ip) AS number\nFROM dns_loglines\nWHERE client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)\n\nUNION ALL\n\nSELECT 'detected' AS type, count(DISTINCT client_ip) AS number\nFROM dns_loglines\nWHERE client_ip IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)", + "refId": "A" + } + ], + "title": "Total number of logs", + "transformations": [ + { + "id": "rowsToFields", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "benign": 2, + "detected": 1, + "malicious": 3, + "undetected": 0 + }, + "renameByName": {} + } + } + ], + "type": "bargauge" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 24, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT client_ip, JSONExtractString(additional_fields, 'domain_name') AS domain_name\nFROM dns_loglines\nWHERE client_ip IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)\nAND client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)\nLIMIT 100", + "refId": "A" + } + ], + "title": "False Positive IPs and domains", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "client_ip": "Client IP address", + "domain_name": "Domain used" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 25, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT client_ip, JSONExtractString(additional_fields, 'domain_name') AS domain_name\nFROM dns_loglines\nWHERE client_ip NOT IN (\n SELECT DISTINCT client_ip\n FROM alerts\n)\nAND client_ip IN ( -- change to notin\n SELECT DISTINCT client_ip\n FROM dgta_dataset\n INNER JOIN dns_loglines ON dgta_dataset.query = JSONExtractString(dns_loglines.additional_fields, 'domain_name')\n WHERE class = 1\n)\nLIMIT 100", + "refId": "A" + } + ], + "title": "False Negative IPs and domains", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "client_ip": "Client IP address", + "domain_name": "Domain used" + } + } + } + ], + "type": "table" + } + ], + "refresh": "auto", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": true + }, + "timezone": "browser", + "title": "Data Tests", + "uid": "cea2xqic5klq8b", + "version": 6, + "weekStart": "" +} diff --git a/docker/dev-get-csv-data.py b/docker/dev-get-csv-data.py new file mode 100644 index 00000000..ca1bbe32 --- /dev/null +++ b/docker/dev-get-csv-data.py @@ -0,0 +1,39 @@ +import gzip +import os +import sys + +sys.path.append(os.getcwd()) +from src.train.dataset import DatasetLoader +from src.base.log_config import get_logger + +logger = get_logger() + + +def create_dgta_dataset_json_gz(base_path="../data"): + logger.info("Loading data for DGTA dataset...") + try: + loader = DatasetLoader(base_path) + dataset = loader.dgta_dataset + + logger.info("Converting to JSON data...") + json_data = dataset.data.write_json() + logger.warning(json_data) + + logger.info("Compressing data...") + with gzip.open("../data/dgta_dataset.json.gz", "wb") as f: + logger.info("Writing compressed data to file...") + f.write(json_data.encode()) + except FileNotFoundError: + logger.warning( + "Dataset was not found in 'data' directory. Skipping this dataset" + ) + return + except Exception as err: + logger.error(err) + return + + logger.info("DGTA dataset: Done") + + +if __name__ == "__main__": + create_dgta_dataset_json_gz() diff --git a/docker/docker-compose.datatests.yml b/docker/docker-compose.datatests.yml new file mode 100644 index 00000000..5083f5ee --- /dev/null +++ b/docker/docker-compose.datatests.yml @@ -0,0 +1,122 @@ +services: + zookeeper: + extends: + file: "docker-compose/base/docker-compose.kafka.yml" + service: zookeeper + + kafka1: + extends: + file: "docker-compose/base/docker-compose.kafka.yml" + service: kafka1 + depends_on: + zookeeper: + condition: service_healthy + + kafka2: + extends: + file: "docker-compose/base/docker-compose.kafka.yml" + service: kafka2 + depends_on: + zookeeper: + condition: service_healthy + + kafka3: + extends: + file: "docker-compose/base/docker-compose.kafka.yml" + service: kafka3 + depends_on: + zookeeper: + condition: service_healthy + + logserver: + extends: + file: "docker-compose/base/docker-compose.pipeline.yml" + service: logserver + depends_on: + kafka1: + condition: service_healthy + kafka2: + condition: service_healthy + kafka3: + condition: service_healthy + + logcollector: + extends: + file: "docker-compose/base/docker-compose.pipeline.yml" + service: logcollector + depends_on: + kafka1: + condition: service_healthy + kafka2: + condition: service_healthy + kafka3: + condition: service_healthy + + prefilter: + extends: + file: "docker-compose/base/docker-compose.pipeline.yml" + service: prefilter + depends_on: + kafka1: + condition: service_healthy + kafka2: + condition: service_healthy + kafka3: + condition: service_healthy + + inspector: + extends: + file: "docker-compose/base/docker-compose.pipeline.yml" + service: inspector + depends_on: + kafka1: + condition: service_healthy + kafka2: + condition: service_healthy + kafka3: + condition: service_healthy + + detector: + extends: + file: "docker-compose/base/docker-compose.pipeline.yml" + service: detector + depends_on: + kafka1: + condition: service_healthy + kafka2: + condition: service_healthy + kafka3: + condition: service_healthy + + clickhouse-server: + extends: + file: "docker-compose/datatests/docker-compose.clickhouse.yml" + service: clickhouse-server + + grafana: + extends: + file: "docker-compose/datatests/docker-compose.grafana.yml" + service: grafana + + monitoring_agent: + extends: + file: "docker-compose/base/docker-compose.monitoring.yml" + service: monitoring_agent + depends_on: + kafka1: + condition: service_healthy + kafka2: + condition: service_healthy + kafka3: + condition: service_healthy + clickhouse-server: + condition: service_healthy + +networks: + heidgaf: + driver: bridge + ipam: + driver: default + config: + - subnet: 172.27.0.0/16 + gateway: 172.27.0.1 diff --git a/docker/docker-compose.dev-query.yml b/docker/docker-compose.dev-query.yml deleted file mode 100644 index ade6ae47..00000000 --- a/docker/docker-compose.dev-query.yml +++ /dev/null @@ -1,56 +0,0 @@ -include: - - "docker-compose.kafka.yml" - -services: - dev-query: - build: - context: .. - dockerfile: docker/dockerfiles/Dockerfile.dev-query - network: host - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy - networks: - heidgaf: - ipv4_address: 172.27.0.100 - memswap_limit: 768m - deploy: - resources: - limits: - cpus: '2' - memory: 512m - reservations: - cpus: '1' - memory: 256m - volumes: - - "${MOUNT_PATH:?MOUNT_PATH not set}:/opt/file.txt" - - clickhouse-server: - image: clickhouse/clickhouse-server:24.3.12.75-alpine - container_name: clickhouse-server - networks: - heidgaf: - ipv4_address: 172.27.0.11 - restart: "unless-stopped" - ports: - - "8123:8123" - - "9000:9000" - healthcheck: - test: [ "CMD-SHELL", "nc -z 127.0.0.1 8123" ] - interval: 10s - timeout: 5s - retries: 3 - - -networks: - heidgaf: - driver: bridge - ipam: - driver: default - config: - - subnet: 172.27.0.0/16 - gateway: 172.27.0.1 diff --git a/docker/docker-compose.monitoring.yml b/docker/docker-compose.monitoring.yml deleted file mode 100644 index 3ec9d85f..00000000 --- a/docker/docker-compose.monitoring.yml +++ /dev/null @@ -1,47 +0,0 @@ -include: - - "docker-compose.kafka.yml" - -services: - clickhouse-server: - image: clickhouse/clickhouse-server:24.3.12.75-alpine - container_name: clickhouse-server - networks: - heidgaf: - ipv4_address: 172.27.0.11 - restart: "unless-stopped" - ports: - - "8123:8123" - - "9000:9000" - healthcheck: - test: [ "CMD-SHELL", "nc -z 127.0.0.1 8123" ] - interval: 10s - timeout: 5s - retries: 3 - - monitoring_agent: - build: - context: .. - dockerfile: docker/dockerfiles/Dockerfile.monitoring - network: host - restart: "unless-stopped" - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy - clickhouse-server: - condition: service_healthy - networks: - heidgaf: - ipv4_address: 172.27.0.12 - -networks: - heidgaf: - driver: bridge - ipam: - driver: default - config: - - subnet: 172.27.0.0/16 - gateway: 172.27.0.1 diff --git a/docker/docker-compose.query.yml b/docker/docker-compose.query.yml new file mode 100644 index 00000000..56febc75 --- /dev/null +++ b/docker/docker-compose.query.yml @@ -0,0 +1,21 @@ +services: + query: + build: + context: .. + dockerfile: docker/dockerfiles/Dockerfile.dev-query + network: host + networks: + docker_heidgaf: + memswap_limit: 768m + deploy: + resources: + limits: + cpus: '2' + memory: 512m + reservations: + cpus: '1' + memory: 256m + +networks: + docker_heidgaf: + external: true diff --git a/docker/docker-compose.send-mock-logs.yml b/docker/docker-compose.send-mock-logs.yml new file mode 100644 index 00000000..13d3cdd8 --- /dev/null +++ b/docker/docker-compose.send-mock-logs.yml @@ -0,0 +1,21 @@ +services: + send-mock-logs: + build: + context: .. + dockerfile: docker/dockerfiles/Dockerfile.dev-mock-logs + network: host + networks: + docker_heidgaf: + memswap_limit: 768m + deploy: + resources: + limits: + cpus: '2' + memory: 512m + reservations: + cpus: '1' + memory: 256m + +networks: + docker_heidgaf: + external: true diff --git a/docker/docker-compose.send-real-logs.yml b/docker/docker-compose.send-real-logs.yml new file mode 100644 index 00000000..ca80e64b --- /dev/null +++ b/docker/docker-compose.send-real-logs.yml @@ -0,0 +1,21 @@ +services: + send-real-logs: + build: + context: .. + dockerfile: docker/dockerfiles/Dockerfile.dev-real-logs + network: host + networks: + docker_heidgaf: + memswap_limit: 768m + deploy: + resources: + limits: + cpus: '2' + memory: 512m + reservations: + cpus: '1' + memory: 256m + +networks: + docker_heidgaf: + external: true diff --git a/docker/docker-compose.swarm.yml b/docker/docker-compose.swarm.yml new file mode 100644 index 00000000..6521da94 --- /dev/null +++ b/docker/docker-compose.swarm.yml @@ -0,0 +1,271 @@ +services: + zookeeper: + image: confluentinc/cp-zookeeper:7.3.2 + networks: + - heidgaf + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_SERVER_ID: 1 + healthcheck: + test: [ "CMD-SHELL", "nc -z localhost 2181" ] + interval: 10s + timeout: 5s + retries: 3 + deploy: + placement: + constraints: [ node.hostname == heidgaf-1 ] + restart_policy: + condition: on-failure + + kafka1: + image: confluentinc/cp-kafka:7.3.2 + networks: + - heidgaf + ports: + - "8097:8097" + - "29092:29092" + depends_on: + - zookeeper + healthcheck: + test: [ "CMD-SHELL", "nc -z localhost 8097" ] + interval: 30s + timeout: 10s + retries: 10 + environment: + KAFKA_BROKER_ID: 1 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://tasks.kafka1:8097,DOCKER://host.docker.internal:29092 + KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + deploy: + placement: + constraints: [ node.hostname == heidgaf-1 ] + restart_policy: + condition: on-failure + + + kafka2: + image: confluentinc/cp-kafka:7.3.2 + networks: + - heidgaf + ports: + - "8098:8098" + - "29093:29093" + depends_on: + - zookeeper + healthcheck: + test: [ "CMD-SHELL", "nc -z localhost 8098" ] + interval: 30s + timeout: 10s + retries: 10 + environment: + KAFKA_BROKER_ID: 2 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://tasks.kafka2:8098,DOCKER://host.docker.internal:29093 + KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + deploy: + placement: + constraints: [ node.hostname == heidgaf-1 ] + restart_policy: + condition: on-failure + + kafka3: + image: confluentinc/cp-kafka:7.3.2 + networks: + - heidgaf + ports: + - "8099:8099" + - "29094:29094" + depends_on: + - zookeeper + healthcheck: + test: [ "CMD-SHELL", "nc -z localhost 8099" ] + interval: 30s + timeout: 10s + retries: 10 + environment: + KAFKA_BROKER_ID: 3 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://tasks.kafka3:8099,DOCKER://host.docker.internal:29094 + KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + deploy: + placement: + constraints: [ node.hostname == heidgaf-1 ] + restart_policy: + condition: on-failure + + logserver: + image: stefan96/heidgaf-logserver + networks: + - heidgaf + deploy: +# resources: +# limits: +# cpus: '2' +# memory: 512m +# reservations: +# cpus: '1' +# memory: 256m + placement: + constraints: [ node.hostname == heidgaf-2 ] + volumes: + - ./default.txt:/opt/file.txt + environment: + - GROUP_ID=log_storage + + logcollector: + image: stefan96/heidgaf-logcollector + networks: + - heidgaf + deploy: +# resources: +# limits: +# cpus: '2' +# memory: 512m +# reservations: +# cpus: '1' +# memory: 256m + placement: + constraints: [ node.hostname == heidgaf-2 ] + environment: + - GROUP_ID=log_collection + + prefilter: + image: stefan96/heidgaf-prefilter + networks: + - heidgaf + deploy: + mode: "replicated" + replicas: 1 +# resources: +# limits: +# cpus: '2' +# memory: 512m +# reservations: +# cpus: '1' +# memory: 256m + placement: + constraints: [ node.hostname == heidgaf-2 ] + environment: + - GROUP_ID=log_filtering + + inspector: + image: stefan96/heidgaf-inspector + networks: + - heidgaf + deploy: + mode: "replicated" + replicas: 2 +# resources: +# limits: +# cpus: '2' +# memory: 512m +# reservations: +# cpus: '1' +# memory: 256m + placement: + constraints: [ node.hostname == heidgaf-2 ] + environment: + - GROUP_ID=data_inspection + - NUMBER_OF_INSTANCES=2 + + detector: + image: stefan96/heidgaf-detector + networks: + - heidgaf + deploy: + mode: "replicated" + replicas: 1 +# resources: +# limits: +# cpus: '2' +# memory: 512m +# reservations: +# cpus: '1' +# memory: 256m +# generic_resources: +# - discrete_resource_spec: +# kind: 'gpu' +# value: 1 + placement: + constraints: [ node.hostname == heidgaf-2 ] + environment: + - GROUP_ID=data_analysis + + clickhouse-server: + image: clickhouse/clickhouse-server:24.3.12.75-alpine + volumes: + - ./create_tables:/docker-entrypoint-initdb.d + networks: + - heidgaf + ports: + - "8123:8123" + - "9000:9000" + healthcheck: + test: [ "CMD-SHELL", "nc -z localhost 8123" ] + interval: 10s + timeout: 5s + retries: 3 + deploy: + placement: + constraints: [ node.hostname == heidgaf-3 ] + restart_policy: + condition: on-failure + + grafana: + image: grafana/grafana:11.2.2-security-01 + networks: + - heidgaf + ports: + - "3000:3000" + volumes: + - ./grafana-provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ./grafana-provisioning/dashboards/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml + - ./grafana-provisioning/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource + healthcheck: + test: [ "CMD-SHELL", "nc -z localhost 3000" ] + interval: 10s + timeout: 5s + retries: 3 + deploy: + placement: + constraints: [ node.hostname == heidgaf-3 ] + restart_policy: + condition: on-failure + + monitoring_agent: + image: monitoring_agent:latest + networks: + - heidgaf + environment: + - GROUP_ID=monitoring_agent + depends_on: + - kafka1 + - kafka2 + - kafka3 + - clickhouse-server + deploy: + placement: + constraints: [ node.hostname == heidgaf-3 ] + restart_policy: + condition: on-failure + +networks: + heidgaf: + external: true diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 436d3a4a..5198da86 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,14 +1,37 @@ -include: - - "docker-compose.kafka.yml" - - "docker-compose.monitoring.yml" - services: - logcollector: - build: - context: .. - dockerfile: docker/dockerfiles/Dockerfile.logcollector - network: host - restart: "unless-stopped" + zookeeper: + extends: + file: "docker-compose/base/docker-compose.kafka.yml" + service: zookeeper + + kafka1: + extends: + file: "docker-compose/base/docker-compose.kafka.yml" + service: kafka1 + depends_on: + zookeeper: + condition: service_healthy + + kafka2: + extends: + file: "docker-compose/base/docker-compose.kafka.yml" + service: kafka2 + depends_on: + zookeeper: + condition: service_healthy + + kafka3: + extends: + file: "docker-compose/base/docker-compose.kafka.yml" + service: kafka3 + depends_on: + zookeeper: + condition: service_healthy + + logserver: + extends: + file: "docker-compose/base/docker-compose.pipeline.yml" + service: logserver depends_on: kafka1: condition: service_healthy @@ -16,25 +39,11 @@ services: condition: service_healthy kafka3: condition: service_healthy - networks: - heidgaf: - ipv4_address: 172.27.0.7 - memswap_limit: 768m - deploy: - resources: - limits: - cpus: '2' - memory: 512m - reservations: - cpus: '1' - memory: 256m - logserver: - build: - context: .. - dockerfile: docker/dockerfiles/Dockerfile.logserver - network: host - restart: "unless-stopped" + logcollector: + extends: + file: "docker-compose/base/docker-compose.pipeline.yml" + service: logcollector depends_on: kafka1: condition: service_healthy @@ -42,28 +51,23 @@ services: condition: service_healthy kafka3: condition: service_healthy - networks: - heidgaf: - ipv4_address: 172.27.0.8 - memswap_limit: 768m - deploy: - resources: - limits: - cpus: '2' - memory: 512m - reservations: - cpus: '1' - memory: 256m - volumes: - - "${MOUNT_PATH:?MOUNT_PATH not set}:/opt/file.txt" + prefilter: + extends: + file: "docker-compose/base/docker-compose.pipeline.yml" + service: prefilter + depends_on: + kafka1: + condition: service_healthy + kafka2: + condition: service_healthy + kafka3: + condition: service_healthy inspector: - build: - context: .. - dockerfile: docker/dockerfiles/Dockerfile.inspector - network: host - restart: "unless-stopped" + extends: + file: "docker-compose/base/docker-compose.pipeline.yml" + service: inspector depends_on: kafka1: condition: service_healthy @@ -71,26 +75,11 @@ services: condition: service_healthy kafka3: condition: service_healthy - networks: - heidgaf: - ipv4_address: 172.27.0.6 - deploy: - mode: "replicated" - replicas: 1 - resources: - limits: - cpus: '2' - memory: 512m - reservations: - cpus: '1' - memory: 256m - prefilter: - build: - context: .. - dockerfile: docker/dockerfiles/Dockerfile.prefilter - network: host - restart: "unless-stopped" + detector: + extends: + file: "docker-compose/base/docker-compose.pipeline.yml" + service: detector depends_on: kafka1: condition: service_healthy @@ -98,26 +87,21 @@ services: condition: service_healthy kafka3: condition: service_healthy - networks: - heidgaf: - ipv4_address: 172.27.0.9 - deploy: - mode: "replicated" - replicas: 1 - resources: - limits: - cpus: '2' - memory: 512m - reservations: - cpus: '1' - memory: 256m - detector: - build: - context: .. - dockerfile: docker/dockerfiles/Dockerfile.detector - network: host - restart: "unless-stopped" + clickhouse-server: + extends: + file: "docker-compose/prod/docker-compose.clickhouse.yml" + service: clickhouse-server + + grafana: + extends: + file: "docker-compose/prod/docker-compose.grafana.yml" + service: grafana + + monitoring_agent: + extends: + file: "docker-compose/base/docker-compose.monitoring.yml" + service: monitoring_agent depends_on: kafka1: condition: service_healthy @@ -125,23 +109,8 @@ services: condition: service_healthy kafka3: condition: service_healthy - networks: - heidgaf: - ipv4_address: 172.27.0.10 - deploy: - mode: "replicated" - replicas: 1 - resources: - limits: - cpus: '2' - memory: 512m - reservations: - cpus: '1' - memory: 256m - devices: - - driver: nvidia - count: 1 # alternatively, use `count: all` for all GPUs - capabilities: [ gpu ] + clickhouse-server: + condition: service_healthy networks: heidgaf: diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose/base/docker-compose.kafka.yml similarity index 85% rename from docker/docker-compose.kafka.yml rename to docker/docker-compose/base/docker-compose.kafka.yml index 8ec9c8a9..1541f130 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose/base/docker-compose.kafka.yml @@ -4,8 +4,7 @@ services: container_name: zookeeper restart: "unless-stopped" networks: - heidgaf: - ipv4_address: 172.27.0.2 + - heidgaf environment: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_SERVER_ID: 1 @@ -20,8 +19,7 @@ services: container_name: kafka1 restart: "unless-stopped" networks: - heidgaf: - ipv4_address: 172.27.0.3 + - heidgaf ports: - "8097:8097" - "29092:29092" @@ -35,10 +33,10 @@ services: retries: 10 environment: KAFKA_BROKER_ID: 1 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT,DOCKER:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://172.27.0.3:8097,DOCKER://host.docker.internal:29092 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://kafka1:8097,DOCKER://host.docker.internal:29092 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer @@ -50,8 +48,7 @@ services: container_name: kafka2 restart: "unless-stopped" networks: - heidgaf: - ipv4_address: 172.27.0.4 + - heidgaf ports: - "8098:8098" - "29093:29093" @@ -67,7 +64,7 @@ services: KAFKA_BROKER_ID: 2 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT,DOCKER:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19092,EXTERNAL://172.27.0.4:8098,DOCKER://host.docker.internal:29093 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19092,EXTERNAL://kafka2:8098,DOCKER://host.docker.internal:29093 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer @@ -78,8 +75,7 @@ services: container_name: kafka3 restart: "unless-stopped" networks: - heidgaf: - ipv4_address: 172.27.0.5 + - heidgaf ports: - "8099:8099" - "29094:29094" @@ -95,7 +91,7 @@ services: KAFKA_BROKER_ID: 3 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT,DOCKER:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19092,EXTERNAL://172.27.0.5:8099,DOCKER://host.docker.internal:29094 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19092,EXTERNAL://kafka3:8099,DOCKER://host.docker.internal:29094 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer @@ -103,9 +99,4 @@ services: networks: heidgaf: - driver: bridge - ipam: - driver: default - config: - - subnet: 172.27.0.0/16 - gateway: 172.27.0.1 + external: true diff --git a/docker/docker-compose/base/docker-compose.monitoring.yml b/docker/docker-compose/base/docker-compose.monitoring.yml new file mode 100644 index 00000000..a70c3563 --- /dev/null +++ b/docker/docker-compose/base/docker-compose.monitoring.yml @@ -0,0 +1,22 @@ +services: + monitoring_agent: + build: + context: ../../.. + dockerfile: docker/dockerfiles/Dockerfile.monitoring + network: host + restart: "unless-stopped" + depends_on: + kafka1: + condition: service_healthy + kafka2: + condition: service_healthy + kafka3: + condition: service_healthy + clickhouse-server: + condition: service_healthy + grafana: + condition: service_healthy + networks: + - heidgaf + environment: + - GROUP_ID=monitoring_agent diff --git a/docker/docker-compose/base/docker-compose.pipeline.yml b/docker/docker-compose/base/docker-compose.pipeline.yml new file mode 100644 index 00000000..5ab77385 --- /dev/null +++ b/docker/docker-compose/base/docker-compose.pipeline.yml @@ -0,0 +1,110 @@ +services: + logserver: + build: + context: ../../.. + dockerfile: docker/dockerfiles/Dockerfile.logserver + network: host + restart: "unless-stopped" + networks: + heidgaf: + memswap_limit: 768m + deploy: + resources: + limits: + cpus: '2' + memory: 512m + reservations: + cpus: '1' + memory: 256m + volumes: + - "${MOUNT_PATH:?MOUNT_PATH not set}:/opt/file.txt" + environment: + - GROUP_ID=log_storage + + logcollector: + build: + context: ../../.. + dockerfile: docker/dockerfiles/Dockerfile.logcollector + network: host + restart: "unless-stopped" + networks: + heidgaf: + memswap_limit: 768m + deploy: + resources: + limits: + cpus: '2' + memory: 512m + reservations: + cpus: '1' + memory: 256m + environment: + - GROUP_ID=log_collection + + prefilter: + build: + context: ../../.. + dockerfile: docker/dockerfiles/Dockerfile.prefilter + network: host + restart: "unless-stopped" + networks: + heidgaf: + deploy: + mode: "replicated" + replicas: 1 + resources: + limits: + cpus: '2' + memory: 512m + reservations: + cpus: '1' + memory: 256m + environment: + - GROUP_ID=log_filtering + + inspector: + build: + context: ../../.. + dockerfile: docker/dockerfiles/Dockerfile.inspector + network: host + restart: "unless-stopped" + networks: + heidgaf: + deploy: + mode: "replicated" + replicas: 2 + resources: + limits: + cpus: '2' + memory: 512m + reservations: + cpus: '1' + memory: 256m + environment: + - GROUP_ID=data_inspection + - NUMBER_OF_INSTANCES=2 + + detector: + build: + context: ../../.. + dockerfile: docker/dockerfiles/Dockerfile.detector + network: host + restart: "unless-stopped" + networks: + heidgaf: + deploy: + mode: "replicated" + replicas: 1 + resources: + limits: + cpus: '2' + memory: 512m +# reservations: +# cpus: '1' +# memory: 256m +# devices: +# - driver: nvidia +# count: 1 # alternatively, use `count: all` for all GPUs +# capabilities: [ gpu ] + environment: + - GROUP_ID=data_analysis diff --git a/docker/docker-compose/datatests/docker-compose.clickhouse.yml b/docker/docker-compose/datatests/docker-compose.clickhouse.yml new file mode 100644 index 00000000..76ec355f --- /dev/null +++ b/docker/docker-compose/datatests/docker-compose.clickhouse.yml @@ -0,0 +1,21 @@ +services: + clickhouse-server: + image: clickhouse/clickhouse-server:24.3.12.75-alpine + container_name: clickhouse-server + volumes: + - ../../create_tables:/create_tables + - ../../create_datatest_tables:/create_datatest_tables + - ../../insert_datatest_data:/insert_datatest_data + - ../../insert_datatest_data/data:/var/lib/clickhouse/user_files/data + - ../../init_datatests.sh:/docker-entrypoint-initdb.d/init.sh + networks: + - heidgaf + restart: "unless-stopped" + ports: + - "8123:8123" + - "9000:9000" + healthcheck: + test: [ "CMD-SHELL", "nc -z 127.0.0.1 8123" ] + interval: 10s + timeout: 5s + retries: 3 diff --git a/docker/docker-compose/datatests/docker-compose.grafana.yml b/docker/docker-compose/datatests/docker-compose.grafana.yml new file mode 100644 index 00000000..a5539d48 --- /dev/null +++ b/docker/docker-compose/datatests/docker-compose.grafana.yml @@ -0,0 +1,23 @@ +services: + grafana: + image: grafana/grafana:11.2.2-security-01 + container_name: grafana + networks: + - heidgaf + restart: "unless-stopped" + ports: + - "3000:3000" + volumes: + - ../../grafana-provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ../../datatests.json:/etc/grafana/provisioning/dashboards/datatests.json + - ../../grafana-provisioning/dashboards/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml + - ../../grafana-provisioning/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource + healthcheck: + test: [ "CMD-SHELL", "nc -z localhost 3000" ] + interval: 10s + timeout: 5s + retries: 3 diff --git a/docker/docker-compose/prod/docker-compose.clickhouse.yml b/docker/docker-compose/prod/docker-compose.clickhouse.yml new file mode 100644 index 00000000..d6483290 --- /dev/null +++ b/docker/docker-compose/prod/docker-compose.clickhouse.yml @@ -0,0 +1,17 @@ +services: + clickhouse-server: + image: clickhouse/clickhouse-server:24.3.12.75-alpine + container_name: clickhouse-server + volumes: + - ../../create_tables:/docker-entrypoint-initdb.d + networks: + - heidgaf + restart: "unless-stopped" + ports: + - "8123:8123" + - "9000:9000" + healthcheck: + test: [ "CMD-SHELL", "nc -z 127.0.0.1 8123" ] + interval: 10s + timeout: 5s + retries: 3 diff --git a/docker/docker-compose/prod/docker-compose.grafana.yml b/docker/docker-compose/prod/docker-compose.grafana.yml new file mode 100644 index 00000000..d6e8839a --- /dev/null +++ b/docker/docker-compose/prod/docker-compose.grafana.yml @@ -0,0 +1,22 @@ +services: + grafana: + image: grafana/grafana:11.2.2-security-01 + container_name: grafana + networks: + - heidgaf + restart: "unless-stopped" + ports: + - "3000:3000" + volumes: + - ../../grafana-provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ../../grafana-provisioning/dashboards/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml + - ../../grafana-provisioning/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource + healthcheck: + test: [ "CMD-SHELL", "nc -z localhost 3000" ] + interval: 10s + timeout: 5s + retries: 3 diff --git a/docker/dockerfiles/Dockerfile.dev-mock-logs b/docker/dockerfiles/Dockerfile.dev-mock-logs new file mode 100644 index 00000000..689f5ee7 --- /dev/null +++ b/docker/dockerfiles/Dockerfile.dev-mock-logs @@ -0,0 +1,16 @@ +FROM python:3.11-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 + +WORKDIR /usr/src/app + +RUN pip --disable-pip-version-check install --no-cache-dir --no-compile marshmallow_dataclass colorlog pyYAML confluent_kafka + +COPY src/base ./src/base +COPY src/mock ./src/mock +COPY config.yaml . +COPY docker/mock_logs.dev.py . + +RUN rm -rf /root/.cache + +CMD [ "python", "mock_logs.dev.py"] diff --git a/docker/dockerfiles/Dockerfile.dev-query b/docker/dockerfiles/Dockerfile.dev-query index f730d1a2..a4792c4b 100644 --- a/docker/dockerfiles/Dockerfile.dev-query +++ b/docker/dockerfiles/Dockerfile.dev-query @@ -1,4 +1,4 @@ -FROM python:3.11-slim-bookworm +FROM python:3.11-bookworm ENV PYTHONDONTWRITEBYTECODE=1 diff --git a/docker/dockerfiles/Dockerfile.dev-real-logs b/docker/dockerfiles/Dockerfile.dev-real-logs new file mode 100644 index 00000000..298f20c6 --- /dev/null +++ b/docker/dockerfiles/Dockerfile.dev-real-logs @@ -0,0 +1,18 @@ +FROM python:3.11-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 + +WORKDIR /usr/src/app + +RUN pip --disable-pip-version-check install --no-cache-dir --no-compile marshmallow_dataclass colorlog pyYAML confluent_kafka numpy polars scikit-learn torch + +COPY src/base ./src/base +COPY src/mock ./src/mock +COPY src/train ./src/train +COPY config.yaml . +COPY docker/real_logs.dev.py . +COPY data ./data + +RUN rm -rf /root/.cache + +CMD [ "python", "real_logs.dev.py"] diff --git a/docker/grafana-provisioning/dashboards/alerts.json b/docker/grafana-provisioning/dashboards/alerts.json new file mode 100644 index 00000000..e02b5476 --- /dev/null +++ b/docker/grafana-provisioning/dashboards/alerts.json @@ -0,0 +1,767 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "# Alerts / ${Granularity}", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 10, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "barRadius": 0, + "barWidth": 1, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "vertical", + "showValue": "never", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT \n toStartOf${Granularity}(toDateTime64(alert_timestamp, 6)) AS time_bucket,\n count(client_ip) AS \"count\"\nFROM alerts\nWHERE alert_timestamp >= toStartOf${Granularity}(toDateTime64($__fromTime, 6)) AND alert_timestamp <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket\nWITH FILL\nFROM toStartOf${Granularity}(toDateTime64($__fromTime, 6))\nTO toStartOf${Granularity}(toDateTime64($__toTime, 6))\nSTEP toInterval${Granularity}(1);", + "refId": "B" + } + ], + "title": "Alerts per Time", + "type": "barchart" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT count(*)\nFROM (\n SELECT DISTINCT client_ip\n FROM alerts\n WHERE alert_timestamp >= $__fromTime AND alert_timestamp <= $__toTime\n);", + "refId": "A" + } + ], + "title": "Number of alerts in time range", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 19, + "y": 3 + }, + "id": 4, + "options": { + "folderUID": "", + "includeVars": true, + "keepTime": true, + "maxItems": 10, + "query": "", + "showFolderNames": false, + "showHeadings": false, + "showRecentlyViewed": false, + "showSearch": true, + "showStarred": false, + "tags": [] + }, + "pluginVersion": "11.2.2+security-01", + "title": "Dashboards", + "transparent": true, + "type": "dashlist" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 9, + "panels": [], + "title": "Heatmaps", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "fieldMinMax": false + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 11 + }, + "hideTimeOverride": false, + "id": 8, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "blue", + "min": 0, + "mode": "opacity", + "reverse": false, + "scale": "linear", + "scheme": "PuBu", + "steps": 62 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "ge" + }, + "tooltip": { + "mode": "single", + "showColorScale": true, + "yHistogram": true + }, + "yAxis": { + "axisLabel": "# Alerts / Day", + "axisPlacement": "left", + "reverse": true, + "unit": "" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT \n alert_month AS time,\n SUM(CASE WHEN day = 1 THEN alert_count ELSE 0 END) AS `1`,\n SUM(CASE WHEN day = 2 THEN alert_count ELSE 0 END) AS `2`,\n SUM(CASE WHEN day = 3 THEN alert_count ELSE 0 END) AS `3`,\n SUM(CASE WHEN day = 4 THEN alert_count ELSE 0 END) AS `4`,\n SUM(CASE WHEN day = 5 THEN alert_count ELSE 0 END) AS `5`,\n SUM(CASE WHEN day = 6 THEN alert_count ELSE 0 END) AS `6`,\n SUM(CASE WHEN day = 7 THEN alert_count ELSE 0 END) AS `7`,\n SUM(CASE WHEN day = 8 THEN alert_count ELSE 0 END) AS `8`,\n SUM(CASE WHEN day = 9 THEN alert_count ELSE 0 END) AS `9`,\n SUM(CASE WHEN day = 10 THEN alert_count ELSE 0 END) AS `10`,\n SUM(CASE WHEN day = 11 THEN alert_count ELSE 0 END) AS `11`,\n SUM(CASE WHEN day = 12 THEN alert_count ELSE 0 END) AS `12`,\n SUM(CASE WHEN day = 13 THEN alert_count ELSE 0 END) AS `13`,\n SUM(CASE WHEN day = 14 THEN alert_count ELSE 0 END) AS `14`,\n SUM(CASE WHEN day = 15 THEN alert_count ELSE 0 END) AS `15`,\n SUM(CASE WHEN day = 16 THEN alert_count ELSE 0 END) AS `16`,\n SUM(CASE WHEN day = 17 THEN alert_count ELSE 0 END) AS `17`,\n SUM(CASE WHEN day = 18 THEN alert_count ELSE 0 END) AS `18`,\n SUM(CASE WHEN day = 19 THEN alert_count ELSE 0 END) AS `19`,\n SUM(CASE WHEN day = 20 THEN alert_count ELSE 0 END) AS `20`,\n SUM(CASE WHEN day = 21 THEN alert_count ELSE 0 END) AS `21`,\n SUM(CASE WHEN day = 22 THEN alert_count ELSE 0 END) AS `22`,\n SUM(CASE WHEN day = 23 THEN alert_count ELSE 0 END) AS `23`,\n SUM(CASE WHEN day = 24 THEN alert_count ELSE 0 END) AS `24`,\n SUM(CASE WHEN day = 25 THEN alert_count ELSE 0 END) AS `25`,\n SUM(CASE WHEN day = 26 THEN alert_count ELSE 0 END) AS `26`,\n SUM(CASE WHEN day = 27 THEN alert_count ELSE 0 END) AS `27`,\n SUM(CASE WHEN day = 28 THEN alert_count ELSE 0 END) AS `28`,\n SUM(CASE WHEN day = 29 THEN alert_count ELSE 0 END) AS `29`,\n SUM(CASE WHEN day = 30 THEN alert_count ELSE 0 END) AS `30`,\n SUM(CASE WHEN day = 31 THEN alert_count ELSE 0 END) AS `31`\nFROM (\n SELECT \n toStartOfMonth(alert_timestamp) AS alert_month,\n toDayOfMonth(alert_timestamp) - 1 AS day,\n COUNT(*) AS alert_count\n FROM alerts\n WHERE alert_timestamp >= toStartOfMonth(now() - INTERVAL 1 MONTH)\n AND alert_timestamp < toStartOfMonth(now())\n GROUP BY alert_month, day\n) AS subquery\nGROUP BY alert_month\nORDER BY alert_month\nWITH FILL\nFROM toStartOfMonth(now() - INTERVAL 1 YEAR)\nTO toStartOfMonth(now())\nSTEP toIntervalMonth(1);\n", + "refId": "A" + } + ], + "timeFrom": "1y", + "title": "# Alerts per day", + "type": "heatmap" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "fieldMinMax": false + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 11 + }, + "hideTimeOverride": false, + "id": 7, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "blue", + "min": 0, + "mode": "opacity", + "reverse": false, + "scale": "linear", + "scheme": "PuBu", + "steps": 62 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "ge" + }, + "tooltip": { + "mode": "single", + "showColorScale": true, + "yHistogram": true + }, + "yAxis": { + "axisLabel": "# Alerts / Hour", + "axisPlacement": "left", + "reverse": true, + "unit": "" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT \n alert_day AS time,\n SUM(CASE WHEN hour = 0 THEN alert_count ELSE 0 END) AS `0`,\n SUM(CASE WHEN hour = 1 THEN alert_count ELSE 0 END) AS `1`,\n SUM(CASE WHEN hour = 2 THEN alert_count ELSE 0 END) AS `2`,\n SUM(CASE WHEN hour = 3 THEN alert_count ELSE 0 END) AS `3`,\n SUM(CASE WHEN hour = 4 THEN alert_count ELSE 0 END) AS `4`,\n SUM(CASE WHEN hour = 5 THEN alert_count ELSE 0 END) AS `5`,\n SUM(CASE WHEN hour = 6 THEN alert_count ELSE 0 END) AS `6`,\n SUM(CASE WHEN hour = 7 THEN alert_count ELSE 0 END) AS `7`,\n SUM(CASE WHEN hour = 8 THEN alert_count ELSE 0 END) AS `8`,\n SUM(CASE WHEN hour = 9 THEN alert_count ELSE 0 END) AS `9`,\n SUM(CASE WHEN hour = 10 THEN alert_count ELSE 0 END) AS `10`,\n SUM(CASE WHEN hour = 11 THEN alert_count ELSE 0 END) AS `11`,\n SUM(CASE WHEN hour = 12 THEN alert_count ELSE 0 END) AS `12`,\n SUM(CASE WHEN hour = 13 THEN alert_count ELSE 0 END) AS `13`,\n SUM(CASE WHEN hour = 14 THEN alert_count ELSE 0 END) AS `14`,\n SUM(CASE WHEN hour = 15 THEN alert_count ELSE 0 END) AS `15`,\n SUM(CASE WHEN hour = 16 THEN alert_count ELSE 0 END) AS `16`,\n SUM(CASE WHEN hour = 17 THEN alert_count ELSE 0 END) AS `17`,\n SUM(CASE WHEN hour = 18 THEN alert_count ELSE 0 END) AS `18`,\n SUM(CASE WHEN hour = 19 THEN alert_count ELSE 0 END) AS `19`,\n SUM(CASE WHEN hour = 20 THEN alert_count ELSE 0 END) AS `20`,\n SUM(CASE WHEN hour = 21 THEN alert_count ELSE 0 END) AS `21`,\n SUM(CASE WHEN hour = 22 THEN alert_count ELSE 0 END) AS `22`,\n SUM(CASE WHEN hour = 23 THEN alert_count ELSE 0 END) AS `23`\nFROM (\n SELECT \n toDateTime(toStartOfDay(alert_timestamp)) AS alert_day,\n toHour(alert_timestamp) AS hour,\n COUNT(*) AS alert_count\n FROM alerts\n WHERE alert_timestamp >= toStartOfMonth(now() - INTERVAL 1 MONTH)\n AND alert_timestamp < toStartOfMonth(now())\n GROUP BY alert_day, hour\n) AS subquery\nGROUP BY alert_day\nORDER BY alert_day\nWITH FILL\nFROM toDateTime(toStartOfMonth(now() - INTERVAL 1 MONTH))\nTO toDateTime(toStartOfMonth(now()))\nSTEP toIntervalDay(1);\n\n", + "refId": "A" + } + ], + "timeFrom": "1M", + "title": "# Alerts per hour", + "type": "heatmap" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "fieldMinMax": false + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 11 + }, + "hideTimeOverride": false, + "id": 6, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "blue", + "min": 0, + "mode": "opacity", + "reverse": false, + "scale": "linear", + "scheme": "PuBu", + "steps": 62 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "ge" + }, + "tooltip": { + "mode": "single", + "showColorScale": true, + "yHistogram": true + }, + "yAxis": { + "axisLabel": "# Alerts / Quarter Hour", + "axisPlacement": "left", + "reverse": true, + "unit": "m" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT \n alert_hour as time,\n SUM(CASE WHEN quarter = 0 THEN alert_count ELSE 0 END) AS `0`,\n SUM(CASE WHEN quarter = 1 THEN alert_count ELSE 0 END) AS `15`,\n SUM(CASE WHEN quarter = 2 THEN alert_count ELSE 0 END) AS `30`,\n SUM(CASE WHEN quarter = 3 THEN alert_count ELSE 0 END) AS `45`\nFROM (\n SELECT \n toDateTime(DATE_FORMAT(alert_timestamp, '%Y-%m-%d %H:00:00')) AS alert_hour,\n FLOOR(MINUTE(alert_timestamp) / 15) AS quarter,\n COUNT(*) AS alert_count\n FROM alerts\n WHERE alert_hour >= toDateTime(DATE_FORMAT(now() - INTERVAL 1 DAY, '%Y-%m-%d %H:00:00'))\n AND alert_timestamp <= now()\n GROUP BY alert_hour, quarter\n) AS subquery\nGROUP BY alert_hour\nORDER BY alert_hour\nWITH FILL\nFROM toDateTime(DATE_FORMAT(now() - INTERVAL 1 DAY, '%Y-%m-%d %H:00:00'))\nTO toDateTime(now())\nSTEP toIntervalHour(1);\n", + "refId": "A" + } + ], + "timeFrom": "24h", + "title": "# Alerts per quarter hour", + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 10, + "panels": [], + "title": "Latest alerts and most active IP addresses", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "filterable": false, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 1, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT DISTINCT client_ip AS \"Client IP address\", arrayStringConcat(JSONExtract(domain_names, 'Array(String)'), ', ') AS \"Domains used\"\nFROM alerts\nORDER BY alert_timestamp DESC\nLIMIT 20", + "refId": "A" + } + ], + "title": "Latest 20 alerts", + "transparent": true, + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "filterable": false, + "inspect": false + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 5, + "options": { + "cellHeight": "lg", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT concat(rowNumberInAllBlocks() + 1, '.') AS \"Rank\", client_ip AS \"Client IP address\", count(logline_id) AS \"# Total Requests\"\nFROM dns_loglines\nWHERE \"Client IP address\" IN (\n SELECT DISTINCT client_ip\n FROM alerts\n WHERE alert_timestamp >= $__fromTime AND alert_timestamp <= $__toTime\n)\nGROUP BY \"Client IP address\"\nORDER BY \"# Total Requests\" DESC\nLIMIT 5", + "refId": "A" + } + ], + "title": "Top 5 most active malicious addresses", + "transparent": true, + "type": "table" + } + ], + "refresh": "auto", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Minute", + "value": "Minute" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT CASE \n WHEN (dateDiff('minute', toDateTime($__fromTime), toDateTime($__toTime))) <= 5 THEN 'Second'\n WHEN (dateDiff('hour', toDateTime($__fromTime), toDateTime($__toTime))) <= 6 THEN 'Minute'\n WHEN (dateDiff('hour', toDateTime($__fromTime), toDateTime($__toTime))) <= 72 THEN 'Hour'\n ELSE 'Day'\nEND AS interval;", + "hide": 2, + "includeAll": false, + "label": "", + "multi": false, + "name": "Granularity", + "options": [], + "query": "SELECT CASE \n WHEN (dateDiff('minute', toDateTime($__fromTime), toDateTime($__toTime))) <= 5 THEN 'Second'\n WHEN (dateDiff('hour', toDateTime($__fromTime), toDateTime($__toTime))) <= 6 THEN 'Minute'\n WHEN (dateDiff('hour', toDateTime($__fromTime), toDateTime($__toTime))) <= 72 THEN 'Hour'\n ELSE 'Day'\nEND AS interval;", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Alerts", + "uid": "de9z3fduc2m0we", + "version": 1, + "weekStart": "" +} diff --git a/docker/grafana-provisioning/dashboards/dashboards.yaml b/docker/grafana-provisioning/dashboards/dashboards.yaml new file mode 100644 index 00000000..e2dd9baf --- /dev/null +++ b/docker/grafana-provisioning/dashboards/dashboards.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: "default" + orgId: 1 + folder: "" + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/docker/grafana-provisioning/dashboards/latencies.json b/docker/grafana-provisioning/dashboards/latencies.json new file mode 100644 index 00000000..8b455b46 --- /dev/null +++ b/docker/grafana-provisioning/dashboards/latencies.json @@ -0,0 +1,3674 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "Median latencies per module shown throughout the \"lifetime\" of a log line, ignoring wait and transport times.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "hidden", + "fillOpacity": 62, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 19, + "x": 0, + "y": 0 + }, + "id": 41, + "options": { + "barRadius": 0.2, + "barWidth": 0.54, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "never", + "stacking": "percent", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "value", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT\n 'Median Latency' AS value,\n (\n SELECT median(value) FROM (\n SELECT dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS value\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out' AND\n sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\n )\n ) AS \"LogServer\",\n (\n SELECT median(value) FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, lt2.timestamp) as value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n ) AS \"LogCollector\",\n (\n SELECT median(value) FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, lt2.timestamp) AS value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n lt2.stage = 'log_collection.batch_handler' AND lt2.status = 'batched' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n ) AS \"BatchHandler\",\n (\n SELECT median(value) FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) as value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n ) AS \"Prefilter\",\n (\n SELECT median(value) FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n\n UNION ALL\n\n SELECT dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS value\n FROM batch_timestamps bt\n INNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\n INNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\n WHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\n )\n ) AS \"Inspector\",\n (\n SELECT median(value) FROM (\n SELECT dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS value\n FROM suspicious_batch_timestamps sbt1\n INNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\n WHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\n )\n ) AS \"Detector\"", + "refId": "A" + } + ], + "title": "Relative module latency", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 77, + "options": { + "folderUID": "", + "includeVars": true, + "keepTime": true, + "maxItems": 10, + "query": "", + "showFolderNames": true, + "showHeadings": false, + "showRecentlyViewed": false, + "showSearch": true, + "showStarred": false, + "tags": [] + }, + "pluginVersion": "11.2.2+security-01", + "title": "Dashboards", + "transparent": true, + "type": "dashlist" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "Comparison of the median latencies per module.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "continuous-GrYlRd" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 14, + "x": 0, + "y": 3 + }, + "id": 79, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.7.0", + "queryType": "table", + "rawSql": "SELECT *\nFROM (\n SELECT 'LogServer' AS name, median(value) as median\n FROM (\n SELECT dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS value\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out' AND\n sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Collector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, lt2.timestamp) as value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'BatchHandler' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, lt2.timestamp) AS value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n lt2.stage = 'log_collection.batch_handler' AND lt2.status = 'batched' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) as value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Inspector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n\n UNION ALL\n\n SELECT dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS value\n FROM batch_timestamps bt\n INNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\n INNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\n WHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Detector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS value\n FROM suspicious_batch_timestamps sbt1\n INNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\n WHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\n )\n)\nWHERE name IN (${include_modules:csv});\n", + "refId": "A" + } + ], + "title": "Module latency comparison", + "transformations": [ + { + "id": "rowsToFields", + "options": { + "mappings": [] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "BatchHandler": 2, + "Collector": 1, + "Detector": 5, + "Inspector": 4, + "LogServer": 0, + "Prefilter": 3 + }, + "renameByName": {} + } + } + ], + "type": "bargauge" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "displayName": "${__field.column}", + "fieldMinMax": false, + "mappings": [], + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "" + }, + "properties": [] + } + ] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 14, + "y": 3 + }, + "id": 43, + "options": { + "displayLabels": [ + "percent", + "name" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": false + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT 'LogServer' AS name, median(value) AS \" \"\nFROM (\n SELECT slt.event_timestamp AS time, dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS value\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out' AND\n sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\n)\n\nUNION ALL\n\nSELECT 'LogCollector' AS name, median(value) AS \" \"\nFROM (\n SELECT lt2.timestamp as time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) as value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n)\n\nUNION ALL\n\nSELECT 'BatchHandler' AS name, median(value) AS \" \"\nFROM (\n SELECT lt2.timestamp AS time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) AS value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n lt2.stage = 'log_collection.batch_handler' AND lt2.status = 'batched' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n)\n\nUNION ALL\n\nSELECT 'Prefilter' AS name, median(value) AS \" \"\nFROM (\n SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n)", + "refId": "Data Preparation" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT 'Inspector' AS name, median(value) AS \" \"\nFROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n\n UNION ALL\n\n SELECT dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS value\n FROM batch_timestamps bt\n INNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\n INNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\n WHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\n )\n\nUNION ALL\n\nSELECT 'Detector' AS name, median(value) AS \" \"\nFROM (\n SELECT dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS value\n FROM suspicious_batch_timestamps sbt1\n INNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\n WHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\n)", + "refId": "Analysis" + } + ], + "type": "piechart" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "Median time that a log line spends in a phase.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "continuous-GrYlRd" + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 11, + "x": 0, + "y": 10 + }, + "id": 78, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "top", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "valueMode": "text" + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.7.0", + "queryType": "table", + "rawSql": "SELECT *\nFROM (\n SELECT 'Data analysis phase' AS name, sum(Median)\n FROM (\n SELECT 'Inspector' AS name, median(value) AS \"Median\"\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n\n UNION ALL\n\n SELECT dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS value\n FROM batch_timestamps bt\n INNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\n INNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\n WHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Detector' AS name, median(value) AS \"Median\"\n FROM (\n SELECT dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS value\n FROM suspicious_batch_timestamps sbt1\n INNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\n WHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\n )\n )\n\n UNION ALL\n\n SELECT 'Data preparation phase' AS name, sum(Median) as median\n FROM (\n SELECT 'LogServer' AS name, median(value) AS \"Median\"\n FROM (\n SELECT slt.event_timestamp AS time, dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS value\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out' AND\n sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'LogCollector' AS name, median(value) AS \"Median\"\n FROM (\n SELECT lt2.timestamp as time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) as value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'BatchHandler' AS name, median(value) AS \"Median\"\n FROM (\n SELECT lt2.timestamp AS time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) AS value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n lt2.stage = 'log_collection.batch_handler' AND lt2.status = 'batched' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, median(value) AS \"Median\"\n FROM (\n SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n )\n);\n", + "refId": "phases" + } + ], + "title": "Duration per phase", + "transformations": [ + { + "id": "rowsToFields", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "Data analysis phase": 1, + "Data preparation phase": 0 + }, + "renameByName": {} + } + } + ], + "type": "bargauge" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "Latency of all modules combined. Approximated through the medians of all single latencies per module.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 11, + "y": 10 + }, + "id": 80, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "top", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "text" + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT 'Including transport and wait time' AS name, sum(median)\nFROM (\n SELECT 'LogServer' AS name, median(value) as median\n FROM (\n SELECT dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS value\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out' AND\n sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'LogCollection' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, lt2.timestamp) as value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'BatchHandler' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, bt2.timestamp) AS value\n FROM logline_timestamps lt1\n INNER JOIN logline_to_batches ltb ON ltb.logline_id = lt1.logline_id\n INNER JOIN batch_timestamps bt2 ON bt2.batch_id = ltb.batch_id\n WHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n bt2.stage = 'log_collection.batch_handler' AND bt2.status = 'completed' AND\n lt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Between BatchHandler and Prefilter' AS name, median(value) AS median\n FROM (\n SELECT dateDiff('microseconds', bt1.timestamp, bt2.timestamp) as value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2\n ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'log_collection.batch_handler' AND bt1.status = 'completed'\n AND bt2.stage = 'log_filtering.prefilter' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) as value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Between Prefilter and Inspector' AS name, median(value) AS median\n FROM (\n SELECT dateDiff('microseconds', bt1.timestamp, bt2.timestamp) as value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2\n ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'log_filtering.prefilter' AND bt1.status = 'finished'\n AND bt2.stage = 'data_inspection.inspector' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Inspector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n\n UNION ALL\n\n SELECT dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS value\n FROM batch_timestamps bt\n INNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\n INNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\n WHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\n )\n\n UNION ALL\n \n SELECT 'Between Inspector and Detector' AS name, median(value) AS median\n FROM (\n SELECT dateDiff('microseconds', bt1.timestamp, bt2.timestamp) as value\n FROM suspicious_batch_timestamps bt1\n INNER JOIN suspicious_batch_timestamps bt2\n ON bt1.suspicious_batch_id = bt2.suspicious_batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'finished'\n AND bt2.stage = 'data_analysis.detector' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n \n UNION ALL\n\n SELECT 'Detector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS value\n FROM suspicious_batch_timestamps sbt1\n INNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\n WHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\n )\n)\n\nUNION ALL\n\nSELECT 'Excluding transport and wait time' AS name, sum(median)\nFROM (\n SELECT 'LogServer' AS name, median(value) as median\n FROM (\n SELECT dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS value\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out' AND\n sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'LogCollection' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, lt2.timestamp) as value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'BatchHandler' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, lt2.timestamp) AS value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt2.logline_id = lt1.logline_id\n WHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n lt2.stage = 'log_collection.batch_handler' AND lt2.status = 'batched' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) as value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Inspector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n\n UNION ALL\n\n SELECT dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS value\n FROM batch_timestamps bt\n INNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\n INNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\n WHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Detector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS value\n FROM suspicious_batch_timestamps sbt1\n INNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\n WHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\n )\n);\n", + "refId": "A" + } + ], + "title": "Total latency", + "transformations": [ + { + "id": "rowsToFields", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "Excluding transport and wait time": 0, + "Including transport and wait time": 1 + }, + "renameByName": {} + } + } + ], + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 33, + "panels": [], + "title": "Per module", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "linearThreshold": 1, + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "latency" + }, + "properties": [ + { + "id": "custom.gradientMode", + "value": "scheme" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Median value" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 15 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 0, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "timeseries", + "rawSql": "SELECT slt.event_timestamp AS time, dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS value\nFROM server_logs sl\nINNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\nWHERE slt.event = 'timestamp_out' AND\n sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\nORDER BY time ASC;", + "refId": "Latency" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT\n toStartOfMinute(time) AS time_bucket,\n median(latency) AS value\nFROM (\n SELECT slt.event_timestamp AS time, dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS latency\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out'\n AND sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\n)\nGROUP BY time_bucket\nORDER BY time_bucket;", + "refId": "Median" + } + ], + "title": "Latency LogServer", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 6, + "y": 15 + }, + "id": 29, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT median(value) AS \"Median\"\nFROM (\n SELECT slt.event_timestamp AS time, dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS value\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out' AND\n sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "title": "Median", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Latency value" + }, + "properties": [ + { + "id": "custom.gradientMode", + "value": "scheme" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Median value" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 15 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 0, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "timeseries", + "rawSql": "SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM batch_timestamps bt1\nINNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\nWHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\nORDER BY time ASC;", + "refId": "Latency" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT\n toStartOfMinute(time) AS time_bucket,\n median(latency) AS value\nFROM (\n SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS latency\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n)\nGROUP BY time_bucket\nORDER BY time_bucket;", + "refId": "Median" + } + ], + "title": "Latency Prefilter", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 18, + "y": 15 + }, + "id": 35, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT median(value) AS \"Median\"\nFROM (\n SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "title": "Median", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 6, + "y": 18 + }, + "id": 28, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT slt.event_timestamp AS time, dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS value\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out' AND\n sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 18, + "y": 18 + }, + "id": 36, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Average value" + }, + "properties": [ + { + "id": "custom.gradientMode", + "value": "scheme" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Median value" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 21 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 0, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "timeseries", + "rawSql": "SELECT lt2.timestamp as time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) as value\nFROM logline_timestamps lt1\nINNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\nWHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\nORDER BY time ASC;", + "refId": "Latency" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT\n toStartOfMinute(time) AS time_bucket,\n median(latency) AS value\nFROM (\n SELECT lt2.timestamp AS time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) AS latency\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n)\nGROUP BY time_bucket\nORDER BY time_bucket;", + "refId": "Median" + } + ], + "title": "Latency Collector", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 6, + "y": 21 + }, + "id": 30, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT median(value) AS \"Median\"\nFROM (\n SELECT lt2.timestamp as time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) as value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "title": "Median", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "latency" + }, + "properties": [ + { + "id": "custom.gradientMode", + "value": "scheme" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Median value" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 21 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 0, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "timeseries", + "rawSql": "SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM batch_timestamps bt1\nINNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\nWHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\nORDER BY time ASC\n\nUNION ALL\n\nSELECT sbt.timestamp AS time, dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS value\nFROM batch_timestamps bt\nINNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\nINNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\nWHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\nORDER BY time ASC;", + "refId": "Latency" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT\n toStartOfMinute(time) AS time_bucket,\n median(latency) as value\nFROM (\n SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS latency\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n ORDER BY time ASC\n\n UNION ALL\n\n SELECT sbt.timestamp AS time, dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS latency\n FROM batch_timestamps bt\n INNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\n INNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\n WHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\n)\nGROUP BY time_bucket\nORDER BY time_bucket;", + "refId": "Median" + } + ], + "title": "Latency Inspector", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 18, + "y": 21 + }, + "id": 37, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT median(value) AS \"Median\"\nFROM (\n SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n\n UNION ALL\n\n SELECT sbt.timestamp AS time, dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS value\n FROM batch_timestamps bt\n INNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\n INNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\n WHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "title": "Median", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 6, + "y": 24 + }, + "id": 31, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT lt2.timestamp as time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) as value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 18, + "y": 24 + }, + "id": 38, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n\n UNION ALL\n\n SELECT sbt.timestamp AS time, dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS value\n FROM batch_timestamps bt\n INNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\n INNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\n WHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "latency" + }, + "properties": [ + { + "id": "custom.gradientMode", + "value": "scheme" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Median value" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 27 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 0, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "timeseries", + "rawSql": "SELECT lt2.timestamp AS time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) AS value\nFROM logline_timestamps lt1\nINNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\nWHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n lt2.stage = 'log_collection.batch_handler' AND lt2.status = 'batched' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\nORDER BY time ASC;", + "refId": "Latency" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT\n toStartOfMinute(time) AS time_bucket,\n median(latency) as value\nFROM (\n SELECT lt2.timestamp AS time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) AS latency\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n lt2.stage = 'log_collection.batch_handler' AND lt2.status = 'batched' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n)\nGROUP BY time_bucket\nORDER BY time_bucket;", + "refId": "Median" + } + ], + "title": "Latency BatchHandler", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 6, + "y": 27 + }, + "id": 32, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT median(value) AS \"Median\"\nFROM (\n SELECT lt2.timestamp AS time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) AS value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n lt2.stage = 'log_collection.batch_handler' AND lt2.status = 'batched' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "title": "Median", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "latency" + }, + "properties": [ + { + "id": "custom.gradientMode", + "value": "scheme" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Median value" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 27 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 0, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "timeseries", + "rawSql": "SELECT sbt2.timestamp AS time, dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS value\nFROM suspicious_batch_timestamps sbt1\nINNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\nWHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\nORDER BY time ASC;", + "refId": "Latency" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT\n toStartOfMinute(time) AS time_bucket,\n median(latency) AS value\nFROM (\n SELECT sbt2.timestamp AS time, dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS latency\n FROM suspicious_batch_timestamps sbt1\n INNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\n WHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\n)\nGROUP BY time_bucket\nORDER BY time_bucket;", + "refId": "Median" + } + ], + "title": "Latency Detector", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 18, + "y": 27 + }, + "id": 39, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT median(value) AS \"Median\"\nFROM (\n SELECT sbt2.timestamp AS time, dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS value\n FROM suspicious_batch_timestamps sbt1\n INNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\n WHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "title": "Median", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 6, + "y": 30 + }, + "id": 34, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT lt2.timestamp AS time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) AS value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n lt2.stage = 'log_collection.batch_handler' AND lt2.status = 'batched' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 18, + "y": 30 + }, + "id": 40, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT sbt2.timestamp AS time, dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS value\n FROM suspicious_batch_timestamps sbt1\n INNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\n WHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "transparent": true, + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 71, + "panels": [], + "title": "Transport and wait time between stages", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Latency value" + }, + "properties": [ + { + "id": "custom.gradientMode", + "value": "scheme" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Median value" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 34 + }, + "id": 74, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 0, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "timeseries", + "rawSql": "SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM batch_timestamps bt1\nINNER JOIN batch_timestamps bt2\n ON bt1.batch_id = bt2.batch_id\nWHERE bt1.stage = 'log_collection.batch_handler' AND bt1.status = 'completed'\n AND bt2.stage = 'log_filtering.prefilter' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\nORDER BY time ASC;", + "refId": "Latency" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT\n toStartOfMinute(time) AS time_bucket,\n median(value) AS value\nFROM (\n SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2\n ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'log_collection.batch_handler' AND bt1.status = 'completed'\n AND bt2.stage = 'log_filtering.prefilter' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n)\nGROUP BY time_bucket\nORDER BY time_bucket;", + "refId": "Median" + } + ], + "title": "Time between BatchHandler and Prefilter", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Latency value" + }, + "properties": [ + { + "id": "custom.gradientMode", + "value": "scheme" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Median value" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 34 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 0, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "timeseries", + "rawSql": "SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM batch_timestamps bt1\nINNER JOIN batch_timestamps bt2\n ON bt1.batch_id = bt2.batch_id\nWHERE bt1.stage = 'log_filtering.prefilter' AND bt1.status = 'finished'\n AND bt2.stage = 'data_inspection.inspector' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\nORDER BY time ASC;", + "refId": "Latency" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT\n toStartOfMinute(time) AS time_bucket,\n median(value) AS value\nFROM (\n SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2\n ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'log_filtering.prefilter' AND bt1.status = 'finished'\n AND bt2.stage = 'data_inspection.inspector' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n)\nGROUP BY time_bucket\nORDER BY time_bucket;", + "refId": "Median" + } + ], + "title": "Time between Prefilter and Inspector", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Latency value" + }, + "properties": [ + { + "id": "custom.gradientMode", + "value": "scheme" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Median value" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 34 + }, + "id": 76, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 0, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "timeseries", + "rawSql": "SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM suspicious_batch_timestamps bt1\nINNER JOIN suspicious_batch_timestamps bt2\n ON bt1.suspicious_batch_id = bt2.suspicious_batch_id\nWHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'finished'\n AND bt2.stage = 'data_analysis.detector' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\nORDER BY time ASC;", + "refId": "Latency" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT\n toStartOfMinute(time) AS time_bucket,\n median(value) AS value\nFROM (\n SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM suspicious_batch_timestamps bt1\n INNER JOIN suspicious_batch_timestamps bt2\n ON bt1.suspicious_batch_id = bt2.suspicious_batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'finished'\n AND bt2.stage = 'data_analysis.detector' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n)\nGROUP BY time_bucket\nORDER BY time_bucket;", + "refId": "Median" + } + ], + "title": "Time between Inspector and Detector", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Median" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 40 + }, + "id": 57, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", median(value) AS \"Median\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT dateDiff('microseconds', bt1.timestamp, bt2.timestamp) as value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2\n ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'log_collection.batch_handler' AND bt1.status = 'completed'\n AND bt2.stage = 'log_filtering.prefilter' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Median" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 40 + }, + "id": 48, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", median(value) AS \"Median\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT dateDiff('microseconds', bt1.timestamp, bt2.timestamp) as value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2\n ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'log_filtering.prefilter' AND bt1.status = 'finished'\n AND bt2.stage = 'data_inspection.inspector' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Median" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 16, + "y": 40 + }, + "id": 49, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", median(value) AS \"Median\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT dateDiff('microseconds', bt1.timestamp, bt2.timestamp) as value\n FROM suspicious_batch_timestamps bt1\n INNER JOIN suspicious_batch_timestamps bt2\n ON bt1.suspicious_batch_id = bt2.suspicious_batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'finished'\n AND bt2.stage = 'data_analysis.detector' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "type": "stat" + } + ], + "refresh": "auto", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "description": "Modules to be included in the comparison graphs and charts", + "hide": 0, + "includeAll": true, + "label": "Modules to include in comparison", + "multi": true, + "name": "include_modules", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "LogServer", + "value": "'LogServer'" + }, + { + "selected": false, + "text": "Collector", + "value": "'Collector'" + }, + { + "selected": false, + "text": "BatchHandler", + "value": "'BatchHandler'" + }, + { + "selected": false, + "text": "Prefilter", + "value": "'Prefilter'" + }, + { + "selected": false, + "text": "Inspector", + "value": "'Inspector'" + }, + { + "selected": false, + "text": "Detector", + "value": "'Detector'" + } + ], + "query": "LogServer : 'LogServer', Collector : 'Collector', BatchHandler : 'BatchHandler', Prefilter : 'Prefilter', Inspector : 'Inspector', Detector : 'Detector'", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "Latencies", + "uid": "fe9rv9qph1af4f", + "version": 1, + "weekStart": "" +} diff --git a/docker/grafana-provisioning/dashboards/log_volumes.json b/docker/grafana-provisioning/dashboards/log_volumes.json new file mode 100644 index 00000000..abbf228c --- /dev/null +++ b/docker/grafana-provisioning/dashboards/log_volumes.json @@ -0,0 +1,2662 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "Number of all log lines that entered the LogServer in the time range, and those being fully processed, either by being filtered out or leading to an alert.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 17, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Entering" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Processed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 0, + "y": 0 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.7.0", + "queryType": "table", + "rawSql": "SELECT timestamp_in, count(DISTINCT message_id) OVER (ORDER BY timestamp_in) AS cumulative_count\nFROM (\n SELECT timestamp_in, message_id\n FROM server_logs\n WHERE timestamp_in >= $__fromTime AND timestamp_in <= $__toTime\n);", + "refId": "A" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.7.0", + "queryType": "table", + "rawSql": "SELECT timestamp, count(DISTINCT value) OVER (ORDER BY timestamp) AS cumulative_count\nFROM (\n SELECT timestamp_failed AS timestamp, message_text AS value\n FROM failed_dns_loglines\n WHERE timestamp >= $__fromTime AND timestamp <= $__toTime\n\n UNION ALL\n\n SELECT timestamp, toString(logline_id) AS value\n FROM logline_timestamps\n WHERE is_active = False\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n);", + "refId": "B" + } + ], + "title": "Entering and processed log lines", + "transformations": [ + { + "filter": { + "id": "byRefId", + "options": "A" + }, + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "cumulative_count": "Entering", + "timestamp_in": "" + } + }, + "topic": "series" + }, + { + "filter": { + "id": "byRefId", + "options": "B" + }, + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "cumulative_count": "Processed" + } + }, + "topic": "series" + } + ], + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "# Log Lines / ${Granularity}", + "axisPlacement": "left", + "fillOpacity": 74, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Entering" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Processed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 9, + "y": 0 + }, + "id": 73, + "options": { + "barRadius": 0, + "barWidth": 0.8, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "vertical", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT \n toStartOf${Granularity}(toDateTime64(timestamp_in, 6)) AS time_bucket,\n count(*) AS \" \"\nFROM server_logs\nWHERE timestamp_in >= toStartOf${Granularity}(toDateTime64($__fromTime, 6)) AND timestamp_in <= toStartOf${Granularity}(toDateTime64($__toTime, 6))\nGROUP BY time_bucket\nORDER BY time_bucket\nWITH FILL\nFROM toStartOf${Granularity}(toDateTime64($__fromTime, 6))\nTO toStartOf${Granularity}(toDateTime64($__toTime, 6))\nSTEP toInterval${Granularity}(1);", + "refId": "Entering" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.7.0", + "queryType": "table", + "rawSql": "SELECT time_bucket, -sum(number) as \"(negative)\"\nFROM (\n SELECT \n toStartOf${Granularity}(toDateTime64(timestamp_failed, 6)) AS time_bucket,\n count(DISTINCT message_text) AS number\n FROM failed_dns_loglines\n WHERE timestamp_failed >= toStartOf${Granularity}(toDateTime64($__fromTime, 6)) AND timestamp_failed <= toStartOf${Granularity}(toDateTime64($__toTime, 6))\n GROUP BY time_bucket\n ORDER BY time_bucket\n WITH FILL\n FROM toStartOf${Granularity}(toDateTime64($__fromTime, 6))\n TO toStartOf${Granularity}(toDateTime64($__toTime, 6))\n STEP toInterval${Granularity}(1)\n\n UNION ALL\n\n SELECT \n toStartOf${Granularity}(toDateTime64(timestamp, 6)) AS time_bucket,\n count(DISTINCT logline_id) AS number\n FROM logline_timestamps\n WHERE is_active = False\n AND timestamp >= toStartOf${Granularity}(toDateTime64($__fromTime, 6)) AND timestamp <= toStartOf${Granularity}(toDateTime64($__toTime, 6))\n GROUP BY time_bucket\n ORDER BY time_bucket\n WITH FILL\n FROM toStartOf${Granularity}(toDateTime64($__fromTime, 6))\n TO toStartOf${Granularity}(toDateTime64($__toTime, 6))\n STEP toInterval${Granularity}(1)\n)\nGROUP BY time_bucket;", + "refId": "Processed" + } + ], + "title": "Entering and processed log lines per ${Granularity}", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 16, + "y": 0 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT 'Suspicious' AS state, MAX(message_count) AS maximal_size\nFROM suspicious_batch_timestamps\nWHERE stage = 'data_inspection.inspector' AND\n status = 'finished' AND\n timestamp >= $__fromTime AND timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Filtered' AS state, MAX(message_count) AS maximal_size\nFROM batch_timestamps\nWHERE stage = 'log_filtering.prefilter' AND\n status = 'finished' AND\n timestamp >= $__fromTime AND timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Initial' AS state, MAX(message_count) AS maximal_size\nFROM batch_timestamps\nWHERE stage = 'log_collection.batch_handler' AND\n status = 'completed' AND\n timestamp >= $__fromTime AND timestamp <= $__toTime", + "refId": "A" + } + ], + "title": "Max. number of entries per Batch", + "transformations": [ + { + "id": "rowsToFields", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "Filtered": 1, + "Initial": 0, + "Suspicious": 2 + }, + "renameByName": { + "Initial": "" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 74, + "options": { + "folderUID": "", + "includeVars": true, + "keepTime": true, + "maxItems": 10, + "query": "", + "showFolderNames": false, + "showHeadings": false, + "showRecentlyViewed": false, + "showSearch": true, + "showStarred": false, + "tags": [] + }, + "pluginVersion": "11.2.2+security-01", + "title": "Dashboards", + "transparent": true, + "type": "dashlist" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 72, + "panels": [], + "title": "Comparison", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "# Log Lines / Module (log10)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 11, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 15, + "x": 0, + "y": 9 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT timestamp, entry_count AS \" \"\nFROM fill_levels\nWHERE stage = 'log_collection.collector'\n AND 'Collector' IN (${include_modules:csv})\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "Collector" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT timestamp, entry_count AS \" \"\nFROM fill_levels\nWHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND 'Prefilter' IN (${include_modules:csv})\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "Prefilter" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT timestamp, entry_count AS \" \"\nFROM fill_levels\nWHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND 'Inspector' IN (${include_modules:csv})\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "Inspector" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT timestamp, entry_count AS \" \"\nFROM fill_levels\nWHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND 'Detector' IN (${include_modules:csv})\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "Detector" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT timestamp, entry_count AS \" \"\nFROM fill_levels\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND 'BatchHandler' IN (${include_modules:csv})\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "BatchHandler" + } + ], + "title": "Log Volume Combined", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "decimals": 0, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 15, + "y": 9 + }, + "id": 47, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 0, + "namePlacement": "top", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "text" + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT *\nFROM (\n SELECT 'Collector' AS name, median(value) AS \"Median\", 2 AS sort_order\n FROM (\n SELECT entry_count as value\n FROM fill_levels\n WHERE stage = 'log_collection.collector' AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'BatchHandler' AS name, median(value) AS \"Median\", 3 AS sort_order\n FROM (\n SELECT entry_count as value\n FROM fill_levels\n WHERE stage = 'log_collection.batch_handler' AND entry_type = 'total_loglines_in_batches' AND\n timestamp >= $__fromTime AND timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, median(value) AS \"Median\", 4 AS sort_order\n FROM (\n SELECT entry_count as value\n FROM fill_levels\n WHERE stage = 'log_filtering.prefilter' AND entry_type = 'total_loglines' AND\n timestamp >= $__fromTime AND timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Inspector' AS name, median(value) AS \"Median\", 5 AS sort_order\n FROM (\n SELECT entry_count as value\n FROM fill_levels\n WHERE stage = 'data_inspection.inspector' AND entry_type = 'total_loglines' AND\n timestamp >= $__fromTime AND timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Detector' AS name, median(value) AS \"Median\", 6 AS sort_order\n FROM (\n SELECT entry_count as value\n FROM fill_levels\n WHERE stage = 'data_analysis.detector' AND entry_type = 'total_loglines' AND\n timestamp >= $__fromTime AND timestamp <= $__toTime\n )\n)\nWHERE name IN (${include_modules:csv})\nORDER BY sort_order ASC;", + "refId": "Fill States" + } + ], + "title": "Log volume comparison", + "transformations": [ + { + "id": "rowsToFields", + "options": { + "mappings": [ + { + "fieldName": "name", + "handlerKey": "field.name" + }, + { + "fieldName": "Median", + "handlerKey": "field.value" + }, + { + "fieldName": "sort_order", + "handlerKey": "__ignore" + } + ] + } + } + ], + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 59, + "panels": [], + "title": "Per Module", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 27, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 19 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT timestamp, entry_count AS \" \"\nFROM fill_levels\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "Batch" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT timestamp, entry_count AS \" \"\nFROM fill_levels\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_buffer'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "Buffer" + } + ], + "title": "BatchHandler Fill State", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 6, + "y": 19 + }, + "id": 60, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT median(entry_count) AS \"Median\"\nFROM (\n SELECT *\n FROM fill_levels\n WHERE stage = 'log_collection.collector'\n AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "title": "Median", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 19 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT *\nFROM fill_levels\nWHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "Batch" + } + ], + "title": "Prefilter Fill State", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 18, + "y": 19 + }, + "id": 64, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT median(entry_count) AS \"Median\"\nFROM (\n SELECT *\n FROM fill_levels\n WHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "title": "Median", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Maximum" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 6, + "y": 22 + }, + "id": 61, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT entry_count AS value\n FROM fill_levels\n WHERE stage = 'log_collection.collector'\n AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Maximum" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 18, + "y": 22 + }, + "id": 67, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT entry_count AS value\n FROM fill_levels\n WHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 11, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 25 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT *\nFROM fill_levels\nWHERE stage = 'log_collection.collector'\n AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "A" + } + ], + "title": "Collector Fill State", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 25 + }, + "id": 62, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT median(entry_count) AS \"Batch Median\"\nFROM (\n SELECT *\n FROM fill_levels\n WHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n)", + "refId": "Batch" + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "yellow", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 9, + "y": 25 + }, + "id": 68, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT median(entry_count) AS \"Buffer Median\"\nFROM (\n SELECT *\n FROM fill_levels\n WHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_buffer'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n)", + "refId": "Buffer" + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 25 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT *\nFROM fill_levels\nWHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "Batch" + } + ], + "title": "Inspector Fill State", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 18, + "y": 25 + }, + "id": 66, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT median(entry_count) AS \"Median\"\nFROM (\n SELECT *\n FROM fill_levels\n WHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "title": "Median", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Maximum" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 6, + "y": 28 + }, + "id": 63, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT entry_count AS value\n FROM fill_levels\n WHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n)", + "refId": "Batch" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT entry_count AS value\n FROM fill_levels\n WHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_buffer'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n)", + "refId": "Buffer" + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Maximum" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 18, + "y": 28 + }, + "id": 65, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT entry_count AS value\n FROM fill_levels\n WHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 31 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT *\nFROM fill_levels\nWHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "Batch" + } + ], + "title": "Detector Fill State", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 18, + "y": 31 + }, + "id": 70, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT median(entry_count) AS \"Median\"\nFROM (\n SELECT *\n FROM fill_levels\n WHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "title": "Median", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Maximum" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 18, + "y": 34 + }, + "id": 69, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT min(value) AS \"Minimum\", avg(value) AS \"Average\", max(value) AS \"Maximum\"\nFROM (\n SELECT entry_count AS value\n FROM fill_levels\n WHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n)", + "refId": "A" + } + ], + "transparent": true, + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 76, + "panels": [], + "title": "Remaining log rate per module", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 75, + "options": { + "edges": {}, + "nodes": { + "arcs": [ + { + "color": "#73BF69", + "field": "arc__success" + }, + { + "color": "#F2495C", + "field": "arc__filteredout" + } + ], + "mainStatUnit": "" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT 'LogCollector' AS id, 'LogCollector' AS title, (\n SELECT count(*)\n FROM logline_timestamps\n WHERE stage = 'log_collection.collector'\n AND status = 'finished'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n) AS successful_count, (\n SELECT count(*)\n FROM failed_dns_loglines\n WHERE timestamp_failed >= $__fromTime AND timestamp_failed <= $__toTime\n) AS filteredout_count,\nsuccessful_count / (successful_count + filteredout_count) AS arc__success,\nfilteredout_count / (successful_count + filteredout_count) AS arc__filteredout,\nif(isNaN(arc__success), '-', CONCAT(toString(round(arc__success * 100, 1)), '%')) AS mainstat\n\nUNION ALL \n\nSELECT 'BatchHandler' AS id, 'BatchHandler' AS title, \n0 AS successful_count, \n0 AS filteredout_count, \nif((\n SELECT count(*) = 0\n FROM batch_timestamps\n WHERE stage = 'log_collection.batch_handler'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n), 0, 1) AS arc__success, \n0 AS arc__filteredout,\nif((\n SELECT count(*) = 0\n FROM batch_timestamps\n WHERE stage = 'log_collection.batch_handler'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n), '-', CONCAT(toString(round(arc__success * 100, 1)), '%')) AS mainstat\n\nUNION ALL\n\nSELECT 'Prefilter' AS id, 'Prefilter' AS title, (\n SELECT sum(message_count)\n FROM batch_timestamps\n WHERE stage = 'log_filtering.prefilter'\n AND status = 'finished'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n) AS successful_count, (\n SELECT count(*)\n FROM logline_timestamps\n WHERE stage = 'log_filtering.prefilter'\n AND status = 'filtered_out'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n) AS filteredout_count,\nsuccessful_count / (successful_count + filteredout_count) AS arc__success,\nfilteredout_count / (successful_count + filteredout_count) AS arc__filteredout,\nif(isNaN(arc__success), '-', CONCAT(toString(round(arc__success * 100, 1)), '%')) AS mainstat\n\nUNION ALL\n\nSELECT 'Inspector' AS id, 'Inspector' AS title, (\n SELECT sum(message_count)\n FROM suspicious_batch_timestamps\n WHERE stage = 'data_inspection.inspector'\n AND status = 'finished'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n) AS successful_count, (\n SELECT sum(message_count)\n FROM batch_timestamps\n WHERE stage = 'data_inspection.inspector'\n AND status = 'filtered_out'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n) AS filteredout_count,\nsuccessful_count / (successful_count + filteredout_count) AS arc__success,\nfilteredout_count / (successful_count + filteredout_count) AS arc__filteredout,\nif(isNaN(arc__success), '-', CONCAT(toString(round(arc__success * 100, 1)), '%')) AS mainstat\n\nUNION ALL\n\nSELECT 'Detector' AS id, 'Detector' AS title, (\n SELECT sum(message_count)\n FROM suspicious_batch_timestamps\n WHERE stage = 'data_analysis.detector'\n AND status = 'finished'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n) AS successful_count, (\n SELECT sum(message_count)\n FROM batch_timestamps\n WHERE stage = 'data_analysis.detector'\n AND status = 'filtered_out'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n) AS filteredout_count,\nsuccessful_count / (successful_count + filteredout_count) AS arc__success,\nfilteredout_count / (successful_count + filteredout_count) AS arc__filteredout,\nif(isNaN(arc__success), '-', CONCAT(toString(round(arc__success * 100, 1)), '%')) AS mainstat", + "refId": "Nodes" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT 'LogCollector_BatchHandler' AS id, 'LogCollector' AS source, 'BatchHandler' AS target\nUNION ALL\nSELECT 'BatchHandler_Prefilter' AS id, 'BatchHandler' AS source, 'Prefilter' AS target\nUNION ALL\nSELECT 'Prefilter_Inspector' AS id, 'Prefilter' AS source, 'Inspector' AS target\nUNION ALL\nSELECT 'Inspector_Detector' AS id, 'Inspector' AS source, 'Detector' AS target", + "refId": "Edges" + } + ], + "transformations": [ + { + "filter": { + "id": "byRefId", + "options": "Nodes" + }, + "id": "organize", + "options": { + "excludeByName": { + "arc__filteredout": false, + "arc__success": false, + "filteredout_count": true, + "id": false, + "mainstat": false, + "secondarystat": false, + "successful_count": true, + "title": false + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "arc__filteredout": "Filtered out", + "arc__success": "Success", + "mainstat": "Success Rate", + "secondarystat": "" + } + }, + "topic": "series" + } + ], + "transparent": true, + "type": "nodeGraph" + } + ], + "refresh": "auto", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Minute", + "value": "Minute" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT CASE \n WHEN (dateDiff('minute', toDateTime($__fromTime), toDateTime($__toTime))) <= 5 THEN 'Second'\n WHEN (dateDiff('hour', toDateTime($__fromTime), toDateTime($__toTime))) <= 6 THEN 'Minute'\n WHEN (dateDiff('hour', toDateTime($__fromTime), toDateTime($__toTime))) <= 72 THEN 'Hour'\n ELSE 'Day'\nEND AS interval;", + "hide": 2, + "includeAll": false, + "multi": false, + "name": "Granularity", + "options": [], + "query": "SELECT CASE \n WHEN (dateDiff('minute', toDateTime($__fromTime), toDateTime($__toTime))) <= 5 THEN 'Second'\n WHEN (dateDiff('hour', toDateTime($__fromTime), toDateTime($__toTime))) <= 6 THEN 'Minute'\n WHEN (dateDiff('hour', toDateTime($__fromTime), toDateTime($__toTime))) <= 72 THEN 'Hour'\n ELSE 'Day'\nEND AS interval;", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "description": "Modules to be included in the comparison graphs and charts", + "hide": 0, + "includeAll": true, + "label": "Modules to include in comparison", + "multi": true, + "name": "include_modules", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "LogServer", + "value": "'LogServer'" + }, + { + "selected": false, + "text": "Collector", + "value": "'Collector'" + }, + { + "selected": false, + "text": "BatchHandler", + "value": "'BatchHandler'" + }, + { + "selected": false, + "text": "Prefilter", + "value": "'Prefilter'" + }, + { + "selected": false, + "text": "Inspector", + "value": "'Inspector'" + }, + { + "selected": false, + "text": "Detector", + "value": "'Detector'" + } + ], + "query": "LogServer : 'LogServer', Collector : 'Collector', BatchHandler : 'BatchHandler', Prefilter : 'Prefilter', Inspector : 'Inspector', Detector : 'Detector'", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "Log Volumes", + "uid": "ae9rw4xedy1a8e", + "version": 1, + "weekStart": "" +} diff --git a/docker/grafana-provisioning/dashboards/overview.json b/docker/grafana-provisioning/dashboards/overview.json new file mode 100644 index 00000000..5ac938ae --- /dev/null +++ b/docker/grafana-provisioning/dashboards/overview.json @@ -0,0 +1,1021 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "fieldMinMax": false + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "hideTimeOverride": false, + "id": 5, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "blue", + "min": 0, + "mode": "opacity", + "reverse": false, + "scale": "linear", + "scheme": "PuBu", + "steps": 62 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "ge" + }, + "tooltip": { + "mode": "single", + "showColorScale": true, + "yHistogram": true + }, + "yAxis": { + "axisLabel": "# Alerts / Quarter Hour", + "axisPlacement": "left", + "reverse": true, + "unit": "m" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.7.0", + "queryType": "table", + "rawSql": "SELECT \n alert_hour as time,\n SUM(CASE WHEN quarter = 0 THEN alert_count ELSE 0 END) AS `0`,\n SUM(CASE WHEN quarter = 1 THEN alert_count ELSE 0 END) AS `15`,\n SUM(CASE WHEN quarter = 2 THEN alert_count ELSE 0 END) AS `30`,\n SUM(CASE WHEN quarter = 3 THEN alert_count ELSE 0 END) AS `45`\nFROM (\n SELECT \n toDateTime(DATE_FORMAT(alert_timestamp, '%Y-%m-%d %H:00:00')) AS alert_hour,\n FLOOR(MINUTE(alert_timestamp) / 15) AS quarter,\n COUNT(*) AS alert_count\n FROM alerts\n WHERE alert_hour >= toDateTime(DATE_FORMAT(now() - INTERVAL 1 DAY, '%Y-%m-%d %H:00:00'))\n AND alert_timestamp <= now()\n GROUP BY alert_hour, quarter\n) AS subquery\nGROUP BY alert_hour\nORDER BY alert_hour\nWITH FILL\nFROM toDateTime(DATE_FORMAT(now() - INTERVAL 1 DAY, '%Y-%m-%d %H:00:00'))\nTO toDateTime(now())\nSTEP toIntervalHour(1);\n", + "refId": "A" + } + ], + "timeFrom": "24h", + "title": "# Alerts per quarter hour", + "type": "heatmap" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "filterable": false, + "inspect": false + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 11, + "x": 8, + "y": 0 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT concat(rowNumberInAllBlocks() + 1, '.') AS \"Rank\", client_ip AS \"Client IP address\", count(logline_id) AS \"# Total Requests\"\nFROM dns_loglines\nWHERE \"Client IP address\" IN (\n SELECT DISTINCT client_ip\n FROM alerts\n WHERE alert_timestamp >= $__fromTime AND alert_timestamp <= $__toTime\n)\nGROUP BY \"Client IP address\"\nORDER BY \"# Total Requests\" DESC\nLIMIT 5", + "refId": "A" + } + ], + "title": "Top 5 most active malicious addresses", + "transparent": true, + "type": "table" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 2, + "options": { + "folderUID": "", + "includeVars": true, + "keepTime": true, + "maxItems": 10, + "query": "", + "showFolderNames": true, + "showHeadings": false, + "showRecentlyViewed": false, + "showSearch": true, + "showStarred": false, + "tags": [] + }, + "pluginVersion": "11.2.2+security-01", + "title": "Dashboards", + "transparent": true, + "type": "dashlist" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "# Log Lines / Module (log10)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 11, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 15, + "x": 0, + "y": 8 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT timestamp, entry_count AS \" \"\nFROM fill_levels\nWHERE stage = 'log_collection.collector'\n AND 'Collector' IN (${include_modules:csv})\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "Collector" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT timestamp, entry_count AS \" \"\nFROM fill_levels\nWHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND 'Prefilter' IN (${include_modules:csv})\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "Prefilter" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT timestamp, entry_count AS \" \"\nFROM fill_levels\nWHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND 'Inspector' IN (${include_modules:csv})\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "Inspector" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT timestamp, entry_count AS \" \"\nFROM fill_levels\nWHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND 'Detector' IN (${include_modules:csv})\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "Detector" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT timestamp, entry_count AS \" \"\nFROM fill_levels\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND 'BatchHandler' IN (${include_modules:csv})\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\nORDER BY timestamp ASC;", + "refId": "BatchHandler" + } + ], + "title": "Log volume combined", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "decimals": 0, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 15, + "y": 8 + }, + "id": 8, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 0, + "namePlacement": "top", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "text" + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.7.0", + "queryType": "table", + "rawSql": "SELECT *\nFROM (\n SELECT 'Collector' AS name, median(value) AS \"Median\", 2 AS sort_order\n FROM (\n SELECT entry_count as value\n FROM fill_levels\n WHERE stage = 'log_collection.collector' AND entry_type = 'total_loglines'\n AND timestamp >= $__fromTime AND timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'BatchHandler' AS name, median(value) AS \"Median\", 3 AS sort_order\n FROM (\n SELECT entry_count as value\n FROM fill_levels\n WHERE stage = 'log_collection.batch_handler' AND entry_type = 'total_loglines_in_batches' AND\n timestamp >= $__fromTime AND timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, median(value) AS \"Median\", 4 AS sort_order\n FROM (\n SELECT entry_count as value\n FROM fill_levels\n WHERE stage = 'log_filtering.prefilter' AND entry_type = 'total_loglines' AND\n timestamp >= $__fromTime AND timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Inspector' AS name, median(value) AS \"Median\", 5 AS sort_order\n FROM (\n SELECT entry_count as value\n FROM fill_levels\n WHERE stage = 'data_inspection.inspector' AND entry_type = 'total_loglines' AND\n timestamp >= $__fromTime AND timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Detector' AS name, median(value) AS \"Median\", 6 AS sort_order\n FROM (\n SELECT entry_count as value\n FROM fill_levels\n WHERE stage = 'data_analysis.detector' AND entry_type = 'total_loglines' AND\n timestamp >= $__fromTime AND timestamp <= $__toTime\n )\n)\nWHERE name IN (${include_modules:csv})\nORDER BY sort_order ASC;", + "refId": "Fill States" + } + ], + "title": "Log volume comparison", + "transformations": [ + { + "id": "rowsToFields", + "options": { + "mappings": [ + { + "fieldName": "name", + "handlerKey": "field.name" + }, + { + "fieldName": "Median", + "handlerKey": "field.value" + }, + { + "fieldName": "sort_order", + "handlerKey": "__ignore" + } + ] + } + } + ], + "type": "bargauge" + }, + { + "datasource": { + "default": false, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "continuous-GrYlRd" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 13, + "x": 0, + "y": 15 + }, + "id": 1, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "hide": false, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.7.0", + "queryType": "table", + "rawSql": "SELECT *\nFROM (\n SELECT 'LogServer' AS name, median(value) as median\n FROM (\n SELECT dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS value\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out' AND\n sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Collector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, lt2.timestamp) as value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'BatchHandler' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, lt2.timestamp) AS value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n lt2.stage = 'log_collection.batch_handler' AND lt2.status = 'batched' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) as value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Inspector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n\n UNION ALL\n\n SELECT dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS value\n FROM batch_timestamps bt\n INNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\n INNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\n WHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Detector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS value\n FROM suspicious_batch_timestamps sbt1\n INNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\n WHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\n )\n)\nWHERE name IN (${include_modules:csv});\n", + "refId": "A" + } + ], + "title": "Latency comparison (per module)", + "transformations": [ + { + "id": "rowsToFields", + "options": { + "mappings": [] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "BatchHandler": 2, + "Collector": 1, + "Detector": 5, + "Inspector": 4, + "LogServer": 0, + "Prefilter": 3 + }, + "renameByName": {} + } + } + ], + "type": "bargauge" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "continuous-GrYlRd" + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 11, + "x": 13, + "y": 15 + }, + "id": 3, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "top", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "valueMode": "text" + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.7.0", + "queryType": "table", + "rawSql": "SELECT *\nFROM (\n SELECT 'Data analysis phase' AS name, sum(Median)\n FROM (\n SELECT 'Inspector' AS name, median(value) AS \"Median\"\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n\n UNION ALL\n\n SELECT dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS value\n FROM batch_timestamps bt\n INNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\n INNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\n WHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Detector' AS name, median(value) AS \"Median\"\n FROM (\n SELECT dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS value\n FROM suspicious_batch_timestamps sbt1\n INNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\n WHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\n )\n )\n\n UNION ALL\n\n SELECT 'Data preparation phase' AS name, sum(Median) as median\n FROM (\n SELECT 'LogServer' AS name, median(value) AS \"Median\"\n FROM (\n SELECT slt.event_timestamp AS time, dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS value\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out' AND\n sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'LogCollector' AS name, median(value) AS \"Median\"\n FROM (\n SELECT lt2.timestamp as time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) as value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'BatchHandler' AS name, median(value) AS \"Median\"\n FROM (\n SELECT lt2.timestamp AS time, dateDiff('microsecond', lt1.timestamp, lt2.timestamp) AS value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n lt2.stage = 'log_collection.batch_handler' AND lt2.status = 'batched' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, median(value) AS \"Median\"\n FROM (\n SELECT bt2.timestamp AS time, dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n )\n);\n", + "refId": "phases" + } + ], + "title": "Latency comparison (per phase)", + "transformations": [ + { + "id": "rowsToFields", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "Data analysis phase": 1, + "Data preparation phase": 0 + }, + "renameByName": {} + } + } + ], + "type": "bargauge" + }, + { + "datasource": { + "default": true, + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 11, + "x": 13, + "y": 19 + }, + "id": 4, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "top", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.6.0", + "queryType": "table", + "rawSql": "SELECT 'Including transport and wait time' AS name, sum(median)\nFROM (\n SELECT 'LogServer' AS name, median(value) as median\n FROM (\n SELECT dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS value\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out' AND\n sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'LogCollection' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, lt2.timestamp) as value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'BatchHandler' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, bt2.timestamp) AS value\n FROM logline_timestamps lt1\n INNER JOIN logline_to_batches ltb ON ltb.logline_id = lt1.logline_id\n INNER JOIN batch_timestamps bt2 ON bt2.batch_id = ltb.batch_id\n WHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n bt2.stage = 'log_collection.batch_handler' AND bt2.status = 'completed' AND\n lt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Between BatchHandler and Prefilter' AS name, median(value) AS median\n FROM (\n SELECT dateDiff('microseconds', bt1.timestamp, bt2.timestamp) as value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2\n ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'log_collection.batch_handler' AND bt1.status = 'completed'\n AND bt2.stage = 'log_filtering.prefilter' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) as value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Between Prefilter and Inspector' AS name, median(value) AS median\n FROM (\n SELECT dateDiff('microseconds', bt1.timestamp, bt2.timestamp) as value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2\n ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'log_filtering.prefilter' AND bt1.status = 'finished'\n AND bt2.stage = 'data_inspection.inspector' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Inspector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n\n UNION ALL\n\n SELECT dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS value\n FROM batch_timestamps bt\n INNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\n INNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\n WHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\n )\n\n UNION ALL\n \n SELECT 'Between Inspector and Detector' AS name, median(value) AS median\n FROM (\n SELECT dateDiff('microseconds', bt1.timestamp, bt2.timestamp) as value\n FROM suspicious_batch_timestamps bt1\n INNER JOIN suspicious_batch_timestamps bt2\n ON bt1.suspicious_batch_id = bt2.suspicious_batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'finished'\n AND bt2.stage = 'data_analysis.detector' AND bt2.status = 'in_process' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n \n UNION ALL\n\n SELECT 'Detector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS value\n FROM suspicious_batch_timestamps sbt1\n INNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\n WHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\n )\n)\n\nUNION ALL\n\nSELECT 'Excluding transport and wait time' AS name, sum(median)\nFROM (\n SELECT 'LogServer' AS name, median(value) as median\n FROM (\n SELECT dateDiff(microsecond, sl.timestamp_in, slt.event_timestamp) AS value\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out' AND\n sl.timestamp_in >= $__fromTime AND slt.event_timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'LogCollection' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, lt2.timestamp) as value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt1.logline_id = lt2.logline_id\n WHERE lt1.status = 'in_process' AND lt2.status = 'finished' AND\n lt1.stage = 'log_collection.collector' AND lt2.stage = 'log_collection.collector' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'BatchHandler' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', lt1.timestamp, lt2.timestamp) AS value\n FROM logline_timestamps lt1\n INNER JOIN logline_timestamps lt2 ON lt2.logline_id = lt1.logline_id\n WHERE lt1.stage = 'log_collection.batch_handler' AND lt1.status = 'in_process' AND\n lt2.stage = 'log_collection.batch_handler' AND lt2.status = 'batched' AND\n lt1.timestamp >= $__fromTime AND lt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) as value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.status = 'in_process' AND bt2.status = 'finished' AND\n bt1.stage = 'log_filtering.prefilter' AND bt2.stage = 'log_filtering.prefilter' AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Inspector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_timestamps bt1\n INNER JOIN batch_timestamps bt2 ON bt1.batch_id = bt2.batch_id\n WHERE bt1.stage = 'data_inspection.inspector' AND bt1.status = 'in_process' AND\n bt2.stage = 'data_inspection.inspector' AND bt2.is_active = False AND\n bt1.timestamp >= $__fromTime AND bt2.timestamp <= $__toTime\n\n UNION ALL\n\n SELECT dateDiff('microsecond', bt.timestamp, sbt.timestamp) AS value\n FROM batch_timestamps bt\n INNER JOIN suspicious_batches_to_batch sbtb ON bt.batch_id = sbtb.batch_id\n INNER JOIN suspicious_batch_timestamps sbt ON sbtb.suspicious_batch_id = sbt.suspicious_batch_id\n WHERE bt.stage = 'data_inspection.inspector' AND bt.status = 'in_process' AND\n sbt.stage = 'data_inspection.inspector' AND sbt.status = 'finished' AND\n bt.timestamp >= $__fromTime AND sbt.timestamp <= $__toTime\n )\n\n UNION ALL\n\n SELECT 'Detector' AS name, median(value) as median\n FROM (\n SELECT dateDiff('microsecond', sbt1.timestamp, sbt2.timestamp) AS value\n FROM suspicious_batch_timestamps sbt1\n INNER JOIN suspicious_batch_timestamps sbt2 ON sbt1.suspicious_batch_id = sbt2.suspicious_batch_id\n WHERE sbt1.stage = 'data_analysis.detector' AND sbt1.status = 'in_process' AND\n sbt2.stage = 'data_analysis.detector' AND sbt2.is_active = False AND\n sbt1.timestamp >= $__fromTime AND sbt2.timestamp <= $__toTime\n )\n);\n", + "refId": "A" + } + ], + "title": "Total latency", + "transformations": [ + { + "id": "rowsToFields", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "Excluding transport and wait time": 0, + "Including transport and wait time": 1 + }, + "renameByName": {} + } + } + ], + "type": "bargauge" + } + ], + "refresh": "auto", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "description": "Modules to be included in the comparison graphs and charts", + "hide": 0, + "includeAll": true, + "label": "Modules to include in comparison", + "multi": true, + "name": "include_modules", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "LogServer", + "value": "'LogServer'" + }, + { + "selected": false, + "text": "Collector", + "value": "'Collector'" + }, + { + "selected": false, + "text": "BatchHandler", + "value": "'BatchHandler'" + }, + { + "selected": false, + "text": "Prefilter", + "value": "'Prefilter'" + }, + { + "selected": false, + "text": "Inspector", + "value": "'Inspector'" + }, + { + "selected": false, + "text": "Detector", + "value": "'Detector'" + } + ], + "query": "LogServer : 'LogServer', Collector : 'Collector', BatchHandler : 'BatchHandler', Prefilter : 'Prefilter', Inspector : 'Inspector', Detector : 'Detector'", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Overview", + "uid": "eebjla27u66f4f", + "version": 5, + "weekStart": "" +} diff --git a/docker/grafana-provisioning/datasources.yaml b/docker/grafana-provisioning/datasources.yaml new file mode 100644 index 00000000..b0708fc4 --- /dev/null +++ b/docker/grafana-provisioning/datasources.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +datasources: + - name: ClickHouse + type: grafana-clickhouse-datasource + jsonData: + host: clickhouse-server + port: 9000 + + protocol: native + + username: default + tlsSkipVerify: false diff --git a/docker/init_datatests.sh b/docker/init_datatests.sh new file mode 100755 index 00000000..783afdbb --- /dev/null +++ b/docker/init_datatests.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +echo "Creating tables for normal operation..." + +for script in /create_tables/*.sql; do + echo "Executing $script..." + clickhouse-client --host=127.0.0.1 --query="$(cat $script)" +done + +echo "Initializing datatest tables..." + +for script in /create_datatest_tables/*.sql; do + echo "Executing $script..." + clickhouse-client --host=127.0.0.1 --query="$(cat $script)" +done + +echo "Inserting data..." + +for script in /insert_datatest_data/*.sql; do + echo "Executing $script..." + clickhouse-client --host=127.0.0.1 --query="$(cat $script)" +done + +echo "Initialization complete!" diff --git a/docker/insert_datatest_data/dgta_dataset.sql b/docker/insert_datatest_data/dgta_dataset.sql new file mode 100644 index 00000000..0a35e70b --- /dev/null +++ b/docker/insert_datatest_data/dgta_dataset.sql @@ -0,0 +1,3 @@ +INSERT INTO dgta_dataset +SELECT * +FROM file('/var/lib/clickhouse/user_files/data/dgta_dataset.json.gz', JSONEachRow); diff --git a/docker/mock_logs.dev.py b/docker/mock_logs.dev.py new file mode 100644 index 00000000..8a417daa --- /dev/null +++ b/docker/mock_logs.dev.py @@ -0,0 +1,25 @@ +import os +import sys + +sys.path.append(os.getcwd()) +from src.base.kafka_handler import SimpleKafkaProduceHandler +from src.mock.log_generator import generate_dns_log_line + +kafka_producer = SimpleKafkaProduceHandler() + + +def main(): + try: + for i in range(50000): + kafka_producer.produce( + "pipeline-logserver_in", f"{generate_dns_log_line('random-ip.de')}" + ) + print("Sent logline", i) + # time.sleep(0.1 * random.uniform(0.1, 1)) + # print(f"{generate_dns_log_line('random-ip.de')}") + except KeyboardInterrupt: + pass + + +if __name__ == "__main__": + main() diff --git a/docker/query.dev.py b/docker/query.dev.py index 8c274c38..db027dcd 100644 --- a/docker/query.dev.py +++ b/docker/query.dev.py @@ -18,7 +18,7 @@ def get_tables(): def query_once(client, tables): for table_name in tables.keys(): - tables[table_name] = client.query(f"SELECT * FROM {table_name};") + tables[table_name] = client.query(f"SELECT * FROM {table_name} LIMIT 10;") return tables diff --git a/docker/real_logs.dev.py b/docker/real_logs.dev.py new file mode 100644 index 00000000..4e73daab --- /dev/null +++ b/docker/real_logs.dev.py @@ -0,0 +1,51 @@ +import os +import sys +import time + +import numpy as np +import polars as pl +from confluent_kafka import KafkaError + +sys.path.append(os.getcwd()) +from src.base.kafka_handler import SimpleKafkaProduceHandler +from src.mock.log_generator import generate_dns_log_line +from src.base.log_config import get_logger +from src.train.dataset import Dataset, DatasetLoader + +logger = get_logger() +kafka_producer = SimpleKafkaProduceHandler() + +if __name__ == "__main__": + try: + data_base_path: str = "./data" + datasets = DatasetLoader(base_path=data_base_path, max_rows=10000) + dataset = Dataset( + data_path="", + data=pl.concat( + [ + datasets.dgta_dataset.data, + # datasets.cic_dataset.data, + # datasets.bambenek_dataset.data, + # datasets.dga_dataset.data, + # datasets.dgarchive_dataset.data, + ] + ), + max_rows=100, + ) + data = dataset.data + print(data) + np.random.seed(None) + while True: + for i in range(0, 10): + random_domain = data.sample(n=1) + logline = generate_dns_log_line(random_domain["query"].item()) + try: + kafka_producer.produce( + "pipeline-logserver_in", logline.encode("utf-8") + ) + logger.info(f"Sent logline: {logline}") + except KafkaError: + logger.warning(KafkaError) + time.sleep(0.1) + except KeyboardInterrupt: + pass diff --git a/src/base/data_classes/clickhouse_connectors.py b/src/base/data_classes/clickhouse_connectors.py index 957fd3ad..f7d98e9e 100644 --- a/src/base/data_classes/clickhouse_connectors.py +++ b/src/base/data_classes/clickhouse_connectors.py @@ -169,9 +169,26 @@ class Alerts: overall_score: float = field( metadata={"marshmallow_field": marshmallow.fields.Float()} ) + domain_names: str = field( + metadata={"marshmallow_field": marshmallow.fields.String()} + ) result: str = field(metadata={"marshmallow_field": marshmallow.fields.String()}) +@dataclass +class FillLevels: + timestamp: datetime.datetime = field( + metadata={ + "marshmallow_field": marshmallow.fields.DateTime("%Y-%m-%d %H:%M:%S.%f") + } + ) + stage: str = field(metadata={"marshmallow_field": marshmallow.fields.String()}) + entry_type: str = field(metadata={"marshmallow_field": marshmallow.fields.String()}) + entry_count: int = field( + metadata={"marshmallow_field": marshmallow.fields.Integer()} + ) + + TABLE_NAME_TO_TYPE = { "server_logs": ServerLogs, "server_logs_timestamps": ServerLogsTimestamps, @@ -183,4 +200,5 @@ class Alerts: "suspicious_batches_to_batch": SuspiciousBatchesToBatch, "suspicious_batch_timestamps": SuspiciousBatchTimestamps, "alerts": Alerts, + "fill_levels": FillLevels, } diff --git a/src/base/kafka_handler.py b/src/base/kafka_handler.py index 7fde8196..5fb41f85 100644 --- a/src/base/kafka_handler.py +++ b/src/base/kafka_handler.py @@ -16,8 +16,8 @@ KafkaError, KafkaException, Producer, - TopicPartition, ) +from confluent_kafka.admin import AdminClient, NewTopic sys.path.append(os.getcwd()) from src.base.data_classes.batch import Batch @@ -26,7 +26,9 @@ logger = get_logger() -CONSUMER_GROUP_ID = "consumer_group" +HOSTNAME = os.getenv("HOSTNAME", "default_tid") +CONSUMER_GROUP_ID = os.getenv("GROUP_ID", "default_gid") +NUMBER_OF_INSTANCES = int(os.getenv("NUMBER_OF_INSTANCES", 1)) config = setup_config() KAFKA_BROKERS = config["environment"]["kafka_brokers"] @@ -114,7 +116,7 @@ def produce(self, topic: str, data: str, key: None | str = None) -> None: self.producer.produce( topic=topic, key=key, - value=data.encode("utf-8"), + value=data, callback=kafka_delivery_report, ) @@ -124,14 +126,14 @@ class ExactlyOnceKafkaProduceHandler(KafkaProduceHandler): Wrapper for the Kafka Producer with Write-Exactly-Once semantics. """ - def __init__(self, transactional_id: str): + def __init__(self): self.brokers = ",".join( [f"{broker['hostname']}:{broker['port']}" for broker in KAFKA_BROKERS] ) conf = { "bootstrap.servers": self.brokers, - "transactional.id": transactional_id, + "transactional.id": HOSTNAME, "enable.idempotence": True, } @@ -161,7 +163,7 @@ def produce(self, topic: str, data: str, key: None | str = None) -> None: self.producer.produce( topic=topic, key=key, - value=data.encode("utf-8"), + value=data, callback=kafka_delivery_report, ) @@ -214,10 +216,12 @@ class KafkaConsumeHandler(KafkaHandler): def __init__(self, topics: str | list[str]) -> None: super().__init__() + # get brokers self.brokers = ",".join( [f"{broker['hostname']}:{broker['port']}" for broker in KAFKA_BROKERS] ) + # create consumer conf = { "bootstrap.servers": self.brokers, "group.id": CONSUMER_GROUP_ID, @@ -225,12 +229,27 @@ def __init__(self, topics: str | list[str]) -> None: "auto.offset.reset": "earliest", "enable.partition.eof": True, } + self.consumer = Consumer(conf) if isinstance(topics, str): topics = [topics] - self.consumer = Consumer(conf) - self.consumer.assign([TopicPartition(topic, 0) for topic in topics]) + # create topics + admin_client = AdminClient( + { + "bootstrap.servers": self.brokers, + } + ) + admin_client.create_topics( + [NewTopic(topic, NUMBER_OF_INSTANCES, 1) for topic in topics] + ) + + # check if topics are created + if not self._all_topics_created(topics): + raise TooManyFailedAttemptsError("Not all topics were created.") + + # subscribe to the topics + self.consumer.subscribe(topics) @abstractmethod def consume(self, *args, **kwargs): @@ -265,6 +284,27 @@ def consume_as_json(self) -> tuple[None | str, dict]: except Exception: raise ValueError("Unknown data format") + def _all_topics_created(self, topics): + number_of_retries_left = 30 + all_topics_created = False + while not all_topics_created: # try for 15 seconds + assigned_topics = self.consumer.list_topics(timeout=10) + + all_topics_created = True + for topic in topics: + if topic not in assigned_topics.topics: + all_topics_created = False + + if not all_topics_created: + number_of_retries_left -= 1 + + if not number_of_retries_left > 0: + return False + + time.sleep(0.5) + + return True + def __del__(self) -> None: if self.consumer: self.consumer.close() diff --git a/src/base/logline_handler.py b/src/base/logline_handler.py index c0199a5a..cbb3713b 100644 --- a/src/base/logline_handler.py +++ b/src/base/logline_handler.py @@ -7,7 +7,13 @@ CONFIG = setup_config() LOGLINE_FIELDS = CONFIG["pipeline"]["log_collection"]["collector"]["logline_format"] -REQUIRED_FIELDS = ["timestamp", "status_code", "client_ip", "record_type"] +REQUIRED_FIELDS = [ + "timestamp", + "status_code", + "client_ip", + "record_type", + "domain_name", +] FORBIDDEN_FIELD_NAMES = [ "logline_id", "batch_id", @@ -176,11 +182,12 @@ def validate_logline(self, logline: str) -> bool: True if the logline contains correct fields in the configured format, False otherwise """ parts = logline.split() + number_of_entries = len(parts) # check number of entries - if len(parts) != self.number_of_fields: + if number_of_entries != self.number_of_fields: logger.warning( - f"Logline contains {len(parts)} value(s), not {self.number_of_fields}." + f"Logline contains {number_of_entries} value(s), not {self.number_of_fields}." ) return False @@ -225,9 +232,8 @@ def __get_fields_as_json(self, logline: str) -> dict: return return_dict.copy() def validate_logline_and_get_fields_as_json(self, logline: str) -> dict: - """ - Validates the fields and returns them as dictionary, with the names of the fields as key, and the field value - as value. + """Validates the fields and returns them as dictionary, with the names of the fields as key, and the field + value as value. Args: logline (str): Logline as string to be validated diff --git a/src/base/utils.py b/src/base/utils.py index 26f6f041..c4bd938e 100644 --- a/src/base/utils.py +++ b/src/base/utils.py @@ -141,32 +141,3 @@ def normalize_ipv6_address( net = ipaddress.IPv6Network((address, prefix_length), strict=False) return net.network_address, prefix_length - - -def generate_unique_transactional_id(base_name: str, bootstrap_servers: str) -> str: - """ - Checks if the given name is already a transactional ID. If so, it a number is added to make it unique. - - Args: - base_name (str): Name of the transactional ID to be checked - bootstrap_servers (str): Kafka brokers as string in the form `'host1:port1,host2:port2'` - - Returns: - Unique transactional ID using the base_name - """ - # TODO: Test and activate - # admin_client = AdminClient({"bootstrap.servers": bootstrap_servers}) - # existing_ids = set() - # - # consumer_groups = admin_client.list_consumer_groups() - # - # for group in consumer_groups: - # existing_ids.add(group.id) - # - # transactional_id = base_name - # counter = 1 - # while transactional_id in existing_ids: - # transactional_id = f"{base_name}-{counter}" - # counter += 1 - - return base_name diff --git a/src/detector/detector.py b/src/detector/detector.py index 412817d8..da6ee708 100644 --- a/src/detector/detector.py +++ b/src/detector/detector.py @@ -68,6 +68,17 @@ def __init__(self) -> None: "suspicious_batch_timestamps" ) self.alerts = ClickHouseKafkaSender("alerts") + self.logline_timestamps = ClickHouseKafkaSender("logline_timestamps") + self.fill_levels = ClickHouseKafkaSender("fill_levels") + + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines", + entry_count=0, + ) + ) def get_and_fill_data(self) -> None: """Consumes data from KafkaConsumeHandler and stores it for processing.""" @@ -99,6 +110,15 @@ def get_and_fill_data(self) -> None: ) ) + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines", + entry_count=len(self.messages), + ) + ) + if not self.messages: logger.info( "Received message:\n" @@ -315,21 +335,75 @@ def send_warning(self) -> None: alert_timestamp=datetime.datetime.now(), suspicious_batch_id=self.suspicious_batch_id, overall_score=overall_score, + domain_names=json.dumps( + [warning["request"] for warning in self.warnings] + ), result=json.dumps(self.warnings), ) ) + + self.suspicious_batch_timestamps.insert( + dict( + suspicious_batch_id=self.suspicious_batch_id, + client_ip=self.key, + stage=module_name, + status="finished", + timestamp=datetime.datetime.now(), + is_active=False, + message_count=len(self.messages), + ) + ) + + logline_ids = set() + for message in self.messages: + logline_ids.add(message["logline_id"]) + + for logline_id in logline_ids: + self.logline_timestamps.insert( + dict( + logline_id=logline_id, + stage=module_name, + status="detected", + timestamp=datetime.datetime.now(), + is_active=False, + ) + ) else: logger.info("No warning produced.") - self.suspicious_batch_timestamps.insert( + self.suspicious_batch_timestamps.insert( + dict( + suspicious_batch_id=self.suspicious_batch_id, + client_ip=self.key, + stage=module_name, + status="filtered_out", + timestamp=datetime.datetime.now(), + is_active=False, + message_count=len(self.messages), + ) + ) + + logline_ids = set() + for message in self.messages: + logline_ids.add(message["logline_id"]) + + for logline_id in logline_ids: + self.logline_timestamps.insert( + dict( + logline_id=logline_id, + stage=module_name, + status="filtered_out", + timestamp=datetime.datetime.now(), + is_active=False, + ) + ) + + self.fill_levels.insert( dict( - suspicious_batch_id=self.suspicious_batch_id, - client_ip=self.key, - stage=module_name, - status="finished", timestamp=datetime.datetime.now(), - is_active=False, - message_count=len(self.messages), + stage=module_name, + entry_type="total_loglines", + entry_count=0, ) ) diff --git a/src/inspector/inspector.py b/src/inspector/inspector.py index 334718f1..cba91582 100644 --- a/src/inspector/inspector.py +++ b/src/inspector/inspector.py @@ -18,7 +18,6 @@ ExactlyOnceKafkaProduceHandler, KafkaMessageFetchException, ) -from src.base.utils import generate_unique_transactional_id from src.base.log_config import get_logger module_name = "data_inspection.inspector" @@ -88,9 +87,8 @@ def __init__(self) -> None: self.messages = [] self.anomalies = [] - transactional_id = generate_unique_transactional_id(module_name, KAFKA_BROKERS) self.kafka_consume_handler = ExactlyOnceKafkaConsumeHandler(CONSUME_TOPIC) - self.kafka_produce_handler = ExactlyOnceKafkaProduceHandler(transactional_id) + self.kafka_produce_handler = ExactlyOnceKafkaProduceHandler() # databases self.batch_timestamps = ClickHouseKafkaSender("batch_timestamps") @@ -100,6 +98,17 @@ def __init__(self) -> None: self.suspicious_batches_to_batch = ClickHouseKafkaSender( "suspicious_batches_to_batch" ) + self.logline_timestamps = ClickHouseKafkaSender("logline_timestamps") + self.fill_levels = ClickHouseKafkaSender("fill_levels") + + self.fill_levels.insert( + dict( + timestamp=datetime.now(), + stage=module_name, + entry_type="total_loglines", + entry_count=0, + ) + ) def get_and_fill_data(self) -> None: """Consumes data from KafkaConsumeHandler and stores it for processing.""" @@ -130,6 +139,15 @@ def get_and_fill_data(self) -> None: ) ) + self.fill_levels.insert( + dict( + timestamp=datetime.now(), + stage=module_name, + entry_type="total_loglines", + entry_count=len(self.messages), + ) + ) + if not self.messages: logger.info( "Received message:\n" @@ -463,12 +481,6 @@ def send_data(self): batch_schema = marshmallow_dataclass.class_schema(Batch)() - self.kafka_produce_handler.produce( - topic=PRODUCE_TOPIC, - data=batch_schema.dumps(data_to_send), - key=key, - ) - self.suspicious_batch_timestamps.insert( dict( suspicious_batch_id=suspicious_batch_id, @@ -480,6 +492,12 @@ def send_data(self): message_count=len(value), ) ) + + self.kafka_produce_handler.produce( + topic=PRODUCE_TOPIC, + data=batch_schema.dumps(data_to_send), + key=key, + ) else: # subnet is not suspicious self.batch_timestamps.insert( dict( @@ -492,6 +510,30 @@ def send_data(self): ) ) + logline_ids = set() + for message in self.messages: + logline_ids.add(message["logline_id"]) + + for logline_id in logline_ids: + self.logline_timestamps.insert( + dict( + logline_id=logline_id, + stage=module_name, + status="filtered_out", + timestamp=datetime.now(), + is_active=False, + ) + ) + + self.fill_levels.insert( + dict( + timestamp=datetime.now(), + stage=module_name, + entry_type="total_loglines", + entry_count=0, + ) + ) + def main(one_iteration: bool = False): """ diff --git a/src/logcollector/batch_handler.py b/src/logcollector/batch_handler.py index 34d9e5f7..6826a55b 100644 --- a/src/logcollector/batch_handler.py +++ b/src/logcollector/batch_handler.py @@ -11,7 +11,7 @@ from src.base.data_classes.batch import Batch from src.base.clickhouse_kafka_sender import ClickHouseKafkaSender from src.base.kafka_handler import ExactlyOnceKafkaProduceHandler -from src.base.utils import setup_config, generate_unique_transactional_id +from src.base.utils import setup_config from src.base.log_config import get_logger module_name = "log_collection.batch_handler" @@ -20,6 +20,7 @@ config = setup_config() BATCH_SIZE = config["pipeline"]["log_collection"]["batch_handler"]["batch_size"] BATCH_TIMEOUT = config["pipeline"]["log_collection"]["batch_handler"]["batch_timeout"] +TIMESTAMP_FORMAT = config["environment"]["timestamp_format"] PRODUCE_TOPIC = config["environment"]["kafka_topics"]["pipeline"][ "batch_sender_to_prefilter" ] @@ -32,9 +33,8 @@ class BufferedBatch: - """ - Data structure for managing the batch, buffer, and timestamps. The batch contains the latest messages and a - buffer that stores the previous batch messages. Also sorts the batches and can return timestamps. + """Data structure for managing the batch, buffer, and timestamps. The batch contains the latest messages and a + buffer that stores the previous batch messages. Sorts the batches and can return timestamps. """ def __init__(self): @@ -45,15 +45,32 @@ def __init__(self): # databases self.logline_to_batches = ClickHouseKafkaSender("logline_to_batches") self.batch_timestamps = ClickHouseKafkaSender("batch_timestamps") + self.fill_levels = ClickHouseKafkaSender("fill_levels") + + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines_in_batches", + entry_count=0, + ) + ) + + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines_in_buffer", + entry_count=0, + ) + ) def add_message(self, key: str, logline_id: uuid.UUID, message: str) -> None: - """ - Adds a given message to the messages list of the given key. If the key already exists, the message is simply - added, otherwise, the key is created. + """Adds message to the key. If the key does not exist yet, it is created first. Args: - logline_id (uuid.UUID): Logline ID of the added message key (str): Key to which the message is added + logline_id (uuid.UUID): Logline ID of the message message (str): Message to be added """ if key in self.batch: # key already has messages associated @@ -74,7 +91,7 @@ def add_message(self, key: str, logline_id: uuid.UUID, message: str) -> None: status="waiting", timestamp=datetime.datetime.now(), is_active=True, - message_count=self.get_number_of_messages(key), + message_count=self.get_message_count_for_batch_key(key), ) ) @@ -102,153 +119,48 @@ def add_message(self, key: str, logline_id: uuid.UUID, message: str) -> None: ) ) - def get_number_of_messages(self, key: str) -> int: - """ - Returns the number of entries in the batch of the latest messages. + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines_in_batches", + entry_count=self.get_message_count_for_batch(), + ) + ) - Args: - key (str): Key for which to return the number of messages + def get_message_count_for_batch(self) -> int: + """Returns the number of all batch entries as a sum over all key's batch entries.""" + return sum(len(key_entry) for key_entry in self.batch.values()) - Returns: - Number of messages associated with the given key as integer + def get_message_count_for_buffer(self) -> int: + """Returns the number of all buffered entries as a sum over all key's buffer entries.""" + return sum(len(key_entry) for key_entry in self.buffer.values()) + + def get_message_count_for_batch_key(self, key: str) -> int: + """Returns the number of all batch messages for a given key. + + Args: + key (str): Key for which message count is returned """ if key in self.batch: return len(self.batch[key]) return 0 - def get_number_of_buffered_messages(self, key: str) -> int: - """ - Returns the number of entries in the buffer of the latest messages. + def get_message_count_for_buffer_key(self, key: str) -> int: + """Returns the number of all buffered messages for a given key. Args: - key (str): Key for which to return the number of messages in the buffer - - Returns: - Number of messages in buffer associated with the given key as integer + key (str): Key for which message count is returned """ if key in self.buffer: return len(self.buffer[key]) return 0 - @staticmethod - def sort_messages( - data: list[tuple[str, str]], timestamp_format: str = "%Y-%m-%dT%H:%M:%S.%fZ" - ) -> list[str]: - """ - Sorts the given list of loglines by their respective timestamps, in ascending order. - - Args: - timestamp_format (str): Format of the timestamps - data (list[tuple[str, str]]): List of loglines to be sorted, with the tuple of strings consisting of 1. the - timestamps of the message and 2. the full message (unchanged, i.e. including - the respective timestamp) - - Returns: - List of log lines as strings sorted by timestamps (ascending) - """ - sorted_data = sorted( - data, key=lambda x: datetime.datetime.strptime(x[0], timestamp_format) - ) - loglines = [message for _, message in sorted_data] - - return loglines - - @staticmethod - def extract_tuples_from_json_formatted_strings( - data: list[str], - ) -> list[tuple[str, str]]: - """ - Args: - data (list[str]): Input list of strings to be prepared - - Returns: - Tuple with timestamps and log lines, which is needed for :func:'sort_messages'. - """ - tuples = [] - - for item in data: - record = json.loads(item) - - timestamp = record.get("timestamp", "") - tuples.append((str(timestamp), item)) - - return tuples - - def get_first_timestamp_of_buffer(self, key: str) -> str | None: - """ - Get the first timestamp of the buffer messages list. - - Returns: - First timestamp of the list of buffer messages for the given key, None if the key's list is empty - """ - entries = self.buffer.get(key) - - return json.loads(entries[0])["timestamp"] if entries and entries[0] else None - - def get_last_timestamp_of_buffer(self, key: str) -> str | None: - """ - Get the last timestamp of the buffer messages list. - - Returns: - Last timestamp of the list of buffer messages for the given key, None if the key's list is empty - """ - entries = self.buffer.get(key) - - return json.loads(entries[-1])["timestamp"] if entries and entries[-1] else None - - def get_first_timestamp_of_batch(self, key: str) -> str | None: - """ - Get the first timestamp of the batch messages list. - - Returns: - First timestamp of the list of batch messages for the given key, None if the key's list is empty - """ - entries = self.batch.get(key) - - return json.loads(entries[0])["timestamp"] if entries and entries[0] else None - - def get_last_timestamp_of_batch(self, key: str) -> str | None: - """ - Get the last timestamp of the batch messages list. - - Returns: - Last timestamp of the list of batch messages for the given key, None if the key's list is empty - """ - entries = self.batch.get(key) - - return json.loads(entries[-1])["timestamp"] if entries and entries[-1] else None - - def sort_buffer(self, key: str): - """ - Sorts the buffer's messages by their timestamp, in ascending order. - - Args: - key (str): Key for which to sort entries - """ - if key in self.buffer: - self.buffer[key] = self.sort_messages( - self.extract_tuples_from_json_formatted_strings(self.buffer[key]) - ) - - def sort_batch(self, key: str): - """ - Sorts the batches messages by their timestamp, in ascending order. - - Args: - key (str): Key for which to sort entries - """ - if key in self.batch: - self.batch[key] = self.sort_messages( - self.extract_tuples_from_json_formatted_strings(self.batch[key]) - ) - def complete_batch(self, key: str) -> dict: - """ - Completes the batch and returns a full data packet including timestamps and messages. Depending on the - stored data, either both batches are added to the packet, or just the latest messages. Raises a ValueError - if no data is available. + """Completes the batch and returns a full data packet including timestamps and messages. Depending on the + stored data, either both batches are added to the packet, or just the latest messages. Args: key (str): Key for which to complete the current batch and return data packet @@ -263,18 +175,46 @@ def complete_batch(self, key: str) -> dict: if self.batch.get(key): if not self.buffer.get(key): # Variant 1: Only batch has entries logger.debug("Variant 1: Only batch has entries. Sending...") - self.sort_batch(key) + self._sort_batch(key) buffer_data = [] - begin_timestamp = self.get_first_timestamp_of_batch(key) + + def _get_first_timestamp_of_batch() -> str | None: + entries = self.batch.get(key) + return ( + json.loads(entries[0])["timestamp"] + if entries and entries[0] + else None + ) + + begin_timestamp = _get_first_timestamp_of_batch() + else: # Variant 2: Buffer and batch have entries logger.debug("Variant 2: Buffer and batch have entries. Sending...") - self.sort_batch(key) - self.sort_buffer(key) + self._sort_batch(key) + self._sort_buffer(key) buffer_data = self.buffer[key] - begin_timestamp = self.get_first_timestamp_of_buffer(key) + + def _get_first_timestamp_of_buffer() -> str | None: + entries = self.buffer.get(key) + return ( + json.loads(entries[0])["timestamp"] + if entries and entries[0] + else None + ) + + begin_timestamp = _get_first_timestamp_of_buffer() batch_id = self.batch_id.get(key) + def _get_last_timestamp_of_batch() -> str | None: + entries = self.batch.get(key) + + return ( + json.loads(entries[-1])["timestamp"] + if entries and entries[-1] + else None + ) + data = { "batch_id": batch_id, "begin_timestamp": datetime.datetime.strptime( @@ -282,7 +222,7 @@ def complete_batch(self, key: str) -> dict: "%Y-%m-%dT%H:%M:%S.%fZ", ), "end_timestamp": datetime.datetime.strptime( - self.get_last_timestamp_of_batch(key), + _get_last_timestamp_of_batch(), "%Y-%m-%dT%H:%M:%S.%fZ", ), "data": buffer_data + self.batch[key], @@ -295,7 +235,7 @@ def complete_batch(self, key: str) -> dict: status="completed", timestamp=datetime.datetime.now(), is_active=True, - message_count=self.get_number_of_messages(key), + message_count=self.get_message_count_for_batch_key(key), ) ) @@ -306,14 +246,43 @@ def complete_batch(self, key: str) -> dict: # Batch ID is not needed anymore del self.batch_id[key] + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines_in_batches", + entry_count=self.get_message_count_for_batch(), + ) + ) + + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines_in_buffer", + entry_count=self.get_message_count_for_buffer(), + ) + ) + return data - if self.buffer: # Variant 3: Only buffer has entries + if self.buffer.get(key): # Variant 3: Only buffer has entries logger.debug("Variant 3: Only buffer has entries.") logger.debug( "Deleting buffer data (has no influence on analysis since it was too long ago)..." ) + del self.buffer[key] + + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines_in_buffer", + entry_count=self.get_message_count_for_buffer(), + ) + ) + else: # Variant 4: No data exists logger.debug("Variant 4: No data exists. Nothing to send.") @@ -336,47 +305,73 @@ def get_stored_keys(self) -> set: return keys_set.copy() + @staticmethod + def _extract_tuples_from_json_formatted_strings( + data: list[str], + ) -> list[tuple[str, str]]: + tuples = [] + + for item in data: + record = json.loads(item) + + timestamp = record.get("timestamp", "") + tuples.append((str(timestamp), item)) + + return tuples + + @staticmethod + def _sort_by_timestamp( + data: list[tuple[str, str]], + timestamp_format: str = TIMESTAMP_FORMAT, + ) -> list[str]: + sorted_data = sorted( + data, key=lambda x: datetime.datetime.strptime(x[0], timestamp_format) + ) + loglines = [message for _, message in sorted_data] + + return loglines + + def _sort_batch(self, key: str): + if key in self.batch: + self.batch[key] = self._sort_by_timestamp( + self._extract_tuples_from_json_formatted_strings(self.batch[key]) + ) + + def _sort_buffer(self, key: str): + if key in self.buffer: + self.buffer[key] = self._sort_by_timestamp( + self._extract_tuples_from_json_formatted_strings(self.buffer[key]) + ) + class BufferedBatchSender: - """ - Adds messages to the :class:`BufferedBatch` and sends them after a timer ran out or the respective batch is full. - """ + """Adds messages to the :class:`BufferedBatch` and sends them after a timer ran out or a key's batch is full.""" def __init__(self): self.topic = PRODUCE_TOPIC self.batch = BufferedBatch() self.timer = None - transactional_id = generate_unique_transactional_id(module_name, KAFKA_BROKERS) - self.kafka_produce_handler = ExactlyOnceKafkaProduceHandler(transactional_id) + self.kafka_produce_handler = ExactlyOnceKafkaProduceHandler() # databases self.logline_timestamps = ClickHouseKafkaSender("logline_timestamps") def __del__(self): - logger.debug(f"Closing KafkaBatchSender ({self.topic=})...") if self.timer: - logger.debug("Timer is active. Cancelling timer...") self.timer.cancel() - logger.debug("Timer cancelled.") self._send_all_batches(reset_timer=False) - logger.debug(f"Closed KafkaBatchSender ({self.topic=}).") - def add_message(self, key: str, message: str) -> None: - """ - Adds the given message to the key specific batch. Checks if the batch is full. If so, it is sent. In the first - execution, the timer starts. The same timer is used for all keys. + """Adds the message to the key's batch and sends it if it is full. In the first execution, a timer starts + whose timeout triggers sending for all batches. Args: message (str): Message to be added to the batch key (str): Key of the message (e.g. subnet of client IP address in a log message) """ - logger.debug(f"Adding message '{message}' to batch.") - logline_id = json.loads(message).get("logline_id") - self.logline_timestamps.insert( dict( logline_id=logline_id, @@ -388,7 +383,6 @@ def add_message(self, key: str, message: str) -> None: ) self.batch.add_message(key, logline_id, message) - self.logline_timestamps.insert( dict( logline_id=logline_id, @@ -400,12 +394,9 @@ def add_message(self, key: str, message: str) -> None: ) logger.debug(f"Batch: {self.batch.batch}") - number_of_messages_for_key = self.batch.get_number_of_messages(key) + number_of_messages_for_key = self.batch.get_message_count_for_batch_key(key) if number_of_messages_for_key >= BATCH_SIZE: - logger.debug( - f"Batch for {key=} is full. Calling _send_batch_for_key({key})..." - ) self._send_batch_for_key(key) logger.info( f"Full batch: Successfully sent batch messages for subnet_id {key}.\n" @@ -414,25 +405,19 @@ def add_message(self, key: str, message: str) -> None: elif not self.timer: # First time setting the timer self._reset_timer() - logger.debug(f"Message '{message}' successfully added to batch for {key=}.") - def _send_all_batches(self, reset_timer: bool = True) -> None: number_of_keys = 0 - total_number_of_batch_messages = 0 - total_number_of_buffer_messages = 0 + total_number_of_batch_messages = self.batch.get_message_count_for_batch() + total_number_of_buffer_messages = self.batch.get_message_count_for_buffer() for key in self.batch.get_stored_keys(): number_of_keys += 1 - total_number_of_batch_messages += self.batch.get_number_of_messages(key) - total_number_of_buffer_messages += ( - self.batch.get_number_of_buffered_messages(key) - ) self._send_batch_for_key(key) if reset_timer: self._reset_timer() - if not total_number_of_batch_messages: + if total_number_of_batch_messages == 0: return if number_of_keys == 1: @@ -471,19 +456,8 @@ def _send_data_packet(self, key: str, data: dict) -> None: ) def _reset_timer(self) -> None: - """ - Resets an existing or starts a new timer with the globally set batch timeout. In the case of a timeout, - all batches are sent. The timer serves as a backup so that all batches are cleared sometimes (prevents subnets - with few messages from never being analyzed). - """ - logger.debug("Resetting timer...") if self.timer: - logger.debug("Cancelling active timer...") self.timer.cancel() - else: - logger.debug("No timer active.") - logger.debug("Starting new timer...") self.timer = Timer(BATCH_TIMEOUT, self._send_all_batches) self.timer.start() - logger.debug("Successfully started new timer.") diff --git a/src/logcollector/collector.py b/src/logcollector/collector.py index 3ceebd59..2873419d 100644 --- a/src/logcollector/collector.py +++ b/src/logcollector/collector.py @@ -33,9 +33,8 @@ class LogCollector: - """ - Consumes incoming log lines from the :class:`LogServer`. Validates all data fields by type and - value, invalid loglines are discarded. All valid loglines are sent to the Batch Sender. + """Consumes incoming log lines from the :class:`LogServer`. Validates all data fields by type and + value, invalid loglines are discarded. All valid loglines are sent to the batch sender. """ def __init__(self) -> None: @@ -48,11 +47,19 @@ def __init__(self) -> None: self.failed_dns_loglines = ClickHouseKafkaSender("failed_dns_loglines") self.dns_loglines = ClickHouseKafkaSender("dns_loglines") self.logline_timestamps = ClickHouseKafkaSender("logline_timestamps") + self.fill_levels = ClickHouseKafkaSender("fill_levels") + + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines", + entry_count=0, + ) + ) async def start(self) -> None: - """ - Starts fetching messages from Kafka and sending them to the :class:`Prefilter`. - """ + """Starts fetching messages from Kafka and sending them to the :class:`Prefilter`.""" logger.info( "LogCollector started:\n" f" ⤷ receiving on Kafka topic '{CONSUME_TOPIC}'" @@ -73,10 +80,8 @@ async def start(self) -> None: logger.info("LogCollector stopped.") async def fetch(self) -> None: - """ - Starts a loop to continuously listen on the configured Kafka topic. If a message is consumed, it is - decoded and stored. - """ + """Starts a loop to continuously listen on the configured Kafka topic. If a message is consumed, it is + decoded and stored.""" loop = asyncio.get_running_loop() while True: @@ -88,16 +93,23 @@ async def fetch(self) -> None: await self.store(datetime.datetime.now(), value) async def send(self) -> None: - """ - Continuously sends the next logline in JSON format to the BatchSender, where it is stored in - a temporary batch before being sent to the Prefilter. Adds a subnet ID to the message, that it retrieves - from the client's IP address. + """Continuously sends the next logline in JSON format to the BatchSender, where it is stored in + a temporary batch before being sent to the :class:`Prefilter`. Adds the subnet ID to the message. """ try: while True: if not self.loglines.empty(): timestamp_in, logline = await self.loglines.get() + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines", + entry_count=self.loglines.qsize(), + ) + ) + try: fields = self.logline_handler.validate_logline_and_get_fields_as_json( logline @@ -113,7 +125,7 @@ async def send(self) -> None: ) continue - subnet_id = self.get_subnet_id( + subnet_id = self._get_subnet_id( ipaddress.ip_address(fields.get("client_ip")) ) @@ -150,10 +162,6 @@ async def send(self) -> None: message_fields = fields.copy() message_fields["logline_id"] = str(logline_id) - self.batch_handler.add_message( - subnet_id, json.dumps(message_fields) - ) - self.logline_timestamps.insert( dict( logline_id=logline_id, @@ -163,16 +171,30 @@ async def send(self) -> None: is_active=True, ) ) + + self.batch_handler.add_message( + subnet_id, json.dumps(message_fields) + ) logger.debug(f"Sent: '{logline}'") else: await asyncio.sleep(0.1) except KeyboardInterrupt: while not self.loglines.empty(): logline = await self.loglines.get() + + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines", + entry_count=self.loglines.qsize(), + ) + ) + fields = self.logline_handler.validate_logline_and_get_fields_as_json( logline ) - subnet_id = self.get_subnet_id( + subnet_id = self._get_subnet_id( ipaddress.ip_address(fields.get("client_ip")) ) @@ -181,8 +203,7 @@ async def send(self) -> None: logger.info("Stopped LogCollector.") async def store(self, timestamp_in: datetime.datetime, message: str): - """ - Stores the given message temporarily. + """Stores the message temporarily. Args: timestamp_in (datetime.datetime): Timestamp of entering the pipeline @@ -190,8 +211,17 @@ async def store(self, timestamp_in: datetime.datetime, message: str): """ await self.loglines.put((timestamp_in, message)) + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines", + entry_count=self.loglines.qsize(), + ) + ) + @staticmethod - def get_subnet_id(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> str: + def _get_subnet_id(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> str: """ Returns the subnet ID of an IP address. diff --git a/src/logserver/server.py b/src/logserver/server.py index 19d9ce97..79a78a1b 100644 --- a/src/logserver/server.py +++ b/src/logserver/server.py @@ -12,7 +12,6 @@ ExactlyOnceKafkaProduceHandler, ) from src.base.clickhouse_kafka_sender import ClickHouseKafkaSender -from src.base.utils import generate_unique_transactional_id from src.base.utils import setup_config from src.base.log_config import get_logger @@ -40,10 +39,8 @@ class LogServer: """ def __init__(self) -> None: - transactional_id = generate_unique_transactional_id(module_name, KAFKA_BROKERS) - self.kafka_consume_handler = SimpleKafkaConsumeHandler(CONSUME_TOPIC) - self.kafka_produce_handler = ExactlyOnceKafkaProduceHandler(transactional_id) + self.kafka_produce_handler = ExactlyOnceKafkaProduceHandler() # databases self.server_logs = ClickHouseKafkaSender("server_logs") diff --git a/src/mock/log_generator.py b/src/mock/log_generator.py index 45430537..1b7d8211 100644 --- a/src/mock/log_generator.py +++ b/src/mock/log_generator.py @@ -16,7 +16,7 @@ def random_ipv6(): MAX_IPV4 = ipaddress.IPv4Address._ALL_ONES # 2 ** 32 - 1 MAX_IPV6 = ipaddress.IPv6Address._ALL_ONES # 2 ** 128 - 1 IP = [random_ipv4, random_ipv6] -RECORD_TYPES = ["AAAA", "A", "PR", "CNAME"] +RECORD_TYPES = ["AAAA", "A", "AAAA", "A", "AAAA", "A", "AAAA", "A", "PR", "CNAME"] def generate_dns_log_line(domain: str): diff --git a/src/monitoring/clickhouse_connector.py b/src/monitoring/clickhouse_connector.py index efe652a2..39c9516e 100644 --- a/src/monitoring/clickhouse_connector.py +++ b/src/monitoring/clickhouse_connector.py @@ -301,6 +301,7 @@ def __init__(self): "alert_timestamp", "suspicious_batch_id", "overall_score", + "domain_names", "result", ] @@ -312,6 +313,7 @@ def insert( alert_timestamp: datetime.datetime, suspicious_batch_id: uuid.UUID, overall_score: float, + domain_names: str, result: str, ) -> None: self._add_to_batch( @@ -320,6 +322,35 @@ def insert( alert_timestamp, suspicious_batch_id, overall_score, + domain_names, result, ] ) + + +class FillLevelsConnector(ClickHouseConnector): + def __init__(self): + column_names = [ + "timestamp", + "stage", + "entry_type", + "entry_count", + ] + + super().__init__("fill_levels", column_names) + + def insert( + self, + timestamp: datetime.datetime, + stage: str, + entry_type: str, + entry_count: int, + ) -> None: + self._add_to_batch( + [ + timestamp, + stage, + entry_type, + entry_count, + ] + ) diff --git a/src/monitoring/monitoring_agent.py b/src/monitoring/monitoring_agent.py index 030dca24..9ce9b135 100644 --- a/src/monitoring/monitoring_agent.py +++ b/src/monitoring/monitoring_agent.py @@ -15,7 +15,10 @@ logger = get_logger() CONFIG = setup_config() -CREATE_TABLES_DIRECTORY = "src/monitoring/create_tables" # TODO: Get from config +CREATE_TABLES_DIRECTORY = "docker/create_tables" # TODO: Get from config +CLICKHOUSE_HOSTNAME = CONFIG["environment"]["monitoring"]["clickhouse_server"][ + "hostname" +] def prepare_all_tables(): @@ -49,6 +52,7 @@ def __init__(self): "suspicious_batches_to_batch": SuspiciousBatchesToBatchConnector(), "suspicious_batch_timestamps": SuspiciousBatchTimestampsConnector(), "alerts": AlertsConnector(), + "fill_levels": FillLevelsConnector(), } self.topics = [f"clickhouse_{table_name}" for table_name in self.connectors] @@ -78,7 +82,6 @@ async def start(self): def main(): - prepare_all_tables() clickhouse_consumer = MonitoringAgent() asyncio.run(clickhouse_consumer.start()) diff --git a/src/prefilter/prefilter.py b/src/prefilter/prefilter.py index c766d3a4..2703b306 100644 --- a/src/prefilter/prefilter.py +++ b/src/prefilter/prefilter.py @@ -1,5 +1,4 @@ import datetime -import json import os import sys import uuid @@ -15,7 +14,6 @@ ExactlyOnceKafkaProduceHandler, KafkaMessageFetchException, ) -from src.base.utils import generate_unique_transactional_id from src.base.log_config import get_logger from src.base.utils import setup_config @@ -54,12 +52,21 @@ def __init__(self): self.logline_handler = LoglineHandler() self.kafka_consume_handler = ExactlyOnceKafkaConsumeHandler(CONSUME_TOPIC) - transactional_id = generate_unique_transactional_id(module_name, KAFKA_BROKERS) - self.kafka_produce_handler = ExactlyOnceKafkaProduceHandler(transactional_id) + self.kafka_produce_handler = ExactlyOnceKafkaProduceHandler() # databases self.batch_timestamps = ClickHouseKafkaSender("batch_timestamps") self.logline_timestamps = ClickHouseKafkaSender("logline_timestamps") + self.fill_levels = ClickHouseKafkaSender("fill_levels") + + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines", + entry_count=0, + ) + ) def get_and_fill_data(self) -> None: """ @@ -88,6 +95,15 @@ def get_and_fill_data(self) -> None: ) ) + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines", + entry_count=len(self.unfiltered_data), + ) + ) + if not self.unfiltered_data: logger.info( f"Received message:\n" @@ -121,6 +137,15 @@ def filter_by_error(self) -> None: ) ) + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines", + entry_count=len(self.filtered_data), + ) + ) + def send_filtered_data(self): """ Sends the filtered data if available via the :class:`KafkaProduceHandler`. @@ -146,6 +171,15 @@ def send_filtered_data(self): ) ) + self.fill_levels.insert( + dict( + timestamp=datetime.datetime.now(), + stage=module_name, + entry_type="total_loglines", + entry_count=0, + ) + ) + batch_schema = marshmallow_dataclass.class_schema(Batch)() self.kafka_produce_handler.produce( topic=PRODUCE_TOPIC, diff --git a/tests/clickhouse/test_clickhouse_connector.py b/tests/clickhouse/test_clickhouse_connector.py index bba9a893..48e12bec 100644 --- a/tests/clickhouse/test_clickhouse_connector.py +++ b/tests/clickhouse/test_clickhouse_connector.py @@ -675,6 +675,7 @@ def test_init(self, mock_clickhouse_batch_sender): "alert_timestamp", "suspicious_batch_id", "overall_score", + "domain_names", "result", ] @@ -698,6 +699,7 @@ def test_insert_all_given(self, mock_clickhouse_batch_sender): alert_timestamp = datetime.datetime(2034, 12, 13, 12, 35, 35, 542635) suspicious_batch_id = uuid.UUID("7299539b-6215-4f6b-b39f-69335aafbeff") overall_score = 15.4 + domain_names = "random.de" result = "test" sut = AlertsConnector() @@ -709,6 +711,7 @@ def test_insert_all_given(self, mock_clickhouse_batch_sender): alert_timestamp=alert_timestamp, suspicious_batch_id=suspicious_batch_id, overall_score=overall_score, + domain_names=domain_names, result=result, ) @@ -719,10 +722,71 @@ def test_insert_all_given(self, mock_clickhouse_batch_sender): datetime.datetime(2034, 12, 13, 12, 35, 35, 542635), uuid.UUID("7299539b-6215-4f6b-b39f-69335aafbeff"), 15.4, + "random.de", "test", ] ) +class TestFillLevelsConnector(unittest.TestCase): + @patch("src.monitoring.clickhouse_connector.ClickHouseBatchSender") + def test_init(self, mock_clickhouse_batch_sender): + # Arrange + mock_clickhouse_batch_sender_instance = MagicMock() + mock_clickhouse_batch_sender.return_value = ( + mock_clickhouse_batch_sender_instance + ) + + expected_table_name = "fill_levels" + expected_column_names = [ + "timestamp", + "stage", + "entry_type", + "entry_count", + ] + + # Act + sut = FillLevelsConnector() + + # Assert + self.assertEqual(expected_table_name, sut._table_name) + self.assertEqual(expected_column_names, sut._column_names) + self.assertEqual(mock_clickhouse_batch_sender_instance, sut._batch_sender) + + mock_clickhouse_batch_sender.assert_called_once_with( + table_name=expected_table_name, + column_names=expected_column_names, + ) + + @patch("src.monitoring.clickhouse_connector.ClickHouseBatchSender") + def test_insert_all_given(self, mock_clickhouse_batch_sender): + # Arrange + timestamp = datetime.datetime(2034, 12, 13, 12, 35, 35, 542635) + stage = "test_stage" + entry_type = "test_entry_type" + entry_count = 17 + + sut = FillLevelsConnector() + + with patch.object(sut, "_add_to_batch", MagicMock()) as mock_add_to_batch: + # Act + sut.insert( + timestamp=timestamp, + stage=stage, + entry_type=entry_type, + entry_count=entry_count, + ) + + # Assert + mock_add_to_batch.assert_called_once_with( + [ + datetime.datetime(2034, 12, 13, 12, 35, 35, 542635), + "test_stage", + "test_entry_type", + 17, + ] + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/detector/test_detector.py b/tests/detector/test_detector.py index 6bc878dc..57963f35 100644 --- a/tests/detector/test_detector.py +++ b/tests/detector/test_detector.py @@ -13,7 +13,8 @@ class TestSha256Sum(unittest.TestCase): @patch("src.detector.detector.ExactlyOnceKafkaConsumeHandler") - def test_sha256_empty_file(self, mock_kafka_consume_handler): + @patch("src.detector.detector.ClickHouseKafkaSender") + def test_sha256_empty_file(self, mock_clickhouse, mock_kafka_consume_handler): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -23,7 +24,10 @@ def test_sha256_empty_file(self, mock_kafka_consume_handler): sut._sha256sum("") @patch("src.detector.detector.ExactlyOnceKafkaConsumeHandler") - def test_sha256_not_existing_file(self, mock_kafka_consume_handler): + @patch("src.detector.detector.ClickHouseKafkaSender") + def test_sha256_not_existing_file( + self, mock_clickhouse, mock_kafka_consume_handler + ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -49,7 +53,8 @@ def setUp(self): "https://heibox.uni-heidelberg.de/d/0d5cbcbe16cd46a58021/", ) @patch("src.detector.detector.ExactlyOnceKafkaConsumeHandler") - def test_get_model(self, mock_kafka_consume_handler): + @patch("src.detector.detector.ClickHouseKafkaSender") + def test_get_model(self, mock_clickhouse, mock_kafka_consume_handler): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -73,7 +78,8 @@ def setUp(self): "https://heibox.uni-heidelberg.de/d/0d5cbcbe16cd46a58021/", ) @patch("src.detector.detector.ExactlyOnceKafkaConsumeHandler") - def test_get_model(self, mock_kafka_consume_handler): + @patch("src.detector.detector.ClickHouseKafkaSender") + def test_get_model(self, mock_clickhouse, mock_kafka_consume_handler): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -205,7 +211,10 @@ def test_get_data_with_return_data( @patch("src.detector.detector.logger") @patch("src.detector.detector.ExactlyOnceKafkaConsumeHandler") - def test_get_data_while_busy(self, mock_kafka_consume_handler, mock_logger): + @patch("src.detector.detector.ClickHouseKafkaSender") + def test_get_data_while_busy( + self, mock_clickhouse, mock_kafka_consume_handler, mock_logger + ): begin = datetime.now() end = begin + timedelta(0, 3) test_batch = Batch( @@ -265,6 +274,7 @@ def test_save_warning(self, mock_clickhouse, mock_kafka_consume_handler): "sha256": "ba1f718179191348fe2abd51644d76191d42a5d967c6844feb3371b6f798bf06", }, ] + sut.messages = [{"logline_id": "test_id"}] open_mock = mock_open() with patch("src.detector.detector.open", open_mock, create=True): sut.send_warning() @@ -290,6 +300,7 @@ def test_save_empty_warning(self, mock_clickhouse, mock_kafka_consume_handler): sut = Detector() sut.warnings = [] + sut.messages = [{"logline_id": "test_id"}] open_mock = mock_open() with patch("src.detector.detector.open", open_mock, create=True): sut.send_warning() @@ -306,7 +317,8 @@ def test_save_empty_warning(self, mock_clickhouse, mock_kafka_consume_handler): "https://heibox.uni-heidelberg.de/d/0d5cbcbe16cd46a58021/", ) @patch("src.detector.detector.ExactlyOnceKafkaConsumeHandler") - def test_save_warning_error(self, mock_kafka_consume_handler): + @patch("src.detector.detector.ClickHouseKafkaSender") + def test_save_warning_error(self, mock_clickhouse, mock_kafka_consume_handler): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -331,8 +343,9 @@ def setUp(self): @patch("src.detector.detector.logger") @patch("src.detector.detector.ExactlyOnceKafkaConsumeHandler") + @patch("src.detector.detector.ClickHouseKafkaSender") def test_clear_data_without_existing_data( - self, mock_kafka_consume_handler, mock_logger + self, mock_clickhouse, mock_kafka_consume_handler, mock_logger ): begin = datetime.now() end = begin + timedelta(0, 3) @@ -355,8 +368,9 @@ def test_clear_data_without_existing_data( @patch("src.detector.detector.logger") @patch("src.detector.detector.ExactlyOnceKafkaConsumeHandler") + @patch("src.detector.detector.ClickHouseKafkaSender") def test_clear_data_with_existing_data( - self, mock_kafka_consume_handler, mock_logger + self, mock_clickhouse, mock_kafka_consume_handler, mock_logger ): begin = datetime.now() end = begin + timedelta(0, 3) diff --git a/tests/inspector/test_inspector.py b/tests/inspector/test_inspector.py index 418124da..8485444a 100644 --- a/tests/inspector/test_inspector.py +++ b/tests/inspector/test_inspector.py @@ -38,9 +38,12 @@ def get_batch(data): class TestInit(unittest.TestCase): @patch("src.inspector.inspector.CONSUME_TOPIC", "test_topic") + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") - def test_init(self, mock_kafka_consume_handler, mock_produce_handler): + def test_init( + self, mock_kafka_consume_handler, mock_produce_handler, mock_clickhouse + ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance mock_produce_handler_instance = MagicMock() @@ -140,10 +143,11 @@ def test_get_data_with_no_return_data( self.assertEqual(end, sut.end_timestamp) self.assertEqual([], sut.messages) + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") def test_get_data_while_busy( - self, mock_kafka_consume_handler, mock_produce_handler + self, mock_kafka_consume_handler, mock_produce_handler, mock_clickhouse ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -166,10 +170,11 @@ def test_get_data_while_busy( class TestClearData(unittest.TestCase): + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") def test_clear_data_without_existing_data( - self, mock_kafka_consume_handler, mock_produce_handler + self, mock_kafka_consume_handler, mock_produce_handler, mock_clickhouse ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -187,10 +192,11 @@ def test_clear_data_without_existing_data( self.assertEqual([], sut.messages) + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") def test_clear_data_with_existing_data( - self, mock_kafka_consume_handler, mock_produce_handler + self, mock_kafka_consume_handler, mock_kafka_produce_handler, mock_clickhouse ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -200,7 +206,7 @@ def test_clear_data_with_existing_data( "data": [], } mock_produce_handler_instance = MagicMock() - mock_produce_handler.return_value = mock_produce_handler_instance + mock_kafka_produce_handler.return_value = mock_produce_handler_instance sut = Inspector() sut.messages = ["test_data"] @@ -215,11 +221,14 @@ def test_clear_data_with_existing_data( class TestDataFunction(unittest.TestCase): + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") @patch("src.inspector.inspector.TIME_TYPE", "ms") @patch("src.inspector.inspector.TIME_RANGE", 1) - def test_count_errors(self, mock_kafka_consume_handler, mock_produce_handler): + def test_count_errors( + self, mock_kafka_consume_handler, mock_produce_handler, mock_clickhouse + ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance mock_produce_handler_instance = MagicMock() @@ -238,11 +247,14 @@ def test_count_errors(self, mock_kafka_consume_handler, mock_produce_handler): sut._count_errors(messages, begin_timestamp, end_timestamp), ) + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") @patch("src.inspector.inspector.TIME_TYPE", "ms") @patch("src.inspector.inspector.TIME_RANGE", 1) - def test_mean_packet_size(self, mock_kafka_consume_handler, mock_produce_handler): + def test_mean_packet_size( + self, mock_kafka_consume_handler, mock_produce_handler, mock_clickhouse + ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance mock_produce_handler_instance = MagicMock() @@ -261,10 +273,11 @@ def test_mean_packet_size(self, mock_kafka_consume_handler, mock_produce_handler sut._mean_packet_size(messages, begin_timestamp, end_timestamp), ) + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") def test_count_errors_empty_messages( - self, mock_kafka_consume_handler, mock_produce_handler + self, mock_kafka_consume_handler, mock_produce_handler, mock_clickhouse ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -283,10 +296,11 @@ def test_count_errors_empty_messages( sut._count_errors([], begin_timestamp, end_timestamp), ) + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") def test_mean_packet_size_empty_messages( - self, mock_kafka_consume_handler, mock_produce_handler + self, mock_kafka_consume_handler, mock_produce_handler, mock_clickhouse ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -308,6 +322,7 @@ def test_mean_packet_size_empty_messages( class TestInspectFunction(unittest.TestCase): @patch("src.inspector.inspector.logger") + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") @patch( @@ -315,7 +330,11 @@ class TestInspectFunction(unittest.TestCase): None, ) def test_inspect_none_models( - self, mock_kafka_consume_handler, mock_produce_handler, mock_logger + self, + mock_kafka_consume_handler, + mock_produce_handler, + mock_clickhouse, + mock_logger, ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -331,6 +350,7 @@ def test_inspect_none_models( sut.inspect() @patch("src.inspector.inspector.logger") + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") @patch( @@ -338,7 +358,11 @@ def test_inspect_none_models( "", ) def test_inspect_empty_models( - self, mock_kafka_consume_handler, mock_produce_handler, mock_logger + self, + mock_kafka_consume_handler, + mock_produce_handler, + mock_clickhouse, + mock_logger, ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -760,6 +784,7 @@ def test_inspect_ensemble_invalid( sut.inspect() @patch("src.inspector.inspector.logger") + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") @patch( @@ -767,7 +792,11 @@ def test_inspect_ensemble_invalid( [{"model": "INVALID", "module": "streamad.model"}], ) def test_invalid_model_univariate( - self, mock_kafka_consume_handler, mock_produce_handler, mock_logger + self, + mock_kafka_consume_handler, + mock_produce_handler, + mock_clickhouse, + mock_logger, ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -779,6 +808,7 @@ def test_invalid_model_univariate( sut.inspect() @patch("src.inspector.inspector.logger") + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") @patch( @@ -787,7 +817,11 @@ def test_invalid_model_univariate( ) @patch("src.inspector.inspector.MODE", "multivariate") def test_invalid_model_multivariate( - self, mock_kafka_consume_handler, mock_produce_handler, mock_logger + self, + mock_kafka_consume_handler, + mock_produce_handler, + mock_clickhouse, + mock_logger, ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -799,6 +833,7 @@ def test_invalid_model_multivariate( sut.inspect() @patch("src.inspector.inspector.logger") + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") @patch( @@ -807,7 +842,11 @@ def test_invalid_model_multivariate( ) @patch("src.inspector.inspector.MODE", "ensemble") def test_invalid_model_ensemble( - self, mock_kafka_consume_handler, mock_produce_handler, mock_logger + self, + mock_kafka_consume_handler, + mock_produce_handler, + mock_clickhouse, + mock_logger, ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance @@ -818,10 +857,13 @@ def test_invalid_model_ensemble( with self.assertRaises(NotImplementedError): sut.inspect() + @patch("src.inspector.inspector.ClickHouseKafkaSender") @patch("src.inspector.inspector.ExactlyOnceKafkaProduceHandler") @patch("src.inspector.inspector.ExactlyOnceKafkaConsumeHandler") @patch("src.inspector.inspector.MODE", "INVALID") - def test_invalid_mode(self, mock_kafka_consume_handler, mock_produce_handler): + def test_invalid_mode( + self, mock_kafka_consume_handler, mock_produce_handler, mock_clickhouse + ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance mock_produce_handler_instance = MagicMock() @@ -843,13 +885,13 @@ def test_send( self, mock_clickhouse, mock_kafka_consume_handler, - mock_produce_handler, + mock_kafka_produce_handler, mock_logger, ): mock_kafka_consume_handler_instance = MagicMock() mock_kafka_consume_handler.return_value = mock_kafka_consume_handler_instance mock_produce_handler_instance = MagicMock() - mock_produce_handler.return_value = mock_produce_handler_instance + mock_kafka_produce_handler.return_value = mock_produce_handler_instance batch_schema = marshmallow_dataclass.class_schema(Batch)() sut = Inspector() @@ -868,7 +910,7 @@ def test_send( sut.send_data() mock_produce_handler_instance.produce.assert_called_once_with( - topic="pipeline.inspector_to_detector", + topic="pipeline-inspector_to_detector", data=batch_schema.dumps( { "batch_id": mock_batch_id, @@ -908,6 +950,7 @@ def test_send_not_suspicious( data["timestamp"] = datetime.strftime( sut.begin_timestamp + timedelta(0, 0, 1), TIMESTAMP_FORMAT ) + data["logline_id"] = uuid.UUID("99a427a6-ba3f-4aa2-b848-210d994d9108") sut.messages = [data] mock_batch_id = uuid.UUID("5ae0872e-5bb9-472c-8c37-8c173213a51f") with patch("src.inspector.inspector.uuid") as mock_uuid: diff --git a/tests/kafka/test_exactly_once_kafka_consume_handler.py b/tests/kafka/test_exactly_once_kafka_consume_handler.py index 381a53ad..a93d2fc8 100644 --- a/tests/kafka/test_exactly_once_kafka_consume_handler.py +++ b/tests/kafka/test_exactly_once_kafka_consume_handler.py @@ -10,11 +10,10 @@ from src.base.data_classes.batch import Batch from src.base.kafka_handler import ExactlyOnceKafkaConsumeHandler -CONSUMER_GROUP_ID = "test_group_id" +CONSUMER_GROUP_ID = "default_gid" class TestInit(unittest.TestCase): - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") @patch( "src.base.kafka_handler.KAFKA_BROKERS", [ @@ -32,8 +31,13 @@ class TestInit(unittest.TestCase): }, ], ) + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") @patch("src.base.kafka_handler.Consumer") - def test_init(self, mock_consumer): + def test_init(self, mock_consumer, mock_admin_client, mock_all_topics_created): mock_consumer_instance = Mock() mock_consumer.return_value = mock_consumer_instance @@ -50,10 +54,8 @@ def test_init(self, mock_consumer): self.assertEqual(mock_consumer_instance, sut.consumer) mock_consumer.assert_called_once_with(expected_conf) - mock_consumer_instance.assign.assert_called_once() + mock_consumer_instance.subscribe.assert_called_once() - @patch("src.base.kafka_handler.logger") - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") @patch( "src.base.kafka_handler.KAFKA_BROKERS", [ @@ -71,8 +73,13 @@ def test_init(self, mock_consumer): }, ], ) + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") @patch("src.base.kafka_handler.Consumer") - def test_init_fail(self, mock_consumer, mock_logger): + def test_init_fail(self, mock_consumer, mock_admin_client, mock_all_topics_created): mock_consumer_instance = Mock() mock_consumer.return_value = mock_consumer_instance @@ -84,7 +91,9 @@ def test_init_fail(self, mock_consumer, mock_logger): "enable.partition.eof": True, } - with patch.object(mock_consumer_instance, "assign", side_effect=KafkaException): + with patch.object( + mock_consumer_instance, "subscribe", side_effect=KafkaException + ): with self.assertRaises(KafkaException): sut = ExactlyOnceKafkaConsumeHandler(topics="test_topic") @@ -114,7 +123,12 @@ class TestConsume(unittest.TestCase): ], ) @patch("src.base.kafka_handler.Consumer") - def setUp(self, mock_consumer): + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") + def setUp(self, mock_admin_client, mock_all_topics_created, mock_consumer): self.mock_consumer = mock_consumer self.topics = ["test_topic_1", "test_topic_2"] self.sut = ExactlyOnceKafkaConsumeHandler(self.topics) @@ -209,7 +223,14 @@ class TestDel(unittest.TestCase): ], ) @patch("src.base.kafka_handler.Consumer") - def test_del_with_existing_consumer(self, mock_consumer): + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") + def test_del_with_existing_consumer( + self, mock_admin_client, mock_all_topics_created, mock_consumer + ): # Arrange mock_consumer_instance = Mock() mock_consumer.return_value = mock_consumer_instance @@ -242,7 +263,14 @@ def test_del_with_existing_consumer(self, mock_consumer): ], ) @patch("src.base.kafka_handler.Consumer") - def test_del_with_existing_consumer(self, mock_consumer): + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") + def test_del_with_existing_consumer( + self, mock_admin_client, mock_all_topics_created, mock_consumer + ): # Arrange mock_consumer_instance = Mock() mock_consumer.return_value = mock_consumer_instance @@ -277,7 +305,12 @@ class TestDict(unittest.TestCase): ], ) @patch("src.base.kafka_handler.Consumer") - def test_dict(self, mock_consumer): + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") + def test_dict(self, mock_admin_client, mock_all_topics_created, mock_consumer): mock_consumer_instance = Mock() mock_consumer.return_value = mock_consumer_instance @@ -304,8 +337,13 @@ class TestConsumeAsObject(unittest.TestCase): }, ], ) + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") @patch("src.base.kafka_handler.Consumer") - def setUp(self, mock_consumer): + def setUp(self, mock_consumer, mock_admin_client, mock_all_topics_created): self.sut = ExactlyOnceKafkaConsumeHandler(topics="test_topic") def test_consume_as_object_no_key_no_value(self): diff --git a/tests/kafka/test_exactly_once_kafka_produce_handler.py b/tests/kafka/test_exactly_once_kafka_produce_handler.py index 57d0b8e7..557774ea 100644 --- a/tests/kafka/test_exactly_once_kafka_produce_handler.py +++ b/tests/kafka/test_exactly_once_kafka_produce_handler.py @@ -7,6 +7,7 @@ class TestInit(unittest.TestCase): + @patch("src.base.kafka_handler.HOSTNAME", "test_transactional_id") @patch( "src.base.kafka_handler.KAFKA_BROKERS", [ @@ -35,7 +36,7 @@ def test_init(self, mock_producer): "enable.idempotence": True, } - sut = ExactlyOnceKafkaProduceHandler("test_transactional_id") + sut = ExactlyOnceKafkaProduceHandler() self.assertIsNone(sut.consumer) self.assertEqual(mock_producer_instance, sut.producer) @@ -68,7 +69,7 @@ def test_init_fail(self, mock_producer, mock_logger): expected_conf = { "bootstrap.servers": "127.0.0.1:9999,127.0.0.2:9998,127.0.0.3:9997", - "transactional.id": "test_transactional_id", + "transactional.id": "default_tid", "enable.idempotence": True, } @@ -76,9 +77,7 @@ def test_init_fail(self, mock_producer, mock_logger): mock_producer_instance, "init_transactions", side_effect=KafkaException ): with self.assertRaises(KafkaException): - sut = ExactlyOnceKafkaProduceHandler( - transactional_id="test_transactional_id" - ) + sut = ExactlyOnceKafkaProduceHandler() mock_producer.assert_called_once_with(expected_conf) mock_producer_instance.init_transactions.assert_called_once() @@ -116,13 +115,13 @@ def test_send_with_data( mock_producer_instance = MagicMock() mock_producer.return_value = mock_producer_instance - sut = ExactlyOnceKafkaProduceHandler(transactional_id="test_transactional_id") + sut = ExactlyOnceKafkaProduceHandler() sut.produce("test_topic", "test_data", key=None) mock_producer_instance.produce.assert_called_once_with( topic="test_topic", key=None, - value="test_data".encode("utf-8"), + value="test_data", callback=mock_kafka_delivery_report, ) mock_commit_transaction_with_retry.assert_called_once() @@ -147,7 +146,7 @@ def test_send_with_data( ) @patch("src.base.kafka_handler.Producer") def test_send_with_empty_data_string(self, mock_producer): - sut = ExactlyOnceKafkaProduceHandler(transactional_id="test_transactional_id") + sut = ExactlyOnceKafkaProduceHandler() sut.produce("test_topic", "", None) mock_producer.begin_transaction.assert_not_called() @@ -188,7 +187,7 @@ def test_send_fail( mock_producer.return_value = mock_producer_instance mock_commit_transaction_with_retry.side_effect = Exception - sut = ExactlyOnceKafkaProduceHandler(transactional_id="test_transactional_id") + sut = ExactlyOnceKafkaProduceHandler() with self.assertRaises(Exception): sut.produce("test_topic", "test_data", key=None) @@ -196,7 +195,7 @@ def test_send_fail( mock_producer_instance.produce.assert_called_once_with( topic="test_topic", key=None, - value="test_data".encode("utf-8"), + value="test_data", callback=mock_kafka_delivery_report, ) @@ -230,7 +229,7 @@ def test_commit_successful(self, mock_sleep, mock_producer): mock_producer.return_value = mock_producer_instance mock_producer.commit_transaction.return_value = None - sut = ExactlyOnceKafkaProduceHandler(transactional_id="test_transactional_id") + sut = ExactlyOnceKafkaProduceHandler() sut.commit_transaction_with_retry() mock_producer_instance.commit_transaction.assert_called_once() @@ -265,7 +264,7 @@ def test_commit_retries_then_successful(self, mock_sleep, mock_producer): None, ] - sut = ExactlyOnceKafkaProduceHandler(transactional_id="test_transactional_id") + sut = ExactlyOnceKafkaProduceHandler() sut.commit_transaction_with_retry() self.assertEqual(mock_producer_instance.commit_transaction.call_count, 2) @@ -298,7 +297,7 @@ def test_commit_retries_and_fails(self, mock_sleep, mock_producer, mock_logger): "Conflicting commit_transaction API call is already in progress" ) - sut = ExactlyOnceKafkaProduceHandler(transactional_id="test_transactional_id") + sut = ExactlyOnceKafkaProduceHandler() with self.assertRaises(RuntimeError) as context: sut.commit_transaction_with_retry() @@ -334,7 +333,7 @@ def test_commit_fails_with_other_exception(self, mock_sleep, mock_producer): "Some other error" ) - sut = ExactlyOnceKafkaProduceHandler(transactional_id="test_transactional_id") + sut = ExactlyOnceKafkaProduceHandler() with self.assertRaises(KafkaException) as context: sut.commit_transaction_with_retry() @@ -366,7 +365,7 @@ def test_del(self, mock_producer): mock_producer_instance = MagicMock() mock_producer.return_value = mock_producer_instance - sut = ExactlyOnceKafkaProduceHandler(transactional_id="test_transactional_id") + sut = ExactlyOnceKafkaProduceHandler() del sut mock_producer_instance.flush.assert_called_once() diff --git a/tests/kafka/test_kafka_consume_handler.py b/tests/kafka/test_kafka_consume_handler.py index 48d3cedd..21412fde 100644 --- a/tests/kafka/test_kafka_consume_handler.py +++ b/tests/kafka/test_kafka_consume_handler.py @@ -2,7 +2,11 @@ import unittest from unittest.mock import patch, MagicMock -from src.base.kafka_handler import KafkaConsumeHandler, KafkaMessageFetchException +from src.base.kafka_handler import ( + KafkaConsumeHandler, + KafkaMessageFetchException, + TooManyFailedAttemptsError, +) class TestInit(unittest.TestCase): @@ -24,8 +28,15 @@ class TestInit(unittest.TestCase): }, ], ) + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") @patch("src.base.kafka_handler.Consumer") - def test_init_successful(self, mock_consumer): + def test_init_successful( + self, mock_consumer, mock_admin_client, mock_all_topics_created + ): # Arrange mock_consumer_instance = MagicMock() mock_consumer.return_value = mock_consumer_instance @@ -45,7 +56,7 @@ def test_init_successful(self, mock_consumer): self.assertEqual(mock_consumer_instance, sut.consumer) mock_consumer.assert_called_once_with(expected_conf) - mock_consumer_instance.assign.assert_called_once() + mock_consumer_instance.subscribe.assert_called_once() @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") @patch( @@ -65,8 +76,62 @@ def test_init_successful(self, mock_consumer): }, ], ) + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=False, + ) + @patch("src.base.kafka_handler.AdminClient") @patch("src.base.kafka_handler.Consumer") - def test_init_successful_with_list(self, mock_consumer): + def test_init_unsuccessful( + self, mock_consumer, mock_admin_client, mock_all_topics_created + ): + # Arrange + mock_consumer_instance = MagicMock() + mock_consumer.return_value = mock_consumer_instance + + expected_conf = { + "bootstrap.servers": "127.0.0.1:9999,127.0.0.2:9998,127.0.0.3:9997", + "group.id": "test_group_id", + "enable.auto.commit": False, + "auto.offset.reset": "earliest", + "enable.partition.eof": True, + } + + # Act + with self.assertRaises(TooManyFailedAttemptsError): + KafkaConsumeHandler(topics="test_topic") + + # Assert + mock_consumer.assert_called_once_with(expected_conf) + mock_consumer_instance.subscribe.assert_not_called() + + @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch( + "src.base.kafka_handler.KAFKA_BROKERS", + [ + { + "hostname": "127.0.0.1", + "port": 9999, + }, + { + "hostname": "127.0.0.2", + "port": 9998, + }, + { + "hostname": "127.0.0.3", + "port": 9997, + }, + ], + ) + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") + @patch("src.base.kafka_handler.Consumer") + def test_init_successful_with_list( + self, mock_consumer, mock_admin_client, mock_all_topics_created + ): # Arrange mock_consumer_instance = MagicMock() mock_consumer.return_value = mock_consumer_instance @@ -86,7 +151,7 @@ def test_init_successful_with_list(self, mock_consumer): self.assertEqual(mock_consumer_instance, sut.consumer) mock_consumer.assert_called_once_with(expected_conf) - mock_consumer_instance.assign.assert_called_once() + mock_consumer_instance.subscribe.assert_called_once() class TestConsume(unittest.TestCase): @@ -108,8 +173,15 @@ class TestConsume(unittest.TestCase): }, ], ) + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") @patch("src.base.kafka_handler.Consumer") - def test_not_implemented(self, mock_consumer): + def test_not_implemented( + self, mock_consumer, mock_admin_client, mock_all_topics_created + ): # Arrange sut = KafkaConsumeHandler(topics="test_topic") @@ -137,8 +209,13 @@ class TestConsumeAsJSON(unittest.TestCase): }, ], ) + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") @patch("src.base.kafka_handler.Consumer") - def setUp(self, mock_consumer): + def setUp(self, mock_consumer, mock_admin_client, mock_all_topics_created): self.sut = KafkaConsumeHandler(topics="test_topic") def test_successful(self): @@ -216,5 +293,63 @@ def test_kafka_message_else(self): self.assertEqual((None, {}), returned_values) +class TestAllTopicsCreated(unittest.TestCase): + @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch( + "src.base.kafka_handler.KAFKA_BROKERS", + [ + { + "hostname": "127.0.0.1", + "port": 9999, + }, + { + "hostname": "127.0.0.2", + "port": 9998, + }, + { + "hostname": "127.0.0.3", + "port": 9997, + }, + ], + ) + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") + @patch("src.base.kafka_handler.Consumer") + def setUp(self, mock_consumer, mock_admin_client, mock_all_topics_created): + self.sut = KafkaConsumeHandler(topics=["test_topic", "another_topic"]) + + @patch("src.base.kafka_handler.Consumer") + def test_with_all_created(self, mock_consumer): + # Arrange + mock_topics = MagicMock() + mock_topics.topics = ["test_topic", "another_topic"] + + self.sut.consumer = MagicMock() + self.sut.consumer.list_topics.return_value = mock_topics + + # Act and Assert + self.assertTrue( + self.sut._all_topics_created(topics=["test_topic", "another_topic"]) + ) + + @patch("src.base.kafka_handler.time.sleep") + @patch("src.base.kafka_handler.Consumer") + def test_with_none_created(self, mock_consumer, mock_sleep): + # Arrange + mock_topics = MagicMock() + mock_topics.topics = [] + + self.sut.consumer = MagicMock() + self.sut.consumer.list_topics.return_value = mock_topics + + # Act and Assert + self.assertFalse( + self.sut._all_topics_created(topics=["test_topic", "another_topic"]) + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/kafka/test_simple_kafka_consume_handler.py b/tests/kafka/test_simple_kafka_consume_handler.py index f69179cd..21a69734 100644 --- a/tests/kafka/test_simple_kafka_consume_handler.py +++ b/tests/kafka/test_simple_kafka_consume_handler.py @@ -25,8 +25,15 @@ class TestInit(unittest.TestCase): }, ], ) + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") @patch("src.base.kafka_handler.Consumer") - def test_init_successful(self, mock_consumer): + def test_init_successful( + self, mock_consumer, mock_admin_client, mock_all_topics_created + ): # Arrange mock_consumer_instance = Mock() mock_consumer.return_value = mock_consumer_instance @@ -46,7 +53,7 @@ def test_init_successful(self, mock_consumer): self.assertEqual(mock_consumer_instance, sut.consumer) mock_consumer.assert_called_once_with(expected_conf) - mock_consumer_instance.assign.assert_called_once() + mock_consumer_instance.subscribe.assert_called_once() class TestConsume(unittest.TestCase): @@ -68,8 +75,13 @@ class TestConsume(unittest.TestCase): }, ], ) + @patch( + "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + return_value=True, + ) + @patch("src.base.kafka_handler.AdminClient") @patch("src.base.kafka_handler.Consumer") - def setUp(self, mock_consumer): + def setUp(self, mock_consumer, mock_admin_client, mock_all_topics_created): self.mock_consumer = mock_consumer self.topics = ["test_topic_1", "test_topic_2"] self.sut = SimpleKafkaConsumeHandler(self.topics) diff --git a/tests/kafka/test_simple_kafka_produce_handler.py b/tests/kafka/test_simple_kafka_produce_handler.py index 8faed81c..9436d07d 100644 --- a/tests/kafka/test_simple_kafka_produce_handler.py +++ b/tests/kafka/test_simple_kafka_produce_handler.py @@ -61,7 +61,7 @@ def test_with_data(self): mock_producer_instance.produce.assert_called_once_with( topic="test_topic", key=None, - value="test_data".encode("utf-8"), + value="test_data", callback=kafka_delivery_report, ) diff --git a/tests/logcollector/test_batch_handler.py b/tests/logcollector/test_batch_handler.py index cc3f56dd..40a99e91 100644 --- a/tests/logcollector/test_batch_handler.py +++ b/tests/logcollector/test_batch_handler.py @@ -31,9 +31,7 @@ def test_init_with_buffer( self.assertEqual(mock_handler_instance, sut.kafka_produce_handler) mock_buffered_batch.assert_called_once() - mock_kafka_produce_handler.assert_called_once_with( - "log_collection.batch_handler" - ) + mock_kafka_produce_handler.assert_called_once() class TestDel(unittest.TestCase): @@ -46,14 +44,12 @@ class TestAddMessage(unittest.TestCase): @patch("src.logcollector.batch_handler.BATCH_SIZE", 1000) @patch("src.logcollector.batch_handler.ExactlyOnceKafkaProduceHandler") @patch("src.logcollector.batch_handler.BufferedBatchSender._reset_timer") - @patch("src.logcollector.batch_handler.BufferedBatch.get_number_of_messages") @patch("src.logcollector.batch_handler.BufferedBatchSender._send_batch_for_key") @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") def test_add_message_normal( self, mock_clickhouse, mock_send_batch, - mock_get_nr_messages, mock_reset_timer, mock_produce_handler, mock_logger, @@ -61,7 +57,6 @@ def test_add_message_normal( # Arrange mock_produce_handler_instance = MagicMock() mock_produce_handler.return_value = mock_produce_handler_instance - mock_get_nr_messages.return_value = 1 key = "test_key" message = json.dumps( @@ -79,7 +74,6 @@ def test_add_message_normal( # Assert mock_send_batch.assert_not_called() - mock_get_nr_messages.assert_called_once_with(key) mock_reset_timer.assert_not_called() @patch("src.logcollector.batch_handler.logger") @@ -399,7 +393,10 @@ class TestResetTimer(unittest.TestCase): @patch("src.logcollector.batch_handler.BATCH_TIMEOUT", 5.9) @patch("src.logcollector.batch_handler.ExactlyOnceKafkaProduceHandler") @patch("src.logcollector.batch_handler.Timer") - def test_reset_timer_with_existing_timer(self, mock_timer, mock_produce_handler): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_reset_timer_with_existing_timer( + self, mock_clickhouse, mock_timer, mock_produce_handler + ): # Arrange mock_timer_instance = MagicMock() mock_timer.return_value = mock_timer_instance @@ -423,7 +420,10 @@ def test_reset_timer_with_existing_timer(self, mock_timer, mock_produce_handler) @patch("src.logcollector.batch_handler.BATCH_TIMEOUT", 4.6) @patch("src.logcollector.batch_handler.ExactlyOnceKafkaProduceHandler") @patch("src.logcollector.batch_handler.Timer") - def test_reset_timer_without_existing_timer(self, mock_timer, mock_produce_handler): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_reset_timer_without_existing_timer( + self, mock_clickhouse, mock_timer, mock_produce_handler + ): # Arrange mock_produce_handler_instance = MagicMock() mock_produce_handler.return_value = mock_produce_handler_instance diff --git a/tests/logcollector/test_buffered_batch.py b/tests/logcollector/test_buffered_batch.py index 31e35ece..da3db282 100644 --- a/tests/logcollector/test_buffered_batch.py +++ b/tests/logcollector/test_buffered_batch.py @@ -7,7 +7,8 @@ class TestInit(unittest.TestCase): - def test_init(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_init(self, mock_clickhouse): # Act sut = BufferedBatch() @@ -135,16 +136,18 @@ def test_add_message_with_existing_other_key(self, mock_clickhouse): class TestGetNumberOfMessages(unittest.TestCase): - def test_get_number_of_messages_with_empty_batch(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_number_of_messages_with_empty_batch(self, mock_clickhouse): # Arrange key = "test_key" sut = BufferedBatch() # Act and Assert - self.assertEqual(0, sut.get_number_of_messages(key)) + self.assertEqual(0, sut.get_message_count_for_batch_key(key)) - def test_get_number_of_messages_with_used_batch_for_key(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_number_of_messages_with_used_batch_for_key(self, mock_clickhouse): # Arrange key = "test_key" message = "test_message" @@ -153,9 +156,12 @@ def test_get_number_of_messages_with_used_batch_for_key(self): sut.batch = {key: [message]} # Act and Assert - self.assertEqual(1, sut.get_number_of_messages(key)) + self.assertEqual(1, sut.get_message_count_for_batch_key(key)) - def test_get_number_of_messages_with_used_batch_for_other_key(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_number_of_messages_with_used_batch_for_other_key( + self, mock_clickhouse + ): # Arrange key = "test_key" other_key = "other_key" @@ -165,10 +171,13 @@ def test_get_number_of_messages_with_used_batch_for_other_key(self): sut.batch = {other_key: [message]} # Act and Assert - self.assertEqual(0, sut.get_number_of_messages(key)) - self.assertEqual(1, sut.get_number_of_messages(other_key)) + self.assertEqual(0, sut.get_message_count_for_batch_key(key)) + self.assertEqual(1, sut.get_message_count_for_batch_key(other_key)) - def test_get_number_of_messages_with_empty_batch_and_used_buffer(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_number_of_messages_with_empty_batch_and_used_buffer( + self, mock_clickhouse + ): # Arrange key = "test_key" other_key = "other_key" @@ -178,10 +187,13 @@ def test_get_number_of_messages_with_empty_batch_and_used_buffer(self): sut.buffer = {other_key: [message]} # Act and Assert - self.assertEqual(0, sut.get_number_of_messages(key)) - self.assertEqual(0, sut.get_number_of_messages(other_key)) + self.assertEqual(0, sut.get_message_count_for_batch_key(key)) + self.assertEqual(0, sut.get_message_count_for_batch_key(other_key)) - def test_get_number_of_messages_with_multiple_keys_and_messages(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_number_of_messages_with_multiple_keys_and_messages( + self, mock_clickhouse + ): # Arrange key_1 = "key_1" key_2 = "key_2" @@ -202,23 +214,27 @@ def test_get_number_of_messages_with_multiple_keys_and_messages(self): sut.buffer = {key_2: [message_4]} # Act and Assert - self.assertEqual(1, sut.get_number_of_messages(key_1)) - self.assertEqual(1, sut.get_number_of_messages(key_2)) - self.assertEqual(2, sut.get_number_of_messages(key_3)) - self.assertEqual(0, sut.get_number_of_messages(key_4)) + self.assertEqual(1, sut.get_message_count_for_batch_key(key_1)) + self.assertEqual(1, sut.get_message_count_for_batch_key(key_2)) + self.assertEqual(2, sut.get_message_count_for_batch_key(key_3)) + self.assertEqual(0, sut.get_message_count_for_batch_key(key_4)) class TestGetNumberOfBufferedMessages(unittest.TestCase): - def test_get_number_of_buffered_messages_with_empty_buffer(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_number_of_buffered_messages_with_empty_buffer(self, mock_clickhouse): # Arrange key = "test_key" sut = BufferedBatch() # Act and Assert - self.assertEqual(0, sut.get_number_of_buffered_messages(key)) + self.assertEqual(0, sut.get_message_count_for_buffer_key(key)) - def test_get_number_of_buffered_messages_with_used_buffer_for_key(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_number_of_buffered_messages_with_used_buffer_for_key( + self, mock_clickhouse + ): # Arrange key = "test_key" message = "test_message" @@ -227,9 +243,12 @@ def test_get_number_of_buffered_messages_with_used_buffer_for_key(self): sut.buffer = {key: [message]} # Act and Assert - self.assertEqual(1, sut.get_number_of_buffered_messages(key)) + self.assertEqual(1, sut.get_message_count_for_buffer_key(key)) - def test_get_number_of_buffered_messages_with_used_buffer_for_other_key(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_number_of_buffered_messages_with_used_buffer_for_other_key( + self, mock_clickhouse + ): # Arrange key = "test_key" other_key = "other_key" @@ -239,10 +258,13 @@ def test_get_number_of_buffered_messages_with_used_buffer_for_other_key(self): sut.buffer = {other_key: [message]} # Act and Assert - self.assertEqual(0, sut.get_number_of_buffered_messages(key)) - self.assertEqual(1, sut.get_number_of_buffered_messages(other_key)) + self.assertEqual(0, sut.get_message_count_for_buffer_key(key)) + self.assertEqual(1, sut.get_message_count_for_buffer_key(other_key)) - def test_get_number_of_buffered_messages_with_empty_buffer_and_used_batch(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_number_of_buffered_messages_with_empty_buffer_and_used_batch( + self, mock_clickhouse + ): # Arrange key = "test_key" other_key = "other_key" @@ -252,10 +274,13 @@ def test_get_number_of_buffered_messages_with_empty_buffer_and_used_batch(self): sut.batch = {other_key: [message]} # Act and Assert - self.assertEqual(0, sut.get_number_of_buffered_messages(key)) - self.assertEqual(0, sut.get_number_of_buffered_messages(other_key)) + self.assertEqual(0, sut.get_message_count_for_buffer_key(key)) + self.assertEqual(0, sut.get_message_count_for_buffer_key(other_key)) - def test_get_number_of_buffered_messages_with_multiple_keys_and_messages(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_number_of_buffered_messages_with_multiple_keys_and_messages( + self, mock_clickhouse + ): # Arrange key_1 = "key_1" key_2 = "key_2" @@ -276,25 +301,27 @@ def test_get_number_of_buffered_messages_with_multiple_keys_and_messages(self): sut.batch = {key_2: [message_4]} # Act and Assert - self.assertEqual(1, sut.get_number_of_buffered_messages(key_1)) - self.assertEqual(1, sut.get_number_of_buffered_messages(key_2)) - self.assertEqual(2, sut.get_number_of_buffered_messages(key_3)) - self.assertEqual(0, sut.get_number_of_buffered_messages(key_4)) + self.assertEqual(1, sut.get_message_count_for_buffer_key(key_1)) + self.assertEqual(1, sut.get_message_count_for_buffer_key(key_2)) + self.assertEqual(2, sut.get_message_count_for_buffer_key(key_3)) + self.assertEqual(0, sut.get_message_count_for_buffer_key(key_4)) class TestSortMessages(unittest.TestCase): - def test_sort_with_empty_list(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_sort_with_empty_list(self, mock_clickhouse): # Arrange list_of_timestamps_and_loglines = [] sut = BufferedBatch() # Act - result = sut.sort_messages(list_of_timestamps_and_loglines) + result = sut._sort_by_timestamp(list_of_timestamps_and_loglines) # Assert self.assertEqual([], result) - def test_sort_with_sorted_list(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_sort_with_sorted_list(self, mock_clickhouse): # Arrange list_of_timestamps_and_loglines = [ ( @@ -325,12 +352,13 @@ def test_sort_with_sorted_list(self): sut = BufferedBatch() # Act - result = sut.sort_messages(list_of_timestamps_and_loglines) + result = sut._sort_by_timestamp(list_of_timestamps_and_loglines) # Assert self.assertEqual(expected_list, result) - def test_sort_with_unsorted_list(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_sort_with_unsorted_list(self, mock_clickhouse): # Arrange list_of_timestamps_and_loglines = [ ( @@ -361,25 +389,27 @@ def test_sort_with_unsorted_list(self): sut = BufferedBatch() # Act - result = sut.sort_messages(list_of_timestamps_and_loglines) + result = sut._sort_by_timestamp(list_of_timestamps_and_loglines) # Assert self.assertEqual(expected_list, result) class TestExtractTuplesFromJson(unittest.TestCase): - def test_empty_data(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_empty_data(self, mock_clickhouse): # Arrange sut = BufferedBatch() data = [] # Act - result = sut.extract_tuples_from_json_formatted_strings(data) + result = sut._extract_tuples_from_json_formatted_strings(data) # Assert self.assertEqual([], result) - def test_with_data(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_with_data(self, mock_clickhouse): # Arrange sut = BufferedBatch() data = [ @@ -410,214 +440,28 @@ def test_with_data(self): ] # Act - result = sut.extract_tuples_from_json_formatted_strings(data) + result = sut._extract_tuples_from_json_formatted_strings(data) # Assert self.assertEqual(expected_result, result) -class TestGetFirstTimestampOfBuffer(unittest.TestCase): - def test_with_data(self): - # Arrange - key = "test_key" - sut = BufferedBatch() - sut.buffer[key] = [ - '{"timestamp": "2024-05-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.105", "dns_ip": "8.8.8.8", "host_domain_name": "www.test.de", "record_type": "A", "response_ip": "fe80::1", "size": "150b"}', - '{"timestamp": "2024-01-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.106", "dns_ip": "8.8.8.8", "host_domain_name": "www.example.com", "record_type": "A", "response_ip": "fe80::2", "size": "200b"}', - '{"timestamp": "2024-12-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.107", "dns_ip": "8.8.8.8", "host_domain_name": "www.sample.com", "record_type": "A", "response_ip": "fe80::3", "size": "250b"}', - ] - sut.batch[key] = [ - '{"timestamp": "2025-05-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.108", "dns_ip": "8.8.8.8", "host_domain_name": "www.example2.com", "record_type": "A", "response_ip": "fe80::4", "size": "300b"}', - '{"timestamp": "2025-01-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.109", "dns_ip": "8.8.8.8", "host_domain_name": "www.test2.de", "record_type": "A", "response_ip": "fe80::5", "size": "350b"}', - '{"timestamp": "2025-12-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.110", "dns_ip": "8.8.8.8", "host_domain_name": "www.sample2.com", "record_type": "A", "response_ip": "fe80::6", "size": "400b"}', - ] - - # Act - result = sut.get_first_timestamp_of_buffer(key) - - # Assert - self.assertEqual("2024-05-21T08:31:28.119Z", result) - - def test_no_data(self): - # Arrange - key = "test_key" - sut = BufferedBatch() - sut.buffer[key] = [] - sut.batch[key] = [] - - # Act - result = sut.get_first_timestamp_of_buffer(key) - - # Assert - self.assertIsNone(result) - - def test_key_does_not_exist(self): - # Arrange - key = "test_key" - sut = BufferedBatch() - - # Act - result = sut.get_first_timestamp_of_buffer(key) - - # Assert - self.assertIsNone(result) - - -class TestGetFirstTimestampOfBatch(unittest.TestCase): - def test_with_data(self): - # Arrange - key = "test_key" - sut = BufferedBatch() - sut.buffer[key] = [ - '{"timestamp": "2024-05-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.105", "dns_ip": "8.8.8.8", "host_domain_name": "www.test.de", "record_type": "A", "response_ip": "fe80::1", "size": "150b"}', - '{"timestamp": "2024-01-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.106", "dns_ip": "8.8.8.8", "host_domain_name": "www.example.com", "record_type": "A", "response_ip": "fe80::2", "size": "200b"}', - '{"timestamp": "2024-12-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.107", "dns_ip": "8.8.8.8", "host_domain_name": "www.sample.com", "record_type": "A", "response_ip": "fe80::3", "size": "250b"}', - ] - sut.batch[key] = [ - '{"timestamp": "2025-05-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.108", "dns_ip": "8.8.8.8", "host_domain_name": "www.example2.com", "record_type": "A", "response_ip": "fe80::4", "size": "300b"}', - '{"timestamp": "2025-01-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.109", "dns_ip": "8.8.8.8", "host_domain_name": "www.test2.de", "record_type": "A", "response_ip": "fe80::5", "size": "350b"}', - '{"timestamp": "2025-12-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.110", "dns_ip": "8.8.8.8", "host_domain_name": "www.sample2.com", "record_type": "A", "response_ip": "fe80::6", "size": "400b"}', - ] - - # Act - result = sut.get_first_timestamp_of_batch(key) - - # Assert - self.assertEqual("2025-05-21T08:31:28.119Z", result) - - def test_no_data(self): - # Arrange - key = "test_key" - sut = BufferedBatch() - sut.buffer[key] = [] - sut.batch[key] = [] - - # Act - result = sut.get_first_timestamp_of_batch(key) - - # Assert - self.assertIsNone(result) - - def test_key_does_not_exist(self): - # Arrange - key = "test_key" - sut = BufferedBatch() - - # Act - result = sut.get_first_timestamp_of_batch(key) - - # Assert - self.assertIsNone(result) - - -class TestGetLastTimestampOfBatch(unittest.TestCase): - def test_with_data(self): - # Arrange - key = "test_key" - sut = BufferedBatch() - sut.buffer[key] = [ - '{"timestamp": "2024-05-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.105", "dns_ip": "8.8.8.8", "host_domain_name": "www.test.de", "record_type": "A", "response_ip": "fe80::1", "size": "150b"}', - '{"timestamp": "2024-01-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.106", "dns_ip": "8.8.8.8", "host_domain_name": "www.example.com", "record_type": "A", "response_ip": "fe80::2", "size": "200b"}', - '{"timestamp": "2024-12-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.107", "dns_ip": "8.8.8.8", "host_domain_name": "www.sample.com", "record_type": "A", "response_ip": "fe80::3", "size": "250b"}', - ] - sut.batch[key] = [ - '{"timestamp": "2025-05-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.108", "dns_ip": "8.8.8.8", "host_domain_name": "www.example2.com", "record_type": "A", "response_ip": "fe80::4", "size": "300b"}', - '{"timestamp": "2025-01-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.109", "dns_ip": "8.8.8.8", "host_domain_name": "www.test2.de", "record_type": "A", "response_ip": "fe80::5", "size": "350b"}', - '{"timestamp": "2025-12-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.110", "dns_ip": "8.8.8.8", "host_domain_name": "www.sample2.com", "record_type": "A", "response_ip": "fe80::6", "size": "400b"}', - ] - - # Act - result = sut.get_last_timestamp_of_batch(key) - - # Assert - self.assertEqual("2025-12-21T08:31:28.119Z", result) - - def test_no_data(self): - # Arrange - key = "test_key" - sut = BufferedBatch() - sut.buffer[key] = [] - sut.batch[key] = [] - - # Act - result = sut.get_last_timestamp_of_batch(key) - - # Assert - self.assertIsNone(result) - - def test_key_does_not_exist(self): - # Arrange - key = "test_key" - sut = BufferedBatch() - - # Act - result = sut.get_last_timestamp_of_batch(key) - - # Assert - self.assertIsNone(result) - - -class TestGetLastTimestampOfBuffer(unittest.TestCase): - def test_with_data(self): - # Arrange - key = "test_key" - sut = BufferedBatch() - sut.buffer[key] = [ - '{"timestamp": "2024-05-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.105", "dns_ip": "8.8.8.8", "host_domain_name": "www.test.de", "record_type": "A", "response_ip": "fe80::1", "size": "150b"}', - '{"timestamp": "2024-01-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.106", "dns_ip": "8.8.8.8", "host_domain_name": "www.example.com", "record_type": "A", "response_ip": "fe80::2", "size": "200b"}', - '{"timestamp": "2024-12-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.107", "dns_ip": "8.8.8.8", "host_domain_name": "www.sample.com", "record_type": "A", "response_ip": "fe80::3", "size": "250b"}', - ] - sut.batch[key] = [ - '{"timestamp": "2025-05-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.108", "dns_ip": "8.8.8.8", "host_domain_name": "www.example2.com", "record_type": "A", "response_ip": "fe80::4", "size": "300b"}', - '{"timestamp": "2025-01-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.109", "dns_ip": "8.8.8.8", "host_domain_name": "www.test2.de", "record_type": "A", "response_ip": "fe80::5", "size": "350b"}', - '{"timestamp": "2025-12-21T08:31:28.119Z", "status": "NOERROR", "client_ip": "192.168.0.110", "dns_ip": "8.8.8.8", "host_domain_name": "www.sample2.com", "record_type": "A", "response_ip": "fe80::6", "size": "400b"}', - ] - - # Act - result = sut.get_last_timestamp_of_buffer(key) - - # Assert - self.assertEqual("2024-12-21T08:31:28.119Z", result) - - def test_no_data(self): - # Arrange - key = "test_key" - sut = BufferedBatch() - sut.buffer[key] = [] - sut.batch[key] = [] - - # Act - result = sut.get_last_timestamp_of_buffer(key) - - # Assert - self.assertIsNone(result) - - def test_key_does_not_exist(self): - # Arrange - key = "test_key" - sut = BufferedBatch() - - # Act - result = sut.get_last_timestamp_of_buffer(key) - - # Assert - self.assertIsNone(result) - - class TestSortBuffer(unittest.TestCase): - def test_sort_empty_buffer(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_sort_empty_buffer(self, mock_clickhouse): # Arrange key = "test_key" sut = BufferedBatch() sut.buffer = [] # Act - sut.sort_buffer(key) + sut._sort_buffer(key) # Assert self.assertEqual([], sut.buffer) - def test_sort_sorted_buffer(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_sort_sorted_buffer(self, mock_clickhouse): # Arrange key = "test_key" sut = BufferedBatch() @@ -638,12 +482,13 @@ def test_sort_sorted_buffer(self): expected_buffer = sut.buffer[key].copy() # Act - sut.sort_buffer(key) + sut._sort_buffer(key) # Assert self.assertEqual(expected_buffer, sut.buffer[key]) - def test_sort_unsorted_buffer(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_sort_unsorted_buffer(self, mock_clickhouse): # Arrange key = "test_key" sut = BufferedBatch() @@ -677,26 +522,28 @@ def test_sort_unsorted_buffer(self): ] # Act - sut.sort_buffer(key) + sut._sort_buffer(key) # Assert self.assertEqual(expected_buffer, sut.buffer[key]) class TestSortBatch(unittest.TestCase): - def test_sort_empty_batch(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_sort_empty_batch(self, mock_clickhouse): # Arrange key = "test_key" sut = BufferedBatch() sut.batch = [] # Act - sut.sort_batch(key) + sut._sort_batch(key) # Assert self.assertEqual([], sut.batch) - def test_sort_sorted_batch(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_sort_sorted_batch(self, mock_clickhouse): # Arrange key = "test_key" sut = BufferedBatch() @@ -717,12 +564,13 @@ def test_sort_sorted_batch(self): expected_batch = sut.batch[key].copy() # Act - sut.sort_batch(key) + sut._sort_batch(key) # Assert self.assertEqual(expected_batch, sut.batch[key]) - def test_sort_unsorted_buffer(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_sort_unsorted_buffer(self, mock_clickhouse): # Arrange key = "test_key" sut = BufferedBatch() @@ -756,7 +604,7 @@ def test_sort_unsorted_buffer(self): ] # Act - sut.sort_batch(key) + sut._sort_batch(key) # Assert self.assertEqual(expected_batch, sut.batch[key]) @@ -848,7 +696,8 @@ def test_complete_batch_variant_2(self, mock_clickhouse): self.assertEqual({key: [message_3, message_4]}, sut.buffer) self.assertEqual({}, sut.batch) - def test_complete_batch_variant_3(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_complete_batch_variant_3(self, mock_clickhouse): # Arrange key = "test_key" @@ -862,7 +711,8 @@ def test_complete_batch_variant_3(self): self.assertEqual({}, sut.batch) self.assertEqual({}, sut.buffer, "Should have been emptied") - def test_complete_batch_variant_4(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_complete_batch_variant_4(self, mock_clickhouse): # Arrange key = "test_key" @@ -877,14 +727,16 @@ def test_complete_batch_variant_4(self): class TestGetStoredKeys(unittest.TestCase): - def test_get_stored_keys_without_any_keys_stored(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_stored_keys_without_any_keys_stored(self, mock_clickhouse): # Arrange sut = BufferedBatch() # Act and Assert self.assertEqual(set(), sut.get_stored_keys()) - def test_get_stored_keys_with_keys_stored_only_in_batch(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_stored_keys_with_keys_stored_only_in_batch(self, mock_clickhouse): # Arrange key_1 = "key_1" key_2 = "key_2" @@ -896,7 +748,8 @@ def test_get_stored_keys_with_keys_stored_only_in_batch(self): # Act and Assert self.assertEqual({key_1, key_2, key_3}, sut.get_stored_keys()) - def test_get_stored_keys_with_keys_stored_only_in_buffer(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_stored_keys_with_keys_stored_only_in_buffer(self, mock_clickhouse): # Arrange key_1 = "key_1" key_2 = "key_2" @@ -908,7 +761,10 @@ def test_get_stored_keys_with_keys_stored_only_in_buffer(self): # Act and Assert self.assertEqual({key_1, key_2, key_3}, sut.get_stored_keys()) - def test_get_stored_keys_with_keys_stored_in_both_batch_and_buffer(self): + @patch("src.logcollector.batch_handler.ClickHouseKafkaSender") + def test_get_stored_keys_with_keys_stored_in_both_batch_and_buffer( + self, mock_clickhouse + ): # Arrange key_1 = "key_1" key_2 = "key_2" diff --git a/tests/logcollector/test_collector.py b/tests/logcollector/test_collector.py index 54f1c743..79f8f97a 100644 --- a/tests/logcollector/test_collector.py +++ b/tests/logcollector/test_collector.py @@ -385,7 +385,7 @@ def test_get_subnet_id_ipv4( sut = LogCollector() # Act - result = sut.get_subnet_id(test_address) + result = sut._get_subnet_id(test_address) # Assert self.assertEqual(expected_result, result) @@ -408,7 +408,7 @@ def test_get_subnet_id_ipv4_zero( sut = LogCollector() # Act - result = sut.get_subnet_id(test_address) + result = sut._get_subnet_id(test_address) # Assert self.assertEqual(expected_result, result) @@ -431,7 +431,7 @@ def test_get_subnet_id_ipv4_max( sut = LogCollector() # Act - result = sut.get_subnet_id(test_address) + result = sut._get_subnet_id(test_address) # Assert self.assertEqual(expected_result, result) @@ -454,7 +454,7 @@ def test_get_subnet_id_ipv6( sut = LogCollector() # Act - result = sut.get_subnet_id(test_address) + result = sut._get_subnet_id(test_address) # Assert self.assertEqual(expected_result, result) @@ -477,7 +477,7 @@ def test_get_subnet_id_ipv6_zero( sut = LogCollector() # Act - result = sut.get_subnet_id(test_address) + result = sut._get_subnet_id(test_address) # Assert self.assertEqual(expected_result, result) @@ -500,7 +500,7 @@ def test_get_subnet_id_ipv6_max( sut = LogCollector() # Act - result = sut.get_subnet_id(test_address) + result = sut._get_subnet_id(test_address) # Assert self.assertEqual(expected_result, result) @@ -525,7 +525,7 @@ def test_get_subnet_id_unsupported_type( # Act & Assert with self.assertRaises(ValueError): # noinspection PyTypeChecker - sut.get_subnet_id(test_address) + sut._get_subnet_id(test_address) @patch("src.logcollector.collector.IPV4_PREFIX_LENGTH", 24) @patch("src.logcollector.collector.IPV6_PREFIX_LENGTH", 48) @@ -547,7 +547,7 @@ def test_get_subnet_id_none( # Act & Assert with self.assertRaises(ValueError): # noinspection PyTypeChecker - sut.get_subnet_id(test_address) + sut._get_subnet_id(test_address) class TestMain(unittest.TestCase): diff --git a/tests/logserver/test_server.py b/tests/logserver/test_server.py index 8f29c540..d6e8855e 100644 --- a/tests/logserver/test_server.py +++ b/tests/logserver/test_server.py @@ -91,15 +91,19 @@ async def mock_gather(*args, **kwargs): class TestSend(unittest.TestCase): @patch("src.logserver.server.PRODUCE_TOPIC", "test_topic") @patch("src.logserver.server.ExactlyOnceKafkaProduceHandler") + @patch("src.logserver.server.SimpleKafkaConsumeHandler") @patch("src.logserver.server.ClickHouseKafkaSender") def test_send( self, mock_clickhouse, + mock_consume_handler, mock_produce_handler, ): # Arrange mock_kafka_produce_handler_instance = MagicMock() + mock_kafka_consume_handler_instance = MagicMock() mock_produce_handler.return_value = mock_kafka_produce_handler_instance + mock_consume_handler.return_value = mock_kafka_consume_handler_instance message = "test_message" sut = LogServer() diff --git a/tests/miscellaneous/test_monitoring_agent.py b/tests/miscellaneous/test_monitoring_agent.py index ead50b03..9c096396 100644 --- a/tests/miscellaneous/test_monitoring_agent.py +++ b/tests/miscellaneous/test_monitoring_agent.py @@ -59,10 +59,12 @@ class TestInit(unittest.TestCase): @patch("src.monitoring.monitoring_agent.SuspiciousBatchesToBatchConnector") @patch("src.monitoring.monitoring_agent.SuspiciousBatchTimestampsConnector") @patch("src.monitoring.monitoring_agent.AlertsConnector") + @patch("src.monitoring.monitoring_agent.FillLevelsConnector") @patch("src.monitoring.monitoring_agent.SimpleKafkaConsumeHandler") def test_init( self, mock_kafka_consumer, + mock_fill_levels, mock_alerts, mock_suspicious_batch_timestamps, mock_suspicious_batches_to_batch, @@ -86,6 +88,7 @@ def test_init( "clickhouse_suspicious_batches_to_batch", "clickhouse_suspicious_batch_timestamps", "clickhouse_alerts", + "clickhouse_fill_levels", ] # Act @@ -110,6 +113,7 @@ class TestStart(unittest.IsolatedAsyncioTestCase): @patch("src.monitoring.monitoring_agent.SuspiciousBatchesToBatchConnector") @patch("src.monitoring.monitoring_agent.SuspiciousBatchTimestampsConnector") @patch("src.monitoring.monitoring_agent.AlertsConnector") + @patch("src.monitoring.monitoring_agent.FillLevelsConnector") @patch("src.monitoring.monitoring_agent.logger") @patch("src.monitoring.monitoring_agent.SimpleKafkaConsumeHandler") @patch("asyncio.get_running_loop") @@ -118,6 +122,7 @@ async def test_handle_kafka_inputs( mock_get_running_loop, mock_kafka_consume, mock_logger, + mock_fill_levels, mock_alerts, mock_suspicious_batch_timestamps, mock_suspicious_batches_to_batch, @@ -167,12 +172,9 @@ async def test_handle_kafka_inputs( class TestMain(unittest.TestCase): - @patch("src.monitoring.monitoring_agent.prepare_all_tables") @patch("src.monitoring.monitoring_agent.MonitoringAgent") @patch("asyncio.run") - def test_main( - self, mock_asyncio_run, mock_monitoring_agent, mock_prepare_all_tables - ): + def test_main(self, mock_asyncio_run, mock_monitoring_agent): # Arrange mock_agent_instance = Mock() mock_monitoring_agent.return_value = mock_agent_instance @@ -181,7 +183,6 @@ def test_main( main() # Assert - mock_prepare_all_tables.assert_called_once() mock_monitoring_agent.assert_called_once() mock_asyncio_run.assert_called_once_with(mock_agent_instance.start()) diff --git a/tests/prefilter/test_prefilter.py b/tests/prefilter/test_prefilter.py index 26caf391..a26cb1ac 100644 --- a/tests/prefilter/test_prefilter.py +++ b/tests/prefilter/test_prefilter.py @@ -13,8 +13,13 @@ class TestInit(unittest.TestCase): @patch("src.prefilter.prefilter.LoglineHandler") @patch("src.prefilter.prefilter.ExactlyOnceKafkaConsumeHandler") @patch("src.prefilter.prefilter.ExactlyOnceKafkaProduceHandler") + @patch("src.prefilter.prefilter.ClickHouseKafkaSender") def test_valid_init( - self, mock_produce_handler, mock_consume_handler, mock_logline_handler + self, + mock_clickhouse, + mock_produce_handler, + mock_consume_handler, + mock_logline_handler, ): sut = Prefilter() @@ -149,8 +154,10 @@ class TestFilterByError(unittest.TestCase): @patch("src.prefilter.prefilter.LoglineHandler") @patch("src.prefilter.prefilter.ExactlyOnceKafkaConsumeHandler") @patch("src.prefilter.prefilter.ExactlyOnceKafkaProduceHandler") + @patch("src.prefilter.prefilter.ClickHouseKafkaSender") def test_filter_by_error_empty_data( self, + mock_clickhouse, mock_produce_handler, mock_consume_handler, mock_logline_handler, @@ -403,8 +410,10 @@ def test_send_with_data( @patch("src.prefilter.prefilter.LoglineHandler") @patch("src.prefilter.prefilter.ExactlyOnceKafkaConsumeHandler") @patch("src.prefilter.prefilter.ExactlyOnceKafkaProduceHandler") + @patch("src.prefilter.prefilter.ClickHouseKafkaSender") def test_send_without_filtered_data_with_unfiltered_data( self, + mock_clickhouse, mock_produce_handler, mock_consume_handler, mock_logline_handler, @@ -425,8 +434,13 @@ def test_send_without_filtered_data_with_unfiltered_data( @patch("src.prefilter.prefilter.LoglineHandler") @patch("src.prefilter.prefilter.ExactlyOnceKafkaConsumeHandler") @patch("src.prefilter.prefilter.ExactlyOnceKafkaProduceHandler") + @patch("src.prefilter.prefilter.ClickHouseKafkaSender") def test_send_without_data( - self, mock_produce_handler, mock_consume_handler, mock_logline_handler + self, + mock_clickhouse, + mock_produce_handler, + mock_consume_handler, + mock_logline_handler, ): mock_produce_handler_instance = MagicMock() mock_produce_handler.return_value = mock_produce_handler_instance @@ -445,8 +459,13 @@ class TestClearData(unittest.TestCase): @patch("src.prefilter.prefilter.LoglineHandler") @patch("src.prefilter.prefilter.ExactlyOnceKafkaConsumeHandler") @patch("src.prefilter.prefilter.ExactlyOnceKafkaProduceHandler") + @patch("src.prefilter.prefilter.ClickHouseKafkaSender") def test_clear_data_with_data( - self, mock_produce_handler, mock_consume_handler, mock_logline_handler + self, + mock_clickhouse, + mock_produce_handler, + mock_consume_handler, + mock_logline_handler, ): first_entry = { "timestamp": "2024-05-21T08:31:28.119Z", @@ -480,8 +499,13 @@ def test_clear_data_with_data( @patch("src.prefilter.prefilter.LoglineHandler") @patch("src.prefilter.prefilter.ExactlyOnceKafkaConsumeHandler") @patch("src.prefilter.prefilter.ExactlyOnceKafkaProduceHandler") + @patch("src.prefilter.prefilter.ClickHouseKafkaSender") def test_clear_data_without_data( - self, mock_produce_handler, mock_consume_handler, mock_logline_handler + self, + mock_clickhouse, + mock_produce_handler, + mock_consume_handler, + mock_logline_handler, ): sut = Prefilter() sut.unfiltered_data = []