Update the full HELK stack

docker/helk-base-images.yml

@@ -0,0 +1,59 @@
+services:
+  helk-base:
+    build: helk-base/
+    container_name: helk-base
+  helk-kafka-base:
+    build: helk-kafka-base/
+    container_name: helk-kafka-base
+    depends_on:
+      - helk-base
+  helk-jupyter-base:
+    build: helk-jupyter-base/
+    container_name: helk-jupyter-base
+    environment:
+      JUPYTER_TYPE: lab
+      JUPYTER_BASE_URL: /jupyter
+    restart: always
+    networks:
+      helk:
+  helk-jupyter-pyspark:
+    build: helk-jupyter-pyspark/
+    container_name: helk-jupyter-pyspark
+    depends_on:
+      - helk-jupyter-base
+    environment:
+      JUPYTER_TYPE: notebook
+      JUPYTER_BASE_URL: /jupyter
+    restart: always
+    networks:
+      helk:
+  helk-jupyter-hunt:
+    build: helk-jupyter-hunt/
+    container_name: helk-jupyter-hunt
+    depends_on:
+      - helk-jupyter-pyspark
+    volumes:
+      - notebooks:/opt/jupyter/notebooks
+    environment:
+      JUPYTER_TYPE: lab
+      JUPYTER_BASE_URL: /jupyter
+    ports:
+      - "8888:8888"
+    restart: always
+    networks:
+      helk:
+  helk-spark-base:
+    build: helk-spark-base/
+    container_name: helk-spark-base
+    depends_on:
+      - helk-base
+
+networks:
+  helk:
+    driver: bridge
+
+volumes:
+  esdata:
+    driver: local
+  notebooks:
+    driver: local

docker/helk-base/Dockerfile

@@ -3,8 +3,8 @@
 # Author: Roberto Rodriguez (@Cyb3rWard0g)
 # License: GPL-3.0
 
-FROM phusion/baseimage:latest
-LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
+FROM phusion/baseimage:noble-1.0.0
+LABEL maintainer="Ignace De Cock @Krapgras"
 LABEL description="Dockerfile HELK Base Image.."
 
 ENV DEBIAN_FRONTEND noninteractive

docker/helk-elastalert/Dockerfile

@@ -1,65 +1,54 @@
-# HELK script: HELK Elastalert Dockerfile
+# HELK script: HELK Elastalert2 Dockerfile
 # HELK build Stage: Alpha
 # Author: Roberto Rodriguez (@Cyb3rWard0g)
 # License: GPL-3.0
 
-FROM otrf/helk-base:latest
-LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
-LABEL description="Dockerfile base for the HELK Elastalert."
+FROM python:3.12-slim
+LABEL maintainer="Ignace De Cock @Krapgras"
+LABEL description="Dockerfile base for the HELK elastalert2."
 
-ENV ESALERT_GID=910
-ENV ESALERT_UID=910
+ENV ESALERT_GID=1000
+ENV ESALERT_UID=1000
 ENV ESALERT_USER=elastalertuser
 ENV ESALERT_HOME=/etc/elastalert
 ENV ESALERT_SIGMA_HOME=/opt/sigma
 
-# *********** Installing Prerequisites ***************
-# -qq : No output except for errors
-RUN apt-get update -qq && apt-get install -qqy --no-install-recommends \
-  python3.6 \
-  libmagic-dev \
-  build-essential \
-  python3-setuptools \
-  git \
-  python3-pip \
-  python3-dev \
-  python3-setuptools \
-  tzdata \
-  # ********* Clean ****************************
-  && apt-get -qy clean \
-  autoremove \
-  && rm -rf /var/lib/apt/lists/* \
-  # ********* Install Elastalert **************
-  && git clone https://github.com/Yelp/elastalert.git ${ESALERT_HOME} \
-  && bash -c 'mkdir -pv /etc/elastalert/rules' \
-  && cd ${ESALERT_HOME} \
-  && sudo pip3 install --upgrade pip \
-  && sudo pip3 install --upgrade setuptools \
-  && pip3 install urllib3 \
-  && pip3 install -U enum34 \
-  #TODO: temporary fix  https://github.com/Yelp/elastalert/issues/2725 also https://github.com/Cyb3rWard0g/HELK/issues/505
-  && pip3 install --force-reinstall elasticsearch==7.0.0 \
-  && pip3 install -r requirements.txt \
-  && python3 setup.py install \
-  # ********* Download SIGMA *******************
-  && git clone https://github.com/Cyb3rWard0g/sigma.git -b helk_neu5ron_updates ${ESALERT_SIGMA_HOME}
+RUN apt update && apt -y upgrade \
+    && apt -y install jq curl gcc libffi-dev git yamllint \
+    && rm -rf /var/lib/apt/lists/* \
+    && git clone https://github.com/jertel/elastalert2.git ${ESALERT_HOME} \
+    && bash -c 'mkdir -pv /etc/elastalert/rules' \
+    && cd ${ESALERT_HOME} \
+    && pip install setuptools wheel \
+    && python setup.py sdist bdist_wheel \
+    && pip install ${ESALERT_HOME}/dist/*.tar.gz \
+    && pip install sigma-cli \
+    && sigma plugin install elasticsearch \
+    && sigma plugin install sysmon \
+    && sigma plugin install windows \
+    && apt -y remove gcc libffi-dev \
+    && apt -y autoremove \
+    # ********* Download SIGMA *******************
+    && git clone https://github.com/SigmaHQ/sigma.git ${ESALERT_SIGMA_HOME}
 
 # ********* Copy Elastalert files **************
 COPY scripts/* ${ESALERT_HOME}/
 COPY config.yaml ${ESALERT_HOME}/
 COPY pull-sigma-config.yaml ${ESALERT_HOME}/
 COPY rules/* ${ESALERT_HOME}/rules/
-COPY sigmac/sigmac-config.yml ${ESALERT_SIGMA_HOME}/sigmac-config.yml
+COPY sigmac/* ${ESALERT_SIGMA_HOME}/
 
 RUN chmod +x ${ESALERT_HOME}/pull-sigma.sh \
-  # ********* Adding Elastalert User *************
-  && groupadd -g ${ESALERT_GID} ${ESALERT_USER} \
-  && useradd -u ${ESALERT_UID} -g ${ESALERT_GID} -d ${ESALERT_HOME} --no-create-home -s /bin/bash ${ESALERT_USER} \
-  && chown -R ${ESALERT_USER}:${ESALERT_USER} ${ESALERT_HOME} ${ESALERT_SIGMA_HOME}
+# ********* Adding Elastalert User *************
+    && groupadd -g ${ESALERT_GID} ${ESALERT_USER} \
+    && useradd -u ${ESALERT_UID} -g ${ESALERT_GID} -d ${ESALERT_HOME} --no-create-home -s /bin/bash ${ESALERT_USER} \
+    && chown -R ${ESALERT_USER}:${ESALERT_USER} ${ESALERT_HOME} ${ESALERT_SIGMA_HOME}
+
+USER ${ESALERT_USER}
 
-USER elastalertuser
+ENV TZ "UTC"
 
-# *********** RUN Elastalert ***************
 WORKDIR ${ESALERT_HOME}
+# *********** RUN Elastalert ***************
 ENTRYPOINT ["./elastalert-entrypoint.sh"]
-CMD ["/bin/bash","-c","elastalert","--verbose"]
+CMD ["/bin/bash","-c","elastalert","--verbose"]

docker/helk-elastalert/scripts/pull-sigma.sh

@@ -22,6 +22,7 @@ HELK_ERROR_FILE="/tmp/helk_error"
 
 # Additional Settings
 helk_sigmac="${ESALERT_SIGMA_HOME}/sigmac-config.yml"
+helk_sigma_filter="${ESALERT_SIGMA_HOME}/elastalert_any_v2.yml"
 
 getYamlKey() {
     python3 -c "import yaml;print(yaml.safe_load(open('$1'))$2)" 2>${HELK_ERROR_FILE}
@@ -125,51 +126,42 @@ echo "------------------------------------------------"
 echo " "
 rule_counter=0
 # Windows rules
-for  rule_category in rules/windows/* ; do
+for rule in $(find rules/windows/* rules-threat-hunting/windows/* -type f -name "*.yml"); do
     echo " "
-    echo -e "${HELK_INFO_TAG} Working on Folder: $rule_category:"
+    echo -e "${HELK_INFO_TAG} Working on Rule: $rule:"
     echo "-------------------------------------------------------------"
-    if [[ "$rule_category" == "rules/windows/process_creation" ]]; then
-        for rule in "${rule_category}"/* ; do
-            if [[ ${rule} != "rules/windows/process_creation/win_mal_adwind.yml" ]]; then
-                if SIGMAremoveNearRules "$rule"; then
-                    continue
-                else
-                    echo "[+++] Processing Windows process creation rule: $rule .."
-                    tools/sigmac -t elastalert -c tools/config/generic/sysmon.yml -c "${helk_sigmac}" --backend-option timestamp_field=etl_processed_time -o "${ESALERT_HOME}"/rules/sigma_sysmon_"$(basename "${rule}")" "$rule"
-                    # Give unique rule name for sysmon
-                    sed -i 's/^name: /name: Sysmon_/' "${ESALERT_HOME}"/rules/sigma_sysmon_"$(basename "${rule}")"
-                    tools/sigmac -t elastalert -c tools/config/generic/windows-audit.yml -c "${helk_sigmac}" --backend-option timestamp_field=etl_processed_time -o "${ESALERT_HOME}"/rules/sigma_"$(basename "${rule}")" "$rule"
-                    rule_counter=$[$rule_counter +1]
-                fi
-            fi
-        done
+    if [[ "$rule" == *"rules/windows/process_creation"* ]] || [[ "$rule" == *"rules-threat-hunting/windows/process_creation"* ]]; then
+        if SIGMAremoveNearRules "$rule"; then
+            continue
+        else
+            echo "[+++] Processing Windows process creation rule: $rule .."
+            sigma convert -t lucene -p "${helk_sigmac}" -p sysmon -p "${helk_sigma_filter}" -o "${ESALERT_HOME}"/rules/sigma_sysmon_"$(basename "${rule}")" "$rule"
+            # Give unique rule name for sysmon
+            sed -i 's/^name: /name: Sysmon_/' "${ESALERT_HOME}"/rules/sigma_sysmon_"$(basename "${rule}")"
+            sigma convert -t lucene -p "${helk_sigmac}" -p windows-audit -p "${helk_sigma_filter}" -o "${ESALERT_HOME}"/rules/sigma_"$(basename "${rule}")" "$rule"
+            rule_counter=$[$rule_counter +1]
+        fi
     else
-        for rule in "${rule_category}"/* ; do
-            if SIGMAremoveNearRules "$rule"; then
-                continue
-            else
-                echo "[+++] Processing additional Windows rule: $rule .."
-                tools/sigmac -t elastalert -c "${helk_sigmac}" --backend-option timestamp_field=etl_processed_time -o "${ESALERT_HOME}"/rules/sigma_"$(basename "${rule}")" "$rule"
-                rule_counter=$[$rule_counter +1]
-            fi
-        done
+        if SIGMAremoveNearRules "$rule"; then
+            continue
+        else
+            echo "[+++] Processing additional Windows rule: $rule .."
+            sigma convert -t lucene -p "${helk_sigmac}" -p "${helk_sigma_filter}" -o "${ESALERT_HOME}"/rules/sigma_"$(basename "${rule}")" "$rule"
+            rule_counter=$[$rule_counter +1]
+        fi
     fi
 done
-# Apt rules
+# emerging-threats rules
 echo " "
 echo -e "${HELK_INFO_TAG} Working on Folder: apt:"
 echo "-------------------------------------------------------------"
-for rule in rules/apt/* ; do
+for rule in $(find rules-emerging-threats/*  -type f -name "*.yml"); do
 
     if SIGMAremoveNearRules "$rule"; then
         continue
     else
-        echo "[+++] Processing apt rule: $rule .."
-        tools/sigmac -t elastalert -c tools/config/generic/sysmon.yml -c "${helk_sigmac}" --backend-option timestamp_field=etl_processed_time -o "${ESALERT_HOME}"/rules/sigma_sysmon_apt_"$(basename "${rule}")" "$rule"
-        # Give unique rule name for sysmon
-        sed -i 's/^name: /name: Sysmon_/' "${ESALERT_HOME}"/rules/sigma_sysmon_apt_"$(basename "${rule}")"
-        tools/sigmac -t elastalert -c tools/config/generic/windows-audit.yml -c "${helk_sigmac}" --backend-option timestamp_field=etl_processed_time -o "${ESALERT_HOME}"/rules/sigma_apt_"$(basename "${rule}")" "$rule"
+        echo "[+++] Processing emerging-threats rule: $rule .."
+        sigma convert -t lucene -p "${helk_sigmac}" -p "${helk_sigma_filter}" -o "${ESALERT_HOME}"/rules/sigma_"$(basename "${rule}")" "$rule"
         rule_counter=$[$rule_counter +1]
     fi
 done
@@ -237,4 +229,11 @@ done
 echo "---------------------------------------------------------"
 echo -e "${HELK_INFO_TAG} [+++] Finished splitting $rule_counter Elastalert rules"
 echo "---------------------------------------------------------"
-echo " "
+echo " "
+echo "---------------------------------------------------------"
+echo -e "${HELK_INFO_TAG} [+++] Removing rules with syntax errors"
+echo "---------------------------------------------------------"
+echo " "
+#lint yaml file with no rules, grep file paths and remove files with syntax errors
+find ${ESALERT_HOME}/rules/* -type f -exec sed -i 's/owner: @/owner: /g' "{}" \;
+yamllint -d "{rules:{}}" --no-warnings ${ESALERT_HOME}/rules/ | grep -F '.' 2>&1 | xargs rm

docker/helk-elastalert/sigmac/elastalert_any.yml

@@ -0,0 +1,22 @@
+postprocessing:
+  - type: template
+    template: |+
+      name: {{ rule.id }}
+      description: {{ rule.title }}
+      owner: {{ rule.author }}
+
+      type: any
+      priority:{% set priority = ({"critical":4, "high":3, "medium":2, "low":1, "informational":0 })%} {{ priority["%s" % rule.level]}}
+      alert:
+        - debug    
+
+      # The Detection Warning miss the index information in the lucene output
+      index: linux-*
+      filter:
+        - query:
+            query_string:
+              query: '{{ query }}'
+
+
+finalizers:
+  - type: concat

docker/helk-elastalert/sigmac/elastalert_any_v2.yml

@@ -0,0 +1,43 @@
+postprocessing:
+  - type: template
+    template: |+
+      name: {{ rule.id }}
+      description: {{ rule.title }}
+      owner: {{ rule.author }}
+
+      type: any
+      priority:{% set priority = ({"critical":4, "high":3, "medium":2, "low":1, "informational":0 })%} {{ priority["%s" % rule.level]}}
+      alert:
+        - debug    
+
+      {% macro index() -%}
+      {% if rule.logsource.product == 'windows' -%}
+      {% if rule.logsource.service == 'application' -%}
+      {% set index = 'logs-endpoint-winevent-application-*' -%}
+      {% elif rule.logsource.service == 'security' -%}
+      {% set index = 'logs-endpoint-winevent-security-*' -%}
+      {% elif rule.logsource.category == 'sysmon' or rule.logsource.service == 'sysmon' -%}
+      {% set index = 'logs-endpoint-winevent-sysmon-*' -%}
+      {% elif rule.logsource.service == 'system' -%}
+      {% set index = 'logs-endpoint-winevent-system-*' -%}  
+      {% elif rule.logsource.category == 'wmi_event' or rule.logsource.service == 'wmi' -%}
+      {% set index = 'logs-endpoint-winevent-wmiactivity-*' -%}
+      {% elif rule.logsource.category in ['ps_classic_start','ps_module','ps_script'] or rule.logsource.service in ['powershell','powershell-classic'] -%}
+      {% set index = 'logs-endpoint-winevent-powershell-*' -%}
+      {% endif -%}
+      {% set index = 'logs-endpoint-winevent-*' -%}
+      {% elif rule.logsource.product == 'linux' -%}
+      {% set index = 'linux-*' -%}
+      {% else -%}
+      {% set index = 'logs-*' -%}
+      {% endif -%}
+      {{- index }}{% endmacro -%}
+      index: {{ index() }}
+      filter:
+        - query:
+            query_string:
+              query: '{{ query }}'
+
+
+finalizers:
+  - type: concat