From fbd828ef8ef2f7a85f9213f65f3682a2edefbb96 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 21 Oct 2020 11:11:37 -0400 Subject: [PATCH] Update pfelk.grok --- roles/logstash/files/patterns/pfelk.grok | 52 ++++++++++++------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/roles/logstash/files/patterns/pfelk.grok b/roles/logstash/files/patterns/pfelk.grok index e2f2ff3..00d31cd 100644 --- a/roles/logstash/files/patterns/pfelk.grok +++ b/roles/logstash/files/patterns/pfelk.grok @@ -1,23 +1,21 @@ # pfelk.grok -########################## -# pfelk GROK Pattern # -# # -# Date 19 September 2020 # -########################## +######### +# 20.10 # +######### PF_LOG_ENTRY %{PF_LOG_DATA}%{PF_IP_SPECIFIC_DATA}%{PF_IP_DATA}%{PF_PROTOCOL_DATA}? -PF_LOG_DATA %{INT:rule_number},%{INT:sub_rule}?,,%{INT:tracker},%{DATA:[interface][name]},(?\b[\w\-]+\b),%{WORD:[event][action]},%{WORD:[network][direction]}, -PF_IP_DATA %{INT:[packet][length]},%{IP:[source][ip]},%{IP:[destination][ip]}, +PF_LOG_DATA %{INT:[rule][id]},%{INT:[rule][sub][id]}?,,%{INT:[rule][uuid]},%{DATA:[interface][name]},(?<[event][reason]>\b[\w\-]+\b),%{WORD:[event][action]},%{WORD:[network][direction]}, PF_IP_SPECIFIC_DATA %{PF_IPv4_SPECIFIC_DATA}|%{PF_IPv6_SPECIFIC_DATA} -PF_IPv4_SPECIFIC_DATA (?<[network][type]>(4)),%{BASE16NUM:tos},%{WORD:ecn}?,%{INT:ttl},%{INT:[packet][id]},%{INT:offset},(?:%{WORD:[ip][flags]}|%{PF_SPEC:[ip][flags]}),%{INT:[network][iana_number]},%{WORD:[network][transport]}, -PF_IPv6_SPECIFIC_DATA (?<[network][type]>(6)),%{BASE16NUM:IPv6_Flag1},%{WORD:IPv6_Flag2},%{WORD:flow_label},%{DATA:[protocol][type]},%{INT:[protocol][id]}, +PF_IPv4_SPECIFIC_DATA (?<[network][type]>(4)),%{BASE16NUM:[ipv4][tos]},%{WORD:[ipv4][ecn]}?,%{INT:[ipv4][ttl]},%{INT:[ipv4][packet][id]},%{INT:[ipv4][offset]},%{WORD:[ipv4][flags]},%{INT:[network][iana_number]},%{WORD:[network][transport]}, +PF_IPv6_SPECIFIC_DATA (?<[network][type]>(6)),%{BASE16NUM:[ipv6][class]},%{WORD:[ipv6][flow_label]},%{WORD:[ipv6][hop_limit]},%{DATA:[protocol][type]},%{INT:[protocol][id]}, +PF_IP_DATA %{INT:[packet][length]},%{IP:[source][ip]},%{IP:[destination][ip]}, PF_PROTOCOL_DATA %{PF_TCP_DATA}|%{PF_UDP_DATA}|%{PF_ICMP_DATA}|%{PF_IGMP_DATA}|%{PF_IPv6_VAR}|%{PF_IPv6_ICMP} # IPv6 -PF_IPv6_VAR %{WORD:Type},%{WORD:Option},%{WORD:Flags},%{WORD:Flags} +PF_IPv6_VAR %{WORD:type},%{WORD:option},%{WORD:Flags},%{WORD:Flags} PF_IPv6_ICMP # PROTOCOL -PF_TCP_DATA %{INT:[source][port]},%{INT:[destination][port]},%{INT:[transport][data_length]},(?(\w*)?),(?(\d*)?):?\d*,(?(\d*)?),(?(\d*)?),(?(\w*)?),%{GREEDYDATA:tcp_options} +PF_TCP_DATA %{INT:[source][port]},%{INT:[destination][port]},%{INT:[transport][data_length]},(?<[tcp][flags]>(\w*)?),(?<[tcp][sequence_number]>(\d*)?):?\d*,(?<[tcp][ack_number]>(\d*)?),(?<[tcp][window]>(\d*)?),(?<[tcp][urg]>(\w*)?),%{GREEDYDATA:[tcp][options]} PF_UDP_DATA %{INT:[source][port]},%{INT:[destination][port]},%{INT:[transport][data_length]}$ PF_IGMP_DATA datalength=%{INT:[network][packets]} PF_ICMP_DATA %{PF_ICMP_TYPE}%{PF_ICMP_RESPONSE} @@ -33,22 +31,22 @@ PF_ICMP_TSTAMP_REPLY %{INT:icmp_tstamp_reply_id},%{INT:icmp_tstamp_reply_sequenc PF_SPEC \+ # DHCPv4 (Optional) -DHCPD_VIA via (%{IP:[dhcpv4][relay_ip]}|(?<[interface][name]>[^: ]+)) +DHCPD_VIA via (%{IP:[dhcpv4][relay][ip]}|(?<[interface][name]>[^: ]+)) DHCPD DHCP(%{DHCPD_DISCOVER}|%{DHCPD_OFFER_ACK}|%{DHCPD_REQUEST}|%{DHCPD_DECLINE}|%{DHCPD_RELEASE}|%{DHCPD_INFORM}|%{DHCPD_LEASE})(: %{GREEDYDATA:[dhcpv4][option][message]})? -DHCPD_DISCOVER (?<[dhcp][operation]>DISCOVER) from %{MAC:[dhcpv4][client_mac]}( \(%{DATA:[dhcpv4][option][hostname]}\))? %{DHCPD_VIA} -DHCPD_OFFER_ACK (?<[dhcp][operation]>(OFFER|N?ACK)) on %{IP:[dhcpv4][client_ip]} to %{MAC:[dhcpv4][client_mac]}( \(%{DATA:[dhcpv4][option][hostname]}\))? %{DHCPD_VIA} -DHCPD_REQUEST (?<[dhcp][operation]>REQUEST) for %{IP:[dhcpv4][client_ip]}( \(%{DATA:[dhcpv4][server_ip]}\))? from %{MAC:[dhcpv4][client_mac]}( \(%{DATA:[dhcpv4][option][hostname]}\))? %{DHCPD_VIA} -DHCPD_DECLINE (?<[dhcp][operation]>DECLINE) of %{IP:[dhcpv4][client_ip]} from %{MAC:[dhcpv4][client_mac]}( \(%{DATA:[dhcpv4][option][hostname]}\))? %{DHCPD_VIA} -DHCPD_RELEASE (?<[dhcp][operation]>RELEASE) of %{IP:[dhcpv4][client_ip]} from %{MAC:[dhcpv4][client_mac]}( \(%{DATA:[dhcpv4][option][hostname]}\))? %{DHCPD_VIA} \((?(not )?found)\) -DHCPD_INFORM (?<[dhcp][operation]>INFORM) from %{IP:[dhcpv4][client_ip]}? %{DHCPD_VIA} -DHCPD_LEASE (?<[dhcp][operation]>LEASE(QUERY|UNKNOWN|ACTIVE|UNASSIGNED)) (from|to) %{IP:[dhcpv4][client_ip]} for (IP %{IP:[dhcpv4][leasequery_ip]}|client-id %{NOTSPACE:[dhcpv4][leasequery_id]}|MAC address %{MAC:[dhcpv4][leasequery_mac]})( \(%{NUMBER:[dhcpv4][leasequery_associated]} associated IPs\))? +DHCPD_DISCOVER (?<[dhcp][operation]>DISCOVER) from %{MAC:[dhcpv4][client][mac]}( \(%{DATA:[dhcpv4][option][hostname]}\))? %{DHCPD_VIA} +DHCPD_OFFER_ACK (?<[dhcp][operation]>(OFFER|N?ACK)) on %{IP:[dhcpv4][client][ip]} to %{MAC:[dhcpv4][client][mac]}( \(%{DATA:[dhcpv4][option][hostname]}\))? %{DHCPD_VIA} +DHCPD_REQUEST (?<[dhcp][operation]>REQUEST) for %{IP:[dhcpv4][client][ip]}( \(%{DATA:[dhcpv4][server][ip]}\))? from %{MAC:[dhcpv4][client][mac]}( \(%{DATA:[dhcpv4][option][hostname]}\))? %{DHCPD_VIA} +DHCPD_DECLINE (?<[dhcp][operation]>DECLINE) of %{IP:[dhcpv4][client][ip]} from %{MAC:[dhcpv4][client][mac]}( \(%{DATA:[dhcpv4][option][hostname]}\))? %{DHCPD_VIA} +DHCPD_RELEASE (?<[dhcp][operation]>RELEASE) of %{IP:[dhcpv4][client][ip]} from %{MAC:[dhcpv4][client][mac]}( \(%{DATA:[dhcpv4][option][hostname]}\))? %{DHCPD_VIA} \((?(not )?found)\) +DHCPD_INFORM (?<[dhcp][operation]>INFORM) from %{IP:[dhcpv4][client][ip]}? %{DHCPD_VIA} +DHCPD_LEASE (?<[dhcp][operation]>LEASE(QUERY|UNKNOWN|ACTIVE|UNASSIGNED)) (from|to) %{IP:[dhcpv4][client][ip]} for (IP %{IP:[dhcpv4][query][ip]}|client-id %{NOTSPACE:[dhcpv4][query][id]}|MAC address %{MAC:[dhcpv4][query][mac]})( \(%{NUMBER:[dhcpv4][query][associated]} associated IPs\))? DHCPGENERAL %{GREEDYDATA:[dhcp][message]} # DHCPv6 (Optional - In Development) DHCPDv6 %{GREEDYDATA:[dhcpv6][operation]} # PF -PF %{DATA:application}(?:\[%{POSINT:[process][id]}\])?: %{GREEDYDATA:pfelk_message} +PF %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]}\])?: %{GREEDYDATA:pfelk_message} PF_CARP_DATA (%{WORD:[carp][type]}),(%{INT:[carp][ttl]}),(%{INT:[carp][vhid]}),(%{INT:[carp][version]}),(%{INT:[carp][advbase]}),(%{INT:[carp][advskew]}) PF_APP (%{DATA:pf_APP}): PF_APP_DATA (%{PF_APP_LOGOUT}|%{PF_APP_LOGIN}|%{PF_APP_ERROR}|%{PF_APP_GEN}) @@ -59,12 +57,14 @@ PF_APP_GEN (%{GREEDYDATA:pf_ACTION}) # OPENVPN OPENVPN (%{OPENVPNIP}|%{OPENVPNUSER}|%{OPENVPNLOG}) -OPENVPNIP %{IP:[vpn_source][ip]}\:%{INT:[vpn_source][port]}%{SPACE}\[%{DATA:vpn_client}\]%{SPACE}Peer%{SPACE}Connection%{SPACE}Initiated%{SPACE}with%{GREEDYDATA} -OPENVPNUSER (%{WORD:openvpn_domain}?\\)?(?\b[+\w\.-]+\b)?/?%{IP:[vpn_source][ip]}:%{INT:[vpn_source][port]} peer info: IV_PLAT=%{WORD:openvpn_plat} -OPENVPNLOG %{GREEDYDATA:openvpn_message} +OPENVPNIP %{IP:[vpn][source][ip]}\:%{INT:[vpn][source][port]}%{SPACE}\[%{DATA:[vpn][client]}\]%{SPACE}Peer%{SPACE}Connection%{SPACE}Initiated%{SPACE}with%{GREEDYDATA} +OPENVPNUSER (%{WORD:[vpn][domain]}?\\)?(?<[vpn][user]>\b[+\w\.-]+\b)?/?%{IP:[vpn][source][ip]}:%{INT:[vpn][source][port]} peer info: IV_PLAT=%{WORD:[vpn][plat]} +OPENVPNLOG %{GREEDYDATA:[vpn][log][message]} + +# UNBOUND - Level 1 (Optional) +UNBOUND %{INT:[unbound][process][pid]}:%{INT:[unbound][process][thread][id]}] %{LOGLEVEL:[unbound][log][level]}: %{IP:[unbound][client][ip]} %{GREEDYDATA:[unbound][dns][question][name]}\. %{WORD:[unbound][dns][answers][type]} %{WORD:[unbound][dns][question][class]} +### Expand with Level 2 & 3 -# UNBOUND (Optional) -UNBOUND %{INT:[unbound][process][id]}:%{INT:[unbound][instance][id]}] %{LOGLEVEL:[unbound][log][level]}: %{IP:[unbound][query][client][ip]} %{GREEDYDATA:[unbound][query][url]}\. %{WORD:[unbound][query][record][type]} %{WORD:[unbound][query][message][flags]} # SURICATA SURICATA \[%{NUMBER:[suricata][rule][uuid]}:%{NUMBER:[suricata][rule][id]}:%{NUMBER:[suricata][rule][version]}\]%{SPACE}%{GREEDYDATA:[suricata][rule][description]}%{SPACE}\[Classification:%{SPACE}%{GREEDYDATA:[suricata][rule][category]}\]%{SPACE}\[Priority:%{SPACE}%{NUMBER:[suricata][priority]}\]%{SPACE}{%{WORD:[network][transport]}}%{SPACE}%{IP:[source][ip]}:%{NUMBER:[source][port]}%{SPACE}->%{SPACE}%{IP:[destination][ip]}:%{NUMBER:[destination][port]} @@ -73,4 +73,4 @@ SURICATA \[%{NUMBER:[suricata][rule][uuid]}:%{NUMBER:[suricata][rule][id]}:%{NUM SNORT \[%{INT:[snort][rule][uuid]}\:%{INT:[snort][rule][reference]}\:%{INT:[snort][rule][version]}\].%{GREEDYDATA:[snort][rule][description]}.\[Classification\: %{DATA:[snort][rule][classification]}\].\[Priority\: %{INT:[snort][priority]}\].\{%{DATA:[network][transport]}\}.%{IP:[source][ip]}(\:%{INT:[source][port]})?.->.%{IP:[destination][ip]}(\:%{INT:[destination][port]})? # HAPROXY -HAPROXY %{DATA:application}(?:\[%{POSINT:[process][pid]}\])?:%{SPACE}%{IP:[haproxy][client][ip]}:%{INT:[haproxy][client][port]} \[%{HAPROXYDATE:haproxy_timestamp}\] %{NOTSPACE:[haproxy][frontend_name]} %{NOTSPACE:[haproxy][backend_name]}/%{NOTSPACE:[haproxy][real_server_name]} %{INT:[haproxy][time_request]}/%{INT:[haproxy][time_queue]}/%{INT:[haproxy][time_backend_connect]}/%{INT:[haproxy][time_backend_response]}/%{NOTSPACE:[haproxy][time_duration]} %{INT:[haproxy][http_status_code]} %{NOTSPACE:[haproxy][bytes_read]} %{DATA:[haproxy][captured_request_cookie]} %{DATA:[haproxy][captured_response_cookie]} %{NOTSPACE:[haproxy][termination_state]} %{INT:[haproxy][actconn]}/%{INT:[haproxy][feconn]}/%{INT:[haproxy][beconn]}/%{INT:[haproxy][srvconn]}/%{NOTSPACE:[haproxy][retries]} %{INT:[haproxy][srv_queue]}/%{INT:[haproxy][backend_queue]} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(|(%{WORD:[haproxy][http_verb]} (%{URIPROTO:[haproxy][http_proto]}://)?(?:%{USER:[haproxy][http_user]}(?::[^@]*)?@)?(?:%{URIHOST:[haproxy][http_host]})?(?:%{URIPATHPARAM:[haproxy][http_request]})?( HTTP/%{NUMBER:[haproxy][http_version]})?))?"? +HAPROXY %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]}\])?:%{SPACE}%{IP:[haproxy][client][ip]}:%{INT:[haproxy][client][port]} \[%{HAPROXYDATE:haproxy_timestamp}\] %{NOTSPACE:[haproxy][frontend_name]} %{NOTSPACE:[haproxy][backend_name]}/%{NOTSPACE:[haproxy][server_name]} %{INT:[haproxy][time_request]}/%{INT:[haproxy][time_queue]}/%{INT:[haproxy][time_backend_connect]}/%{INT:[haproxy][time_backend_response]}/%{NOTSPACE:[haproxy][time_duration]} %{INT:[haproxy][http_status_code]} %{NOTSPACE:[haproxy][bytes_read]} %{DATA:[haproxy][captured_request_cookie]} %{DATA:[haproxy][captured_response_cookie]} %{NOTSPACE:[haproxy][termination_state]} %{INT:[haproxy][actconn]}/%{INT:[haproxy][feconn]}/%{INT:[haproxy][beconn]}/%{INT:[haproxy][srvconn]}/%{NOTSPACE:[haproxy][retries]} %{INT:[haproxy][srv_queue]}/%{INT:[haproxy][backend_queue]} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(|(%{WORD:[haproxy][http_verb]} (%{URIPROTO:[haproxy][http_proto]}://)?(?:%{USER:[haproxy][http_user]}(?::[^@]*)?@)?(?:%{URIHOST:[haproxy][http_host]})?(?:%{URIPATHPARAM:[haproxy][http_request]})?( HTTP/%{NUMBER:[haproxy][http_version]})?))?"?