Log raw events and errors containing events to a separate file

CHANGELOG.next.asciidoc

@@ -129,6 +129,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d
 - The Elasticsearch output can now configure performance presets with the `preset` configuration field. {pull}37259[37259]
 - Upgrade to elastic-agent-libs v0.7.3 and golang.org/x/crypto v0.17.0. {pull}37544[37544]
 - Make more selective the Pod autodiscovery upon node and namespace update events. {issue}37338[37338] {pull}37431[37431]
+- Raw event data logged by outputs on error is now logged to a different log file {pull}37475[37475]
 
 *Auditbeat*
 

NOTICE.txt

@@ -12700,12 +12700,12 @@ SOFTWARE
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.7.5
+Dependency : github.com/belimawr/elastic-agent-libs
+Version: v0.2.9-0.20240122163001-efb117578ab2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.7.5/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/belimawr/elastic-agent-libs@v0.2.9-0.20240122163001-efb117578ab2/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

auditbeat/auditbeat.reference.yml

@@ -1544,6 +1544,46 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events (that may contain
+# sensitive information) together with other log messages, a different
+# log file, only for log entries containing raw events, is used. It will
+# use the same level, selectors and all other configurations from the
+# default logger, but it will have it's own file configuration.
+
+# Having a different log file for raw events also prevents event data
+# from drowning out the regular log files.
+#logging.sensitive:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/auditbeat
+
+    # The name of the files where the logs are written to.
+    #name: auditbeat-sensitive-data
+
+    # Configure log file size limit. If the limit is reached, log file will be
+    # automatically rotated.
+    #rotateeverybytes: 5242880 # = 5MB
+
+    # Number of rotated log files to keep. The oldest files will be deleted first.
+    #keepfiles: 5
+
+    # The permissions mask to apply when rotating log files. The default value is 0600.
+    # Must be a valid Unix-style file permissions mask expressed in octal notation.
+    #permissions: 0600
+
+    # Enable log file rotation on time intervals in addition to the size-based rotation.
+    # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h
+    # are boundary-aligned with minutes, hours, days, weeks, months, and years as
+    # reported by the local system clock. All other intervals are calculated from the
+    # Unix epoch. Defaults to disabled.
+    #interval: 0
+
+    # Rotate existing logs on startup rather than appending them to the existing
+    # file. Defaults to true.
+    # rotateonstartup: true
+
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

auditbeat/auditbeat.yml

@@ -169,6 +169,20 @@ processors:
 # "publisher", "service".
 #logging.selectors: ["*"]
 
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events together with other
+# log messages, a different log file, only for log entries containing raw events,
+# is used. It will use the same level, selectors and all other configurations
+# from the default logger, but it will have it's own file configuration.
+#logging.sensitive:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/auditbeat
+
+    # The name of the files where the logs are written to.
+    #name: auditbeat-sensitive-data
+
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

auditbeat/tests/system/requirements.txt

@@ -1 +1 @@
-protobuf==3.19.5 #Temporary change because of protobuf new version bug: https://github.com/protocolbuffers/protobuf/issues/10051
+protobuf==3.19.5

dev-tools/requirements.txt

@@ -1,3 +1,3 @@
 elasticsearch
 requests
-protobuf==3.19.5 #Temporary change because of protobuf new version bug: https://github.com/protocolbuffers/protobuf/issues/10051
+protobuf==3.19.5

filebeat/filebeat.reference.yml

@@ -2640,6 +2640,46 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events (that may contain
+# sensitive information) together with other log messages, a different
+# log file, only for log entries containing raw events, is used. It will
+# use the same level, selectors and all other configurations from the
+# default logger, but it will have it's own file configuration.
+
+# Having a different log file for raw events also prevents event data
+# from drowning out the regular log files.
+#logging.sensitive:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/filebeat
+
+    # The name of the files where the logs are written to.
+    #name: filebeat-sensitive-data
+
+    # Configure log file size limit. If the limit is reached, log file will be
+    # automatically rotated.
+    #rotateeverybytes: 5242880 # = 5MB
+
+    # Number of rotated log files to keep. The oldest files will be deleted first.
+    #keepfiles: 5
+
+    # The permissions mask to apply when rotating log files. The default value is 0600.
+    # Must be a valid Unix-style file permissions mask expressed in octal notation.
+    #permissions: 0600
+
+    # Enable log file rotation on time intervals in addition to the size-based rotation.
+    # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h
+    # are boundary-aligned with minutes, hours, days, weeks, months, and years as
+    # reported by the local system clock. All other intervals are calculated from the
+    # Unix epoch. Defaults to disabled.
+    #interval: 0
+
+    # Rotate existing logs on startup rather than appending them to the existing
+    # file. Defaults to true.
+    # rotateonstartup: true
+
 # ============================= X-Pack Monitoring ==============================
 # Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

filebeat/filebeat.yml

@@ -186,6 +186,20 @@ processors:
 # "publisher", "service".
 #logging.selectors: ["*"]
 
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events together with other
+# log messages, a different log file, only for log entries containing raw events,
+# is used. It will use the same level, selectors and all other configurations
+# from the default logger, but it will have it's own file configuration.
+#logging.sensitive:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/filebeat
+
+    # The name of the files where the logs are written to.
+    #name: filebeat-sensitive-data
+
 # ============================= X-Pack Monitoring ==============================
 # Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

filebeat/tests/integration/sensitive_log_file_test.go

@@ -0,0 +1,131 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build integration
+
+package integration
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/beats/v7/libbeat/tests/integration"
+)
+
+var eventsLogFileCfg = `
+filebeat.inputs:
+  - type: filestream
+    id: filestream-input-id
+    enabled: true
+    parsers:
+      - ndjson:
+          target: ""
+          overwrite_keys: true
+          expand_keys: true
+          add_error_key: true
+          ignore_decoding_error: false
+    paths:
+      - %s
+
+output:
+  elasticsearch:
+    hosts:
+      - localhost:9200
+    protocol: http
+    username: admin
+    password: testing
+
+logging:
+  level: debug
+  files:
+    events:
+      files:
+        name: filebeat-sensitive-data
+`
+
+func TestEventsLoggerESOutput(t *testing.T) {
+	// First things first, ensure ES is running and we can connect to it.
+	// If ES is not running, the test will timeout and the only way to know
+	// what caused it is going through Filebeat's logs.
+	integration.EnsureESIsRunning(t)
+
+	filebeat := integration.NewBeat(
+		t,
+		"filebeat",
+		"../../filebeat.test",
+	)
+
+	logFilePath := filepath.Join(filebeat.TempDir(), "log.log")
+	filebeat.WriteConfigFile(fmt.Sprintf(eventsLogFileCfg, logFilePath))
+
+	logFile, err := os.Create(logFilePath)
+	if err != nil {
+		t.Fatalf("could not create file '%s': %s", logFilePath, err)
+	}
+
+	_, _ = logFile.WriteString(`
+{"message":"foo bar","int":10,"string":"str"}
+{"message":"another message","int":20,"string":"str2"}
+{"message":"index failure","int":"not a number","string":10}
+{"message":"second index failure","int":"not a number","string":10}
+`)
+	if err := logFile.Sync(); err != nil {
+		t.Fatalf("could not sync log file '%s': %s", logFilePath, err)
+	}
+	if err := logFile.Close(); err != nil {
+		t.Fatalf("could not close log file '%s': %s", logFilePath, err)
+	}
+
+	filebeat.Start()
+
+	// Wait for a log entry that indicates an entry in the events
+	// logger file.
+	msg := "Cannot index event (status=400)"
+	require.Eventually(t, func() bool {
+		return filebeat.LogContains(msg)
+	}, time.Minute, 100*time.Millisecond,
+		fmt.Sprintf("String '%s' not found on Filebeat logs", msg))
+
+	glob := filepath.Join(filebeat.TempDir(), "filebeat-sensitive-data*.ndjson")
+	files, err := filepath.Glob(glob)
+	if err != nil {
+		t.Fatalf("could not read files matching glob '%s': %s", glob, err)
+	}
+	if len(files) != 1 {
+		t.Fatalf("there must be only one file matching the glob '%s', found: %s", glob, files)
+	}
+
+	eventsLogFile := files[0]
+	data, err := os.ReadFile(eventsLogFile)
+	if err != nil {
+		t.Fatalf("could not read '%s': %s", eventsLogFile, err)
+	}
+
+	strData := string(data)
+	eventMsg := "not a number"
+	if !strings.Contains(strData, eventMsg) {
+		t.Errorf("expecting to find '%s' on '%s'", eventMsg, eventsLogFile)
+		t.Errorf("Contents:\n%s", strData)
+		t.FailNow()
+	}
+}

go.mod

@@ -419,3 +419,5 @@ replace (
 
 // Exclude this version because the version has an invalid checksum.
 exclude github.com/docker/distribution v2.8.0+incompatible
+
+replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240122163001-efb117578ab2

go.sum

@@ -373,6 +373,8 @@ github.com/awslabs/goformation/v4 v4.1.0 h1:JRxIW0IjhYpYDrIZOTJGMu2azXKI+OK5dP56
 github.com/awslabs/goformation/v4 v4.1.0/go.mod h1:MBDN7u1lMNDoehbFuO4uPvgwPeolTMA2TzX1yO6KlxI=
 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 h1:lxW5Q6K2IisyF5tlr6Ts0W4POGWQZco05MJjFmoeIHs=
 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5/go.mod h1:0Qr1uMHFmHsIYMcG4T7BJ9yrJtWadhOmpABCX69dwuc=
+github.com/belimawr/elastic-agent-libs v0.2.9-0.20240122163001-efb117578ab2 h1:QOTo5kTJ8oqdrSOH8/OhSkEMA3mnRltGg52M9YyH7Zo=
+github.com/belimawr/elastic-agent-libs v0.2.9-0.20240122163001-efb117578ab2/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
 github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps=
@@ -662,8 +664,6 @@ github.com/elastic/elastic-agent-autodiscover v0.6.7 h1:+KVjltN0rPsBrU8b156gV4lO
 github.com/elastic/elastic-agent-autodiscover v0.6.7/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4=
 github.com/elastic/elastic-agent-client/v7 v7.6.0 h1:FEn6FjzynW4TIQo5G096Tr7xYK/P5LY9cSS6wRbXZTc=
 github.com/elastic/elastic-agent-client/v7 v7.6.0/go.mod h1:GlUKrbVd/O1CRAZonpBeN3J0RlVqP6VGcrBjFWca+aM=
-github.com/elastic/elastic-agent-libs v0.7.5 h1:4UMqB3BREvhwecYTs/L23oQp1hs/XUkcunPlmTZn5yg=
-github.com/elastic/elastic-agent-libs v0.7.5/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI=
 github.com/elastic/elastic-agent-system-metrics v0.9.1 h1:r0ofKHgPpl+W09ie7tzGcCDC0d4NZbQUv37rSgHf4FM=

heartbeat/heartbeat.reference.yml

@@ -1636,6 +1636,46 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events (that may contain
+# sensitive information) together with other log messages, a different
+# log file, only for log entries containing raw events, is used. It will
+# use the same level, selectors and all other configurations from the
+# default logger, but it will have it's own file configuration.
+
+# Having a different log file for raw events also prevents event data
+# from drowning out the regular log files.
+#logging.sensitive:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/heartbeat
+
+    # The name of the files where the logs are written to.
+    #name: heartbeat-sensitive-data
+
+    # Configure log file size limit. If the limit is reached, log file will be
+    # automatically rotated.
+    #rotateeverybytes: 5242880 # = 5MB
+
+    # Number of rotated log files to keep. The oldest files will be deleted first.
+    #keepfiles: 5
+
+    # The permissions mask to apply when rotating log files. The default value is 0600.
+    # Must be a valid Unix-style file permissions mask expressed in octal notation.
+    #permissions: 0600
+
+    # Enable log file rotation on time intervals in addition to the size-based rotation.
+    # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h
+    # are boundary-aligned with minutes, hours, days, weeks, months, and years as
+    # reported by the local system clock. All other intervals are calculated from the
+    # Unix epoch. Defaults to disabled.
+    #interval: 0
+
+    # Rotate existing logs on startup rather than appending them to the existing
+    # file. Defaults to true.
+    # rotateonstartup: true
+
 # ============================= X-Pack Monitoring ==============================
 # Heartbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

heartbeat/heartbeat.yml

@@ -152,6 +152,20 @@ processors:
 # "publisher", "service".
 #logging.selectors: ["*"]
 
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events together with other
+# log messages, a different log file, only for log entries containing raw events,
+# is used. It will use the same level, selectors and all other configurations
+# from the default logger, but it will have it's own file configuration.
+#logging.sensitive:
+  #files:
+    # Configure the path where the logs are written. The default is the logs directory
+    # under the home path (the binary location).
+    #path: /var/log/heartbeat
+
+    # The name of the files where the logs are written to.
+    #name: heartbeat-sensitive-data
+
 # ============================= X-Pack Monitoring ==============================
 # Heartbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

heartbeat/tests/system/requirements.txt

@@ -1 +1 @@
-protobuf==3.19.5 #Temporary change because of protobuf new version bug: https://github.com/protocolbuffers/protobuf/issues/10051
+protobuf==3.19.5