Exclude integration error logs by default. (#1816)

* Exclude integration error logs by default (Except the one causing to exit with status > 1) but allow including them using filters.
* Add Error and Fatal Logs inclussion on log_configuration.md

Author: alvarocabanas

docs/log_configuration.md

@@ -41,7 +41,7 @@ strongly recommend using the new configuration.
 The new log configuration has added two new options to filter logs based on key-values (fields). They can be used in
 order to remove logging noise in a troubleshooting scenario.
 
-By default, all entries will be included* in the logs (`include_filters`). To exclude some entries, we must define the
+By default, all entries will be included* in the logs (`include_filters`) except the integration execution errors. To exclude some entries, we must define the
 key-values to remove using the `exclude_filters` option. The following text is a usual agent's log line:
 
 `time="2022-06-10T15:46:38Z" level=debug msg="Integration instances finished their execution. Waiting until next interval." component=integrations.runner.Runner integration_name=nri-flex runner_uid=c03734e49d`
@@ -79,6 +79,16 @@ log:
       - nri-nginx
 ```
 
+* Only general information from an integration execution is logged, Error and Fatal logs are only visible under debug level. If we want to always show them under any log level for a specific integration we'll need to add it under the include_filters:
+
+```yaml
+log:
+  include_filters:
+    integration_name:
+      - nri-flex
+      - nri-nginx
+```
+
 * The integration supervisor trace logs are excluded by default due the big amount of produced log entries, the
   following configuration would enable them:
 

internal/integrations/v4/runner/runner.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/newrelic/infrastructure-agent/internal/agent/instrumentation"
 	"github.com/newrelic/infrastructure-agent/internal/integrations/v4/constants"
+	"github.com/newrelic/infrastructure-agent/pkg/config" //nolint:depguard
 	"github.com/newrelic/infrastructure-agent/pkg/entity/host"
 
 	"github.com/newrelic/infrastructure-agent/internal/integrations/v4/cache"
@@ -310,20 +311,29 @@ func (r *runner) handleStderr(stderr <-chan []byte) {
 				logLine := r.log.WithFields(logrus.Fields(fields))
 				logMessage := "integration log"
 
+				lineWasParsed = true
+
+				if r.log.IsDebugEnabled() {
+					// If Debug is enabled the level is ignored.
+					logLine.Debug(logMessage)
+
+					break
+				}
+
 				lvl, ok := lvl.(string)
 				if ok {
 					switch strings.ToLower(lvl) {
-					// We will set all this levels as debug to not be shown by default
+					// We only want to respect Fatal and Error levels.
 					case "info", "debug", "warn", "warning":
 						logLine.Debug(logMessage)
 					case "trace":
 						logLine.Trace(logMessage)
 					default:
-						logLine.Error(logMessage)
+						// This error log line has the filtered-by-default "integration-errors" component.
+						// Unless included on the config, this log lines won't be logged by the agent.
+						logLine.WithComponent(config.IntegrationsErrorsValue).Error(logMessage)
 					}
 
-					lineWasParsed = true
-
 					break
 				}
 			}

internal/integrations/v4/runner/runner_test.go

@@ -200,8 +200,6 @@ func Test_runner_Run_Integration_Log(t *testing.T) {
 	hook := new(test.Hook)
 	log.AddHook(hook)
 
-	log.SetLevel(logrus.TraceLevel)
-
 	testCases := []struct {
 		name           string
 		logLine        string
@@ -224,7 +222,7 @@ func Test_runner_Run_Integration_Log(t *testing.T) {
 			name:           "SDK_Trace_log",
 			logLine:        "[TRACE] This is a trace message",
 			expectedLogMsg: "This is a trace message",
-			expectedLevel:  logrus.TraceLevel,
+			expectedLevel:  logrus.DebugLevel,
 		},
 		{
 			name:           "SDK_Warning_log",
@@ -260,7 +258,7 @@ func Test_runner_Run_Integration_Log(t *testing.T) {
 			name:           "Logrus_Trace_log",
 			logLine:        "level=trace msg=\"This is a trace message\"",
 			expectedLogMsg: "This is a trace message",
-			expectedLevel:  logrus.TraceLevel,
+			expectedLevel:  logrus.DebugLevel,
 		},
 		{
 			name:           "Logrus_Warning_log",
@@ -291,7 +289,8 @@ func Test_runner_Run_Integration_Log(t *testing.T) {
 	for _, tt := range testCases {
 		testCase := tt
 		t.Run(testCase.name, func(t *testing.T) {
-			t.Parallel()
+			log.SetLevel(testCase.expectedLevel)
+
 			// GIVEN a runner that receives a cfg request without a handle function.
 			def, err := integration.NewDefinition(config.ConfigEntry{
 				InstanceName: testCase.name,

pkg/config/config.go

@@ -43,6 +43,9 @@ const (
 	FeatureTrace         = "feature"
 	ProcessTrace         = "process"
 
+	IntegrationsErrorsField = "component"
+	IntegrationsErrorsValue = "integration-errors"
+
 	// LogFilterWildcard will match everything.
 	LogFilterWildcard = "*"
 
@@ -1332,6 +1335,9 @@ func (lc *LogConfig) AttachDefaultFilters() {
 
 	// Exclude by default supervisor and feature traces.
 	lc.ExcludeFilters[TracesFieldName] = append(lc.ExcludeFilters[TracesFieldName], SupervisorTrace, FeatureTrace, ProcessTrace)
+
+	// Exclude all integration error logs by default
+	lc.ExcludeFilters[IntegrationsErrorsField] = append(lc.ExcludeFilters[IntegrationsErrorsField], IntegrationsErrorsValue)
 }
 
 // HasIncludeFilter returns true if key-value pair are included in the filtering configuration.

pkg/config/config_darwin_test.go

@@ -115,7 +115,9 @@ func Test_ParseLogConfigRule_EnvVar(t *testing.T) {
 		Forward:              nil,
 		SmartLevelEntryLimit: &expectedSmartLevelEntryLimit,
 		IncludeFilters:       map[string][]interface{}{"component": {"ProcessSample", "StorageSample"}},
-		ExcludeFilters:       map[string][]interface{}{TracesFieldName: {SupervisorTrace, FeatureTrace, ProcessTrace}},
+		ExcludeFilters: map[string][]interface{}{
+			TracesFieldName: {SupervisorTrace, FeatureTrace, ProcessTrace}, IntegrationsErrorsField: {IntegrationsErrorsValue},
+		},
 		Rotate: LogRotateConfig{
 			MaxSizeMb:          intPtr(0),
 			MaxFiles:           0,

pkg/config/config_linux_test.go

@@ -196,7 +196,9 @@ func Test_ParseLogConfigRule_EnvVar(t *testing.T) {
 		Forward:              nil,
 		SmartLevelEntryLimit: &expectedSmartLevelEntryLimit,
 		IncludeFilters:       map[string][]interface{}{"component": {"ProcessSample", "StorageSample"}},
-		ExcludeFilters:       map[string][]interface{}{TracesFieldName: {SupervisorTrace, FeatureTrace, ProcessTrace}},
+		ExcludeFilters: map[string][]interface{}{
+			TracesFieldName: {SupervisorTrace, FeatureTrace, ProcessTrace}, IntegrationsErrorsField: {IntegrationsErrorsValue},
+		},
 		Rotate: LogRotateConfig{
 			MaxSizeMb:          intPtr(0),
 			MaxFiles:           0,

pkg/config/config_test.go

@@ -783,6 +783,7 @@ func TestLoadLogConfig_BackwardsCompatability(t *testing.T) {
 	}
 }
 
+//nolint:exhaustruct,lll
 func TestLoadLogConfig_Populate(t *testing.T) {
 	// TODO: migrate to generic function with go1.18
 	boolPtr := func(a bool) *bool {
@@ -796,10 +797,10 @@ func TestLoadLogConfig_Populate(t *testing.T) {
 		c                 Config
 		expectedLogConfig LogConfig
 	}{
-		{"Verbose disabled (info level) and custom log file", Config{Verbose: 0, LogFile: "agent.log"}, LogConfig{Level: LogLevelInfo, File: "agent.log", ToStdout: boolPtr(false), Forward: boolPtr(false), ExcludeFilters: LogFilters{"traces": []interface{}{"supervisor", "feature", "process"}}, SmartLevelEntryLimit: intPtr(0)}},
-		{"Smart Verbose enabled with defined limit", Config{Verbose: 2, SmartVerboseModeEntryLimit: 200}, LogConfig{Level: LogLevelSmart, File: "", ToStdout: boolPtr(false), Forward: boolPtr(false), ExcludeFilters: LogFilters{"traces": []interface{}{"supervisor", "feature", "process"}}, SmartLevelEntryLimit: intPtr(200)}},
-		{"Forward Verbose enabled and stdout", Config{Verbose: 3, LogToStdout: true}, LogConfig{Level: LogLevelDebug, File: "", ToStdout: boolPtr(true), Forward: boolPtr(true), ExcludeFilters: LogFilters{"traces": []interface{}{"supervisor", "feature", "process"}}, SmartLevelEntryLimit: intPtr(0)}},
-		{"Trace Verbose enabled and file", Config{Verbose: 4, LogFile: "agent.log"}, LogConfig{Level: LogLevelTrace, File: "agent.log", ToStdout: boolPtr(false), Forward: boolPtr(false), ExcludeFilters: LogFilters{"traces": []interface{}{"supervisor", "feature", "process"}}, SmartLevelEntryLimit: intPtr(0)}},
+		{"Verbose disabled (info level) and custom log file", Config{Verbose: 0, LogFile: "agent.log"}, LogConfig{Level: LogLevelInfo, File: "agent.log", ToStdout: boolPtr(false), Forward: boolPtr(false), ExcludeFilters: LogFilters{"traces": []interface{}{"supervisor", "feature", "process"}, "component": []interface{}{"integration-errors"}}, SmartLevelEntryLimit: intPtr(0)}},
+		{"Smart Verbose enabled with defined limit", Config{Verbose: 2, SmartVerboseModeEntryLimit: 200}, LogConfig{Level: LogLevelSmart, File: "", ToStdout: boolPtr(false), Forward: boolPtr(false), ExcludeFilters: LogFilters{"traces": []interface{}{"supervisor", "feature", "process"}, "component": []interface{}{"integration-errors"}}, SmartLevelEntryLimit: intPtr(200)}},
+		{"Forward Verbose enabled and stdout", Config{Verbose: 3, LogToStdout: true}, LogConfig{Level: LogLevelDebug, File: "", ToStdout: boolPtr(true), Forward: boolPtr(true), ExcludeFilters: LogFilters{"traces": []interface{}{"supervisor", "feature", "process"}, "component": []interface{}{"integration-errors"}}, SmartLevelEntryLimit: intPtr(0)}},
+		{"Trace Verbose enabled and file", Config{Verbose: 4, LogFile: "agent.log"}, LogConfig{Level: LogLevelTrace, File: "agent.log", ToStdout: boolPtr(false), Forward: boolPtr(false), ExcludeFilters: LogFilters{"traces": []interface{}{"supervisor", "feature", "process"}, "component": []interface{}{"integration-errors"}}, SmartLevelEntryLimit: intPtr(0)}},
 	}
 
 	for _, tt := range configs {

pkg/config/config_windows_test.go

@@ -113,7 +113,9 @@ func Test_ParseLogConfigRule_EnvVar(t *testing.T) {
 		Forward:              nil,
 		SmartLevelEntryLimit: &expectedSmartLevelEntryLimit,
 		IncludeFilters:       map[string][]interface{}{"component": {"ProcessSample", "StorageSample"}},
-		ExcludeFilters:       map[string][]interface{}{TracesFieldName: {SupervisorTrace, FeatureTrace, ProcessTrace}},
+		ExcludeFilters: map[string][]interface{}{
+			TracesFieldName: {SupervisorTrace, FeatureTrace, ProcessTrace}, IntegrationsErrorsField: {IntegrationsErrorsValue},
+		},
 		Rotate: LogRotateConfig{
 			MaxSizeMb:          intPtr(100),
 			MaxFiles:           5,