Merge branch 'main' into metricbeat/windows-arm

.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml

@@ -39,10 +39,10 @@ steps:
         - github_commit_status:
             context: "deploy/k8s checks"
 
-    - label: "K8S Test/K8S version: v1.30.2"
-      key: "k8s-test-130"
+    - label: "K8S Test/K8S version: v1.31.0"
+      key: "k8s-test-131"
       env:
-        K8S_VERSION: "v1.30.2"
+        K8S_VERSION: "v1.31.0"
         MODULE: "${MODULE}"
       commands: |
         set -euo pipefail
@@ -59,12 +59,12 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "deploy/k8s test v1.30.2"
+            context: "deploy/k8s test v1.31.0"
 
-    - label: "K8S Test/K8S version: v1.29.4"
-      key: "k8s-test-129"
+    - label: "K8S Test/K8S version: v1.30.4"
+      key: "k8s-test-130"
       env:
-        K8S_VERSION: "v1.29.4"
+        K8S_VERSION: "v1.30.4"
         MODULE: "${MODULE}"
       commands: |
         set -euo pipefail
@@ -81,12 +81,12 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "deploy/k8s test v1.29.4"
+            context: "deploy/k8s test v1.30.4"
 
-    - label: "K8S Test/K8S version: v1.28.9"
-      key: "k8s-test-128"
+    - label: "K8S Test/K8S version: v1.29.8"
+      key: "k8s-test-129"
       env:
-        K8S_VERSION: "v1.28.9"
+        K8S_VERSION: "v1.29.8"
         MODULE: "${MODULE}"
       commands: |
         set -euo pipefail
@@ -103,12 +103,12 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "deploy/k8s test v1.28.9"
+            context: "deploy/k8s test v1.29.8"
 
-    - label: "K8S Test/K8S version: v1.27.13"
-      key: "k8s-test-127"
+    - label: "K8S Test/K8S version: v1.28.13"
+      key: "k8s-test-128"
       env:
-        K8S_VERSION: "v1.27.13"
+        K8S_VERSION: "v1.28.13"
         MODULE: "${MODULE}"
       commands: |
         set -euo pipefail
@@ -125,4 +125,4 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "deploy/k8s test v1.27.13"
+            context: "deploy/k8s test v1.28.13"

.golangci.yml

@@ -66,6 +66,7 @@ linters:
     - wastedassign # wastedassign finds wasted assignment statements.
     - gomodguard # check for blocked dependencies
     - depguard
+    - gomoddirectives
 
 # all available settings of specific linters
 linters-settings:
@@ -127,6 +128,25 @@ linters-settings:
               - github.com/gofrs/uuid/v5
             reason: "Use one uuid library consistently across the codebase"
 
+  gomoddirectives:
+    # Forbid local `replace` directives
+    replace-local: false
+
+    # Forbid any `replace` directives that are intended temporarily only during
+    # development. The modules listed below are intended to be replaced permanently.
+    replace-allow-list:
+      - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumption
+      - github.com/Shopify/sarama
+      - github.com/apoydence/eachers
+      - github.com/dop251/goja
+      - github.com/dop251/goja_nodejs
+      - github.com/fsnotify/fsevents
+      - github.com/fsnotify/fsnotify
+      - github.com/google/gopacket
+      - github.com/insomniacslk/dhcp
+      - github.com/meraki/dashboard-api-go/v3
+      - github.com/snowflakedb/gosnowflake
+
   gosimple:
     # Select the Go version to target. The default is '1.13'.
     go: "1.22.7"

CHANGELOG-developer.next.asciidoc

@@ -205,6 +205,8 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Add a configuration option for TCP/UDP network type. {issue}40407[40407] {pull}40623[40623]
 - Added debug logging to parquet reader in x-pack/libbeat/reader. {pull}40651[40651]
 - Added filebeat debug histograms for s3 object size and events per processed s3 object. {pull}40775[40775]
+- Simplified GCS input state checkpoint calculation logic. {issue}40878[40878] {pull}40937[40937] 
+- Simplified Azure Blob Storage input state checkpoint calculation logic. {issue}40674[40674] {pull}40936[40936]
 
 ==== Deprecated
 

CHANGELOG.asciidoc

@@ -7,7 +7,17 @@
 === Beats version 8.15.2
 https://github.com/elastic/beats/compare/v8.15.0\...v8.15.2[View commits]
 
-==== Breaking changes
+==== Known issues
+
+*Affecting all Beats*
+
+- Beats Docker images do not log to stderr by default. The workaround is to pass the CLI flag `-e` or to set `logging.to_stderr: true` in the configuration file. {issue}41118[41118]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix bug that prevented the Elasticsearch output from recovering from an interrupted connection. {issue}40705[40705] {pull}40769[40796]
 
 *Metricbeat*
 
@@ -17,6 +27,12 @@ https://github.com/elastic/beats/compare/v8.15.0\...v8.15.2[View commits]
 === Beats version 8.15.1
 https://github.com/elastic/beats/compare/v8.15.0\...v8.15.1[View commits]
 
+==== Known issues
+
+*Affecting all Beats*
+
+- Beats Docker images do not log to stderr by default. The workaround is to pass the CLI flag `-e` or to set `logging.to_stderr: true` in the configuration file. {issue}41118[41118]
+
 ==== Bugfixes
 
 *Affecting all Beats*
@@ -65,6 +81,10 @@ https://github.com/elastic/beats/compare/v8.14.3\...v8.15.0[View commits]
 
 ==== Known issues
 
+*Affecting all Beats*
+
+- Beats Docker images do not log to stderr by default. The workaround is to pass the CLI flag `-e` or to set `logging.to_stderr: true` in the configuration file. {issue}41118[41118]
+
 *Filebeat*
 
 - The Azure EventHub input in Filebeat is not found when running on Windows. Please refrain from upgrading to 8.15. See {issue}40608[40608] for details.

CHANGELOG.next.asciidoc

@@ -44,6 +44,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Journald: `seek` and `since` behaviour have been simplified, if there is a cursor (state) `seek` and `since` are ignored and the cursor is used. {pull}40061[40061]
 - Redis: Added replication role as a field to submitted slowlogs
 - Added `container.image.name` to `journald` Filebeat input's Docker-specific translated fields. {pull}40450[40450]
+- Remove deprecated awscloudwatch field from Filebeat. {pull}41089[41089]
 
 
 *Heartbeat*
@@ -168,6 +169,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Ensure netflow custom field configuration is applied. {issue}40735[40735] {pull}40730[40730]
 - Fix replace processor handling of zero string replacement validation. {pull}40751[40751]
 - Fix long filepaths in diagnostics exceeding max path limits on Windows. {pull}40909[40909]
+- Add backup and delete for AWS S3 polling mode feature back. {pull}41071[41071]
+- Fix a bug in Salesforce input to only handle responses with 200 status code {pull}41015[41015]
 
 
 *Heartbeat*
@@ -204,6 +207,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Packetbeat*
 
+- Fix upload of bundled ingest pipelines on Windows. {pull}41110[41110]
 
 *Winlogbeat*
 
@@ -304,10 +308,13 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Disable event normalization for netflow input {pull}40635[40635]
 - Allow attribute selection in the Active Directory entity analytics provider. {issue}40482[40482] {pull}40662[40662]
 - Improve error quality when CEL program does not correctly return an events array. {pull}40580[40580]
+- Added support for Microsoft Entra ID RBAC authentication. {issue}40434[40434] {pull}40879[40879]
 - Add `use_kubeadm` config option for filebeat (both filbeat.input and autodiscovery) in order to toggle kubeadm-config api requests {pull}40301[40301]
 - Make HTTP library function inclusion non-conditional in CEL input. {pull}40912[40912]
 - Add support for Crowdstrike streaming API to the streaming input. {issue}40264[40264] {pull}40838[40838]
 - Add support to CEL for reading host environment variables. {issue}40762[40762] {pull}40779[40779]
+- Add CSV decoder to awss3 input. {pull}40896[40896]
+- Change request trace logging to include headers instead of complete request. {pull}41072[41072]
 
 *Auditbeat*
 
@@ -321,6 +328,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Added status to monitor run log report.
 - Upgrade node to latest LTS v18.20.3. {pull}40038[40038]
 - Add journey duration to synthetics browser events. {pull}40230[40230]
+- Add monitor status reporter under managed mode. {pull}41077[41077]
 
 *Metricbeat*