Merge branch 'main' into yang.song/OTEL-2359

.gitlab-ci.yml

@@ -380,7 +380,7 @@ variables:
   if: ($DEPLOY_AGENT == "true" || $DDR_WORKFLOW_ID != null) && $BUCKET_BRANCH == "beta" && $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$/
 
 .if_scheduled_main: &if_scheduled_main
-  if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "main"
+  if: ($CI_PIPELINE_SOURCE == "schedule" || ($DDR_WORKFLOW_ID != null && $APPS =~ /^beta-build-/)) && $CI_COMMIT_BRANCH == "main"
 
 # Rule to trigger jobs only when a branch matches the mergequeue pattern.
 .if_mergequeue: &if_mergequeue
@@ -644,11 +644,6 @@ workflow:
 .on_all_builds:
   - <<: *if_run_all_builds
 
-.on_all_builds_manual:
-  - <<: *if_run_all_builds
-    when: manual
-    allow_failure: true
-
 .on_e2e_tests:
   - <<: *if_installer_tests
 

.gitlab/binary_build/system_probe.yml

@@ -12,9 +12,7 @@
     - find "$CI_BUILDS_DIR" ! -path '*DataDog/datadog-agent*' -delete || true  # Allow failure, we can't remove parent folders of datadog-agent
   script:
     - inv check-go-version
-    - inv -e system-probe.build --strip-object-files
-    # fail if references to glibc >= 2.18
-    - objdump -p $CI_PROJECT_DIR/$SYSTEM_PROBE_BINARIES_DIR/system-probe | egrep 'GLIBC_2\.(1[8-9]|[2-9][0-9])' && exit 1
+    - inv -e system-probe.build-object-files --strip-object-files
     - inv -e system-probe.save-build-outputs $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz
   variables:
     KUBERNETES_MEMORY_REQUEST: "6Gi"

.gitlab/common/test_infra_version.yml

@@ -5,5 +5,5 @@
 
 ---
 variables:
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: f1880fef5f48
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 9f38bc4eab68
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ''

.gitlab/deploy_containers/deploy_containers_a7.yml

@@ -207,13 +207,13 @@ deploy_containers_latest-a7:
         IMG_SOURCES: "%BASE%-win1809-servercore-amd64"
         IMG_DESTINATIONS: ${AGENT_REPOSITORY}:7-servercore-ltsc2019,${AGENT_REPOSITORY}:latest-servercore-ltsc2019
       - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7"
-        IMG_SOURCES: "BASE%-winltsc2022-servercore-amd64"
+        IMG_SOURCES: "%BASE%-winltsc2022-servercore-amd64"
         IMG_DESTINATIONS: ${AGENT_REPOSITORY}:7-servercore-ltsc2022,${AGENT_REPOSITORY}:latest-servercore-ltsc2022
       - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx"
         IMG_SOURCES: "%BASE%-winltsc2022-servercore-amd64"
         IMG_DESTINATIONS: ${AGENT_REPOSITORY}:7-servercore-ltsc2022-jmx,${AGENT_REPOSITORY}:latest-servercore-ltsc2022-jmx
       - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx"
-        IMG_SOURCES: "%BASE%-winltsc1809-servercore-amd64"
+        IMG_SOURCES: "%BASE%-win1809-servercore-amd64"
         IMG_DESTINATIONS: ${AGENT_REPOSITORY}:7-servercore-ltsc2019-jmx,${AGENT_REPOSITORY}:latest-servercore-ltsc2019-jmx
 
 deploy_containers_latest-a7_internal:

.gitlab/deploy_cws_instrumentation/deploy_cws_instrumentation.yml

@@ -20,7 +20,7 @@ include:
 # will push the `7.xx.y-rc.z` tags
 deploy_containers-cws-instrumentation-rc-versioned:
   extends: .deploy_containers-cws-instrumentation-base
-  rules: !reference [.on_deploy_rc]
+  rules: !reference [.on_deploy_manual_auto_on_rc]
 
 # will update the `rc` tag
 deploy_containers-cws-instrumentation-rc-mutable:

.gitlab/dev_container_deploy/docker_linux.yml

@@ -126,7 +126,7 @@ dca_dev_branch:
 dca_dev_branch_multiarch:
   extends: .docker_publish_job_definition
   stage: dev_container_deploy
-  rules: !reference [.on_all_builds_manual]
+  rules: !reference [.manual]
   needs:
     - docker_build_cluster_agent_amd64
     - docker_build_cluster_agent_arm64
@@ -149,7 +149,7 @@ dca_dev_master:
 cws_instrumentation_dev_branch_multiarch:
   extends: .docker_publish_job_definition
   stage: dev_container_deploy
-  rules: !reference [.on_all_builds_manual]
+  rules: !reference [.manual]
   needs:
     - docker_build_cws_instrumentation_amd64
     - docker_build_cws_instrumentation_arm64

.gitlab/e2e/e2e.yml

@@ -316,7 +316,6 @@ new-e2e-cws:
     - qa_agent
     - qa_dca
   variables:
-    SHOULD_RUN_IN_FLAKES_FINDER: "false" # Currently broken in flake finder ADXT-687
     TARGETS: ./tests/cws
     TEAM: csm-threats-agent
     CWS_INSTRUMENTATION_FULLIMAGEPATH: 669783387624.dkr.ecr.us-east-1.amazonaws.com/cws-instrumentation:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
@@ -334,6 +333,7 @@ new-e2e-discovery:
   needs:
     - !reference [.needs_new_e2e_template]
     - deploy_deb_testing-a7_x64
+    - qa_agent
   rules:
     - !reference [.on_discovery_or_e2e_changes]
     - !reference [.manual]
@@ -533,7 +533,6 @@ new-e2e-windows-systemprobe:
   variables:
     TARGETS: ./tests/sysprobe-functional
     TEAM: windows-kernel-integrations
-    SHOULD_RUN_IN_FLAKES_FINDER: "false" # Currently broken in flake finder ADXT-687
   parallel:
     matrix:
       - EXTRA_PARAMS: --run TestUSMAutoTaggingSuite
@@ -672,6 +671,7 @@ generate-flakes-finder-pipeline:
     - qa_dogstatsd
     - qa_agent
     - qa_agent_ot
+    - tests_windows_sysprobe_x64
   tags: ["arch:amd64"]
   script:
     - inv -e testwasher.generate-flake-finder-pipeline
@@ -689,6 +689,7 @@ trigger-flakes-finder:
   variables:
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
     PARENT_COMMIT_SHA: $CI_COMMIT_SHORT_SHA
+    PARENT_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA
   trigger:
     include:
       - artifact: flake-finder-gitlab-ci.yml

.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml

@@ -12,7 +12,7 @@ internal_kubernetes_deploy_experimental:
       when: always
     - if: $CI_COMMIT_BRANCH != "main"
       when: never
-    - if: $DDR != "true"
+    - if: $DDR_WORKFLOW_ID == null
       when: never
     - if: $APPS !~ "/^datadog-agent/"
       when: never
@@ -74,7 +74,7 @@ notify-slack:
       when: always
     - if: $CI_COMMIT_BRANCH != "main"
       when: never
-    - if: $DDR != "true"
+    - if: $DDR_WORKFLOW_ID == null
       when: never
     - if: $APPS !~ "/^datadog-agent/"
       when: never

.gitlab/trigger_release/trigger_release.yml

@@ -37,8 +37,6 @@ trigger_auto_staging_release:
     AUTO_RELEASE: "true"
     TARGET_REPO: staging
   rules:
-    - if: $DDR == "true"
-      when: never
     - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+){0,1}$/
       when: never
     - !reference [.on_deploy]
@@ -73,13 +71,13 @@ generate_windows_gitlab_runner_bump_pr:
   needs: ["trigger_auto_staging_release"]
   tags: ["arch:amd64"]
   rules:
-    - if: $DDR == "true"
+    - if: $DDR_WORKFLOW_ID != null
       when: never
     - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+){0,1}$/
       when: never
     - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$/
 
-  script: 
+  script:
     # We are using the agent platform auto PR github app to access the buildenv repository (already used for macOS builds)
     - !reference [.setup_github_app_agent_platform_auto_pr]
     - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-notifications.txt

cmd/secrethelper/providers/k8s_secret.go

@@ -6,18 +6,17 @@
 package providers
 
 import (
-	"context"
 	"fmt"
 	"strings"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-
 	"github.com/DataDog/datadog-agent/comp/core/secrets"
 )
 
+// KubeSecretGetter is a function that fetches a secret from k8s
+type KubeSecretGetter func(string, string) (map[string][]byte, error)
+
 // ReadKubernetesSecret reads a secrets store in k8s
-func ReadKubernetesSecret(kubeClient kubernetes.Interface, path string) secrets.SecretVal {
+func ReadKubernetesSecret(readSecretFromKubeClient KubeSecretGetter, path string) secrets.SecretVal {
 	splitName := strings.Split(path, "/")
 
 	if len(splitName) != 3 {
@@ -26,12 +25,12 @@ func ReadKubernetesSecret(kubeClient kubernetes.Interface, path string) secrets.
 
 	namespace, name, key := splitName[0], splitName[1], splitName[2]
 
-	secret, err := kubeClient.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+	secret, err := readSecretFromKubeClient(namespace, name)
 	if err != nil {
 		return secrets.SecretVal{ErrorMsg: err.Error()}
 	}
 
-	value, ok := secret.Data[key]
+	value, ok := secret[key]
 	if !ok {
 		return secrets.SecretVal{ErrorMsg: fmt.Sprintf("key %s not found in secret %s/%s", key, namespace, name)}
 	}

cmd/secrethelper/providers/k8s_secret_test.go

@@ -6,6 +6,7 @@
 package providers
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -74,7 +75,15 @@ func TestReadKubernetesSecret(t *testing.T) {
 			}
 			kubeClient := fake.NewSimpleClientset(kubeObjects...)
 
-			resolvedSecret := ReadKubernetesSecret(kubeClient, test.secretPath)
+			secretGetter := func(namespace, name string) (map[string][]byte, error) {
+				secret, err := kubeClient.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+				if err != nil {
+					return nil, err
+				}
+				return secret.Data, nil
+			}
+
+			resolvedSecret := ReadKubernetesSecret(secretGetter, test.secretPath)
 
 			if test.expectedError != "" {
 				assert.Equal(t, test.expectedError, resolvedSecret.ErrorMsg)

cmd/secrethelper/secret_helper.go

@@ -32,11 +32,9 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/spf13/cobra"
 	"go.uber.org/fx"
-	"k8s.io/client-go/kubernetes"
 
 	"github.com/DataDog/datadog-agent/cmd/secrethelper/providers"
 	"github.com/DataDog/datadog-agent/comp/core/secrets"
@@ -51,9 +49,6 @@ const (
 	k8sSecretPrefix         = "k8s_secret"
 )
 
-// NewKubeClient returns a new kubernetes.Interface
-type NewKubeClient func(timeout time.Duration, qps float32, burst int) (kubernetes.Interface, error)
-
 // cliParams are the command-line arguments for this subcommand
 type cliParams struct {
 	usePrefixes bool
@@ -100,17 +95,17 @@ func readCmd(cliParams *cliParams) error {
 		dir = cliParams.args[0]
 	}
 
-	return readSecrets(os.Stdin, os.Stdout, dir, cliParams.usePrefixes, apiserver.GetKubeClient)
+	return readSecrets(os.Stdin, os.Stdout, dir, cliParams.usePrefixes, apiserver.GetKubeSecret)
 }
 
-func readSecrets(r io.Reader, w io.Writer, dir string, usePrefixes bool, newKubeClientFunc NewKubeClient) error {
+func readSecrets(r io.Reader, w io.Writer, dir string, usePrefixes bool, kubeSecretGetter providers.KubeSecretGetter) error {
 	inputSecrets, err := parseInputSecrets(r)
 	if err != nil {
 		return err
 	}
 
 	if usePrefixes {
-		return writeFetchedSecrets(w, readSecretsUsingPrefixes(inputSecrets, dir, newKubeClientFunc))
+		return writeFetchedSecrets(w, readSecretsUsingPrefixes(inputSecrets, dir, kubeSecretGetter))
 	}
 
 	return writeFetchedSecrets(w, readSecretsFromFile(inputSecrets, dir))
@@ -161,7 +156,7 @@ func readSecretsFromFile(secretsList []string, dir string) map[string]secrets.Se
 	return res
 }
 
-func readSecretsUsingPrefixes(secretsList []string, rootPath string, newKubeClientFunc NewKubeClient) map[string]secrets.SecretVal {
+func readSecretsUsingPrefixes(secretsList []string, rootPath string, kubeSecretGetter providers.KubeSecretGetter) map[string]secrets.SecretVal {
 	res := make(map[string]secrets.SecretVal)
 
 	for _, secretID := range secretsList {
@@ -175,12 +170,7 @@ func readSecretsUsingPrefixes(secretsList []string, rootPath string, newKubeClie
 		case filePrefix:
 			res[secretID] = providers.ReadSecretFile(id)
 		case k8sSecretPrefix:
-			kubeClient, err := newKubeClientFunc(10*time.Second, 0, 0) // Default QPS and burst to Kube client defaults using 0
-			if err != nil {
-				res[secretID] = secrets.SecretVal{Value: "", ErrorMsg: err.Error()}
-			} else {
-				res[secretID] = providers.ReadKubernetesSecret(kubeClient, id)
-			}
+			res[secretID] = providers.ReadKubernetesSecret(kubeSecretGetter, id)
 		default:
 			res[secretID] = secrets.SecretVal{Value: "", ErrorMsg: fmt.Sprintf("provider not supported: %s", prefix)}
 		}

cmd/secrethelper/secret_helper_test.go

@@ -7,29 +7,35 @@ package secrethelper
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"path/filepath"
 	"strings"
 	"testing"
-	"time"
 
-	"github.com/DataDog/datadog-agent/pkg/util/fxutil"
 	"github.com/stretchr/testify/assert"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
+
+	"github.com/DataDog/datadog-agent/pkg/util/fxutil"
 )
 
 func TestReadSecrets(t *testing.T) {
-	newKubeClientFunc := func(_ time.Duration, _ float32, _ int) (kubernetes.Interface, error) {
-		return fake.NewSimpleClientset(&v1.Secret{
+	newKubeClientFunc := func(namespace, name string) (map[string][]byte, error) {
+		kubeClient := fake.NewSimpleClientset(&v1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "some_name",
 				Namespace: "some_namespace",
 			},
 			Data: map[string][]byte{"some_key": []byte("some_value")},
-		}), nil
+		})
+
+		secret, err := kubeClient.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			return nil, err
+		}
+		return secret.Data, nil
 	}
 
 	tests := []struct {

cmd/system-probe/config/adjust_usm.go

@@ -71,6 +71,15 @@ func adjustUSM(cfg model.Config) {
 	applyDefault(cfg, smNS("max_postgres_stats_buffered"), 100000)
 	applyDefault(cfg, smNS("max_redis_stats_buffered"), 100000)
 
+	// kernel_buffer_pages determines the number of pages allocated *per CPU*
+	// for buffering kernel data, whether using a perf buffer or a ring buffer.
+	applyDefault(cfg, smNS("kernel_buffer_pages"), 16)
+
+	// data_channel_size defines the size of the Go channel that buffers events.
+	// Each event has a fixed size of approximately 4KB (sizeof(batch_data_t)).
+	// By setting this value to 100, the channel will buffer up to ~400KB of data in the Go heap memory.
+	applyDefault(cfg, smNS("data_channel_size"), 100)
+
 	validateInt(cfg, smNS("http_notification_threshold"), cfg.GetInt(smNS("max_tracked_http_connections"))/2, func(v int) error {
 		limit := cfg.GetInt(smNS("max_tracked_http_connections"))
 		if v >= limit {

comp/api/api/def/go.mod

@@ -11,5 +11,5 @@ require (
 	go.uber.org/dig v1.18.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
 )

comp/api/authtoken/go.mod

@@ -107,8 +107,8 @@ require (
 	go.uber.org/dig v1.18.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect
-	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect
+	golang.org/x/sys v0.29.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect