Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -514,6 +514,10 @@ DD_INSTRUMENTATION_TELEMETRY_URL: |
telemetry is in use (otherwise telemetry is sent to the agent using
<see cref="Datadog.Trace.Configuration.ExporterSettings.AgentUri"/> instead)

DD_INTERNAL_DYNAMIC_INSTRUMENTATION_MAX_PROBES_PER_TYPE: |
Internal configuration key to optionally limit the maximum number of probes of each type.
Default value is <c>0</c>, no probe-count limit is enforced.

DD_INTERNAL_PROFILING_NATIVE_ENGINE_PATH: |
This variable is set by the native loader after trying to load the profiler.
It means the profiler is _there_ though it may not be _loaded_. This only works
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1105,6 +1105,14 @@
"product": "Telemetry"
}
],
"DD_INTERNAL_DYNAMIC_INSTRUMENTATION_MAX_PROBES_PER_TYPE": [
{
"implementation": "A",
"type": "int",
"default": "0",
"product": "Debugger"
}
],
"DD_INTERNAL_RCM_POLL_INTERVAL": [
{
"implementation": "A",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,28 +15,25 @@ namespace Datadog.Trace.Debugger.Configurations
{
internal sealed class ConfigurationUpdater
{
private const int MaxAllowedLogProbes = 100;
private const int MaxAllowedMetricProbes = 100;
private const int MaxAllowedSpanProbes = 100;
private const int MaxAllowedSpanDecorationProbes = 100;

private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor<ConfigurationUpdater>();

private readonly string? _env;
private readonly string? _version;
private readonly int _maxProbesPerType;

private ProbeConfiguration _currentConfiguration;

private ConfigurationUpdater(string? env, string? version)
private ConfigurationUpdater(string? env, string? version, int maxProbesPerType)
{
_env = env;
_version = version;
_maxProbesPerType = maxProbesPerType;
_currentConfiguration = new ProbeConfiguration();
}

public static ConfigurationUpdater Create(string? environment, string? serviceVersion)
public static ConfigurationUpdater Create(string? environment, string? serviceVersion, int maxProbesPerType)
{
return new ConfigurationUpdater(environment, serviceVersion);
return new ConfigurationUpdater(environment, serviceVersion, maxProbesPerType);
}

public List<UpdateResult> AcceptAdded(ProbeConfiguration configuration)
Expand Down Expand Up @@ -77,21 +74,26 @@ private ProbeConfiguration ApplyConfigurationFilters(ProbeConfiguration configur
return new ProbeConfiguration()
{
ServiceConfiguration = configuration.ServiceConfiguration,
LogProbes = Filter(configuration.LogProbes, MaxAllowedLogProbes),
MetricProbes = Filter(configuration.MetricProbes, MaxAllowedMetricProbes),
SpanProbes = Filter(configuration.SpanProbes, MaxAllowedSpanProbes),
SpanDecorationProbes = Filter(configuration.SpanDecorationProbes, MaxAllowedSpanDecorationProbes)
LogProbes = Filter(configuration.LogProbes),
MetricProbes = Filter(configuration.MetricProbes),
SpanProbes = Filter(configuration.SpanProbes),
SpanDecorationProbes = Filter(configuration.SpanDecorationProbes)
};

T[] Filter<T>(T[] probes, int maxAllowedProbes)
T[] Filter<T>(T[] probes)
where T : ProbeDefinition
{
return
var filtered =
probes
.Where(probe => probe.Language == TracerConstants.Language)
.Where(IsEnvAndVersionMatch)
.Take(maxAllowedProbes)
.ToArray();
.Where(IsEnvAndVersionMatch);

if (_maxProbesPerType > 0)
{
filtered = filtered.Take(_maxProbesPerType);
}

return filtered.ToArray();

bool IsEnvAndVersionMatch(ProbeDefinition probe)
{
Expand Down
2 changes: 1 addition & 1 deletion tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ internal static DynamicInstrumentation CreateDynamicInstrumentation(IDiscoverySe
var diagnosticsUploader = CreateDiagnosticsUploader(discoveryService, debuggerSettings, gitMetadataTagsProvider, GetApiFactory(tracerSettings, true), diagnosticsSink);
var lineProbeResolver = LineProbeResolver.Create(debuggerSettings.ThirdPartyDetectionExcludes, debuggerSettings.ThirdPartyDetectionIncludes);
var probeStatusPoller = ProbeStatusPoller.Create(diagnosticsSink, debuggerSettings);
var configurationUpdater = ConfigurationUpdater.Create(tracerSettings.Manager.InitialMutableSettings.Environment, tracerSettings.Manager.InitialMutableSettings.ServiceVersion);
var configurationUpdater = ConfigurationUpdater.Create(tracerSettings.Manager.InitialMutableSettings.Environment, tracerSettings.Manager.InitialMutableSettings.ServiceVersion, debuggerSettings.MaxProbesPerType);

var statsd = GetDogStatsd(tracerSettings);

Expand Down
10 changes: 9 additions & 1 deletion tracer/src/Datadog.Trace/Debugger/DebuggerSettings.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// <copyright file="DebuggerSettings.cs" company="Datadog">
// <copyright file="DebuggerSettings.cs" company="Datadog">
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
// </copyright>
Expand All @@ -23,6 +23,7 @@ internal sealed record DebuggerSettings
public const int DefaultMaxNumberOfItemsInCollectionToCopy = 100;
public const int DefaultMaxNumberOfFieldsToCopy = 20;
public const int DefaultMaxStringLength = 1000;
public const int DefaultMaxProbesPerType = 0;

private const int DefaultUploadBatchSize = 100;
public const int DefaultSymbolBatchSizeInBytes = 100000;
Expand Down Expand Up @@ -63,6 +64,11 @@ public DebuggerSettings(IConfigurationSource? source, IConfigurationTelemetry te
.AsInt32(DefaultSymbolBatchSizeInBytes, batchSize => batchSize > 0)
.Value;

MaxProbesPerType = config
.WithKeys(ConfigurationKeys.Debugger.InternalDynamicInstrumentationMaxProbesPerType)
.AsInt32(DefaultMaxProbesPerType, maxProbes => maxProbes >= 0)
.Value;

var thirdPartyIncludes = config
.WithKeys(ConfigurationKeys.Debugger.ThirdPartyDetectionIncludes)
.AsString()?
Expand Down Expand Up @@ -165,6 +171,8 @@ public DebuggerSettings(IConfigurationSource? source, IConfigurationTelemetry te

public int SymbolDatabaseBatchSizeInBytes { get; }

public int MaxProbesPerType { get; }

public ImmutableHashSet<string> ThirdPartyDetectionIncludes { get; }

public ImmutableHashSet<string> ThirdPartyDetectionExcludes { get; }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ namespace Datadog.Trace.Debugger.Instrumentation.Collections
/// </summary>
internal abstract class EverGrowingCollection<TPayload>
{
private static readonly int[] CapacityRiskThresholds = [1024, 4096, 16384, 65536];

private int _highestReportedRiskThreshold = -1;

protected IDatadogLogger Log { get; } = DatadogLogging.GetLoggerFor(typeof(EverGrowingCollection<TPayload>));

protected object ItemsLocker { get; } = new object();
Expand Down Expand Up @@ -44,17 +48,41 @@ private void EnsureCapacity(int min)
{
// Enlarge the _items array as needed. Note that there is an implicit by-design memory-leak here,
// in the sense that this unbounded collection can only grow and is never shrunk.
// This is fine, for the time being, because we are limited to 100 simultaneous probes anyhow, so it is extremely unlikely
// we will ever reach any substantial memory usage here.
// After removing the default 100 probes cap, a long-running process with high probe churn can
// retain large backing arrays. We keep this implementation for now and emit warning logs as
// capacity crosses risk thresholds so we can monitor and decide if additional safeguards are needed.
if (Items.Length < min)
{
var previousCapacity = Items.Length;
var newCapacity = Items.Length * 2;
if (newCapacity < min)
{
newCapacity = min + 1;
}

Capacity = newCapacity;
ReportCapacityRiskIfNeeded(previousCapacity, newCapacity);
}
}

private void ReportCapacityRiskIfNeeded(int previousCapacity, int newCapacity)
{
for (var i = _highestReportedRiskThreshold + 1; i < CapacityRiskThresholds.Length; i++)
{
var threshold = CapacityRiskThresholds[i];
if (newCapacity >= threshold)
{
Log.Warning<int, int, int>(
"Collection capacity increased from {PreviousCapacity} to {NewCapacity} and reached risk threshold {Threshold} for retained memory.",
previousCapacity,
newCapacity,
threshold);
_highestReportedRiskThreshold = i;
}
else
{
break;
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,12 @@ internal static class Debugger
/// </summary>
public const string RateLimitSeconds = "DD_EXCEPTION_REPLAY_RATE_LIMIT_SECONDS";

/// <summary>
/// Internal configuration key to optionally limit the maximum number of probes of each type.
/// Default value is <c>0</c>, no probe-count limit is enforced.
/// </summary>
public const string InternalDynamicInstrumentationMaxProbesPerType = "DD_INTERNAL_DYNAMIC_INSTRUMENTATION_MAX_PROBES_PER_TYPE";

/// <summary>
/// Configuration key for the maximum symbol size to upload (in bytes).
/// Default value is 1 mb.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,12 @@ internal static class Debugger
/// </summary>
public const string RateLimitSeconds = "DD_EXCEPTION_REPLAY_RATE_LIMIT_SECONDS";

/// <summary>
/// Internal configuration key to optionally limit the maximum number of probes of each type.
/// Default value is <c>0</c>, no probe-count limit is enforced.
/// </summary>
public const string InternalDynamicInstrumentationMaxProbesPerType = "DD_INTERNAL_DYNAMIC_INSTRUMENTATION_MAX_PROBES_PER_TYPE";

/// <summary>
/// Configuration key for the maximum symbol size to upload (in bytes).
/// Default value is 1 mb.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,12 @@ internal static class Debugger
/// </summary>
public const string RateLimitSeconds = "DD_EXCEPTION_REPLAY_RATE_LIMIT_SECONDS";

/// <summary>
/// Internal configuration key to optionally limit the maximum number of probes of each type.
/// Default value is <c>0</c>, no probe-count limit is enforced.
/// </summary>
public const string InternalDynamicInstrumentationMaxProbesPerType = "DD_INTERNAL_DYNAMIC_INSTRUMENTATION_MAX_PROBES_PER_TYPE";

/// <summary>
/// Configuration key for the maximum symbol size to upload (in bytes).
/// Default value is 1 mb.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,12 @@ internal static class Debugger
/// </summary>
public const string RateLimitSeconds = "DD_EXCEPTION_REPLAY_RATE_LIMIT_SECONDS";

/// <summary>
/// Internal configuration key to optionally limit the maximum number of probes of each type.
/// Default value is <c>0</c>, no probe-count limit is enforced.
/// </summary>
public const string InternalDynamicInstrumentationMaxProbesPerType = "DD_INTERNAL_DYNAMIC_INSTRUMENTATION_MAX_PROBES_PER_TYPE";

/// <summary>
/// Configuration key for the maximum symbol size to upload (in bytes).
/// Default value is 1 mb.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,43 @@ public void InvalidUploadFlushInterval_DefaultUsed(string value)
settings.UploadFlushIntervalMilliseconds.Should().Be(0);
}

[Fact]
public void MaxProbesPerType_DefaultUsed_WhenNotSet()
{
var settings = new DebuggerSettings(
new NameValueConfigurationSource(new()),
NullConfigurationTelemetry.Instance);

settings.MaxProbesPerType.Should().Be(0);
}

[Theory]
[InlineData("0", 0)]
[InlineData("1", 1)]
[InlineData("100", 100)]
public void MaxProbesPerType_UsesProvidedValue_WhenValid(string value, int expected)
{
var settings = new DebuggerSettings(
new NameValueConfigurationSource(new() { { ConfigurationKeys.Debugger.InternalDynamicInstrumentationMaxProbesPerType, value }, }),
NullConfigurationTelemetry.Instance);

settings.MaxProbesPerType.Should().Be(expected);
}

[Theory]
[InlineData("-1")]
[InlineData("")]
[InlineData("abc")]
[InlineData(null)]
public void MaxProbesPerType_DefaultUsed_WhenInvalid(string value)
{
var settings = new DebuggerSettings(
new NameValueConfigurationSource(new() { { ConfigurationKeys.Debugger.InternalDynamicInstrumentationMaxProbesPerType, value }, }),
NullConfigurationTelemetry.Instance);

settings.MaxProbesPerType.Should().Be(0);
}

public class DebuggerSettingsCodeOriginTests
{
[Theory]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public async Task DynamicInstrumentationEnabled_ServicesCalled()
var logUploader = new LogUploaderMock();
var diagnosticsUploader = new UploaderMock();
var probeStatusPoller = new ProbeStatusPollerMock();
var updater = ConfigurationUpdater.Create("env", "version");
var updater = ConfigurationUpdater.Create("env", "version", 0);

var debugger = new DynamicInstrumentation(settings, discoveryService, rcmSubscriptionManagerMock, lineProbeResolver, snapshotUploader, logUploader, diagnosticsUploader, probeStatusPoller, updater, NoOpStatsd.Instance);
debugger.Initialize();
Expand Down Expand Up @@ -77,7 +77,7 @@ public void DynamicInstrumentationDisabled_ServicesNotCalled()
var logUploader = new LogUploaderMock();
var diagnosticsUploader = new UploaderMock();
var probeStatusPoller = new ProbeStatusPollerMock();
var updater = ConfigurationUpdater.Create(string.Empty, string.Empty);
var updater = ConfigurationUpdater.Create(string.Empty, string.Empty, 0);

var debugger = new DynamicInstrumentation(settings, discoveryService, rcmSubscriptionManagerMock, lineProbeResolver, snapshotUploader, logUploader, diagnosticsUploader, probeStatusPoller, updater, NoOpStatsd.Instance);
debugger.Initialize();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,7 @@
"DD_DYNAMIC_INSTRUMENTATION_SYMBOL_DATABASE_BATCH_SIZE_BYTES": "dynamic_instrumentation_symbol_database_batch_size_bytes",
"DD_SYMBOL_DATABASE_UPLOAD_ENABLED": "symbol_database_upload_enabled",
"DD_DYNAMIC_INSTRUMENTATION_SYMBOL_DATABASE_UPLOAD_ENABLED": "dynamic_instrumentation_symbol_database_upload_enabled",
"DD_INTERAL_FORCE_SYMBOL_DATABASE_UPLOAD": "internal_force_symbol_database_upload",
"DD_INTERNAL_DYNAMIC_INSTRUMENTATION_MAX_PROBES_PER_TYPE": "dynamic_instrumentation_max_probes_per_type",
"DD_THIRD_PARTY_DETECTION_INCLUDES": "third_party_detection_includes",
"DD_THIRD_PARTY_DETECTION_EXCLUDES": "third_party_detection_excludes",
"DD_SYMBOL_DATABASE_THIRD_PARTY_DETECTION_INCLUDES": "symbol_database_third_party_detection_includes",
Expand Down
Loading