Skip to content

Commit 7c6440c

Browse files
tonyredondotonyredondo
authored
[Test Optimization] ci: git caching + env improvements (#8072)
## Summary of changes - Add git command caching and telemetry improvements for CI visibility. - Update GitInfo discovery and CI environment logging. - Update impacted tests for new git discovery behavior. JIRA: SDTEST-3226 ## Reason for change Reduce git command overhead and improve visibility into CI git metadata collection. ## Implementation details - Disk cache keyed by RunId in GitCommandHelper with safe.directory handling. - JSON annotations in ProcessHelpers for cache serialization. - Adjusted CI runner git discovery logic and tests. ## Test coverage CI passes then all changes are good. ## Other details - Stacked PRs (current marked): - PR1 #8070 - PR2 #8071 - **CURRENT**: PR3 #8072 - PR4 #8073
1 parent 886954d commit 7c6440c

File tree

12 files changed

+271
-663
lines changed

12 files changed

+271
-663
lines changed

‎tracer/src/Datadog.Trace.Tools.Runner/CiUtils.cs‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -134,13 +134,6 @@ public static async Task<InitResults> InitializeCiCommandsAsync(
134134

135135
// Change the .git search folder to the CurrentDirectory or WorkingFolder
136136
var ciValues = testOptimization.CIValues;
137-
ciValues.GitSearchFolder = Environment.CurrentDirectory;
138-
if (string.IsNullOrEmpty(ciValues.WorkspacePath))
139-
{
140-
// In case we cannot get the WorkspacePath we fallback to the default configuration.
141-
ciValues.GitSearchFolder = null;
142-
}
143-
144137
var client = TestOptimizationClient.Create(ciValues.WorkspacePath ?? Environment.CurrentDirectory, testOptimization);
145138
if (testOptimizationSettings.GitUploadEnabled != false || testOptimizationSettings.IntelligentTestRunnerEnabled)
146139
{

‎tracer/src/Datadog.Trace/Ci/CiEnvironment/CIEnvironmentValuesGenerics.cs‎

Lines changed: 63 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
using Datadog.Trace.Ci.Tags;
1616
using Datadog.Trace.Configuration;
1717
using Datadog.Trace.Logging;
18+
using Datadog.Trace.Util;
19+
using Datadog.Trace.Vendors.Serilog.Events;
1820

1921
namespace Datadog.Trace.Ci.CiEnvironment;
2022

@@ -27,6 +29,7 @@ internal abstract class CIEnvironmentValues<TValueProvider>(TValueProvider value
2729

2830
internal static CIEnvironmentValues Create(TValueProvider valueProvider)
2931
{
32+
Log.Debug("CIEnvironmentValues: Creating instance.");
3033
if (!string.IsNullOrEmpty(valueProvider.GetValue(PlatformKeys.Ci.Travis.Name)))
3134
{
3235
return new TravisEnvironmentValues<TValueProvider>(valueProvider);
@@ -108,6 +111,35 @@ internal static CIEnvironmentValues Create(TValueProvider valueProvider)
108111

109112
protected override void Setup(IGitInfo gitInfo)
110113
{
114+
if (gitInfo.Errors.Count > 0)
115+
{
116+
var sb = StringBuilderCache.Acquire();
117+
sb.AppendLine();
118+
foreach (var err in gitInfo.Errors)
119+
{
120+
sb.AppendLine(" Error: " + err);
121+
}
122+
123+
Log.Warning("CIEnvironmentValues: Errors detected in the local gitInfo: {Errors}", StringBuilderCache.GetStringAndRelease(sb));
124+
}
125+
126+
if (Log.IsEnabled(LogEventLevel.Debug))
127+
{
128+
var sb = StringBuilderCache.Acquire();
129+
sb.AppendLine();
130+
var values = ValueProvider.GetValues();
131+
foreach (var field in GetAllFieldsRecursive(typeof(PlatformKeys.Ci)))
132+
{
133+
var fieldName = field.GetValue(null) as string;
134+
if (!StringUtil.IsNullOrEmpty(fieldName) && values.TryGetValue<string>(fieldName, out var value))
135+
{
136+
sb.AppendFormat("\t{0}={1}{2}", fieldName, value == string.Empty ? "(empty)" : $"\"{value}\"", Environment.NewLine);
137+
}
138+
}
139+
140+
Log.Debug("CIEnvironmentValues: Values detected:{Values}", StringBuilderCache.GetStringAndRelease(sb));
141+
}
142+
111143
OnInitialize(gitInfo);
112144

113145
// **********
@@ -160,7 +192,7 @@ protected override void Setup(IGitInfo gitInfo)
160192
}
161193
else
162194
{
163-
Log.Warning("Git commit in .git folder is different from the one in the environment variables. [{GitCommit} != {EnvVarCommit}]", gitInfo.Commit, Commit);
195+
Log.Warning("CIEnvironmentValues: Git commit in .git folder is different from the one in the environment variables. [{GitCommit} != {EnvVarCommit}]", gitInfo.Commit, Commit);
164196
}
165197

166198
// **********
@@ -182,7 +214,7 @@ protected override void Setup(IGitInfo gitInfo)
182214
}
183215
else
184216
{
185-
Log.Warning("Error fetching data for git commit '{HeadCommit}'", HeadCommit);
217+
Log.Warning("CIEnvironmentValues: Error fetching data for git commit '{HeadCommit}'", HeadCommit);
186218
}
187219
}
188220

@@ -210,11 +242,11 @@ protected override void Setup(IGitInfo gitInfo)
210242
{
211243
if (string.IsNullOrEmpty(defaultValue))
212244
{
213-
Log.ErrorSkipTelemetry("DD_GIT_REPOSITORY_URL is set with an empty value, and the Git repository could not be automatically extracted");
245+
Log.ErrorSkipTelemetry("CIEnvironmentValues: DD_GIT_REPOSITORY_URL is set with an empty value, and the Git repository could not be automatically extracted");
214246
}
215247
else
216248
{
217-
Log.ErrorSkipTelemetry("DD_GIT_REPOSITORY_URL is set with an empty value, defaulting to '{Default}'", defaultValue);
249+
Log.ErrorSkipTelemetry("CIEnvironmentValues: DD_GIT_REPOSITORY_URL is set with an empty value, defaulting to '{Default}'", defaultValue);
218250
}
219251

220252
return false;
@@ -224,11 +256,11 @@ protected override void Setup(IGitInfo gitInfo)
224256
{
225257
if (string.IsNullOrEmpty(defaultValue))
226258
{
227-
Log.ErrorSkipTelemetry("DD_GIT_REPOSITORY_URL is set with an invalid value ('{Value}'), and the Git repository could not be automatically extracted", value);
259+
Log.ErrorSkipTelemetry("CIEnvironmentValues: DD_GIT_REPOSITORY_URL is set with an invalid value ('{Value}'), and the Git repository could not be automatically extracted", value);
228260
}
229261
else
230262
{
231-
Log.ErrorSkipTelemetry("DD_GIT_REPOSITORY_URL is set with an invalid value ('{Value}'), defaulting to '{Default}'", value, defaultValue);
263+
Log.ErrorSkipTelemetry("CIEnvironmentValues: DD_GIT_REPOSITORY_URL is set with an invalid value ('{Value}'), defaulting to '{Default}'", value, defaultValue);
232264
}
233265

234266
return false;
@@ -240,7 +272,7 @@ protected override void Setup(IGitInfo gitInfo)
240272

241273
if (string.IsNullOrEmpty(defaultValue))
242274
{
243-
Log.Warning("The Git repository couldn't be automatically extracted.");
275+
Log.Warning("CIEnvironmentValues: The Git repository couldn't be automatically extracted.");
244276
}
245277

246278
// If not set use the default value
@@ -258,11 +290,11 @@ protected override void Setup(IGitInfo gitInfo)
258290
{
259291
if (string.IsNullOrEmpty(defaultValue))
260292
{
261-
Log.Error("DD_GIT_COMMIT_SHA must be a full-length git SHA, and the The Git commit sha couldn't be automatically extracted.");
293+
Log.Error("CIEnvironmentValues: DD_GIT_COMMIT_SHA must be a full-length git SHA ({Value}), and the The Git commit sha couldn't be automatically extracted.", value);
262294
}
263295
else
264296
{
265-
Log.Error("DD_GIT_COMMIT_SHA must be a full-length git SHA, defaulting to '{Default}", defaultValue);
297+
Log.Error("CIEnvironmentValues: DD_GIT_COMMIT_SHA must be a full-length git SHA ({Value}), defaulting to '{Default}", value, defaultValue);
266298
}
267299

268300
return false;
@@ -274,7 +306,7 @@ protected override void Setup(IGitInfo gitInfo)
274306

275307
if (string.IsNullOrEmpty(defaultValue))
276308
{
277-
Log.Warning("The Git commit sha couldn't be automatically extracted.");
309+
Log.Warning("CIEnvironmentValues: The Git commit sha couldn't be automatically extracted.");
278310
}
279311

280312
// If not set use the default value
@@ -297,19 +329,19 @@ protected override void Setup(IGitInfo gitInfo)
297329
{
298330
value = value.Trim();
299331
if (value.Length != 40 || !IsHex(value))
332+
{
333+
if (string.IsNullOrEmpty(defaultValue))
300334
{
301-
if (string.IsNullOrEmpty(defaultValue))
302-
{
303-
Log.Error("DD_GIT_PULL_REQUEST_BASE_BRANCH_SHA must be a full-length git SHA, and the The Git commit sha couldn't be automatically extracted.");
304-
}
305-
else
306-
{
307-
Log.Error("DD_GIT_CODD_GIT_PULL_REQUEST_BASE_BRANCH_SHAMMIT_SHA must be a full-length git SHA, defaulting to '{Default}", defaultValue);
308-
}
309-
310-
return false;
335+
Log.Error("CIEnvironmentValues: DD_GIT_PULL_REQUEST_BASE_BRANCH_SHA must be a full-length git SHA ({Value}), and the The Git commit sha couldn't be automatically extracted.", value);
336+
}
337+
else
338+
{
339+
Log.Error("CIEnvironmentValues: DD_GIT_CODD_GIT_PULL_REQUEST_BASE_BRANCH_SHAMMIT_SHA must be a full-length git SHA ({Value}), defaulting to '{Default}", value, defaultValue);
311340
}
312341

342+
return false;
343+
}
344+
313345
// All ok!
314346
return true;
315347
}
@@ -397,6 +429,17 @@ protected void SetVariablesIfNotEmpty(Dictionary<string, string?>? dictionary, s
397429
}
398430
}
399431

432+
private static IEnumerable<System.Reflection.FieldInfo> GetAllFieldsRecursive(Type type)
433+
{
434+
var fields = new List<System.Reflection.FieldInfo>(type.GetFields(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Static));
435+
foreach (var nestedType in type.GetNestedTypes(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic))
436+
{
437+
fields.AddRange(GetAllFieldsRecursive(nestedType));
438+
}
439+
440+
return fields;
441+
}
442+
400443
[MethodImpl(MethodImplOptions.AggressiveInlining)]
401444
protected string? ExpandPath(string? path)
402445
{

‎tracer/src/Datadog.Trace/Ci/CiEnvironment/GitCommandGitInfoProvider.cs‎

Lines changed: 15 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using System.IO;
1010
using System.Linq;
1111
using Datadog.Trace.Logging;
12+
using Datadog.Trace.Telemetry.Metrics;
1213
using Datadog.Trace.Util;
1314

1415
namespace Datadog.Trace.Ci.CiEnvironment;
@@ -23,34 +24,21 @@ private GitCommandGitInfoProvider()
2324

2425
protected override bool TryGetFrom(DirectoryInfo gitDirectory, [NotNullWhen(true)] out IGitInfo? gitInfo)
2526
{
27+
using var cd = CodeDurationRef.Create();
2628
var localGitInfo = new GitInfo
2729
{
28-
SourceRoot = gitDirectory.Parent?.FullName
30+
SourceRoot = gitDirectory.Name == ".git" ? gitDirectory.Parent?.FullName : gitDirectory.FullName
2931
};
3032

3133
gitInfo = localGitInfo;
3234

3335
try
3436
{
35-
// Ensure we have permissions to read the git directory
36-
var safeDirectory = ProcessHelpers.RunCommand(
37-
new ProcessHelpers.Command(
38-
cmd: "git",
39-
arguments: $"config --global --add safe.directory {gitDirectory.FullName}",
40-
workingDirectory: gitDirectory.FullName,
41-
useWhereIsIfFileNotFound: true));
42-
if (safeDirectory?.ExitCode != 0)
43-
{
44-
localGitInfo.Errors.Add($"Error setting safe.directory: {safeDirectory?.Error}");
45-
}
46-
4737
// Get the repository URL
48-
var repositoryOutput = ProcessHelpers.RunCommand(
49-
new ProcessHelpers.Command(
50-
cmd: "git",
51-
arguments: "ls-remote --get-url",
52-
workingDirectory: gitDirectory.FullName,
53-
useWhereIsIfFileNotFound: true));
38+
var repositoryOutput = GitCommandHelper.RunGitCommand(
39+
localGitInfo.SourceRoot,
40+
"ls-remote --get-url",
41+
MetricTags.CIVisibilityCommands.GetRepository);
5442
if (repositoryOutput?.ExitCode == 0)
5543
{
5644
localGitInfo.Repository = repositoryOutput.Output.Trim();
@@ -61,12 +49,10 @@ protected override bool TryGetFrom(DirectoryInfo gitDirectory, [NotNullWhen(true
6149
}
6250

6351
// Get the branch name
64-
var branchOutput = ProcessHelpers.RunCommand(
65-
new ProcessHelpers.Command(
66-
cmd: "git",
67-
arguments: "rev-parse --abbrev-ref HEAD",
68-
workingDirectory: gitDirectory.FullName,
69-
useWhereIsIfFileNotFound: true));
52+
var branchOutput = GitCommandHelper.RunGitCommand(
53+
localGitInfo.SourceRoot,
54+
"rev-parse --abbrev-ref HEAD",
55+
MetricTags.CIVisibilityCommands.GetBranch);
7056
if (branchOutput?.ExitCode == 0 && branchOutput.Output.Trim() is { Length: > 0 } branchName && branchName != "HEAD")
7157
{
7258
localGitInfo.Branch = branchName;
@@ -77,12 +63,10 @@ protected override bool TryGetFrom(DirectoryInfo gitDirectory, [NotNullWhen(true
7763
}
7864

7965
// Get the remaining data from the log -1
80-
var gitLogOutput = ProcessHelpers.RunCommand(
81-
new ProcessHelpers.Command(
82-
cmd: "git",
83-
arguments: """log -1 --pretty='%H|,|%at|,|%an|,|%ae|,|%ct|,|%cn|,|%ce|,|%B'""",
84-
workingDirectory: gitDirectory.FullName,
85-
useWhereIsIfFileNotFound: true));
66+
var gitLogOutput = GitCommandHelper.RunGitCommand(
67+
localGitInfo.SourceRoot,
68+
"""log -1 --pretty='%H|,|%at|,|%an|,|%ae|,|%ct|,|%cn|,|%ce|,|%B'""",
69+
MetricTags.CIVisibilityCommands.Fetch);
8670
if (gitLogOutput?.ExitCode != 0)
8771
{
8872
localGitInfo.Errors.Add($"Error getting git log: {gitLogOutput?.Error}");

‎tracer/src/Datadog.Trace/Ci/CiEnvironment/GitInfo.cs‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@ namespace Datadog.Trace.Ci.CiEnvironment;
1515
internal sealed class GitInfo : IGitInfo
1616
{
1717
private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(GitInfo));
18+
private static readonly string? RuntimeFolder = Path.GetDirectoryName(typeof(string).Assembly.Location);
19+
private static readonly string DatadogTraceToolsRunnerAssembly = "Datadog.Trace.Tools.Runner.dll";
1820
private static IGitInfoProvider[] _gitInfoProviders = [
19-
ManualParserGitInfoProvider.Instance,
2021
GitCommandGitInfoProvider.Instance,
2122
];
2223

@@ -121,8 +122,30 @@ public static IGitInfo GetFrom(string folder)
121122
public static IGitInfo GetCurrent()
122123
{
123124
List<string>? errors = null;
124-
var baseDirectory = AppDomain.CurrentDomain.BaseDirectory;
125-
var gitDirectory = GetParentGitFolder(baseDirectory) ?? GetParentGitFolder(Environment.CurrentDirectory);
125+
List<string> searchPaths = new();
126+
127+
var baseDirectory = AppContext.BaseDirectory;
128+
if (!baseDirectory.Contains("/dotnet/sdk") && !string.IsNullOrWhiteSpace(RuntimeFolder) && !baseDirectory.Contains(RuntimeFolder))
129+
{
130+
if (!File.Exists(Path.Combine(baseDirectory, DatadogTraceToolsRunnerAssembly)))
131+
{
132+
searchPaths.Add(baseDirectory);
133+
}
134+
}
135+
136+
searchPaths.Add(Environment.CurrentDirectory);
137+
138+
DirectoryInfo? gitDirectory = null;
139+
foreach (var sp in searchPaths)
140+
{
141+
Log.Information("GitInfo: Using directory: {Directory}", sp);
142+
gitDirectory = GetParentGitFolder(sp);
143+
if (gitDirectory is not null)
144+
{
145+
break;
146+
}
147+
}
148+
126149
if (gitDirectory != null)
127150
{
128151
foreach (var provider in _gitInfoProviders)

‎tracer/src/Datadog.Trace/Ci/CiEnvironment/GitInfoProvider.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,15 @@ internal abstract class GitInfoProvider : IGitInfoProvider
1414
{
1515
public bool TryGetFrom(string folder, [NotNullWhen(true)] out IGitInfo? gitInfo)
1616
{
17+
if (!folder.EndsWith("/.git") && !folder.EndsWith("\\.git"))
18+
{
19+
var tentativeFolder = Path.Combine(folder, ".git");
20+
if (Directory.Exists(tentativeFolder))
21+
{
22+
folder = tentativeFolder;
23+
}
24+
}
25+
1726
// Try to load git metadata from the folder
1827
if (TryGetFrom(new DirectoryInfo(folder), out gitInfo))
1928
{

0 commit comments

Comments
 (0)