Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

try to make MemoryDiagnoserTests more stable #1766

Closed
wants to merge 5 commits into from
Closed

try to make MemoryDiagnoserTests more stable #1766

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f87b62a
skip AwaitingTasksShouldNotInterfereAllocationResults for InProcessTo…
adamsitnik Aug 4, 2021
2eb9e73
play with the tests
adamsitnik Aug 4, 2021
3b2d736
reduce the number of test cases (1 should be enough)
adamsitnik Aug 4, 2021
0e57b77
these tests are passing on CoreRT now
adamsitnik Aug 4, 2021
a2e930f
use ranges for MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks test
adamsitnik Aug 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 38 additions & 74 deletions tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Diagnosers;
using BenchmarkDotNet.Engines;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.IntegrationTests.Xunit;
using BenchmarkDotNet.Jobs;
Expand Down Expand Up @@ -37,8 +38,9 @@ public static IEnumerable<object[]> GetToolchains()
: new[]
{
new object[] { Job.Default.GetToolchain() },
new object[] { InProcessEmitToolchain.Instance },
#if !NETFRAMEWORK
#if NETFRAMEWORK
new object[] { InProcessEmitToolchain.Instance }, // it seems to be unstable for .NET 5
#else
// we don't want to test CoreRT twice (for .NET 4.6 and 5.0) when running the integration tests (these tests take a lot of time)
// we test against specific version to keep this test stable
new object[] { CoreRtToolchain.CreateBuilder().UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "6.0.0-preview.1.21074.3").ToToolchain() }
Expand All @@ -47,10 +49,7 @@ public static IEnumerable<object[]> GetToolchains()

public class AccurateAllocations
{
[Benchmark] public byte[] EightBytesArray() => new byte[8];
[Benchmark] public byte[] SixtyFourBytesArray() => new byte[64];

[Benchmark] public Task<int> AllocateTask() => Task.FromResult(default(int));
[Benchmark] public byte[] SmallObjectHeap() => new byte[8];
}

[Theory, MemberData(nameof(GetToolchains))]
Expand All @@ -62,10 +61,7 @@ public void MemoryDiagnoserIsAccurate(IToolchain toolchain)

AssertAllocations(toolchain, typeof(AccurateAllocations), new Dictionary<string, long>
{
{ nameof(AccurateAllocations.EightBytesArray), 8 + objectAllocationOverhead + arraySizeOverhead },
{ nameof(AccurateAllocations.SixtyFourBytesArray), 64 + objectAllocationOverhead + arraySizeOverhead },

{ nameof(AccurateAllocations.AllocateTask), CalculateRequiredSpace<Task<int>>() },
{ nameof(AccurateAllocations.SmallObjectHeap), 8 + objectAllocationOverhead + arraySizeOverhead },
});
}

Expand All @@ -92,7 +88,7 @@ private void AllocateUntilGcWakesUp()
}
}

[Theory(Skip = "#1542 Tiered JIT Thread allocates memory in the background"), MemberData(nameof(GetToolchains))]
[Theory, MemberData(nameof(GetToolchains))]
[Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup(IToolchain toolchain)
{
Expand Down Expand Up @@ -194,14 +190,10 @@ public byte[] SixtyFourBytesArray()
}
}

[Theory(Skip = "#1542 Tiered JIT Thread allocates memory in the background"), MemberData(nameof(GetToolchains))]
//[TheoryNetCoreOnly("Only .NET Core 2.0+ API is bug free for this case"), MemberData(nameof(GetToolchains))]
[TheoryNetCoreOnly("Only .NET Core 2.0+ API is bug free for this case"), MemberData(nameof(GetToolchains))]
[Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain)
{
if (toolchain is CoreRtToolchain) // the fix has not yet been backported to CoreRT
return;

long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word
long arraySizeOverhead = IntPtr.Size; // array length

Expand Down Expand Up @@ -237,52 +229,56 @@ public void Allocate()
}
}

[TheoryNetCore30(".NET Core 3.0 preview6+ exposes a GC.GetTotalAllocatedBytes method which makes it possible to work"), MemberData(nameof(GetToolchains))]
[Theory, MemberData(nameof(GetToolchains))]
[Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolchain)
{
if (toolchain is CoreRtToolchain) // the API has not been yet ported to CoreRT
return;

long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word
long arraySizeOverhead = IntPtr.Size; // array length
long memoryAllocatedPerArray = (MultiThreadedAllocation.Size + objectAllocationOverhead + arraySizeOverhead);
long threadStartAndJoinOverhead = 112; // this is more or less a magic number taken from memory profiler
long allocatedMemoryPerThread = memoryAllocatedPerArray + threadStartAndJoinOverhead;
long maxThreadStartAndJoinOverhead = RuntimeInformation.IsFullFramework ? 2500 : 600; // these are more or less a magic numbers taken from memory profiler and test runs

AssertAllocations(toolchain, typeof(MultiThreadedAllocation), new Dictionary<string, long>
AssertAllocations(toolchain, typeof(MultiThreadedAllocation), new Dictionary<string, (long low, long high)>
{
{ nameof(MultiThreadedAllocation.Allocate), allocatedMemoryPerThread * MultiThreadedAllocation.ThreadsCount }
{ nameof(MultiThreadedAllocation.Allocate), (
low: memoryAllocatedPerArray * MultiThreadedAllocation.ThreadsCount,
high: (memoryAllocatedPerArray + maxThreadStartAndJoinOverhead) * MultiThreadedAllocation.ThreadsCount) }
});
}

private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary<string, long> benchmarksAllocationsValidators)
{
var config = CreateConfig(toolchain);
var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config);
AssertAllocations(toolchain, benchmarkType, benchmarksAllocationsValidators, (benchmark, gcStats, expectedBytes) =>
{
Assert.Equal(expectedBytes, gcStats.GetBytesAllocatedPerOperation(benchmark));

var summary = BenchmarkRunner.Run(benchmarks);
if (expectedBytes == 0)
{
Assert.Equal(0, gcStats.GetTotalAllocatedBytes(excludeAllocationQuantumSideEffects: true));
}
});
}

foreach (var benchmarkAllocationsValidator in benchmarksAllocationsValidators)
private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary<string, (long low, long high)> benchmarksAllocationsValidators)
{
AssertAllocations(toolchain, benchmarkType, benchmarksAllocationsValidators, (benchmark, gcStats, expectedBytes) =>
{
// CoreRT is missing some of the CoreCLR threading/task related perf improvements, so sizeof(Task<int>) calculated for CoreCLR < sizeof(Task<int>) on CoreRT
// see https://github.com/dotnet/corert/issues/5705 for more
if (benchmarkAllocationsValidator.Key == nameof(AccurateAllocations.AllocateTask) && toolchain is CoreRtToolchain)
continue;
Assert.InRange(gcStats.GetBytesAllocatedPerOperation(benchmark), expectedBytes.low, expectedBytes.high);
});
}

var allocatingBenchmarks = benchmarks.BenchmarksCases.Where(benchmark => benchmark.DisplayInfo.Contains(benchmarkAllocationsValidator.Key));
private void AssertAllocations<T>(IToolchain toolchain, Type benchmarkType, Dictionary<string, T> benchmarksAllocationsValidators, Action<BenchmarkCase, GcStats, T> assert)
{
var config = CreateConfig(toolchain);
var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config);

foreach (var benchmark in allocatingBenchmarks)
{
var benchmarkReport = summary.Reports.Single(report => report.BenchmarkCase == benchmark);
var summary = BenchmarkRunner.Run(benchmarks);

Assert.Equal(benchmarkAllocationsValidator.Value, benchmarkReport.GcStats.GetBytesAllocatedPerOperation(benchmark));
foreach (var report in summary.Reports)
{
var benchmark = report.BenchmarkCase;

if (benchmarkAllocationsValidator.Value == 0)
{
Assert.Equal(0, benchmarkReport.GcStats.GetTotalAllocatedBytes(excludeAllocationQuantumSideEffects: true));
}
}
assert(benchmark, report.GcStats, benchmarksAllocationsValidators[benchmark.Descriptor.WorkloadMethod.Name]);
}
}

Expand All @@ -293,41 +289,9 @@ private IConfig CreateConfig(IToolchain toolchain)
.WithWarmupCount(0) // don't run warmup to save some time for our CI runs
.WithIterationCount(1) // single iteration is enough for us
.WithGcForce(false)
.WithEnvironmentVariable("COMPlus_TieredCompilation", "0") // Tiered JIT can allocate some memory on a background thread, let's disable it to make our tests less flaky (#1542)
.WithToolchain(toolchain))
.AddColumnProvider(DefaultColumnProviders.Instance)
.AddDiagnoser(MemoryDiagnoser.Default)
.AddLogger(toolchain.IsInProcess ? ConsoleLogger.Default : new OutputLogger(output)); // we can't use OutputLogger for the InProcess toolchains because it allocates memory on the same thread

// note: don't copy, never use in production systems (it should work but I am not 100% sure)
private int CalculateRequiredSpace<T>()
{
int total = SizeOfAllFields<T>();

if (!typeof(T).GetTypeInfo().IsValueType)
total += IntPtr.Size * 2; // pointer to method table + object header word

if (total % IntPtr.Size != 0) // aligning..
total += IntPtr.Size - (total % IntPtr.Size);

return total;
}

// note: don't copy, never use in production systems (it should work but I am not 100% sure)
private int SizeOfAllFields<T>()
{
int GetSize(Type type)
{
var sizeOf = typeof(Unsafe).GetTypeInfo().GetMethod(nameof(Unsafe.SizeOf));

return (int)sizeOf.MakeGenericMethod(type).Invoke(null, null);
}

return typeof(T)
.GetAllFields()
.Where(field => !field.IsStatic && !field.IsLiteral)
.Distinct()
.Sum(field => GetSize(field.FieldType));
}
}
}
16 changes: 0 additions & 16 deletions tests/BenchmarkDotNet.Tests/XUnit/TheoryNetCore30Attribute.cs
View file
Open in desktop

This file was deleted.