Skip to content

Commit 134f41e

Browse files
TedHartMSbadrishc
andauthored
[C#] Add initial Benchmarkdotnet tests (#751)
* Add a couple BenchmarkDotNet tests * add BenchmarkDotNet README Co-authored-by: Badrish Chandramouli <[email protected]>
1 parent 4c6bd9d commit 134f41e

File tree

8 files changed

+241
-1
lines changed

8 files changed

+241
-1
lines changed

Diff for: .gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -199,3 +199,5 @@ nativebin/
199199
cs/.idea/
200200
cs/remote/.idea
201201
cs/libdpr/.idea
202+
203+
cs/**/BenchmarkDotNet.Artifacts/

Diff for: cs/FASTER.sln

+13
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TstRunner", "playground\Tst
9696
EndProject
9797
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EpvsSample", "samples\EpvsSample\EpvsSample.csproj", "{DC3E0640-9A36-43D0-AA37-A1B61B0BFBC9}"
9898
EndProject
99+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "performance", "performance", "{5E4C9997-3350-4761-9FC9-F27649848B1D}"
100+
EndProject
101+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDotNetTests", "performance\BenchmarkDotNet\BenchmarkDotNetTests.csproj", "{AF996720-DB6C-4ED7-9693-B9531F0B119A}"
102+
EndProject
99103
Global
100104
GlobalSection(SolutionConfigurationPlatforms) = preSolution
101105
Debug|Any CPU = Debug|Any CPU
@@ -311,6 +315,14 @@ Global
311315
{80519947-DB86-4B21-8C11-B74186788BBD}.Release|Any CPU.Build.0 = Release|Any CPU
312316
{80519947-DB86-4B21-8C11-B74186788BBD}.Release|x64.ActiveCfg = Release|Any CPU
313317
{80519947-DB86-4B21-8C11-B74186788BBD}.Release|x64.Build.0 = Release|Any CPU
318+
{AF996720-DB6C-4ED7-9693-B9531F0B119A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
319+
{AF996720-DB6C-4ED7-9693-B9531F0B119A}.Debug|Any CPU.Build.0 = Debug|Any CPU
320+
{AF996720-DB6C-4ED7-9693-B9531F0B119A}.Debug|x64.ActiveCfg = Debug|Any CPU
321+
{AF996720-DB6C-4ED7-9693-B9531F0B119A}.Debug|x64.Build.0 = Debug|Any CPU
322+
{AF996720-DB6C-4ED7-9693-B9531F0B119A}.Release|Any CPU.ActiveCfg = Release|Any CPU
323+
{AF996720-DB6C-4ED7-9693-B9531F0B119A}.Release|Any CPU.Build.0 = Release|Any CPU
324+
{AF996720-DB6C-4ED7-9693-B9531F0B119A}.Release|x64.ActiveCfg = Release|Any CPU
325+
{AF996720-DB6C-4ED7-9693-B9531F0B119A}.Release|x64.Build.0 = Release|Any CPU
314326
EndGlobalSection
315327
GlobalSection(SolutionProperties) = preSolution
316328
HideSolutionNode = FALSE
@@ -344,6 +356,7 @@ Global
344356
{E8C7FB0F-38B8-468A-B1CA-8793DF8F2693} = {E6026D6A-01C5-4582-B2C1-64751490DABE}
345357
{A265D9D2-3FEA-48BB-B1CC-273ECFEA0611} = {E6026D6A-01C5-4582-B2C1-64751490DABE}
346358
{DC3E0640-9A36-43D0-AA37-A1B61B0BFBC9} = {62BC1134-B6E1-476A-B894-7CA278A8B6DE}
359+
{AF996720-DB6C-4ED7-9693-B9531F0B119A} = {5E4C9997-3350-4761-9FC9-F27649848B1D}
347360
EndGlobalSection
348361
GlobalSection(ExtensibilityGlobals) = postSolution
349362
SolutionGuid = {A0750637-2CCB-4139-B25E-F2CE740DCFAC}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<TargetFramework>net6.0</TargetFramework>
4+
</PropertyGroup>
5+
<PropertyGroup>
6+
<PlatformTarget>AnyCPU</PlatformTarget>
7+
<DebugType>pdbonly</DebugType>
8+
<DebugSymbols>true</DebugSymbols>
9+
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
10+
<Optimize>true</Optimize>
11+
<Configuration>Release</Configuration>
12+
<IsPackable>false</IsPackable>
13+
<ApplicationIcon />
14+
<OutputType>Exe</OutputType>
15+
<StartupObject />
16+
</PropertyGroup>
17+
<ItemGroup>
18+
<PackageReference Include="BenchmarkDotNet" Version="0.13.0" />
19+
<PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.13.0" />
20+
</ItemGroup>
21+
<ItemGroup>
22+
<ProjectReference Include="..\..\src\core\FASTER.core.csproj" />
23+
</ItemGroup>
24+
</Project>
+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT license.
3+
4+
using BenchmarkDotNet.Running;
5+
using System.IO;
6+
7+
namespace BenchmarkDotNetTests
8+
{
9+
public class BenchmarkDotNetTestsApp
10+
{
11+
public static string TestDirectory => Path.Combine(Path.GetDirectoryName(typeof(BenchmarkDotNetTestsApp).Assembly.Location), "Tests");
12+
13+
public static void Main(string[] args)
14+
{
15+
BenchmarkSwitcher.FromAssembly(typeof(BenchmarkDotNetTestsApp).Assembly).Run(args);
16+
}
17+
}
18+
}

Diff for: cs/performance/BenchmarkDotNet/LightEpochTests.cs

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT license.
3+
4+
using BenchmarkDotNet.Attributes;
5+
using BenchmarkDotNet.Configs;
6+
using FASTER.core;
7+
8+
#pragma warning disable 0649 // Field 'field' is never assigned to, and will always have its default value 'value'; happens due to [Params(..)]
9+
10+
namespace BenchmarkDotNetTests
11+
{
12+
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory, BenchmarkLogicalGroupRule.ByParams)]
13+
public class LightEpochTests
14+
{
15+
[Params(10_000_000, 100_000_000, 1_000_000_000)]
16+
public int NumIterations;
17+
18+
[BenchmarkCategory("LightEpoch"), Benchmark]
19+
public void AcquireAndRelease()
20+
{
21+
var epoch = new LightEpoch();
22+
for (int i = 0; i < NumIterations; i++)
23+
{
24+
epoch.Resume();
25+
epoch.Suspend();
26+
}
27+
}
28+
}
29+
}

Diff for: cs/performance/BenchmarkDotNet/README.md

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
## BenchmarkDotNet tests
2+
3+
[BenchmarkDotNet](https://benchmarkdotnet.org) is a micro-benchmarking suite that supports annotations on functions, somewhat similar to unit tests.
4+
5+
To run:
6+
- Build Release
7+
- Execute `bin/Release/[platform]/BenchmarkDotNetTests.exe -f *LightEpoch*` to run the LightEpoch tests. See [BenchmarkDotNet](https://benchmarkdotnet.org) and related documentation for more details.
8+
9+
Currently we have two very simple benchmarks; more will be added later.
10+
- LightEpochTests: Currently runs a microbenchmark of epoch Acquire/Release.
11+
- SyncVsAsyncTests: Compares sync vs. async API performance.
12+
13+
BenchmarkDotNet does not support cross-run comparisons; currently, just save the output to a file and diff. A script will be written in the future to automate these comparisons.
14+
15+
### Adding a Test
16+
Use one of the existing tests as an example, and add an entry in this doc describing it briefly.

Diff for: cs/performance/BenchmarkDotNet/SyncVsAsyncTests.cs

+135
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT license.
3+
4+
using BenchmarkDotNet.Attributes;
5+
using BenchmarkDotNet.Configs;
6+
using FASTER.core;
7+
using System;
8+
using System.IO;
9+
using System.Threading.Tasks;
10+
11+
#pragma warning disable 0649 // Field 'field' is never assigned to, and will always have its default value 'value'; happens due to [Params(..)]
12+
13+
namespace BenchmarkDotNetTests
14+
{
15+
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory, BenchmarkLogicalGroupRule.ByParams)]
16+
public class SyncVsAsync
17+
{
18+
[Params(100, 1_000_000)]
19+
public int NumRecords;
20+
21+
//[ParamsAllValues]
22+
//bool useAsync;
23+
24+
FasterKV<long, long> store;
25+
IDevice logDevice;
26+
string logDirectory;
27+
28+
void SetupStore()
29+
{
30+
logDirectory = BenchmarkDotNetTestsApp.TestDirectory;
31+
var logFilename = Path.Combine(logDirectory, $"{nameof(SyncVsAsync)}_{Guid.NewGuid()}.log");
32+
logDevice = Devices.CreateLogDevice(logFilename, preallocateFile: true, deleteOnClose: true, useIoCompletionPort: true);
33+
var logSettings = new LogSettings
34+
{
35+
LogDevice = logDevice
36+
};
37+
store = new FasterKV<long, long>(1L << 20, logSettings);
38+
}
39+
40+
void PopulateStoreSync()
41+
{
42+
using var session = store.For(new SimpleFunctions<long, long>()).NewSession<SimpleFunctions<long, long>>();
43+
for (long ii = 0; ii < NumRecords; ++ii)
44+
session.Upsert(ii, ii);
45+
}
46+
47+
async ValueTask PopulateStoreAsync()
48+
{
49+
using var session = store.For(new SimpleFunctions<long, long>()).NewSession<SimpleFunctions<long, long>>();
50+
for (long ii = 0; ii < NumRecords; ++ii)
51+
{
52+
var result = await session.UpsertAsync(ii, ii);
53+
while (result.Status.IsPending)
54+
result = await result.CompleteAsync().ConfigureAwait(false);
55+
}
56+
}
57+
58+
[GlobalSetup(Targets = new[] { nameof(InsertSync), nameof(InsertAsync) })]
59+
public void SetupEmptyStore() => SetupStore();
60+
61+
[GlobalSetup(Targets = new[] { nameof(RMWSync), nameof(RMWAsync), nameof(ReadSync), nameof(ReadAsync) })]
62+
public void SetupPopulatedStore()
63+
{
64+
SetupStore();
65+
PopulateStoreSync();
66+
}
67+
68+
[GlobalCleanup]
69+
public void TearDown()
70+
{
71+
store?.Dispose();
72+
store = null;
73+
logDevice?.Dispose();
74+
logDevice = null;
75+
try
76+
{
77+
Directory.Delete(logDirectory);
78+
}
79+
catch { }
80+
}
81+
82+
[BenchmarkCategory("Insert"), Benchmark(Baseline = true)]
83+
public void InsertSync() => PopulateStoreSync();
84+
85+
[BenchmarkCategory("Insert"), Benchmark]
86+
public ValueTask InsertAsync() => PopulateStoreAsync();
87+
88+
[BenchmarkCategory("RMW"), Benchmark(Baseline = true)]
89+
public void RMWSync()
90+
{
91+
using var session = store.For(new SimpleFunctions<long, long>()).NewSession<SimpleFunctions<long, long>>();
92+
for (long ii = 0; ii < NumRecords; ++ii)
93+
session.RMW(ii, ii * 2);
94+
session.CompletePending();
95+
}
96+
97+
[BenchmarkCategory("RMW"), Benchmark]
98+
public async ValueTask RMWAsync()
99+
{
100+
using var session = store.For(new SimpleFunctions<long, long>()).NewSession<SimpleFunctions<long, long>>();
101+
for (long ii = 0; ii < NumRecords; ++ii)
102+
{
103+
var result = await session.RMWAsync(ii, ii * 2);
104+
while (result.Status.IsPending)
105+
result = await result.CompleteAsync().ConfigureAwait(false);
106+
}
107+
}
108+
109+
[BenchmarkCategory("Read"), Benchmark(Baseline = true)]
110+
public void ReadSync()
111+
{
112+
using var session = store.For(new SimpleFunctions<long, long>()).NewSession<SimpleFunctions<long, long>>();
113+
for (long ii = 0; ii < NumRecords; ++ii)
114+
{
115+
var (status, output) = session.Read(ii);
116+
if (status.IsPending)
117+
{
118+
session.CompletePendingWithOutputs(out var completedOutputs, wait: true);
119+
completedOutputs.Dispose();
120+
}
121+
}
122+
session.CompletePending();
123+
}
124+
125+
[BenchmarkCategory("Read"), Benchmark]
126+
public async ValueTask ReadAsync()
127+
{
128+
using var session = store.For(new SimpleFunctions<long, long>()).NewSession<SimpleFunctions<long, long>>();
129+
for (long ii = 0; ii < NumRecords; ++ii)
130+
{
131+
var (status, output) = (await session.ReadAsync(ii).ConfigureAwait(false)).Complete();
132+
}
133+
}
134+
}
135+
}

Diff for: cs/playground/AsyncStress/IFasterWrapper.cs

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
using FASTER.core;
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT license.
3+
4+
using FASTER.core;
25
using System.Threading.Tasks;
36

47
namespace AsyncStress

0 commit comments

Comments
 (0)