Skip to content

Commit 8717e88

Browse files
bitfasterbitfaster
authored
parameterize throughput tests (#275)
* size * runner * test constants
1 parent 0edd8a4 commit 8717e88

File tree

15 files changed

+107
-49
lines changed

15 files changed

+107
-49
lines changed

‎BitFaster.Caching.HitRateAnalysis/Arc/RunnerConfig.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ public RunnerConfig(string name, int[] cacheSizes, Uri dataUri)
2525

2626
public ArcDataFile File => this.file;
2727

28-
public static RunnerConfig Database = new RunnerConfig("results.arc.database.csv", new[] { 1000000, 2000000, 3000000, 4000000, 5000000, 6000000, 7000000, 8000000 }, new Uri("https://github.com/bitfaster/cache-datasets/releases/download/v1.0/DS1.lis.gz"));
29-
public static RunnerConfig Search = new RunnerConfig("results.arc.search.csv", new[] { 100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000 }, new Uri("https://github.com/bitfaster/cache-datasets/releases/download/v1.0/S3.lis.gz"));
28+
public static RunnerConfig Database = new RunnerConfig("results.arc.database.csv", new[] { 1_000_000, 2_000_000, 3_000_000, 4_000_000, 5_000_000, 6_000_000, 7_000_000, 8_000_000 }, new Uri("https://github.com/bitfaster/cache-datasets/releases/download/v1.0/DS1.lis.gz"));
29+
public static RunnerConfig Search = new RunnerConfig("results.arc.search.csv", new[] { 100_000, 200_000, 300_000, 400_000, 500_000, 600_000, 700_000, 800_000 }, new Uri("https://github.com/bitfaster/cache-datasets/releases/download/v1.0/S3.lis.gz"));
3030
public static RunnerConfig Oltp = new RunnerConfig("results.arc.oltp.csv", new[] { 250, 500, 750, 1000, 1250, 1500, 1750, 2000 }, new Uri("https://github.com/bitfaster/cache-datasets/releases/download/v1.0/OLTP.lis.gz"));
3131
}
3232
}

‎BitFaster.Caching.HitRateAnalysis/Wikibench/Runner.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public static async Task Run()
3838
a.TestKey(url);
3939
}
4040

41-
if (count++ % 100000 == 0)
41+
if (count++ % 100_000 == 0)
4242
{
4343
Console.WriteLine($"Processed {count} URIs...");
4444
}

‎BitFaster.Caching.HitRateAnalysis/Zipfian/Runner.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@ public class Runner
2121
// distribution (about 45 / 20).
2222

2323
// Took 1 million samples
24-
const int sampleCount = 1000000;
24+
const int sampleCount = 1_000_000;
2525

2626
// Simulated a database of 50,000 pages and
2727
// buffer sizes ranging from 2,500 (5%) items to 20,000
2828
// (40%) items.
29-
const int n = 50000;
29+
const int n = 50_000;
3030

3131
public static void Run()
3232
{

‎BitFaster.Caching.ThroughputAnalysis/BitFaster.Caching.ThroughputAnalysis.csproj‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@
3333
<None Update="NUMAExec.bat">
3434
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
3535
</None>
36+
<None Update="SizeExec.bat">
37+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
38+
</None>
3639
</ItemGroup>
3740

3841
</Project>

‎BitFaster.Caching.ThroughputAnalysis/CommandParser.cs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+

2+
namespace BitFaster.Caching.ThroughputAnalysis
3+
{
4+
public class CommandParser
5+
{
6+
public static (Mode, int) Parse(string[] args)
7+
{
8+
// arg[0] == mode, arg[1] == size
9+
if (args.Length == 2)
10+
{
11+
if (int.TryParse(args[0], out int modeArg))
12+
{
13+
if (int.TryParse(args[1], out int size))
14+
{
15+
return ((Mode)modeArg, size);
16+
}
17+
}
18+
}
19+
20+
Mode mode = Mode.Read;
21+
var menu = new EasyConsole.Menu()
22+
.Add("Read", () => mode = Mode.Read)
23+
.Add("Read + Write", () => mode = Mode.ReadWrite)
24+
.Add("Update", () => mode = Mode.Update)
25+
.Add("Evict", () => mode = Mode.Evict)
26+
.Add("All", () => mode = Mode.All);
27+
28+
menu.Display();
29+
30+
return (mode, 500);
31+
}
32+
}
33+
}

‎BitFaster.Caching.ThroughputAnalysis/ConfigFactory.cs‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,27 +8,47 @@ namespace BitFaster.Caching.ThroughputAnalysis
88
{
99
public class ConfigFactory
1010
{
11-
const double s = 0.86; // Zipf s parameter, controls distribution
12-
const int n = 500; // number of unique items for Zipf
13-
const int maxThreads = 52;
14-
const int sampleCount = 2000;
11+
const double s = 0.86; // Zipf s parameter, controls distribution
1512

16-
public static (ThroughputBenchmarkBase, IThroughputBenchConfig, int) Create(Mode mode, int repeatCount)
13+
public static (ThroughputBenchmarkBase, IThroughputBenchConfig, int) Create(Mode mode, int cacheSize, int maxThreads)
1714
{
15+
int iterations = GetIterationCount(cacheSize);
16+
int samples = GetSampleCount(cacheSize);
17+
int n = cacheSize; // number of unique items for Zipf
18+
1819
switch (mode)
1920
{
2021
case Mode.Read:
21-
return (new ReadThroughputBenchmark(), new ZipfConfig(repeatCount, sampleCount, s, n), n);
22+
return (new ReadThroughputBenchmark(), new ZipfConfig(iterations, samples, s, n), cacheSize);
2223
case Mode.ReadWrite:
2324
// cache holds 10% of all items
24-
return (new ReadThroughputBenchmark(), new ZipfConfig(repeatCount, sampleCount, s, n), n / 10);
25+
cacheSize = cacheSize / 10;
26+
return (new ReadThroughputBenchmark(), new ZipfConfig(iterations, samples, s, n), cacheSize);
2527
case Mode.Update:
26-
return (new UpdateThroughputBenchmark(), new ZipfConfig(repeatCount, sampleCount, s, n), n);
28+
return (new UpdateThroughputBenchmark(), new ZipfConfig(iterations, samples, s, n), cacheSize);
2729
case Mode.Evict:
28-
return (new ReadThroughputBenchmark(), new EvictionConfig(repeatCount, sampleCount, maxThreads), n);
30+
return (new ReadThroughputBenchmark(), new EvictionConfig(iterations, samples, maxThreads), cacheSize);
2931
}
3032

3133
throw new InvalidOperationException();
3234
}
35+
36+
private static int GetIterationCount(int cacheSize) => cacheSize switch
37+
{
38+
< 500 => 400,
39+
< 5_000 => 200,
40+
< 10_000 => 100,
41+
< 100_000 => 50,
42+
< 1_000_000 => 25,
43+
< 10_000_000 => 5,
44+
_ => 1
45+
};
46+
47+
private static int GetSampleCount(int cacheSize) => cacheSize switch
48+
{
49+
< 5_000 => cacheSize * 4,
50+
< 5_000_000 => cacheSize * 2,
51+
_ => cacheSize
52+
};
3353
}
3454
}

‎BitFaster.Caching.ThroughputAnalysis/Exporter.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@ public void CaptureRows(IEnumerable<ICacheFactory> caches)
4646
}
4747
}
4848

49-
public void ExportCsv(Mode mode)
49+
public void ExportCsv(Mode mode, int cacheSize)
5050
{
51-
using (var textWriter = File.CreateText($"Results{mode}.csv"))
51+
using (var textWriter = File.CreateText($"Results_{mode}_{cacheSize}.csv"))
5252
using (var csv = new CsvWriter(textWriter, CultureInfo.InvariantCulture))
5353
{
5454
foreach (DataColumn column in resultTable.Columns)

‎BitFaster.Caching.ThroughputAnalysis/Program.cs‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,7 @@
33

44
Host.PrintInfo();
55

6-
Mode mode = Mode.Read;
6+
var (mode, size) = CommandParser.Parse(args);
77

8-
var menu = new EasyConsole.Menu()
9-
.Add("Read", () => mode = Mode.Read)
10-
.Add("Read + Write", () => mode = Mode.ReadWrite)
11-
.Add("Update", () => mode = Mode.Update)
12-
.Add("Evict", () => mode = Mode.Evict)
13-
.Add("All", () => mode = Mode.All);
14-
15-
menu.Display();
16-
Runner.Run(mode);
8+
Runner.Run(mode, size);
179
Console.WriteLine("Done.");

‎BitFaster.Caching.ThroughputAnalysis/Runner.cs‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,26 +8,25 @@ namespace BitFaster.Caching.ThroughputAnalysis
88
public class Runner
99
{
1010
private static readonly int maxThreads = Host.GetAvailableCoreCount() * 2;
11-
private const int repeatCount = 400;
1211

13-
public static void Run(Mode mode)
12+
public static void Run(Mode mode, int cacheSize)
1413
{
1514
ThreadPool.SetMinThreads(maxThreads, maxThreads);
1615

1716
foreach (Mode value in Enum.GetValues(mode.GetType()))
1817
{
1918
if (mode.HasFlag(value) && value != Mode.All)
2019
{
21-
RunTest(value);
20+
RunTest(value, cacheSize);
2221
}
2322
}
2423
}
2524

26-
private static void RunTest(Mode mode)
25+
private static void RunTest(Mode mode, int cacheSize)
2726
{
2827
Console.WriteLine("Generating input distribution...");
2928

30-
var (bench, dataConfig, capacity) = ConfigFactory.Create(mode, repeatCount);
29+
var (bench, dataConfig, capacity) = ConfigFactory.Create(mode, cacheSize, maxThreads);
3130

3231
var cachesToTest = new List<ICacheFactory>();
3332
cachesToTest.Add(new ClassicLruFactory(capacity));
@@ -40,7 +39,7 @@ private static void RunTest(Mode mode)
4039
exporter.Initialize(cachesToTest);
4140

4241
Console.WriteLine();
43-
Console.WriteLine($"Running {mode}...");
42+
Console.WriteLine($"Running {mode} with size {cacheSize} over {maxThreads} threads...");
4443
Console.WriteLine();
4544

4645
foreach (int tc in Enumerable.Range(1, maxThreads).ToArray())
@@ -63,7 +62,7 @@ private static void RunTest(Mode mode)
6362

6463
exporter.CaptureRows(cachesToTest);
6564

66-
exporter.ExportCsv(mode);
65+
exporter.ExportCsv(mode, cacheSize);
6766

6867
//ConsoleTable
6968
// .From(resultTable)

‎BitFaster.Caching.ThroughputAnalysis/SizeExec.bat‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
cls
2+
3+
@echo off
4+
set DOTNET_Thread_UseAllCpuGroups=1
5+
6+
call BitFaster.Caching.ThroughputAnalysis.exe 4 100
7+
call BitFaster.Caching.ThroughputAnalysis.exe 4 10000
8+
call BitFaster.Caching.ThroughputAnalysis.exe 4 1000000
9+
call BitFaster.Caching.ThroughputAnalysis.exe 4 10000000

0 commit comments

Comments
 (0)