Skip to content

Commit 8ae9e2a

Browse files
bitfasterbitfaster
authored
Analyze hit rate with sequential scan (#40)
* scantest * update readme * update * fix charts * fix dupe %
1 parent 1bdb8fe commit 8ae9e2a

File tree

4 files changed

+192
-40
lines changed

4 files changed

+192
-40
lines changed

‎BitFaster.Caching.HitRateAnalysis/AnalysisResult.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,14 @@ public class AnalysisResult
1818

1919
public int Samples { get; set; }
2020

21+
public bool IsScan { get; set; }
22+
2123
public double CacheSizePercent { get; set; }
2224

2325
public double HitRatio { get; set; }
2426

27+
public TimeSpan Duration { get; set; }
28+
2529
public static void WriteToFile(string path, IEnumerable<AnalysisResult> results)
2630
{
2731
using (var writer = new StreamWriter(path))

‎BitFaster.Caching.HitRateAnalysis/Program.cs‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.Diagnostics;
34
using BitFaster.Caching.HitRateAnalysis;
45
using BitFaster.Caching.Lru;
56
using MathNet.Numerics;
@@ -30,7 +31,7 @@ static void Main(string[] args)
3031
double[] sValues = { 0.5, 0.86 };
3132

3233
// % of total number of items
33-
double[] cacheSizes = { 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4 };
34+
double[] cacheSizes = { 0.0125, 0.025, 0.05, 0.075, 0.1, 0.125, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4 };
3435

3536
List<Analysis> analysis = new List<Analysis>();
3637

@@ -52,8 +53,10 @@ static void Main(string[] args)
5253
for (int i = 0; i < sValues.Length; i++)
5354
{
5455
Console.WriteLine($"Generating Zipfan distribution with {sampleCount} samples, s = {sValues[i]}, N = {n}");
56+
var sw = Stopwatch.StartNew();
5557
zipdfDistribution[i] = new int[sampleCount];
5658
Zipf.Samples(zipdfDistribution[i], sValues[i], n);
59+
Console.WriteLine($"Took {sw.Elapsed}.");
5760
}
5861

5962
List<AnalysisResult> results = new List<AnalysisResult>();
@@ -68,13 +71,44 @@ static void Main(string[] args)
6871
var concurrentLru = new ConcurrentLru<int, int>(1, cacheSize, EqualityComparer<int>.Default);
6972
var classicLru = new ClassicLru<int, int>(1, cacheSize, EqualityComparer<int>.Default);
7073

74+
var concurrentLruScan = new ConcurrentLru<int, int>(1, cacheSize, EqualityComparer<int>.Default);
75+
var classicLruScan = new ClassicLru<int, int>(1, cacheSize, EqualityComparer<int>.Default);
76+
7177
var d = a.s == 0.5 ? 0 : 1;
7278

79+
var lruSw = Stopwatch.StartNew();
7380
for (int i = 0; i < sampleCount; i++)
7481
{
7582
concurrentLru.GetOrAdd(zipdfDistribution[d][i], func);
83+
}
84+
lruSw.Stop();
85+
Console.WriteLine($"concurrentLru size={cacheSize} took {lruSw.Elapsed}.");
86+
87+
var clruSw = Stopwatch.StartNew();
88+
for (int i = 0; i < sampleCount; i++)
89+
{
7690
classicLru.GetOrAdd(zipdfDistribution[d][i], func);
7791
}
92+
clruSw.Stop();
93+
Console.WriteLine($"classic lru size={cacheSize} took {clruSw.Elapsed}.");
94+
95+
var lruSwScan = Stopwatch.StartNew();
96+
for (int i = 0; i < sampleCount; i++)
97+
{
98+
concurrentLruScan.GetOrAdd(zipdfDistribution[d][i], func);
99+
concurrentLruScan.GetOrAdd(i % n, func);
100+
}
101+
lruSwScan.Stop();
102+
Console.WriteLine($"concurrentLruScan lru size={cacheSize} took {lruSwScan.Elapsed}.");
103+
104+
var clruSwScan = Stopwatch.StartNew();
105+
for (int i = 0; i < sampleCount; i++)
106+
{
107+
classicLruScan.GetOrAdd(zipdfDistribution[d][i], func);
108+
classicLruScan.GetOrAdd(i % n, func);
109+
}
110+
clruSwScan.Stop();
111+
Console.WriteLine($"classicLruScan lru size={cacheSize} took {clruSwScan.Elapsed}.");
78112

79113
results.Add(new AnalysisResult
80114
{
@@ -83,7 +117,9 @@ static void Main(string[] args)
83117
s = a.s,
84118
CacheSizePercent = a.CacheSizePercent * 100.0,
85119
Samples = a.Samples,
120+
IsScan = false,
86121
HitRatio = classicLru.HitRatio * 100.0,
122+
Duration = clruSw.Elapsed,
87123
});
88124

89125
results.Add(new AnalysisResult
@@ -93,7 +129,33 @@ static void Main(string[] args)
93129
s = a.s,
94130
CacheSizePercent = a.CacheSizePercent * 100.0,
95131
Samples = a.Samples,
132+
IsScan = false,
96133
HitRatio = concurrentLru.HitRatio * 100.0,
134+
Duration = lruSw.Elapsed,
135+
});
136+
137+
results.Add(new AnalysisResult
138+
{
139+
Cache = "ClassicLru",
140+
N = a.N,
141+
s = a.s,
142+
CacheSizePercent = a.CacheSizePercent * 100.0,
143+
Samples = a.Samples,
144+
IsScan = true,
145+
HitRatio = classicLruScan.HitRatio * 100.0,
146+
Duration = clruSwScan.Elapsed,
147+
});
148+
149+
results.Add(new AnalysisResult
150+
{
151+
Cache = "ConcurrentLru",
152+
N = a.N,
153+
s = a.s,
154+
CacheSizePercent = a.CacheSizePercent * 100.0,
155+
Samples = a.Samples,
156+
IsScan = true,
157+
HitRatio = concurrentLruScan.HitRatio * 100.0,
158+
Duration = lruSwScan.Elapsed,
97159
});
98160
}
99161

‎BitFaster.Caching.HitRateAnalysis/results.csv‎

Lines changed: 97 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,97 @@
1-
Cache,N,s,Samples,CacheSizePercent,HitRatio
2-
ClassicLru,50000,0.5,1000000,2.5,5.8545
3-
ConcurrentLru,50000,0.5,1000000,2.5,8.8714
4-
ClassicLru,50000,0.5,1000000,5,10.4898
5-
ConcurrentLru,50000,0.5,1000000,5,13.6905
6-
ClassicLru,50000,0.5,1000000,10,18.3537
7-
ConcurrentLru,50000,0.5,1000000,10,21.301000000000002
8-
ClassicLru,50000,0.5,1000000,15,25.2029
9-
ConcurrentLru,50000,0.5,1000000,15,27.750200000000003
10-
ClassicLru,50000,0.5,1000000,20,31.443900000000003
11-
ConcurrentLru,50000,0.5,1000000,20,33.4923
12-
ClassicLru,50000,0.5,1000000,25,37.1243
13-
ConcurrentLru,50000,0.5,1000000,25,38.8337
14-
ClassicLru,50000,0.5,1000000,30,42.4998
15-
ConcurrentLru,50000,0.5,1000000,30,43.7457
16-
ClassicLru,50000,0.5,1000000,35,47.5692
17-
ConcurrentLru,50000,0.5,1000000,35,48.4488
18-
ClassicLru,50000,0.5,1000000,40,52.3858
19-
ConcurrentLru,50000,0.5,1000000,40,52.9031
20-
ClassicLru,50000,0.86,1000000,2.5,35.9167
21-
ConcurrentLru,50000,0.86,1000000,2.5,40.2709
22-
ClassicLru,50000,0.86,1000000,5,44.2585
23-
ConcurrentLru,50000,0.86,1000000,5,47.7731
24-
ClassicLru,50000,0.86,1000000,10,53.8678
25-
ConcurrentLru,50000,0.86,1000000,10,56.2986
26-
ClassicLru,50000,0.86,1000000,15,60.2531
27-
ConcurrentLru,50000,0.86,1000000,15,61.9663
28-
ClassicLru,50000,0.86,1000000,20,65.1358
29-
ConcurrentLru,50000,0.86,1000000,20,66.3116
30-
ClassicLru,50000,0.86,1000000,25,69.1442
31-
ConcurrentLru,50000,0.86,1000000,25,69.9031
32-
ClassicLru,50000,0.86,1000000,30,72.5617
33-
ConcurrentLru,50000,0.86,1000000,30,72.95270000000001
34-
ClassicLru,50000,0.86,1000000,35,75.54610000000001
35-
ConcurrentLru,50000,0.86,1000000,35,75.6668
36-
ClassicLru,50000,0.86,1000000,40,78.20899999999999
37-
ConcurrentLru,50000,0.86,1000000,40,78.0539
1+
Cache,N,s,Samples,IsScan,CacheSizePercent,HitRatio,Duration
2+
ClassicLru,50000,0.5,1000000,False,1.25,3.1815,00:00:00.2219936
3+
ConcurrentLru,50000,0.5,1000000,False,1.25,5.8393,00:00:00.2553361
4+
ClassicLru,50000,0.5,1000000,True,1.25,1.46085,00:00:00.3538231
5+
ConcurrentLru,50000,0.5,1000000,True,1.25,2.8294,00:00:00.4064362
6+
ClassicLru,50000,0.5,1000000,False,2.5,5.8342,00:00:00.1863446
7+
ConcurrentLru,50000,0.5,1000000,False,2.5,8.8863,00:00:00.1956831
8+
ClassicLru,50000,0.5,1000000,True,2.5,2.83075,00:00:00.3777144
9+
ConcurrentLru,50000,0.5,1000000,True,2.5,4.4782,00:00:00.3984226
10+
ClassicLru,50000,0.5,1000000,False,5,10.4677,00:00:00.1988212
11+
ConcurrentLru,50000,0.5,1000000,False,5,13.6737,00:00:00.1985374
12+
ClassicLru,50000,0.5,1000000,True,5,5.35485,00:00:00.4012853
13+
ConcurrentLru,50000,0.5,1000000,True,5,7.24645,00:00:00.4069157
14+
ClassicLru,50000,0.5,1000000,False,7.5,14.5759,00:00:00.2001636
15+
ConcurrentLru,50000,0.5,1000000,False,7.5,17.6997,00:00:00.1955304
16+
ClassicLru,50000,0.5,1000000,True,7.5,7.7721,00:00:00.4027166
17+
ConcurrentLru,50000,0.5,1000000,True,7.5,9.6753,00:00:00.4025873
18+
ClassicLru,50000,0.5,1000000,False,10,18.348300000000002,00:00:00.2080376
19+
ConcurrentLru,50000,0.5,1000000,False,10,21.299599999999998,00:00:00.1955030
20+
ClassicLru,50000,0.5,1000000,True,10,10.07585,00:00:00.4116798
21+
ConcurrentLru,50000,0.5,1000000,True,10,11.9522,00:00:00.4029283
22+
ClassicLru,50000,0.5,1000000,False,1.25,3.1815,00:00:00.1864771
23+
ConcurrentLru,50000,0.5,1000000,False,1.25,5.8393,00:00:00.1990631
24+
ClassicLru,50000,0.5,1000000,True,1.25,1.46085,00:00:00.3600207
25+
ConcurrentLru,50000,0.5,1000000,True,1.25,2.8294,00:00:00.4222347
26+
ClassicLru,50000,0.5,1000000,False,15,25.181700000000003,00:00:00.2282900
27+
ConcurrentLru,50000,0.5,1000000,False,15,27.7221,00:00:00.2087622
28+
ClassicLru,50000,0.5,1000000,True,15,14.496400000000001,00:00:00.4531845
29+
ConcurrentLru,50000,0.5,1000000,True,15,16.217699999999997,00:00:00.4159303
30+
ClassicLru,50000,0.5,1000000,False,20,31.4061,00:00:00.2372111
31+
ConcurrentLru,50000,0.5,1000000,False,20,33.5112,00:00:00.1965954
32+
ClassicLru,50000,0.5,1000000,True,20,18.72495,00:00:00.4799955
33+
ConcurrentLru,50000,0.5,1000000,True,20,20.244699999999998,00:00:00.4276315
34+
ClassicLru,50000,0.5,1000000,False,25,37.160399999999996,00:00:00.2471366
35+
ConcurrentLru,50000,0.5,1000000,False,25,38.822,00:00:00.2056509
36+
ClassicLru,50000,0.5,1000000,True,25,22.76605,00:00:00.5113088
37+
ConcurrentLru,50000,0.5,1000000,True,25,24.1382,00:00:00.4348177
38+
ClassicLru,50000,0.5,1000000,False,30,42.5495,00:00:00.2536968
39+
ConcurrentLru,50000,0.5,1000000,False,30,43.802600000000005,00:00:00.2038566
40+
ClassicLru,50000,0.5,1000000,True,30,26.716450000000002,00:00:00.5487216
41+
ConcurrentLru,50000,0.5,1000000,True,30,27.87425,00:00:00.4446285
42+
ClassicLru,50000,0.5,1000000,False,35,47.5886,00:00:00.2674640
43+
ConcurrentLru,50000,0.5,1000000,False,35,48.4817,00:00:00.2008129
44+
ClassicLru,50000,0.5,1000000,True,35,30.5348,00:00:00.5652294
45+
ConcurrentLru,50000,0.5,1000000,True,35,31.53125,00:00:00.4446676
46+
ClassicLru,50000,0.5,1000000,False,40,52.39509999999999,00:00:00.2679638
47+
ConcurrentLru,50000,0.5,1000000,False,40,52.912099999999995,00:00:00.1968226
48+
ClassicLru,50000,0.5,1000000,True,40,34.319300000000005,00:00:00.5922026
49+
ConcurrentLru,50000,0.5,1000000,True,40,35.1464,00:00:00.4419703
50+
ClassicLru,50000,0.86,1000000,False,1.25,28.7188,00:00:00.1590029
51+
ConcurrentLru,50000,0.86,1000000,False,1.25,33.7497,00:00:00.1745940
52+
ClassicLru,50000,0.86,1000000,True,1.25,11.44975,00:00:00.3356978
53+
ConcurrentLru,50000,0.86,1000000,True,1.25,16.35335,00:00:00.3678454
54+
ClassicLru,50000,0.86,1000000,False,2.5,35.9255,00:00:00.1521401
55+
ConcurrentLru,50000,0.86,1000000,False,2.5,40.2911,00:00:00.1459784
56+
ClassicLru,50000,0.86,1000000,True,2.5,14.897750000000002,00:00:00.3460469
57+
ConcurrentLru,50000,0.86,1000000,True,2.5,19.612199999999998,00:00:00.3438083
58+
ClassicLru,50000,0.86,1000000,False,5,44.2184,00:00:00.1503402
59+
ConcurrentLru,50000,0.86,1000000,False,5,47.7282,00:00:00.1351654
60+
ClassicLru,50000,0.86,1000000,True,5,19.223100000000002,00:00:00.3556054
61+
ConcurrentLru,50000,0.86,1000000,True,5,23.50135,00:00:00.3422126
62+
ClassicLru,50000,0.86,1000000,False,7.5,49.681999999999995,00:00:00.1527913
63+
ConcurrentLru,50000,0.86,1000000,False,7.5,52.61130000000001,00:00:00.1313441
64+
ClassicLru,50000,0.86,1000000,True,7.5,22.28575,00:00:00.3620021
65+
ConcurrentLru,50000,0.86,1000000,True,7.5,26.25925,00:00:00.3426238
66+
ClassicLru,50000,0.86,1000000,False,10,53.8481,00:00:00.1599107
67+
ConcurrentLru,50000,0.86,1000000,False,10,56.3201,00:00:00.1267778
68+
ClassicLru,50000,0.86,1000000,True,10,24.804750000000002,00:00:00.3666843
69+
ConcurrentLru,50000,0.86,1000000,True,10,28.510999999999996,00:00:00.3390340
70+
ClassicLru,50000,0.86,1000000,False,1.25,28.7188,00:00:00.1596049
71+
ConcurrentLru,50000,0.86,1000000,False,1.25,33.7497,00:00:00.1566299
72+
ClassicLru,50000,0.86,1000000,True,1.25,11.44975,00:00:00.3432512
73+
ConcurrentLru,50000,0.86,1000000,True,1.25,16.35335,00:00:00.3590832
74+
ClassicLru,50000,0.86,1000000,False,15,60.2324,00:00:00.1584608
75+
ConcurrentLru,50000,0.86,1000000,False,15,61.978100000000005,00:00:00.1257132
76+
ClassicLru,50000,0.86,1000000,True,15,28.9737,00:00:00.3945479
77+
ConcurrentLru,50000,0.86,1000000,True,15,32.244499999999995,00:00:00.3474522
78+
ClassicLru,50000,0.86,1000000,False,20,65.12790000000001,00:00:00.1625112
79+
ConcurrentLru,50000,0.86,1000000,False,20,66.2993,00:00:00.1190971
80+
ClassicLru,50000,0.86,1000000,True,20,32.4771,00:00:00.4151890
81+
ConcurrentLru,50000,0.86,1000000,True,20,35.393550000000005,00:00:00.3661938
82+
ClassicLru,50000,0.86,1000000,False,25,69.0926,00:00:00.1765459
83+
ConcurrentLru,50000,0.86,1000000,False,25,69.8668,00:00:00.1220536
84+
ClassicLru,50000,0.86,1000000,True,25,35.5605,00:00:00.4374009
85+
ConcurrentLru,50000,0.86,1000000,True,25,38.201049999999995,00:00:00.3574605
86+
ClassicLru,50000,0.86,1000000,False,30,72.4841,00:00:00.1744888
87+
ConcurrentLru,50000,0.86,1000000,False,30,72.91159999999999,00:00:00.1207647
88+
ClassicLru,50000,0.86,1000000,True,30,38.38225,00:00:00.4761851
89+
ConcurrentLru,50000,0.86,1000000,True,30,40.8101,00:00:00.3642631
90+
ClassicLru,50000,0.86,1000000,False,35,75.457,00:00:00.1756699
91+
ConcurrentLru,50000,0.86,1000000,False,35,75.5849,00:00:00.1246051
92+
ClassicLru,50000,0.86,1000000,True,35,40.98865,00:00:00.5014744
93+
ConcurrentLru,50000,0.86,1000000,True,35,43.285050000000005,00:00:00.3813605
94+
ClassicLru,50000,0.86,1000000,False,40,78.0983,00:00:00.1715080
95+
ConcurrentLru,50000,0.86,1000000,False,40,77.9631,00:00:00.1156444
96+
ClassicLru,50000,0.86,1000000,True,40,43.426550000000006,00:00:00.4955222
97+
ConcurrentLru,50000,0.86,1000000,True,40,45.6681,00:00:00.3754359

‎README.md‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,10 +105,36 @@ When the cache is small, below 15% of the total key space, ConcurrentLru outperf
105105
<table>
106106
<tr>
107107
<td>
108-
<img src="https://user-images.githubusercontent.com/12851828/84707621-e2a62480-af13-11ea-91e7-726911bce162.png" width="400"/>
108+
<img src="https://user-images.githubusercontent.com/12851828/84844130-a00d4680-affe-11ea-8f7a-e3c66180d8b9.png" width="250"/>
109109
</td>
110110
<td>
111-
<img src="https://user-images.githubusercontent.com/12851828/84707663-f81b4e80-af13-11ea-96d4-1ba71444d333.png" width="400"/>
111+
<img src="https://user-images.githubusercontent.com/12851828/84844172-b6b39d80-affe-11ea-9a29-cbdae6020246.png" width="250"/>
112+
</td>
113+
</tr>
114+
</table>
115+
116+
This is the same test, but interleaving a sequential scan of every key. In this case, ConcurrentLru performs better across the board.
117+
118+
<table>
119+
<tr>
120+
<td>
121+
<img src="https://user-images.githubusercontent.com/12851828/84841922-a4366580-aff8-11ea-93dd-568d60cd82d9.png" width="250"/>
122+
</td>
123+
<td>
124+
<img src="https://user-images.githubusercontent.com/12851828/84842237-730a6500-aff9-11ea-9a46-40141adff920.png" width="250"/>
125+
</td>
126+
</tr>
127+
</table>
128+
129+
These charts summarize the percentage increase in hit rate ConcurrentLru vs LRU. Increase during sequential scan can be > 90%, depending on cache size relative to data set.
130+
131+
<table>
132+
<tr>
133+
<td>
134+
<img src="https://user-images.githubusercontent.com/12851828/84843966-283f1c00-affe-11ea-99c9-20aa01f307f0.png" width="250"/>
135+
</td>
136+
<td>
137+
<img src="https://user-images.githubusercontent.com/12851828/84844003-3d1baf80-affe-11ea-9266-e83efe2e8c35.png" width="250"/>
112138
</td>
113139
</tr>
114140
</table>

0 commit comments

Comments
 (0)