Skip to content

Commit 0a5251b

Browse files
RomanxRomanx
committed
Add tests for SkippingDeadNodes for Sticky #1810 
1 parent 9a14cd2 commit 0a5251b

File tree

2 files changed

+179
-1
lines changed

2 files changed

+179
-1
lines changed

‎src/Tests/ClientConcepts/ConnectionPooling/Sticky/SkipDeadNodes.doc.cs‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
using System;
2+
using System.Diagnostics.CodeAnalysis;
3+
using System.Linq;
4+
using System.Threading.Tasks;
5+
using Elasticsearch.Net;
6+
using FluentAssertions;
7+
using Tests.Framework;
8+
using static Tests.Framework.TimesHelper;
9+
using static Elasticsearch.Net.AuditEvent;
10+
11+
namespace Tests.ClientConcepts.ConnectionPooling.Sticky
12+
{
13+
public class SkipDeadNodes
14+
{
15+
/** Sticky - Skipping Dead Nodes
16+
* When selecting nodes the connection pool will try and skip all the nodes that are marked dead.
17+
*/
18+
19+
protected int NumberOfNodes = 3;
20+
21+
[U] public void EachViewDoesNotSkip()
22+
{
23+
var seeds = Enumerable.Range(9200, NumberOfNodes).Select(p => new Node(new Uri("http://localhost:" + p))).ToList();
24+
var pool = new StickyConnectionPool(seeds);
25+
for (var i = 0; i < 20; i++)
26+
{
27+
var node = pool.CreateView().First();
28+
node.Uri.Port.Should().Be(9200);
29+
node = pool.CreateView().First();
30+
node.Uri.Port.Should().Be(9200);
31+
node = pool.CreateView().First();
32+
node.Uri.Port.Should().Be(9200);
33+
}
34+
}
35+
36+
[U] public void EachViewSeesNextButSkipsTheDeadNode()
37+
{
38+
var seeds = Enumerable.Range(9200, NumberOfNodes).Select(p => new Node(new Uri("http://localhost:" + p))).ToList();
39+
seeds.First().MarkDead(DateTime.Now.AddDays(1));
40+
var pool = new StickyConnectionPool(seeds);
41+
for (var i = 0; i < 20; i++)
42+
{
43+
var node = pool.CreateView().First();
44+
node.Uri.Port.Should().Be(9201);
45+
node = pool.CreateView().First();
46+
node.Uri.Port.Should().Be(9201);
47+
}
48+
/** After we marke the first node alive again we expect it to be hit again*/
49+
seeds.First().MarkAlive();
50+
for (var i = 0; i < 20; i++)
51+
{
52+
var node = pool.CreateView().First();
53+
node.Uri.Port.Should().Be(9200);
54+
node = pool.CreateView().First();
55+
node.Uri.Port.Should().Be(9200);
56+
node = pool.CreateView().First();
57+
node.Uri.Port.Should().Be(9200);
58+
}
59+
}
60+
61+
[U] public void ViewSeesResurrectedNodes()
62+
{
63+
var dateTimeProvider = new TestableDateTimeProvider();
64+
var seeds = Enumerable.Range(9200, NumberOfNodes).Select(p => new Node(new Uri("http://localhost:" + p))).ToList();
65+
seeds.First().MarkDead(dateTimeProvider.Now().AddDays(1));
66+
var pool = new StickyConnectionPool(seeds, dateTimeProvider: dateTimeProvider);
67+
for (var i = 0; i < 20; i++)
68+
{
69+
var node = pool.CreateView().First();
70+
node.Uri.Port.Should().Be(9201);
71+
node = pool.CreateView().First();
72+
node.Uri.Port.Should().Be(9201);
73+
}
74+
/** If we forward our clock 2 days the node that was marked dead until tomorrow (or yesterday!) should be resurrected */
75+
dateTimeProvider.ChangeTime(d => d.AddDays(2));
76+
var n = pool.CreateView().First();
77+
n.Uri.Port.Should().Be(9200);
78+
n = pool.CreateView().First();
79+
n.Uri.Port.Should().Be(9200);
80+
n = pool.CreateView().First();
81+
n.Uri.Port.Should().Be(9200);
82+
n.IsResurrected.Should().BeTrue();
83+
}
84+
85+
[U, SuppressMessage("AsyncUsage", "AsyncFixer001:Unnecessary async/await usage", Justification = "Its a test")]
86+
public async Task FallsOverDeadNodes()
87+
{
88+
/** A cluster with 2 nodes where the second node fails on ping */
89+
var audit = new Auditor(() => Framework.Cluster
90+
.Nodes(4)
91+
.ClientCalls(p => p.Succeeds(Always))
92+
.ClientCalls(p => p.OnPort(9200).FailAlways())
93+
.ClientCalls(p => p.OnPort(9201).FailAlways())
94+
.StickyConnectionPool()
95+
.Settings(p => p.DisablePing())
96+
);
97+
98+
await audit.TraceCalls(
99+
/** The first call goes to 9200 which succeeds */
100+
new ClientCall {
101+
{ BadResponse, 9200},
102+
{ BadResponse, 9201},
103+
{ HealthyResponse, 9202},
104+
{ pool => pool.Nodes.Where(n=>!n.IsAlive).Should().HaveCount(2) }
105+
},
106+
/** The 2nd call does a ping on 9201 because its used for the first time.
107+
* It fails so we wrap over to node 9202 */
108+
new ClientCall {
109+
{ HealthyResponse, 9202},
110+
/** Finally we assert that the connectionpool has one node that is marked as dead */
111+
{ pool => pool.Nodes.Where(n=>!n.IsAlive).Should().HaveCount(2) }
112+
},
113+
new ClientCall {
114+
{ HealthyResponse, 9202},
115+
{ pool => pool.Nodes.Where(n=>!n.IsAlive).Should().HaveCount(2) }
116+
}
117+
);
118+
}
119+
120+
[U, SuppressMessage("AsyncUsage", "AsyncFixer001:Unnecessary async/await usage", Justification = "Its a test")]
121+
public async Task PicksADifferentNodeEachTimeAnodeIsDown()
122+
{
123+
/** A cluster with 2 nodes where the second node fails on ping */
124+
var audit = new Auditor(() => Framework.Cluster
125+
.Nodes(4)
126+
.ClientCalls(p => p.Fails(Always))
127+
.StickyConnectionPool()
128+
.Settings(p => p.DisablePing())
129+
);
130+
131+
await audit.TraceCalls(
132+
/** All the calls fail */
133+
new ClientCall {
134+
{ BadResponse, 9200},
135+
{ BadResponse, 9201},
136+
{ BadResponse, 9202},
137+
{ BadResponse, 9203},
138+
{ MaxRetriesReached },
139+
{ pool => pool.Nodes.Where(n=>!n.IsAlive).Should().HaveCount(4) }
140+
},
141+
/** After all our registered nodes are marked dead we want to sample a single dead node
142+
* each time to quickly see if the cluster is back up. We do not want to retry all 4
143+
* nodes
144+
*/
145+
new ClientCall {
146+
{ AllNodesDead },
147+
{ Resurrection, 9200},
148+
{ BadResponse, 9200},
149+
{ pool => pool.Nodes.Where(n=>!n.IsAlive).Should().HaveCount(4) }
150+
},
151+
new ClientCall {
152+
{ AllNodesDead },
153+
{ Resurrection, 9201},
154+
{ BadResponse, 9201},
155+
{ pool => pool.Nodes.Where(n=>!n.IsAlive).Should().HaveCount(4) }
156+
},
157+
new ClientCall {
158+
{ AllNodesDead },
159+
{ Resurrection, 9202},
160+
{ BadResponse, 9202},
161+
{ pool => pool.Nodes.Where(n=>!n.IsAlive).Should().HaveCount(4) }
162+
},
163+
new ClientCall {
164+
{ AllNodesDead },
165+
{ Resurrection, 9203},
166+
{ BadResponse, 9203},
167+
{ pool => pool.Nodes.Where(n=>!n.IsAlive).Should().HaveCount(4) }
168+
}
169+
);
170+
}
171+
}
172+
}

‎src/Tests/Framework/VirtualClustering/VirtualCluster.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,12 @@ public SealedVirtualCluster SniffingConnectionPool(Func<IList<Node>, IEnumerable
7878
var nodes = seedNodesSelector?.Invoke(this._nodes) ?? this._nodes;
7979
return new SealedVirtualCluster(this, new SniffingConnectionPool(nodes, randomize: false, dateTimeProvider: this.DateTimeProvider), this.DateTimeProvider);
8080
}
81-
}
81+
82+
public SealedVirtualCluster StickyConnectionPool(Func<IList<Node>, IEnumerable<Node>> seedNodesSelector = null)
83+
{
84+
var nodes = seedNodesSelector?.Invoke(this._nodes) ?? this._nodes;
85+
return new SealedVirtualCluster(this, new StickyConnectionPool(nodes, dateTimeProvider: this.DateTimeProvider), this.DateTimeProvider);
86+
}
87+
}
8288

8389
}

0 commit comments

Comments
 (0)