Skip to content

Commit 402529a

Browse files
HowardWoloskyHowardWolosky
authored
Attempting to increase reliability of some Pester tests - take 2 (#268)
This is re-implementing #265 in a more robust way, thanks to a suggestion from @X-Guardian in #267. Instead of adding in one-off sleeps throughout the test codebase, there is now a new configuration value `StateChangeDelaySeconds`) that will allow us to insert a delay before returning the result of _any_ state change request. This should ideally consistently add reliability throughout the entire test codebase.
1 parent f406cc5 commit 402529a

10 files changed

+24
-234
lines changed

‎GitHubConfiguration.ps1‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,11 @@ function Set-GitHubConfiguration
147147
.PARAMETER RetryDelaySeconds
148148
The number of seconds to wait before retrying a command again after receiving a 202 response.
149149
150+
.PARAMETER StateChangeDelaySeconds
151+
The number of seconds to wait before returning the result after executing a command that
152+
may result in a state change on the server. This is intended to only be used during test
153+
execution in order to increase reliability.
154+
150155
.PARAMETER SuppressNoTokenWarning
151156
If an Access Token has not been configured, this module will provide a warning to the user
152157
informing them of this, once per session. If it is expected that this module will regularly
@@ -228,6 +233,8 @@ function Set-GitHubConfiguration
228233

229234
[int] $RetryDelaySeconds,
230235

236+
[int] $StateChangeDelaySeconds,
237+
231238
[switch] $SuppressNoTokenWarning,
232239

233240
[switch] $SuppressTelemetryReminder,
@@ -312,6 +319,7 @@ function Get-GitHubConfiguration
312319
'LogTimeAsUtc',
313320
'MultiRequestProgressThreshold',
314321
'RetryDelaySeconds',
322+
'StateChangeDelaySeconds',
315323
'SuppressNoTokenWarning',
316324
'SuppressTelemetryReminder',
317325
'TestConfigSettingsHash',
@@ -659,6 +667,7 @@ function Import-GitHubConfiguration
659667
'logTimeAsUtc' = $false
660668
'multiRequestProgressThreshold' = 10
661669
'retryDelaySeconds' = 30
670+
'stateChangeDelaySeconds' = 0
662671
'suppressNoTokenWarning' = $false
663672
'suppressTelemetryReminder' = $false
664673
'webRequestTimeoutSec' = 0

‎GitHubCore.ps1‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,16 @@ function Invoke-GHRestMethod
321321
}
322322
}
323323

324+
# Allow for a delay after a command that may result in a state change in order to
325+
#increase the reliability of the UT's which attempt multiple successive state change
326+
# on the same object.
327+
$stateChangeDelaySeconds = $(Get-GitHubConfiguration -Name 'StateChangeDelaySeconds')
328+
$stateChangeMethods = @('Delete', 'Post', 'Patch', 'Put')
329+
if (($stateChangeDelaySeconds -gt 0) -and ($Method -in $stateChangeMethods))
330+
{
331+
Start-Sleep -Seconds $stateChangeDelaySeconds
332+
}
333+
324334
if ($ExtendedResult)
325335
{
326336
$finalResultEx = @{

‎Tests/Common.ps1‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,10 @@ function Initialize-CommonTestSetup
101101
Set-GitHubConfiguration -MultiRequestProgressThreshold 0 # Status corrupts the raw CI logs for Linux and Mac, and makes runs take slightly longer.
102102
Set-GitHubConfiguration -DisableUpdateCheck # The update check is unnecessary during tests.
103103

104-
# The number of seconds to sleep after performing some operations to ensure that successive
105-
# API calls properly reflect previously updated state.
106-
$script:defaultSleepSecondsForReliability = 3
104+
# We execute so many successive state changing commands on the same object that sometimes
105+
# GitHub gets confused. We'll add an intentional delay to slow down our execution in an effort
106+
# to increase the reliability of the tests.
107+
Set-GitHubConfiguration -StateChangeDelaySeconds 3
107108
}
108109

109110
Initialize-CommonTestSetup

‎Tests/GitHubAssignees.tests.ps1‎

Lines changed: 0 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -108,21 +108,13 @@ try
108108
Context 'Adding and removing an assignee via parameters' {
109109
$issue = $repo | New-GitHubIssue -Title "Test issue"
110110

111-
# The CI build has been unreliable with this test.
112-
# Adding a short sleep to ensure successive queries reflect updated state.
113-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
114-
115111
It 'Should have no assignees when created' {
116112
$issue.assignee.login | Should -BeNullOrEmpty
117113
$issue.assignees | Should -BeNullOrEmpty
118114
}
119115

120116
$updatedIssue = Add-GitHubAssignee -OwnerName $script:ownerName -RepositoryName $repo.name -Issue $issue.number -Assignee $owner.login
121117

122-
# The CI build has been unreliable with this test.
123-
# Adding a short sleep to ensure successive queries reflect updated state.
124-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
125-
126118
It 'Should have returned the same issue' {
127119
$updatedIssue.number | Should -Be $issue.number
128120
}
@@ -155,21 +147,13 @@ try
155147
Context 'Adding an assignee with the repo on the pipeline' {
156148
$issue = $repo | New-GitHubIssue -Title "Test issue"
157149

158-
# The CI build has been unreliable with this test.
159-
# Adding a short sleep to ensure successive queries reflect updated state.
160-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
161-
162150
It 'Should have no assignees when created' {
163151
$issue.assignee.login | Should -BeNullOrEmpty
164152
$issue.assignees | Should -BeNullOrEmpty
165153
}
166154

167155
$updatedIssue = $repo | Add-GitHubAssignee -Issue $issue.number -Assignee $owner.login
168156

169-
# The CI build has been unreliable with this test.
170-
# Adding a short sleep to ensure successive queries reflect updated state.
171-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
172-
173157
It 'Should have returned the same issue' {
174158
$updatedIssue.number | Should -Be $issue.number
175159
}
@@ -202,21 +186,13 @@ try
202186
Context 'Adding an assignee with the issue on the pipeline' {
203187
$issue = $repo | New-GitHubIssue -Title "Test issue"
204188

205-
# The CI build has been unreliable with this test.
206-
# Adding a short sleep to ensure successive queries reflect updated state.
207-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
208-
209189
It 'Should have no assignees when created' {
210190
$issue.assignee.login | Should -BeNullOrEmpty
211191
$issue.assignees | Should -BeNullOrEmpty
212192
}
213193

214194
$updatedIssue = $issue | Add-GitHubAssignee -Assignee $owner.login
215195

216-
# The CI build has been unreliable with this test.
217-
# Adding a short sleep to ensure successive queries reflect updated state.
218-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
219-
220196
It 'Should have returned the same issue' {
221197
$updatedIssue.number | Should -Be $issue.number
222198
}
@@ -249,21 +225,13 @@ try
249225
Context 'Adding an assignee with the assignee user object on the pipeline' {
250226
$issue = $repo | New-GitHubIssue -Title "Test issue"
251227

252-
# The CI build has been unreliable with this test.
253-
# Adding a short sleep to ensure successive queries reflect updated state.
254-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
255-
256228
It 'Should have no assignees when created' {
257229
$issue.assignee.login | Should -BeNullOrEmpty
258230
$issue.assignees | Should -BeNullOrEmpty
259231
}
260232

261233
$updatedIssue = $owner | Add-GitHubAssignee -OwnerName $script:ownerName -RepositoryName $repo.name -Issue $issue.number
262234

263-
# The CI build has been unreliable with this test.
264-
# Adding a short sleep to ensure successive queries reflect updated state.
265-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
266-
267235
It 'Should have returned the same issue' {
268236
$updatedIssue.number | Should -Be $issue.number
269237
}

‎Tests/GitHubContents.tests.ps1‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,6 @@ try
3737
BeforeAll {
3838
# AutoInit will create a readme with the GUID of the repo name
3939
$repo = New-GitHubRepository -RepositoryName ($repoGuid) -AutoInit
40-
41-
# The CI build has been unreliable with this test.
42-
# Adding a short sleep to ensure successive queries reflect updated state.
43-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
4440
}
4541

4642
AfterAll {
@@ -262,12 +258,7 @@ try
262258
Describe 'GitHubContents/Set-GitHubContent' {
263259
BeforeAll {
264260
$repoName = [Guid]::NewGuid().Guid
265-
266261
$repo = New-GitHubRepository -RepositoryName $repoName -AutoInit
267-
268-
# The CI build has been unreliable with this test.
269-
# Adding a short sleep to ensure successive queries reflect updated state.
270-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
271262
}
272263

273264
Context 'When setting new file content' {

‎Tests/GitHubIssues.tests.ps1‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -161,9 +161,6 @@ try
161161
for ($i = 0; $i -lt 4; $i++)
162162
{
163163
$newIssues += New-GitHubIssue -OwnerName $script:ownerName -RepositoryName $repo.name -Title ([Guid]::NewGuid().Guid)
164-
165-
# Needed to ensure that there is a unique creation timestamp between issues
166-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
167164
}
168165

169166
$newIssues[0] = Set-GitHubIssue -OwnerName $script:ownerName -RepositoryName $repo.name -Issue $newIssues[0].number -State Closed
@@ -550,10 +547,6 @@ try
550547

551548
Lock-GitHubIssue -OwnerName $script:OwnerName -RepositoryName $repo.name -Issue $issue.number
552549

553-
# The CI build has been unreliable with this test.
554-
# Adding a short sleep to ensure successive queries reflect updated state.
555-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
556-
557550
$timeline = @(Get-GitHubIssueTimeline -OwnerName $script:OwnerName -RepositoryName $repo.name -Issue $issue.number)
558551
It 'Should have an event now' {
559552
$timeline.Count | Should -Be 1

‎Tests/GitHubLabels.tests.ps1‎

Lines changed: 0 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -80,11 +80,6 @@ try
8080
BeforeAll {
8181
$repositoryName = [Guid]::NewGuid().Guid
8282
$repo = New-GitHubRepository -RepositoryName $repositoryName
83-
84-
# The CI build has been unreliable with this test.
85-
# Adding a short sleep to ensure successive queries reflect updated state.
86-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
87-
8883
Initialize-GitHubLabel -OwnerName $script:ownerName -RepositoryName $repositoryName -Label $defaultLabels
8984
}
9085

@@ -209,10 +204,6 @@ try
209204
BeforeAll {
210205
$repositoryName = [Guid]::NewGuid().Guid
211206
$repo = New-GitHubRepository -RepositoryName $repositoryName
212-
213-
# The CI build has been unreliable with this test.
214-
# Adding a short sleep to ensure successive queries reflect updated state.
215-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
216207
}
217208

218209
AfterAll {
@@ -327,10 +318,6 @@ try
327318
BeforeAll {
328319
$repositoryName = [Guid]::NewGuid().Guid
329320
$repo = New-GitHubRepository -RepositoryName $repositoryName
330-
331-
# The CI build has been unreliable with this test.
332-
# Adding a short sleep to ensure successive queries reflect updated state.
333-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
334321
}
335322

336323
AfterAll {
@@ -339,11 +326,6 @@ try
339326

340327
Context 'Removing a label with parameters' {
341328
$label = $repo | New-GitHubLabel -Label 'test' -Color 'CCCCCC'
342-
343-
# The CI build has been unreliable with this test.
344-
# Adding a short sleep to ensure successive queries reflect updated state.
345-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
346-
347329
Remove-GitHubLabel -OwnerName $script:ownerName -RepositoryName $repositoryName -Label $label.name -Force
348330

349331
It 'Should be gone after being removed by parameter' {
@@ -353,11 +335,6 @@ try
353335

354336
Context 'Removing a label with the repo on the pipeline' {
355337
$label = $repo | New-GitHubLabel -Label 'test' -Color 'CCCCCC'
356-
357-
# The CI build has been unreliable with this test.
358-
# Adding a short sleep to ensure successive queries reflect updated state.
359-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
360-
361338
$repo | Remove-GitHubLabel -Label $label.name -Confirm:$false
362339

363340
It 'Should be gone after being removed by parameter' {
@@ -367,11 +344,6 @@ try
367344

368345
Context 'Removing a label with the name on the pipeline' {
369346
$label = $repo | New-GitHubLabel -Label 'test' -Color 'CCCCCC'
370-
371-
# The CI build has been unreliable with this test.
372-
# Adding a short sleep to ensure successive queries reflect updated state.
373-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
374-
375347
$label.name | Remove-GitHubLabel -OwnerName $script:ownerName -RepositoryName $repositoryName -Force
376348

377349
It 'Should be gone after being removed by parameter' {
@@ -381,11 +353,6 @@ try
381353

382354
Context 'Removing a label with the label object on the pipeline' {
383355
$label = $repo | New-GitHubLabel -Label 'test' -Color 'CCCCCC'
384-
385-
# The CI build has been unreliable with this test.
386-
# Adding a short sleep to ensure successive queries reflect updated state.
387-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
388-
389356
$label | Remove-GitHubLabel -Force
390357

391358
It 'Should be gone after being removed by parameter' {
@@ -398,10 +365,6 @@ try
398365
BeforeAll {
399366
$repositoryName = [Guid]::NewGuid().Guid
400367
$repo = New-GitHubRepository -RepositoryName $repositoryName
401-
402-
# The CI build has been unreliable with this test.
403-
# Adding a short sleep to ensure successive queries reflect updated state.
404-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
405368
}
406369

407370
AfterAll {
@@ -579,10 +542,6 @@ try
579542
BeforeAll {
580543
$repositoryName = [Guid]::NewGuid().Guid
581544
$repo = New-GitHubRepository -RepositoryName $repositoryName
582-
583-
# The CI build has been unreliable with this test.
584-
# Adding a short sleep to ensure successive queries reflect updated state.
585-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
586545
}
587546

588547
AfterAll {
@@ -657,11 +616,6 @@ try
657616
BeforeAll {
658617
$repositoryName = [Guid]::NewGuid().Guid
659618
$repo = New-GitHubRepository -RepositoryName $repositoryName
660-
661-
# The CI build has been unreliable with this test.
662-
# Adding a short sleep to ensure successive queries reflect updated state.
663-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
664-
665619
$repo | Initialize-GitHubLabel -Label $defaultLabels
666620
}
667621

@@ -890,11 +844,6 @@ try
890844
BeforeAll {
891845
$repositoryName = [Guid]::NewGuid().Guid
892846
$repo = New-GitHubRepository -RepositoryName $repositoryName
893-
894-
# The CI build has been unreliable with this test.
895-
# Adding a short sleep to ensure successive queries reflect updated state.
896-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
897-
898847
$repo | Initialize-GitHubLabel -Label $defaultLabels
899848
}
900849

@@ -961,11 +910,6 @@ try
961910
BeforeAll {
962911
$repositoryName = [Guid]::NewGuid().Guid
963912
$repo = New-GitHubRepository -RepositoryName $repositoryName
964-
965-
# The CI build has been unreliable with this test.
966-
# Adding a short sleep to ensure successive queries reflect updated state.
967-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
968-
969913
$repo | Initialize-GitHubLabel -Label $defaultLabels
970914
}
971915

@@ -1133,11 +1077,6 @@ try
11331077
BeforeAll {
11341078
$repositoryName = [Guid]::NewGuid().Guid
11351079
$repo = New-GitHubRepository -RepositoryName $repositoryName
1136-
1137-
# The CI build has been unreliable with this test.
1138-
# Adding a short sleep to ensure successive queries reflect updated state.
1139-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
1140-
11411080
$repo | Initialize-GitHubLabel -Label $defaultLabels
11421081
}
11431082

@@ -1287,11 +1226,6 @@ try
12871226
BeforeAll {
12881227
$repositoryName = [Guid]::NewGuid().Guid
12891228
$repo = New-GitHubRepository -RepositoryName $repositoryName
1290-
1291-
# The CI build has been unreliable with this test.
1292-
# Adding a short sleep to ensure successive queries reflect updated state.
1293-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
1294-
12951229
$repo | Initialize-GitHubLabel -Label $defaultLabels
12961230

12971231
$milestone = $repo | New-GitHubMilestone -Title 'test milestone'

‎Tests/GitHubProjects.tests.ps1‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -611,10 +611,6 @@ try
611611
$project = New-GitHubProject -OwnerName $script:ownerName -RepositoryName $repo.name -ProjectName $defaultRepoProject -Description $defaultRepoProjectDesc
612612
$null = Remove-GitHubProject -Project $project.id -Confirm:$false
613613
It 'Project should be removed' {
614-
# The CI build has been unreliable with this test.
615-
# Adding a short sleep to ensure successive queries reflect updated state.
616-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
617-
618614
{Get-GitHubProject -Project $project.id} | Should -Throw
619615
}
620616
}
@@ -623,10 +619,6 @@ try
623619
$project = $repo | New-GitHubProject -ProjectName $defaultRepoProject -Description $defaultRepoProjectDesc
624620
$project | Remove-GitHubProject -Force
625621
It 'Project should be removed' {
626-
# The CI build has been unreliable with this test.
627-
# Adding a short sleep to ensure successive queries reflect updated state.
628-
Start-Sleep -Seconds $script:defaultSleepSecondsForReliability
629-
630622
{$project | Get-GitHubProject} | Should -Throw
631623
}
632624
}

0 commit comments

Comments
 (0)