Reducing warning messages during unit tests configuration (#132)

During UT's, `Reset-GitHubConfiguration` and `Clear-GitHubAuthentication` are called frequently.
This change moves their warning messages to be in Verbose output instead, in order to reduce
the noise that is commonly seen when they're called.

Additionally, the warning that is seen at the start of running a UT test suite is now
completely suppressed if the user has modified `Tests/Config/Settings.ps1` (presumably to
have it reference accounts that they have access for). This is achieved by storing the hash of the Settings file in the code, along with documentation to remind users to update it if the Settings
file is ever permanently changed.

Finally, an additional test has been added to the CI pipeline to validate that the Settings file
content matches the hash that is present in the code.

Author: smaglio81

GitHubConfiguration.ps1

@@ -288,6 +288,7 @@ function Get-GitHubConfiguration
             'RetryDelaySeconds',
             'SuppressNoTokenWarning',
             'SuppressTelemetryReminder',
+            'TestConfigSettingsHash',
             'WebRequestTimeoutSec')]
         [string] $Name
     )
@@ -481,6 +482,9 @@ function Reset-GitHubConfiguration
         Reset-GitHubConfiguration
 
         Deletes the local configuration file and loads in all default configration values.
+
+    .NOTES
+        This command will not clear your authentication token.  Please use Clear-GitHubAuthentication to accomplish that.
 #>
     [CmdletBinding(SupportsShouldProcess)]
     param(
@@ -504,7 +508,7 @@ function Reset-GitHubConfiguration
 
     Initialize-GitHubConfiguration
 
-    Write-Log -Message "This has not cleared your authentication token.  Call Clear-GitHubAuthentication to accomplish that." -Level Warning
+    Write-Log -Message "This has not cleared your authentication token.  Call Clear-GitHubAuthentication to accomplish that." -Level Verbose
 }
 
 function Read-GitHubConfiguration
@@ -622,6 +626,13 @@ function Import-GitHubConfiguration
         'suppressNoTokenWarning' = $false
         'suppressTelemetryReminder' = $false
         'webRequestTimeoutSec' = 0
+
+        # This hash is generated by using Helper.ps1's Get-Sha512Hash in Tests/Config/Settings.ps1 like so:
+        #    . ./Helpers.ps1; Get-Sha512Hash -PlainText (Get-Content -Path ./Tests/Config/Settings.ps1 -Raw -Encoding Utf8)
+        # The hash is used to identify if the user has made changes to the config file prior to running the UT's locally.
+        # It intentionally cannot be modified via Set-GitHubConfiguration and must be updated directly in the
+        # source code here should the default Settings.ps1 file ever be changed.
+        'testConfigSettingsHash' = 'A76CA42A587D10247F887F9257DB7BF5F988E8714A7C0E29D7B100A20F5D35B8E3306AC5B9BBC8851EC19846A90BB3C80FC7C594D0347A772B2B10BADB1B3E68'
     }
 
     $jsonObject = Read-GitHubConfiguration -Path $Path
@@ -909,6 +920,9 @@ function Clear-GitHubAuthentication
         Clear-GitHubAuthentication
 
         Clears out any GitHub API token from memory, as well as from local file storage.
+
+    .NOTES
+        This command will not clear your configuration settings.  Please use Reset-GitHubConfiguration to accomplish that.
 #>
     [CmdletBinding(SupportsShouldProcess)]
     param(
@@ -937,7 +951,7 @@ function Clear-GitHubAuthentication
         }
     }
 
-    Write-Log -Message "This has not cleared your configuration settings.  Call Reset-GitHubConfiguration to accomplish that." -Level Warning
+    Write-Log -Message "This has not cleared your configuration settings.  Call Reset-GitHubConfiguration to accomplish that." -Level Verbose
 }
 
 function Get-AccessToken

Tests/Common.ps1

@@ -33,18 +33,26 @@ function Initialize-CommonTestSetup
     param()
 
     $moduleRootPath = Split-Path -Path $PSScriptRoot -Parent
-    . (Join-Path -Path $moduleRootPath -ChildPath 'Tests\Config\Settings.ps1')
+    $settingsPath = Join-Path -Path $moduleRootPath -ChildPath 'Tests/Config/Settings.ps1'
+    . $settingsPath
     Import-Module -Name (Join-Path -Path $moduleRootPath -ChildPath 'PowerShellForGitHub.psd1') -Force
 
+    $originalSettingsHash = (Get-GitHubConfiguration -Name TestConfigSettingsHash)
+    $currentSettingsHash = Get-SHA512Hash -PlainText (Get-Content -Path $settingsPath -Raw -Encoding Utf8)
+    $settingsAreUnaltered = $originalSettingsHash -eq $currentSettingsHash
+
     if ([string]::IsNullOrEmpty($env:ciAccessToken))
     {
-        $message = @(
-            'The tests are using the configuration settings defined in Tests\Config\Settings.ps1.',
-            'If you haven''t locally modified those values, your tests are going to fail since you',
-            'don''t have access to the default accounts referenced.  If that is the case, you should',
-            'cancel the existing tests, modify the values to ones you have access to, call',
-            'Set-GitHubAuthentication to cache your AccessToken, and then try running the tests again.')
-        Write-Warning -Message ($message -join [Environment]::NewLine)
+        if ($settingsAreUnaltered)
+        {
+            $message = @(
+                'The tests are using the configuration settings defined in Tests/Config/Settings.ps1.',
+                'If you haven''t locally modified those values, your tests are going to fail since you',
+                'don''t have access to the default accounts referenced.  If that is the case, you should',
+                'cancel the existing tests, modify the values to ones you have access to, call',
+                'Set-GitHubAuthentication to cache your AccessToken, and then try running the tests again.')
+            Write-Warning -Message ($message -join [Environment]::NewLine)
+        }
     }
     else
     {

Tests/Config/Settings.ps1

@@ -1,6 +1,20 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
+<#
+    .NOTES
+    A hash of this file is stored in GitHubConfiguration.ps1 within the Import-GitHubConfiguration function
+    as TestConfigSettingsHash (and can be retrieved by calling Get-GitHubConfiguration -Name TestConfigSettingsHash).
+    It is used when trying to detect if the file has been updated with personal settings before unit tests
+    are run locally (since unit tests will otherwise fail because users do not have account permissions to
+    access the accounts below).
+
+    If this file is being modified as part of an intended change to the official repo, please be sure to
+    additionally update the value of TestConfigSettingsHash in GitHubConfiguration.ps1 by running:
+
+        . ./Helpers.ps1; Get-Sha512Hash -PlainText (Get-Content -Path ./Tests/Config/Settings.ps1 -Raw -Encoding Utf8)
+#>
+
 # The account that the tests will be running under
 $script:ownerName = 'PowerShellForGitHubTeam'
 

build/pipelines/azure-pipelines.ci.yaml

@@ -23,19 +23,22 @@ jobs:
   pool:
     vmImage: 'vs2017-win2016'
   steps:
+  - template: ./templates/verify-testConfigSettingsHash.yaml
   - template: ./templates/run-staticAnalysis.yaml
   - template: ./templates/run-unitTests.yaml
 
 - job: Linux
   pool:
     vmImage: 'ubuntu-16.04'
   steps:
+  - template: ./templates/verify-testConfigSettingsHash.yaml
   - template: ./templates/run-staticAnalysis.yaml
   - template: ./templates/run-unitTests.yaml
 
 - job: macOS
   pool:
     vmImage: 'macOS-10.13'
   steps:
+  - template: ./templates/verify-testConfigSettingsHash.yaml
   - template: ./templates/run-staticAnalysis.yaml
   - template: ./templates/run-unitTests.yaml

build/pipelines/templates/verify-testConfigSettingsHash.yaml

@@ -0,0 +1,38 @@
+#
+# This template contains a script which verifies if the $testConfigSettingsHash (GitHubConfiguration.ps1)
+# value matches the Get-SHA512Hash value of ./Tests/Config/Settings.ps1. Both hash values need to be
+# the same in order to ensure the correct warning messages are presented to developers running UTs.
+#
+
+steps:
+  - powershell: |
+      . ./Helpers.ps1
+      $content = Get-Content -Path ./Tests/Config/Settings.ps1 -Raw -Encoding UTF8
+      $hash = Get-SHA512Hash -PlainText $content
+      $configurationFile = Get-Content -Path ./GitHubConfiguration.ps1 -Raw -Encoding Utf8
+      if($configurationFile -match "'testConfigSettingsHash' = '([^']+)'")
+      {
+        if($hash -ne $Matches[1])
+        {
+          $configHash = $Matches[1]
+          $message = @(
+            "`$testConfigSettingsHash value does not match the Get-SHA512Hash of file ./Tests/Config/Settings.ps1. If the contents of ",
+            "Settings.ps1 has been updated, please update the `$testConfigSettingsHash value in ./GitHubConfiguration.ps1's ",
+            "Import-GitHubConfiguration function. You can generate the new hash value using the commands:",
+            "",
+            "`t. ./Helpers.ps1",
+            "`tGet-SHA512Hash -PlainText (Get-Content -Path ./Tests/Config/Settings.ps1 -Raw -Encoding Utf8)",
+            "",
+            "`$testConfigSettingsHash = $configHash",
+            "Get-SHA512Hash(Settings.ps1) = $hash")
+          throw ($message -join [Environment]::NewLine)
+        }
+      } else
+      {
+        $message = @(
+          "`$testConfigSettingsHash value not found in ./GitHubConfiguration.ps1. Please ensure ",
+          "the default hash value is set within the Import-GitHubConfiguration function.")
+        throw ($message -join [Environment]::NewLine)
+      }
+    displayName: 'Set GitHubConfiguration.ps1 testConfigSettingsHash value.' 
+