osx: dynamically skip keychain test if invalid

Dynamically skip the macOS Keychain read/write/delete test if the
Keychain is in a "strange" state.

There is an unknown issue that the keychain can sometimes get itself
in where all API calls result in an errSecAuthFailed error. The only
solution seems to be a machine restart; not possible in CI!

The problem has plagued others who are calling the same Keychain APIs
from C# such as the MSAL.NET team - they don't know either. It might
have something to do with the code signing signature of the binary
(our collective best theory).

It's probably only diagnosable at this point by Apple, but we don't
have a reliable way to reproduce the problem.

Author: mjcheetham

src/shared/Microsoft.Git.CredentialManager.Tests/Interop/MacOS/MacOSKeychainTests.cs

@@ -2,15 +2,17 @@
 // Licensed under the MIT license.
 using System;
 using Xunit;
+using Microsoft.Git.CredentialManager.Interop;
 using Microsoft.Git.CredentialManager.Interop.MacOS;
+using Microsoft.Git.CredentialManager.Interop.MacOS.Native;
 
 namespace Microsoft.Git.CredentialManager.Tests.Interop.MacOS
 {
     public class MacOSKeychainTests
     {
         private const string TestNamespace = "git-test";
 
-        [PlatformFact(Platforms.MacOS)]
+        [SkippablePlatformFact(Platforms.MacOS)]
         public void MacOSKeychain_ReadWriteDelete()
         {
             var keychain = new MacOSKeychain(TestNamespace);
@@ -32,6 +34,19 @@ public void MacOSKeychain_ReadWriteDelete()
                 Assert.Equal(account, outCredential.Account);
                 Assert.Equal(password, outCredential.Password);
             }
+            // There is an unknown issue that the keychain can sometimes get itself in where all API calls
+            // result in an errSecAuthFailed error. The only solution seems to be a machine restart, which
+            // isn't really possible in CI!
+            // The problem has plagued others who are calling the same Keychain APIs from C# such as the
+            // MSAL.NET team - they don't know either. It might have something to do with the code signing
+            // signature of the binary (our collective best theory).
+            // It's probably only diagnosable at this point by Apple, but we don't have a reliable way to
+            // reproduce the problem.
+            // For now we will just mark the test as "skipped" when we hit this problem.
+            catch (InteropException iex) when (iex.ErrorCode == SecurityFramework.ErrorSecAuthFailed)
+            {
+                AssertEx.Skip("macOS Keychain is in an invalid state (errSecAuthFailed)");
+            }
             finally
             {
                 // Ensure we clean up after ourselves even in case of 'get' failures

src/shared/TestInfrastructure/AssertEx.cs

@@ -0,0 +1,17 @@
+using Xunit;
+
+namespace Microsoft.Git.CredentialManager.Tests
+{
+    public static class AssertEx
+    {
+        /// <summary>
+        /// Requires the fact or theory be marked with the <see cref="SkippableFactAttribute"/>
+        /// or <see cref="SkippableTheoryAttribute"/>.
+        /// </summary>
+        /// <param name="reason">Reason the test has been skipped.</param>
+        public static void Skip(string reason)
+        {
+            Xunit.Skip.If(true, reason);
+        }
+    }
+}

src/shared/TestInfrastructure/PlatformAttributes.cs

@@ -28,6 +28,28 @@ public PlatformTheoryAttribute(Platforms platforms)
         }
     }
 
+    public class SkippablePlatformFactAttribute : SkippableFactAttribute
+    {
+        public SkippablePlatformFactAttribute(Platforms platforms)
+        {
+            Xunit.Skip.IfNot(
+                XunitHelpers.IsSupportedPlatform(platforms),
+                "Test not supported on this platform."
+            );
+        }
+    }
+
+    public class SkippablePlatformTheoryAttribute : SkippableTheoryAttribute
+    {
+        public SkippablePlatformTheoryAttribute(Platforms platforms)
+        {
+            Xunit.Skip.IfNot(
+                XunitHelpers.IsSupportedPlatform(platforms),
+                "Test not supported on this platform."
+            );
+        }
+    }
+
     internal static class XunitHelpers
     {
         public static bool IsSupportedPlatform(Platforms platforms)

src/shared/TestInfrastructure/TestInfrastructure.csproj

@@ -11,6 +11,7 @@
   <ItemGroup>
     <PackageReference Include="Moq" Version="4.10.1" />
     <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="Xunit.SkippableFact" Version="1.4.13" />
   </ItemGroup>
 
   <ItemGroup>