diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckData.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckData.cs new file mode 100644 index 0000000000..3c9fb1663b --- /dev/null +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckData.cs @@ -0,0 +1,142 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using Azure; + +namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks +{ + /// + /// A helper for providing data with a health check result. + /// + internal partial class HealthCheckData + { + // exposed to the HealthCheckResult through IReadOnlyDictionary. + private readonly Dictionary _data = []; + + /// + /// Gets or sets the area of the health check data failure. + /// + /// + /// This is the area that has failed. Such as "configuration", "connectivity", etc. + /// + public string Area + { + get => GetOrDefault(); + set => Set(value); + } + + /// + /// Gets or sets the configuration section related to the health check data. + /// + /// + /// Useful for when the component being checked is related to a specific configuration section. + /// + public string ConfigurationSection + { + get => GetOrDefault(); + set => Set(value); + } + + /// + /// Gets or sets the status code related to the health check data. + /// For HTTP related related checks, this is the HTTP status code. + /// + public int StatusCode + { + get => GetOrDefault(); + set => Set(value); + } + + /// + /// Gets or sets the error code related to the health check data. + /// + /// + /// For Azure SDK related checks, this is typically the RequestFailedException.ErrorCode value. + /// + public string ErrorCode + { + get => GetOrDefault(); + set => Set(value); + } + + /// + /// Sets exception details into the health check data. + /// + /// The exception to set details from. + /// + /// This will set various properties based on the type of exception. + /// + public void SetExceptionDetails(Exception ex) + { + ArgumentNullException.ThrowIfNull(ex); + if (ex is AggregateException aggregate) + { + // Azure SDK will retry a few times in some cases, leading to multiple inner exceptions. + // We only care about the last one. + ex = aggregate.InnerExceptions.Last(); + } + + if (ex is TimeoutException) + { + ErrorCode = "Timeout"; + } + else if (ex is OperationCanceledException) + { + ErrorCode = "OperationCanceled"; + } + else if (ex is RequestFailedException rfe) + { + StatusCode = rfe.Status; + ErrorCode = rfe.ErrorCode; + } + } + + private void Set(T value, [CallerMemberName] string key = null) + { + _data[key] = value; + } + + private T GetOrDefault([CallerMemberName] string key = null, T defaultValue = default) + { + if (_data.TryGetValue(key, out var value) && value is T typedValue) + { + return typedValue; + } + + return defaultValue; + } + } + + // Partial class down here to separate IReadOnlyDictionary implementation details. + internal partial class HealthCheckData : IReadOnlyDictionary + { + IEnumerable IReadOnlyDictionary.Keys + => _data.Keys; + + IEnumerable IReadOnlyDictionary.Values + => _data.Values; + + int IReadOnlyCollection>.Count + => _data.Count; + + object IReadOnlyDictionary.this[string key] + => _data[key]; + + bool IReadOnlyDictionary.ContainsKey(string key) + => _data.ContainsKey(key); + + IEnumerator> IEnumerable>.GetEnumerator() + => _data.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() + => _data.GetEnumerator(); + + bool IReadOnlyDictionary.TryGetValue(string key, out object value) + => _data.TryGetValue(key, out value); + } +} diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs index ea051c7bc6..42b95cd08c 100644 --- a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs @@ -1,10 +1,11 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. using System; using System.Collections.Generic; using System.Linq; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; @@ -27,6 +28,7 @@ public static IHealthChecksBuilder AddWebJobsScriptHealthChecks(this IHealthChec builder .AddWebHostHealthCheck() .AddScriptHostHealthCheck() + .AddWebJobsStorageHealthCheck() .AddTelemetryPublisher(HealthCheckTags.Liveness, HealthCheckTags.Readiness) .UseDynamicHealthCheckService(); return builder; @@ -126,6 +128,24 @@ public static IHealthChecksBuilder AddScriptHostHealthCheck(this IHealthChecksBu return builder; } + /// + /// Adds a health check for the WebJobs storage account. + /// + /// The builder to register health checks with. + /// The original builder, for call chaining. + public static IHealthChecksBuilder AddWebJobsStorageHealthCheck(this IHealthChecksBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + // Ensure singleton as this health check refreshes in the background. + builder.Services.TryAddSingleton(); + builder.AddCheck( + HealthCheckNames.WebJobsStorage, + tags: [HealthCheckTags.Configuration, HealthCheckTags.Connectivity, HealthCheckTags.WebJobsStorage], + timeout: TimeSpan.FromSeconds(10)); + return builder; + } + /// /// Filters a health report to include only specified entries. /// diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckNames.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckNames.cs index 2410594cc7..180c4fb38b 100644 --- a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckNames.cs +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckNames.cs @@ -19,5 +19,10 @@ internal static class HealthCheckNames /// The 'azure.functions.script_host.lifecycle' check monitors the lifecycle of the script host. /// public const string ScriptHostLifeCycle = Prefix + "script_host.lifecycle"; + + /// + /// The 'azure.functions.web_jobs.storage' check monitors connectivity to the WebJobs storage account. + /// + public const string WebJobsStorage = Prefix + "web_jobs.storage"; } } diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs index 5fe097b69b..2f3bfdadb5 100644 --- a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks @@ -8,7 +8,8 @@ namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks /// internal static class HealthCheckTags { - private const string Prefix = "azure.functions."; + private const string FuncPrefix = "azure.functions."; + private const string WebJobsPrefix = FuncPrefix + "web_jobs."; /// /// The 'azure.functions.liveness' tag is used for liveness checks in the Azure Functions host. @@ -16,7 +17,7 @@ internal static class HealthCheckTags /// /// Liveness checks are used to determine if the host is alive and responsive. /// - public const string Liveness = Prefix + "liveness"; + public const string Liveness = FuncPrefix + "liveness"; /// /// The 'azure.functions.readiness' tag is used for readiness checks in the Azure Functions host. @@ -24,7 +25,7 @@ internal static class HealthCheckTags /// /// Readiness checks are used to determine if the host is ready to process requests. /// - public const string Readiness = Prefix + "readiness"; + public const string Readiness = FuncPrefix + "readiness"; /// /// The 'azure.functions.configuration' tag is used for configuration-related health checks in the Azure Functions host. @@ -32,6 +33,19 @@ internal static class HealthCheckTags /// /// These are typically customer configuration related, such as configuring AzureWebJobsStorage access. /// - public const string Configuration = Prefix + "configuration"; + public const string Configuration = FuncPrefix + "configuration"; + + /// + /// The 'azure.functions.connectivity' tag is used for connectivity-related health checks in the Azure Functions host. + /// + /// + /// These are typically related to connectivity to external services, such as Azure Storage. + /// + public const string Connectivity = FuncPrefix + "connectivity"; + + /// + /// The "azure.functions.web_jobs.storage" tag is used for health checks related to the WebJobs storage account. + /// + public const string WebJobsStorage = WebJobsPrefix + "storage"; } } diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs new file mode 100644 index 0000000000..7017ab747c --- /dev/null +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs @@ -0,0 +1,187 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.Blobs; +using Microsoft.Azure.WebJobs.Host.Storage; +using Microsoft.Azure.WebJobs.Script.Extensions; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks +{ + /// + /// Health check for the AzureWebJobsStorage used by WebJobs. + /// + /// The blob storage provider. + /// + /// Checking connectivity to Azure Storage can be expensive and time consuming - especially + /// waiting for a connection timeout in the event of no connectivity. To speed up health checks + /// this class will periodically refresh the health check in the background. On + /// , this class will wait for + /// the first result. After that it will keep returning the most recent result of the background + /// refresh. + /// + internal class WebJobsStorageHealthCheck : IHealthCheck, IAsyncDisposable + { + private const string ConfigSection = "AzureWebJobsStorage"; + + // Refresh in the background every 30 seconds by default. + private static readonly TimeSpan DefaultPeriod = TimeSpan.FromSeconds(30); + private readonly CancellationTokenSource _cts = new(); + private readonly Lazy _initialized; + private readonly IEnvironment _environment; + private readonly IAzureBlobStorageProvider _provider; + + private HealthCheckResult _last; + private HealthCheckContext _context; + private BlobServiceClient _client; + + /// + /// Initializes a new instance of the class. + /// + /// The blob storage provider. + /// The environment. + public WebJobsStorageHealthCheck( + IAzureBlobStorageProvider provider, IEnvironment environment) + { + ArgumentNullException.ThrowIfNull(provider); + ArgumentNullException.ThrowIfNull(environment); + _provider = provider; + _environment = environment; + _initialized = new(RunInBackgroundAsync); + } + + public async ValueTask DisposeAsync() + { + await _cts.CancelNoThrowAsync().ConfigureAwait(false); + _cts.Dispose(); + } + + public async Task CheckHealthAsync( + HealthCheckContext context, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(context); + + // TODO: we don't have access to StandbyOptions in this assembly. + // Switch to using that when/if it ever moves here. It isn't worth moving to the other assembly + // just to get access to that type, since technically IEnvironment is sufficient here. + // TODO: a better approach would be to dynamically register only when specialization completes. + // However, that is not supported with the current health check infrastructure -- it captures + // the set of health checks at startup, and does not allow dynamic changes. We can revisit that later. + if (_environment.IsPlaceholderModeEnabled()) + { + // This health check runs in the WebHost context, as misconfigured AzureWebJobsStorage may prevent + // the ScriptHost from starting. So we want to ensure this check runs independent of ScriptHost + // starting. But we also want to avoid false negatives during placeholder mode. + return HealthCheckResult.Healthy("Placeholder mode. Check skipped."); + } + + _context = context; + await _initialized.Value.WaitAsync(cancellationToken).ConfigureAwait(false); + return _last; + } + + private async Task CheckHealthCoreAsync(CancellationToken cancellation) + { + if (!TryGetClient(out BlobServiceClient client, out HealthCheckResult result)) + { + return result; + } + + try + { + // Right now we only check if we can access blobs. The functions host doesn't use queues or tables directly (although extensions may). + // We use this API to check connectivity to Blob storage. We don't check permissions/role assignments in depth for a couple reasons: + // 1. It is expensive + // 2. Permissions/role assignments needed by the host can change over time. + // So we settle for just connectivity here. Insufficient permissions will show up as errors elsewhere. + // See https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-app?tabs=dotnet#configure-permissions-for-access-to-blob-and-queue-data + await client + .GetBlobContainersAsync(cancellationToken: cancellation) + .AsPages(pageSizeHint: 1) + .GetAsyncEnumerator(cancellation) + .MoveNextAsync() + .ConfigureAwait(false); + + return HealthCheckResult.Healthy(); + } + catch (Exception ex) when (!ex.IsFatal()) + { + HealthCheckData data = GetData(ex, "connectivity"); + return HealthCheckResult.Unhealthy($"Unable to access {ConfigSection}", ex, data); + } + } + + private async Task RunInBackgroundAsync() + { + // Ensure we have at least one result to return right away. + CancellationToken cancellation = _cts.Token; + _last = await CheckHealthCoreAsync(cancellation).ConfigureAwait(false); + + // Kick off background refresh. + Task.Run( + async () => + { + while (!cancellation.IsCancellationRequested) + { + TimeSpan delay = _context?.Registration?.Period ?? DefaultPeriod; + await Task.Delay(delay, cancellation); + _last = await CheckHealthCoreAsync(cancellation).ConfigureAwait(false); + } + }) + .Forget(); + } + + private static BlobServiceClient GetClient(IAzureBlobStorageProvider provider) + { + if (provider is HostAzureBlobStorageProvider hostProvider) + { + // Avoid TryCreate* to capture the original exception on failures. + return hostProvider.CreateBlobServiceClient(ConnectionStringNames.Storage); + } + + if (provider.TryCreateBlobServiceClientFromConnection(ConnectionStringNames.Storage, out BlobServiceClient blobServiceClient)) + { + return blobServiceClient; + } + + // TODO: need a better exception type. + throw new InvalidOperationException("Failed to create BlobServiceClient."); + } + + private bool TryGetClient(out BlobServiceClient client, out HealthCheckResult result) + { + try + { + // Only cache the client on success. On failure, we want to re-fetch the client next time + // in case configuration has changed. This is to be defensive in-case config is still updating + // from specialization. + _client ??= GetClient(_provider); + client = _client; + result = HealthCheckResult.Healthy(); + return true; + } + catch (Exception ex) when (!ex.IsFatal()) + { + client = null; + HealthCheckData data = GetData(ex, "configuration"); + result = HealthCheckResult.Unhealthy($"Unable to create client for {ConfigSection}", ex, data); + return false; + } + } + + private static HealthCheckData GetData(Exception ex, string area) + { + HealthCheckData data = new() + { + Area = area, + ConfigurationSection = ConfigSection, + }; + + data.SetExceptionDetails(ex); + return data; + } + } +} diff --git a/src/WebJobs.Script/StorageProvider/HostAzureBlobStorageProvider.cs b/src/WebJobs.Script/StorageProvider/HostAzureBlobStorageProvider.cs index 2794241dad..2411160652 100644 --- a/src/WebJobs.Script/StorageProvider/HostAzureBlobStorageProvider.cs +++ b/src/WebJobs.Script/StorageProvider/HostAzureBlobStorageProvider.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System; @@ -78,11 +78,9 @@ public virtual bool TryCreateHostingBlobContainerClient(out BlobContainerClient public virtual bool TryCreateBlobServiceClientFromConnection(string connection, out BlobServiceClient client) { - var connectionToUse = connection ?? ConnectionStringNames.Storage; - try { - client = _blobServiceClientProvider.Create(connectionToUse, Configuration); + client = CreateBlobServiceClient(connection); return true; } catch (Exception e) @@ -92,5 +90,15 @@ public virtual bool TryCreateBlobServiceClientFromConnection(string connection, return false; } } + + public virtual BlobServiceClient CreateBlobServiceClient(string connection) + { + if (string.IsNullOrWhiteSpace(connection)) + { + connection = ConnectionStringNames.Storage; + } + + return _blobServiceClientProvider.Create(connection, Configuration); + } } } diff --git a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs index 9b369537f6..6b475a22f4 100644 --- a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs +++ b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs @@ -71,6 +71,9 @@ public void AddWebJobsScriptHealthChecks_RegistersExpectedServices() builder.Verify(b => b.Add(IsRegistration( HealthCheckNames.ScriptHostLifeCycle, HealthCheckTags.Readiness)), Times.Once); + builder.Verify(b => b.Add(IsRegistration( + HealthCheckNames.WebJobsStorage, HealthCheckTags.Configuration)), + Times.Once); builder.Verify(b => b.Services, Times.AtLeastOnce); builder.VerifyNoOtherCalls(); @@ -118,6 +121,34 @@ public void AddScriptHostHealthCheck_RegistersScriptHostHealthCheck() builder.VerifyNoOtherCalls(); } + [Fact] + public void AddWebJobsStorageHealthCheck_RegistersWebJobsStorageHealthCheck() + { + // arrange + ServiceCollection services = new(); + Mock builder = new(MockBehavior.Strict); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Add(It.IsAny())).Returns(builder.Object); + + // act + IHealthChecksBuilder returned = builder.Object.AddWebJobsStorageHealthCheck(); + + // assert + returned.Should().BeSameAs(builder.Object); + builder.Verify(b => b.Add(IsRegistration( + HealthCheckNames.WebJobsStorage, HealthCheckTags.Configuration, HealthCheckTags.Connectivity, HealthCheckTags.WebJobsStorage)), + Times.Once); + builder.Verify(b => b.Services, Times.AtLeastOnce); + builder.VerifyNoOtherCalls(); + services.Should().ContainSingle() + .Which.Should().Satisfy(sd => + { + sd.Lifetime.Should().Be(ServiceLifetime.Singleton); + sd.ServiceType.Should().Be(); + sd.ImplementationType.Should().Be(); + }); + } + [Fact] public void Filter_ReturnsFilteredHealthReport() { @@ -258,7 +289,7 @@ public void UseDynamicHealthCheckService_AddHealthChecksCalled_ReplacesHealthChe VerifyDynamicHealthCheckService(services); } - private static HealthCheckRegistration IsRegistration(string name, string tag) + private static HealthCheckRegistration IsRegistration(string name, params string[] tags) where T : IHealthCheck { static bool IsType(HealthCheckRegistration registration) @@ -273,7 +304,7 @@ static bool IsType(HealthCheckRegistration registration) return Match.Create(r => { - return r.Name == name && r.Tags.Contains(tag) && IsType(r); + return r.Name == name && tags.All(t => r.Tags.Contains(t)) && IsType(r); }); } diff --git a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/WebJobsStorageHealthCheckTests.cs b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/WebJobsStorageHealthCheckTests.cs new file mode 100644 index 0000000000..455779f663 --- /dev/null +++ b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/WebJobsStorageHealthCheckTests.cs @@ -0,0 +1,194 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using AwesomeAssertions; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Microsoft.Azure.WebJobs.Host.Storage; +using Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Moq; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Script.Tests.Diagnostics.HealthChecks +{ + public class WebJobsStorageHealthCheckTests + { + private readonly Mock _provider = new(); + private readonly TestEnvironment _environment = new(); + private readonly Mock _mockBlobServiceClient = new(); + private readonly HealthCheckContext _healthCheckContext = new(); + + [Fact] + public void Constructor_WithNullProvider_ThrowsArgumentNullException() + { + // Act & Assert + TestHelpers.Act(() => new WebJobsStorageHealthCheck(null, _environment)) + .Should().Throw() + .WithParameterName("provider"); + } + + [Fact] + public void Constructor_WithNullEnvironment_ThrowsArgumentNullException() + { + // Act & Assert + TestHelpers.Act(() => new WebJobsStorageHealthCheck(_provider.Object, null)) + .Should().Throw() + .WithParameterName("environment"); + } + + [Fact] + public async Task CheckHealthAsync_WithNullContext_ThrowsArgumentNullException() + { + // Arrange + WebJobsStorageHealthCheck healthCheck = new(_provider.Object, _environment); + + // Act & Assert + await TestHelpers.Act(async () => await healthCheck.CheckHealthAsync(null, default)) + .Should().ThrowAsync() + .WithParameterName("context"); + } + + [Fact] + public async Task CheckHealthAsync_InPlaceholderMode_ReturnsHealthyWithSkippedMessage() + { + // Arrange + _environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1"); + WebJobsStorageHealthCheck healthCheck = new(_provider.Object, _environment); + + // Act + HealthCheckResult result = await healthCheck.CheckHealthAsync(_healthCheckContext, default); + + // Assert + result.Status.Should().Be(HealthStatus.Healthy); + result.Description.Should().Be("Placeholder mode. Check skipped."); + result.Exception.Should().BeNull(); + } + + [Fact] + public async Task CheckHealthAsync_ChecksBlobConnectivity() + { + // Arrange + Page page = Page.FromValues([], null, Mock.Of()); + AsyncPageable pageable = AsyncPageable.FromPages([page]); + SetupGetContainers(pageable); + BlobServiceClient client = _mockBlobServiceClient.Object; + _provider.Setup(p => p.TryCreateBlobServiceClientFromConnection( + ConnectionStringNames.Storage, out client)).Returns(true); + + WebJobsStorageHealthCheck healthCheck = new(_provider.Object, _environment); + + // Act + HealthCheckResult result = await healthCheck.CheckHealthAsync(_healthCheckContext, default); + + // Assert + VerifyGetContainersCalled(Times.Once()); + result.Status.Should().Be(HealthStatus.Healthy); + result.Exception.Should().BeNull(); + } + + [Fact] + public async Task CheckHealthAsync_Twice_ReturnsCachedResult() + { + // Arrange + Page page = Page.FromValues([], null, Mock.Of()); + AsyncPageable pageable = AsyncPageable.FromPages([page]); + SetupGetContainers(pageable); + BlobServiceClient client = _mockBlobServiceClient.Object; + _provider.Setup(p => p.TryCreateBlobServiceClientFromConnection( + ConnectionStringNames.Storage, out client)).Returns(true); + + WebJobsStorageHealthCheck healthCheck = new(_provider.Object, _environment); + + // Act + HealthCheckContext context = new() + { + Registration = new HealthCheckRegistration("test", healthCheck, null, null) + { + Period = Timeout.InfiniteTimeSpan, + }, + }; + HealthCheckResult result1 = await healthCheck.CheckHealthAsync(context, default); + HealthCheckResult result2 = await healthCheck.CheckHealthAsync(context, default); + + // Assert + VerifyGetContainersCalled(Times.Once()); + result1.Status.Should().Be(HealthStatus.Healthy); + result1.Exception.Should().BeNull(); + result2.Status.Should().Be(HealthStatus.Healthy); + result2.Exception.Should().BeNull(); + } + + [Fact] + public async Task CheckHealthAsync_GetClientFails_Unhealthy() + { + // Arrange + InvalidOperationException ex = new("Failed to create BlobServiceClient."); + BlobServiceClient client = null; + _provider.Setup(p => p.TryCreateBlobServiceClientFromConnection( + ConnectionStringNames.Storage, out client)).Throws(ex); + + WebJobsStorageHealthCheck healthCheck = new(_provider.Object, _environment); + + // Act + HealthCheckResult result = await healthCheck.CheckHealthAsync(_healthCheckContext, default); + + // Assert + result.Status.Should().Be(HealthStatus.Unhealthy); + result.Exception.Should().Be(ex); + result.Data.Should().Contain("Area", "configuration"); + result.Data.Should().Contain("ConfigurationSection", "AzureWebJobsStorage"); + } + + [Fact] + public async Task CheckHealthAsync_WithException_Unhealthy() + { + // Arrange + RequestFailedException rfe = new( + 401, "Some exception message", "SomeErrorCode", null); + SetupGetContainers(rfe); + BlobServiceClient client = _mockBlobServiceClient.Object; + _provider.Setup(p => p.TryCreateBlobServiceClientFromConnection( + ConnectionStringNames.Storage, out client)).Returns(true); + + WebJobsStorageHealthCheck healthCheck = new(_provider.Object, _environment); + + // Act + HealthCheckResult result = await healthCheck.CheckHealthAsync(_healthCheckContext, default); + + // Assert + VerifyGetContainersCalled(Times.Once()); + result.Status.Should().Be(HealthStatus.Unhealthy); + result.Exception.Should().Be(rfe); + result.Description.Should().Be("Unable to access AzureWebJobsStorage"); + result.Data.Should().Contain("Area", "connectivity"); + result.Data.Should().Contain("ConfigurationSection", "AzureWebJobsStorage"); + result.Data.Should().Contain("StatusCode", 401); + result.Data.Should().Contain("ErrorCode", "SomeErrorCode"); + } + + private void SetupGetContainers(AsyncPageable pageable) + { + _mockBlobServiceClient.Setup(c => c.GetBlobContainersAsync( + BlobContainerTraits.None, BlobContainerStates.None, null, It.IsAny())) + .Returns(pageable); + } + + private void SetupGetContainers(RequestFailedException ex) + { + _mockBlobServiceClient.Setup(c => c.GetBlobContainersAsync( + BlobContainerTraits.None, BlobContainerStates.None, null, It.IsAny())) + .Throws(ex); + } + + private void VerifyGetContainersCalled(Times times) + { + _mockBlobServiceClient.Verify(c => c.GetBlobContainersAsync( + BlobContainerTraits.None, BlobContainerStates.None, null, It.IsAny()), times); + } + } +} diff --git a/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj b/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj index 03c6f2e8b2..aab7d2c915 100644 --- a/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj +++ b/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj @@ -17,8 +17,8 @@ - - + +