From efafd99fd2c4993345bd88882cb796a9c3abf6cf Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 3 Nov 2025 15:35:01 -0800 Subject: [PATCH 1/6] Add AzureWebJobsStorage health check --- .../HealthChecks/HealthCheckData.cs | 113 ++++++++++ .../HealthChecks/HealthCheckExtensions.cs | 17 +- .../HealthChecks/HealthCheckNames.cs | 5 + .../HealthChecks/HealthCheckTags.cs | 16 +- .../HealthChecks/WebJobsStorageHealthCheck.cs | 187 +++++++++++++++++ .../HostAzureBlobStorageProvider.cs | 16 +- .../HealthCheckExtensionsTests.cs | 21 ++ .../WebJobsStorageHealthCheckTests.cs | 194 ++++++++++++++++++ 8 files changed, 559 insertions(+), 10 deletions(-) create mode 100644 src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckData.cs create mode 100644 src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs create mode 100644 test/WebJobs.Script.Tests/Diagnostics/HealthChecks/WebJobsStorageHealthCheckTests.cs diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckData.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckData.cs new file mode 100644 index 0000000000..6914849c3a --- /dev/null +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckData.cs @@ -0,0 +1,113 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using Azure; + +namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks +{ + /// + /// A helper for providing data with a health check result. + /// + internal partial class HealthCheckData + { + // exposed to the HealthCheckResult through IReadOnlyDictionary. + private readonly Dictionary _data = []; + + public string Source + { + get => GetOrDefault(); + set => Set(value); + } + + public string ConfigurationSection + { + get => GetOrDefault(); + set => Set(value); + } + + public int StatusCode + { + get => GetOrDefault(); + set => Set(value); + } + + public string ErrorCode + { + get => GetOrDefault(); + set => Set(value); + } + + public void SetExceptionDetails(Exception ex) + { + ArgumentNullException.ThrowIfNull(ex); + if (ex is AggregateException aggregate) + { + // Azure SDK will retry a few times in some cases, leading to multiple inner exceptions. + // We only care about the last one. + ex = aggregate.InnerExceptions.Last(); + } + + if (ex is TimeoutException) + { + ErrorCode = "Timeout"; + } + else if (ex is OperationCanceledException) + { + ErrorCode = "OperationCanceled"; + } + else if (ex is RequestFailedException rfe) + { + StatusCode = rfe.Status; + ErrorCode = rfe.ErrorCode; + } + } + + private void Set(T value, [CallerMemberName] string key = null) + { + _data[key] = value; + } + + private T GetOrDefault([CallerMemberName] string key = null, T defaultValue = default) + { + if (_data.TryGetValue(key, out var value) && value is T typedValue) + { + return typedValue; + } + + return defaultValue; + } + } + + // Partial class down here to separate IReadOnlyDictionary implementation details. + internal partial class HealthCheckData : IReadOnlyDictionary + { + IEnumerable IReadOnlyDictionary.Keys + => _data.Keys; + + IEnumerable IReadOnlyDictionary.Values + => _data.Values; + + int IReadOnlyCollection>.Count + => _data.Count; + + object IReadOnlyDictionary.this[string key] + => _data[key]; + + bool IReadOnlyDictionary.ContainsKey(string key) + => _data.ContainsKey(key); + + IEnumerator> IEnumerable>.GetEnumerator() + => _data.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() + => _data.GetEnumerator(); + + bool IReadOnlyDictionary.TryGetValue(string key, out object value) + => _data.TryGetValue(key, out value); + } +} diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs index ea051c7bc6..643970a4ff 100644 --- a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs @@ -1,10 +1,11 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. using System; using System.Collections.Generic; using System.Linq; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; @@ -27,6 +28,7 @@ public static IHealthChecksBuilder AddWebJobsScriptHealthChecks(this IHealthChec builder .AddWebHostHealthCheck() .AddScriptHostHealthCheck() + .AddWebJobsStorageHealthCheck() .AddTelemetryPublisher(HealthCheckTags.Liveness, HealthCheckTags.Readiness) .UseDynamicHealthCheckService(); return builder; @@ -126,6 +128,19 @@ public static IHealthChecksBuilder AddScriptHostHealthCheck(this IHealthChecksBu return builder; } + public static IHealthChecksBuilder AddWebJobsStorageHealthCheck(this IHealthChecksBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + // Ensure singleton as this health check refreshes in the background. + builder.Services.TryAddSingleton(); + builder.AddCheck( + HealthCheckNames.WebJobsStorage, + tags: [HealthCheckTags.Configuration, HealthCheckTags.WebJobsStorage], + timeout: TimeSpan.FromSeconds(10)); + return builder; + } + /// /// Filters a health report to include only specified entries. /// diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckNames.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckNames.cs index 2410594cc7..180c4fb38b 100644 --- a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckNames.cs +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckNames.cs @@ -19,5 +19,10 @@ internal static class HealthCheckNames /// The 'azure.functions.script_host.lifecycle' check monitors the lifecycle of the script host. /// public const string ScriptHostLifeCycle = Prefix + "script_host.lifecycle"; + + /// + /// The 'azure.functions.web_jobs.storage' check monitors connectivity to the WebJobs storage account. + /// + public const string WebJobsStorage = Prefix + "web_jobs.storage"; } } diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs index 5fe097b69b..0365044794 100644 --- a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks @@ -8,7 +8,8 @@ namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks /// internal static class HealthCheckTags { - private const string Prefix = "azure.functions."; + private const string FuncPrefix = "azure.functions."; + private const string WebJobsPrefix = "azure.web_jobs."; /// /// The 'azure.functions.liveness' tag is used for liveness checks in the Azure Functions host. @@ -16,7 +17,7 @@ internal static class HealthCheckTags /// /// Liveness checks are used to determine if the host is alive and responsive. /// - public const string Liveness = Prefix + "liveness"; + public const string Liveness = FuncPrefix + "liveness"; /// /// The 'azure.functions.readiness' tag is used for readiness checks in the Azure Functions host. @@ -24,7 +25,7 @@ internal static class HealthCheckTags /// /// Readiness checks are used to determine if the host is ready to process requests. /// - public const string Readiness = Prefix + "readiness"; + public const string Readiness = FuncPrefix + "readiness"; /// /// The 'azure.functions.configuration' tag is used for configuration-related health checks in the Azure Functions host. @@ -32,6 +33,11 @@ internal static class HealthCheckTags /// /// These are typically customer configuration related, such as configuring AzureWebJobsStorage access. /// - public const string Configuration = Prefix + "configuration"; + public const string Configuration = FuncPrefix + "configuration"; + + /// + /// The "azure.web_jobs.storage" tag is used for health checks related to the WebJobs storage account. + /// + public const string WebJobsStorage = WebJobsPrefix + "storage"; } } diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs new file mode 100644 index 0000000000..facaf4267d --- /dev/null +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs @@ -0,0 +1,187 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.Blobs; +using Microsoft.Azure.WebJobs.Host.Storage; +using Microsoft.Azure.WebJobs.Script.Extensions; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks +{ + /// + /// Health check for the AzureWebJobsStorage used by WebJobs. + /// + /// The blob storage provider. + /// + /// Checking connectivity to Azure Storage can be expensive and time consuming - especially + /// waiting for a connection timeout in the event of no connectivity. To speed up health checks + /// this class will periodically refresh the health check in the background. On + /// , this class will wait for + /// the first result. After that it will keep returning the most recent result of the background + /// refresh. + /// + internal class WebJobsStorageHealthCheck : IHealthCheck, IAsyncDisposable + { + private const string ConfigSection = "AzureWebJobsStorage"; + + // Refresh in the background every 30 seconds by default. + private static readonly TimeSpan DefaultPeriod = TimeSpan.FromSeconds(30); + private readonly CancellationTokenSource _cts = new(); + private readonly Lazy _initialized; + private readonly IEnvironment _environment; + private readonly IAzureBlobStorageProvider _provider; + + private HealthCheckResult _last; + private HealthCheckContext _context; + private BlobServiceClient _client; + + /// + /// Initializes a new instance of the class. + /// + /// The blob storage provider. + /// The environment. + public WebJobsStorageHealthCheck( + IAzureBlobStorageProvider provider, IEnvironment environment) + { + ArgumentNullException.ThrowIfNull(provider); + ArgumentNullException.ThrowIfNull(environment); + _provider = provider; + _environment = environment; + _initialized = new(RunInBackgroundAsync); + } + + public async ValueTask DisposeAsync() + { + await _cts.CancelNoThrowAsync().ConfigureAwait(false); + _cts.Dispose(); + } + + public async Task CheckHealthAsync( + HealthCheckContext context, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(context); + + // TODO: we don't have access to StandbyOptions in this assembly. + // Switch to using that when/if it ever moves here. It isn't worth moving to the other assembly + // just to get access to that type, since technically IEnvironment is sufficient here. + // TODO: a better approach would be to dynamically register only when specialization completes. + // However, that is not supported with the current health check infrastructure -- it captures + // the set of health checks at startup, and does not allow dynamic changes. We can revisit that later. + if (_environment.IsPlaceholderModeEnabled()) + { + // This health check runs in the WebHost context, as misconfigured AzureWebJobsStorage may prevent + // the ScriptHost from starting. So we want to ensure this check runs independent of ScriptHost + // starting. But we also want to avoid false negatives during placeholder mode. + return HealthCheckResult.Healthy("Placeholder mode. Check skipped."); + } + + _context = context; + await _initialized.Value.WaitAsync(cancellationToken).ConfigureAwait(false); + return _last; + } + + private async Task CheckHealthCoreAsync(CancellationToken cancellation) + { + if (!TryGetClient(out BlobServiceClient client, out HealthCheckResult result)) + { + return result; + } + + try + { + // Right now we only check if we can access blobs. The functions host doesn't use queues or tables directly (although extensions may). + // We use this API to check connectivity to Blob storage. We don't check permissions/role assignments in depth for a couple reasons: + // 1. It is expensive + // 2. Permissions/role assignments needed by the host can change over time. + // So we settle for just connectivity here. Insufficient permissions will show up as errors elsewhere. + // See https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-app?tabs=dotnet#configure-permissions-for-access-to-blob-and-queue-data + await client + .GetBlobContainersAsync(cancellationToken: cancellation) + .AsPages(pageSizeHint: 1) + .GetAsyncEnumerator(cancellation) + .MoveNextAsync() + .ConfigureAwait(false); + + return HealthCheckResult.Healthy(); + } + catch (Exception ex) + { + HealthCheckData data = GetData(ex, "connectivity"); + return HealthCheckResult.Unhealthy($"Unable to access {ConfigSection}", ex, data); + } + } + + private async Task RunInBackgroundAsync() + { + // Ensure we have at least one result to return right away. + CancellationToken cancellation = _cts.Token; + _last = await CheckHealthCoreAsync(cancellation).ConfigureAwait(false); + + // Kick off background refresh. + Task.Run( + async () => + { + while (!cancellation.IsCancellationRequested) + { + TimeSpan delay = _context?.Registration?.Period ?? DefaultPeriod; + await Task.Delay(delay); + _last = await CheckHealthCoreAsync(cancellation).ConfigureAwait(false); + } + }) + .Forget(); + } + + private static BlobServiceClient GetClient(IAzureBlobStorageProvider provider) + { + if (provider is HostAzureBlobStorageProvider hostProvider) + { + // Avoid TryCreate* to capture the original exception on failures. + return hostProvider.CreateBlobServiceClient(ConnectionStringNames.Storage); + } + + if (provider.TryCreateBlobServiceClientFromConnection(ConnectionStringNames.Storage, out BlobServiceClient blobServiceClient)) + { + return blobServiceClient; + } + + // TODO: need a better exception type. + throw new InvalidOperationException("Failed to create BlobServiceClient."); + } + + private bool TryGetClient(out BlobServiceClient client, out HealthCheckResult result) + { + try + { + // Only cache the client on success. On failure, we want to re-fetch the client next time + // in case configuration has changed. This is to be defensive in-case config is still updating + // from specialization. + _client ??= GetClient(_provider); + client = _client; + result = HealthCheckResult.Healthy(); + return true; + } + catch (Exception ex) + { + client = null; + HealthCheckData data = GetData(ex, "configuration"); + result = HealthCheckResult.Unhealthy($"Unable to create client for {ConfigSection}", ex, data); + return false; + } + } + + private static HealthCheckData GetData(Exception ex, string source) + { + HealthCheckData data = new() + { + Source = source, + ConfigurationSection = ConfigSection, + }; + + data.SetExceptionDetails(ex); + return data; + } + } +} diff --git a/src/WebJobs.Script/StorageProvider/HostAzureBlobStorageProvider.cs b/src/WebJobs.Script/StorageProvider/HostAzureBlobStorageProvider.cs index 2794241dad..2411160652 100644 --- a/src/WebJobs.Script/StorageProvider/HostAzureBlobStorageProvider.cs +++ b/src/WebJobs.Script/StorageProvider/HostAzureBlobStorageProvider.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System; @@ -78,11 +78,9 @@ public virtual bool TryCreateHostingBlobContainerClient(out BlobContainerClient public virtual bool TryCreateBlobServiceClientFromConnection(string connection, out BlobServiceClient client) { - var connectionToUse = connection ?? ConnectionStringNames.Storage; - try { - client = _blobServiceClientProvider.Create(connectionToUse, Configuration); + client = CreateBlobServiceClient(connection); return true; } catch (Exception e) @@ -92,5 +90,15 @@ public virtual bool TryCreateBlobServiceClientFromConnection(string connection, return false; } } + + public virtual BlobServiceClient CreateBlobServiceClient(string connection) + { + if (string.IsNullOrWhiteSpace(connection)) + { + connection = ConnectionStringNames.Storage; + } + + return _blobServiceClientProvider.Create(connection, Configuration); + } } } diff --git a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs index 9b369537f6..a9454b182b 100644 --- a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs +++ b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs @@ -71,6 +71,9 @@ public void AddWebJobsScriptHealthChecks_RegistersExpectedServices() builder.Verify(b => b.Add(IsRegistration( HealthCheckNames.ScriptHostLifeCycle, HealthCheckTags.Readiness)), Times.Once); + builder.Verify(b => b.Add(IsRegistration( + HealthCheckNames.WebJobsStorage, HealthCheckTags.Configuration)), + Times.Once); builder.Verify(b => b.Services, Times.AtLeastOnce); builder.VerifyNoOtherCalls(); @@ -118,6 +121,24 @@ public void AddScriptHostHealthCheck_RegistersScriptHostHealthCheck() builder.VerifyNoOtherCalls(); } + [Fact] + public void AdWebJobsStorageHealthCheck_RegistersWebJobsStorageHealthCheck() + { + // arrange + Mock builder = new(MockBehavior.Strict); + builder.Setup(b => b.Add(It.IsAny())).Returns(builder.Object); + + // act + IHealthChecksBuilder returned = builder.Object.AddWebJobsStorageHealthCheck(); + + // assert + returned.Should().BeSameAs(builder.Object); + builder.Verify(b => b.Add(IsRegistration( + HealthCheckNames.WebJobsStorage, HealthCheckTags.Configuration)), + Times.Once); + builder.VerifyNoOtherCalls(); + } + [Fact] public void Filter_ReturnsFilteredHealthReport() { diff --git a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/WebJobsStorageHealthCheckTests.cs b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/WebJobsStorageHealthCheckTests.cs new file mode 100644 index 0000000000..e32ab69e8c --- /dev/null +++ b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/WebJobsStorageHealthCheckTests.cs @@ -0,0 +1,194 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using AwesomeAssertions; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Microsoft.Azure.WebJobs.Host.Storage; +using Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Moq; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Script.Tests.Diagnostics.HealthChecks +{ + public class WebJobsStorageHealthCheckTests + { + private readonly Mock _provider = new(); + private readonly TestEnvironment _environment = new(); + private readonly Mock _mockBlobServiceClient = new(); + private readonly HealthCheckContext _healthCheckContext = new(); + + [Fact] + public void Constructor_WithNullProvider_ThrowsArgumentNullException() + { + // Act & Assert + TestHelpers.Act(() => new WebJobsStorageHealthCheck(null, _environment)) + .Should().Throw() + .WithParameterName("provider"); + } + + [Fact] + public void Constructor_WithNullEnvironment_ThrowsArgumentNullException() + { + // Act & Assert + TestHelpers.Act(() => new WebJobsStorageHealthCheck(_provider.Object, null)) + .Should().Throw() + .WithParameterName("environment"); + } + + [Fact] + public async Task CheckHealthAsync_WithNullContext_ThrowsArgumentNullException() + { + // Arrange + WebJobsStorageHealthCheck healthCheck = new(_provider.Object, _environment); + + // Act & Assert + await TestHelpers.Act(async () => await healthCheck.CheckHealthAsync(null, default)) + .Should().ThrowAsync() + .WithParameterName("context"); + } + + [Fact] + public async Task CheckHealthAsync_InPlaceholderMode_ReturnsHealthyWithSkippedMessage() + { + // Arrange + _environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1"); + WebJobsStorageHealthCheck healthCheck = new(_provider.Object, _environment); + + // Act + HealthCheckResult result = await healthCheck.CheckHealthAsync(_healthCheckContext, default); + + // Assert + result.Status.Should().Be(HealthStatus.Healthy); + result.Description.Should().Be("Placeholder mode. Check skipped."); + result.Exception.Should().BeNull(); + } + + [Fact] + public async Task CheckHealthAsync_ChecksBlobConnectivity() + { + // Arrange + Page page = Page.FromValues([], null, Mock.Of()); + AsyncPageable pageable = AsyncPageable.FromPages([page]); + SetupGetContainers(pageable); + BlobServiceClient client = _mockBlobServiceClient.Object; + _provider.Setup(p => p.TryCreateBlobServiceClientFromConnection( + ConnectionStringNames.Storage, out client)).Returns(true); + + WebJobsStorageHealthCheck healthCheck = new(_provider.Object, _environment); + + // Act + HealthCheckResult result = await healthCheck.CheckHealthAsync(_healthCheckContext, default); + + // Assert + VerifyGetContainersCalled(Times.Once()); + result.Status.Should().Be(HealthStatus.Healthy); + result.Exception.Should().BeNull(); + } + + [Fact] + public async Task CheckHealthAsync_Twice_ReturnsCachedResult() + { + // Arrange + Page page = Page.FromValues([], null, Mock.Of()); + AsyncPageable pageable = AsyncPageable.FromPages([page]); + SetupGetContainers(pageable); + BlobServiceClient client = _mockBlobServiceClient.Object; + _provider.Setup(p => p.TryCreateBlobServiceClientFromConnection( + ConnectionStringNames.Storage, out client)).Returns(true); + + WebJobsStorageHealthCheck healthCheck = new(_provider.Object, _environment); + + // Act + HealthCheckContext context = new() + { + Registration = new HealthCheckRegistration("test", healthCheck, null, null) + { + Period = Timeout.InfiniteTimeSpan, + }, + }; + HealthCheckResult result1 = await healthCheck.CheckHealthAsync(context, default); + HealthCheckResult result2 = await healthCheck.CheckHealthAsync(context, default); + + // Assert + VerifyGetContainersCalled(Times.Once()); + result1.Status.Should().Be(HealthStatus.Healthy); + result1.Exception.Should().BeNull(); + result2.Status.Should().Be(HealthStatus.Healthy); + result2.Exception.Should().BeNull(); + } + + [Fact] + public async Task CheckHealthAsync_GetClientFails_Unhealthy() + { + // Arrange + InvalidOperationException ex = new("Failed to create BlobServiceClient."); + BlobServiceClient client = null; + _provider.Setup(p => p.TryCreateBlobServiceClientFromConnection( + ConnectionStringNames.Storage, out client)).Throws(ex); + + WebJobsStorageHealthCheck healthCheck = new(_provider.Object, _environment); + + // Act + HealthCheckResult result = await healthCheck.CheckHealthAsync(_healthCheckContext, default); + + // Assert + result.Status.Should().Be(HealthStatus.Unhealthy); + result.Exception.Should().Be(ex); + result.Data.Should().Contain("Source", "configuration"); + result.Data.Should().Contain("ConfigurationSection", "AzureWebJobsStorage"); + } + + [Fact] + public async Task CheckHealthAsync_WithException_Unhealthy() + { + // Arrange + RequestFailedException rfe = new( + 401, "Some exception message", "SomeErrorCode", null); + SetupGetContainers(rfe); + BlobServiceClient client = _mockBlobServiceClient.Object; + _provider.Setup(p => p.TryCreateBlobServiceClientFromConnection( + ConnectionStringNames.Storage, out client)).Returns(true); + + WebJobsStorageHealthCheck healthCheck = new(_provider.Object, _environment); + + // Act + HealthCheckResult result = await healthCheck.CheckHealthAsync(_healthCheckContext, default); + + // Assert + VerifyGetContainersCalled(Times.Once()); + result.Status.Should().Be(HealthStatus.Unhealthy); + result.Exception.Should().Be(rfe); + result.Description.Should().Be("Unable to access AzureWebJobsStorage"); + result.Data.Should().Contain("Source", "connectivity"); + result.Data.Should().Contain("ConfigurationSection", "AzureWebJobsStorage"); + result.Data.Should().Contain("StatusCode", 401); + result.Data.Should().Contain("ErrorCode", "SomeErrorCode"); + } + + private void SetupGetContainers(AsyncPageable pageable) + { + _mockBlobServiceClient.Setup(c => c.GetBlobContainersAsync( + BlobContainerTraits.None, BlobContainerStates.None, null, It.IsAny())) + .Returns(pageable); + } + + private void SetupGetContainers(RequestFailedException ex) + { + _mockBlobServiceClient.Setup(c => c.GetBlobContainersAsync( + BlobContainerTraits.None, BlobContainerStates.None, null, It.IsAny())) + .Throws(ex); + } + + private void VerifyGetContainersCalled(Times times) + { + _mockBlobServiceClient.Verify(c => c.GetBlobContainersAsync( + BlobContainerTraits.None, BlobContainerStates.None, null, It.IsAny()), times); + } + } +} From aae0cdeee5cedb92978642b9180b2f3436801720 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 3 Nov 2025 16:29:51 -0800 Subject: [PATCH 2/6] Address copilot review --- .../Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs | 6 +++--- .../Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs index facaf4267d..cfd917ff2c 100644 --- a/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs @@ -107,7 +107,7 @@ await client return HealthCheckResult.Healthy(); } - catch (Exception ex) + catch (Exception ex) when (!ex.IsFatal()) { HealthCheckData data = GetData(ex, "connectivity"); return HealthCheckResult.Unhealthy($"Unable to access {ConfigSection}", ex, data); @@ -127,7 +127,7 @@ private async Task RunInBackgroundAsync() while (!cancellation.IsCancellationRequested) { TimeSpan delay = _context?.Registration?.Period ?? DefaultPeriod; - await Task.Delay(delay); + await Task.Delay(delay, cancellation); _last = await CheckHealthCoreAsync(cancellation).ConfigureAwait(false); } }) @@ -163,7 +163,7 @@ private bool TryGetClient(out BlobServiceClient client, out HealthCheckResult re result = HealthCheckResult.Healthy(); return true; } - catch (Exception ex) + catch (Exception ex) when (!ex.IsFatal()) { client = null; HealthCheckData data = GetData(ex, "configuration"); diff --git a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs index a9454b182b..bc269abcf4 100644 --- a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs +++ b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs @@ -122,7 +122,7 @@ public void AddScriptHostHealthCheck_RegistersScriptHostHealthCheck() } [Fact] - public void AdWebJobsStorageHealthCheck_RegistersWebJobsStorageHealthCheck() + public void AddWebJobsStorageHealthCheck_RegistersWebJobsStorageHealthCheck() { // arrange Mock builder = new(MockBehavior.Strict); From 3445c74a31f62117354453ca87c19b52d3c4b9dc Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Tue, 4 Nov 2025 12:46:10 -0800 Subject: [PATCH 3/6] Add comments to HealthCheckData --- .../HealthChecks/HealthCheckData.cs | 31 ++++++++++++++++++- .../HealthChecks/WebJobsStorageHealthCheck.cs | 4 +-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckData.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckData.cs index 6914849c3a..3c9fb1663b 100644 --- a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckData.cs +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckData.cs @@ -18,30 +18,59 @@ internal partial class HealthCheckData // exposed to the HealthCheckResult through IReadOnlyDictionary. private readonly Dictionary _data = []; - public string Source + /// + /// Gets or sets the area of the health check data failure. + /// + /// + /// This is the area that has failed. Such as "configuration", "connectivity", etc. + /// + public string Area { get => GetOrDefault(); set => Set(value); } + /// + /// Gets or sets the configuration section related to the health check data. + /// + /// + /// Useful for when the component being checked is related to a specific configuration section. + /// public string ConfigurationSection { get => GetOrDefault(); set => Set(value); } + /// + /// Gets or sets the status code related to the health check data. + /// For HTTP related related checks, this is the HTTP status code. + /// public int StatusCode { get => GetOrDefault(); set => Set(value); } + /// + /// Gets or sets the error code related to the health check data. + /// + /// + /// For Azure SDK related checks, this is typically the RequestFailedException.ErrorCode value. + /// public string ErrorCode { get => GetOrDefault(); set => Set(value); } + /// + /// Sets exception details into the health check data. + /// + /// The exception to set details from. + /// + /// This will set various properties based on the type of exception. + /// public void SetExceptionDetails(Exception ex) { ArgumentNullException.ThrowIfNull(ex); diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs index cfd917ff2c..7017ab747c 100644 --- a/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/WebJobsStorageHealthCheck.cs @@ -172,11 +172,11 @@ private bool TryGetClient(out BlobServiceClient client, out HealthCheckResult re } } - private static HealthCheckData GetData(Exception ex, string source) + private static HealthCheckData GetData(Exception ex, string area) { HealthCheckData data = new() { - Source = source, + Area = area, ConfigurationSection = ConfigSection, }; From cdab4f097b3b1897df15541b4044f9021c43a77d Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Tue, 4 Nov 2025 16:03:02 -0800 Subject: [PATCH 4/6] Fix unit tests --- .../Diagnostics/HealthChecks/HealthCheckExtensions.cs | 5 +++++ .../HealthChecks/HealthCheckExtensionsTests.cs | 10 ++++++++++ .../HealthChecks/WebJobsStorageHealthCheckTests.cs | 4 ++-- test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj | 4 ++-- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs index 643970a4ff..af20fcee19 100644 --- a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs @@ -128,6 +128,11 @@ public static IHealthChecksBuilder AddScriptHostHealthCheck(this IHealthChecksBu return builder; } + /// + /// Adds a health check for the WebJobs storage account. + /// + /// The builder to register health checks with. + /// The original builder, for call chaining. public static IHealthChecksBuilder AddWebJobsStorageHealthCheck(this IHealthChecksBuilder builder) { ArgumentNullException.ThrowIfNull(builder); diff --git a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs index bc269abcf4..3de39cd6a9 100644 --- a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs +++ b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs @@ -125,7 +125,9 @@ public void AddScriptHostHealthCheck_RegistersScriptHostHealthCheck() public void AddWebJobsStorageHealthCheck_RegistersWebJobsStorageHealthCheck() { // arrange + ServiceCollection services = new(); Mock builder = new(MockBehavior.Strict); + builder.Setup(b => b.Services).Returns(services); builder.Setup(b => b.Add(It.IsAny())).Returns(builder.Object); // act @@ -136,7 +138,15 @@ public void AddWebJobsStorageHealthCheck_RegistersWebJobsStorageHealthCheck() builder.Verify(b => b.Add(IsRegistration( HealthCheckNames.WebJobsStorage, HealthCheckTags.Configuration)), Times.Once); + builder.Verify(b => b.Services, Times.AtLeastOnce); builder.VerifyNoOtherCalls(); + services.Should().ContainSingle() + .Which.Should().Satisfy(sd => + { + sd.Lifetime.Should().Be(ServiceLifetime.Singleton); + sd.ServiceType.Should().Be(); + sd.ImplementationType.Should().Be(); + }); } [Fact] diff --git a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/WebJobsStorageHealthCheckTests.cs b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/WebJobsStorageHealthCheckTests.cs index e32ab69e8c..455779f663 100644 --- a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/WebJobsStorageHealthCheckTests.cs +++ b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/WebJobsStorageHealthCheckTests.cs @@ -140,7 +140,7 @@ public async Task CheckHealthAsync_GetClientFails_Unhealthy() // Assert result.Status.Should().Be(HealthStatus.Unhealthy); result.Exception.Should().Be(ex); - result.Data.Should().Contain("Source", "configuration"); + result.Data.Should().Contain("Area", "configuration"); result.Data.Should().Contain("ConfigurationSection", "AzureWebJobsStorage"); } @@ -165,7 +165,7 @@ public async Task CheckHealthAsync_WithException_Unhealthy() result.Status.Should().Be(HealthStatus.Unhealthy); result.Exception.Should().Be(rfe); result.Description.Should().Be("Unable to access AzureWebJobsStorage"); - result.Data.Should().Contain("Source", "connectivity"); + result.Data.Should().Contain("Area", "connectivity"); result.Data.Should().Contain("ConfigurationSection", "AzureWebJobsStorage"); result.Data.Should().Contain("StatusCode", 401); result.Data.Should().Contain("ErrorCode", "SomeErrorCode"); diff --git a/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj b/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj index 03c6f2e8b2..aab7d2c915 100644 --- a/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj +++ b/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj @@ -17,8 +17,8 @@ - - + + From 6ccbbad0ace36d08a5801ae3bfbfc173e1808a21 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 6 Nov 2025 13:57:54 -0800 Subject: [PATCH 5/6] Add Connectivity tag --- .../Diagnostics/HealthChecks/HealthCheckExtensions.cs | 2 +- .../Diagnostics/HealthChecks/HealthCheckTags.cs | 8 ++++++++ .../HealthChecks/HealthCheckExtensionsTests.cs | 6 +++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs index af20fcee19..42b95cd08c 100644 --- a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs @@ -141,7 +141,7 @@ public static IHealthChecksBuilder AddWebJobsStorageHealthCheck(this IHealthChec builder.Services.TryAddSingleton(); builder.AddCheck( HealthCheckNames.WebJobsStorage, - tags: [HealthCheckTags.Configuration, HealthCheckTags.WebJobsStorage], + tags: [HealthCheckTags.Configuration, HealthCheckTags.Connectivity, HealthCheckTags.WebJobsStorage], timeout: TimeSpan.FromSeconds(10)); return builder; } diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs index 0365044794..373a8b702e 100644 --- a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs @@ -35,6 +35,14 @@ internal static class HealthCheckTags /// public const string Configuration = FuncPrefix + "configuration"; + /// + /// The 'azure.functions.connectivity' tag is used for connectivity-related health checks in the Azure Functions host. + /// + /// + /// These are typically related to connectivity to external services, such as Azure Storage. + /// + public const string Connectivity = FuncPrefix + "connectivity"; + /// /// The "azure.web_jobs.storage" tag is used for health checks related to the WebJobs storage account. /// diff --git a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs index 3de39cd6a9..6b475a22f4 100644 --- a/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs +++ b/test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs @@ -136,7 +136,7 @@ public void AddWebJobsStorageHealthCheck_RegistersWebJobsStorageHealthCheck() // assert returned.Should().BeSameAs(builder.Object); builder.Verify(b => b.Add(IsRegistration( - HealthCheckNames.WebJobsStorage, HealthCheckTags.Configuration)), + HealthCheckNames.WebJobsStorage, HealthCheckTags.Configuration, HealthCheckTags.Connectivity, HealthCheckTags.WebJobsStorage)), Times.Once); builder.Verify(b => b.Services, Times.AtLeastOnce); builder.VerifyNoOtherCalls(); @@ -289,7 +289,7 @@ public void UseDynamicHealthCheckService_AddHealthChecksCalled_ReplacesHealthChe VerifyDynamicHealthCheckService(services); } - private static HealthCheckRegistration IsRegistration(string name, string tag) + private static HealthCheckRegistration IsRegistration(string name, params string[] tags) where T : IHealthCheck { static bool IsType(HealthCheckRegistration registration) @@ -304,7 +304,7 @@ static bool IsType(HealthCheckRegistration registration) return Match.Create(r => { - return r.Name == name && r.Tags.Contains(tag) && IsType(r); + return r.Name == name && tags.All(t => r.Tags.Contains(t)) && IsType(r); }); } From 4d48aa25f043987179e2de6bb6372437541b88fa Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 6 Nov 2025 14:00:08 -0800 Subject: [PATCH 6/6] Update WebJobs tag --- .../Diagnostics/HealthChecks/HealthCheckTags.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs index 373a8b702e..2f3bfdadb5 100644 --- a/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs +++ b/src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs @@ -9,7 +9,7 @@ namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks internal static class HealthCheckTags { private const string FuncPrefix = "azure.functions."; - private const string WebJobsPrefix = "azure.web_jobs."; + private const string WebJobsPrefix = FuncPrefix + "web_jobs."; /// /// The 'azure.functions.liveness' tag is used for liveness checks in the Azure Functions host. @@ -44,7 +44,7 @@ internal static class HealthCheckTags public const string Connectivity = FuncPrefix + "connectivity"; /// - /// The "azure.web_jobs.storage" tag is used for health checks related to the WebJobs storage account. + /// The "azure.functions.web_jobs.storage" tag is used for health checks related to the WebJobs storage account. /// public const string WebJobsStorage = WebJobsPrefix + "storage"; }