Include HealthChecks from ScriptHost scope (#11410)

* Include HealthChecks from ScriptHost scope

* Update release_notes.md

* Update release_notes.md

* Fix unit test

* Address PR comments

* Fix tests

Author: jviau

release_notes.md

@@ -8,6 +8,9 @@
 - Add JitTrace Files for v4.1044
 - Remove duplicate function names from sync triggers payload(#11371)
 - Avoid emitting empty tag values for health check metrics (#11393)
+- Run health checks from the active ScriptHost (#11410)
+- Publish health check metrics to legacy AppInsights SDK (#11365)
+- Fix tag filter for health check live & ready endpoints (#11363)
 - Functions host to take a customer specified port in Custom Handler scenario (#11408)
 - Updated to version 1.5.8 of Microsoft.Azure.AppService.Middleware.Functions (#11416)
 - Enabling worker indexing for Logic Apps app kind behind an enviornment setting

src/WebJobs.Script.WebHost/DependencyInjection/ServiceProviderExtensions.cs

@@ -1,4 +1,4 @@
-// These extensions are based on Orchard (https://github.com/OrchardCMS/OrchardCore)
+// These extensions are based on Orchard (https://github.com/OrchardCMS/OrchardCore)
 // BSD 3 - Clause License
 //  https://opensource.org/licenses/BSD-3-Clause
 //
@@ -7,11 +7,12 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 
 namespace Microsoft.Azure.WebJobs.Script.WebHost.DependencyInjection
 {
@@ -20,13 +21,16 @@ public static class ServiceProviderExtensions
         /// <summary>
         /// Things we don't want to copy down to child containers because...
         /// </summary>
-        private static readonly HashSet<Type> ChildContainerIgnoredTypes = new()
-        {
-            typeof(IStartupFilter),        // This would re-add middlewares to the host pipeline
+        private static readonly HashSet<Type> ChildContainerIgnoredTypes =
+        [
+            typeof(IStartupFilter),        // This would re-add middleware to the host pipeline
             typeof(IManagedHostedService), // These shouldn't be instantiated twice
             typeof(IHostedService),        // These shouldn't be instantiated twice
-            typeof(ILoggerProvider),        // These shouldn't be instantiated twice
-        };
+            typeof(ILoggerProvider),       // These shouldn't be instantiated twice
+            typeof(HealthCheckService),    // Child container should instantiate its own.
+            typeof(IConfigureOptions<HealthCheckServiceOptions>), // Child container should instantiate its own.
+            typeof(IPostConfigureOptions<HealthCheckServiceOptions>), // Child container should instantiate its own.
+        ];
 
         /// <summary>
         /// Creates a child container.

src/WebJobs.Script/Diagnostics/HealthChecks/DynamicHealthCheckService.cs

@@ -0,0 +1,114 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Logging;
+
+#nullable enable
+
+namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks
+{
+    /// <summary>
+    /// Service to manage health checks. This service supports both the WebHost and ScriptHost health checks.
+    /// </summary>
+    /// <remarks>
+    /// The approach taken is to perform individual health checks on both the WebHost and ScriptHost, then merge the results.
+    /// This is to ensure individual health checks run in the correct service scope (WebHost vs ScriptHost) from where they
+    /// are registered.
+    /// </remarks>
+    public sealed partial class DynamicHealthCheckService : HealthCheckService
+    {
+        private readonly ConcurrentDictionary<string, DateTimeOffset> _conflictLogBackoff = new();
+
+        private readonly HealthCheckService _webHostCheck;
+        private readonly IScriptHostManager _manager;
+        private readonly ILogger _logger;
+
+        public DynamicHealthCheckService(
+            HealthCheckService webHostCheck,
+            IScriptHostManager manager,
+            ILogger<DynamicHealthCheckService> logger)
+        {
+            _webHostCheck = webHostCheck ?? throw new ArgumentNullException(nameof(webHostCheck));
+            _manager = manager ?? throw new ArgumentNullException(nameof(manager));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        public override async Task<HealthReport> CheckHealthAsync(
+            Func<HealthCheckRegistration, bool>? predicate,
+            CancellationToken cancellationToken = default)
+        {
+            Task<HealthReport> webHostReport = _webHostCheck.CheckHealthAsync(predicate, cancellationToken);
+            Task<HealthReport?> scriptHostReport = CheckScriptHostHealthAsync(predicate, cancellationToken);
+            return MergeReports(await webHostReport, await scriptHostReport);
+        }
+
+        private HealthReport MergeReports(HealthReport left, HealthReport? right)
+        {
+            if (right == null)
+            {
+                return left;
+            }
+
+            // Merge the entries of both reports. If there are any duplicate keys, keep the left one.
+            Dictionary<string, HealthReportEntry> entries = new(left.Entries);
+            foreach ((string key, HealthReportEntry value) in right.Entries)
+            {
+                if (!entries.TryAdd(key, value))
+                {
+                    bool log = true;
+                    if (_conflictLogBackoff.TryGetValue(key, out DateTimeOffset lastLogTime))
+                    {
+                        // ensure we log this only once per hour
+                        log = DateTimeOffset.UtcNow - lastLogTime > TimeSpan.FromHours(1);
+                    }
+
+                    if (log)
+                    {
+                        _conflictLogBackoff[key] = DateTimeOffset.UtcNow;
+                        Log.DuplicateHealthCheckEntry(_logger, key);
+                    }
+                }
+            }
+
+            // take the worst status.
+            HealthStatus status = left.Status < right.Status
+                ? left.Status : right.Status;
+
+            // take the longest duration.
+            TimeSpan duration = left.TotalDuration > right.TotalDuration
+                ? left.TotalDuration : right.TotalDuration;
+
+            return new(entries, status, duration);
+        }
+
+        private async Task<HealthReport?> CheckScriptHostHealthAsync(
+            Func<HealthCheckRegistration, bool>? predicate,
+            CancellationToken cancellationToken = default)
+        {
+            HealthCheckService? scriptHostCheck = _manager.Services.GetService<HealthCheckService>();
+            if (scriptHostCheck is null)
+            {
+                Log.ScriptHostNoHealthCheckService(_logger);
+                return null;
+            }
+
+            return await scriptHostCheck.CheckHealthAsync(predicate, cancellationToken);
+        }
+
+        private static partial class Log
+        {
+            [LoggerMessage(0, LogLevel.Debug, "Script host does not have a health check service. Skipping script host health checks.")]
+            public static partial void ScriptHostNoHealthCheckService(ILogger logger);
+
+            [LoggerMessage(1, LogLevel.Warning, "Duplicate health check entry '{HealthCheck}' found when merging health check reports. Keeping the first entry.")]
+            public static partial void DuplicateHealthCheckEntry(ILogger logger, string healthCheck);
+        }
+    }
+}

src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs

@@ -26,7 +26,38 @@ public static IHealthChecksBuilder AddWebJobsScriptHealthChecks(this IHealthChec
             builder
                 .AddWebHostHealthCheck()
                 .AddScriptHostHealthCheck()
-                .AddTelemetryPublisher(HealthCheckTags.Liveness, HealthCheckTags.Readiness);
+                .AddTelemetryPublisher(HealthCheckTags.Liveness, HealthCheckTags.Readiness)
+                .UseDynamicHealthCheckService();
+            return builder;
+        }
+
+        /// <summary>
+        /// Replaces the default <see cref="HealthCheckService"/> with a <see cref="DynamicHealthCheckService"/>.
+        /// </summary>
+        /// <param name="builder">The builder to register health checks with.</param>
+        /// <returns>The original builder, for call chaining.</returns>
+        /// <exception cref="InvalidOperationException">If IServiceCollection.AddHealthChecks() is not called first.</exception>
+        public static IHealthChecksBuilder UseDynamicHealthCheckService(this IHealthChecksBuilder builder)
+        {
+            ArgumentNullException.ThrowIfNull(builder);
+
+            // AddHealthChecks() must be called first to register the default HealthCheckService.
+            // We will then remove the registration and replace it with our own.
+            // We use the existing registration to manually create an instance of the service.
+            ServiceDescriptor descriptor = builder.Services.Where(x => x.ServiceType == typeof(HealthCheckService))
+                .FirstOrDefault()
+                ?? throw new InvalidOperationException(
+                    $"Ensure IServiceCollection.AddHealthChecks() is called before {nameof(UseDynamicHealthCheckService)}.");
+
+            builder.Services.Remove(descriptor);
+            builder.Services.AddSingleton<HealthCheckService, DynamicHealthCheckService>(sp =>
+            {
+                // Manually create the existing HealthCheckService instance so we can wrap it.
+                // We don't want to have to re-implement its logic ourselves.
+                HealthCheckService webHost = (HealthCheckService)sp.CreateInstance(descriptor);
+                return ActivatorUtilities.CreateInstance<DynamicHealthCheckService>(sp, webHost);
+            });
+
             return builder;
         }
 

src/WebJobs.Script/Extensions/IServiceProviderExtensions.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
@@ -36,5 +36,35 @@ public static TService GetScriptHostServiceOrNull<TService>(this IServiceProvide
 
             return null;
         }
+
+        /// <summary>
+        /// Creates an instance of the service described by the <see cref="ServiceDescriptor"/>.
+        /// This method does not necessarily respect the service lifetime, so be cautious when using it.
+        /// </summary>
+        /// <param name="serviceProvider">The service provider to query.</param>
+        /// <param name="descriptor">The descriptor to instantiate a service from.</param>
+        /// <returns>The instantiated service.</returns>
+        public static object CreateInstance(this IServiceProvider serviceProvider, ServiceDescriptor descriptor)
+        {
+            ArgumentNullException.ThrowIfNull(serviceProvider);
+            ArgumentNullException.ThrowIfNull(descriptor);
+
+            if (descriptor.ImplementationType is { } type)
+            {
+                return ActivatorUtilities.GetServiceOrCreateInstance(serviceProvider, type);
+            }
+
+            if (descriptor.ImplementationInstance is { } instance)
+            {
+                return instance;
+            }
+
+            if (descriptor.ImplementationFactory is { } factory)
+            {
+                return factory(serviceProvider);
+            }
+
+            throw new ArgumentException($"Could not get service for descriptor.", nameof(descriptor));
+        }
     }
 }

src/WebJobs.Script/Host/IScriptHostManager.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
@@ -19,8 +19,16 @@ public interface IScriptHostManager
         /// </summary>
         event EventHandler<ActiveHostChangedEventArgs> ActiveHostChanged;
 
+        /// <summary>
+        /// Gets the current state of the script host.
+        /// </summary>
         ScriptHostState State { get; }
 
+        /// <summary>
+        /// Gets the services from the current script host.
+        /// </summary>
+        IServiceProvider Services { get; }
+
         /// <summary>
         /// Gets the last host <see cref="Exception"/> that has occurred.
         /// </summary>

test/WebJobs.Script.Tests.Shared/TestHelpers.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
@@ -654,6 +654,8 @@ event EventHandler IScriptHostManager.HostInitializing
 
             Exception IScriptHostManager.LastError => throw new NotImplementedException();
 
+            IServiceProvider IScriptHostManager.Services => this;
+
             public void OnActiveHostChanged()
             {
                 ActiveHostChanged?.Invoke(this, new ActiveHostChangedEventArgs(null, null));