Implementing a resolver that resolves worker configurations from specified probing paths (#11258)

Author: surgupta-msft

release_notes.md

@@ -6,6 +6,7 @@
 - Emit diagnostic warning for deprecated Azure Functions Proxies usage (#11405)
 - Update Python Worker Version to [4.40.2](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.40.2)
 - Add JitTrace Files for v4.1044
+- Implementing a resolver that resolves worker configurations from specified probing paths (#11258)
 - Remove duplicate function names from sync triggers payload(#11371)
 - Avoid emitting empty tag values for health check metrics (#11393)
 - Run health checks from the active ScriptHost (#11410)

src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs

@@ -238,8 +238,12 @@ public static void AddWebJobsScriptHost(this IServiceCollection services, IConfi
             // Refresh WorkerConfigurationResolverOptions and LanguageWorkerOptions when HostBuiltChangeTokenSource is triggered.
             services.ConfigureOptionsWithChangeTokenSource<WorkerConfigurationResolverOptions, WorkerConfigurationResolverOptionsSetup, HostBuiltChangeTokenSource<WorkerConfigurationResolverOptions>>();
             services.ConfigureOptionsWithChangeTokenSource<LanguageWorkerOptions, LanguageWorkerOptionsSetup, HostBuiltChangeTokenSource<LanguageWorkerOptions>>();
+            services.AddSingleton(SystemRuntimeInformation.Instance);
+            services.AddSingleton<IWorkerConfigurationResolver, WorkerConfigurationResolver>();
+            services.AddSingleton<IWorkerConfigurationProvider, DefaultWorkerConfigurationProvider>();
+            services.AddSingleton<IWorkerConfigurationProvider, DynamicWorkerConfigurationProvider>();
+            services.AddSingleton<IWorkerConfigurationProvider, ExplicitWorkerConfigurationProvider>();
 
-            services.AddSingleton<IWorkerConfigurationResolver, DefaultWorkerConfigurationResolver>();
             services.TryAddSingleton<IDependencyValidator, DependencyValidator>();
             services.TryAddSingleton<IJobHostMiddlewarePipeline>(s => DefaultMiddlewarePipeline.Empty);
 

src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs

@@ -70,6 +70,29 @@ internal string WorkerIndexingDisabledApps
             }
         }
 
+        /// <summary>
+        /// Gets a string delimited by '|' that contains the names of the language workers available through probing paths outside of the Host.
+        /// </summary>
+        internal string WorkersAvailableForDynamicResolution
+        {
+            get
+            {
+                return GetFeature(RpcWorkerConstants.WorkersAvailableForDynamicResolution) ?? string.Empty;
+            }
+        }
+
+        /// <summary>
+        /// Gets a string delimited by '|' that contains the versions of language workers to be ignored during probing outside of the Host.
+        /// Example value: "Worker1Name:Version1|Worker1Name:Version2|Worker2Name:Version1|Worker3Name:Version1".
+        /// </summary>
+        internal string IgnoredWorkerVersions
+        {
+            get
+            {
+                return GetFeature(RpcWorkerConstants.IgnoredWorkerVersions) ?? string.Empty;
+            }
+        }
+
         /// <summary>
         /// Gets a value indicating whether Linux Log Backoff is disabled in the hosting config.
         /// </summary>

src/WebJobs.Script/Diagnostics/Extensions/LoggerExtension.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;

src/WebJobs.Script/Environment/EnvironmentExtensions.cs

@@ -278,6 +278,16 @@ public static bool IsWindowsElasticPremium(this IEnvironment environment)
             return string.Equals(value, ScriptConstants.ElasticPremiumSku, StringComparison.OrdinalIgnoreCase);
         }
 
+        /// <summary>
+        /// Gets a value indicating whether the application is running in a Windows App Service environment.
+        /// </summary>
+        /// <param name="environment">The environment to verify.</param>
+        /// <returns><see cref="true"/> if running in a Windows App Service app; otherwise, false.</returns>
+        public static bool IsHostedWindowsEnvironment(this IEnvironment environment)
+        {
+            return environment.IsWindowsAzureManagedHosting() || environment.IsWindowsConsumption() || environment.IsWindowsElasticPremium();
+        }
+
         public static bool IsDynamicSku(this IEnvironment environment)
         {
             return environment.IsConsumptionSku() || environment.IsWindowsElasticPremium();
@@ -696,6 +706,16 @@ public static bool IsInProc(this IEnvironment environment, string workerRuntime
             return string.IsNullOrEmpty(workerRuntime) || string.Equals(workerRuntime, RpcWorkerConstants.DotNetLanguageWorkerName, StringComparison.OrdinalIgnoreCase);
         }
 
+        /// <summary>
+        /// Returns the Antares platform release channel specified by the environment variable.
+        /// Value of this setting could be "LATEST", "STANDARD" or "EXTENDED".
+        /// If the environment variable is not set, the method returns the default value "LATEST".
+        /// </summary>
+        public static string GetPlatformReleaseChannel(this IEnvironment environment)
+        {
+            return environment.GetEnvironmentVariable(AntaresPlatformReleaseChannel) ?? ScriptConstants.LatestPlatformChannelNameUpper;
+        }
+
         public static bool IsApplicationInsightsAgentEnabled(this IEnvironment environment)
         {
             // cache the value of the environment variable

src/WebJobs.Script/Environment/EnvironmentSettingNames.cs

@@ -35,6 +35,7 @@ public static class EnvironmentSettingNames
         public const string AppInsightsAgent = "APPLICATIONINSIGHTS_ENABLE_AGENT";
         public const string FunctionsExtensionVersion = "FUNCTIONS_EXTENSION_VERSION";
         public const string FunctionWorkerRuntime = "FUNCTIONS_WORKER_RUNTIME";
+        public const string WorkerProbingPaths = "WORKER_PROBING_PATHS";
         public const string ContainerName = "CONTAINER_NAME";
         public const string WebsitePodName = "WEBSITE_POD_NAME";
         public const string LegionServiceHost = "LEGION_SERVICE_HOST";

src/WebJobs.Script/JsonSerializerOptionsProvider.cs

@@ -19,6 +19,12 @@ public static class JsonSerializerOptionsProvider
         /// </summary>
         public static readonly JsonSerializerOptions Options = CreateJsonOptions();
 
+        /// <summary>
+        /// Shared Json serializer with the following setting:
+        /// - PropertyNameCaseInsensitive: true.
+        /// </summary>
+        public static readonly JsonSerializerOptions CaseInsensitiveJsonSerializerOptions = CreateCaseInsensitiveJsonOptions();
+
         private static JsonSerializerOptions CreateJsonOptions()
         {
             var options = new JsonSerializerOptions
@@ -32,5 +38,10 @@ private static JsonSerializerOptions CreateJsonOptions()
 
             return options;
         }
+
+        private static JsonSerializerOptions CreateCaseInsensitiveJsonOptions()
+        {
+            return new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
+        }
     }
-}
+}

src/WebJobs.Script/ScriptConstants.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using NuGet.Versioning;
 
@@ -133,6 +134,7 @@ public static class ScriptConstants
         public const string FeatureFlagDisableWebHostLogForwarding = "DisableWebHostLogForwarding";
         public const string FeatureFlagDisableMergedWebHostScriptHostConfiguration = "DisableMergedConfiguration";
         public const string FeatureFlagEnableWorkerIndexing = "EnableWorkerIndexing";
+        public const string FeatureFlagDisableDynamicWorkerResolution = "DisableDynamicWorkerResolution";
         public const string FeatureFlagEnableDebugTracing = "EnableDebugTracing";
         public const string FeatureFlagEnableProxies = "EnableProxies";
         public const string FeatureFlagStrictHISModeEnabled = "StrictHISModeEnabled";
@@ -251,6 +253,7 @@ public static class ScriptConstants
         public static readonly long DefaultMaxRequestBodySize = 104857600;
 
         public static readonly ImmutableArray<string> SystemLogCategoryPrefixes = ImmutableArray.Create("Microsoft.Azure.WebJobs.", "Function.", "Worker.", "Host.");
+        public static readonly IReadOnlySet<string> HostCapabilities = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
 
         public static readonly string FunctionMetadataDirectTypeKey = "DirectType";
         public static readonly string LiveLogsSessionAIKey = "#AzFuncLiveLogsSessionId";

src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationProvider.cs

@@ -0,0 +1,53 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO.Abstractions;
+using Microsoft.Azure.WebJobs.Script.Diagnostics;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions;
+using Microsoft.Azure.WebJobs.Script.Workers.Profiles;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration
+{
+    /// <summary>
+    /// This class resolves worker configurations by scanning the "workers" directory within the Host for worker config files.
+    /// </summary>
+    internal sealed class DefaultWorkerConfigurationProvider : WorkerConfigurationProviderBase
+    {
+        private readonly IFileSystem _fileSystem;
+        private readonly ILogger _logger;
+
+        public DefaultWorkerConfigurationProvider(
+                        ILoggerFactory loggerFactory,
+                        IMetricsLogger metricsLogger,
+                        IFileSystem fileSystem,
+                        IWorkerProfileManager workerProfileManager,
+                        ISystemRuntimeInformation systemRuntimeInformation,
+                        IOptionsMonitor<WorkerConfigurationResolverOptions> workerConfigurationResolverOptions)
+                        : base(metricsLogger, workerProfileManager, systemRuntimeInformation, workerConfigurationResolverOptions)
+        {
+            ArgumentNullException.ThrowIfNull(loggerFactory);
+            _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryWorkerConfig);
+            _fileSystem = fileSystem;
+        }
+
+        public override ILogger Logger { get => _logger; }
+
+        public override int Priority { get => 2; }
+
+        public override void PopulateWorkerConfigs(Dictionary<string, RpcWorkerConfig> workerRuntimeToConfigMap)
+        {
+            Logger.DefaultWorkersDirectoryPath(WorkerResolverOptions.WorkersRootDirPath);
+
+            // Resolves worker configurations by scanning the "workers" directory within the Host.
+            foreach (var workerPath in _fileSystem.Directory.EnumerateDirectories(WorkerResolverOptions.WorkersRootDirPath))
+            {
+                var workerDirName = _fileSystem.Path.GetFileName(workerPath);
+                AddProvider(WorkerResolverOptions, workerDirName, workerPath, workerRuntimeToConfigMap);
+            }
+        }
+    }
+}