Optionally use max parallelism value from the client settings API (#5132)

b-barthel · web-flow · commit 493c993dfe95 · 2025-03-05T14:49:58.000-05:00
Optionally use our existing client settings API to set the default if it is not overridden on a pipeline level. If the client setting is not available or set, continue using the 192 default.
diff --git a/src/Agent.Plugins/Artifact/FileContainerProvider.cs b/src/Agent.Plugins/Artifact/FileContainerProvider.cs
@@ -170,7 +170,7 @@ private async Task DownloadFileContainerAsync(IEnumerable<FileContainerItem> ite
                         (dedupClient, clientTelemetry) = DedupManifestArtifactClientFactory.Instance.CreateDedupClient(
                             this.connection,
                             domainId,
-                            DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
+                            DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context, clientSettings),
                             clientSettings.GetRedirectTimeout(),
                             false,
                             (str) => this.tracer.Info(str),
diff --git a/src/Agent.Plugins/Artifact/PipelineArtifactServer.cs b/src/Agent.Plugins/Artifact/PipelineArtifactServer.cs
@@ -60,7 +60,6 @@ internal async Task UploadAsync(
                     context.IsSystemDebugTrue(),
                     (str) => context.Output(str),
                     connection,
-                    DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
                     domainId,
                     clientSettings,
                     context,
diff --git a/src/Agent.Worker/Build/FileContainerServer.cs b/src/Agent.Worker/Build/FileContainerServer.cs
@@ -364,7 +364,12 @@ private async Task<UploadResult> BlobUploadAsync(IAsyncCommandContext context, I
                         Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.BuildArtifact,
                         DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, tracer),
                         token);
-
+                int maxParallelism = context.GetHostContext().GetService<IConfigurationStore>().GetSettings().MaxDedupParallelism;
+                if (maxParallelism == 0)
+                {
+                    // if we have a client setting for max parallelism, use that:
+                    maxParallelism = DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(clientSettings, msg => context.Output(msg));
+                }
                 // Check if the pipeline has an override domain set, if not, use the default domain from the client settings.
                 string overrideDomain = AgentKnobs.SendBuildArtifactsToBlobstoreDomain.GetValue(context).AsString();
                 IDomainId domainId = String.IsNullOrWhiteSpace(overrideDomain) ? clientSettings.GetDefaultDomainId() : DomainIdFactory.Create(overrideDomain);
@@ -373,7 +378,7 @@ private async Task<UploadResult> BlobUploadAsync(IAsyncCommandContext context, I
                     .CreateDedupClient(
                         _connection,
                         domainId,
-                        context.GetHostContext().GetService<IConfigurationStore>().GetSettings().MaxDedupParallelism,
+                        maxParallelism,
                         clientSettings.GetRedirectTimeout(),
                         verbose,
                         tracer,
diff --git a/src/Microsoft.VisualStudio.Services.Agent/Blob/BlobstoreClientSettings.cs b/src/Microsoft.VisualStudio.Services.Agent/Blob/BlobstoreClientSettings.cs
@@ -2,26 +2,26 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.VisualStudio.Services.BlobStore.WebApi;
-using Microsoft.VisualStudio.Services.Content.Common.Tracing;
-using Microsoft.VisualStudio.Services.WebApi;
-using Microsoft.VisualStudio.Services.Content.Common;
 using Agent.Sdk;
-using Microsoft.VisualStudio.Services.BlobStore.Common;
+using Agent.Sdk.Knob;
 using BuildXL.Cache.ContentStore.Hashing;
+using Microsoft.VisualStudio.Services.BlobStore.Common;
+using Microsoft.VisualStudio.Services.BlobStore.WebApi;
 using Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts;
-using Agent.Sdk.Knob;
-using System.Collections.Generic;
+using Microsoft.VisualStudio.Services.Content.Common;
+using Microsoft.VisualStudio.Services.Content.Common.Tracing;
+using Microsoft.VisualStudio.Services.WebApi;
 
 namespace Microsoft.VisualStudio.Services.Agent.Blob
 {
     public class BlobstoreClientSettings
     {
         private readonly ClientSettingsInfo clientSettings;
         private readonly IAppTraceSource tracer;
-        
+
         private BlobstoreClientSettings(ClientSettingsInfo settings, IAppTraceSource tracer)
         {
             clientSettings = settings;
@@ -37,8 +37,8 @@ public static async Task<BlobstoreClientSettings> GetClientSettingsAsync(
             BlobStore.WebApi.Contracts.Client? client,
             IAppTraceSource tracer,
             CancellationToken cancellationToken)
-        {            
-            if(client.HasValue)
+        {
+            if (client.HasValue)
             {
                 try
                 {
@@ -75,8 +75,8 @@ public IDomainId GetDefaultDomainId()
                 {
                     tracer.Info($"Error converting the domain id '{clientSettings.Properties[ClientSettingsConstants.DefaultDomainId]}': {exception.Message}.  Falling back to default.");
                 }
-            } 
-            else 
+            }
+            else
             {
                 tracer.Verbose($"No client settings found, using the default domain id '{domainId}'.");
             }
@@ -117,5 +117,18 @@ public HashType GetClientHashType(AgentTaskPluginExecutionContext context)
                 return null;
             }
         }
+
+        public int? GetMaxParallelism()
+        {
+            const string MaxParallelism = "MaxParallelism";
+            if (int.TryParse(clientSettings?.Properties.GetValueOrDefault(MaxParallelism), out int maxParallelism))
+            {
+                return maxParallelism;
+            }
+            else
+            {
+                return null;
+            }
+        }
     }
 }
diff --git a/src/Microsoft.VisualStudio.Services.Agent/Blob/DedupManifestArtifactClientFactory.cs b/src/Microsoft.VisualStudio.Services.Agent/Blob/DedupManifestArtifactClientFactory.cs
@@ -4,18 +4,15 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.VisualStudio.Services.BlobStore.WebApi;
-using Microsoft.VisualStudio.Services.Content.Common.Tracing;
-using Microsoft.VisualStudio.Services.WebApi;
-using Microsoft.VisualStudio.Services.Content.Common;
-using Microsoft.VisualStudio.Services.BlobStore.Common.Telemetry;
 using Agent.Sdk;
+using BuildXL.Cache.ContentStore.Hashing;
 using Microsoft.TeamFoundation.DistributedTask.WebApi;
 using Microsoft.VisualStudio.Services.BlobStore.Common;
-using BuildXL.Cache.ContentStore.Hashing;
-using Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts;
-using Agent.Sdk.Knob;
-using System.Collections.Generic;
+using Microsoft.VisualStudio.Services.BlobStore.Common.Telemetry;
+using Microsoft.VisualStudio.Services.BlobStore.WebApi;
+using Microsoft.VisualStudio.Services.Content.Common;
+using Microsoft.VisualStudio.Services.Content.Common.Tracing;
+using Microsoft.VisualStudio.Services.WebApi;
 
 namespace Microsoft.VisualStudio.Services.Agent.Blob
 {
@@ -36,7 +33,6 @@ public interface IDedupManifestArtifactClientFactory
             bool verbose,
             Action<string> traceOutput,
             VssConnection connection,
-            int maxParallelism,
             IDomainId domainId,
             BlobstoreClientSettings clientSettings,
             AgentTaskPluginExecutionContext context,
@@ -74,13 +70,6 @@ public interface IDedupManifestArtifactClientFactory
             bool verbose,
             Action<string> traceOutput,
             CancellationToken cancellationToken);
-
-        /// <summary>
-        /// Gets the maximum parallelism to use for dedup related downloads and uploads.
-        /// </summary>
-        /// <param name="context">Context which may specify overrides for max parallelism</param>
-        /// <returns>max parallelism</returns>
-        int GetDedupStoreClientMaxParallelism(AgentTaskPluginExecutionContext context);
     }
 
     public class DedupManifestArtifactClientFactory : IDedupManifestArtifactClientFactory
@@ -118,7 +107,6 @@ private DedupManifestArtifactClientFactory()
                     context.IsSystemDebugTrue(),
                     (str) => context.Output(str),
                     connection,
-                    DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
                     domainId,
                     clientSettings,
                     context,
@@ -129,26 +117,22 @@ private DedupManifestArtifactClientFactory()
             bool verbose,
             Action<string> traceOutput,
             VssConnection connection,
-            int maxParallelism,
             IDomainId domainId,
             BlobstoreClientSettings clientSettings,
             AgentTaskPluginExecutionContext context,
             CancellationToken cancellationToken)
         {
             const int maxRetries = 5;
             var tracer = CreateArtifactsTracer(verbose, traceOutput);
-            if (maxParallelism == 0)
-            {
-                maxParallelism = DefaultDedupStoreClientMaxParallelism;
-            }
+            int maxParallelism = DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context, clientSettings);
 
             traceOutput($"Max dedup parallelism: {maxParallelism}");
             traceOutput($"DomainId: {domainId}");
 
             IDedupStoreHttpClient dedupStoreHttpClient = GetDedupStoreHttpClient(connection, domainId, maxRetries, tracer, cancellationToken);
 
             var telemetry = new BlobStoreClientTelemetry(tracer, dedupStoreHttpClient.BaseAddress);
-            HashType hashType= clientSettings.GetClientHashType(context);
+            HashType hashType = clientSettings.GetClientHashType(context);
 
             if (hashType == BuildXL.Cache.ContentStore.Hashing.HashType.Dedup1024K)
             {
@@ -222,32 +206,37 @@ private static IDedupStoreHttpClient GetDedupStoreHttpClient(VssConnection conne
             return (client, telemetry);
         }
 
-        public int GetDedupStoreClientMaxParallelism(AgentTaskPluginExecutionContext context)
+        public int GetDedupStoreClientMaxParallelism(AgentTaskPluginExecutionContext context, BlobstoreClientSettings clientSettings)
         {
             ConfigureEnvironmentVariables(context);
 
-            int parallelism = DefaultDedupStoreClientMaxParallelism;
-
-            if (context.Variables.TryGetValue("AZURE_PIPELINES_DEDUP_PARALLELISM", out VariableValue v))
+            // prefer the pipeline variable over the client settings
+            if (context.Variables.TryGetValue(DedupParallelism, out VariableValue v))
             {
-                if (!int.TryParse(v.Value, out parallelism))
-                {
-                    context.Output($"Could not parse the value of AZURE_PIPELINES_DEDUP_PARALLELISM, '{v.Value}', as an integer. Defaulting to {DefaultDedupStoreClientMaxParallelism}");
-                    parallelism = DefaultDedupStoreClientMaxParallelism;
-                }
-                else
+                if (int.TryParse(v.Value, out int parallelism))
                 {
                     context.Output($"Overriding default max parallelism with {parallelism}");
+                    return parallelism;
                 }
             }
-            else
+            return GetDedupStoreClientMaxParallelism(clientSettings, msg => context.Output(msg));
+        }
+
+        public int GetDedupStoreClientMaxParallelism(BlobstoreClientSettings clientSettings, Action<string> logOutput)
+        {
+            // if we have a client setting for max parallelism, use that:
+            int? maxParallelism = clientSettings?.GetMaxParallelism();
+            if (maxParallelism.HasValue)
             {
-                context.Output($"Using default max parallelism.");
+                logOutput($"Using max parallelism from client settings: {maxParallelism}");
+                return maxParallelism.Value;
             }
-
-            return parallelism;
+            // if we get here, nothing left to do but use the default:
+            logOutput($"Using default max parallelism.");
+            return DefaultDedupStoreClientMaxParallelism;
         }
 
+        public static string DedupParallelism = "AZURE_PIPELINES_DEDUP_PARALLELISM";
         private static readonly string[] EnvironmentVariables = new[] { "VSO_DEDUP_REDIRECT_TIMEOUT_IN_SEC" };
 
         private static void ConfigureEnvironmentVariables(AgentTaskPluginExecutionContext context)
@@ -269,7 +258,6 @@ private static void ConfigureEnvironmentVariables(AgentTaskPluginExecutionContex
             }
         }
 
-
         public static IAppTraceSource CreateArtifactsTracer(bool verbose, Action<string> traceOutput)
         {
             return new CallbackAppTraceSource(
diff --git a/src/Microsoft.VisualStudio.Services.Agent/JobServer.cs b/src/Microsoft.VisualStudio.Services.Agent/JobServer.cs
@@ -172,11 +172,18 @@ public async Task<BlobIdentifierWithBlocks> UploadLogToBlobStore(Stream blob, st
 
         public async Task<(DedupIdentifier dedupId, ulong length)> UploadAttachmentToBlobStore(bool verbose, string itemPath, Guid planId, Guid jobId, CancellationToken cancellationToken)
         {
-            int maxParallelism = HostContext.GetService<IConfigurationStore>().GetSettings().MaxDedupParallelism;
             var clientSettings = await BlobstoreClientSettings.GetClientSettingsAsync(
-                _connection, 
-                client: null, 
+                _connection,
+                client: null,
                 DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, (str) => Trace.Info(str)), cancellationToken);
+
+            int maxParallelism = HostContext.GetService<IConfigurationStore>().GetSettings().MaxDedupParallelism;
+            if (maxParallelism == 0)
+            {
+                // if we have a client setting for max parallelism, use that - this won't log anything:
+                maxParallelism = DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(clientSettings, msg => { });
+            }
+
             var (dedupClient, clientTelemetry) = DedupManifestArtifactClientFactory.Instance
                 .CreateDedupClient(
                     _connection,
diff --git a/src/Test/L0/Plugin/TestFileShareProvider/MockDedupManifestArtifactClientFactory.cs b/src/Test/L0/Plugin/TestFileShareProvider/MockDedupManifestArtifactClientFactory.cs
@@ -6,13 +6,11 @@
 using System.Threading.Tasks;
 using Agent.Sdk;
 using Microsoft.VisualStudio.Services.Agent.Blob;
+using Microsoft.VisualStudio.Services.BlobStore.Common;
+using Microsoft.VisualStudio.Services.BlobStore.Common.Telemetry;
 using Microsoft.VisualStudio.Services.BlobStore.WebApi;
-using Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts;
 using Microsoft.VisualStudio.Services.Content.Common.Tracing;
 using Microsoft.VisualStudio.Services.WebApi;
-using Microsoft.VisualStudio.Services.BlobStore.Common.Telemetry;
-using Agent.Plugins.PipelineArtifact;
-using Microsoft.VisualStudio.Services.BlobStore.Common;
 
 namespace Microsoft.VisualStudio.Services.Agent.Tests
 {
@@ -40,7 +38,6 @@ public class MockDedupManifestArtifactClientFactory : IDedupManifestArtifactClie
             bool verbose,
             Action<string> traceOutput,
             VssConnection connection,
-            int maxParallelism,
             IDomainId domainId,
             BlobstoreClientSettings clientSettings,
             AgentTaskPluginExecutionContext context,
