Skip to content

Commit 8fad84e

Browse files
b-barthelBrian Barthel
b-barthel
and
Brian Barthel
authored
Add agent knob to override chunk size, along with l0 tests for the client settings class (#5249)
Adds a new agent knob that can be used to override the chunk size used to publish pipeline artifacts, and a test suite for the blobstoreclientsettings class. Co-authored-by: Brian Barthel <brbarthe@microsoft.com>
1 parent e8f91bf commit 8fad84e

File tree

5 files changed

+264
-3
lines changed

5 files changed

+264
-3
lines changed

‎src/Agent.Sdk/AssemblyInfo.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
using System.Runtime.CompilerServices;
22

3-
[assembly: InternalsVisibleTo("Test")]
3+
[assembly: InternalsVisibleTo("Test")]

‎src/Agent.Sdk/Knob/AgentKnobs.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,13 @@ public class AgentKnobs
386386
new RuntimeKnobSource("AGENT_FAIL_ON_INCOMPATIBLE_OS"),
387387
new BuiltInDefaultKnobSource("false"));
388388

389+
public static readonly Knob OverridePipelineArtifactChunkSize = new Knob(
390+
nameof(OverridePipelineArtifactChunkSize),
391+
"Overrides the chunk size used in this pipeline for pipeline artifact publish.",
392+
new RuntimeKnobSource("OVERRIDE_PIPELINE_ARTIFACT_CHUNKSIZE"),
393+
new EnvironmentKnobSource("OVERRIDE_PIPELINE_ARTIFACT_CHUNKSIZE"),
394+
new BuiltInDefaultKnobSource(string.Empty));
395+
389396
public static readonly Knob AgentEnablePipelineArtifactLargeChunkSize = new Knob(
390397
nameof(AgentEnablePipelineArtifactLargeChunkSize),
391398
"Enables large chunk size for pipeline artifacts.",

‎src/Microsoft.VisualStudio.Services.Agent/AssemblyInfo.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
using System.Runtime.CompilerServices;
2+
3+
[assembly: InternalsVisibleTo("Test")]

‎src/Microsoft.VisualStudio.Services.Agent/Blob/BlobstoreClientSettings.cs‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ public class BlobstoreClientSettings
2222
private readonly ClientSettingsInfo clientSettings;
2323
private readonly IAppTraceSource tracer;
2424

25-
private BlobstoreClientSettings(ClientSettingsInfo settings, IAppTraceSource tracer)
25+
internal BlobstoreClientSettings(ClientSettingsInfo settings, IAppTraceSource tracer)
2626
{
2727
clientSettings = settings;
2828
this.tracer = tracer;
@@ -90,7 +90,8 @@ public HashType GetClientHashType(AgentTaskPluginExecutionContext context)
9090
// Note: 9/6/2023 Remove the below check in couple of months.
9191
if (AgentKnobs.AgentEnablePipelineArtifactLargeChunkSize.GetValue(context).AsBoolean())
9292
{
93-
if (clientSettings != null && clientSettings.Properties.ContainsKey(ClientSettingsConstants.ChunkSize))
93+
// grab the client settings from the server first if available:
94+
if (clientSettings?.Properties.ContainsKey(ClientSettingsConstants.ChunkSize) == true)
9495
{
9596
try
9697
{
@@ -101,6 +102,29 @@ public HashType GetClientHashType(AgentTaskPluginExecutionContext context)
101102
tracer.Info($"Error converting the chunk size '{clientSettings.Properties[ClientSettingsConstants.ChunkSize]}': {exception.Message}. Falling back to default.");
102103
}
103104
}
105+
106+
// now check if this pipeline has an override chunk size set, and use that if available:
107+
string overrideChunkSize = AgentKnobs.OverridePipelineArtifactChunkSize.GetValue(context).AsString();
108+
if (!String.IsNullOrEmpty(overrideChunkSize))
109+
{
110+
try
111+
{
112+
HashTypeExtensions.Deserialize(overrideChunkSize, out HashType overrideHashType);
113+
if (ChunkerHelper.IsHashTypeChunk(overrideHashType))
114+
{
115+
hashType = overrideHashType;
116+
tracer.Info($"Overriding chunk size to '{overrideChunkSize}'.");
117+
}
118+
else
119+
{
120+
tracer.Info($"Override chunk size '{overrideChunkSize}' is not a valid chunk type. Falling back to client settings.");
121+
}
122+
}
123+
catch (Exception exception)
124+
{
125+
tracer.Info($"Error overriding the chunk size to '{overrideChunkSize}': {exception.Message}. Falling back to client settings.");
126+
}
127+
}
104128
}
105129

106130
return ChunkerHelper.IsHashTypeChunk(hashType) ? hashType : ChunkerHelper.DefaultChunkHashType;

‎src/Test/L0/Plugin/BlobstoreClientSettingsL0.cs‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Threading;
4+
using System.Threading.Tasks;
5+
using Agent.Sdk;
6+
using Agent.Sdk.Knob;
7+
using BuildXL.Cache.ContentStore.Hashing;
8+
using Microsoft.VisualStudio.Services.Agent.Blob;
9+
using Microsoft.VisualStudio.Services.Agent.Worker;
10+
using Microsoft.VisualStudio.Services.BlobStore.Common;
11+
using Microsoft.VisualStudio.Services.BlobStore.WebApi;
12+
using Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts;
13+
using Microsoft.VisualStudio.Services.Common;
14+
using Microsoft.VisualStudio.Services.Content.Common;
15+
using Microsoft.VisualStudio.Services.Content.Common.Tracing;
16+
using Microsoft.VisualStudio.Services.WebApi;
17+
using Moq;
18+
using Xunit;
19+
20+
namespace Microsoft.VisualStudio.Services.Agent.Tests
21+
{
22+
23+
public class BlobstoreClientSettingsL0
24+
{
25+
private const string OverrideChunkSize = "OVERRIDE_PIPELINE_ARTIFACT_CHUNKSIZE";
26+
private const string EnablePipelineArtifactLargeChunkSize = "AGENT_ENABLE_PIPELINEARTIFACT_LARGE_CHUNK_SIZE";
27+
[Fact]
28+
public void GetDefaultDomainId_ReturnsDefault_WhenNoSettings()
29+
{
30+
// Arrange
31+
var tracer = new Mock<IAppTraceSource>();
32+
var settings = new BlobstoreClientSettings(null, tracer.Object);
33+
34+
// Act
35+
var result = settings.GetDefaultDomainId();
36+
37+
// Assert
38+
Assert.Equal(WellKnownDomainIds.DefaultDomainId, result);
39+
}
40+
41+
[Fact]
42+
public void GetDefaultDomainId_ReturnsDomainId_WhenSettingsPresent()
43+
{
44+
// Arrange
45+
var tracer = new Mock<IAppTraceSource>();
46+
var domainId = Guid.NewGuid().ToString();
47+
var clientSettings = new ClientSettingsInfo
48+
{
49+
Properties = new Dictionary<string, string>
50+
{
51+
{ ClientSettingsConstants.DefaultDomainId, domainId }
52+
}
53+
};
54+
var settings = new BlobstoreClientSettings(clientSettings, tracer.Object);
55+
56+
// Act
57+
var result = settings.GetDefaultDomainId();
58+
59+
// Assert
60+
Assert.NotNull(result);
61+
}
62+
63+
[Fact]
64+
public void GetClientHashType_EnablePipelineArtifactLargeChunkSize_EnablesOrDisablesChunkSizing()
65+
{
66+
// Arrange
67+
var tracer = new Mock<IAppTraceSource>();
68+
var clientSettings = new ClientSettingsInfo { Properties = new Dictionary<string, string>()
69+
{
70+
{ ClientSettingsConstants.ChunkSize, HashType.Dedup1024K.ToString() }
71+
}
72+
};
73+
var settings = new BlobstoreClientSettings(clientSettings, tracer.Object);
74+
75+
var environment = new LocalEnvironment();
76+
var context = new Mock<AgentTaskPluginExecutionContext>();
77+
78+
context.As<IKnobValueContext>()
79+
.Setup(x => x.GetScopedEnvironment())
80+
.Returns(environment);
81+
82+
context.As<IKnobValueContext>()
83+
.Setup(x => x.GetVariableValueOrDefault(EnablePipelineArtifactLargeChunkSize ))
84+
.Returns("false");
85+
environment.SetEnvironmentVariable(EnablePipelineArtifactLargeChunkSize, "false");
86+
87+
// Act
88+
var result = settings.GetClientHashType(context.Object);
89+
90+
// make sure if we enable it, it uses the client settings
91+
Assert.Equal(ChunkerHelper.DefaultChunkHashType, result);
92+
context.As<IKnobValueContext>()
93+
.Setup(x => x.GetVariableValueOrDefault(EnablePipelineArtifactLargeChunkSize))
94+
.Returns("true");
95+
environment.SetEnvironmentVariable(EnablePipelineArtifactLargeChunkSize, "true");
96+
97+
// Act
98+
result = settings.GetClientHashType(context.Object);
99+
100+
// Assert
101+
Assert.Equal(HashType.Dedup1024K, result);
102+
}
103+
104+
[Fact]
105+
public void GetClientHashType_PipelineOverride()
106+
{
107+
// Arrange
108+
var tracer = new Mock<IAppTraceSource>();
109+
var clientSettings = new ClientSettingsInfo
110+
{
111+
Properties = new Dictionary<string, string>()
112+
{
113+
{ ClientSettingsConstants.ChunkSize, HashType.Dedup64K.ToString() }
114+
}
115+
};
116+
var settings = new BlobstoreClientSettings(clientSettings, tracer.Object);
117+
118+
var environment = new LocalEnvironment();
119+
var context = new Mock<AgentTaskPluginExecutionContext>();
120+
121+
context.As<IKnobValueContext>()
122+
.Setup(x => x.GetScopedEnvironment())
123+
.Returns(environment);
124+
125+
context.As<IKnobValueContext>()
126+
.Setup(x => x.GetVariableValueOrDefault(EnablePipelineArtifactLargeChunkSize))
127+
.Returns("true");
128+
environment.SetEnvironmentVariable(EnablePipelineArtifactLargeChunkSize, "true");
129+
130+
context.As<IKnobValueContext>()
131+
.Setup(x => x.GetVariableValueOrDefault(OverrideChunkSize))
132+
.Returns(HashType.Dedup1024K.ToString());
133+
environment.SetEnvironmentVariable(OverrideChunkSize, HashType.Dedup1024K.ToString());
134+
135+
// Act
136+
var result = settings.GetClientHashType(context.Object);
137+
138+
// we should successfully override the chunk size in the client settings:
139+
Assert.Equal(HashType.Dedup1024K, result);
140+
141+
// now let's setup a bad override and make sure it falls back to the client settings:
142+
clientSettings.Properties[ClientSettingsConstants.ChunkSize] = HashType.Dedup1024K.ToString();
143+
context.As<IKnobValueContext>()
144+
.Setup(x => x.GetVariableValueOrDefault(OverrideChunkSize))
145+
.Returns("nonsense");
146+
environment.SetEnvironmentVariable(OverrideChunkSize, "nonsense");
147+
148+
// Act
149+
result = settings.GetClientHashType(context.Object);
150+
151+
// Assert
152+
Assert.Equal(HashType.Dedup1024K, result);
153+
}
154+
155+
[Fact]
156+
public void GetRedirectTimeout_ReturnsNull_WhenNotPresent()
157+
{
158+
// Arrange
159+
var tracer = new Mock<IAppTraceSource>();
160+
var clientSettings = new ClientSettingsInfo { Properties = new Dictionary<string, string>() };
161+
var settings = new BlobstoreClientSettings(clientSettings, tracer.Object);
162+
163+
// Act
164+
var result = settings.GetRedirectTimeout();
165+
166+
// Assert
167+
Assert.Null(result);
168+
}
169+
170+
[Fact]
171+
public void GetRedirectTimeout_ReturnsValue_WhenPresent()
172+
{
173+
// Arrange
174+
var tracer = new Mock<IAppTraceSource>();
175+
var clientSettings = new ClientSettingsInfo
176+
{
177+
Properties = new Dictionary<string, string>
178+
{
179+
{ ClientSettingsConstants.RedirectTimeout, "42" }
180+
}
181+
};
182+
var settings = new BlobstoreClientSettings(clientSettings, tracer.Object);
183+
184+
// Act
185+
var result = settings.GetRedirectTimeout();
186+
187+
// Assert
188+
Assert.Equal(42, result);
189+
}
190+
191+
[Fact]
192+
public void GetMaxParallelism_ReturnsNull_WhenNotPresent()
193+
{
194+
// Arrange
195+
var tracer = new Mock<IAppTraceSource>();
196+
var clientSettings = new ClientSettingsInfo { Properties = new Dictionary<string, string>() };
197+
var settings = new BlobstoreClientSettings(clientSettings, tracer.Object);
198+
199+
// Act
200+
var result = settings.GetMaxParallelism();
201+
202+
// Assert
203+
Assert.Null(result);
204+
}
205+
206+
[Fact]
207+
public void GetMaxParallelism_ReturnsValue_WhenPresent()
208+
{
209+
// Arrange
210+
var tracer = new Mock<IAppTraceSource>();
211+
var clientSettings = new ClientSettingsInfo
212+
{
213+
Properties = new Dictionary<string, string>
214+
{
215+
{ "MaxParallelism", "8" }
216+
}
217+
};
218+
var settings = new BlobstoreClientSettings(clientSettings, tracer.Object);
219+
220+
// Act
221+
var result = settings.GetMaxParallelism();
222+
223+
// Assert
224+
Assert.Equal(8, result);
225+
}
226+
}
227+
}

0 commit comments

Comments
 (0)