Add support for overriding github owned storage multipart upload chunk size via environment variable

RELEASENOTES.md

@@ -1 +1 @@
-
+- Allow overriding the multipart upload chunk size when using GitHub owned storage. Set the value of the chunk (min 5Mib) in bytes in a environment variable called GITHUB_OWNED_STORAGE_MULTIPART_BYTES (default value 100Mib). Only set this if you a proxy that doesn't allow the default size upload or if you are on a very slow connection.

src/Octoshift/Services/ArchiveUploader.cs

@@ -15,6 +15,7 @@ public class ArchiveUploader
     private readonly GithubClient _client;
     private readonly OctoLogger _log;
     internal int _streamSizeLimit = 100 * 1024 * 1024; // 100 MiB
+    private const int MIN_MULTIPART_BYTES = 5 * 1024 * 1024; // 5 MiB minimum size for multipart upload. Don't allow overrides smaller than this.
     private readonly RetryPolicy _retryPolicy;
 
     private const string BASE_URL = "https://uploads.github.com";
@@ -24,6 +25,8 @@ public ArchiveUploader(GithubClient client, OctoLogger log, RetryPolicy retryPol
         _client = client;
         _log = log;
         _retryPolicy = retryPolicy;
+
+        SetStreamSizeLimitFromEnvironment();
     }
     public virtual async Task<string> Upload(Stream archiveContent, string archiveName, string orgDatabaseId)
     {
@@ -160,4 +163,22 @@ private Uri GetNextUrl(IEnumerable<KeyValuePair<string, IEnumerable<string>>> he
         }
         throw new OctoshiftCliException("Location header is missing in the response, unable to retrieve next URL for multipart upload.");
     }
+
+    private void SetStreamSizeLimitFromEnvironment()
+    {
+        var envValue = Environment.GetEnvironmentVariable("GITHUB_OWNED_STORAGE_MULTIPART_BYTES");
+        if (!int.TryParse(envValue, out var limit) || limit <= 0)
+        {
+            return;
+        }
+
+        if (limit < MIN_MULTIPART_BYTES)
+        {
+            _log.LogWarning($"GITHUB_OWNED_STORAGE_MULTIPART_BYTES is set to {limit} bytes, but the minimum value is {MIN_MULTIPART_BYTES} bytes. Using default value of {_streamSizeLimit} bytes.");
+            return;
+        }
+
+        _streamSizeLimit = limit;
+        _log.LogInformation($"Stream size limit set to {_streamSizeLimit} bytes.");
+    }
 }

src/OctoshiftCLI.Tests/Octoshift/Services/ArchiveUploadersTests.cs

@@ -11,11 +11,12 @@
 
 namespace OctoshiftCLI.Tests.Octoshift.Services;
 
-public class ArchiveUploaderTests
+public class ArchiveUploaderTests : IDisposable
 {
     private readonly Mock<GithubClient> _githubClientMock;
     private readonly Mock<OctoLogger> _logMock;
     private readonly ArchiveUploader _archiveUploader;
+    private const string ENV_VAR_NAME = "GITHUB_OWNED_STORAGE_MULTIPART_BYTES";
 
     public ArchiveUploaderTests()
     {
@@ -25,6 +26,12 @@
         _archiveUploader = new ArchiveUploader(_githubClientMock.Object, _logMock.Object, retryPolicy);
     }
 
+    public void Dispose()
+    {
+        // Clean up environment variable after each test
+        Environment.SetEnvironmentVariable(ENV_VAR_NAME, null);
+    }
+
     [Fact]
     public async Task Upload_Should_Throw_ArgumentNullException_When_Archive_Content_Is_Null()
     {
@@ -37,6 +44,156 @@
         await Assert.ThrowsAsync<ArgumentNullException>(() => _archiveUploader.Upload(nullStream, archiveName, orgDatabaseId));
     }
 
+    [Fact]
+    public void Constructor_Should_Use_Valid_Environment_Variable_Value()
+    {
+        // Arrange
+        var customSize = 10 * 1024 * 1024; // 10 MiB
+        Environment.SetEnvironmentVariable(ENV_VAR_NAME, customSize.ToString());
+
+        var logMock = TestHelpers.CreateMock<OctoLogger>();
+        var githubClientMock = TestHelpers.CreateMock<GithubClient>();
+        var retryPolicy = new RetryPolicy(logMock.Object);
+
+        // Act
+        var archiveUploader = new ArchiveUploader(githubClientMock.Object, logMock.Object, retryPolicy);
+
+        // Assert
+        archiveUploader._streamSizeLimit.Should().Be(customSize);
+        logMock.Verify(x => x.LogInformation($"Stream size limit set to {customSize} bytes."), Times.Once);
+    }
+
+    [Fact]
+    public void Constructor_Should_Use_Default_When_Environment_Variable_Not_Set()
+    {
+        // Arrange
+        Environment.SetEnvironmentVariable(ENV_VAR_NAME, null);
+        var defaultSize = 100 * 1024 * 1024; // 100 MiB
+
+        var logMock = TestHelpers.CreateMock<OctoLogger>();
+        var githubClientMock = TestHelpers.CreateMock<GithubClient>();
+        var retryPolicy = new RetryPolicy(logMock.Object);
+
+        // Act
+        var archiveUploader = new ArchiveUploader(githubClientMock.Object, logMock.Object, retryPolicy);
+
+        // Assert
+        archiveUploader._streamSizeLimit.Should().Be(defaultSize);
+    }
+
+    [Fact]
+    public void Constructor_Should_Use_Default_When_Environment_Variable_Is_Invalid()
+    {
+        // Arrange
+        Environment.SetEnvironmentVariable(ENV_VAR_NAME, "invalid_value");
+        var defaultSize = 100 * 1024 * 1024; // 100 MiB
+
+        var logMock = TestHelpers.CreateMock<OctoLogger>();
+        var githubClientMock = TestHelpers.CreateMock<GithubClient>();
+        var retryPolicy = new RetryPolicy(logMock.Object);
+
+        // Act
+        var archiveUploader = new ArchiveUploader(githubClientMock.Object, logMock.Object, retryPolicy);
+
+        // Assert
+        archiveUploader._streamSizeLimit.Should().Be(defaultSize);
+    }
+
+    [Fact]
+    public void Constructor_Should_Use_Default_When_Environment_Variable_Is_Zero()
+    {
+        // Arrange
+        Environment.SetEnvironmentVariable(ENV_VAR_NAME, "0");
+        var defaultSize = 100 * 1024 * 1024; // 100 MiB
+
+        var logMock = TestHelpers.CreateMock<OctoLogger>();
+        var githubClientMock = TestHelpers.CreateMock<GithubClient>();
+        var retryPolicy = new RetryPolicy(logMock.Object);
+
+        // Act
+        var archiveUploader = new ArchiveUploader(githubClientMock.Object, logMock.Object, retryPolicy);
+
+        // Assert
+        archiveUploader._streamSizeLimit.Should().Be(defaultSize);
+    }
+
+    [Fact]
+    public void Constructor_Should_Use_Default_When_Environment_Variable_Is_Negative()
+    {
+        // Arrange
+        Environment.SetEnvironmentVariable(ENV_VAR_NAME, "-1000");
+        var defaultSize = 100 * 1024 * 1024; // 100 MiB
+
+        var logMock = TestHelpers.CreateMock<OctoLogger>();
+        var githubClientMock = TestHelpers.CreateMock<GithubClient>();
+        var retryPolicy = new RetryPolicy(logMock.Object);
+
+        // Act
+        var archiveUploader = new ArchiveUploader(githubClientMock.Object, logMock.Object, retryPolicy);
+
+        // Assert
+        archiveUploader._streamSizeLimit.Should().Be(defaultSize);
+    }
+
+    [Fact]
+    public void Constructor_Should_Use_Default_And_Log_Warning_When_Environment_Variable_Below_Minimum()
+    {
+        // Arrange
+        var belowMinimumSize = 1024 * 1024; // 1 MiB (below 5 MiB minimum)
+        var defaultSize = 100 * 1024 * 1024; // 100 MiB
+        var minSize = 5 * 1024 * 1024; // 5 MiB minimum
+        Environment.SetEnvironmentVariable(ENV_VAR_NAME, belowMinimumSize.ToString());
+
+        var logMock = TestHelpers.CreateMock<OctoLogger>();
+        var githubClientMock = TestHelpers.CreateMock<GithubClient>();
+        var retryPolicy = new RetryPolicy(logMock.Object);
+
+        // Act
+        var archiveUploader = new ArchiveUploader(githubClientMock.Object, logMock.Object, retryPolicy);
+
+        // Assert
+        archiveUploader._streamSizeLimit.Should().Be(defaultSize);
+        logMock.Verify(x => x.LogWarning($"GITHUB_OWNED_STORAGE_MULTIPART_BYTES is set to {belowMinimumSize} bytes, but the minimum value is {minSize} bytes. Using default value of {defaultSize} bytes."), Times.Once);
+    }
+
+    [Fact]
+    public void Constructor_Should_Accept_Value_Equal_To_Minimum()
+    {
+        // Arrange
+        var minimumSize = 5 * 1024 * 1024; // 5 MiB minimum
+        Environment.SetEnvironmentVariable(ENV_VAR_NAME, minimumSize.ToString());
+
+        var logMock = TestHelpers.CreateMock<OctoLogger>();
+        var githubClientMock = TestHelpers.CreateMock<GithubClient>();
+        var retryPolicy = new RetryPolicy(logMock.Object);
+
+        // Act
+        var archiveUploader = new ArchiveUploader(githubClientMock.Object, logMock.Object, retryPolicy);
+
+        // Assert
+        archiveUploader._streamSizeLimit.Should().Be(minimumSize);
+        logMock.Verify(x => x.LogInformation($"Stream size limit set to {minimumSize} bytes."), Times.Once);
+    }
+
+    [Fact]
+    public void Constructor_Should_Accept_Large_Valid_Value()
+    {
+        // Arrange
+        var largeSize = 500 * 1024 * 1024; // 500 MiB
+        Environment.SetEnvironmentVariable(ENV_VAR_NAME, largeSize.ToString());
+
+        var logMock = TestHelpers.CreateMock<OctoLogger>();
+        var githubClientMock = TestHelpers.CreateMock<GithubClient>();
+        var retryPolicy = new RetryPolicy(logMock.Object);
+
+        // Act
+        var archiveUploader = new ArchiveUploader(githubClientMock.Object, logMock.Object, retryPolicy);
+
+        // Assert
+        archiveUploader._streamSizeLimit.Should().Be(largeSize);
+        logMock.Verify(x => x.LogInformation($"Stream size limit set to {largeSize} bytes."), Times.Once);
+    }
+
     [Fact]
     public async Task Upload_Should_Upload_All_Chunks_When_Stream_Exceeds_Limit()
     {
@@ -128,8 +285,7 @@
             .ThrowsAsync(new TimeoutException("The operation was canceled."))
             .ThrowsAsync(new TimeoutException("The operation was canceled."))
             .ReturnsAsync((It.IsAny<string>(), new[] { new KeyValuePair<string, IEnumerable<string>>("Location", [secondUploadUrl]) }));
-
-        _githubClientMock // second PATCH request
+ _githubClientMock // second PATCH request
             .Setup(m => m.PatchWithFullResponseAsync($"{baseUrl}{secondUploadUrl}",
                 It.Is<HttpContent>(x => x.ReadAsByteArrayAsync().Result.ToJson() == new byte[] { 3 }.ToJson()), null))
             .ReturnsAsync((It.IsAny<string>(), new[] { new KeyValuePair<string, IEnumerable<string>>("Location", [lastUrl]) }));
@@ -183,7 +339,7 @@
         _githubClientMock // first PATCH request
             .Setup(m => m.PatchWithFullResponseAsync($"{baseUrl}{firstUploadUrl}",
                 It.Is<HttpContent>(x => x.ReadAsByteArrayAsync().Result.ToJson() == new byte[] { 1, 2 }.ToJson()), null))
-            .ReturnsAsync((It.IsAny<string>(), new[] { new KeyValuePair<string, IEnumerable<string>>("Location", [secondUploadUrl]) }));
+                    .ReturnsAsync((It.IsAny<string>(), new[] { new KeyValuePair<string, IEnumerable<string>>("Location", [secondUploadUrl]) }));
 
         _githubClientMock // second PATCH request
             .Setup(m => m.PatchWithFullResponseAsync($"{baseUrl}{secondUploadUrl}",
@@ -237,9 +393,9 @@
         _githubClientMock // first PATCH request
             .Setup(m => m.PatchWithFullResponseAsync($"{baseUrl}{firstUploadUrl}",
                 It.Is<HttpContent>(x => x.ReadAsByteArrayAsync().Result.ToJson() == new byte[] { 1, 2 }.ToJson()), null))
-            .ReturnsAsync((It.IsAny<string>(), new[] { new KeyValuePair<string, IEnumerable<string>>("Location", [secondUploadUrl]) }));
+                    .ReturnsAsync((It.IsAny<string>(), new[] { new KeyValuePair<string, IEnumerable<string>>("Location", [secondUploadUrl]) }));
 
-        _githubClientMock // second PATCH request
+ _githubClientMock // second PATCH request
             .Setup(m => m.PatchWithFullResponseAsync($"{baseUrl}{secondUploadUrl}",
                 It.Is<HttpContent>(x => x.ReadAsByteArrayAsync().Result.ToJson() == new byte[] { 3 }.ToJson()), null))
             .ReturnsAsync((It.IsAny<string>(), new[] { new KeyValuePair<string, IEnumerable<string>>("Location", [lastUrl]) }));