Build the read-ahead mechanism into SftpFileStream (#1705)

* Build the read-ahead mechanism into SftpFileStream

This change unifies the SFTP download implementations that exist via DownloadFile and
via SftpFileStream, by rewriting SftpFileStream to perform the same "read-aheads" as
DownloadFile. This brings the performance of downloads via SftpFileStream in line with
DownloadFile, such that the latter is now effectively SftpFileStream.CopyTo. It also
brings the recently added DownloadFileAsync up to speed since that was implemented via
SftpFileStream.CopyToAsync.

The methodology is a mix of the previous one and that within OpenSSH: the first call to
SftpFileStream.Read sends one read request to the server. The second sends two and when
not interrupted by Write or similar, the number of in-flight read requests continues to
scale up in this fashion.

I have measured CopyTo to be 3-20x faster than before, depending on file size and server
round-trip time.

* Check CanSeek in ReadAllBytes

* Squeeze out some performance

Author: Rob-Hague

src/Renci.SshNet/Common/Extensions.cs

@@ -3,6 +3,7 @@
 using System.Globalization;
 #if !NET
 using System.IO;
+using System.Threading.Tasks;
 #endif
 using System.Net;
 using System.Net.Sockets;
@@ -398,6 +399,29 @@ internal static void ReadExactly(this Stream stream, byte[] buffer, int offset,
                 totalRead += read;
             }
         }
+
+        internal static Task<T> WaitAsync<T>(this Task<T> task, CancellationToken cancellationToken)
+        {
+            if (task.IsCompleted || !cancellationToken.CanBeCanceled)
+            {
+                return task;
+            }
+
+            return WaitCore();
+
+            async Task<T> WaitCore()
+            {
+                TaskCompletionSource<T> tcs = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+                using var reg = cancellationToken.Register(
+                    () => tcs.TrySetCanceled(cancellationToken),
+                    useSynchronizationContext: false);
+
+                var completedTask = await Task.WhenAny(task, tcs.Task).ConfigureAwait(false);
+
+                return await completedTask.ConfigureAwait(false);
+            }
+        }
 #endif
     }
 }

src/Renci.SshNet/IServiceFactory.cs

@@ -83,18 +83,6 @@ internal partial interface IServiceFactory
         /// <exception cref="SshConnectionException">No key exchange algorithm is supported by both client and server.</exception>
         IKeyExchange CreateKeyExchange(IDictionary<string, Func<IKeyExchange>> clientAlgorithms, string[] serverAlgorithms);
 
-        /// <summary>
-        /// Creates an <see cref="ISftpFileReader"/> for the specified file and with the specified
-        /// buffer size.
-        /// </summary>
-        /// <param name="fileName">The file to read.</param>
-        /// <param name="sftpSession">The SFTP session to use.</param>
-        /// <param name="bufferSize">The size of buffer.</param>
-        /// <returns>
-        /// An <see cref="ISftpFileReader"/>.
-        /// </returns>
-        ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize);
-
         /// <summary>
         /// Creates a new <see cref="ISftpResponseFactory"/> instance.
         /// </summary>

src/Renci.SshNet/ServiceFactory.cs

@@ -4,8 +4,6 @@
 using System.Net.Sockets;
 using System.Text;
 
-using Microsoft.Extensions.Logging;
-
 using Renci.SshNet.Common;
 using Renci.SshNet.Connection;
 using Renci.SshNet.Messages.Transport;
@@ -118,51 +116,6 @@ public INetConfSession CreateNetConfSession(ISession session, int operationTimeo
             return new NetConfSession(session, operationTimeout);
         }
 
-        /// <summary>
-        /// Creates an <see cref="ISftpFileReader"/> for the specified file and with the specified
-        /// buffer size.
-        /// </summary>
-        /// <param name="fileName">The file to read.</param>
-        /// <param name="sftpSession">The SFTP session to use.</param>
-        /// <param name="bufferSize">The size of buffer.</param>
-        /// <returns>
-        /// An <see cref="ISftpFileReader"/>.
-        /// </returns>
-        public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize)
-        {
-            const int defaultMaxPendingReads = 10;
-
-            // Issue #292: Avoid overlapping SSH_FXP_OPEN and SSH_FXP_LSTAT requests for the same file as this
-            // causes a performance degradation on Sun SSH
-            var openAsyncResult = sftpSession.BeginOpen(fileName, Flags.Read, callback: null, state: null);
-            var handle = sftpSession.EndOpen(openAsyncResult);
-
-            var statAsyncResult = sftpSession.BeginLStat(fileName, callback: null, state: null);
-
-            long? fileSize;
-            int maxPendingReads;
-
-            var chunkSize = sftpSession.CalculateOptimalReadLength(bufferSize);
-
-            // fallback to a default maximum of pending reads when remote server does not allow us to obtain
-            // the attributes of the file
-            try
-            {
-                var fileAttributes = sftpSession.EndLStat(statAsyncResult);
-                fileSize = fileAttributes.Size;
-                maxPendingReads = Math.Min(100, (int)Math.Ceiling((double)fileAttributes.Size / chunkSize) + 1);
-            }
-            catch (SshException ex)
-            {
-                fileSize = null;
-                maxPendingReads = defaultMaxPendingReads;
-
-                sftpSession.SessionLoggerFactory.CreateLogger<ServiceFactory>().LogInformation(ex, "Failed to obtain size of file. Allowing maximum {MaxPendingReads} pending reads", maxPendingReads);
-            }
-
-            return sftpSession.CreateFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize);
-        }
-
         /// <summary>
         /// Creates a new <see cref="ISftpResponseFactory"/> instance.
         /// </summary>

src/Renci.SshNet/Sftp/ISftpFileReader.cs

@@ -67,11 +67,10 @@ internal interface ISftpSession : ISubsystemSession
         /// Asynchronously performs a <c>SSH_FXP_FSTAT</c> request.
         /// </summary>
         /// <param name="handle">The handle.</param>
-        /// <param name="nullOnError">If set to <see langword="true"/>, <see langword="null"/> is returned in case of an error.</param>
         /// <returns>
         /// The file attributes.
         /// </returns>
-        SftpFileAttributes RequestFStat(byte[] handle, bool nullOnError);
+        SftpFileAttributes RequestFStat(byte[] handle);
 
         /// <summary>
         /// Asynchronously performs a <c>SSH_FXP_FSTAT</c> request.
@@ -522,19 +521,5 @@ void RequestWrite(byte[] handle,
         /// Currently, we do not take the remote window size into account.
         /// </remarks>
         uint CalculateOptimalWriteLength(uint bufferSize, byte[] handle);
-
-        /// <summary>
-        /// Creates an <see cref="ISftpFileReader"/> for reading the content of the file represented by a given <paramref name="handle"/>.
-        /// </summary>
-        /// <param name="handle">The handle of the file to read.</param>
-        /// <param name="sftpSession">The SFTP session.</param>
-        /// <param name="chunkSize">The maximum number of bytes to read with each chunk.</param>
-        /// <param name="maxPendingReads">The maximum number of pending reads.</param>
-        /// <param name="fileSize">The size of the file or <see langword="null"/> when the size could not be determined.</param>
-        /// <returns>
-        /// An <see cref="ISftpFileReader"/> for reading the content of the file represented by the
-        /// specified <paramref name="handle"/>.
-        /// </returns>
-        ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize);
     }
 }