Add support for resuming SFTP file upload/download

src/Renci.SshNet/IServiceFactory.cs

@@ -90,10 +90,11 @@ internal partial interface IServiceFactory
         /// <param name="fileName">The file to read.</param>
         /// <param name="sftpSession">The SFTP session to use.</param>
         /// <param name="bufferSize">The size of buffer.</param>
+        /// <param name="offset">The offset to resume from.</param>
         /// <returns>
         /// An <see cref="ISftpFileReader"/>.
         /// </returns>
-        ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize);
+        ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize, ulong offset = 0);
 
         /// <summary>
         /// Creates a new <see cref="ISftpResponseFactory"/> instance.

src/Renci.SshNet/ServiceFactory.cs

@@ -127,10 +127,11 @@ public INetConfSession CreateNetConfSession(ISession session, int operationTimeo
         /// <param name="fileName">The file to read.</param>
         /// <param name="sftpSession">The SFTP session to use.</param>
         /// <param name="bufferSize">The size of buffer.</param>
+        /// <param name="offset">The offset to resume from.</param>
         /// <returns>
         /// An <see cref="ISftpFileReader"/>.
         /// </returns>
-        public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize)
+        public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize, ulong offset = 0)
         {
             const int defaultMaxPendingReads = 10;
 
@@ -152,7 +153,9 @@ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSe
             {
                 var fileAttributes = sftpSession.EndLStat(statAsyncResult);
                 fileSize = fileAttributes.Size;
-                maxPendingReads = Math.Min(100, (int)Math.Ceiling((double)fileAttributes.Size / chunkSize) + 1);
+
+                // calculate maxPendingReads based on remaining size, not total filesize (needed for resume support)
+                maxPendingReads = Math.Min(100, (int) Math.Ceiling((double)(fileSize - (long)offset) / chunkSize) + 1);
             }
             catch (SshException ex)
             {
@@ -162,7 +165,7 @@ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSe
                 DiagnosticAbstraction.Log(string.Format("Failed to obtain size of file. Allowing maximum {0} pending reads: {1}", maxPendingReads, ex));
             }
 
-            return sftpSession.CreateFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize);
+            return sftpSession.CreateFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize, offset);
         }
 
         /// <summary>

src/Renci.SshNet/Sftp/ISftpSession.cs

@@ -494,10 +494,11 @@ void RequestWrite(byte[] handle,
         /// <param name="chunkSize">The maximum number of bytes to read with each chunk.</param>
         /// <param name="maxPendingReads">The maximum number of pending reads.</param>
         /// <param name="fileSize">The size of the file or <see langword="null"/> when the size could not be determined.</param>
+        /// <param name="offset">The offset to resume from.</param>
         /// <returns>
         /// An <see cref="ISftpFileReader"/> for reading the content of the file represented by the
         /// specified <paramref name="handle"/>.
         /// </returns>
-        ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize);
+        ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize, ulong offset = 0);
     }
 }

src/Renci.SshNet/Sftp/SftpFileReader.cs

@@ -56,7 +56,8 @@ internal sealed class SftpFileReader : ISftpFileReader
         /// <param name="chunkSize">The size of a individual read-ahead chunk.</param>
         /// <param name="maxPendingReads">The maximum number of pending reads.</param>
         /// <param name="fileSize">The size of the file, if known; otherwise, <see langword="null"/>.</param>
-        public SftpFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize)
+        /// <param name="offset">The offset to resume from.</param>
+        public SftpFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize, ulong offset = 0)
         {
             _handle = handle;
             _sftpSession = sftpSession;
@@ -69,6 +70,11 @@ public SftpFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, i
             _disposingWaitHandle = new ManualResetEvent(initialState: false);
             _waitHandles = _sftpSession.CreateWaitHandleArray(_disposingWaitHandle, _semaphore.AvailableWaitHandle);
 
+            // When resuming a download (offset > 0), set the initial offset of the remote file to
+            // the same offset as the local output file. Read-ahead also starts at the same offset.
+            _offset = offset;
+            _readAheadOffset = offset;
+
             StartReadAhead();
         }
 

src/Renci.SshNet/Sftp/SftpSession.cs

@@ -239,13 +239,14 @@ public async Task<string> GetCanonicalPathAsync(string path, CancellationToken c
         /// <param name="chunkSize">The maximum number of bytes to read with each chunk.</param>
         /// <param name="maxPendingReads">The maximum number of pending reads.</param>
         /// <param name="fileSize">The size of the file or <see langword="null"/> when the size could not be determined.</param>
+        /// <param name="offset">The offset to resume from.</param>
         /// <returns>
         /// An <see cref="ISftpFileReader"/> for reading the content of the file represented by the
         /// specified <paramref name="handle"/>.
         /// </returns>
-        public ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize)
+        public ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize, ulong offset = 0)
         {
-            return new SftpFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize);
+            return new SftpFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize, offset);
         }
 
         internal string GetFullRemotePath(string path)

src/Renci.SshNet/SftpClient.cs

@@ -979,7 +979,12 @@ public void UploadFile(Stream input, string path, bool canOverride, Action<ulong
 
             var flags = Flags.Write | Flags.Truncate;
 
-            if (canOverride)
+            if (input.Position > 0)
+            {
+                // if the local stream position is not zero, open the remote file in APPEND mode to resume upload
+                flags = Flags.Write | Flags.Append;
+            }
+            else if (canOverride)
             {
                 flags |= Flags.CreateNewOrOpen;
             }
@@ -1117,7 +1122,12 @@ public IAsyncResult BeginUploadFile(Stream input, string path, bool canOverride,
 
             var flags = Flags.Write | Flags.Truncate;
 
-            if (canOverride)
+            if (input.Position > 0)
+            {
+                // if the local stream position is not zero, open the remote file in APPEND mode to resume upload
+                flags = Flags.Write | Flags.Append;
+            }
+            else if (canOverride)
             {
                 flags |= Flags.CreateNewOrOpen;
             }
@@ -2432,7 +2442,8 @@ private void InternalUploadFile(Stream input, string path, Flags flags, SftpUplo
 
             var handle = _sftpSession.RequestOpen(fullPath, flags);
 
-            ulong offset = 0;
+            // Set the initial offset of the remote file to the same as the local file to allow resuming
+            var offset = (ulong)input.Position;
 
             // create buffer of optimal length
             var buffer = new byte[_sftpSession.CalculateOptimalWriteLength(_bufferSize, handle)];

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_EndLStatThrowsSshException.cs

@@ -62,7 +62,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Throws(new SshException());
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 10, null))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 10, null, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsAlmostSixTimesGreaterThanChunkSize.cs

@@ -66,7 +66,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsEqualToChunkSize.cs

@@ -66,7 +66,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsExactlyFiveTimesGreaterThanChunkSize.cs

@@ -66,7 +66,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 6, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 6, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLessThanChunkSize.cs

@@ -66,7 +66,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLittleMoreThanFiveTimesGreaterThanChunkSize.cs

@@ -66,7 +66,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanMaxPendingReadsTimesChunkSize.cs

@@ -68,7 +68,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, _maxPendingReads, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, _maxPendingReads, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsZero.cs

@@ -66,7 +66,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 1, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 1, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }