From 43b60cf4ab4744314be59c1746b927f7ba8f46d9 Mon Sep 17 00:00:00 2001 From: wakabayashik Date: Tue, 3 Sep 2019 13:46:17 +0900 Subject: [PATCH 1/3] improve rate control for high level API --- .../Client/FtpClient_HighLevelDownload.cs | 54 ++++++++++++++----- FluentFTP/Client/FtpClient_HighLevelUpload.cs | 54 ++++++++++++++----- 2 files changed, 84 insertions(+), 24 deletions(-) diff --git a/FluentFTP/Client/FtpClient_HighLevelDownload.cs b/FluentFTP/Client/FtpClient_HighLevelDownload.cs index f03fc4967..84ab0d8df 100644 --- a/FluentFTP/Client/FtpClient_HighLevelDownload.cs +++ b/FluentFTP/Client/FtpClient_HighLevelDownload.cs @@ -620,13 +620,27 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest // we read until EOF instead of reading a specific number of bytes var readToEnd = fileLen <= 0; + const int rateControlResolution = 100; + const int chunkSizeMin = 64; + var rateLimitBytes = DownloadRateLimit != 0 ? (long)DownloadRateLimit * 1024 : 0; + var chunkSize = Math.Max(TransferChunkSize, chunkSizeMin); + if (rateLimitBytes > 0) { + // reduce chunk size to optimize rate control + while (chunkSize > chunkSizeMin) { + var chunkLenInMs = 1000L * chunkSize / rateLimitBytes; + if (chunkLenInMs <= rateControlResolution) { + break; + } + chunkSize = Math.Max(chunkSize >> 1, chunkSizeMin); + } + } + // loop till entire file downloaded - var buffer = new byte[TransferChunkSize]; + var buffer = new byte[chunkSize]; var offset = restartPosition; var transferStarted = DateTime.Now; var sw = new Stopwatch(); - long rateLimitBytes = DownloadRateLimit != 0 ? DownloadRateLimit * 1024 : 0; while (offset < fileLen || readToEnd) { try { // read a chunk of bytes from the FTP stream @@ -649,7 +663,7 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest // honor the rate limit var swTime = (int) sw.ElapsedMilliseconds; - if (rateLimitBytes > 0 && swTime >= 1000) { + if (rateLimitBytes > 0) { var timeShouldTake = limitCheckBytes / rateLimitBytes * 1000; if (timeShouldTake > swTime) { #if CORE14 @@ -658,9 +672,10 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest Thread.Sleep((int) (timeShouldTake - swTime)); #endif } - - limitCheckBytes = 0; - sw.Restart(); + else if (swTime > timeShouldTake + rateControlResolution) { + limitCheckBytes = 0; + sw.Restart(); + } } } @@ -738,13 +753,27 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest // we read until EOF instead of reading a specific number of bytes var readToEnd = fileLen <= 0; + const int rateControlResolution = 100; + const int chunkSizeMin = 64; + var rateLimitBytes = DownloadRateLimit != 0 ? (long)DownloadRateLimit * 1024 : 0; + var chunkSize = Math.Max(TransferChunkSize, chunkSizeMin); + if (rateLimitBytes > 0) { + // reduce chunk size to optimize rate control + while (chunkSize > chunkSizeMin) { + var chunkLenInMs = 1000L * chunkSize / rateLimitBytes; + if (chunkLenInMs <= rateControlResolution) { + break; + } + chunkSize = Math.Max(chunkSize >> 1, chunkSizeMin); + } + } + // loop till entire file downloaded - var buffer = new byte[TransferChunkSize]; + var buffer = new byte[chunkSize]; var offset = restartPosition; var transferStarted = DateTime.Now; var sw = new Stopwatch(); - long rateLimitBytes = DownloadRateLimit != 0 ? DownloadRateLimit * 1024 : 0; while (offset < fileLen || readToEnd) { try { @@ -768,15 +797,16 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest // honor the rate limit var swTime = (int) sw.ElapsedMilliseconds; - if (rateLimitBytes > 0 && swTime >= 1000) { + if (rateLimitBytes > 0) { var timeShouldTake = limitCheckBytes / rateLimitBytes * 1000; if (timeShouldTake > swTime) { await Task.Delay((int) (timeShouldTake - swTime), token); token.ThrowIfCancellationRequested(); } - - limitCheckBytes = 0; - sw.Restart(); + else if (swTime > timeShouldTake + rateControlResolution) { + limitCheckBytes = 0; + sw.Restart(); + } } } diff --git a/FluentFTP/Client/FtpClient_HighLevelUpload.cs b/FluentFTP/Client/FtpClient_HighLevelUpload.cs index be7a37b8f..d79a98c69 100644 --- a/FluentFTP/Client/FtpClient_HighLevelUpload.cs +++ b/FluentFTP/Client/FtpClient_HighLevelUpload.cs @@ -683,13 +683,27 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR upStream = OpenAppend(remotePath, UploadDataType, checkFileExistsAgain); } + const int rateControlResolution = 100; + const int chunkSizeMin = 64; + var rateLimitBytes = UploadRateLimit != 0 ? (long)UploadRateLimit * 1024 : 0; + var chunkSize = Math.Max(TransferChunkSize, chunkSizeMin); + if (rateLimitBytes > 0) { + // reduce chunk size to optimize rate control + while (chunkSize > chunkSizeMin) { + var chunkLenInMs = 1000L * chunkSize / rateLimitBytes; + if (chunkLenInMs <= rateControlResolution) { + break; + } + chunkSize = Math.Max(chunkSize >> 1, chunkSizeMin); + } + } + // loop till entire file uploaded var fileLen = fileData.Length; - var buffer = new byte[TransferChunkSize]; + var buffer = new byte[chunkSize]; var transferStarted = DateTime.Now; var sw = new Stopwatch(); - long rateLimitBytes = UploadRateLimit != 0 ? UploadRateLimit * 1024 : 0; // Fix #288 - Upload hangs with only a few bytes left if (fileLen < upStream.Length) { @@ -719,7 +733,7 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR // honor the speed limit var swTime = (int) sw.ElapsedMilliseconds; - if (rateLimitBytes > 0 && swTime >= 1000) { + if (rateLimitBytes > 0) { var timeShouldTake = limitCheckBytes / rateLimitBytes * 1000; if (timeShouldTake > swTime) { #if CORE14 @@ -728,9 +742,10 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR Thread.Sleep((int) (timeShouldTake - swTime)); #endif } - - limitCheckBytes = 0; - sw.Restart(); + else if (swTime > timeShouldTake + rateControlResolution) { + limitCheckBytes = 0; + sw.Restart(); + } } } @@ -877,13 +892,27 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR upStream = await OpenAppendAsync(remotePath, UploadDataType, checkFileExistsAgain, token); } + const int rateControlResolution = 100; + const int chunkSizeMin = 64; + var rateLimitBytes = UploadRateLimit != 0 ? (long)UploadRateLimit * 1024 : 0; + var chunkSize = Math.Max(TransferChunkSize, chunkSizeMin); + if (rateLimitBytes > 0) { + // reduce chunk size to optimize rate control + while (chunkSize > chunkSizeMin) { + var chunkLenInMs = 1000L * chunkSize / rateLimitBytes; + if (chunkLenInMs <= rateControlResolution) { + break; + } + chunkSize = Math.Max(chunkSize >> 1, chunkSizeMin); + } + } + // loop till entire file uploaded var fileLen = fileData.Length; - var buffer = new byte[TransferChunkSize]; + var buffer = new byte[chunkSize]; var transferStarted = DateTime.Now; var sw = new Stopwatch(); - long rateLimitBytes = UploadRateLimit != 0 ? UploadRateLimit * 1024 : 0; // Fix #288 - Upload hangs with only a few bytes left if (fileLen < upStream.Length) { @@ -913,15 +942,16 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR // honor the rate limit var swTime = (int) sw.ElapsedMilliseconds; - if (rateLimitBytes > 0 && swTime >= 1000) { + if (rateLimitBytes > 0) { var timeShouldTake = limitCheckBytes / rateLimitBytes * 1000; if (timeShouldTake > swTime) { await Task.Delay((int) (timeShouldTake - swTime), token); token.ThrowIfCancellationRequested(); } - - limitCheckBytes = 0; - sw.Restart(); + else if (swTime > timeShouldTake + rateControlResolution) { + limitCheckBytes = 0; + sw.Restart(); + } } } From f5375bcea7593b35af34c2fd3ace2ea5e77cada0 Mon Sep 17 00:00:00 2001 From: wakabayashik Date: Thu, 5 Sep 2019 11:03:20 +0900 Subject: [PATCH 2/3] fixed potential bugs which might occur 24 days after from beginning of the transfer. (int overflow) --- FluentFTP/Client/FtpClient_HighLevelDownload.cs | 12 ++++++------ FluentFTP/Client/FtpClient_HighLevelUpload.cs | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/FluentFTP/Client/FtpClient_HighLevelDownload.cs b/FluentFTP/Client/FtpClient_HighLevelDownload.cs index 84ab0d8df..228f7e483 100644 --- a/FluentFTP/Client/FtpClient_HighLevelDownload.cs +++ b/FluentFTP/Client/FtpClient_HighLevelDownload.cs @@ -645,7 +645,7 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest try { // read a chunk of bytes from the FTP stream var readBytes = 1; - double limitCheckBytes = 0; + long limitCheckBytes = 0; long bytesProcessed = 0; sw.Start(); @@ -662,9 +662,9 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest } // honor the rate limit - var swTime = (int) sw.ElapsedMilliseconds; + var swTime = sw.ElapsedMilliseconds; if (rateLimitBytes > 0) { - var timeShouldTake = limitCheckBytes / rateLimitBytes * 1000; + var timeShouldTake = limitCheckBytes * 1000 / rateLimitBytes; if (timeShouldTake > swTime) { #if CORE14 Task.Delay((int) (timeShouldTake - swTime)).Wait(); @@ -779,7 +779,7 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest try { // read a chunk of bytes from the FTP stream var readBytes = 1; - double limitCheckBytes = 0; + long limitCheckBytes = 0; long bytesProcessed = 0; sw.Start(); @@ -796,9 +796,9 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest } // honor the rate limit - var swTime = (int) sw.ElapsedMilliseconds; + var swTime = sw.ElapsedMilliseconds; if (rateLimitBytes > 0) { - var timeShouldTake = limitCheckBytes / rateLimitBytes * 1000; + var timeShouldTake = limitCheckBytes * 1000 / rateLimitBytes; if (timeShouldTake > swTime) { await Task.Delay((int) (timeShouldTake - swTime), token); token.ThrowIfCancellationRequested(); diff --git a/FluentFTP/Client/FtpClient_HighLevelUpload.cs b/FluentFTP/Client/FtpClient_HighLevelUpload.cs index d79a98c69..a335b7bf3 100644 --- a/FluentFTP/Client/FtpClient_HighLevelUpload.cs +++ b/FluentFTP/Client/FtpClient_HighLevelUpload.cs @@ -714,7 +714,7 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR try { // read a chunk of bytes from the file int readBytes; - double limitCheckBytes = 0; + long limitCheckBytes = 0; long bytesProcessed = 0; sw.Start(); @@ -732,9 +732,9 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR } // honor the speed limit - var swTime = (int) sw.ElapsedMilliseconds; + var swTime = sw.ElapsedMilliseconds; if (rateLimitBytes > 0) { - var timeShouldTake = limitCheckBytes / rateLimitBytes * 1000; + var timeShouldTake = limitCheckBytes * 1000 / rateLimitBytes; if (timeShouldTake > swTime) { #if CORE14 Task.Delay((int) (timeShouldTake - swTime)).Wait(); @@ -923,7 +923,7 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR try { // read a chunk of bytes from the file int readBytes; - double limitCheckBytes = 0; + long limitCheckBytes = 0; long bytesProcessed = 0; sw.Start(); @@ -941,9 +941,9 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR } // honor the rate limit - var swTime = (int) sw.ElapsedMilliseconds; + var swTime = sw.ElapsedMilliseconds; if (rateLimitBytes > 0) { - var timeShouldTake = limitCheckBytes / rateLimitBytes * 1000; + var timeShouldTake = limitCheckBytes * 1000 / rateLimitBytes; if (timeShouldTake > swTime) { await Task.Delay((int) (timeShouldTake - swTime), token); token.ThrowIfCancellationRequested(); From f0fd156b3a35abfd0d610aa23afb2df183564ae3 Mon Sep 17 00:00:00 2001 From: wakabayashik Date: Thu, 5 Sep 2019 12:05:44 +0900 Subject: [PATCH 3/3] modified so that chunk size is adjusted only if TransferChunkSize is omitted --- FluentFTP/Client/FtpClient_HighLevelDownload.cs | 12 ++++++------ FluentFTP/Client/FtpClient_HighLevelUpload.cs | 12 ++++++------ FluentFTP/Client/FtpClient_Properties.cs | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/FluentFTP/Client/FtpClient_HighLevelDownload.cs b/FluentFTP/Client/FtpClient_HighLevelDownload.cs index 228f7e483..1e5797f87 100644 --- a/FluentFTP/Client/FtpClient_HighLevelDownload.cs +++ b/FluentFTP/Client/FtpClient_HighLevelDownload.cs @@ -621,11 +621,11 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest var readToEnd = fileLen <= 0; const int rateControlResolution = 100; - const int chunkSizeMin = 64; var rateLimitBytes = DownloadRateLimit != 0 ? (long)DownloadRateLimit * 1024 : 0; - var chunkSize = Math.Max(TransferChunkSize, chunkSizeMin); - if (rateLimitBytes > 0) { + var chunkSize = TransferChunkSize; + if (m_transferChunkSize == null && rateLimitBytes > 0) { // reduce chunk size to optimize rate control + const int chunkSizeMin = 64; while (chunkSize > chunkSizeMin) { var chunkLenInMs = 1000L * chunkSize / rateLimitBytes; if (chunkLenInMs <= rateControlResolution) { @@ -754,11 +754,11 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest var readToEnd = fileLen <= 0; const int rateControlResolution = 100; - const int chunkSizeMin = 64; var rateLimitBytes = DownloadRateLimit != 0 ? (long)DownloadRateLimit * 1024 : 0; - var chunkSize = Math.Max(TransferChunkSize, chunkSizeMin); - if (rateLimitBytes > 0) { + var chunkSize = TransferChunkSize; + if (m_transferChunkSize == null && rateLimitBytes > 0) { // reduce chunk size to optimize rate control + const int chunkSizeMin = 64; while (chunkSize > chunkSizeMin) { var chunkLenInMs = 1000L * chunkSize / rateLimitBytes; if (chunkLenInMs <= rateControlResolution) { diff --git a/FluentFTP/Client/FtpClient_HighLevelUpload.cs b/FluentFTP/Client/FtpClient_HighLevelUpload.cs index a335b7bf3..adf128355 100644 --- a/FluentFTP/Client/FtpClient_HighLevelUpload.cs +++ b/FluentFTP/Client/FtpClient_HighLevelUpload.cs @@ -684,11 +684,11 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR } const int rateControlResolution = 100; - const int chunkSizeMin = 64; var rateLimitBytes = UploadRateLimit != 0 ? (long)UploadRateLimit * 1024 : 0; - var chunkSize = Math.Max(TransferChunkSize, chunkSizeMin); - if (rateLimitBytes > 0) { + var chunkSize = TransferChunkSize; + if (m_transferChunkSize == null && rateLimitBytes > 0) { // reduce chunk size to optimize rate control + const int chunkSizeMin = 64; while (chunkSize > chunkSizeMin) { var chunkLenInMs = 1000L * chunkSize / rateLimitBytes; if (chunkLenInMs <= rateControlResolution) { @@ -893,11 +893,11 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR } const int rateControlResolution = 100; - const int chunkSizeMin = 64; var rateLimitBytes = UploadRateLimit != 0 ? (long)UploadRateLimit * 1024 : 0; - var chunkSize = Math.Max(TransferChunkSize, chunkSizeMin); - if (rateLimitBytes > 0) { + var chunkSize = TransferChunkSize; + if (m_transferChunkSize == null && rateLimitBytes > 0) { // reduce chunk size to optimize rate control + const int chunkSizeMin = 64; while (chunkSize > chunkSizeMin) { var chunkLenInMs = 1000L * chunkSize / rateLimitBytes; if (chunkLenInMs <= rateControlResolution) { diff --git a/FluentFTP/Client/FtpClient_Properties.cs b/FluentFTP/Client/FtpClient_Properties.cs index 7ec567fb9..668eb018b 100644 --- a/FluentFTP/Client/FtpClient_Properties.cs +++ b/FluentFTP/Client/FtpClient_Properties.cs @@ -644,7 +644,7 @@ public int BulkListingLength { } - private int m_transferChunkSize = 65536; + private int? m_transferChunkSize; /// /// Gets or sets the number of bytes transferred in a single chunk (a single FTP command). @@ -652,7 +652,7 @@ public int BulkListingLength { /// to transfer large files in multiple chunks. /// public int TransferChunkSize { - get => m_transferChunkSize; + get => m_transferChunkSize ?? 65536; set => m_transferChunkSize = value; }