Upload: Auto limit max chunk size on HTTP-413

Signed-off-by: Juergen Kellerer <juergen@k123.eu>

src/libsync/owncloudpropagator.cpp

@@ -673,7 +673,8 @@ const SyncOptions &OwncloudPropagator::syncOptions() const
 void OwncloudPropagator::setSyncOptions(const SyncOptions &syncOptions)
 {
     _syncOptions = syncOptions;
-    _chunkSize = syncOptions._initialChunkSize;
+    _syncOptions.verifyChunkSizes();
+    _chunkSize = _syncOptions._initialChunkSize;
 }
 
 bool OwncloudPropagator::localFileNameClash(const QString &relFile)

src/libsync/propagateupload.cpp

@@ -676,6 +676,31 @@ void PropagateUploadFileCommon::commonErrorHandling(AbstractNetworkJob *job)
     SyncFileItem::Status status = classifyError(job->reply()->error(), _item->_httpErrorCode,
         &propagator()->_anotherSyncNeeded, replyContent);
 
+    // Request entity to large: Chunk size is larger than supported by the server
+    if (_item->_httpErrorCode == 413 && _item->_requestBodySize > 0 && status == SyncFileItem::NormalError) {
+        auto opts = propagator()->syncOptions();
+        auto chunkSize = _item->_requestBodySize;
+
+        // Set new bound as global limit (we need to guess here, as sent bytes are no good indicator what is allowed)
+        auto maxChunkSize = chunkSize / 2 + qMax(qint64(0), opts._minChunkSize) / 2;
+        SyncOptions::limitMaxChunkSize(maxChunkSize);
+        opts.verifyChunkSizes();
+
+        // Apply to this propagator
+        propagator()->setSyncOptions(opts);
+        opts = propagator()->syncOptions();
+
+        // Trigger retry
+        if (opts._maxChunkSize < chunkSize) {
+            propagator()->_anotherSyncNeeded = true;
+            status = SyncFileItem::SoftError;
+        }
+
+        errorString = tr("Upload of %1 exceeds the max upload size allowed by the server. Reducing max upload size to %2")
+                        .arg(Utility::octetsToString(chunkSize))
+                        .arg(Utility::octetsToString(opts._maxChunkSize));
+    }
+
     // Insufficient remote storage.
     if (_item->_httpErrorCode == 507) {
         // Update the quota expectation

src/libsync/propagateuploadng.cpp

@@ -386,6 +386,7 @@ void PropagateUploadFileNG::slotPutFinished()
     if (err != QNetworkReply::NoError) {
         _item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
         _item->_requestId = job->requestId();
+        _item->_requestBodySize = job->device()->size();
         commonErrorHandling(job);
         return;
     }

src/libsync/propagateuploadv1.cpp

@@ -215,6 +215,7 @@ void PropagateUploadFileV1::slotPutFinished()
     _item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
     _item->_responseTimeStamp = job->responseTimestamp();
     _item->_requestId = job->requestId();
+    _item->_requestBodySize = job->device()->size();
     QNetworkReply::NetworkError err = job->reply()->error();
     if (err != QNetworkReply::NoError) {
         commonErrorHandling(job);

src/libsync/syncfileitem.h

@@ -268,6 +268,7 @@ class OWNCLOUDSYNC_EXPORT SyncFileItem
     QString _errorString; // Contains a string only in case of error
     QByteArray _responseTimeStamp;
     QByteArray _requestId; // X-Request-Id of the failed request
+    qint64 _requestBodySize = -1; // the number of bytes sent. -1 if unknown.
     quint32 _affectedItems = 1; // the number of affected items by the operation on this item.
     // usually this value is 1, but for removes on dirs, it might be much higher.
 

src/libsync/syncoptions.cpp

@@ -19,6 +19,12 @@
 
 using namespace OCC;
 
+// limit cannot be lower than 512KB
+const qint64 lowestMaxChunkSizeLimit = 512 * 1024;
+
+// initial limit -1 (no limit)
+QAtomicInteger<qint64> SyncOptions::_maxChunkSizeLimit = -1;
+
 SyncOptions::SyncOptions()
     : _vfs(new VfsOff)
 {
@@ -51,10 +57,35 @@ void SyncOptions::fillFromEnvironmentVariables()
 
 void SyncOptions::verifyChunkSizes()
 {
+    qint64 maxUpperLimit = _maxChunkSizeLimit;
+    if (maxUpperLimit > 0) {
+        _initialChunkSize = qMin(maxUpperLimit, _initialChunkSize);
+        _minChunkSize = qMin(maxUpperLimit, _minChunkSize);
+        _maxChunkSize = qMin(maxUpperLimit, _maxChunkSize);
+    }
+
     _minChunkSize = qMin(_minChunkSize, _initialChunkSize);
     _maxChunkSize = qMax(_maxChunkSize, _initialChunkSize);
 }
 
+void SyncOptions::limitMaxChunkSize(qint64 maxChunkSizeLimit)
+{
+    if (maxChunkSizeLimit < 0) {
+        // Disable the limit
+        _maxChunkSizeLimit = -1;
+    } else {
+        // Lower the limit concurrently (but never increase)
+        auto newLimit = qMax(lowestMaxChunkSizeLimit, maxChunkSizeLimit);
+
+        for (auto limit = _maxChunkSizeLimit; ; limit = _maxChunkSizeLimit) {
+            if (newLimit >= limit && limit > 0)
+                break;
+            if (_maxChunkSizeLimit.testAndSetOrdered(limit, newLimit))
+                break;
+        }
+    }
+}
+
 QRegularExpression SyncOptions::fileRegex() const
 {
     return _fileRegex;

src/libsync/syncoptions.h

@@ -17,6 +17,7 @@
 #include "owncloudlib.h"
 #include "common/vfs.h"
 
+#include <QAtomicInteger>
 #include <QRegularExpression>
 #include <QSharedPointer>
 #include <QString>
@@ -88,6 +89,8 @@ class OWNCLOUDSYNC_EXPORT SyncOptions
      */
     void verifyChunkSizes();
 
+    /** Limits the max values of _minChunkSize and _maxChunkSize, set to -1 (default) to disable the limit */
+    static void limitMaxChunkSize(qint64 maxChunkSizeLimit);
 
     /** A regular expression to match file names
      * If no pattern is provided the default is an invalid regular expression.
@@ -105,6 +108,11 @@ class OWNCLOUDSYNC_EXPORT SyncOptions
     void setPathPattern(const QString &pattern);
 
 private:
+    /** Global max value for _minChunkSize & _maxChunkSize
+     * Only applied in verifyChunkSizes() if > 0
+     * */
+    static QAtomicInteger<qint64> _maxChunkSizeLimit;
+
     /**
      * Only sync files that mathc the expression
      * Invalid pattern by default.

translations/client_en.ts

@@ -2432,6 +2432,11 @@ It is not advisable to use it.</source>
         <source>Local file changed during sync.</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <location filename="../src/libsync/propagateupload.cpp" line="687"/>
+        <source>Upload of %1 exceeds the max upload size allowed by the server. Reducing max upload size to %2</source>
+        <translation type="unfinished"></translation>
+    </message>
     <message>
         <location filename="../src/libsync/propagateupload.cpp" line="225"/>
         <location filename="../src/libsync/propagateupload.cpp" line="603"/>