From 7353c36b5a9fb500a133c1051549a962a63ca3a4 Mon Sep 17 00:00:00 2001 From: Vincent Petry Date: Thu, 26 Jul 2018 15:31:12 +0200 Subject: [PATCH] Revert "Revert "[stable10] Retry chunks in web UI on stalled uploads"" --- apps/files/js/app.js | 4 +- apps/files/js/file-upload.js | 135 ++++++++++++++++++---- apps/files/js/filelist.js | 4 +- apps/files/lib/App.php | 6 +- apps/files/tests/js/fileUploadSpec.js | 155 +++++++++++++++++++++++++- 5 files changed, 272 insertions(+), 32 deletions(-) diff --git a/apps/files/js/app.js b/apps/files/js/app.js index a3524036699a..1c23d3745e14 100644 --- a/apps/files/js/app.js +++ b/apps/files/js/app.js @@ -95,7 +95,9 @@ }, config: this._filesConfig, enableUpload: true, - maxChunkSize: OC.appConfig.files && OC.appConfig.files.max_chunk_size + maxChunkSize: OC.appConfig.files && OC.appConfig.files.max_chunk_size, + uploadStallTimeout: OC.appConfig.files && OC.appConfig.files.upload_stall_timeout, + uploadStallRetries: OC.appConfig.files && OC.appConfig.files.upload_stall_retries } ); this.files.initialize(); diff --git a/apps/files/js/file-upload.js b/apps/files/js/file-upload.js index 1d70501f943e..8108a9addc63 100644 --- a/apps/files/js/file-upload.js +++ b/apps/files/js/file-upload.js @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 + * Copyright (c) 2018 * * This file is licensed under the Affero General Public License version 3 * or later. @@ -332,10 +332,22 @@ OC.FileUpload.prototype = { if (this.data.isChunked) { this._deleteChunkFolder(); } + this.data.stalled = false; this.data.abort(); this.deleteUpload(); }, + /** + * retry the upload + */ + retry: function() { + if (!this.data.stalled) { + console.log('Retrying upload ' + this.id); + this.data.stalled = true; + this.data.abort(); + } + }, + /** * Fail the upload */ @@ -484,6 +496,17 @@ OC.Uploader.prototype = _.extend({ */ davClient: null, + /** + * Upload progressbar element + * + * @type Object + */ + $uploadprogressbar: null, + + /** + * @type int + */ + _uploadStallTimeout: 60, /** * Function that will allow us to know if Ajax uploads are supported * @link https://github.com/New-Bamboo/example-ajax-upload/blob/master/public/index.html @@ -806,34 +829,40 @@ OC.Uploader.prototype = _.extend({ var self = this; window.clearInterval(this._progressBarInterval); $('#uploadprogresswrapper .stop').fadeOut(); - $('#uploadprogressbar').fadeOut(function() { + this.$uploadprogressbar.fadeOut(function() { self.$uploadEl.trigger(new $.Event('resized')); }); }, _showProgressBar: function() { - $('#uploadprogressbar').fadeIn(); + this.$uploadprogressbar.fadeIn(); this.$uploadEl.trigger(new $.Event('resized')); if (this._progressBarInterval) { window.clearInterval(this._progressBarInterval); } this._progressBarInterval = window.setInterval(_.bind(this._updateProgressBar, this), 1000); this._lastProgress = 0; - this._lastProgressStalledSeconds = 0; }, - + _updateProgressBar: function() { - var progress = parseInt($('#uploadprogressbar').attr('data-loaded'), 10); - var total = parseInt($('#uploadprogressbar').attr('data-total'), 10); + var progress = parseInt(this.$uploadprogressbar.attr('data-loaded'), 10); + var total = parseInt(this.$uploadprogressbar.attr('data-total'), 10); if (progress !== this._lastProgress) { this._lastProgress = progress; - this._lastProgressStalledSeconds = 0; + this._lastProgressTime = new Date().getTime(); } else { - if (this._lastProgressStalledSeconds < 1) { - this._lastProgressStalledSeconds++; - } else if (progress >= total) { + if (progress >= total) { // change message if we stalled at 100% - $('#uploadprogressbar .label .desktop').text(t('core', 'Processing files...')); + this.$uploadprogressbar.find('.label .desktop').text(t('core', 'Processing files...')); + } else if (new Date().getTime() - this._lastProgressTime >= this._uploadStallTimeout * 1000 ) { + // TODO: move to "fileuploadprogress" event instead and use data.uploadedBytes + // stalling needs to be checked here because the file upload no longer triggers events + // restart upload + this.log('progress stalled'); // retry chunk (and prevent IE from dying) + _.each(this._uploads, function(upload) { + // FIXME: harden by only retry pending, not the finished ones + upload.retry(); + }); } } }, @@ -870,6 +899,9 @@ OC.Uploader.prototype = _.extend({ var self = this; options = options || {}; + this._uploads = {}; + this._knownDirs = {}; + this.fileList = options.fileList; this.filesClient = options.filesClient || OC.Files.getClient(); this.davClient = new OC.Files.Client({ @@ -883,10 +915,15 @@ OC.Uploader.prototype = _.extend({ if (options.url) { this.url = options.url; } + if (options.uploadStallTimeout) { + this._uploadStallTimeout = options.uploadStallTimeout; + } $uploadEl = $($uploadEl); this.$uploadEl = $uploadEl; + this.$uploadprogressbar = $('#uploadprogressbar'); + if ($uploadEl.exists()) { $('#uploadprogresswrapper .stop').on('click', function() { self.cancelUploads(); @@ -897,10 +934,12 @@ OC.Uploader.prototype = _.extend({ dropZone: options.dropZone, // restrict dropZone to content div autoUpload: false, sequentialUploads: true, + maxRetries: options.uploadStallRetries || 3, + retryTimeout: 500, //singleFileUploads is on by default, so the data.files array will always have length 1 /** * on first add of every selection - * - check all files of originalFiles array with files in dir + * - on conflict show dialog * - skip all -> remember as single skip action for all conflicting files * - replace all -> remember as single replace action for all conflicting files @@ -1061,6 +1100,53 @@ OC.Uploader.prototype = _.extend({ }, fail: function(e, data) { var upload = self.getUpload(data); + if (upload && upload.data && upload.data.stalled) { + self.log('retry', e, upload); + // jQuery Widget Factory uses "namespace-widgetname" since version 1.10.0: + var fu = $(this).data('blueimp-fileupload') || $(this).data('fileupload'), + retries = upload.data.retries || 0, + retry = function () { + var uid = OC.getCurrentUser().uid; + upload.uploader.davClient.getFolderContents( + 'uploads/' + uid + '/' + upload.getId() + ) + .done(function (status, files) { + data.uploadedBytes = 0; + _.each(files, function(file) { + // only count numeric file names to omit .file and .file.zsync + if (!isNaN(parseFloat(file.name)) + && isFinite(file.name) + // only count full chunks + && file.size === fu.options.maxChunkSize + ) { + data.uploadedBytes += file.size; + } + }); + + // clear the previous data: + data.data = null; + // overwrite chunk + delete data.headers['If-None-Match']; + data.submit(); + }) + .fail(function (status, ex) { + self.log('failed to retry', status, ex); + fu._trigger('fail', e, data); + }); + }; + if (upload && upload.data && upload.data.stalled && + data.uploadedBytes < data.files[0].size && + retries < fu.options.maxRetries) { + retries += 1; + upload.data.retries = retries; + window.setTimeout(retry, retries * fu.options.retryTimeout); + return; + } + fu.prototype + .options.fail.call(this, e, data); + return; + } + var status = null; if (upload) { status = upload.getResponseStatus(); @@ -1158,14 +1244,14 @@ OC.Uploader.prototype = _.extend({ self.log('progress handle fileuploadstart', e, data); $('#uploadprogresswrapper .stop').show(); $('#uploadprogresswrapper .label').show(); - $('#uploadprogressbar').progressbar({value: 0}); - $('#uploadprogressbar .ui-progressbar-value'). + self.$uploadprogressbar.progressbar({value: 0}); + self.$uploadprogressbar.find('.ui-progressbar-value'). html('' + t('files', 'Uploading...') + '' + t('files', '...') + ''); - $('#uploadprogressbar').tipsy({gravity:'n', fade:true, live:true}); + self.$uploadprogressbar.tipsy({gravity:'n', fade:true, live:true}); self._showProgressBar(); self.trigger('start', e, data); }); @@ -1184,25 +1270,25 @@ OC.Uploader.prototype = _.extend({ lastSize = data.loaded; diffSize = diffSize / diffUpdate; // apply timing factor, eg. 1mb/2s = 0.5mb/s var remainingSeconds = ((data.total - data.loaded) / diffSize); - if(remainingSeconds >= 0) { + if (isFinite(remainingSeconds) && remainingSeconds >= 0) { bufferTotal = bufferTotal - (buffer[bufferIndex]) + remainingSeconds; buffer[bufferIndex] = remainingSeconds; //buffer to make it smoother bufferIndex = (bufferIndex + 1) % bufferSize; } var smoothRemainingSeconds = (bufferTotal / bufferSize); //seconds var h = moment.duration(smoothRemainingSeconds, "seconds").humanize(); - $('#uploadprogressbar').attr('data-loaded', data.loaded); - $('#uploadprogressbar').attr('data-total', data.total); - $('#uploadprogressbar .label .mobile').text(h); - $('#uploadprogressbar .label .desktop').text(h); - $('#uploadprogressbar').attr('original-title', + self.$uploadprogressbar.attr('data-loaded', data.loaded); + self.$uploadprogressbar.attr('data-total', data.total); + self.$uploadprogressbar.find('.label .mobile').text(h); + self.$uploadprogressbar.find('.label .desktop').text(h); + self.$uploadprogressbar.attr('original-title', t('files', '{loadedSize} of {totalSize} ({bitrate})' , { loadedSize: humanFileSize(data.loaded), totalSize: humanFileSize(data.total), bitrate: humanFileSize(data.bitrate) + '/s' }) ); - $('#uploadprogressbar').progressbar('value', progress); + self.$uploadprogressbar.progressbar('value', progress); self.trigger('progressall', e, data); }); fileupload.on('fileuploadstop', function(e, data) { @@ -1232,6 +1318,9 @@ OC.Uploader.prototype = _.extend({ '/' + encodeURIComponent(chunkId); delete data.contentRange; delete data.headers['Content-Range']; + + // reset retries + upload.data.retries = 0; }); fileupload.on('fileuploadchunkdone', function(e, data) { $(data.xhr().upload).unbind('progress'); diff --git a/apps/files/js/filelist.js b/apps/files/js/filelist.js index 282f1ce0082b..c729ed2b7060 100644 --- a/apps/files/js/filelist.js +++ b/apps/files/js/filelist.js @@ -344,7 +344,9 @@ fileList: this, filesClient: this.filesClient, dropZone: $('#content'), - maxChunkSize: options.maxChunkSize + maxChunkSize: options.maxChunkSize, + uploadStallTimeout: options.uploadStallTimeout, + uploadStallRetries: options.uploadStallRetries }); this.setupUploadEvents(this._uploader); diff --git a/apps/files/lib/App.php b/apps/files/lib/App.php index a2cb13889c10..ce07aa099c95 100644 --- a/apps/files/lib/App.php +++ b/apps/files/lib/App.php @@ -45,8 +45,12 @@ public static function getNavigationManager() { public static function extendJsConfig($array) { $maxChunkSize = (int)(\OC::$server->getConfig()->getAppValue('files', 'max_chunk_size', (10 * 1024 * 1024))); + $uploadStallTimeout = (int)(\OC::$server->getConfig()->getAppValue('files', 'upload_stall_timeout', 60)); // in seconds + $uploadStallRetries = (int)(\OC::$server->getConfig()->getAppValue('files', 'upload_stall_retries', 100)); $array['array']['oc_appconfig']['files'] = [ - 'max_chunk_size' => $maxChunkSize + 'max_chunk_size' => $maxChunkSize, + 'upload_stall_timeout' => $uploadStallTimeout, + 'upload_stall_retries' => $uploadStallRetries ]; } } diff --git a/apps/files/tests/js/fileUploadSpec.js b/apps/files/tests/js/fileUploadSpec.js index 6d65b8749235..95864ab1df5e 100644 --- a/apps/files/tests/js/fileUploadSpec.js +++ b/apps/files/tests/js/fileUploadSpec.js @@ -24,6 +24,8 @@ describe('OC.Upload tests', function() { var testFile; var uploader; var failStub; + var currentUserStub; + var getFolderContentsStub; beforeEach(function() { testFile = { @@ -37,6 +39,7 @@ describe('OC.Upload tests', function() { '' + '' + // 50 MB // TODO: handlebars! + '
' + '
' + 'New' + '