From 710dae1ad1b66e18614ee4a7faa0f158ec015371 Mon Sep 17 00:00:00 2001 From: Matthieu Gallien Date: Fri, 13 Sep 2024 12:25:29 +0200 Subject: [PATCH] count the files deletion and warn if threshold is exceeded Signed-off-by: Matthieu Gallien --- src/libsync/configfile.cpp | 17 ++- src/libsync/configfile.h | 3 + src/libsync/syncengine.cpp | 293 +++++++++++++++++++++---------------- src/libsync/syncengine.h | 9 ++ 4 files changed, 197 insertions(+), 125 deletions(-) diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp index 05dcfbbf6e531..22900c8674cda 100644 --- a/src/libsync/configfile.cpp +++ b/src/libsync/configfile.cpp @@ -59,6 +59,7 @@ static constexpr char fullLocalDiscoveryIntervalC[] = "fullLocalDiscoveryInterva static constexpr char notificationRefreshIntervalC[] = "notificationRefreshInterval"; static constexpr char monoIconsC[] = "monoIcons"; static constexpr char promptDeleteC[] = "promptDeleteAllFiles"; +static constexpr char deleteFilesThresholdC[] = "deleteFilesThreshold"; static constexpr char crashReporterC[] = "crashReporter"; static constexpr char optionalServerNotificationsC[] = "optionalServerNotifications"; static constexpr char showCallNotificationsC[] = "showCallNotifications"; @@ -113,6 +114,8 @@ static constexpr char certPasswd[] = "http_certificatePasswd"; static const QSet validUpdateChannels { QStringLiteral("stable"), QStringLiteral("beta") }; static constexpr auto macFileProviderModuleEnabledC = "macFileProviderModuleEnabled"; + +static constexpr int deleteFilesThresholdDefaultValue = 100; } namespace OCC { @@ -1019,7 +1022,7 @@ bool ConfigFile::showMainDialogAsNormalWindow() const { bool ConfigFile::promptDeleteFiles() const { QSettings settings(configFile(), QSettings::IniFormat); - return settings.value(QLatin1String(promptDeleteC), false).toBool(); + return settings.value(QLatin1String(promptDeleteC), true).toBool(); } void ConfigFile::setPromptDeleteFiles(bool promptDeleteFiles) @@ -1028,6 +1031,18 @@ void ConfigFile::setPromptDeleteFiles(bool promptDeleteFiles) settings.setValue(QLatin1String(promptDeleteC), promptDeleteFiles); } +int ConfigFile::deleteFilesThreshold() const +{ + QSettings settings(configFile(), QSettings::IniFormat); + return settings.value(QLatin1String(deleteFilesThresholdC), deleteFilesThresholdDefaultValue).toInt(); +} + +void ConfigFile::setDeleteFilesThreshold(int thresholdValue) +{ + QSettings settings(configFile(), QSettings::IniFormat); + settings.setValue(QLatin1String(deleteFilesThresholdC), thresholdValue); +} + bool ConfigFile::monoIcons() const { QSettings settings(configFile(), QSettings::IniFormat); diff --git a/src/libsync/configfile.h b/src/libsync/configfile.h index 7ae1f98df207b..c3a2ccbf2c3df 100644 --- a/src/libsync/configfile.h +++ b/src/libsync/configfile.h @@ -93,6 +93,9 @@ class OWNCLOUDSYNC_EXPORT ConfigFile [[nodiscard]] bool promptDeleteFiles() const; void setPromptDeleteFiles(bool promptDeleteFiles); + [[nodiscard]] int deleteFilesThreshold() const; + void setDeleteFilesThreshold(int thresholdValue); + [[nodiscard]] bool crashReporter() const; void setCrashReporter(bool enabled); diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 2d6f22a52796e..6a092b88fcc68 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -799,132 +799,11 @@ void SyncEngine::slotDiscoveryFinished() _progressInfo->_status = ProgressInfo::Reconcile; emit transmissionProgress(*_progressInfo); - // qCInfo(lcEngine) << "Permissions of the root folder: " << _csync_ctx->remote.root_perms.toString(); - auto finish = [this]{ - auto databaseFingerprint = _journal->dataFingerprint(); - // If databaseFingerprint is empty, this means that there was no information in the database - // (for example, upgrading from a previous version, or first sync, or server not supporting fingerprint) - if (!databaseFingerprint.isEmpty() && _discoveryPhase - && _discoveryPhase->_dataFingerprint != databaseFingerprint) { - qCInfo(lcEngine) << "data fingerprint changed, assume restore from backup" << databaseFingerprint << _discoveryPhase->_dataFingerprint; - restoreOldFiles(_syncItems); - } - - if (_discoveryPhase->_anotherSyncNeeded && !_discoveryPhase->_filesNeedingScheduledSync.empty()) { - slotScheduleFilesDelayedSync(); - } else if (_discoveryPhase->_anotherSyncNeeded && _anotherSyncNeeded == NoFollowUpSync) { - _anotherSyncNeeded = ImmediateFollowUp; - } - - if (!_discoveryPhase->_filesUnscheduleSync.empty()) { - slotUnscheduleFilesDelayedSync(); - } - - if (_discoveryPhase->_hasDownloadRemovedItems && _discoveryPhase->_hasUploadErrorItems) { - for (const auto &item : qAsConst(_syncItems)) { - if (item->_instruction == CSYNC_INSTRUCTION_ERROR && item->_direction == SyncFileItem::Up) { - item->_instruction = CSYNC_INSTRUCTION_IGNORE; - } - } - _anotherSyncNeeded = ImmediateFollowUp; - } - - Q_ASSERT(std::is_sorted(_syncItems.begin(), _syncItems.end())); - - qCInfo(lcEngine) << "#### Reconcile (aboutToPropagate) #################################################### " << _stopWatch.addLapTime(QStringLiteral("Reconcile (aboutToPropagate)")) << "ms"; - - _localDiscoveryPaths.clear(); - - // To announce the beginning of the sync - emit aboutToPropagate(_syncItems); - - qCInfo(lcEngine) << "#### Reconcile (aboutToPropagate OK) #################################################### "<< _stopWatch.addLapTime(QStringLiteral("Reconcile (aboutToPropagate OK)")) << "ms"; - - // it's important to do this before ProgressInfo::start(), to announce start of new sync - _progressInfo->_status = ProgressInfo::Propagation; - emit transmissionProgress(*_progressInfo); - _progressInfo->startEstimateUpdates(); - - // post update phase script: allow to tweak stuff by a custom script in debug mode. - if (!qEnvironmentVariableIsEmpty("OWNCLOUD_POST_UPDATE_SCRIPT")) { - #ifndef NDEBUG - const QString script = qEnvironmentVariable("OWNCLOUD_POST_UPDATE_SCRIPT"); - - qCDebug(lcEngine) << "Post Update Script: " << script; - auto scriptArgs = script.split(QRegularExpression("\\s+"), Qt::SkipEmptyParts); - if (scriptArgs.size() > 0) { - const auto scriptExecutable = scriptArgs.takeFirst(); - QProcess::execute(scriptExecutable, scriptArgs); - } -#else - qCWarning(lcEngine) << "**** Attention: POST_UPDATE_SCRIPT installed, but not executed because compiled with NDEBUG"; - #endif - } - - // do a database commit - _journal->commit(QStringLiteral("post treewalk")); - - _propagator = QSharedPointer( - new OwncloudPropagator(_account, _localPath, _remotePath, _journal, _bulkUploadBlackList)); - _propagator->setSyncOptions(_syncOptions); - connect(_propagator.data(), &OwncloudPropagator::itemCompleted, - this, &SyncEngine::slotItemCompleted); - connect(_propagator.data(), &OwncloudPropagator::progress, - this, &SyncEngine::slotProgress); - connect(_propagator.data(), &OwncloudPropagator::finished, this, &SyncEngine::slotPropagationFinished, Qt::QueuedConnection); - connect(_propagator.data(), &OwncloudPropagator::seenLockedFile, this, &SyncEngine::seenLockedFile); - connect(_propagator.data(), &OwncloudPropagator::touchedFile, this, &SyncEngine::slotAddTouchedFile); - connect(_propagator.data(), &OwncloudPropagator::insufficientLocalStorage, this, &SyncEngine::slotInsufficientLocalStorage); - connect(_propagator.data(), &OwncloudPropagator::insufficientRemoteStorage, this, &SyncEngine::slotInsufficientRemoteStorage); - connect(_propagator.data(), &OwncloudPropagator::newItem, this, &SyncEngine::slotNewItem); - - // apply the network limits to the propagator - setNetworkLimits(_uploadLimit, _downloadLimit); - - deleteStaleDownloadInfos(_syncItems); - deleteStaleUploadInfos(_syncItems); - deleteStaleErrorBlacklistEntries(_syncItems); - _journal->commit(QStringLiteral("post stale entry removal")); - - // Emit the started signal only after the propagator has been set up. - if (_needsUpdate) - Q_EMIT started(); - - _propagator->start(std::move(_syncItems)); - - qCInfo(lcEngine) << "#### Post-Reconcile end #################################################### " << _stopWatch.addLapTime(QStringLiteral("Post-Reconcile Finished")) << "ms"; - }; - - if (!_hasNoneFiles && _hasRemoveFile) { - qCInfo(lcEngine) << "All the files are going to be changed, asking the user"; - int side = 0; // > 0 means more deleted on the server. < 0 means more deleted on the client - foreach (const auto &it, _syncItems) { - if (it->_instruction == CSYNC_INSTRUCTION_REMOVE) { - side += it->_direction == SyncFileItem::Down ? 1 : -1; - } - } - - QPointer guard = new QObject(); - QPointer self = this; - auto callback = [this, self, finish, guard](bool cancel) -> void { - // use a guard to ensure its only called once... - // qpointer to self to ensure we still exist - if (!guard || !self) { - return; - } - guard->deleteLater(); - if (cancel) { - qCInfo(lcEngine) << "User aborted sync"; - finalize(false); - return; - } else { - finish(); - } - }; - emit aboutToRemoveAllFiles(side >= 0 ? SyncFileItem::Down : SyncFileItem::Up, callback); + if (handleMassDeletion()) { return; } - finish(); + + finishSync(); } void SyncEngine::slotCleanPollsJobAborted(const QString &error, const ErrorCategory errorCategory) @@ -1101,6 +980,172 @@ void SyncEngine::restoreOldFiles(SyncFileItemVector &syncItems) } } +void SyncEngine::cancelSyncOrContinue(bool cancel) +{ + if (cancel) { + qCInfo(lcEngine) << "User aborted sync"; + finalize(false); + } else { + finishSync(); + } +} + +void SyncEngine::finishSync() +{ + auto databaseFingerprint = _journal->dataFingerprint(); + // If databaseFingerprint is empty, this means that there was no information in the database + // (for example, upgrading from a previous version, or first sync, or server not supporting fingerprint) + if (!databaseFingerprint.isEmpty() && _discoveryPhase + && _discoveryPhase->_dataFingerprint != databaseFingerprint) { + qCInfo(lcEngine) << "data fingerprint changed, assume restore from backup" << databaseFingerprint << _discoveryPhase->_dataFingerprint; + restoreOldFiles(_syncItems); + } + + if (_discoveryPhase && _discoveryPhase->_anotherSyncNeeded && !_discoveryPhase->_filesNeedingScheduledSync.empty()) { + slotScheduleFilesDelayedSync(); + } else if (_discoveryPhase && _discoveryPhase->_anotherSyncNeeded && _anotherSyncNeeded == NoFollowUpSync) { + _anotherSyncNeeded = ImmediateFollowUp; + } + + if (_discoveryPhase && !_discoveryPhase->_filesUnscheduleSync.empty()) { + slotUnscheduleFilesDelayedSync(); + } + + if (_discoveryPhase && _discoveryPhase->_hasDownloadRemovedItems && _discoveryPhase->_hasUploadErrorItems) { + for (const auto &item : qAsConst(_syncItems)) { + if (item->_instruction == CSYNC_INSTRUCTION_ERROR && item->_direction == SyncFileItem::Up) { + // item->_instruction = CSYNC_INSTRUCTION_IGNORE; + } + } + _anotherSyncNeeded = ImmediateFollowUp; + } + + Q_ASSERT(std::is_sorted(_syncItems.begin(), _syncItems.end())); + + qCInfo(lcEngine) << "#### Reconcile (aboutToPropagate) #################################################### " << _stopWatch.addLapTime(QStringLiteral("Reconcile (aboutToPropagate)")) << "ms"; + + _localDiscoveryPaths.clear(); + + // To announce the beginning of the sync + emit aboutToPropagate(_syncItems); + + qCInfo(lcEngine) << "#### Reconcile (aboutToPropagate OK) #################################################### "<< _stopWatch.addLapTime(QStringLiteral("Reconcile (aboutToPropagate OK)")) << "ms"; + + // it's important to do this before ProgressInfo::start(), to announce start of new sync + _progressInfo->_status = ProgressInfo::Propagation; + emit transmissionProgress(*_progressInfo); + _progressInfo->startEstimateUpdates(); + + // post update phase script: allow to tweak stuff by a custom script in debug mode. + if (!qEnvironmentVariableIsEmpty("OWNCLOUD_POST_UPDATE_SCRIPT")) { +#ifndef NDEBUG + const QString script = qEnvironmentVariable("OWNCLOUD_POST_UPDATE_SCRIPT"); + + qCDebug(lcEngine) << "Post Update Script: " << script; + auto scriptArgs = script.split(QRegularExpression("\\s+"), Qt::SkipEmptyParts); + if (scriptArgs.size() > 0) { + const auto scriptExecutable = scriptArgs.takeFirst(); + QProcess::execute(scriptExecutable, scriptArgs); + } +#else + qCWarning(lcEngine) << "**** Attention: POST_UPDATE_SCRIPT installed, but not executed because compiled with NDEBUG"; +#endif + } + + // do a database commit + _journal->commit(QStringLiteral("post treewalk")); + + _propagator = QSharedPointer( + new OwncloudPropagator(_account, _localPath, _remotePath, _journal, _bulkUploadBlackList)); + _propagator->setSyncOptions(_syncOptions); + connect(_propagator.data(), &OwncloudPropagator::itemCompleted, + this, &SyncEngine::slotItemCompleted); + connect(_propagator.data(), &OwncloudPropagator::progress, + this, &SyncEngine::slotProgress); + connect(_propagator.data(), &OwncloudPropagator::finished, this, &SyncEngine::slotPropagationFinished, Qt::QueuedConnection); + connect(_propagator.data(), &OwncloudPropagator::seenLockedFile, this, &SyncEngine::seenLockedFile); + connect(_propagator.data(), &OwncloudPropagator::touchedFile, this, &SyncEngine::slotAddTouchedFile); + connect(_propagator.data(), &OwncloudPropagator::insufficientLocalStorage, this, &SyncEngine::slotInsufficientLocalStorage); + connect(_propagator.data(), &OwncloudPropagator::insufficientRemoteStorage, this, &SyncEngine::slotInsufficientRemoteStorage); + connect(_propagator.data(), &OwncloudPropagator::newItem, this, &SyncEngine::slotNewItem); + + // apply the network limits to the propagator + setNetworkLimits(_uploadLimit, _downloadLimit); + + deleteStaleDownloadInfos(_syncItems); + deleteStaleUploadInfos(_syncItems); + deleteStaleErrorBlacklistEntries(_syncItems); + _journal->commit(QStringLiteral("post stale entry removal")); + + // Emit the started signal only after the propagator has been set up. + if (_needsUpdate) + Q_EMIT started(); + + _propagator->start(std::move(_syncItems)); + + qCInfo(lcEngine) << "#### Post-Reconcile end #################################################### " << _stopWatch.addLapTime(QStringLiteral("Post-Reconcile Finished")) << "ms"; +} + +bool SyncEngine::handleMassDeletion() +{ + const auto displayDialog = ConfigFile().promptDeleteFiles(); + const auto allFilesDeleted = !_hasNoneFiles && _hasRemoveFile; + + auto deletionCounter = 0; + for (const auto &oneItem : qAsConst(_syncItems)) { + if (oneItem->_instruction == CSYNC_INSTRUCTION_REMOVE) { + if (oneItem->isDirectory()) { + const auto result = _journal->listFilesInPath(oneItem->_file.toUtf8(), [&deletionCounter] (const auto &oneRecord) { + if (oneRecord.isFile()) { + ++deletionCounter; + } + }); + if (!result) { + qCDebug(lcEngine()) << "unable to find the number of files within a deleted folder:" << oneItem->_file; + } + } else { + ++deletionCounter; + } + } + } + const auto filesDeletedThresholdExceeded = deletionCounter > ConfigFile().deleteFilesThreshold(); + + if ((allFilesDeleted || filesDeletedThresholdExceeded) && displayDialog) { + qCInfo(lcEngine) << "All the files are going to be changed, asking the user"; + int side = 0; // > 0 means more deleted on the server. < 0 means more deleted on the client + for (const auto &it : qAsConst(_syncItems)) { + if (it->_instruction == CSYNC_INSTRUCTION_REMOVE) { + side += it->_direction == SyncFileItem::Down ? 1 : -1; + } + } + + promptUserBeforePropagation([this, side](auto &&callback){ + emit aboutToRemoveAllFiles(side >= 0 ? SyncFileItem::Down : SyncFileItem::Up, callback); + }); + return true; + } + return false; +} + +template +void SyncEngine::promptUserBeforePropagation(T &&lambda) +{ + QPointer guard = new QObject(); + QPointer self = this; + auto callback = [this, self, guard](bool cancel) -> void { + // use a guard to ensure its only called once... + // qpointer to self to ensure we still exist + if (!guard || !self) { + return; + } + guard->deleteLater(); + + cancelSyncOrContinue(cancel); + }; + + lambda(callback); +} + void SyncEngine::slotAddTouchedFile(const QString &fn) { QElapsedTimer now; diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h index af234725cf2d8..6b54e7aa3bcb1 100644 --- a/src/libsync/syncengine.h +++ b/src/libsync/syncengine.h @@ -354,6 +354,15 @@ private slots: */ void restoreOldFiles(SyncFileItemVector &syncItems); + void cancelSyncOrContinue(bool cancel); + + void finishSync(); + + bool handleMassDeletion(); + + template + void promptUserBeforePropagation(T &&lambda); + // true if there is at least one file which was not changed on the server bool _hasNoneFiles = false;