diff --git a/doc/conffile.rst b/doc/conffile.rst index 9b3c2ffe038..3fdcc179b76 100644 --- a/doc/conffile.rst +++ b/doc/conffile.rst @@ -27,6 +27,8 @@ Some interesting values that can be set on the configuration file are: +---------------------------------+---------------+--------------------------------------------------------------------------------------------------------+ | ``forceSyncInterval`` | ``7200000`` | The duration of no activity after which a synchronization run shall be triggered automatically. | +---------------------------------+---------------+--------------------------------------------------------------------------------------------------------+ +| ``fullLocalDiscoveryInterval`` | ``3600000`` | The interval after which the next synchronization will perform a full local discovery. | ++---------------------------------+---------------+--------------------------------------------------------------------------------------------------------+ | ``notificationRefreshInterval`` | ``300000`` | Specifies the default interval of checking for new server notifications in milliseconds. | +---------------------------------+---------------+--------------------------------------------------------------------------------------------------------+ @@ -62,4 +64,4 @@ Some interesting values that can be set on the configuration file are: | | | ``2`` for No Proxy. | + + +--------------------------------------------------------------------------------------------------------+ | | | ``3`` for HTTP(S) Proxy. | -+---------------------------------+---------------+--------------------------------------------------------------------------------------------------------+ \ No newline at end of file ++---------------------------------+---------------+--------------------------------------------------------------------------------------------------------+ diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index b827657c755..ad8c68aa808 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -547,13 +547,28 @@ bool SyncJournalDb::checkConnect() return sqlFail("prepare _getFileRecordQueryByFileId", *_getFileRecordQueryByFileId); } + // This query is used to skip discovery and fill the tree from the + // database instead _getFilesBelowPathQuery.reset(new SqlQuery(_db)); if (_getFilesBelowPathQuery->prepare( GET_FILE_RECORD_QUERY - " WHERE path > (?1||'/') AND path < (?1||'0') ORDER BY path||'/' ASC")) { + " WHERE path > (?1||'/') AND path < (?1||'0')" + // We want to ensure that the contents of a directory are sorted + // directly behind the directory itself. Without this ORDER BY + // an ordering like foo, foo-2, foo/file would be returned. + // With the trailing /, we get foo-2, foo, foo/file. This property + // is used in fill_tree_from_db(). + " ORDER BY path||'/' ASC")) { return sqlFail("prepare _getFilesBelowPathQuery", *_getFilesBelowPathQuery); } + _getAllFilesQuery.reset(new SqlQuery(_db)); + if (_getAllFilesQuery->prepare( + GET_FILE_RECORD_QUERY + " ORDER BY path||'/' ASC")) { + return sqlFail("prepare _getAllFilesQuery", *_getAllFilesQuery); + } + _setFileRecordQuery.reset(new SqlQuery(_db)); if (_setFileRecordQuery->prepare("INSERT OR REPLACE INTO metadata " "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId) " @@ -704,6 +719,7 @@ void SyncJournalDb::close() _getFileRecordQueryByInode.reset(0); _getFileRecordQueryByFileId.reset(0); _getFilesBelowPathQuery.reset(0); + _getAllFilesQuery.reset(0); _setFileRecordQuery.reset(0); _setFileRecordChecksumQuery.reset(0); _setFileRecordLocalMetadataQuery.reset(0); @@ -1133,16 +1149,23 @@ bool SyncJournalDb::getFilesBelowPath(const QByteArray &path, const std::functio if (!checkConnect()) return false; - _getFilesBelowPathQuery->reset_and_clear_bindings(); - _getFilesBelowPathQuery->bindValue(1, path); + // Since the path column doesn't store the starting /, the getFilesBelowPathQuery + // can't be used for the root path "". It would scan for (path > '/' and path < '0') + // and find nothing. So, unfortunately, we have to use a different query for + // retrieving the whole tree. + auto &query = path.isEmpty() ? _getAllFilesQuery : _getFilesBelowPathQuery; - if (!_getFilesBelowPathQuery->exec()) { + query->reset_and_clear_bindings(); + if (query == _getFilesBelowPathQuery) + query->bindValue(1, path); + + if (!query->exec()) { return false; } - while (_getFilesBelowPathQuery->next()) { + while (query->next()) { SyncJournalFileRecord rec; - fillFileRecordFromGetQuery(rec, *_getFilesBelowPathQuery); + fillFileRecordFromGetQuery(rec, *query); rowCallback(rec); } diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h index cfdced9a349..91556a80785 100644 --- a/src/common/syncjournaldb.h +++ b/src/common/syncjournaldb.h @@ -244,6 +244,7 @@ class OCSYNC_EXPORT SyncJournalDb : public QObject QScopedPointer _getFileRecordQueryByInode; QScopedPointer _getFileRecordQueryByFileId; QScopedPointer _getFilesBelowPathQuery; + QScopedPointer _getAllFilesQuery; QScopedPointer _setFileRecordQuery; QScopedPointer _setFileRecordChecksumQuery; QScopedPointer _setFileRecordLocalMetadataQuery; diff --git a/src/csync/csync.cpp b/src/csync/csync.cpp index 4f0c1991622..9c8593e3170 100644 --- a/src/csync/csync.cpp +++ b/src/csync/csync.cpp @@ -314,6 +314,9 @@ int csync_s::reinitialize() { local.files.clear(); remote.files.clear(); + local_discovery_style = LocalDiscoveryStyle::FilesystemOnly; + locally_touched_dirs.clear(); + status = CSYNC_STATUS_INIT; SAFE_FREE(error_string); diff --git a/src/csync/csync_private.h b/src/csync/csync_private.h index 2c29ece7034..bef8a75126d 100644 --- a/src/csync/csync_private.h +++ b/src/csync/csync_private.h @@ -38,6 +38,7 @@ #include #include #include +#include #include "common/syncjournaldb.h" #include "config_csync.h" @@ -70,6 +71,11 @@ enum csync_replica_e { REMOTE_REPLICA }; +enum class LocalDiscoveryStyle { + FilesystemOnly, //< read all local data from the filesystem + DatabaseAndFilesystem, //< read from the db, except for listed paths +}; + /* * This is a structurere similar to QStringRef @@ -190,6 +196,16 @@ struct OCSYNC_EXPORT csync_s { */ bool read_remote_from_db = false; + LocalDiscoveryStyle local_discovery_style = LocalDiscoveryStyle::FilesystemOnly; + + /** + * List of folder-relative directory paths that should be scanned on the + * filesystem if the local_discovery_style suggests it. + * + * Their parents will be scanned too. The paths don't start with a /. + */ + std::set locally_touched_dirs; + bool ignore_hidden_files = true; csync_s(const char *localUri, OCC::SyncJournalDb *statedb); diff --git a/src/csync/csync_update.cpp b/src/csync/csync_update.cpp index 0271b1c569f..35c73e625ac 100644 --- a/src/csync/csync_update.cpp +++ b/src/csync/csync_update.cpp @@ -435,28 +435,31 @@ static bool fill_tree_from_db(CSYNC *ctx, const char *uri) { int64_t count = 0; QByteArray skipbase; - auto rowCallback = [ctx, &count, &skipbase](const OCC::SyncJournalFileRecord &rec) { - /* When selective sync is used, the database may have subtrees with a parent - * whose etag (md5) is _invalid_. These are ignored and shall not appear in the - * remote tree. - * Sometimes folders that are not ignored by selective sync get marked as - * _invalid_, but that is not a problem as the next discovery will retrieve - * their correct etags again and we don't run into this case. - */ - if( rec._etag == "_invalid_") { - qCDebug(lcUpdate, "%s selective sync excluded", rec._path.constData()); - skipbase = rec._path; - skipbase += '/'; - return; - } + auto &files = ctx->current == LOCAL_REPLICA ? ctx->local.files : ctx->remote.files; + auto rowCallback = [ctx, &count, &skipbase, &files](const OCC::SyncJournalFileRecord &rec) { + if (ctx->current == REMOTE_REPLICA) { + /* When selective sync is used, the database may have subtrees with a parent + * whose etag is _invalid_. These are ignored and shall not appear in the + * remote tree. + * Sometimes folders that are not ignored by selective sync get marked as + * _invalid_, but that is not a problem as the next discovery will retrieve + * their correct etags again and we don't run into this case. + */ + if (rec._etag == "_invalid_") { + qCDebug(lcUpdate, "%s selective sync excluded", rec._path.constData()); + skipbase = rec._path; + skipbase += '/'; + return; + } - /* Skip over all entries with the same base path. Note that this depends - * strongly on the ordering of the retrieved items. */ - if( !skipbase.isEmpty() && rec._path.startsWith(skipbase) ) { - qCDebug(lcUpdate, "%s selective sync excluded because the parent is", rec._path.constData()); - return; - } else { - skipbase.clear(); + /* Skip over all entries with the same base path. Note that this depends + * strongly on the ordering of the retrieved items. */ + if (!skipbase.isEmpty() && rec._path.startsWith(skipbase)) { + qCDebug(lcUpdate, "%s selective sync excluded because the parent is", rec._path.constData()); + return; + } else { + skipbase.clear(); + } } std::unique_ptr st = csync_file_stat_t::fromSyncJournalFileRecord(rec); @@ -477,7 +480,7 @@ static bool fill_tree_from_db(CSYNC *ctx, const char *uri) } /* store into result list. */ - ctx->remote.files[rec._path] = std::move(st); + files[rec._path] = std::move(st); ++count; }; @@ -522,6 +525,26 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn, int rc = 0; bool do_read_from_db = (ctx->current == REMOTE_REPLICA && ctx->remote.read_from_db); + const char *db_uri = uri; + + if (ctx->current == LOCAL_REPLICA + && ctx->local_discovery_style == LocalDiscoveryStyle::DatabaseAndFilesystem) { + const char *local_uri = uri + strlen(ctx->local.uri); + if (*local_uri == '/') + ++local_uri; + db_uri = local_uri; + do_read_from_db = true; + + // Minor bug: local_uri doesn't have a trailing /. Example: Assume it's "d/foo" + // and we want to check whether we should read from the db. Assume "d/foo a" is + // in locally_touched_dirs. Then this check will say no, don't read from the db! + // (because "d/foo" < "d/foo a" < "d/foo/bar") + // C++14: Could skip the conversion to QByteArray here. + auto it = ctx->locally_touched_dirs.lower_bound(QByteArray(local_uri)); + if (it != ctx->locally_touched_dirs.end() && it->startsWith(local_uri)) { + do_read_from_db = false; + } + } if (!depth) { mark_current_item_ignored(ctx, previous_fs, CSYNC_STATUS_INDIVIDUAL_TOO_DEEP); @@ -533,7 +556,7 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn, // if the etag of this dir is still the same, its content is restored from the // database. if( do_read_from_db ) { - if( ! fill_tree_from_db(ctx, uri) ) { + if( ! fill_tree_from_db(ctx, db_uri) ) { errno = ENOENT; ctx->status_code = CSYNC_STATUS_OPENDIR_ERROR; goto error; diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index c61c4ea68a2..176a4932302 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -448,12 +448,26 @@ int Folder::slotWipeErrorBlacklist() void Folder::slotWatchedPathChanged(const QString &path) { + if (!path.startsWith(this->path())) { + qCDebug(lcFolder) << "Changed path is not contained in folder, ignoring:" << path; + return; + } + + auto relativePath = path.midRef(this->path().size()); + + // Add to list of locally modified paths + // + // We do this before checking for our own sync-related changes to make + // extra sure to not miss relevant changes. + auto relativePathBytes = relativePath.toUtf8(); + _localDiscoveryPaths.insert(relativePathBytes); + qCDebug(lcFolder) << "local discovery: inserted" << relativePath << "due to file watcher"; + // The folder watcher fires a lot of bogus notifications during // a sync operation, both for actual user files and the database // and log. Therefore we check notifications against operations // the sync is doing to filter out our own changes. #ifdef Q_OS_MAC - Q_UNUSED(path) // On OSX the folder watcher does not report changes done by our // own process. Therefore nothing needs to be done here! #else @@ -465,15 +479,12 @@ void Folder::slotWatchedPathChanged(const QString &path) #endif // Check that the mtime actually changed. - if (path.startsWith(this->path())) { - auto relativePath = path.mid(this->path().size()); - SyncJournalFileRecord record; - if (_journal.getFileRecord(relativePath, &record) - && record.isValid() - && !FileSystem::fileChanged(path, record._fileSize, record._modtime)) { - qCInfo(lcFolder) << "Ignoring spurious notification for file" << relativePath; - return; // probably a spurious notification - } + SyncJournalFileRecord record; + if (_journal.getFileRecord(relativePathBytes, &record) + && record.isValid() + && !FileSystem::fileChanged(path, record._fileSize, record._modtime)) { + qCInfo(lcFolder) << "Ignoring spurious notification for file" << relativePath; + return; // probably a spurious notification } emit watchedFileChangedExternally(path); @@ -645,6 +656,36 @@ void Folder::startSync(const QStringList &pathList) setDirtyNetworkLimits(); setSyncOptions(); + static qint64 fullLocalDiscoveryInterval = []() { + auto interval = ConfigFile().fullLocalDiscoveryInterval(); + QByteArray env = qgetenv("OWNCLOUD_FULL_LOCAL_DISCOVERY_INTERVAL"); + if (!env.isEmpty()) { + interval = env.toLongLong(); + } + return interval; + }(); + if (_folderWatcher && _folderWatcher->isReliable() + && _timeSinceLastFullLocalDiscovery.isValid() + && (fullLocalDiscoveryInterval < 0 + || _timeSinceLastFullLocalDiscovery.elapsed() < fullLocalDiscoveryInterval)) { + qCInfo(lcFolder) << "Allowing local discovery to read from the database"; + _engine->setLocalDiscoveryOptions(LocalDiscoveryStyle::DatabaseAndFilesystem, _localDiscoveryPaths); + + if (lcFolder().isDebugEnabled()) { + QByteArrayList paths; + for (auto &path : _localDiscoveryPaths) + paths.append(path); + qCDebug(lcFolder) << "local discovery paths: " << paths; + } + + _previousLocalDiscoveryPaths = std::move(_localDiscoveryPaths); + } else { + qCInfo(lcFolder) << "Forbidding local discovery to read from the database"; + _engine->setLocalDiscoveryOptions(LocalDiscoveryStyle::FilesystemOnly); + _previousLocalDiscoveryPaths.clear(); + } + _localDiscoveryPaths.clear(); + _engine->setIgnoreHiddenFiles(_definition.ignoreHiddenFiles); QMetaObject::invokeMethod(_engine.data(), "startSync", Qt::QueuedConnection); @@ -783,6 +824,24 @@ void Folder::slotSyncFinished(bool success) journalDb()->setSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, QStringList()); } + // bug: This function uses many different criteria for "sync was successful" - investigate! + if ((_syncResult.status() == SyncResult::Success + || _syncResult.status() == SyncResult::Problem) + && success) { + if (_engine->lastLocalDiscoveryStyle() == LocalDiscoveryStyle::FilesystemOnly) { + _timeSinceLastFullLocalDiscovery.start(); + } + qCDebug(lcFolder) << "Sync success, forgetting last sync's local discovery path list"; + } else { + // On overall-failure we can't forget about last sync's local discovery + // paths yet, reuse them for the next sync again. + // C++17: Could use std::set::merge(). + _localDiscoveryPaths.insert( + _previousLocalDiscoveryPaths.begin(), _previousLocalDiscoveryPaths.end()); + qCDebug(lcFolder) << "Sync failed, keeping last sync's local discovery path list"; + } + _previousLocalDiscoveryPaths.clear(); + emit syncStateChange(); // The syncFinished result that is to be triggered here makes the folderman @@ -843,10 +902,31 @@ void Folder::slotItemCompleted(const SyncFileItemPtr &item) { // add new directories or remove gone away dirs to the watcher if (item->isDirectory() && item->_instruction == CSYNC_INSTRUCTION_NEW) { - FolderMan::instance()->addMonitorPath(alias(), path() + item->_file); + if (_folderWatcher) + _folderWatcher->addPath(path() + item->_file); } if (item->isDirectory() && item->_instruction == CSYNC_INSTRUCTION_REMOVE) { - FolderMan::instance()->removeMonitorPath(alias(), path() + item->_file); + if (_folderWatcher) + _folderWatcher->removePath(path() + item->_file); + } + + // Success and failure of sync items adjust what the next sync is + // supposed to do. + // + // For successes, we want to wipe the file from the list to ensure we don't + // rediscover it even if this overall sync fails. + // + // For failures, we want to add the file to the list so the next sync + // will be able to retry it. + if (item->_status == SyncFileItem::Success + || item->_status == SyncFileItem::FileIgnored + || item->_status == SyncFileItem::Restoration + || item->_status == SyncFileItem::Conflict) { + if (_previousLocalDiscoveryPaths.erase(item->_file.toUtf8())) + qCDebug(lcFolder) << "local discovery: wiped" << item->_file; + } else { + _localDiscoveryPaths.insert(item->_file.toUtf8()); + qCDebug(lcFolder) << "local discovery: inserted" << item->_file << "due to sync failure"; } _syncResult.processCompletedItem(item); @@ -901,6 +981,11 @@ void Folder::slotScheduleThisFolder() FolderMan::instance()->scheduleFolder(this); } +void Folder::slotNextSyncFullLocalDiscovery() +{ + _timeSinceLastFullLocalDiscovery.invalidate(); +} + void Folder::scheduleThisFolderSoon() { if (!_scheduleSelfTimer.isActive()) { @@ -913,6 +998,20 @@ void Folder::setSaveBackwardsCompatible(bool save) _saveBackwardsCompatible = save; } +void Folder::registerFolderWatcher() +{ + if (_folderWatcher) + return; + if (!QDir(path()).exists()) + return; + + _folderWatcher.reset(new FolderWatcher(path(), this)); + connect(_folderWatcher.data(), &FolderWatcher::pathChanged, + this, &Folder::slotWatchedPathChanged); + connect(_folderWatcher.data(), &FolderWatcher::lostChanges, + this, &Folder::slotNextSyncFullLocalDiscovery); +} + void Folder::slotAboutToRemoveAllFiles(SyncFileItem::Direction dir, bool *cancel) { ConfigFile cfgFile; diff --git a/src/gui/folder.h b/src/gui/folder.h index 7b503a6d604..2e7f2a7c0fc 100644 --- a/src/gui/folder.h +++ b/src/gui/folder.h @@ -27,6 +27,7 @@ #include #include +#include class QThread; class QSettings; @@ -36,6 +37,7 @@ namespace OCC { class SyncEngine; class AccountState; class SyncRunFileLog; +class FolderWatcher; /** * @brief The FolderDefinition class @@ -227,6 +229,13 @@ class Folder : public QObject */ void setSaveBackwardsCompatible(bool save); + /** + * Sets up this folder's folderWatcher if possible. + * + * May be called several times. + */ + void registerFolderWatcher(); + signals: void syncStateChange(); void syncStarted(); @@ -304,6 +313,9 @@ private slots: */ void slotScheduleThisFolder(); + /** Ensures that the next sync performs a full local discovery. */ + void slotNextSyncFullLocalDiscovery(); + private: bool setIgnoredFiles(); @@ -338,6 +350,7 @@ private slots: QString _lastEtag; QElapsedTimer _timeSinceLastSyncDone; QElapsedTimer _timeSinceLastSyncStart; + QElapsedTimer _timeSinceLastFullLocalDiscovery; qint64 _lastSyncDuration; /// The number of syncs that failed in a row. @@ -365,6 +378,29 @@ private slots: * path. */ bool _saveBackwardsCompatible; + + /** + * Watches this folder's local directory for changes. + * + * Created by registerFolderWatcher(), triggers slotWatchedPathChanged() + */ + QScopedPointer _folderWatcher; + + /** + * The paths that should be checked by the next local discovery. + * + * Mostly a collection of files the filewatchers have reported as touched. + * Also includes files that have had errors in the last sync run. + */ + std::set _localDiscoveryPaths; + + /** + * The paths that the current sync run used for local discovery. + * + * For failing syncs, this list will be merged into _localDiscoveryPaths + * again when the sync is done to make sure everything is retried. + */ + std::set _previousLocalDiscoveryPaths; }; } diff --git a/src/gui/folderman.cpp b/src/gui/folderman.cpp index b819e75692e..517d5dc9957 100644 --- a/src/gui/folderman.cpp +++ b/src/gui/folderman.cpp @@ -104,9 +104,6 @@ void FolderMan::unloadFolder(Folder *f) _socketApi->slotUnregisterPath(f->alias()); - if (_folderWatchers.contains(f->alias())) { - _folderWatchers.remove(f->alias()); - } _folderMap.remove(f->alias()); disconnect(f, &Folder::syncStarted, @@ -147,54 +144,18 @@ int FolderMan::unloadAndDeleteAllFolders() return cnt; } -// add a monitor to the local file system. If there is a change in the -// file system, the method slotFolderMonitorFired is triggered through -// the SignalMapper -void FolderMan::registerFolderMonitor(Folder *folder) +void FolderMan::registerFolderWithSocketApi(Folder *folder) { if (!folder) return; if (!QDir(folder->path()).exists()) return; - if (!_folderWatchers.contains(folder->alias())) { - FolderWatcher *fw = new FolderWatcher(folder->path(), folder); - - // Connect the pathChanged signal, which comes with the changed path, - // to the signal mapper which maps to the folder alias. The changed path - // is lost this way, but we do not need it for the current implementation. - connect(fw, &FolderWatcher::pathChanged, folder, &Folder::slotWatchedPathChanged); - - _folderWatchers.insert(folder->alias(), fw); - } - // register the folder with the socket API if (folder->canSync()) _socketApi->slotRegisterPath(folder->alias()); } -void FolderMan::addMonitorPath(const QString &alias, const QString &path) -{ - if (!alias.isEmpty() && _folderWatchers.contains(alias)) { - FolderWatcher *fw = _folderWatchers[alias]; - - if (fw) { - fw->addPath(path); - } - } -} - -void FolderMan::removeMonitorPath(const QString &alias, const QString &path) -{ - if (!alias.isEmpty() && _folderWatchers.contains(alias)) { - FolderWatcher *fw = _folderWatchers[alias]; - - if (fw) { - fw->removePath(path); - } - } -} - int FolderMan::setupFolders() { unloadAndDeleteAllFolders(); @@ -750,7 +711,8 @@ void FolderMan::slotStartScheduledFolderSync() if (folder) { // Safe to call several times, and necessary to try again if // the folder path didn't exist previously. - registerFolderMonitor(folder); + folder->registerFolderWatcher(); + registerFolderWithSocketApi(folder); _currentSyncFolder = folder; folder->startSync(QStringList()); @@ -964,7 +926,8 @@ Folder *FolderMan::addFolderInternal(FolderDefinition folderDefinition, connect(folder, &Folder::watchedFileChangedExternally, &folder->syncEngine().syncFileStatusTracker(), &SyncFileStatusTracker::slotPathTouched); - registerFolderMonitor(folder); + folder->registerFolderWatcher(); + registerFolderWithSocketApi(folder); return folder; } diff --git a/src/gui/folderman.h b/src/gui/folderman.h index 5802789c848..002fb9b4503 100644 --- a/src/gui/folderman.h +++ b/src/gui/folderman.h @@ -109,9 +109,6 @@ class FolderMan : public QObject static SyncResult accountStatus(const QList &folders); - void removeMonitorPath(const QString &alias, const QString &path); - void addMonitorPath(const QString &alias, const QString &path); - // Escaping of the alias which is used in QSettings AND the file // system, thus need to be escaped. static QString escapeAlias(const QString &); @@ -284,7 +281,9 @@ private slots: // finds all folder configuration files // and create the folders QString getBackupName(QString fullPathName) const; - void registerFolderMonitor(Folder *folder); + + // makes the folder known to the socket api + void registerFolderWithSocketApi(Folder *folder); // restarts the application (Linux only) void restartApplication(); @@ -298,9 +297,6 @@ private slots: QPointer _lastSyncFolder; bool _syncEnabled; - /// Watching for file changes in folders - QMap _folderWatchers; - /// Starts regular etag query jobs QTimer _etagPollTimer; /// The currently running etag query diff --git a/src/gui/folderwatcher.cpp b/src/gui/folderwatcher.cpp index 0f5fee413ee..443956d1984 100644 --- a/src/gui/folderwatcher.cpp +++ b/src/gui/folderwatcher.cpp @@ -68,6 +68,11 @@ bool FolderWatcher::pathIsIgnored(const QString &path) return false; } +bool FolderWatcher::isReliable() const +{ + return _isReliable; +} + void FolderWatcher::changeDetected(const QString &path) { QStringList paths(path); diff --git a/src/gui/folderwatcher.h b/src/gui/folderwatcher.h index 334abf1b755..ac40a2ea3bc 100644 --- a/src/gui/folderwatcher.h +++ b/src/gui/folderwatcher.h @@ -72,13 +72,29 @@ class FolderWatcher : public QObject /* Check if the path is ignored. */ bool pathIsIgnored(const QString &path); + /** + * Returns false if the folder watcher can't be trusted to capture all + * notifications. + * + * For example, this can happen on linux if the inotify user limit from + * /proc/sys/fs/inotify/max_user_watches is exceeded. + */ + bool isReliable() const; + signals: /** Emitted when one of the watched directories or one * of the contained files is changed. */ void pathChanged(const QString &path); - /** Emitted if an error occurs */ - void error(const QString &error); + /** + * Emitted if some notifications were lost. + * + * Would happen, for example, if the number of pending notifications + * exceeded the allocated buffer size on Windows. Note that the folder + * watcher could still be able to capture all future notifications - + * i.e. isReliable() is orthogonal to losing changes occasionally. + */ + void lostChanges(); protected slots: // called from the implementations to indicate a change in path @@ -93,6 +109,7 @@ protected slots: QTime _timer; QSet _lastPaths; Folder *_folder; + bool _isReliable = true; friend class FolderWatcherPrivate; }; diff --git a/src/gui/folderwatcher_linux.cpp b/src/gui/folderwatcher_linux.cpp index f0e0eb0e48d..37d79211adc 100644 --- a/src/gui/folderwatcher_linux.cpp +++ b/src/gui/folderwatcher_linux.cpp @@ -78,6 +78,12 @@ void FolderWatcherPrivate::inotifyRegisterPath(const QString &path) IN_CLOSE_WRITE | IN_ATTRIB | IN_MOVE | IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_MOVE_SELF | IN_UNMOUNT | IN_ONLYDIR); if (wd > -1) { _watches.insert(wd, path); + } else { + // If we're running out of memory or inotify watches, become + // unreliable. + if (errno == ENOMEM || errno == ENOSPC) { + _parent->_isReliable = false; + } } } } diff --git a/src/gui/folderwatcher_win.cpp b/src/gui/folderwatcher_win.cpp index 19c623f5663..4f963fbe24a 100644 --- a/src/gui/folderwatcher_win.cpp +++ b/src/gui/folderwatcher_win.cpp @@ -101,6 +101,7 @@ void WatcherThread::watchChanges(size_t fileNotifyBufferSize, DWORD errorCode = GetLastError(); if (errorCode == ERROR_NOTIFY_ENUM_DIR) { qCDebug(lcFolderWatcher) << "The buffer for changes overflowed! Triggering a generic change and resizing"; + emit lostChanges(); emit changed(_path); *increaseBufferSize = true; } else { @@ -199,6 +200,8 @@ FolderWatcherPrivate::FolderWatcherPrivate(FolderWatcher *p, const QString &path _thread = new WatcherThread(path); connect(_thread, SIGNAL(changed(const QString &)), _parent, SLOT(changeDetected(const QString &))); + connect(_thread, SIGNAL(lostChanges()), + _parent, SIGNAL(lostChanges())); _thread->start(); } diff --git a/src/gui/folderwatcher_win.h b/src/gui/folderwatcher_win.h index 687ec891d7b..7ea6730840d 100644 --- a/src/gui/folderwatcher_win.h +++ b/src/gui/folderwatcher_win.h @@ -53,6 +53,7 @@ class WatcherThread : public QThread signals: void changed(const QString &path); + void lostChanges(); private: QString _path; diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp index 6fd3df6f04d..dc7f31d2566 100644 --- a/src/libsync/configfile.cpp +++ b/src/libsync/configfile.cpp @@ -36,6 +36,7 @@ #include #define DEFAULT_REMOTE_POLL_INTERVAL 30000 // default remote poll time in milliseconds +#define DEFAULT_FULL_LOCAL_DISCOVERY_INTERVAL (60 * 60 * 1000) // 1 hour #define DEFAULT_MAX_LOG_LINES 20000 namespace OCC { @@ -45,6 +46,7 @@ Q_LOGGING_CATEGORY(lcConfigFile, "sync.configfile", QtInfoMsg) //static const char caCertsKeyC[] = "CaCertificates"; only used from account.cpp static const char remotePollIntervalC[] = "remotePollInterval"; static const char forceSyncIntervalC[] = "forceSyncInterval"; +static const char fullLocalDiscoveryIntervalC[] = "fullLocalDiscoveryInterval"; static const char notificationRefreshIntervalC[] = "notificationRefreshInterval"; static const char monoIconsC[] = "monoIcons"; static const char promptDeleteC[] = "promptDeleteAllFiles"; @@ -406,6 +408,13 @@ quint64 ConfigFile::forceSyncInterval(const QString &connection) const return interval; } +qint64 ConfigFile::fullLocalDiscoveryInterval() const +{ + QSettings settings(configFile(), QSettings::IniFormat); + settings.beginGroup(defaultConnection()); + return settings.value(QLatin1String(fullLocalDiscoveryIntervalC), DEFAULT_FULL_LOCAL_DISCOVERY_INTERVAL).toLongLong(); +} + quint64 ConfigFile::notificationRefreshInterval(const QString &connection) const { QString con(connection); diff --git a/src/libsync/configfile.h b/src/libsync/configfile.h index b9eda8eb0cd..4ef69862313 100644 --- a/src/libsync/configfile.h +++ b/src/libsync/configfile.h @@ -72,6 +72,13 @@ class OWNCLOUDSYNC_EXPORT ConfigFile /* Force sync interval, in milliseconds */ quint64 forceSyncInterval(const QString &connection = QString()) const; + /** + * Interval in milliseconds within which full local discovery is required + * + * Use -1 to disable regular full local discoveries. + */ + qint64 fullLocalDiscoveryInterval() const; + bool monoIcons() const; void setMonoIcons(bool); diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 8bb33686581..7da9d3bc4f1 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -813,6 +813,7 @@ void SyncEngine::startSync() } _csync_ctx->read_remote_from_db = true; + _lastLocalDiscoveryStyle = _csync_ctx->local_discovery_style; bool ok; auto selectiveSyncBlackList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncBlackList, &ok); @@ -1555,6 +1556,12 @@ AccountPtr SyncEngine::account() const return _account; } +void SyncEngine::setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set dirs) +{ + _csync_ctx->local_discovery_style = style; + _csync_ctx->locally_touched_dirs = std::move(dirs); +} + void SyncEngine::abort() { if (_propagator) diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h index 44c8ca7a276..0b200bb8dab 100644 --- a/src/libsync/syncengine.h +++ b/src/libsync/syncengine.h @@ -25,6 +25,7 @@ #include #include #include +#include #include @@ -92,6 +93,7 @@ class OWNCLOUDSYNC_EXPORT SyncEngine : public QObject AccountPtr account() const; SyncJournalDb *journal() const { return _journal; } QString localPath() const { return _localPath; } + /** * Minimum age, in milisecond, of a file that can be uploaded. * Files more recent than that are not going to be uploaeded as they are considered @@ -99,6 +101,22 @@ class OWNCLOUDSYNC_EXPORT SyncEngine : public QObject */ static qint64 minimumFileAgeForUpload; // in ms + /** + * Control whether local discovery should read from filesystem or db. + * + * If style is Partial, the paths is a set of file paths relative to + * the synced folder. All the parent directories of these paths will not + * be read from the db and scanned on the filesystem. + * + * Note, the style and paths are only retained for the next sync and + * revert afterwards. Use _lastLocalDiscoveryStyle to discover the last + * sync's style. + */ + void setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set dirs = {}); + + /** Access the last sync run's local discovery style */ + LocalDiscoveryStyle lastLocalDiscoveryStyle() const { return _lastLocalDiscoveryStyle; } + signals: void csyncUnavailable(); @@ -272,6 +290,9 @@ private slots: /** List of unique errors that occurred in a sync run. */ QSet _uniqueErrors; + + /** The kind of local discovery the last sync run used */ + LocalDiscoveryStyle _lastLocalDiscoveryStyle = LocalDiscoveryStyle::DatabaseAndFilesystem; }; } diff --git a/test/testsyncengine.cpp b/test/testsyncengine.cpp index 9c1b7a2c021..139fbf42b9e 100644 --- a/test/testsyncengine.cpp +++ b/test/testsyncengine.cpp @@ -613,6 +613,40 @@ private slots: QCOMPARE(nDELETE, 5); QCOMPARE(fakeFolder.currentLocalState(), remoteInfo); } + + // Check correct behavior when local discovery is partially drawn from the db + void testLocalDiscoveryStyle() + { + FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() }; + + // More subdirectories are useful for testing + fakeFolder.localModifier().mkdir("A/X"); + fakeFolder.localModifier().mkdir("A/Y"); + fakeFolder.localModifier().insert("A/X/x1"); + fakeFolder.localModifier().insert("A/Y/y1"); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + // Test begins + fakeFolder.localModifier().insert("A/a3"); + fakeFolder.localModifier().insert("A/X/x2"); + fakeFolder.localModifier().insert("A/Y/y2"); + fakeFolder.localModifier().insert("B/b3"); + fakeFolder.remoteModifier().insert("C/c3"); + + fakeFolder.syncEngine().setLocalDiscoveryOptions(LocalDiscoveryStyle::DatabaseAndFilesystem, { "A/X" }); + QVERIFY(fakeFolder.syncOnce()); + QVERIFY(fakeFolder.currentRemoteState().find("A/a3")); + QVERIFY(fakeFolder.currentRemoteState().find("A/X/x2")); + QVERIFY(!fakeFolder.currentRemoteState().find("A/Y/y2")); + QVERIFY(!fakeFolder.currentRemoteState().find("B/b3")); + QVERIFY(fakeFolder.currentLocalState().find("C/c3")); + QCOMPARE(fakeFolder.syncEngine().lastLocalDiscoveryStyle(), LocalDiscoveryStyle::DatabaseAndFilesystem); + + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + QCOMPARE(fakeFolder.syncEngine().lastLocalDiscoveryStyle(), LocalDiscoveryStyle::FilesystemOnly); + } }; QTEST_GUILESS_MAIN(TestSyncEngine)