Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support blocking monitor loop and full scan separation #2519

Closed
wants to merge 9 commits into from
Closed

Support blocking monitor loop and full scan separation #2519

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
116a7b9
Replace inotify polling
JC-comp Oct 18, 2023
90c6200
Add webhook support for blocking wait
JC-comp Oct 20, 2023
7d11a8d
Refine shutdown cleanup
JC-comp Oct 20, 2023
3db932e
Seperate webhook callback
JC-comp Oct 22, 2023
0d1b044
Fix missing inotify event
JC-comp Oct 22, 2023
51163d5
Fix backup action
JC-comp Oct 22, 2023
8ac4038
Merge 58d17d5e898b7b3d428a693e6eac760aa11c8bb2
JC-comp Oct 25, 2023
9f0019b
Merge branch 'onedrive-v2.5.0-alpha-3' into blocking_loop
JC-comp Oct 25, 2023
4aa32d2
Fix retry interval wait
JC-comp Oct 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 105 additions & 34 deletions src/main.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ int main(string[] cliArgs) {
// DEVELOPER OPTIONS OUTPUT VARIABLES
bool displayMemoryUsage = false;
bool displaySyncOptions = false;
bool monitorFailures = false;

// Define 'exit' and 'failure' scopes
scope(exit) {
Expand Down Expand Up @@ -606,6 +607,7 @@ int main(string[] cliArgs) {
}

// Configure the monitor class
Tid workerTid;
filesystemMonitor = new Monitor(appConfig, selectiveSync);

// Delegated function for when inotify detects a new local directory has been created
Expand Down Expand Up @@ -685,6 +687,7 @@ int main(string[] cliArgs) {
try {
log.log("Initialising filesystem inotify monitoring ...");
filesystemMonitor.initialise();
workerTid = filesystemMonitor.watch();
log.log("Performing initial syncronisation to ensure consistent local state ...");
} catch (MonitorException e) {
// monitor class initialisation failed
Expand All @@ -709,6 +712,7 @@ int main(string[] cliArgs) {
MonoTime lastGitHubCheckTime = MonoTime.currTime();
string loopStartOutputMessage = "################################################## NEW LOOP ##################################################";
string loopStopOutputMessage = "################################################ LOOP COMPLETE ###############################################";
bool notificationReceived = false;

while (performMonitor) {
// Do we need to validate the runtimeSyncDirectory to check for the presence of a '.nosync' file - the disk may have been ejected ..
Expand All @@ -726,40 +730,19 @@ int main(string[] cliArgs) {
}

// Webhook Notification Handling
bool notificationReceived = false;

// Check for notifications pushed from Microsoft to the webhook
if (webhookEnabled) {
// Create a subscription on the first run, or renew the subscription
// on subsequent runs when it is about to expire.
oneDriveApiInstance.createOrRenewSubscription();

// Process incoming notifications if any.

// Empirical evidence shows that Microsoft often sends multiple
// notifications for one single change, so we need a loop to exhaust
// all signals that were queued up by the webhook. The notifications
// do not contain any actual changes, and we will always rely do the
// delta endpoint to sync to latest. Therefore, only one sync run is
// good enough to catch up for multiple notifications.
for (int signalCount = 0;; signalCount++) {
const auto signalExists = receiveTimeout(dur!"seconds"(-1), (ulong _) {});
if (signalExists) {
notificationReceived = true;
} else {
if (notificationReceived) {
log.log("Received ", signalCount," refresh signals from the webhook");
}
break;
}
}
}

// Get the current time this loop is starting
auto currentTime = MonoTime.currTime();

// Do we perform a sync with OneDrive?
if (notificationReceived || (currentTime - lastCheckTime > checkOnlineInterval) || (monitorLoopFullCount == 0)) {
if ((currentTime - lastCheckTime >= checkOnlineInterval) || (monitorLoopFullCount == 0)) {
// Increment relevant counters
monitorLoopFullCount++;
fullScanFrequencyLoopCount++;
Expand Down Expand Up @@ -837,8 +820,8 @@ int main(string[] cliArgs) {
performStandardSyncProcess(localPath, filesystemMonitor);
}

// Discard any inotify events generated as part of any sync operation
filesystemMonitor.update(false);
// Handle any new inotify events
filesystemMonitor.update(true);

// Detail the outcome of the sync process
displaySyncOutcome();
Expand Down Expand Up @@ -881,8 +864,77 @@ int main(string[] cliArgs) {
}
}
}
// Sleep the monitor thread for 1 second, loop around and pick up any inotify changes
Thread.sleep(dur!"seconds"(1));

if(performMonitor) {
auto nextCheckTime = lastCheckTime + checkOnlineInterval;
currentTime = MonoTime.currTime();
auto sleepTime = nextCheckTime - currentTime;
log.vdebug("Sleep for ", sleepTime);

if(filesystemMonitor.initialised || webhookEnabled) {
if(filesystemMonitor.initialised) {
// If local monitor is on
// start the worker and wait for event
if(!filesystemMonitor.isWorking()) {
workerTid.send(1);
}
}

if(webhookEnabled) {
// if onedrive webhook is enabled
// update sleep time based on renew interval
Duration nextWebhookCheckDuration = oneDriveApiInstance.getNextExpirationCheckDuration();
if (nextWebhookCheckDuration < sleepTime) {
sleepTime = nextWebhookCheckDuration;
log.vdebug("Update sleeping time to ", sleepTime);
}
notificationReceived = false;
}

int res = 1;
// Process incoming notifications if any.
auto signalExists = receiveTimeout(sleepTime,
(int msg) {
res = msg;
},
(ulong _) {
notificationReceived = true;
}
);
log.vdebug("signalExists = ", signalExists);
log.vdebug("worker status = ", res);
log.vdebug("notificationReceived = ", notificationReceived);

// Empirical evidence shows that Microsoft often sends multiple
// notifications for one single change, so we need a loop to exhaust
// all signals that were queued up by the webhook. The notifications
// do not contain any actual changes, and we will always rely do the
// delta endpoint to sync to latest. Therefore, only one sync run is
// good enough to catch up for multiple notifications.
int signalCount = notificationReceived ? 1 : 0;
for (;; signalCount++) {
signalExists = receiveTimeout(dur!"seconds"(-1), (ulong _) {});
if (signalExists) {
notificationReceived = true;
} else {
if (notificationReceived) {
log.log("Received ", signalCount," refresh signals from the webhook");
oneDriveWebhookCallback();
}
break;
}
}

if(res == -1) {
log.error("Error: Monitor worker failed.");
monitorFailures = true;
performMonitor = false;
}
} else {
// no hooks available, nothing to check
Thread.sleep(sleepTime);
}
}
}
}
} else {
Expand All @@ -893,7 +945,7 @@ int main(string[] cliArgs) {
}

// Exit application using exit scope
if (!syncEngineInstance.syncFailures) {
if (!syncEngineInstance.syncFailures && !monitorFailures) {
return EXIT_SUCCESS;
} else {
return EXIT_FAILURE;
Expand All @@ -907,7 +959,6 @@ void performStandardExitProcess() {
itemDB.performVacuum();
object.destroy(itemDB);
}

// Free other objects and memory
if (appConfig !is null) {
// Cleanup any existing dry-run elements ... these should never be left hanging around
Expand All @@ -917,13 +968,33 @@ void performStandardExitProcess() {
if (oneDriveApiInstance !is null) object.destroy(oneDriveApiInstance);
if (selectiveSync !is null) object.destroy(selectiveSync);
if (syncEngineInstance !is null) object.destroy(syncEngineInstance);

if (filesystemMonitor !is null) {
// cleanup hooks
if (filesystemMonitor !is null && filesystemMonitor.initialised) {
filesystemMonitor.shutdown();
object.destroy(filesystemMonitor);
}
}

void oneDriveWebhookCallback() {
// If we are in a --download-only method of operation, there is no filesystem monitoring, so no inotify events to check
if (!appConfig.getValueBool("download_only")) {
try {
// Process any inotify events
filesystemMonitor.update(true);
} catch (MonitorException e) {
// Catch any exceptions thrown by inotify / monitor engine
log.error("ERROR: The following inotify error was generated: ", e.msg);
}
}

// Download data from OneDrive last
syncEngineInstance.syncOneDriveAccountToLocalDisk();
if (appConfig.getValueBool("monitor")) {
// Handle any new inotify events
filesystemMonitor.update(true);
}
}

void performUploadOnlySyncProcess(string localPath, Monitor filesystemMonitor = null) {
// Perform the local database consistency check, picking up locally modified data and uploading this to OneDrive
syncEngineInstance.performDatabaseConsistencyAndIntegrityCheck();
Expand Down Expand Up @@ -973,16 +1044,16 @@ void performStandardSyncProcess(string localPath, Monitor filesystemMonitor = nu
// Download data from OneDrive last
syncEngineInstance.syncOneDriveAccountToLocalDisk();
if (appConfig.getValueBool("monitor")) {
// Cancel out any inotify events from downloading data
filesystemMonitor.update(false);
// Handle any new inotify events
filesystemMonitor.update(true);
}
} else {
// Normal sync
// Download data from OneDrive first
syncEngineInstance.syncOneDriveAccountToLocalDisk();
if (appConfig.getValueBool("monitor")) {
// Cancel out any inotify events from downloading data
filesystemMonitor.update(false);
// Handle any new inotify events
filesystemMonitor.update(true);
}


Expand Down
Loading