From be3d28bf3cc71471580858c3eb066295fc3286df Mon Sep 17 00:00:00 2001 From: abraunegg Date: Sun, 28 Apr 2024 17:18:25 +1000 Subject: [PATCH] Release files for 2.5.0-rc2 * Release files for 2.5.0-rc2 * Code changes from 2.5.0-rc1 --> 2.5.0-rc2 (#2686) * Update docs regarding Ubuntu 24.04 --- configure | 20 +- configure.ac | 2 +- docs/application-config-options.md | 2 +- docs/install.md | 3 +- docs/ubuntu-package-install.md | 27 + src/clientSideFiltering.d | 7 + src/config.d | 29 +- src/curlEngine.d | 328 ++++-- src/log.d | 14 +- src/main.d | 315 +++--- src/monitor.d | 8 +- src/onedrive.d | 1401 ++++++++++--------------- src/sync.d | 1559 ++++++++++------------------ src/util.d | 244 +++-- src/webhook.d | 6 +- 15 files changed, 1817 insertions(+), 2148 deletions(-) diff --git a/configure b/configure index ef0d5acc1..a39f764e6 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for onedrive v2.5.0-rc1. +# Generated by GNU Autoconf 2.69 for onedrive v2.5.0-rc2. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='onedrive' PACKAGE_TARNAME='onedrive' -PACKAGE_VERSION='v2.5.0-rc1' -PACKAGE_STRING='onedrive v2.5.0-rc1' +PACKAGE_VERSION='v2.5.0-rc2' +PACKAGE_STRING='onedrive v2.5.0-rc2' PACKAGE_BUGREPORT='https://github.com/abraunegg/onedrive' PACKAGE_URL='' @@ -1219,7 +1219,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures onedrive v2.5.0-rc1 to adapt to many kinds of systems. +\`configure' configures onedrive v2.5.0-rc2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1280,7 +1280,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of onedrive v2.5.0-rc1:";; + short | recursive ) echo "Configuration of onedrive v2.5.0-rc2:";; esac cat <<\_ACEOF @@ -1393,7 +1393,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -onedrive configure v2.5.0-rc1 +onedrive configure v2.5.0-rc2 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1410,7 +1410,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by onedrive $as_me v2.5.0-rc1, which was +It was created by onedrive $as_me v2.5.0-rc2, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2162,7 +2162,7 @@ fi -PACKAGE_DATE="March 2024" +PACKAGE_DATE="April 2024" @@ -3159,7 +3159,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by onedrive $as_me v2.5.0-rc1, which was +This file was extended by onedrive $as_me v2.5.0-rc2, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -3212,7 +3212,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -onedrive config.status v2.5.0-rc1 +onedrive config.status v2.5.0-rc2 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index e4c9ce9be..54395bf0c 100644 --- a/configure.ac +++ b/configure.ac @@ -9,7 +9,7 @@ dnl - commit the changed files (configure.ac, configure) dnl - tag the release AC_PREREQ([2.69]) -AC_INIT([onedrive],[v2.5.0-rc1], [https://github.com/abraunegg/onedrive], [onedrive]) +AC_INIT([onedrive],[v2.5.0-rc2], [https://github.com/abraunegg/onedrive], [onedrive]) AC_CONFIG_SRCDIR([src/main.d]) diff --git a/docs/application-config-options.md b/docs/application-config-options.md index 0aed32ac0..041f39456 100644 --- a/docs/application-config-options.md +++ b/docs/application-config-options.md @@ -206,7 +206,7 @@ _**Description:**_ This setting controls the timeout duration, in seconds, for w _**Value Type:**_ Integer -_**Default Value:**_ 240 +_**Default Value:**_ 60 _**Config Example:**_ `data_timeout = "300"` diff --git a/docs/install.md b/docs/install.md index e79ce5680..87692a933 100644 --- a/docs/install.md +++ b/docs/install.md @@ -31,6 +31,7 @@ Only the current release version or greater is supported. Earlier versions are n | Ubuntu 20.04 | [onedrive](https://packages.ubuntu.com/focal/onedrive) |Ubuntu 20.04 package |❌|✔|✔|✔| **Note:** Do not install from Ubuntu Universe as the package is obsolete and is not supported

For a supported application version, it is recommended that for Ubuntu that you install from OpenSuSE Build Service using the Ubuntu Package Install [Instructions](ubuntu-package-install.md) | | Ubuntu 22.04 | [onedrive](https://packages.ubuntu.com/jammy/onedrive) |Ubuntu 22.04 package |❌|✔|✔|✔| **Note:** Do not install from Ubuntu Universe as the package is obsolete and is not supported

For a supported application version, it is recommended that for Ubuntu that you install from OpenSuSE Build Service using the Ubuntu Package Install [Instructions](ubuntu-package-install.md) | | Ubuntu 23.04 | [onedrive](https://packages.ubuntu.com/lunar/onedrive) |Ubuntu 23.04 package |❌|✔|✔|✔| **Note:** Do not install from Ubuntu Universe as the package is obsolete and is not supported

For a supported application version, it is recommended that for Ubuntu that you install from OpenSuSE Build Service using the Ubuntu Package Install [Instructions](ubuntu-package-install.md) | +| Ubuntu 24.04 | [onedrive](https://packages.ubuntu.com/noble/onedrive) |Ubuntu 24.04 package |❌|✔|✔|✔| **Note:** Do not install from Ubuntu Universe as the package is obsolete and is not supported

For a supported application version, it is recommended that for Ubuntu that you install from OpenSuSE Build Service using the Ubuntu Package Install [Instructions](ubuntu-package-install.md) | | Void Linux | [onedrive](https://voidlinux.org/packages/?arch=x86_64&q=onedrive) |Void Linux x86_64 package|✔|✔|❌|❌| | ## Building from Source - High Level Requirements @@ -61,7 +62,7 @@ Ubuntu Linux 18.x LTS reached the end of its five-year LTS window on May 31th 20 ### Dependencies: Debian 9 Debian 9 reached the end of its five-year support window on June 30th 2022 and is no longer supported. -### Dependencies: Ubuntu 20.x -> Ubuntu 23.x / Debian 10 -> Debian 12 - x86_64 +### Dependencies: Ubuntu 20.x -> Ubuntu 24.x / Debian 10 -> Debian 12 - x86_64 These dependencies are also applicable for all Ubuntu based distributions such as: * Lubuntu * Linux Mint diff --git a/docs/ubuntu-package-install.md b/docs/ubuntu-package-install.md index 04d8c905c..5e6a36023 100644 --- a/docs/ubuntu-package-install.md +++ b/docs/ubuntu-package-install.md @@ -171,6 +171,7 @@ If required, review the table below based on your 'lsb_release' information to p | Ubuntu 22.10 / Kinetic | Use [Ubuntu 22.10](#distribution-ubuntu-2210) instructions below | | Ubuntu 23.04 / Lunar | Use [Ubuntu 23.04](#distribution-ubuntu-2304) instructions below | | Ubuntu 23.10 / Mantic | Use [Ubuntu 23.10](#distribution-ubuntu-2310) instructions below | +| Ubuntu 24.04 / Noble | Use [Ubuntu 24.04](#distribution-ubuntu-2404) instructions below | > [!IMPORTANT] > If your Linux distribution and release is not in the table above, you have 2 options: @@ -423,6 +424,32 @@ Run: `sudo apt install --no-install-recommends --no-install-suggests onedrive` #### Step 5: Read 'Known Issues' with these packages Read and understand the [known issues](#known-issues-with-installing-from-the-above-packages) with these packages below, taking any action that is needed. +### Distribution: Ubuntu 24.04 +The packages support the following platform architectures: +|  i686  | x86_64 | ARMHF | AARCH64 | +|:----:|:------:|:-----:|:-------:| +|❌|✔|❌|✔| + +#### Step 1: Add the OpenSuSE Build Service repository release key +Add the OpenSuSE Build Service repository release key using the following command: +```text +wget -qO - https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_24.04/Release.key | gpg --dearmor | sudo tee /usr/share/keyrings/obs-onedrive.gpg > /dev/null +``` + +#### Step 2: Add the OpenSuSE Build Service repository +Add the OpenSuSE Build Service repository using the following command: +```text +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/obs-onedrive.gpg] https://download.opensuse.org/repositories/home:/npreining:/debian-ubuntu-onedrive/xUbuntu_24.04/ ./" | sudo tee /etc/apt/sources.list.d/onedrive.list +``` + +#### Step 3: Update your apt package cache +Run: `sudo apt-get update` + +#### Step 4: Install 'onedrive' +Run: `sudo apt install --no-install-recommends --no-install-suggests onedrive` + +#### Step 5: Read 'Known Issues' with these packages +Read and understand the [known issues](#known-issues-with-installing-from-the-above-packages) with these packages below, taking any action that is needed. ## Known Issues with Installing from the above packages diff --git a/src/clientSideFiltering.d b/src/clientSideFiltering.d index 5e9a70fc9..8b8d15a48 100644 --- a/src/clientSideFiltering.d +++ b/src/clientSideFiltering.d @@ -30,6 +30,13 @@ class ClientSideFiltering { this.appConfig = appConfig; } + ~this() { + object.destroy(appConfig); + object.destroy(paths); + object.destroy(fileMask); + object.destroy(directoryMask); + } + // Initialise the required items bool initialise() { // Log what is being done diff --git a/src/config.d b/src/config.d index c68d691c0..d70407f21 100644 --- a/src/config.d +++ b/src/config.d @@ -61,17 +61,17 @@ class ApplicationConfig { // HTTP Struct items, used for configuring HTTP() // Curl Timeout Handling // libcurl dns_cache_timeout timeout - immutable int defaultDnsTimeout = 60; + immutable int defaultDnsTimeout = 60; // in seconds // Connect timeout for HTTP|HTTPS connections // Controls CURLOPT_CONNECTTIMEOUT - immutable int defaultConnectTimeout = 10; - // Default data timeout for HTTP + immutable int defaultConnectTimeout = 10; // in seconds + // Default data timeout for HTTP operations // curl.d has a default of: _defaultDataTimeout = dur!"minutes"(2); - immutable int defaultDataTimeout = 240; + immutable int defaultDataTimeout = 60; // in seconds // Maximum time any operation is allowed to take // This includes dns resolution, connecting, data transfer, etc. // Controls CURLOPT_TIMEOUT - immutable int defaultOperationTimeout = 3600; + immutable int defaultOperationTimeout = 3600; // in seconds // Specify what IP protocol version should be used when communicating with OneDrive immutable int defaultIpProtocol = 0; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only // Specify how many redirects should be allowed @@ -682,11 +682,22 @@ class ApplicationConfig { } auto file = File(filename, "r"); - scope(exit) file.close(); - scope(failure) file.close(); - + string lineBuffer; + + scope(exit) { + file.close(); + object.destroy(file); + object.destroy(lineBuffer); + } + + scope(failure) { + file.close(); + object.destroy(file); + object.destroy(lineBuffer); + } + foreach (line; file.byLine()) { - string lineBuffer = stripLeft(line).to!string; + lineBuffer = stripLeft(line).to!string; if (lineBuffer.empty || lineBuffer[0] == ';' || lineBuffer[0] == '#') continue; auto c = lineBuffer.matchFirst(configRegex); if (c.empty) { diff --git a/src/curlEngine.d b/src/curlEngine.d index efc2a6de0..fc07ed611 100644 --- a/src/curlEngine.d +++ b/src/curlEngine.d @@ -3,7 +3,7 @@ module curlEngine; // What does this module require to function? import std.net.curl; -import etc.c.curl: CurlOption; +import etc.c.curl; import std.datetime; import std.conv; import std.file; @@ -13,6 +13,7 @@ import std.range; // What other modules that we have created do we need to import? import log; +import util; class CurlResponse { HTTP.Method method; @@ -20,23 +21,28 @@ class CurlResponse { const(char)[][const(char)[]] requestHeaders; const(char)[] postBody; + bool hasResponse; string[string] responseHeaders; HTTP.StatusLine statusLine; char[] content; + this() { + reset(); + } + void reset() { method = HTTP.Method.undefined; - url = null; + url = ""; requestHeaders = null; - postBody = null; - + postBody = []; + hasResponse = false; responseHeaders = null; - object.destroy(statusLine); - content = null; + statusLine.reset(); + content = []; } void addRequestHeader(const(char)[] name, const(char)[] value) { - requestHeaders[name] = value; + requestHeaders[to!string(name)] = to!string(value); } void connect(HTTP.Method method, const(char)[] url) { @@ -56,6 +62,7 @@ class CurlResponse { }; void update(HTTP *http) { + hasResponse = true; this.responseHeaders = http.responseHeaders(); this.statusLine = http.statusLine; } @@ -65,40 +72,48 @@ class CurlResponse { } // Return the current value of retryAfterValue - ulong getRetryAfterValue() { - ulong delayBeforeRetry; - // is retry-after in the response headers + int getRetryAfterValue() { + int delayBeforeRetry; + // Is 'retry-after' in the response headers if ("retry-after" in responseHeaders) { // Set the retry-after value addLogEntry("curlEngine.http.perform() => Received a 'Retry-After' Header Response with the following value: " ~ to!string(responseHeaders["retry-after"]), ["debug"]); addLogEntry("curlEngine.http.perform() => Setting retryAfterValue to: " ~ responseHeaders["retry-after"], ["debug"]); - delayBeforeRetry = to!ulong(responseHeaders["retry-after"]); + delayBeforeRetry = to!int(responseHeaders["retry-after"]); } else { // Use a 120 second delay as a default given header value was zero // This value is based on log files and data when determining correct process for 429 response handling delayBeforeRetry = 120; // Update that we are over-riding the provided value with a default - addLogEntry("HTTP Response Header retry-after value was 0 - Using a preconfigured default of: " ~ to!string(delayBeforeRetry), ["debug"]); + addLogEntry("HTTP Response Header retry-after value was missing - Using a preconfigured default of: " ~ to!string(delayBeforeRetry), ["debug"]); } - - return delayBeforeRetry; // default to 60 seconds + return delayBeforeRetry; } - - const string parseHeaders(const(string[string]) headers) { - string responseHeadersStr = ""; - foreach (const(char)[] header; headers.byKey()) { - responseHeadersStr ~= "> " ~ header ~ ": " ~ headers[header] ~ "\n"; + + const string parseRequestHeaders(const(const(char)[][const(char)[]]) headers) { + string requestHeadersStr = ""; + foreach (string header; headers.byKey()) { + if (header == "Authorization") { + continue; + } + // Use the 'in' operator to safely check if the key exists in the associative array. + if (auto val = header in headers) { + requestHeadersStr ~= "< " ~ header ~ ": " ~ *val ~ "\n"; + } } - return responseHeadersStr; + return requestHeadersStr; } - - const string parseHeaders(const(const(char)[][const(char)[]]) headers) { + const string parseResponseHeaders(const(immutable(char)[][immutable(char)[]]) headers) { string responseHeadersStr = ""; - foreach (string header; headers.byKey()) { - if (header == "Authorization") - continue; - responseHeadersStr ~= "< " ~ header ~ ": " ~ headers[header] ~ "\n"; + // Ensure response headers is not null and iterate over keys safely. + if (headers !is null) { + foreach (const(char)[] header; headers.byKey()) { + // Check if the key actually exists before accessing it to avoid RangeError. + if (auto val = header in headers) { // 'in' checks for the key and returns a pointer to the value if found. + responseHeadersStr ~= "> " ~ header ~ ": " ~ *val ~ "\n"; // Dereference pointer to get the value. + } + } } return responseHeadersStr; } @@ -110,14 +125,14 @@ class CurlResponse { string str = ""; str ~= format("< %s %s\n", method, url); if (!requestHeaders.empty) { - str ~= parseHeaders(requestHeaders); + str ~= parseRequestHeaders(requestHeaders); } if (!postBody.empty) { - str ~= format("----\n%s\n----\n", postBody); + str ~= format("\n----\n%s\n----\n", postBody); } str ~= format("< %s\n", statusLine); if (!responseHeaders.empty) { - str ~= parseHeaders(responseHeaders); + str ~= parseResponseHeaders(responseHeaders); } return str; } @@ -128,7 +143,7 @@ class CurlResponse { string str = ""; if (!content.empty) { - str ~= format("----\n%s\n----\n", content); + str ~= format("\n----\n%s\n----\n", content); } return str; } @@ -136,73 +151,120 @@ class CurlResponse { override string toString() const { string str = "Curl debugging: \n"; str ~= dumpDebug(); - str ~= "Curl response: \n"; - str ~= dumpResponse(); + if (hasResponse) { + str ~= "Curl response: \n"; + str ~= dumpResponse(); + } return str; } - - CurlResponse dup() { - CurlResponse copy = new CurlResponse(); - copy.method = method; - copy.url = url; - copy.requestHeaders = requestHeaders; - copy.postBody = postBody; - - copy.responseHeaders = responseHeaders; - copy.statusLine = statusLine; - copy.content = content; - - return copy; - } } class CurlEngine { - __gshared CurlEngine[] curlEnginePool; + // Shared pool of CurlEngine instances accessible across all threads + __gshared CurlEngine[] curlEnginePool; // __gshared is used to declare a variable that is shared across all threads + + HTTP http; + File uploadFile; + CurlResponse response; + bool keepAlive; + ulong dnsTimeout; + string internalThreadId; + + this() { + http = HTTP(); // Directly initializes HTTP using its default constructor + response = null; // Initialize as null + internalThreadId = generateAlphanumericString(); + } - static CurlEngine get() { - synchronized(CurlEngine.classinfo) { + // The destructor should only clean up resources owned directly by this instance + ~this() { + // Is the file still open? + if (uploadFile.isOpen()) { + uploadFile.close(); + } + + // Is 'response' cleared? + if (response !is null) { + object.destroy(response); // Destroy, then set to null + response = null; + } + + // Is the actual http instance is stopped? + if (!http.isStopped) { + // HTTP instance was not stopped .. need to stop it + http.shutdown(); + object.destroy(http); // Destroy, however we cant set to null + } + } + + static CurlEngine getCurlInstance() { + synchronized (CurlEngine.classinfo) { + // What is the current pool size + addLogEntry("CURL ENGINE AVAILABLE POOL SIZE: " ~ to!string(curlEnginePool.length), ["debug"]); + if (curlEnginePool.empty) { - return new CurlEngine; + addLogEntry("CURL ENGINE POOL EMPTY - CONSTRUCTING A NEW CURL ENGINE INSTANCE" , ["debug"]); + return new CurlEngine; // Constructs a new CurlEngine with a fresh HTTP instance } else { - CurlEngine curlEngine = curlEnginePool[$-1]; + CurlEngine curlEngine = curlEnginePool[$ - 1]; curlEnginePool.popBack(); - return curlEngine; - } - } - } - - static releaseAll() { - synchronized(CurlEngine.classinfo) { - foreach(curlEngine; curlEnginePool) { - curlEngine.shutdown(); - object.destroy(curlEngine); + + // Is this engine stopped? + if (curlEngine.http.isStopped) { + // return a new curl engine as a stopped one cannot be used + addLogEntry("CURL ENGINE WAS STOPPED - CONSTRUCTING A NEW CURL ENGINE INSTANCE" , ["debug"]); + return new CurlEngine; // Constructs a new CurlEngine with a fresh HTTP instance + } else { + // return an existing curl engine + addLogEntry("CURL ENGINE WAS VALID - RETURNED AN EXISTING CURL ENGINE INSTANCE" , ["debug"]); + addLogEntry("CURL ENGINE ID: " ~ curlEngine.internalThreadId, ["debug"]); + return curlEngine; + } } - curlEnginePool = null; } } - - void release() { - cleanUp(); - synchronized(CurlEngine.classinfo) { - curlEnginePool ~= this; - } - } - - HTTP http; - bool keepAlive; - ulong dnsTimeout; - CurlResponse response; - - this() { - http = HTTP(); - response = new CurlResponse(); - } - - ~this() { - object.destroy(http); - object.destroy(response); - } + + static void releaseAllCurlInstances() { + synchronized (CurlEngine.classinfo) { + // What is the current pool size + addLogEntry("CURL ENGINES TO RELEASE: " ~ to!string(curlEnginePool.length), ["debug"]); + + // Safely iterate and clean up each CurlEngine instance + foreach (curlEngineInstance; curlEnginePool) { + try { + curlEngineInstance.cleanup(); // Cleanup instance by resetting values + curlEngineInstance.shutdownCurlHTTPInstance(); // Assume proper cleanup of any resources used by HTTP + } catch (Exception e) { + // Log the error or handle it appropriately + // e.g., writeln("Error during cleanup/shutdown: ", e.toString()); + } + // It's safe to destroy the object here assuming no other references exist + object.destroy(curlEngineInstance); // Destroy, then set to null + curlEngineInstance = null; + } + // Clear the array after all instances have been handled + curlEnginePool.length = 0; // More explicit than curlEnginePool = []; + } + } + + // Destroy all curl instances + static void destroyAllCurlInstances() { + addLogEntry("DESTROY ALL CURL ENGINES", ["debug"]); + // Release all 'curl' instances + releaseAllCurlInstances(); + } + + // We are releasing a curl instance back to the pool + void releaseEngine() { + addLogEntry("CurlEngine releaseEngine() CALLED", ["debug"]); + addLogEntry("CURRENT CURL ENGINE AVAILABLE POOL SIZE: " ~ to!string(curlEnginePool.length), ["debug"]); + cleanup(); + synchronized (CurlEngine.classinfo) { + curlEnginePool ~= this; + addLogEntry("CURL ENGINE POOL SIZE AFTER RELEASE BACK TO POOL: " ~ to!string(curlEnginePool.length), ["debug"]); + } + } void initialise(ulong dnsTimeout, ulong connectTimeout, ulong dataTimeout, ulong operationTimeout, int maxRedirects, bool httpsDebug, string userAgent, bool httpProtocol, ulong userRateLimit, ulong protocolVersion, bool keepAlive=true) { // Setting this to false ensures that when we close the curl instance, any open sockets are closed - which we need to do when running @@ -287,12 +349,24 @@ class CurlEngine { } } + void setResponseHolder(CurlResponse response) { + if (response is null) { + // Create a response instance if it doesn't already exist + if (this.response is null) + this.response = new CurlResponse(); + } else { + this.response = response; + } + } + void addRequestHeader(const(char)[] name, const(char)[] value) { + setResponseHolder(null); http.addRequestHeader(name, value); response.addRequestHeader(name, value); } void connect(HTTP.Method method, const(char)[] url) { + setResponseHolder(null); if (!keepAlive) addRequestHeader("Connection", "close"); http.method = method; @@ -301,6 +375,7 @@ class CurlEngine { } void setContent(const(char)[] contentType, const(char)[] sendData) { + setResponseHolder(null); addRequestHeader("Content-Type", contentType); if (sendData) { http.contentLength = sendData.length; @@ -316,16 +391,33 @@ class CurlEngine { } } - void setFile(File* file, ulong offsetSize) { + void setFile(string filepath, string contentRange, ulong offset, ulong offsetSize) { + setResponseHolder(null); + // open file as read-only in binary mode + uploadFile = File(filepath, "rb"); + + if (contentRange.empty) { + offsetSize = uploadFile.size(); + } else { + addRequestHeader("Content-Range", contentRange); + uploadFile.seek(offset); + } + + // Setup progress bar to display + http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) { + return 0; + }; + addRequestHeader("Content-Type", "application/octet-stream"); - http.onSend = data => file.rawRead(data).length; + http.onSend = data => uploadFile.rawRead(data).length; http.contentLength = offsetSize; } CurlResponse execute() { scope(exit) { - cleanUp(); + cleanup(); } + setResponseHolder(null); http.onReceive = (ubyte[] data) { response.content ~= data; // HTTP Server Response Code Debugging if --https-debug is being used @@ -334,20 +426,17 @@ class CurlEngine { }; http.perform(); response.update(&http); - return response.dup; + return response; } CurlResponse download(string originalFilename, string downloadFilename) { - // Threshold for displaying download bar - long thresholdFileSize = 4 * 2^^20; // 4 MiB - - CurlResponse response = new CurlResponse(); + setResponseHolder(null); // open downloadFilename as write in binary mode auto file = File(downloadFilename, "wb"); // function scopes scope(exit) { - cleanUp(); + cleanup(); if (file.isOpen()){ // close open file file.close(); @@ -368,27 +457,44 @@ class CurlEngine { return response; } - void cleanUp() { + void cleanup() { // Reset any values to defaults, freeing any set objects - http.clearRequestHeaders(); - http.onSend = null; - http.onReceive = null; - http.onReceiveHeader = null; - http.onReceiveStatusLine = null; - http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) { - return 0; - }; - http.contentLength = 0; - response.reset(); + addLogEntry("CurlEngine cleanup() CALLED", ["debug"]); + + // Is the instance is stopped? + if (!http.isStopped) { + // A stopped instance is not usable, these cannot be reset + http.clearRequestHeaders(); + http.onSend = null; + http.onReceive = null; + http.onReceiveHeader = null; + http.onReceiveStatusLine = null; + http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) { + return 0; + }; + http.contentLength = 0; + http.flushCookieJar(); + http.clearSessionCookies(); + http.clearAllCookies(); + } + + // set the response to null + response = null; + + // close file if open + if (uploadFile.isOpen()){ + // close open file + uploadFile.close(); + } } - void shutdown() { + void shutdownCurlHTTPInstance() { // Shut down the curl instance & close any open sockets - http.shutdown(); - } - - void setDisableSSLVerifyPeer() { - addLogEntry("CAUTION: Switching off CurlOption.ssl_verifypeer ... this makes the application insecure.", ["debug"]); - http.handle.set(CurlOption.ssl_verifypeer, 0); + addLogEntry("HTTP SHUTDOWN CALLED ..." , ["debug"]); + + // Is the instance is stopped? + if (!http.isStopped) { + http.shutdown(); + } } } \ No newline at end of file diff --git a/src/log.d b/src/log.d index 21977c275..904fc22ab 100644 --- a/src/log.d +++ b/src/log.d @@ -52,7 +52,14 @@ class LogBuffer { flushThread.isDaemon(true); flushThread.start(); } - + + // The destructor should only clean up resources owned directly by this instance + ~this() { + object.destroy(bufferLock); + object.destroy(condReady); + object.destroy(flushThread); + } + void shutdown() { synchronized(bufferLock) { if (!isRunning) return; // Prevent multiple shutdowns @@ -62,6 +69,7 @@ class LogBuffer { flushThread.join(); // Wait for the flush thread to finish flush(); // Perform a final flush to ensure all data is processed } + shared void logThisMessage(string message, string[] levels = ["info"]) { // Generate the timestamp for this log entry auto timeStamp = leftJustify(Clock.currTime().toString(), 28, '0'); @@ -100,7 +108,9 @@ class LogBuffer { // Use dnotify's functionality for GUI notifications, if GUI notifications is enabled version(Notifications) { try { - auto n = new Notification("Log Notification", message, "IGNORED"); + auto n = new Notification("OneDrive Client", message, "IGNORED"); + // Show notification for 10 seconds + n.timeout = 10; n.show(); } catch (NotificationError e) { sendGUINotification = false; diff --git a/src/main.d b/src/main.d index cd5a0a766..77535faeb 100644 --- a/src/main.d +++ b/src/main.d @@ -4,6 +4,7 @@ module main; // What does this module require to function? import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit; import core.stdc.signal; +import core.sys.posix.signal; import core.memory; import core.time; import core.thread; @@ -44,7 +45,19 @@ ItemDatabase itemDB; ClientSideFiltering selectiveSync; Monitor filesystemMonitor; +// Class variables +// Flag for performing a synchronised shutdown +bool shutdownInProgress = false; +// Flag if a --dry-run is being performed, as, on shutdown, once config is destroyed, we have no reference here +bool dryRun = false; +// Configure the runtime database file path so that it is available to us on shutdown so objects can be destroyed and removed if required +// - Typically this will be the default, but in a --dry-run scenario, we use a separate database file +string runtimeDatabaseFile = ""; + int main(string[] cliArgs) { + // Setup CTRL-C handler + setupSignalHandler(); + // Application Start Time - used during monitor loop to detail how long it has been running for auto applicationStartTime = Clock.currTime(); // Disable buffering on stdout - this is needed so that when we are using plain write() it will go to the terminal without flushing @@ -63,8 +76,7 @@ int main(string[] cliArgs) { // What is the runtime syncronisation directory that will be used // Typically this will be '~/OneDrive' .. however tilde expansion is unreliable string runtimeSyncDirectory = ""; - // Configure the runtime database file path. Typically this will be the default, but in a --dry-run scenario, we use a separate database file - string runtimeDatabaseFile = ""; + // Verbosity Logging Count - this defines if verbose or debug logging is being used long verbosityCount = 0; // Application Logging Level @@ -86,15 +98,15 @@ int main(string[] cliArgs) { scope(exit) { // Detail what scope was called addLogEntry("Exit scope was called", ["debug"]); - // Perform exit tasks - performStandardExitProcess("exitScope"); + // Perform synchronised exit + performSynchronisedExitProcess("exitScope"); } scope(failure) { // Detail what scope was called addLogEntry("Failure scope was called", ["debug"]); - // Perform exit tasks - performStandardExitProcess("failureScope"); + // Perform synchronised exit + performSynchronisedExitProcess("failureScope"); } // Read in application options as passed in @@ -140,23 +152,6 @@ int main(string[] cliArgs) { // If we need to enable logging to a file, we can only do this once we know the application configuration which is done slightly later on initialiseLogging(verboseLogging, debugLogging); - /** - // most used - addLogEntry("Basic 'info' message", ["info"]); .... or just use addLogEntry("Basic 'info' message"); - addLogEntry("Basic 'verbose' message", ["verbose"]); - addLogEntry("Basic 'debug' message", ["debug"]); - // GUI notify only - addLogEntry("Basic 'notify' ONLY message and displayed in GUI if notifications are enabled", ["notify"]); - // info and notify - addLogEntry("Basic 'info and notify' message and displayed in GUI if notifications are enabled", ["info", "notify"]); - // log file only - addLogEntry("Information sent to the log file only, and only if logging to a file is enabled", ["logFileOnly"]); - // Console only (session based upload|download) - addLogEntry("Basic 'Console only with new line' message", ["consoleOnly"]); - // Console only with no new line - addLogEntry("Basic 'Console only with no new line' message", ["consoleOnlyNoNewLine"]); - **/ - // Log application start time, log line has start time addLogEntry("Application started", ["debug"]); @@ -188,6 +183,9 @@ int main(string[] cliArgs) { // Update the current runtime application configuration (default or 'config' fileread-in options) from any passed in command line arguments appConfig.updateFromArgs(cliArgs); + // Configure dryRun so that this can be used here & during shutdown + dryRun = appConfig.getValueBool("dry_run"); + // As early as possible, now re-configure the logging class, given that we have read in any applicable 'config' file and updated the application running config from CLI input: // - Enable logging to a file if this is required // - Disable GUI notifications if this has been configured @@ -264,9 +262,9 @@ int main(string[] cliArgs) { // Check for --dry-run operation or a 'no-sync' operation where the 'dry-run' DB copy should be used // If this has been requested, we need to ensure that all actions are performed against the dry-run database copy, and, // no actual action takes place - such as deleting files if deleted online, moving files if moved online or local, downloading new & changed files, uploading new & changed files - if ((appConfig.getValueBool("dry_run")) || (appConfig.hasNoSyncOperationBeenRequested())) { + if (dryRun || (appConfig.hasNoSyncOperationBeenRequested())) { - if (appConfig.getValueBool("dry_run")) { + if (dryRun) { // This is a --dry-run operation addLogEntry("DRY-RUN Configured. Output below shows what 'would' have occurred."); } @@ -279,26 +277,29 @@ int main(string[] cliArgs) { // In a --dry-run --resync scenario, we should not copy the existing database file if (!appConfig.getValueBool("resync")) { // Copy the existing DB file to the dry-run copy - if (appConfig.getValueBool("dry_run")) { + if (dryRun) { addLogEntry("DRY-RUN: Copying items.sqlite3 to items-dryrun.sqlite3 to use for dry run operations"); } copy(appConfig.databaseFilePath,appConfig.databaseFilePathDryRun); } else { // No database copy due to --resync - if (appConfig.getValueBool("dry_run")) { + if (dryRun) { addLogEntry("DRY-RUN: No database copy created for --dry-run due to --resync also being used"); } } } // update runtimeDatabaseFile now that we are using the dry run path runtimeDatabaseFile = appConfig.databaseFilePathDryRun; + } else { + // Cleanup any existing dry-run elements ... these should never be left hanging around + cleanupDryRunDatabaseFiles(appConfig.databaseFilePathDryRun); } // Handle --logout as separate item, do not 'resync' on a --logout if (appConfig.getValueBool("logout")) { addLogEntry("--logout requested", ["debug"]); addLogEntry("Deleting the saved authentication status ..."); - if (!appConfig.getValueBool("dry_run")) { + if (!dryRun) { safeRemove(appConfig.refreshTokenFilePath); } else { // --dry-run scenario ... technically we should not be making any local file changes ....... @@ -312,7 +313,7 @@ int main(string[] cliArgs) { if (appConfig.getValueBool("reauth")) { addLogEntry("--reauth requested", ["debug"]); addLogEntry("Deleting the saved authentication status ... re-authentication requested"); - if (!appConfig.getValueBool("dry_run")) { + if (!dryRun) { safeRemove(appConfig.refreshTokenFilePath); } else { // --dry-run scenario ... technically we should not be making any local file changes ....... @@ -433,9 +434,9 @@ int main(string[] cliArgs) { // Flag that we were able to initalise the API in the application config oneDriveApiInstance.debugOutputConfiguredAPIItems(); - - oneDriveApiInstance.shutdown(); + oneDriveApiInstance.releaseCurlEngine(); object.destroy(oneDriveApiInstance); + oneDriveApiInstance = null; // Need to configure the itemDB and syncEngineInstance for 'sync' and 'non-sync' operations addLogEntry("Opening the item database ...", ["verbose"]); @@ -824,10 +825,6 @@ int main(string[] cliArgs) { } }; - // Handle SIGINT and SIGTERM - signal(SIGINT, &exitHandler); - signal(SIGTERM, &exitHandler); - // Initialise the local filesystem monitor class using inotify to monitor for local filesystem changes // If we are in a --download-only method of operation, we do not enable local filesystem monitoring if (!appConfig.getValueBool("download_only")) { @@ -997,13 +994,17 @@ int main(string[] cliArgs) { addLogEntry("End Monitor Loop Time: " ~ to!string(endFunctionProcessingTime), ["debug"]); addLogEntry("Elapsed Monitor Loop Processing Time: " ~ to!string((endFunctionProcessingTime - startFunctionProcessingTime)), ["debug"]); - // Display memory details before cleanup + // Release all the curl instances used during this loop + // New curl instances will be established on next loop + CurlEngine.releaseAllCurlInstances(); + + // Display memory details before garbage collection if (displayMemoryUsage) displayMemoryUsagePreGC(); - // Perform Garbage Cleanup + // Perform Garbage Collection GC.collect(); // Return free memory to the OS GC.minimize(); - // Display memory details after cleanup + // Display memory details after garbage collection if (displayMemoryUsage) displayMemoryUsagePostGC(); // Log that this loop is complete @@ -1111,78 +1112,6 @@ int main(string[] cliArgs) { } } -void performStandardExitProcess(string scopeCaller = null) { - // Who called this function - if (!scopeCaller.empty) { - addLogEntry("Running performStandardExitProcess due to: " ~ scopeCaller, ["debug"]); - } - - // Shutdown the OneDrive Webhook instance - if (oneDriveWebhook !is null) { - oneDriveWebhook.stop(); - object.destroy(oneDriveWebhook); - } - - // Shutdown the sync engine - if (syncEngineInstance !is null) { - addLogEntry("Shutdown Sync Engine instance", ["debug"]); - object.destroy(syncEngineInstance); - } - - // Shutdown the client side filtering objects - if (selectiveSync !is null) { - addLogEntry("Shutdown Client Side Filtering instance", ["debug"]); - selectiveSync.shutdown(); - object.destroy(selectiveSync); - } - - // Shutdown the application configuration objects - if (appConfig !is null) { - addLogEntry("Shutdown Application Configuration instance", ["debug"]); - // Cleanup any existing dry-run elements ... these should never be left hanging around - cleanupDryRunDatabaseFiles(appConfig.databaseFilePathDryRun); - object.destroy(appConfig); - } - - // Shutdown any local filesystem monitoring - if (filesystemMonitor !is null) { - addLogEntry("Shutdown Filesystem Monitoring instance", ["debug"]); - filesystemMonitor.shutdown(); - object.destroy(filesystemMonitor); - } - - // Shutdown the database - if (itemDB !is null) { - addLogEntry("Shutdown Database instance", ["debug"]); - // Make sure the .wal file is incorporated into the main db before we exit - if (itemDB.isDatabaseInitialised()) { - itemDB.performVacuum(); - } - object.destroy(itemDB); - } - - // Shutdown cached sockets - CurlEngine.releaseAll(); - - // Set all objects to null - if (scopeCaller == "failureScope") { - // Set these to be null due to failure scope - prevent 'ERROR: Unable to perform a database vacuum: out of memory' when the exit scope is then called - addLogEntry("Setting ALL Class Objects to null due to failure scope", ["debug"]); - itemDB = null; - appConfig = null; - oneDriveWebhook = null; - selectiveSync = null; - syncEngineInstance = null; - } else { - addLogEntry("Waiting for all internal threads to complete before exiting application", ["verbose"]); - addLogEntry("Application exit", ["debug"]); - addLogEntry("#######################################################################################################################################", ["logFileOnly"]); - // Destroy the shared logging buffer - (cast() logBuffer).shutdown(); - object.destroy(logBuffer); - } -} - void oneDriveWebhookCallback() { // If we are in a --download-only method of operation, there is no filesystem monitoring, so no inotify events to check if (!appConfig.getValueBool("download_only")) { @@ -1352,7 +1281,7 @@ void processResyncDatabaseRemoval(string databaseFilePathToRemove) { destroy(itemDB); // delete application sync state addLogEntry("Deleting the saved application sync status ..."); - if (!appConfig.getValueBool("dry_run")) { + if (!dryRun) { safeRemove(databaseFilePathToRemove); } else { // --dry-run scenario ... technically we should not be making any local file changes ....... @@ -1364,7 +1293,7 @@ void cleanupDryRunDatabaseFiles(string dryRunDatabaseFile) { // Temp variables string dryRunShmFile = dryRunDatabaseFile ~ "-shm"; string dryRunWalFile = dryRunDatabaseFile ~ "-wal"; - + // If the dry run database exists, clean this up if (exists(dryRunDatabaseFile)) { // remove the existing file @@ -1406,23 +1335,159 @@ auto assumeNoGC(T) (T t) if (isFunctionPointer!T || isDelegate!T) { return cast(SetFunctionAttributes!(T, functionLinkage!T, attrs)) t; } -// Catch CTRL-C +// Configure the signal handler to catch SIGINT (CTRL-C) and SIGTERM (kill) +void setupSignalHandler() { + sigaction_t sa; + sa.sa_flags = SA_RESETHAND | SA_NODEFER; // Use reset and no defer flags to handle reentrant signals + sa.sa_handler = &exitHandler; // Direct function pointer assignment + sigemptyset(&sa.sa_mask); // Initialize the signal set to empty + + // Register the signal handler for SIGINT + if (sigaction(SIGINT, &sa, null) != 0) { + writeln("FATAL: Failed to install SIGINT handler"); + exit(-1); + } + + // Register the signal handler for SIGTERM + if (sigaction(SIGTERM, &sa, null) != 0) { + writeln("FATAL: Failed to install SIGTERM handler"); + exit(-1); + } +} + +// Catch SIGINT (CTRL-C) and SIGTERM (kill), handle rapid repeat presses extern(C) nothrow @nogc @system void exitHandler(int value) { + + if (shutdownInProgress) { + return; // Ignore subsequent presses + } + shutdownInProgress = true; + try { assumeNoGC ( () { - addLogEntry("Got termination signal, performing clean up"); + addLogEntry("\nReceived termination signal, initiating cleanup"); // Wait for all parallel jobs that depend on the database to complete addLogEntry("Waiting for any existing upload|download process to complete"); - taskPool.finish(true); - // Was itemDb initialised? - if (itemDB.isDatabaseInitialised()) { - // Make sure the .wal file is incorporated into the main db before we exit - addLogEntry("Shutting down DB connection and merging temporary data"); - itemDB.performVacuum(); - object.destroy(itemDB); - } - performStandardExitProcess(); + syncEngineInstance.shutdown(); + + // Perform the shutdown process + performSynchronisedExitProcess("exitHandler"); })(); - } catch(Exception e) {} - exit(0); + } catch(Exception e) { + // Any output here will cause a GC allocation + // - Error: `@nogc` function `main.exitHandler` cannot call non-@nogc function `std.stdio.writeln!string.writeln` + // - Error: cannot use operator `~` in `@nogc` function `main.exitHandler` + // writeln("Exception during shutdown: " ~ e.msg); + } + // Exit the process with the provided exit code + exit(value); + +} + +// Handle application exit +void performSynchronisedExitProcess(string scopeCaller = null) { + synchronized { + // Logging the caller of the shutdown procedure + if (!scopeCaller.empty) { + addLogEntry("performSynchronisedExitProcess called by: " ~ scopeCaller, ["debug"]); + } + + // Perform cleanup and shutdown of various services and resources + try { + // Shutdown the OneDrive Webhook instance + shutdownOneDriveWebhook(); + // Shutdown the client side filtering objects + shutdownSelectiveSync(); + // Destroy all 'curl' instances + destroyCurlInstances(); + // Shutdown the sync engine + shutdownSyncEngine(); + // Shutdown any local filesystem monitoring + shutdownFilesystemMonitor(); + // Shutdown the database + shutdownDatabase(); + // Shutdown the application configuration objects + shutdownAppConfig(); + } catch (Exception e) { + addLogEntry("Error during performStandardExitProcess: " ~ e.toString(), ["error"]); + } + + // Finalise all logging and destroy log buffer + shutdownApplicationLogging(); + + // Perform Garbage Cleanup + GC.collect(); + // Return free memory to the OS + GC.minimize(); + } +} + +void shutdownOneDriveWebhook() { + if (oneDriveWebhook !is null) { + addLogEntry("Shutdown OneDrive Webhook instance", ["debug"]); + oneDriveWebhook.stop(); + object.destroy(oneDriveWebhook); + oneDriveWebhook = null; + } +} + +void shutdownFilesystemMonitor() { + if (filesystemMonitor !is null) { + addLogEntry("Shutdown Filesystem Monitoring instance", ["debug"]); + filesystemMonitor.shutdown(); + object.destroy(filesystemMonitor); + filesystemMonitor = null; + } +} + +void shutdownSelectiveSync() { + if (selectiveSync !is null) { + addLogEntry("Shutdown Client Side Filtering instance", ["debug"]); + selectiveSync.shutdown(); + object.destroy(selectiveSync); + selectiveSync = null; + } +} + +void shutdownSyncEngine() { + if (syncEngineInstance !is null) { + addLogEntry("Shutdown Sync Engine instance", ["debug"]); + syncEngineInstance.shutdown(); // Make sure any running thread completes first + object.destroy(syncEngineInstance); + syncEngineInstance = null; + } +} + +void shutdownDatabase() { + if (itemDB !is null && itemDB.isDatabaseInitialised()) { + addLogEntry("Shutdown Database instance", ["debug"]); + itemDB.performVacuum(); + object.destroy(itemDB); + itemDB = null; + } +} + +void shutdownAppConfig() { + if (appConfig !is null) { + addLogEntry("Shutdown Application Configuration instance", ["debug"]); + if (dryRun) { + // We were running with --dry-run , clean up the applicable database + cleanupDryRunDatabaseFiles(runtimeDatabaseFile); + } + object.destroy(appConfig); + appConfig = null; + } +} + +void destroyCurlInstances() { + CurlEngine.destroyAllCurlInstances(); +} + +void shutdownApplicationLogging() { + // Log that we are exitintg + addLogEntry("Application is exiting.", ["debug"]); + addLogEntry("#######################################################################################################################################", ["logFileOnly"]); + // Destroy the shared logging buffer + (cast() logBuffer).shutdown(); + object.destroy(logBuffer); } \ No newline at end of file diff --git a/src/monitor.d b/src/monitor.d index 18861b945..73cb8a79b 100644 --- a/src/monitor.d +++ b/src/monitor.d @@ -140,8 +140,7 @@ class MonitorBackgroundWorker { } } -void startMonitorJob(shared(MonitorBackgroundWorker) worker, Tid callerTid) -{ +void startMonitorJob(shared(MonitorBackgroundWorker) worker, Tid callerTid) { try { worker.watch(callerTid); } catch (OwnerTerminated error) { @@ -282,6 +281,11 @@ final class Monitor { this.selectiveSync = selectiveSync; } + // The destructor should only clean up resources owned directly by this instance + ~this() { + object.destroy(worker); + } + // Initialise the monitor class void initialise() { // Configure the variables diff --git a/src/onedrive.d b/src/onedrive.d index af22545f2..dce2dafd9 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -12,7 +12,7 @@ import std.file; import std.exception; import std.regex; import std.json; -import std.algorithm.searching; +import std.algorithm; import std.net.curl; import std.datetime; import std.path; @@ -33,31 +33,42 @@ import curlEngine; // Shared variables between classes shared bool debugHTTPResponseOutput = false; +// Define the 'OneDriveException' class class OneDriveException: Exception { // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/errors int httpStatusCode; + const CurlResponse response; JSONValue error; - @safe pure this(int httpStatusCode, string reason, string file = __FILE__, size_t line = __LINE__) { + this(int httpStatusCode, string reason, const CurlResponse response, string file = __FILE__, size_t line = __LINE__) { this.httpStatusCode = httpStatusCode; - this.error = error; - string msg = format("HTTP request returned status code %d (%s)", httpStatusCode, reason); + this.response = response; + this.error = response.json(); + string msg = format("HTTP request returned status code %d (%s)\n%s", httpStatusCode, reason, toJSON(error, true)); super(msg, file, line); } - this(int httpStatusCode, string reason, ref const JSONValue error, string file = __FILE__, size_t line = __LINE__) { - this.httpStatusCode = httpStatusCode; - this.error = error; - string msg = format("HTTP request returned status code %d (%s)\n%s", httpStatusCode, reason, toJSON(error, true)); - super(msg, file, line); + this(int httpStatusCode, string reason, string file = __FILE__, size_t line = __LINE__) { + this.response = null; + super(msg, file, line, null); } } +// Define the 'OneDriveError' class +class OneDriveError: Error { + this(string msg) { + super(msg); + } +} + +// Define the 'OneDriveApi' class class OneDriveApi { - // Class variables + // Class variables that use other classes ApplicationConfig appConfig; CurlEngine curlEngine; + CurlResponse response; + // Class variables string clientId = ""; string companyName = ""; string authUrl = ""; @@ -76,11 +87,12 @@ class OneDriveApi { const(char)[] refreshToken = ""; bool dryRun = false; bool debugResponse = false; - ulong retryAfterValue = 0; + bool keepAlive = false; this(ApplicationConfig appConfig) { // Configure the class varaible to consume the application configuration this.appConfig = appConfig; + this.curlEngine = null; // Configure the major API Query URL's, based on using application configuration // These however can be updated by config option 'azure_ad_endpoint', thus handled differently @@ -99,16 +111,29 @@ class OneDriveApi { siteSearchUrl = appConfig.globalGraphEndpoint ~ "/v1.0/sites?search"; siteDriveUrl = appConfig.globalGraphEndpoint ~ "/v1.0/sites/"; - // Subscriptions subscriptionUrl = appConfig.globalGraphEndpoint ~ "/v1.0/subscriptions"; } + // The destructor should only clean up resources owned directly by this instance + ~this() { + if (curlEngine !is null) { + curlEngine = null; + } + + if (response !is null) { + response = null; + } + } + // Initialise the OneDrive API class bool initialise(bool keepAlive=true) { // Initialise the curl engine - curlEngine = CurlEngine.get(); - curlEngine.initialise(appConfig.getValueLong("dns_timeout"), appConfig.getValueLong("connect_timeout"), appConfig.getValueLong("data_timeout"), appConfig.getValueLong("operation_timeout"), appConfig.defaultMaxRedirects, appConfig.getValueBool("debug_https"), appConfig.getValueString("user_agent"), appConfig.getValueBool("force_http_11"), appConfig.getValueLong("rate_limit"), appConfig.getValueLong("ip_protocol_version"), keepAlive); + this.keepAlive = keepAlive; + if (curlEngine is null) { + curlEngine = CurlEngine.getCurlInstance(); + curlEngine.initialise(appConfig.getValueLong("dns_timeout"), appConfig.getValueLong("connect_timeout"), appConfig.getValueLong("data_timeout"), appConfig.getValueLong("operation_timeout"), appConfig.defaultMaxRedirects, appConfig.getValueBool("debug_https"), appConfig.getValueString("user_agent"), appConfig.getValueBool("force_http_11"), appConfig.getValueLong("rate_limit"), appConfig.getValueLong("ip_protocol_version"), keepAlive); + } // Authorised value to return bool authorised = false; @@ -309,12 +334,12 @@ class OneDriveApi { // update appConfig.refreshToken appConfig.refreshToken = refreshToken; } - } catch (FileException e) { + } catch (FileException exception) { authorised = authorise(); - } catch (std.utf.UTFException e) { + } catch (std.utf.UTFException exception) { // path contains characters which generate a UTF exception addLogEntry("Cannot read refreshToken from: " ~ appConfig.refreshTokenFilePath); - addLogEntry(" Error Reason:" ~ e.msg); + addLogEntry(" Error Reason:" ~ exception.msg); authorised = false; } } @@ -324,11 +349,12 @@ class OneDriveApi { addLogEntry("refreshToken is empty !!!!!!!!!! This will cause 4xx errors ... CODING TO DO TO HANDLE ?????"); } } + // Return if we are authorised addLogEntry("Authorised State: " ~ to!string(authorised), ["debug"]); return authorised; } - + // If the API has been configured correctly, print the items that been configured void debugOutputConfiguredAPIItems() { // Debug output of configured URL's @@ -353,15 +379,17 @@ class OneDriveApi { addLogEntry("Configured siteDriveUrl: " ~ siteDriveUrl, ["debug"]); } - // Shutdown OneDrive API Curl Engine - void shutdown() { - // Release curl instance + // Release CurlEngine bask to the Curl Engine Pool + void releaseCurlEngine() { + // Log that this was called + addLogEntry("OneDrive API releaseCurlEngine() CALLED", ["debug"]); + // Release curl instance back to the pool if (curlEngine !is null) { - curlEngine.release(); + curlEngine.releaseEngine(); curlEngine = null; } } - + // Authenticate this client against Microsoft OneDrive API bool authorise() { @@ -384,17 +412,17 @@ class OneDriveApi { auto authUrlFile = File(authUrl, "w"); authUrlFile.write(url); authUrlFile.close(); - } catch (FileException e) { + } catch (FileException exception) { // There was a file system error // display the error message - displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + displayFileSystemErrorMessage(exception.msg, getFunctionName!({})); // Must force exit here, allow logging to be done Thread.sleep(dur!("msecs")(500)); exit(-1); - } catch (ErrnoException e) { + } catch (ErrnoException exception) { // There was a file system error // display the error message - displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + displayFileSystemErrorMessage(exception.msg, getFunctionName!({})); // Must force exit here, allow logging to be done Thread.sleep(dur!("msecs")(500)); exit(-1); @@ -409,9 +437,9 @@ class OneDriveApi { // read response from provided from OneDrive try { response = cast(char[]) read(responseUrl); - } catch (OneDriveException e) { + } catch (OneDriveException exception) { // exception generated - displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); return false; } @@ -419,7 +447,7 @@ class OneDriveApi { try { std.file.remove(authUrl); std.file.remove(responseUrl); - } catch (FileException e) { + } catch (FileException exception) { addLogEntry("Cannot remove files " ~ authUrl ~ " " ~ responseUrl); return false; } @@ -450,15 +478,11 @@ class OneDriveApi { } c.popFront(); // skip the whole match redeemToken(c.front); - - return true; - } // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get JSONValue getDefaultDriveDetails() { - checkAccessTokenExpired(); string url; url = driveUrl; return get(url); @@ -466,7 +490,6 @@ class OneDriveApi { // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get JSONValue getDefaultRootDetails() { - checkAccessTokenExpired(); string url; url = driveUrl ~ "/root"; return get(url); @@ -474,7 +497,6 @@ class OneDriveApi { // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get JSONValue getDriveIdRoot(string driveId) { - checkAccessTokenExpired(); string url; url = driveByIdUrl ~ driveId ~ "/root"; return get(url); @@ -482,7 +504,6 @@ class OneDriveApi { // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_get JSONValue getDriveQuota(string driveId) { - checkAccessTokenExpired(); string url; url = driveByIdUrl ~ driveId ~ "/"; url ~= "?select=quota"; @@ -492,7 +513,6 @@ class OneDriveApi { // Return the details of the specified path, by giving the path we wish to query // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get JSONValue getPathDetails(string path) { - checkAccessTokenExpired(); string url; if ((path == ".")||(path == "/")) { url = driveUrl ~ "/root/"; @@ -505,7 +525,6 @@ class OneDriveApi { // Return the details of the specified item based on its driveID and itemID // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get JSONValue getPathDetailsById(string driveId, string id) { - checkAccessTokenExpired(); string url; url = driveByIdUrl ~ driveId ~ "/items/" ~ id; //url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; @@ -515,24 +534,20 @@ class OneDriveApi { // Return all the items that are shared with the user // https://docs.microsoft.com/en-us/graph/api/drive-sharedwithme JSONValue getSharedWithMe() { - checkAccessTokenExpired(); return get(sharedWithMeUrl); } // Create a shareable link for an existing file on OneDrive based on the accessScope JSON permissions // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createlink JSONValue createShareableLink(string driveId, string id, JSONValue accessScope) { - checkAccessTokenExpired(); string url; url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/createLink"; - curlEngine.http.addRequestHeader("Content-Type", "application/json"); return post(url, accessScope.toString()); } // Return the requested details of the specified path on the specified drive id and path // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get JSONValue getPathDetailsByDriveId(string driveId, string path) { - checkAccessTokenExpired(); string url; // https://learn.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online // Required format: /drives/{drive-id}/root:/{item-path}: @@ -540,13 +555,16 @@ class OneDriveApi { return get(url); } + // Track changes for a given driveId // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta - JSONValue viewChangesByItemId(string driveId, string id, string deltaLink) { - checkAccessTokenExpired(); - - // If Business Account add addIncludeFeatureRequestHeader() which should add Prefer: Include-Feature=AddToOneDrive + // Your app begins by calling delta without any parameters. The service starts enumerating the drive's hierarchy, returning pages of items and either an @odata.nextLink or an @odata.deltaLink, as described below. + // Your app should continue calling with the @odata.nextLink until you no longer see an @odata.nextLink returned, or you see a response with an empty set of changes. + // After you have finished receiving all the changes, you may apply them to your local state. To check for changes in the future, call delta again with the @odata.deltaLink from the previous successful response. + JSONValue getChangesByItemId(string driveId, string id, string deltaLink) { + string[string] requestHeaders; + // If Business Account add Prefer: Include-Feature=AddToOneDrive if ((appConfig.accountType != "personal") && ( appConfig.getValueBool("sync_business_shared_items"))) { - addIncludeFeatureRequestHeader(); + addIncludeFeatureRequestHeader(&requestHeaders); } string url; @@ -556,16 +574,15 @@ class OneDriveApi { } else { url = deltaLink; } - return get(url); + return get(url, false, requestHeaders); } // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_list_children JSONValue listChildren(string driveId, string id, string nextLink) { - checkAccessTokenExpired(); - + string[string] requestHeaders; // If Business Account add addIncludeFeatureRequestHeader() which should add Prefer: Include-Feature=AddToOneDrive if ((appConfig.accountType != "personal") && ( appConfig.getValueBool("sync_business_shared_items"))) { - addIncludeFeatureRequestHeader(); + addIncludeFeatureRequestHeader(&requestHeaders); } string url; @@ -576,12 +593,11 @@ class OneDriveApi { } else { url = nextLink; } - return get(url); + return get(url, false, requestHeaders); } // https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_search JSONValue searchDriveForPath(string driveId, string path) { - checkAccessTokenExpired(); string url; url = "https://graph.microsoft.com/v1.0/drives/" ~ driveId ~ "/root/search(q='" ~ encodeComponent(path) ~ "')"; return get(url); @@ -589,108 +605,74 @@ class OneDriveApi { // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_update JSONValue updateById(const(char)[] driveId, const(char)[] id, JSONValue data, const(char)[] eTag = null) { - checkAccessTokenExpired(); + string[string] requestHeaders; const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id; - if (eTag) curlEngine.http.addRequestHeader("If-Match", eTag); - curlEngine.http.addRequestHeader("Content-Type", "application/json"); - return patch(url, data.toString()); + if (eTag) requestHeaders["If-Match"] = to!string(eTag); + return patch(url, data.toString(), requestHeaders); } // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delete void deleteById(const(char)[] driveId, const(char)[] id, const(char)[] eTag = null) { - checkAccessTokenExpired(); + // string[string] requestHeaders; const(char)[] url = driveByIdUrl ~ driveId ~ "/items/" ~ id; //TODO: investigate why this always fail with 412 (Precondition Failed) - //if (eTag) http.addRequestHeader("If-Match", eTag); + // if (eTag) requestHeaders["If-Match"] = eTag; performDelete(url); } // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_post_children JSONValue createById(string parentDriveId, string parentId, JSONValue item) { - checkAccessTokenExpired(); string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ "/children"; - curlEngine.http.addRequestHeader("Content-Type", "application/json"); return post(url, item.toString()); } // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content JSONValue simpleUpload(string localPath, string parentDriveId, string parentId, string filename) { - checkAccessTokenExpired(); string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/content"; - return upload(localPath, url); + return put(url, localPath); } // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content JSONValue simpleUploadReplace(string localPath, string driveId, string id) { - checkAccessTokenExpired(); string url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/content"; - return upload(localPath, url); + return put(url, localPath); } // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession //JSONValue createUploadSession(string parentDriveId, string parentId, string filename, string eTag = null, JSONValue item = null) { JSONValue createUploadSession(string parentDriveId, string parentId, string filename, const(char)[] eTag = null, JSONValue item = null) { - checkAccessTokenExpired(); + // string[string] requestHeaders; string url = driveByIdUrl ~ parentDriveId ~ "/items/" ~ parentId ~ ":/" ~ encodeComponent(filename) ~ ":/createUploadSession"; // eTag If-Match header addition commented out for the moment // At some point, post the creation of this upload session the eTag is being 'updated' by OneDrive, thus when uploadFragment() is used // this generates a 412 Precondition Failed and then a 416 Requested Range Not Satisfiable // This needs to be investigated further as to why this occurs - //if (eTag) curlEngine.http.addRequestHeader("If-Match", eTag); - curlEngine.http.addRequestHeader("Content-Type", "application/json"); + // if (eTag) requestHeaders["If-Match"] = eTag; return post(url, item.toString()); } // https://dev.onedrive.com/items/upload_large_files.htm JSONValue uploadFragment(string uploadUrl, string filepath, long offset, long offsetSize, long fileSize) { - checkAccessTokenExpired(); // open file as read-only in binary mode // If we upload a modified file, with the current known online eTag, this gets changed when the session is started - thus, the tail end of uploading // a fragment fails with a 412 Precondition Failed and then a 416 Requested Range Not Satisfiable // For the moment, comment out adding the If-Match header in createUploadSession, which then avoids this issue - auto file = File(filepath, "rb"); - file.seek(offset); string contentRange = "bytes " ~ to!string(offset) ~ "-" ~ to!string(offset + offsetSize - 1) ~ "/" ~ to!string(fileSize); addLogEntry("", ["debug"]); // Add an empty newline before log output addLogEntry("contentRange: " ~ contentRange, ["debug"]); - - // function scopes - scope(exit) { - curlEngine.http.clearRequestHeaders(); - curlEngine.http.onSend = null; - curlEngine.http.onReceive = null; - curlEngine.http.onReceiveHeader = null; - curlEngine.http.onReceiveStatusLine = null; - curlEngine.http.contentLength = 0; - // close file if open - if (file.isOpen()){ - // close open file - file.close(); - } - } - curlEngine.connect(HTTP.Method.put, uploadUrl); - curlEngine.http.addRequestHeader("Content-Range", contentRange); - curlEngine.http.onSend = data => file.rawRead(data).length; - // convert offsetSize to ulong - curlEngine.http.contentLength = to!ulong(offsetSize); - JSONValue response; - response = performHTTPOperation(); - checkHttpResponseCode(response); - return response; + return put(uploadUrl, filepath, true, contentRange, offset, offsetSize); } // https://dev.onedrive.com/items/upload_large_files.htm JSONValue requestUploadStatus(string uploadUrl) { - checkAccessTokenExpired(); return get(uploadUrl, true); } // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/site_search?view=odsp-graph-online JSONValue o365SiteSearch(string nextLink) { - checkAccessTokenExpired(); string url; // configure URL to query if (nextLink.empty) { @@ -703,14 +685,12 @@ class OneDriveApi { // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/drive_list?view=odsp-graph-online JSONValue o365SiteDrives(string site_id){ - checkAccessTokenExpired(); string url; url = siteDriveUrl ~ site_id ~ "/drives"; return get(url); } JSONValue createSubscription(string notificationUrl, SysTime expirationDateTime) { - checkAccessTokenExpired(); string driveId = appConfig.getValueString("drive_id"); string url = subscriptionUrl; @@ -752,15 +732,14 @@ class OneDriveApi { // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_get_content void downloadById(const(char)[] driveId, const(char)[] id, string saveToPath, long fileSize) { - checkAccessTokenExpired(); scope(failure) { if (exists(saveToPath)) { // try and remove the file, catch error try { remove(saveToPath); - } catch (FileException e) { + } catch (FileException exception) { // display the error message - displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + displayFileSystemErrorMessage(exception.msg, getFunctionName!({})); } } } @@ -776,9 +755,9 @@ class OneDriveApi { // Configure the applicable permissions for the folder addLogEntry("Setting directory permissions for: " ~ newPath, ["debug"]); newPath.setAttributes(appConfig.returnRequiredDirectoryPermisions()); - } catch (FileException e) { + } catch (FileException exception) { // display the error message - displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + displayFileSystemErrorMessage(exception.msg, getFunctionName!({})); } } @@ -798,42 +777,37 @@ class OneDriveApi { return siteSearchUrl; } - // Return the current value of retryAfterValue - ulong getRetryAfterValue() { - return retryAfterValue; + // Private OneDrive API Functions + private void addIncludeFeatureRequestHeader(string[string]* headers) { + addLogEntry("Adding 'Include-Feature=AddToOneDrive' API request header as 'sync_business_shared_items' config option is enabled", ["debug"]); + (*headers)["Prefer"] = "Include-Feature=AddToOneDrive"; } - // Reset the current value of retryAfterValue to 0 after it has been used - void resetRetryAfterValue() { - retryAfterValue = 0; - } - - // Private functions - private void addAccessTokenHeader() { - curlEngine.http.addRequestHeader("Authorization", appConfig.accessToken); - } - - private void addIncludeFeatureRequestHeader() { - addLogEntry("Adding 'Include-Feature=AddToOneDrive' API request header as 'sync_business_shared_items' config option is enabled", ["debug"]); - curlEngine.http.addRequestHeader("Prefer", "Include-Feature=AddToOneDrive"); + private void redeemToken(char[] authCode){ + char[] postData = + "client_id=" ~ clientId ~ + "&redirect_uri=" ~ redirectUrl ~ + "&code=" ~ authCode ~ + "&grant_type=authorization_code"; + acquireToken(postData); } private void acquireToken(char[] postData) { JSONValue response; try { - response = post(tokenUrl, postData); - } catch (OneDriveException e) { + response = post(tokenUrl, postData, true, "application/x-www-form-urlencoded"); + } catch (OneDriveException exception) { // an error was generated - if ((e.httpStatusCode == 400) || (e.httpStatusCode == 401)) { + if ((exception.httpStatusCode == 400) || (exception.httpStatusCode == 401)) { // Handle an unauthorised client - handleClientUnauthorised(e.httpStatusCode, e.msg); + handleClientUnauthorised(exception.httpStatusCode, exception.msg); } else { - if (e.httpStatusCode >= 500) { + if (exception.httpStatusCode >= 500) { // There was a HTTP 5xx Server Side Error - retry acquireToken(postData); } else { - displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); } } } @@ -857,10 +831,9 @@ class OneDriveApi { addLogEntry("Please login to https://account.live.com/consent/Manage and remove your existing application access consent"); addLogEntry(); // force exit - shutdown(); + releaseCurlEngine(); // Must force exit here, allow logging to be done - Thread.sleep(dur!("msecs")(500)); - exit(-1); + forceExit(); } } } @@ -901,9 +874,9 @@ class OneDriveApi { std.file.write(appConfig.refreshTokenFilePath, refreshToken); addLogEntry("Setting file permissions for: " ~ appConfig.refreshTokenFilePath, ["debug"]); appConfig.refreshTokenFilePath.setAttributes(appConfig.returnRequiredFilePermisions()); - } catch (FileException e) { + } catch (FileException exception) { // display the error message - displayFileSystemErrorMessage(e.msg, getFunctionName!({})); + displayFileSystemErrorMessage(exception.msg, getFunctionName!({})); } } } else { @@ -914,756 +887,530 @@ class OneDriveApi { } else { addLogEntry("Invalid response from the OneDrive API. Unable to initialise OneDrive API instance."); // Must force exit here, allow logging to be done - Thread.sleep(dur!("msecs")(500)); - exit(-1); + forceExit(); } } + private void newToken() { + addLogEntry("Need to generate a new access token for Microsoft OneDrive", ["debug"]); + auto postData = appender!(string)(); + postData ~= "client_id=" ~ clientId; + postData ~= "&redirect_uri=" ~ redirectUrl; + postData ~= "&refresh_token=" ~ to!string(refreshToken); + postData ~= "&grant_type=refresh_token"; + acquireToken(postData.data.dup); + } + private void checkAccessTokenExpired() { - try { - if (Clock.currTime() >= appConfig.accessTokenExpiration) { - addLogEntry("Microsoft OneDrive Access Token has EXPIRED. Must generate a new Microsoft OneDrive Access Token", ["debug"]); - newToken(); - } else { - addLogEntry("Existing Microsoft OneDrive Access Token Expires: " ~ to!string(appConfig.accessTokenExpiration), ["debug"]); - } - } catch (OneDriveException e) { - if (e.httpStatusCode == 400 || e.httpStatusCode == 401) { - // flag error and notify - addLogEntry(); - addLogEntry("ERROR: Refresh token invalid, use --reauth to authorize the client again.", ["info", "notify"]); - addLogEntry(); - // set error message - e.msg ~= "\nRefresh token invalid, use --reauth to authorize the client again"; - } + if (Clock.currTime() >= appConfig.accessTokenExpiration) { + addLogEntry("Microsoft OneDrive Access Token has EXPIRED. Must generate a new Microsoft OneDrive Access Token", ["debug"]); + newToken(); + } else { + addLogEntry("Existing Microsoft OneDrive Access Token Expires: " ~ to!string(appConfig.accessTokenExpiration), ["debug"]); } } - private void performDelete(const(char)[] url) { - scope(exit) curlEngine.http.clearRequestHeaders(); - curlEngine.connect(HTTP.Method.del, url); - addAccessTokenHeader(); - JSONValue response; - response = performHTTPOperation(); - checkHttpResponseCode(response); + private string getAccessToken() { + checkAccessTokenExpired(); + return to!string(appConfig.accessToken); + } + + private void addAccessTokenHeader(string[string]* requestHeaders) { + (*requestHeaders)["Authorization"] = getAccessToken(); + } + + private void connect(HTTP.Method method, const(char)[] url, bool skipToken, CurlResponse response, string[string] requestHeaders=null) { + addLogEntry("Request URL = " ~ to!string(url), ["debug"]); + // Check access token first in case the request is overridden + if (!skipToken) addAccessTokenHeader(&requestHeaders); + curlEngine.setResponseHolder(response); + foreach(k, v; requestHeaders) { + curlEngine.addRequestHeader(k, v); + } + curlEngine.connect(method, url); + } + + private void performDelete(const(char)[] url, string[string] requestHeaders=null, string callingFunction=__FUNCTION__, int lineno=__LINE__) { + bool validateJSONResponse = false; + oneDriveErrorHandlerWrapper((CurlResponse response) { + connect(HTTP.Method.del, url, false, response, requestHeaders); + return curlEngine.execute(); + }, validateJSONResponse, callingFunction, lineno); } - private void downloadFile(const(char)[] url, string filename, long fileSize) { + private void downloadFile(const(char)[] url, string filename, long fileSize, string callingFunction=__FUNCTION__, int lineno=__LINE__) { // Threshold for displaying download bar long thresholdFileSize = 4 * 2^^20; // 4 MiB // To support marking of partially-downloaded files, string originalFilename = filename; string downloadFilename = filename ~ ".partial"; - - // open downloadFilename as write in binary mode - auto file = File(downloadFilename, "wb"); - - // function scopes - scope(exit) { - curlEngine.http.clearRequestHeaders(); - curlEngine.http.onSend = null; - curlEngine.http.onReceive = null; - curlEngine.http.onReceiveHeader = null; - curlEngine.http.onReceiveStatusLine = null; - curlEngine.http.contentLength = 0; - // Reset onProgress to not display anything for next download - curlEngine.http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) - { - return 0; - }; - // close file if open - if (file.isOpen()){ - // close open file - file.close(); - } - } - - curlEngine.connect(HTTP.Method.get, url); - addAccessTokenHeader(); - - curlEngine.http.onReceive = (ubyte[] data) { - file.rawWrite(data); - return data.length; - }; - - if (fileSize >= thresholdFileSize){ - // Download Progress variables - size_t expected_total_segments = 20; - ulong start_unix_time = Clock.currTime.toUnixTime(); - int h, m, s; - string etaString; - bool barInit = false; - real previousProgressPercent = -1.0; - real percentCheck = 5.0; - size_t segmentCount = -1; - - // Setup progress bar to display - curlEngine.http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) { - // For each onProgress, what is the % of dlnow to dltotal - // floor - rounds down to nearest whole number - real currentDLPercent = floor(double(dlnow)/dltotal*100); - string downloadLogEntry = "Downloading: " ~ filename ~ " ... "; + bool validateJSONResponse = false; + oneDriveErrorHandlerWrapper((CurlResponse response) { + connect(HTTP.Method.get, url, false, response); + + if (fileSize >= thresholdFileSize){ + // Download Progress variables + size_t expected_total_segments = 20; + ulong start_unix_time = Clock.currTime.toUnixTime(); + int h, m, s; + string etaString; + bool barInit = false; + real previousProgressPercent = -1.0; + real percentCheck = 5.0; + long segmentCount = -1; - // Have we started downloading? - if (currentDLPercent > 0){ - // We have started downloading - addLogEntry("", ["debug"]); // Debug new line only - addLogEntry("Data Received = " ~ to!string(dlnow), ["debug"]); - addLogEntry("Expected Total = " ~ to!string(dltotal), ["debug"]); - addLogEntry("Percent Complete = " ~ to!string(currentDLPercent), ["debug"]); + // Setup progress bar to display + curlEngine.http.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) { + // For each onProgress, what is the % of dlnow to dltotal + // floor - rounds down to nearest whole number + real currentDLPercent = floor(double(dlnow)/dltotal*100); + string downloadLogEntry = "Downloading: " ~ filename ~ " ... "; - // Every 5% download we need to increment the download bar - - // Has the user set a data rate limit? - // when using rate_limit, we will get odd download rates, for example: - // Percent Complete = 24 - // Data Received = 13080163 - // Expected Total = 52428800 - // Percent Complete = 24 - // Data Received = 13685777 - // Expected Total = 52428800 - // Percent Complete = 26 <---- jumps to 26% missing 25%, thus fmod misses incrementing progress bar - // Data Received = 13685777 - // Expected Total = 52428800 - // Percent Complete = 26 - - if (appConfig.getValueLong("rate_limit") > 0) { - // User configured rate limit - // How much data should be in each segment to qualify for 5% - ulong dataPerSegment = to!ulong(floor(double(dltotal)/expected_total_segments)); - // How much data received do we need to validate against - ulong thisSegmentData = dataPerSegment * segmentCount; - ulong nextSegmentData = dataPerSegment * (segmentCount + 1); + // Have we started downloading? + if (currentDLPercent > 0){ + // We have started downloading + addLogEntry("", ["debug"]); // Debug new line only + addLogEntry("Data Received = " ~ to!string(dlnow), ["debug"]); + addLogEntry("Expected Total = " ~ to!string(dltotal), ["debug"]); + addLogEntry("Percent Complete = " ~ to!string(currentDLPercent), ["debug"]); - // Has the data that has been received in a 5% window that we need to increment the progress bar at - if ((dlnow > thisSegmentData) && (dlnow < nextSegmentData) && (previousProgressPercent != currentDLPercent) || (dlnow == dltotal)) { - // Downloaded data equals approx 5% - addLogEntry("Incrementing Progress Bar using calculated 5% of data received", ["debug"]); + // Every 5% download we need to increment the download bar + + // Has the user set a data rate limit? + // when using rate_limit, we will get odd download rates, for example: + // Percent Complete = 24 + // Data Received = 13080163 + // Expected Total = 52428800 + // Percent Complete = 24 + // Data Received = 13685777 + // Expected Total = 52428800 + // Percent Complete = 26 <---- jumps to 26% missing 25%, thus fmod misses incrementing progress bar + // Data Received = 13685777 + // Expected Total = 52428800 + // Percent Complete = 26 + + if (appConfig.getValueLong("rate_limit") > 0) { + // User configured rate limit + // How much data should be in each segment to qualify for 5% + ulong dataPerSegment = to!ulong(floor(double(dltotal)/expected_total_segments)); + // How much data received do we need to validate against + ulong thisSegmentData = dataPerSegment * segmentCount; + ulong nextSegmentData = dataPerSegment * (segmentCount + 1); - // 100% check - if (currentDLPercent != 100) { - // Not 100% yet - // Calculate the output + // Has the data that has been received in a 5% window that we need to increment the progress bar at + if ((dlnow > thisSegmentData) && (dlnow < nextSegmentData) && (previousProgressPercent != currentDLPercent) || (dlnow == dltotal)) { + // Downloaded data equals approx 5% + addLogEntry("Incrementing Progress Bar using calculated 5% of data received", ["debug"]); + + // 100% check + if (currentDLPercent != 100) { + // Not 100% yet + // Calculate the output + segmentCount++; + auto eta = calc_eta(segmentCount, expected_total_segments, start_unix_time); + dur!"seconds"(eta).split!("hours", "minutes", "seconds")(h, m, s); + etaString = format!"| ETA %02d:%02d:%02d"( h, m, s); + string percentage = leftJustify(to!string(currentDLPercent) ~ "%", 5, ' '); + addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]); + } else { + // 100% done + ulong end_unix_time = Clock.currTime.toUnixTime(); + auto upload_duration = cast(int)(end_unix_time - start_unix_time); + dur!"seconds"(upload_duration).split!("hours", "minutes", "seconds")(h, m, s); + etaString = format!"| DONE in %02d:%02d:%02d"( h, m, s); + string percentage = leftJustify(to!string(currentDLPercent) ~ "%", 5, ' '); + addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]); + } + + // update values + addLogEntry("Setting previousProgressPercent to " ~ to!string(currentDLPercent), ["debug"]); + previousProgressPercent = currentDLPercent; + addLogEntry("Incrementing segmentCount", ["debug"]); segmentCount++; - auto eta = calc_eta(segmentCount, expected_total_segments, start_unix_time); - dur!"seconds"(eta).split!("hours", "minutes", "seconds")(h, m, s); - etaString = format!"| ETA %02d:%02d:%02d"( h, m, s); - string percentage = leftJustify(to!string(currentDLPercent) ~ "%", 5, ' '); - addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]); - } else { - // 100% done - ulong end_unix_time = Clock.currTime.toUnixTime(); - auto upload_duration = cast(int)(end_unix_time - start_unix_time); - dur!"seconds"(upload_duration).split!("hours", "minutes", "seconds")(h, m, s); - etaString = format!"| DONE in %02d:%02d:%02d"( h, m, s); - string percentage = leftJustify(to!string(currentDLPercent) ~ "%", 5, ' '); - addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]); } - - // update values - addLogEntry("Setting previousProgressPercent to " ~ to!string(currentDLPercent), ["debug"]); - previousProgressPercent = currentDLPercent; - addLogEntry("Incrementing segmentCount", ["debug"]); - segmentCount++; + } else { + // Is currentDLPercent divisible by 5 leaving remainder 0 and does previousProgressPercent not equal currentDLPercent + if ((isIdentical(fmod(currentDLPercent, percentCheck), 0.0)) && (previousProgressPercent != currentDLPercent)) { + // currentDLPercent matches a new increment + addLogEntry("Incrementing Progress Bar using fmod match", ["debug"]); + + // 100% check + if (currentDLPercent != 100) { + // Not 100% yet + // Calculate the output + segmentCount++; + auto eta = calc_eta(segmentCount, expected_total_segments, start_unix_time); + dur!"seconds"(eta).split!("hours", "minutes", "seconds")(h, m, s); + etaString = format!"| ETA %02d:%02d:%02d"( h, m, s); + string percentage = leftJustify(to!string(currentDLPercent) ~ "%", 5, ' '); + addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]); + } else { + // 100% done + ulong end_unix_time = Clock.currTime.toUnixTime(); + auto upload_duration = cast(int)(end_unix_time - start_unix_time); + dur!"seconds"(upload_duration).split!("hours", "minutes", "seconds")(h, m, s); + etaString = format!"| DONE in %02d:%02d:%02d"( h, m, s); + string percentage = leftJustify(to!string(currentDLPercent) ~ "%", 5, ' '); + addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]); + } + + // update values + previousProgressPercent = currentDLPercent; + } } } else { - // Is currentDLPercent divisible by 5 leaving remainder 0 and does previousProgressPercent not equal currentDLPercent - if ((isIdentical(fmod(currentDLPercent, percentCheck), 0.0)) && (previousProgressPercent != currentDLPercent)) { - // currentDLPercent matches a new increment - addLogEntry("Incrementing Progress Bar using fmod match", ["debug"]); - - // 100% check - if (currentDLPercent != 100) { - // Not 100% yet - // Calculate the output - segmentCount++; - auto eta = calc_eta(segmentCount, expected_total_segments, start_unix_time); - dur!"seconds"(eta).split!("hours", "minutes", "seconds")(h, m, s); - etaString = format!"| ETA %02d:%02d:%02d"( h, m, s); - string percentage = leftJustify(to!string(currentDLPercent) ~ "%", 5, ' '); - addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]); - } else { - // 100% done - ulong end_unix_time = Clock.currTime.toUnixTime(); - auto upload_duration = cast(int)(end_unix_time - start_unix_time); - dur!"seconds"(upload_duration).split!("hours", "minutes", "seconds")(h, m, s); - etaString = format!"| DONE in %02d:%02d:%02d"( h, m, s); - string percentage = leftJustify(to!string(currentDLPercent) ~ "%", 5, ' '); - addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]); - } - - // update values - previousProgressPercent = currentDLPercent; + if ((currentDLPercent == 0) && (!barInit)) { + // Calculate the output + segmentCount++; + etaString = "| ETA --:--:--"; + string percentage = leftJustify(to!string(currentDLPercent) ~ "%", 5, ' '); + addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]); + barInit = true; } } - } else { - if ((currentDLPercent == 0) && (!barInit)) { - // Calculate the output - segmentCount++; - etaString = "| ETA --:--:--"; - string percentage = leftJustify(to!string(currentDLPercent) ~ "%", 5, ' '); - addLogEntry(downloadLogEntry ~ percentage ~ etaString, ["consoleOnly"]); - barInit = true; - } - } - return 0; - }; - - // Perform download - try { - // try and catch any curl error - curlEngine.http.perform(); - // Check the HTTP Response headers - needed for correct 429 handling - // check will be performed in checkHttpCode() - // Reset onProgress to not display anything for next download done using exit scope - } catch (CurlException e) { - displayOneDriveErrorMessage(e.msg, getFunctionName!({})); - } - } else { - // No progress bar - try { - // try and catch any curl error - curlEngine.http.perform(); - // Check the HTTP Response headers - needed for correct 429 handling - // check will be performed in checkHttpCode() - } catch (CurlException e) { - displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + return 0; + }; + } else { + // No progress bar } - } - - // Rename downloaded file - rename(downloadFilename, originalFilename); - - // Check the HTTP response code, which, if a 429, will also check response headers - checkHttpCode(); + + return curlEngine.download(originalFilename, downloadFilename); + }, validateJSONResponse, callingFunction, lineno); } - private JSONValue get(string url, bool skipToken = false) { - scope(exit) curlEngine.http.clearRequestHeaders(); - addLogEntry("Request URL = " ~ url, ["debug"]); - curlEngine.connect(HTTP.Method.get, url); - if (!skipToken) addAccessTokenHeader(); // HACK: requestUploadStatus - JSONValue response; - response = performHTTPOperation(); - checkHttpResponseCode(response); - // OneDrive API Response Debugging if --https-debug is being used - if (debugResponse){ - addLogEntry("OneDrive API Response: " ~ to!string(response), ["debug"]); - } - return response; - } - - private void newToken() { - addLogEntry("Need to generate a new access token for Microsoft OneDrive", ["debug"]); - auto postData = appender!(string)(); - postData ~= "client_id=" ~ clientId; - postData ~= "&redirect_uri=" ~ redirectUrl; - postData ~= "&refresh_token=" ~ to!string(refreshToken); - postData ~= "&grant_type=refresh_token"; - acquireToken(postData.data.dup); + private JSONValue get(string url, bool skipToken = false, string[string] requestHeaders=null, string callingFunction=__FUNCTION__, int lineno=__LINE__) { + bool validateJSONResponse = true; + return oneDriveErrorHandlerWrapper((CurlResponse response) { + connect(HTTP.Method.get, url, skipToken, response, requestHeaders); + return curlEngine.execute(); + }, validateJSONResponse, callingFunction, lineno); } - private auto patch(T)(const(char)[] url, const(T)[] patchData) { - scope(exit) curlEngine.http.clearRequestHeaders(); - curlEngine.connect(HTTP.Method.patch, url); - addAccessTokenHeader(); - auto response = perform(patchData); - checkHttpResponseCode(response); - return response; + private JSONValue patch(const(char)[] url, const(char)[] patchData, string[string] requestHeaders=null, const(char)[] contentType = "application/json", string callingFunction=__FUNCTION__, int lineno=__LINE__) { + bool validateJSONResponse = true; + return oneDriveErrorHandlerWrapper((CurlResponse response) { + connect(HTTP.Method.patch, url, false, response, requestHeaders); + curlEngine.setContent(contentType, patchData); + return curlEngine.execute(); + }, validateJSONResponse, callingFunction, lineno); } - - private auto post(T)(string url, const(T)[] postData) { - scope(exit) curlEngine.http.clearRequestHeaders(); - curlEngine.connect(HTTP.Method.post, url); - addAccessTokenHeader(); - auto response = perform(postData); - checkHttpResponseCode(response); - return response; + + private JSONValue post(const(char)[] url, const(char)[] postData, bool skipToken = false, const(char)[] contentType = "application/json", string callingFunction=__FUNCTION__, int lineno=__LINE__) { + bool validateJSONResponse = true; + return oneDriveErrorHandlerWrapper((CurlResponse response) { + connect(HTTP.Method.post, url, skipToken, response); + curlEngine.setContent(contentType, postData); + return curlEngine.execute(); + }, validateJSONResponse, callingFunction, lineno); } - private JSONValue perform(const(void)[] sendData) { - scope(exit) { - curlEngine.http.onSend = null; - curlEngine.http.contentLength = 0; - } - if (sendData) { - curlEngine.http.contentLength = sendData.length; - curlEngine.http.onSend = (void[] buf) { - import std.algorithm: min; - size_t minLen = min(buf.length, sendData.length); - if (minLen == 0) return 0; - buf[0 .. minLen] = sendData[0 .. minLen]; - sendData = sendData[minLen .. $]; - return minLen; - }; - } else { - curlEngine.http.onSend = buf => 0; - } - JSONValue response; - response = performHTTPOperation(); - return response; + private JSONValue put(const(char)[] url, string filepath, bool skipToken=false, string contentRange=null, ulong offset=0, ulong offsetSize=0, string callingFunction=__FUNCTION__, int lineno=__LINE__) { + bool validateJSONResponse = true; + return oneDriveErrorHandlerWrapper((CurlResponse response) { + connect(HTTP.Method.put, url, skipToken, response); + curlEngine.setFile(filepath, contentRange, offset, offsetSize); + return curlEngine.execute(); + }, validateJSONResponse, callingFunction, lineno); } - - private JSONValue performHTTPOperation() { - scope(exit) curlEngine.http.onReceive = null; - char[] content; - JSONValue json; - - curlEngine.http.onReceive = (ubyte[] data) { - content ~= data; - // HTTP Server Response Code Debugging if --https-debug is being used - if (debugResponse){ - addLogEntry("onedrive.performHTTPOperation() => OneDrive HTTP Server Response: " ~ to!string(curlEngine.http.statusLine.code), ["debug"]); - } - return data.length; - }; - try { - // Attempt to perform the action - curlEngine.http.perform(); - // Check the HTTP Response headers - needed for correct 429 handling - checkHTTPResponseHeaders(); - } catch (CurlException e) { - // Parse and display error message received from OneDrive - addLogEntry("onedrive.performHTTPOperation() Generated a OneDrive CurlException", ["debug"]); - auto errorArray = splitLines(e.msg); - string errorMessage = errorArray[0]; + // Wrapper function for all requests to OneDrive API + // - This should throw a OneDriveException so that this exception can be handled appropriately elsewhere in the application + private JSONValue oneDriveErrorHandlerWrapper(CurlResponse delegate(CurlResponse response) executer, bool validateJSONResponse, string callingFunction, int lineno) { + // Create a new 'curl' response + response = new CurlResponse(); + + // Other wrapper variables + int retryAttempts = 0; + int baseBackoffInterval = 1; // Base backoff interval in seconds + int maxRetryCount = 175200; // Approx 365 days based on maxBackoffInterval + appConfig.defaultDataTimeout + //int maxRetryCount = 5; // Temp + int maxBackoffInterval = 120; // Maximum backoff interval in seconds + int thisBackOffInterval = 0; + int timestampAlign = 0; + JSONValue result; + SysTime currentTime; + SysTime retryTime; + bool retrySuccess = false; + bool transientError = false; + + while (!retrySuccess) { + // Reset thisBackOffInterval + thisBackOffInterval = 0; + transientError = false; - // what is contained in the curl error message? - if (canFind(errorMessage, "Couldn't connect to server on handle") || canFind(errorMessage, "Couldn't resolve host name on handle") || canFind(errorMessage, "Timeout was reached on handle")) { - // This is a curl timeout - // or is this a 408 request timeout - // https://github.com/abraunegg/onedrive/issues/694 - // Back off & retry with incremental delay - int retryCount = 10000; - int retryAttempts = 0; - int backoffInterval = 0; - int maxBackoffInterval = 3600; - int timestampAlign = 0; - bool retrySuccess = false; - SysTime currentTime; - - // Connectivity to Microsoft OneDrive was lost - addLogEntry("Internet connectivity to Microsoft OneDrive service has been interrupted .. re-trying in the background"); - - // what caused the initial curl exception? - if (canFind(errorMessage, "Couldn't connect to server on handle")) addLogEntry("Unable to connect to server - HTTPS access blocked?", ["debug"]); - if (canFind(errorMessage, "Couldn't resolve host name on handle")) addLogEntry("Unable to resolve server - DNS access blocked?", ["debug"]); - if (canFind(errorMessage, "Timeout was reached on handle")) { - // Common cause is libcurl trying IPv6 DNS resolution when there are only IPv4 DNS servers available - addLogEntry("A libcurl timeout was triggered - data too slow, no DNS resolution response, no server response ... use --debug-https to diagnose this issue further.", ["verbose"]); - addLogEntry("A common cause is IPv6 DNS resolution. Investigate 'ip_protocol_version' to only use IPv4 network communication to potentially resolve this issue.", ["verbose"]); - } - - while (!retrySuccess){ - try { - // configure libcurl to perform a fresh connection - addLogEntry("Configuring libcurl to use a fresh connection for re-try", ["debug"]); - curlEngine.http.handle.set(CurlOption.fresh_connect,1); - // try the access - curlEngine.http.perform(); - // Check the HTTP Response headers - needed for correct 429 handling - checkHTTPResponseHeaders(); - // no error from http.perform() on re-try - addLogEntry("Internet connectivity to Microsoft OneDrive service has been restored"); + if (retryAttempts >= 1) { + // re-try log entry & clock time + retryTime = Clock.currTime(); + retryTime.fracSecs = Duration.zero; + addLogEntry("Retrying the respective OneDrive Graph API call ... (" ~ to!string(retryTime) ~ ")"); + } + + try { + response.reset(); + response = executer(response); + // Check for a valid response + if (response.hasResponse) { + // Process the response + result = response.json(); + // Print response if 'debugResponse' is flagged + if (debugResponse){ + addLogEntry("OneDrive API Response: " ~ response.dumpResponse(), ["debug"]); + } + // Check http response code, raise a OneDriveException if the operation was not successfully performed + // '100 != 2' This condition checks if the response code does not start with 2. In the context of HTTP response codes, the 2xx series represents successful responses. + // '!= 302' This condition explicitly checks if the response code is not 302. The 302 status code represents a temporary redirect, indicating that the requested resource has been temporarily moved to a different URI. + if (response.statusLine.code / 100 != 2 && response.statusLine.code != 302) { + // Not a 2xx or 302 response code + // Every other HTTP status code, including those from the 1xx (Informational), 3xx (other Redirection codes excluding 302), 4xx (Client Error), and 5xx (Server Error) series, will trigger the following line of code. + throw new OneDriveException(response.statusLine.code, response.statusLine.reason, response); + } + // Do we need to validate the JSON response? + if (validateJSONResponse) { + if (result.type() != JSONType.object) { + throw new OneDriveException(0, "Caller request a non null JSON response, get null instead", response); + } + } + + // If retryAtempts is greater than 1, it means we were re-trying the request + if (retryAttempts > 1) { + // No error from http.perform() on re-try + if (!transientError) { + // Log that Internet access has been restored + addLogEntry("Internet connectivity to Microsoft OneDrive service has been restored"); + } // unset the fresh connect option as this then creates performance issues if left enabled addLogEntry("Unsetting libcurl to use a fresh connection as this causes a performance impact if left enabled", ["debug"]); curlEngine.http.handle.set(CurlOption.fresh_connect,0); - // connectivity restored - retrySuccess = true; - } catch (CurlException e) { - // when was the exception generated - currentTime = Clock.currTime(); - // Increment retry attempts - retryAttempts++; - if (canFind(e.msg, "Couldn't connect to server on handle") || canFind(e.msg, "Couldn't resolve host name on handle") || canFind(errorMessage, "Timeout was reached on handle")) { - // no access to Internet - addLogEntry(); - addLogEntry("ERROR: There was a timeout in accessing the Microsoft OneDrive service - Internet connectivity issue?"); - // what is the error reason to assis the user as what to check - if (canFind(e.msg, "Couldn't connect to server on handle")) { - addLogEntry(" - Check HTTPS access or Firewall Rules"); - timestampAlign = 9; - } - if (canFind(e.msg, "Couldn't resolve host name on handle")) { - addLogEntry(" - Check DNS resolution or Firewall Rules"); - timestampAlign = 0; - } - - // increment backoff interval - backoffInterval++; - int thisBackOffInterval = retryAttempts*backoffInterval; - - // display retry information - currentTime.fracSecs = Duration.zero; - auto timeString = currentTime.toString(); - addLogEntry(" Retry attempt: " ~ to!string(retryAttempts), ["verbose"]); - addLogEntry(" This attempt timestamp: " ~ timeString, ["verbose"]); - if (thisBackOffInterval > maxBackoffInterval) { - thisBackOffInterval = maxBackoffInterval; + } + + // On successful processing, break out of the loop + break; + } else { + // Throw a custom 506 error + // Whilst this error code is a bit more esoteric and typically involves content negotiation issues that lead to a configuration error on the server, but it could be loosely + // interpreted to signal that the response received didn't meet the expected criteria or format. + throw new OneDriveException(506, "Received an unexpected response from Microsoft OneDrive", response); + } + // A 'curl' exception was thrown + } catch (CurlException exception) { + // Handle 'curl' exception errors + + // Detail the curl exception, debug output only + addLogEntry("Handling a specific Curl exception:", ["debug"]); + addLogEntry(to!string(response), ["debug"]); + + // Parse and display error message received from OneDrive + addLogEntry(callingFunction ~ "() - Generated a OneDrive CurlException", ["debug"]); + auto errorArray = splitLines(exception.msg); + string errorMessage = errorArray[0]; + + // Configure libcurl to perform a fresh connection + setFreshConnectOption(); + + // What is contained in the curl error message? + if (canFind(errorMessage, "Couldn't connect to server on handle") || canFind(errorMessage, "Couldn't resolve host name on handle") || canFind(errorMessage, "Timeout was reached on handle")) { + // Connectivity to Microsoft OneDrive was lost + addLogEntry("Internet connectivity to Microsoft OneDrive service has been interrupted .. re-trying in the background"); + + // What caused the initial curl exception? + if (canFind(errorMessage, "Couldn't resolve host name on handle")) addLogEntry("Unable to resolve server - DNS access blocked?", ["debug"]); + if (canFind(errorMessage, "Couldn't connect to server on handle")) addLogEntry("Unable to connect to server - HTTPS access blocked?", ["debug"]); + if (canFind(errorMessage, "Timeout was reached on handle")) { + // Common cause is libcurl trying IPv6 DNS resolution when there are only IPv4 DNS servers available + addLogEntry("A libcurl timeout has been triggered - data transfer too slow, no DNS resolution response, no server response", ["verbose"]); + // There are 3 common causes for this issue: + // 1. Usually poor DNS resolution where libcurl flip/flops to use IPv6 and is unable to resolve + // 2. A device between the user and Microsoft OneDrive is unable to correctly handle HTTP/2 communication + // 3. No Internet access from this system at this point in time + addLogEntry(" - IPv6 DNS resolution issues may be causing timeouts. Consider setting 'ip_protocol_version' to IPv4 to potentially avoid this", ["verbose"]); + addLogEntry(" - HTTP/2 compatibility issues might also be interfering with your system. Use 'force_http_11' to switch to HTTP/1.1 to potentially avoid this", ["verbose"]); + addLogEntry(" - If these options do not resolve this timeout issue, please use --debug-https to diagnose this issue further.", ["verbose"]); + } + } else { + // Some other 'libcurl' error was returned + if (canFind(errorMessage, "Problem with the SSL CA cert (path? access rights?) on handle")) { + // error setting certificate verify locations: + // CAfile: /etc/pki/tls/certs/ca-bundle.crt + // CApath: none + // + // Tell the Curl Engine to bypass SSL check - essentially SSL is passing back a bad value due to 'stdio' compile time option + // Further reading: + // https://github.com/curl/curl/issues/6090 + // https://github.com/openssl/openssl/issues/7536 + // https://stackoverflow.com/questions/45829588/brew-install-fails-curl77-error-setting-certificate-verify + // https://forum.dlang.org/post/vwvkbubufexgeuaxhqfl@forum.dlang.org + + addLogEntry("Problem with reading the local SSL CA cert via libcurl - please repair your system SSL CA Certificates"); + throw new OneDriveError("OneDrive operation encountered an issue with libcurl reading the local SSL CA Certificates"); + } else { + // Was this a curl initialization error? + if (canFind(errorMessage, "Failed initialization on handle")) { + // initialization error ... prevent a run-away process if we have zero disk space + ulong localActualFreeSpace = getAvailableDiskSpace("."); + if (localActualFreeSpace == 0) { + throw new OneDriveError("Zero disk space detected"); } - - // detail when the next attempt will be tried - // factor in the delay for curl to generate the exception - otherwise the next timestamp appears to be 'out' even though technically correct - auto nextRetry = currentTime + dur!"seconds"(thisBackOffInterval) + dur!"seconds"(timestampAlign); - addLogEntry(" Next retry in approx: " ~ to!string((thisBackOffInterval + timestampAlign)) ~ " seconds", ["verbose"]); - addLogEntry(" Next retry approx: " ~ to!string(nextRetry), ["verbose"]); - // thread sleep - Thread.sleep(dur!"seconds"(thisBackOffInterval)); - } - if (retryAttempts == retryCount) { - // we have attempted to re-connect X number of times - // false set this to true to break out of while loop - retrySuccess = true; + } else { + // Unknown error + displayGeneralErrorMessage(exception, callingFunction, lineno); } } } - if (retryAttempts >= retryCount) { - addLogEntry(" ERROR: Unable to reconnect to the Microsoft OneDrive service after " ~ to!string(retryCount) ~ " attempts lasting over 1.2 years!"); - throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?"); - } - } else { - - // what error was returned? - if (canFind(errorMessage, "Problem with the SSL CA cert (path? access rights?) on handle")) { - // error setting certificate verify locations: - // CAfile: /etc/pki/tls/certs/ca-bundle.crt - // CApath: none - // - // Tell the Curl Engine to bypass SSL check - essentially SSL is passing back a bad value due to 'stdio' compile time option - // Further reading: - // https://github.com/curl/curl/issues/6090 - // https://github.com/openssl/openssl/issues/7536 - // https://stackoverflow.com/questions/45829588/brew-install-fails-curl77-error-setting-certificate-verify - // https://forum.dlang.org/post/vwvkbubufexgeuaxhqfl@forum.dlang.org + // A OneDrive API exception was thrown + } catch (OneDriveException exception) { + // https://developer.overdrive.com/docs/reference-guide + // https://learn.microsoft.com/en-us/onedrive/developer/rest-api/concepts/errors?view=odsp-graph-online + // https://learn.microsoft.com/en-us/graph/errors + + /** + HTTP/1.1 Response handling + + Errors in the OneDrive API are returned using standard HTTP status codes, as well as a JSON error response object. The following HTTP status codes should be expected. + + Status code Status message Description + 100 Continue Continue + 200 OK Request was handled OK + 201 Created This means you've made a successful POST to checkout, lock in a format, or place a hold + 204 No Content This means you've made a successful DELETE to remove a hold or return a title + + 400 Bad Request Cannot process the request because it is malformed or incorrect. + 401 Unauthorized Required authentication information is either missing or not valid for the resource. + 403 Forbidden Access is denied to the requested resource. The user might not have enough permission. + 404 Not Found The requested resource doesn’t exist. + 405 Method Not Allowed The HTTP method in the request is not allowed on the resource. + 406 Not Acceptable This service doesn’t support the format requested in the Accept header. + 408 Request Time out CUSTOM ERROR - Not expected from OneDrive, but can be used to handle Internet connection failures the same (fallback and try again) + 409 Conflict The current state conflicts with what the request expects. For example, the specified parent folder might not exist. + 410 Gone The requested resource is no longer available at the server. + 411 Length Required A Content-Length header is required on the request. + 412 Precondition Failed A precondition provided in the request (such as an if-match header) does not match the resource's current state. + 413 Request Entity Too Large The request size exceeds the maximum limit. + 415 Unsupported Media Type The content type of the request is a format that is not supported by the service. + 416 Requested Range Not Satisfiable The specified byte range is invalid or unavailable. + 422 Unprocessable Entity Cannot process the request because it is semantically incorrect. + 423 Locked The file is currently checked out or locked for editing by another user + 429 Too Many Requests Client application has been throttled and should not attempt to repeat the request until an amount of time has elapsed. + + 500 Internal Server Error There was an internal server error while processing the request. + 501 Not Implemented The requested feature isn’t implemented. + 502 Bad Gateway The service was unreachable + 503 Service Unavailable The service is temporarily unavailable. You may repeat the request after a delay. There may be a Retry-After header. + 504 Gateway Timeout The server, which is acting as a gateway or proxy, did not receive a timely response from an upstream server it needed to access in order to complete the request + 506 Variant Also Negotiates CUSTOM ERROR - Received an unexpected response from Microsoft OneDrive + 507 Insufficient Storage The maximum storage quota has been reached. + 509 Bandwidth Limit Exceeded Your app has been throttled for exceeding the maximum bandwidth cap. Your app can retry the request again after more time has elapsed. + + HTTP/2 Response handling + + 0 OK - addLogEntry("Problem with reading the SSL CA cert via libcurl - please repair your system SSL CA Certificates"); - // Must force exit here, allow logging to be done. If needed later, we could re-use setDisableSSLVerifyPeer() - Thread.sleep(dur!("msecs")(500)); - exit(-1); - } else { - // Log that an error was returned - addLogEntry("ERROR: OneDrive returned an error with the following message:"); - // Some other error was returned - addLogEntry(" Error Message: " ~ errorMessage); - addLogEntry(" Calling Function: " ~ getFunctionName!({})); + **/ + + // Detail the OneDriveAPI exception, debug output only + addLogEntry("Handling a OneDrive API exception:", ["debug"]); + addLogEntry(to!string(response), ["debug"]); + + // Parse and display error message received from OneDrive + addLogEntry(callingFunction ~ "() - Generated a OneDriveException", ["debug"]); - // Was this a curl initialization error? - if (canFind(errorMessage, "Failed initialization on handle")) { - // initialization error ... prevent a run-away process if we have zero disk space - ulong localActualFreeSpace = getAvailableDiskSpace("."); - if (localActualFreeSpace == 0) { - // force exit - shutdown(); - // Must force exit here, allow logging to be done - Thread.sleep(dur!("msecs")(500)); - exit(-1); + // Configure libcurl to perform a fresh connection on API retry + setFreshConnectOption(); + + // Perform action based on the HTTP Status Code + switch(exception.httpStatusCode) { + + // 0 - OK ... HTTP/2 version of 200 OK + case 0: + break; + // 100 - Continue + case 100: + break; + // 408 - Request Time Out + // 429 - Too Many Requests, backoff + case 408,429: + // If OneDrive sends a status code 429 then this function will be used to process the Retry-After response header which contains the value by which we need to wait + if (exception.httpStatusCode == 408) { + addLogEntry("Handling a OneDrive HTTP 408 Response Code (Request Time Out) - Internal Thread ID: " ~ to!string(curlEngine.internalThreadId)); + } else { + addLogEntry("Handling a OneDrive HTTP 429 Response Code (Too Many Requests) - Internal Thread ID: " ~ to!string(curlEngine.internalThreadId)); } - } + // Read in the Retry-After HTTP header as set and delay as per this value before retrying the request + thisBackOffInterval = response.getRetryAfterValue(); + addLogEntry("Using Retry-After Value = " ~ to!string(thisBackOffInterval), ["debug"]); + addLogEntry("Using Retry-After Value = " ~ to!string(thisBackOffInterval)); + transientError = true; + break; + // Transient errors + // 503 - Service Unavailable + // 504 - Gateway Timeout + case 503,504: + // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request + auto errorArray = splitLines(exception.msg); + addLogEntry(to!string(errorArray[0]) ~ " when attempting to query the OneDrive API Service - retrying applicable request in 30 seconds - Internal Thread ID: " ~ to!string(curlEngine.internalThreadId)); + addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); + // Transient error - try again in 30 seconds + thisBackOffInterval = 30; + transientError = true; + break; + // Default + default: + // This exception should be then passed back to the original calling function for handling a OneDriveException + throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); } + + // A FileSystem exception was thrown + } catch (ErrnoException exception) { + // There was a file system error + // display the error message + displayFileSystemErrorMessage(exception.msg, callingFunction); + throw new OneDriveException(0, "There was a file system error during OneDrive request: " ~ exception.msg, response); } - // return an empty JSON for handling - return json; - } - - try { - json = content.parseJSON(); - } catch (JSONException e) { - // Log that a JSON Exception was caught, dont output the HTML response from OneDrive - addLogEntry("JSON Exception caught when performing HTTP operations - use --debug-https to diagnose further", ["debug"]); - } - return json; - } - - private void redeemToken(char[] authCode){ - char[] postData = - "client_id=" ~ clientId ~ - "&redirect_uri=" ~ redirectUrl ~ - "&code=" ~ authCode ~ - "&grant_type=authorization_code"; - acquireToken(postData); - } - - private JSONValue upload(string filepath, string url) { - checkAccessTokenExpired(); - // open file as read-only in binary mode - auto file = File(filepath, "rb"); - // function scopes - scope(exit) { - curlEngine.http.clearRequestHeaders(); - curlEngine.http.onSend = null; - curlEngine.http.onReceive = null; - curlEngine.http.onReceiveHeader = null; - curlEngine.http.onReceiveStatusLine = null; - curlEngine.http.contentLength = 0; - // close file if open - if (file.isOpen()){ - // close open file - file.close(); + // Increment re-try counter + retryAttempts++; + + // Has maxRetryCount been reached? + if (retryAttempts > maxRetryCount) { + addLogEntry("ERROR: Unable to reconnect to the Microsoft OneDrive service after " ~ to!string(retryAttempts) ~ " attempts lasting approximately 365 days"); + throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?", response); + } else { + // Was 'thisBackOffInterval' set by a 429 event ? + if (thisBackOffInterval == 0) { + // Calculate and apply exponential backoff upto a maximum of 120 seconds before the API call is re-tried + thisBackOffInterval = calculateBackoff(retryAttempts, baseBackoffInterval, maxBackoffInterval); + } + + // When are we re-trying the API call? + currentTime = Clock.currTime(); + currentTime.fracSecs = Duration.zero; + auto timeString = currentTime.toString(); + addLogEntry("Retry attempt: " ~ to!string(retryAttempts) ~ " - Internal Thread ID: " ~ to!string(curlEngine.internalThreadId), ["verbose"]); + addLogEntry(" This attempt timestamp: " ~ timeString, ["verbose"]); + // Detail when the next attempt will be tried + // Factor in the delay for curl to generate the exception - otherwise the next timestamp appears to be 'out' even though technically correct + auto nextRetry = currentTime + dur!"seconds"(thisBackOffInterval) + dur!"seconds"(timestampAlign); + addLogEntry(" Next retry in approx: " ~ to!string((thisBackOffInterval + timestampAlign)) ~ " seconds"); + addLogEntry(" Next retry approx: " ~ to!string(nextRetry), ["verbose"]); + + // Thread sleep + Thread.sleep(dur!"seconds"(thisBackOffInterval)); } } - curlEngine.connect(HTTP.Method.put, url); - addAccessTokenHeader(); - curlEngine.http.addRequestHeader("Content-Type", "application/octet-stream"); - curlEngine.http.onSend = data => file.rawRead(data).length; - curlEngine.http.contentLength = file.size; - JSONValue response; - response = performHTTPOperation(); - checkHttpResponseCode(response); - return response; + // Return the result + return result; } - private void checkHTTPResponseHeaders() { - // Get the HTTP Response headers - needed for correct 429 handling - auto responseHeaders = curlEngine.http.responseHeaders(); - if (debugResponse){ - addLogEntry("curlEngine.http.perform() => HTTP Response Headers: " ~ to!string(responseHeaders), ["debug"]); - } - - // is retry-after in the response headers - if ("retry-after" in curlEngine.http.responseHeaders) { - // Set the retry-after value - addLogEntry("curlEngine.http.perform() => Received a 'Retry-After' Header Response with the following value: " ~ to!string(curlEngine.http.responseHeaders["retry-after"]), ["debug"]); - addLogEntry("curlEngine.http.perform() => Setting retryAfterValue to: " ~ to!string(curlEngine.http.responseHeaders["retry-after"]), ["debug"]); - retryAfterValue = to!ulong(curlEngine.http.responseHeaders["retry-after"]); - } - } - - private void checkHttpResponseCode(JSONValue response) { - switch(curlEngine.http.statusLine.code) { - // 0 - OK ... HTTP2 version of 200 OK - case 0: - break; - // 100 - Continue - case 100: - break; - // 200 - OK - case 200: - // No Log .. - break; - // 201 - Created OK - // 202 - Accepted - // 204 - Deleted OK - case 201,202,204: - // Log if --debug-https logging is used - if (debugHTTPResponseOutput) { - addLogEntry("OneDrive Response: '" ~ to!string(curlEngine.http.statusLine.code) ~ " - " ~ to!string(curlEngine.http.statusLine.reason) ~ "'", ["debug"]); - } - break; - - // 302 - resource found and available at another location, redirect - case 302: - // Log if --debug-https logging is used - if (debugHTTPResponseOutput) { - addLogEntry("OneDrive Response: '" ~ to!string(curlEngine.http.statusLine.code) ~ " - " ~ to!string(curlEngine.http.statusLine.reason) ~ "'", ["debug"]); - } - break; - - // 400 - Bad Request - case 400: - // Bad Request .. how should we act? - // make sure this is thrown so that it is caught - throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); - - // 403 - Forbidden - case 403: - // OneDrive responded that the user is forbidden - addLogEntry("OneDrive returned a 'HTTP 403 - Forbidden' - gracefully handling error", ["verbose"]); - - // Throw this as a specific exception so this is caught when performing 'siteQuery = onedrive.o365SiteSearch(nextLink);' call - throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); - - // 412 - Precondition Failed - case 412: - // Throw this as a specific exception so this is caught when performing sync.uploadLastModifiedTime - throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); - - // Server side (OneDrive) Errors - // 500 - Internal Server Error - // 502 - Bad Gateway - // 503 - Service Unavailable - // 504 - Gateway Timeout (Issue #320) - case 500: - // Throw this as a specific exception so this is caught - throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); - - case 502: - // Throw this as a specific exception so this is caught - throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); - - case 503: - // Throw this as a specific exception so this is caught - throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); - - case 504: - // Throw this as a specific exception so this is caught - throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); - - // Default - all other errors that are not a 2xx or a 302 - default: - if (curlEngine.http.statusLine.code / 100 != 2 && curlEngine.http.statusLine.code != 302) { - throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason, response); - } - } + // Calculates the delay for exponential backoff + private int calculateBackoff(int retryAttempts, int baseInterval, int maxInterval) { + int backoffTime = min(pow(2, retryAttempts) * baseInterval, maxInterval); + return backoffTime; } - private void checkHttpCode() { - // https://dev.onedrive.com/misc/errors.htm - // https://developer.overdrive.com/docs/reference-guide - - /* - HTTP/1.1 Response handling - - Errors in the OneDrive API are returned using standard HTTP status codes, as well as a JSON error response object. The following HTTP status codes should be expected. - - Status code Status message Description - 100 Continue Continue - 200 OK Request was handled OK - 201 Created This means you've made a successful POST to checkout, lock in a format, or place a hold - 204 No Content This means you've made a successful DELETE to remove a hold or return a title - - 400 Bad Request Cannot process the request because it is malformed or incorrect. - 401 Unauthorized Required authentication information is either missing or not valid for the resource. - 403 Forbidden Access is denied to the requested resource. The user might not have enough permission. - 404 Not Found The requested resource doesn’t exist. - 405 Method Not Allowed The HTTP method in the request is not allowed on the resource. - 406 Not Acceptable This service doesn’t support the format requested in the Accept header. - 408 Request Time out Not expected from OneDrive, but can be used to handle Internet connection failures the same (fallback and try again) - 409 Conflict The current state conflicts with what the request expects. For example, the specified parent folder might not exist. - 410 Gone The requested resource is no longer available at the server. - 411 Length Required A Content-Length header is required on the request. - 412 Precondition Failed A precondition provided in the request (such as an if-match header) does not match the resource's current state. - 413 Request Entity Too Large The request size exceeds the maximum limit. - 415 Unsupported Media Type The content type of the request is a format that is not supported by the service. - 416 Requested Range Not Satisfiable The specified byte range is invalid or unavailable. - 422 Unprocessable Entity Cannot process the request because it is semantically incorrect. - 429 Too Many Requests Client application has been throttled and should not attempt to repeat the request until an amount of time has elapsed. - - 500 Internal Server Error There was an internal server error while processing the request. - 501 Not Implemented The requested feature isn’t implemented. - 502 Bad Gateway The service was unreachable - 503 Service Unavailable The service is temporarily unavailable. You may repeat the request after a delay. There may be a Retry-After header. - 507 Insufficient Storage The maximum storage quota has been reached. - 509 Bandwidth Limit Exceeded Your app has been throttled for exceeding the maximum bandwidth cap. Your app can retry the request again after more time has elapsed. - - HTTP/2 Response handling - - 0 OK - - */ - - switch(curlEngine.http.statusLine.code) - { - // 0 - OK ... HTTP2 version of 200 OK - case 0: - break; - // 100 - Continue - case 100: - break; - // 200 - OK - case 200: - // No Log .. - break; - // 201 - Created OK - // 202 - Accepted - // 204 - Deleted OK - case 201,202,204: - // Log if --debug-https logging is used - if (debugHTTPResponseOutput) { - addLogEntry("OneDrive Response: '" ~ to!string(curlEngine.http.statusLine.code) ~ " - " ~ to!string(curlEngine.http.statusLine.reason) ~ "'", ["debug"]); - } - break; - - // 302 - resource found and available at another location, redirect - case 302: - // Log if --debug-https logging is used - if (debugHTTPResponseOutput) { - addLogEntry("OneDrive Response: '" ~ to!string(curlEngine.http.statusLine.code) ~ " - " ~ to!string(curlEngine.http.statusLine.reason) ~ "'", ["debug"]); - } - break; - - // 400 - Bad Request - case 400: - // Bad Request .. how should we act? - addLogEntry("OneDrive returned a 'HTTP 400 - Bad Request' - gracefully handling error", ["verbose"]); - break; - - // 403 - Forbidden - case 403: - // OneDrive responded that the user is forbidden - addLogEntry("OneDrive returned a 'HTTP 403 - Forbidden' - gracefully handling error", ["verbose"]); - throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason); - - // 404 - Item not found - case 404: - // Item was not found - do not throw an exception - addLogEntry("OneDrive returned a 'HTTP 404 - Item not found' - gracefully handling error", ["verbose"]); - break; - - // 408 - Request Timeout - case 408: - // Request to connect to OneDrive service timed out - addLogEntry("Request Timeout - gracefully handling error", ["verbose"]); - throw new OneDriveException(408, "Request Timeout - HTTP 408 or Internet down?"); - - // 409 - Conflict - case 409: - // Conflict handling .. how should we act? This only really gets triggered if we are using --local-first & we remove items.db as the DB thinks the file is not uploaded but it is - addLogEntry("OneDrive returned a 'HTTP 409 - Conflict' - gracefully handling error", ["verbose"]); - break; - - // 412 - Precondition Failed - case 412: - // A precondition provided in the request (such as an if-match header) does not match the resource's current state. - addLogEntry("OneDrive returned a 'HTTP 412 - Precondition Failed' - gracefully handling error", ["verbose"]); - break; - - // 415 - Unsupported Media Type - case 415: - // Unsupported Media Type ... sometimes triggered on image files, especially PNG - addLogEntry("OneDrive returned a 'HTTP 415 - Unsupported Media Type' - gracefully handling error", ["verbose"]); - break; - - // 429 - Too Many Requests - case 429: - // Too many requests in a certain time window - // Check the HTTP Response headers - needed for correct 429 handling - checkHTTPResponseHeaders(); - // https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online - addLogEntry("OneDrive returned a 'HTTP 429 - Too Many Requests' - gracefully handling error", ["verbose"]); - throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason); - - // Server side (OneDrive) Errors - // 500 - Internal Server Error - // 502 - Bad Gateway - // 503 - Service Unavailable - // 504 - Gateway Timeout (Issue #320) - case 500: - // No actions - addLogEntry("OneDrive returned a 'HTTP 500 Internal Server Error' - gracefully handling error", ["verbose"]); - break; - - case 502: - // No actions - addLogEntry("OneDrive returned a 'HTTP 502 Bad Gateway Error' - gracefully handling error", ["verbose"]); - break; - - case 503: - // No actions - addLogEntry("OneDrive returned a 'HTTP 503 Service Unavailable Error' - gracefully handling error", ["verbose"]); - break; - - case 504: - // No actions - addLogEntry("OneDrive returned a 'HTTP 504 Gateway Timeout Error' - gracefully handling error", ["verbose"]); - break; - - // "else" - default: - throw new OneDriveException(curlEngine.http.statusLine.code, curlEngine.http.statusLine.reason); - } + // Configure libcurl to perform a fresh connection + private void setFreshConnectOption() { + addLogEntry("Configuring libcurl to use a fresh connection for re-try", ["debug"]); + curlEngine.http.handle.set(CurlOption.fresh_connect,1); } } \ No newline at end of file diff --git a/src/sync.d b/src/sync.d index 59b4bedbd..5d3e1e971 100644 --- a/src/sync.d +++ b/src/sync.d @@ -80,7 +80,6 @@ struct DeltaLinkDetails { class SyncEngine { // Class Variables ApplicationConfig appConfig; - OneDriveApi oneDriveApiInstance; ItemDatabase itemDB; ClientSideFiltering selectiveSync; @@ -184,6 +183,11 @@ class SyncEngine { // Configure this class instance this(ApplicationConfig appConfig, ItemDatabase itemDB, ClientSideFiltering selectiveSync) { + + // Create the specific task pool to process items in parallel + processPool = new TaskPool(to!int(appConfig.getValueLong("threads"))); + addLogEntry("PROCESS POOL WORKER THREADS: " ~ to!string(processPool.size), ["debug"]); + // Configure the class varaible to consume the application configuration this.appConfig = appConfig; // Configure the class varaible to consume the database configuration @@ -298,54 +302,63 @@ class SyncEngine { } } + // The destructor should only clean up resources owned directly by this instance + ~this() { + processPool.finish(true); + } + // Initialise the Sync Engine class bool initialise() { + // Control whether the worker threads are daemon threads. A daemon thread is automatically terminated when all non-daemon threads have terminated. + processPool.isDaemon(true); - // Create common parallel thread pool - processPool = taskPool(); - processPool.isDaemon(true); // Control whether the worker threads are daemon threads. A daemon thread is automatically terminated when all non-daemon threads have terminated. - - // create a new instance of the OneDrive API + // Create a new instance of the OneDrive API + OneDriveApi oneDriveApiInstance; oneDriveApiInstance = new OneDriveApi(appConfig); + + // Can the API be initialised successfully? if (oneDriveApiInstance.initialise()) { + // Get the relevant default drive details try { - // Get the relevant default account & drive details getDefaultDriveDetails(); } catch (accountDetailsException exception) { // details could not be queried addLogEntry(exception.msg); // Shutdown this API instance, as we will create API instances as required, when required - oneDriveApiInstance.shutdown(); + oneDriveApiInstance.releaseCurlEngine(); // Free object and memory - object.destroy(oneDriveApiInstance); + //object.destroy(oneDriveApiInstance); + //oneDriveApiInstance = null; // Must force exit here, allow logging to be done forceExit(); } + // Get the relevant default root details try { - // Get the relevant default account & drive details getDefaultRootDetails(); } catch (accountDetailsException exception) { // details could not be queried addLogEntry(exception.msg); // Shutdown this API instance, as we will create API instances as required, when required - oneDriveApiInstance.shutdown(); + oneDriveApiInstance.releaseCurlEngine(); // Free object and memory - object.destroy(oneDriveApiInstance); + //object.destroy(oneDriveApiInstance); + //oneDriveApiInstance = null; // Must force exit here, allow logging to be done forceExit(); } + // Display details try { - // Display details displaySyncEngineDetails(); } catch (accountDetailsException exception) { - // details could not be queried + // Details could not be queried addLogEntry(exception.msg); // Shutdown this API instance, as we will create API instances as required, when required - oneDriveApiInstance.shutdown(); + oneDriveApiInstance.releaseCurlEngine(); // Free object and memory - object.destroy(oneDriveApiInstance); + //object.destroy(oneDriveApiInstance); + //oneDriveApiInstance = null; // Must force exit here, allow logging to be done forceExit(); } @@ -353,9 +366,10 @@ class SyncEngine { // API could not be initialised addLogEntry("OneDrive API could not be initialised with previously used details"); // Shutdown this API instance, as we will create API instances as required, when required - oneDriveApiInstance.shutdown(); + oneDriveApiInstance.releaseCurlEngine(); // Free object and memory - object.destroy(oneDriveApiInstance); + //object.destroy(oneDriveApiInstance); + //oneDriveApiInstance = null; // Must force exit here, allow logging to be done forceExit(); } @@ -364,59 +378,42 @@ class SyncEngine { addLogEntry("Sync Engine Initialised with new Onedrive API instance", ["verbose"]); // Shutdown this API instance, as we will create API instances as required, when required - oneDriveApiInstance.shutdown(); - - // Free object and memory - object.destroy(oneDriveApiInstance); + oneDriveApiInstance.releaseCurlEngine(); return true; } + // Shutdown the sync engine, wait for anything in processPool to complete + void shutdown() { + addLogEntry("SYNC-ENGINE: Waiting for all internal threads to complete", ["debug"]); + processPool.finish(true); + } + // Get Default Drive Details for this Account void getDefaultDriveDetails() { // Function variables JSONValue defaultOneDriveDriveDetails; + // Create a new instance of the OneDrive API + OneDriveApi getDefaultDriveApiInstance; + getDefaultDriveApiInstance = new OneDriveApi(appConfig); + getDefaultDriveApiInstance.initialise(); + // Get Default Drive Details for this Account try { addLogEntry("Getting Account Default Drive Details", ["debug"]); - defaultOneDriveDriveDetails = oneDriveApiInstance.getDefaultDriveDetails(); + defaultOneDriveDriveDetails = getDefaultDriveApiInstance.getDefaultDriveDetails(); } catch (OneDriveException exception) { - addLogEntry("defaultOneDriveDriveDetails = oneDriveApiInstance.getDefaultDriveDetails() generated a OneDriveException", ["debug"]); + addLogEntry("defaultOneDriveDriveDetails = getDefaultDriveApiInstance.getDefaultDriveDetails() generated a OneDriveException", ["debug"]); string thisFunctionName = getFunctionName!({}); if ((exception.httpStatusCode == 400) || (exception.httpStatusCode == 401)) { // Handle the 400 | 401 error handleClientUnauthorised(exception.httpStatusCode, exception.msg); - } - - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(oneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) ||(exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to query Account Default Drive Details - retrying applicable request in 30 seconds"); - addLogEntry("defaultOneDriveDriveDetails = oneDriveApiInstance.getDefaultDriveDetails() previously threw an error - retrying", ["debug"]); - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429 and 504 - but loop back calling this function - addLogEntry("Retrying Function: getDefaultDriveDetails()", ["debug"]); - getDefaultDriveDetails(); } else { - // Default operation if not 408,429,503,504 errors - // display what the error is + // Default operation if not 400,401 errors + // - 408,429,503,504 errors are handled as a retry within getDefaultDriveApiInstance + // Display what the error is displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); } } @@ -477,6 +474,11 @@ class SyncEngine { // Handle the invalid JSON response throw new accountDetailsException(); } + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + getDefaultDriveApiInstance.releaseCurlEngine(); + object.destroy(getDefaultDriveApiInstance); + getDefaultDriveApiInstance = null; } // Get Default Root Details for this Account @@ -485,46 +487,26 @@ class SyncEngine { // Function variables JSONValue defaultOneDriveRootDetails; + // Create a new instance of the OneDrive API + OneDriveApi getDefaultRootApiInstance; + getDefaultRootApiInstance = new OneDriveApi(appConfig); + getDefaultRootApiInstance.initialise(); + // Get Default Root Details for this Account try { addLogEntry("Getting Account Default Root Details", ["debug"]); - defaultOneDriveRootDetails = oneDriveApiInstance.getDefaultRootDetails(); + defaultOneDriveRootDetails = getDefaultRootApiInstance.getDefaultRootDetails(); } catch (OneDriveException exception) { - addLogEntry("defaultOneDriveRootDetails = oneDriveApiInstance.getDefaultRootDetails() generated a OneDriveException", ["debug"]); + addLogEntry("defaultOneDriveRootDetails = getDefaultRootApiInstance.getDefaultRootDetails() generated a OneDriveException", ["debug"]); string thisFunctionName = getFunctionName!({}); if ((exception.httpStatusCode == 400) || (exception.httpStatusCode == 401)) { // Handle the 400 | 401 error handleClientUnauthorised(exception.httpStatusCode, exception.msg); - } - - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(oneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to query Account Default Root Details - retrying applicable request in 30 seconds"); - addLogEntry("defaultOneDriveRootDetails = oneDriveApiInstance.getDefaultRootDetails() previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - getDefaultRootDetails(); } else { - // Default operation if not 408,429,503,504 errors - // display what the error is + // Default operation if not 400,401 errors + // - 408,429,503,504 errors are handled as a retry within getDefaultRootApiInstance + // Display what the error is displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); } } @@ -541,19 +523,39 @@ class SyncEngine { // Handle the invalid JSON response throw new accountDetailsException(); } + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + getDefaultRootApiInstance.releaseCurlEngine(); + object.destroy(getDefaultRootApiInstance); + getDefaultRootApiInstance = null; } - // Reset syncFailures to false + // Reset syncFailures to false based on file activity void resetSyncFailures() { - // Reset syncFailures to false if these are both empty - if (syncFailures) { - if ((fileDownloadFailures.empty) && (fileUploadFailures.empty)) { - addLogEntry("Resetting syncFailures = false"); + // Log initial status and any non-empty arrays + string logMessage = "Evaluating reset of syncFailures: "; + if (fileDownloadFailures.length > 0) { + logMessage ~= "fileDownloadFailures is not empty; "; + } + if (fileUploadFailures.length > 0) { + logMessage ~= "fileUploadFailures is not empty; "; + } + + // Check if both arrays are empty to reset syncFailures + if (fileDownloadFailures.length == 0 && fileUploadFailures.length == 0) { + if (syncFailures) { syncFailures = false; + logMessage ~= "Resetting syncFailures to false."; } else { - addLogEntry("File activity array's not empty - not resetting syncFailures"); + logMessage ~= "syncFailures already false."; } + } else { + // Indicate no reset of syncFailures due to non-empty conditions + logMessage ~= "Not resetting syncFailures due to non-empty arrays."; } + + // Log the final decision and conditions + addLogEntry(logMessage, ["debug"]); } // Perform a sync of the OneDrive Account @@ -569,7 +571,7 @@ class SyncEngine { addLogEntry("Perform a Full Scan True-Up: " ~ to!string(appConfig.fullScanTrueUpRequired), ["debug"]); // Fetch the API response of /delta to track changes that were performed online - fetchOneDriveDeltaAPIResponse(null, null, null); + fetchOneDriveDeltaAPIResponse(); // Process any download activities or cleanup actions processDownloadActivities(); @@ -833,11 +835,9 @@ class SyncEngine { } // Create a new API Instance for querying /delta and initialise it - // Reuse the socket to speed up - bool keepAlive = true; OneDriveApi getDeltaQueryOneDriveApiInstance; getDeltaQueryOneDriveApiInstance = new OneDriveApi(appConfig); - getDeltaQueryOneDriveApiInstance.initialise(keepAlive); + getDeltaQueryOneDriveApiInstance.initialise(); for (;;) { responseBundleCount++; @@ -937,10 +937,10 @@ class SyncEngine { // To finish off the JSON processing items, this is needed to reflect this in the log addLogEntry("------------------------------------------------------------------", ["debug"]); - // Shutdown this API instance, as we will create API instances as required, when required - getDeltaQueryOneDriveApiInstance.shutdown(); - // Free object and memory + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + getDeltaQueryOneDriveApiInstance.releaseCurlEngine(); object.destroy(getDeltaQueryOneDriveApiInstance); + getDeltaQueryOneDriveApiInstance = null; // Log that we have finished querying the /delta API if (appConfig.verbosityCount == 0) { @@ -2227,9 +2227,10 @@ class SyncEngine { if (!dryRun) { // Attempt to download the file as there is enough free space locally OneDriveApi downloadFileOneDriveApiInstance; - downloadFileOneDriveApiInstance = new OneDriveApi(appConfig); + try { // Initialise API instance + downloadFileOneDriveApiInstance = new OneDriveApi(appConfig); downloadFileOneDriveApiInstance.initialise(); // OneDrive Business Shared Files - update the driveId where to get the file from @@ -2239,10 +2240,11 @@ class SyncEngine { // Perform the download downloadFileOneDriveApiInstance.downloadById(downloadDriveId, downloadItemId, newItemPath, jsonFileSize); - downloadFileOneDriveApiInstance.shutdown(); - // Free object and memory + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + downloadFileOneDriveApiInstance.releaseCurlEngine(); object.destroy(downloadFileOneDriveApiInstance); + downloadFileOneDriveApiInstance = null; } catch (OneDriveException exception) { addLogEntry("downloadFileOneDriveApiInstance.downloadById(downloadDriveId, downloadItemId, newItemPath, jsonFileSize); generated a OneDriveException", ["debug"]); @@ -2254,37 +2256,10 @@ class SyncEngine { addLogEntry("Unable to download this file as this was shared as read-only without download permission: " ~ newItemPath); downloadFailed = true; } else { - - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(downloadFileOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to download an item from OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - downloadFileItem(onedriveJSONItem); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); - } + // Default operation if not a 403 error + // - 408,429,503,504 errors are handled as a retry within downloadFileOneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); } } catch (FileException e) { // There was a file system error @@ -2525,96 +2500,37 @@ class SyncEngine { addLogEntry("------------------------------------------------------------------", ["debug"]); try { - deltaChangesBundle = getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink); + deltaChangesBundle = getDeltaQueryOneDriveApiInstance.getChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink); } catch (OneDriveException exception) { // caught an exception - addLogEntry("getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink) generated a OneDriveException", ["debug"]); + addLogEntry("getDeltaQueryOneDriveApiInstance.getChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink) generated a OneDriveException", ["debug"]); auto errorArray = splitLines(exception.msg); string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(getDeltaQueryOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - addLogEntry(to!string(errorArray[0]) ~ " when attempting to query OneDrive API for Delta Changes - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // dont retry request, loop back to calling function - addLogEntry("Looping back after failure", ["debug"]); - deltaChangesBundle = null; + + // Error handling operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within getDeltaQueryOneDriveApiInstance + if (exception.httpStatusCode == 410) { + addLogEntry(); + addLogEntry("WARNING: The OneDrive API responded with an error that indicates the locally stored deltaLink value is invalid"); + // Essentially the 'providedDeltaLink' that we have stored is no longer available ... re-try without the stored deltaLink + addLogEntry("WARNING: Retrying OneDrive API call without using the locally stored deltaLink value"); + // Configure an empty deltaLink + addLogEntry("Delta link expired for 'getDeltaQueryOneDriveApiInstance.getChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink)', setting 'deltaLink = null'", ["debug"]); + string emptyDeltaLink = ""; + // retry with empty deltaLink + deltaChangesBundle = getDeltaQueryOneDriveApiInstance.getChangesByItemId(selectedDriveId, selectedItemId, emptyDeltaLink); } else { - // Default operation if not 408,429,503,504 errors - if (exception.httpStatusCode == 410) { - addLogEntry(); - addLogEntry("WARNING: The OneDrive API responded with an error that indicates the locally stored deltaLink value is invalid"); - // Essentially the 'providedDeltaLink' that we have stored is no longer available ... re-try without the stored deltaLink - addLogEntry("WARNING: Retrying OneDrive API call without using the locally stored deltaLink value"); - // Configure an empty deltaLink - addLogEntry("Delta link expired for 'getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, providedDeltaLink)', setting 'deltaLink = null'", ["debug"]); - string emptyDeltaLink = ""; - // retry with empty deltaLink - deltaChangesBundle = getDeltaQueryOneDriveApiInstance.viewChangesByItemId(selectedDriveId, selectedItemId, emptyDeltaLink); - } else { - // display what the error is - addLogEntry("CODING TO DO: Hitting this failure error output"); - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - deltaChangesBundle = null; - } + // Display what the error is + addLogEntry("CODING TO DO: Hitting this failure error output after getting a httpStatusCode == 410 when the API responded the deltaLink was invalid"); + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + deltaChangesBundle = null; } } return deltaChangesBundle; } - // Common code to handle a 408 or 429 response from the OneDrive API - void handleOneDriveThrottleRequest(OneDriveApi activeOneDriveApiInstance) { - - // If OneDrive sends a status code 429 then this function will be used to process the Retry-After response header which contains the value by which we need to wait - addLogEntry("Handling a OneDrive HTTP 429 Response Code (Too Many Requests)", ["debug"]); - - // Read in the Retry-After HTTP header as set and delay as per this value before retrying the request - auto retryAfterValue = activeOneDriveApiInstance.getRetryAfterValue(); - addLogEntry("Using Retry-After Value = " ~ to!string(retryAfterValue), ["debug"]); - - // HTTP request returned status code 429 (Too Many Requests) - // https://github.com/abraunegg/onedrive/issues/133 - // https://github.com/abraunegg/onedrive/issues/815 - - ulong delayBeforeRetry = 0; - if (retryAfterValue != 0) { - // Use the HTTP Response Header Value - delayBeforeRetry = retryAfterValue; - } else { - // Use a 120 second delay as a default given header value was zero - // This value is based on log files and data when determining correct process for 429 response handling - delayBeforeRetry = 120; - // Update that we are over-riding the provided value with a default - addLogEntry("HTTP Response Header retry-after value was 0 - Using a preconfigured default of: " ~ to!string(delayBeforeRetry), ["debug"]); - } - - // Sleep thread as per request - addLogEntry("Thread sleeping due to 'HTTP request returned status code 429' - The request has been throttled"); - addLogEntry("Sleeping for " ~ to!string(delayBeforeRetry) ~ " seconds"); - Thread.sleep(dur!"seconds"(delayBeforeRetry)); - - // Reset retry-after value to zero as we have used this value now and it may be changed in the future to a different value - activeOneDriveApiInstance.resetRetryAfterValue(); - } - // If the JSON response is not correct JSON object, exit void invalidJSONResponseFromOneDriveAPI() { addLogEntry("ERROR: Query of the OneDrive API returned an invalid JSON response"); @@ -2789,28 +2705,30 @@ class SyncEngine { // What eTag value do we use? string eTagValue; if (appConfig.accountType == "personal") { + // Nullify the eTag to avoid 412 errors as much as possible eTagValue = null; } else { eTagValue = eTag; } JSONValue response; - // Create a new OneDrive API instance OneDriveApi uploadLastModifiedTimeApiInstance; - uploadLastModifiedTimeApiInstance = new OneDriveApi(appConfig); - uploadLastModifiedTimeApiInstance.initialise(); // Try and update the online last modified time try { + // Create a new OneDrive API instance + uploadLastModifiedTimeApiInstance = new OneDriveApi(appConfig); + uploadLastModifiedTimeApiInstance.initialise(); // Use this instance response = uploadLastModifiedTimeApiInstance.updateById(driveId, id, data, eTagValue); - // Shut the instance down - uploadLastModifiedTimeApiInstance.shutdown(); - // Free object and memory + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + uploadLastModifiedTimeApiInstance.releaseCurlEngine(); object.destroy(uploadLastModifiedTimeApiInstance); + uploadLastModifiedTimeApiInstance = null; // Do we actually save the response? - // Special case here .. if the DB record item (originItem) is a remote object, thus, if we same the 'response' we will have a DB FOREIGN KEY constraint failed problem + // Special case here .. if the DB record item (originItem) is a remote object, thus, if we save the 'response' we will have a DB FOREIGN KEY constraint failed problem // Update 'originItem.mtime' with the correct timestamp // Update 'originItem.size' with the correct size from the response // Update 'originItem.eTag' with the correct eTag from the response @@ -2823,47 +2741,27 @@ class SyncEngine { // Is the response a valid JSON object - validation checking done in saveItem saveItem(response); } - } catch (OneDriveException exception) { - string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(uploadLastModifiedTimeApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to update the timestamp on an item on OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function + // Handle a 412 - A precondition provided in the request (such as an if-match header) does not match the resource's current state. + if (exception.httpStatusCode == 412) { + // OneDrive threw a 412 error, most likely: ETag does not match current item's value + addLogEntry("OneDrive returned a 'HTTP 412 - Precondition Failed' when attempting file time stamp update - gracefully handling error", ["verbose"]); + addLogEntry("File Metadata Update Failed - OneDrive eTag / cTag match issue", ["debug"]); addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - uploadLastModifiedTime(originItem, driveId, id, mtime, eTag); - return; + // Retry without eTag + uploadLastModifiedTime(originItem, driveId, id, mtime, null); } else { - // Default operation if not 408,429,503,504 errors - if (exception.httpStatusCode == 409) { - // ETag does not match current item's value - use a null eTag - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - uploadLastModifiedTime(originItem, driveId, id, mtime, null); - } else { - // display what the error is - displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); - } + // Any other error that should be handled + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); } + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + uploadLastModifiedTimeApiInstance.releaseCurlEngine(); + object.destroy(uploadLastModifiedTimeApiInstance); + uploadLastModifiedTimeApiInstance = null; } } @@ -3702,6 +3600,7 @@ class SyncEngine { } } + // Process all the changed local items in parrallel void processChangedLocalItemsToUploadInParallel(string[3][] array) { foreach (i, localItemDetails; processPool.parallel(array)) { @@ -3886,7 +3785,7 @@ class SyncEngine { } } - // Perform the upload of a locally modified file to OneDrive + // Perform the upload of a locally modified file to OneDrive JSONValue performModifiedFileUpload(Item dbItem, string localFilePath, ulong thisFileSizeLocal) { // Function variables @@ -3923,55 +3822,29 @@ class SyncEngine { // Do we use simpleUpload or create an upload session? bool useSimpleUpload = false; - // Try and get the absolute latest object details from online + // Try and get the absolute latest object details from online, so we get the latest eTag to try and avoid a 412 eTag error try { currentOnlineData = uploadFileOneDriveApiInstance.getPathDetailsById(targetDriveId, targetItemId); } catch (OneDriveException exception) { - string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to obtain latest file details from OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); - } + // Display what the error is + // - 408,429,503,504 errors are handled as a retry within uploadFileOneDriveApiInstance + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); } // Was a valid JSON response provided? if (currentOnlineData.type() == JSONType.object) { // Does the response contain an eTag? if (hasETag(currentOnlineData)) { - // Use the value returned from online + // Use the value returned from online as this will attempt to avoid a 412 response if we are creating a session upload currentETag = currentOnlineData["eTag"].str; } else { - // Use the database value + // Use the database value - greater potential for a 412 error to occur if we are creating a session upload + addLogEntry("Online data for file returned zero eTag - using database eTag value", ["debug"]); currentETag = dbItem.eTag; } } else { - // no valid JSON response + // no valid JSON response - greater potential for a 412 error to occur if we are creating a session upload + addLogEntry("Online data returned was invalid - using database eTag value", ["debug"]); currentETag = dbItem.eTag; } @@ -4029,45 +3902,23 @@ class SyncEngine { } catch (OneDriveException exception) { // Function name string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 403 if ((exception.httpStatusCode == 403) && (appConfig.getValueBool("sync_business_shared_files"))) { // We attempted to upload a file, that was shared with us, but this was shared with us as read-only addLogEntry("Unable to upload this modified file as this was shared as read-only: " ~ localFilePath); + } + // HTTP request returned status code 423 + // Resolve https://github.com/abraunegg/onedrive/issues/36 + if (exception.httpStatusCode == 423) { + // The file is currently checked out or locked for editing by another user + // We cant upload this file at this time + addLogEntry("Unable to upload this modified file as this is currently checked out or locked for editing by another user: " ~ localFilePath); } else { // Handle all other HTTP status codes - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to upload a modified file to OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); - } + // - 408,429,503,504 errors are handled as a retry within uploadFileOneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); } - } catch (FileException e) { // filesystem error displayFileSystemErrorMessage(e.msg, getFunctionName!({})); @@ -4089,37 +3940,19 @@ class SyncEngine { // We attempted to upload a file, that was shared with us, but this was shared with us as read-only addLogEntry("Unable to upload this modified file as this was shared as read-only: " ~ localFilePath); return uploadResponse; - } - - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to create an upload session on OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal); + } + // HTTP request returned status code 423 + // Resolve https://github.com/abraunegg/onedrive/issues/36 + if (exception.httpStatusCode == 423) { + // The file is currently checked out or locked for editing by another user + // We cant upload this file at this time + addLogEntry("Unable to upload this modified file as this is currently checked out or locked for editing by another user: " ~ localFilePath); + return uploadResponse; } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + // Handle all other HTTP status codes + // - 408,429,503,504 errors are handled as a retry within uploadFileOneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); } } catch (FileException e) { @@ -4133,35 +3966,12 @@ class SyncEngine { } catch (OneDriveException exception) { // Function name string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to upload a file via a session to OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - performModifiedFileUpload(dbItem, localFilePath, thisFileSizeLocal); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); - } + + // Handle all other HTTP status codes + // - 408,429,503,504 errors are handled as a retry within uploadFileOneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } catch (FileException e) { writeln("DEBUG TO REMOVE: Modified file upload FileException Handling (Perform the Upload using the session)"); displayFileSystemErrorMessage(e.msg, getFunctionName!({})); @@ -4175,10 +3985,11 @@ class SyncEngine { // Debug Log the modified upload response addLogEntry("Modified File Upload Response: " ~ to!string(uploadResponse), ["debug"]); - // Shutdown this API instance, as we will create API instances as required, when required instance - uploadFileOneDriveApiInstance.shutdown(); - // Free object and memory + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + uploadFileOneDriveApiInstance.releaseCurlEngine(); object.destroy(uploadFileOneDriveApiInstance); + uploadFileOneDriveApiInstance = null; + // Return JSON return uploadResponse; } @@ -4206,10 +4017,12 @@ class SyncEngine { getCurrentDriveQuotaApiInstance.initialise(); addLogEntry("Seeking available quota for this drive id: " ~ driveId, ["debug"]); currentDriveQuota = getCurrentDriveQuotaApiInstance.getDriveQuota(driveId); - // Shut this API instance down - getCurrentDriveQuotaApiInstance.shutdown(); - // Free object and memory + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + getCurrentDriveQuotaApiInstance.releaseCurlEngine(); object.destroy(getCurrentDriveQuotaApiInstance); + getCurrentDriveQuotaApiInstance = null; + } catch (OneDriveException e) { addLogEntry("currentDriveQuota = onedrive.getDriveQuota(driveId) generated a OneDriveException", ["debug"]); // If an exception occurs, it's unclear if quota is restricted, but quota details are not available @@ -4271,6 +4084,7 @@ class SyncEngine { processNewLocalItemsToUpload(); } + // Scan the local filesystem for new data to upload void scanLocalFilesystemPathForNewDataToUpload(string path) { // To improve logging output for this function, what is the 'logical path' we are scanning for file & folder differences? string logPath; @@ -4743,36 +4557,11 @@ class SyncEngine { parentItem = makeItem(onlinePathData); } else { string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(createDirectoryOnlineOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to create a remote directory on OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - createDirectoryOnline(thisNewPathToCreate); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - } + + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); } } } @@ -4916,7 +4705,7 @@ class SyncEngine { // but when we attempted to create it, OneDrive responded that it now already exists addLogEntry("OneDrive reported that " ~ thisNewPathToCreate ~ " already exists .. OneDrive API race condition", ["verbose"]); // Shutdown this API instance, as we will create API instances as required, when required - createDirectoryOnlineOneDriveApiInstance.shutdown(); + createDirectoryOnlineOneDriveApiInstance.releaseCurlEngine(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); return; @@ -4925,7 +4714,7 @@ class SyncEngine { addLogEntry("OneDrive generated an error when creating this path: " ~ thisNewPathToCreate); displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); // Shutdown this API instance, as we will create API instances as required, when required - createDirectoryOnlineOneDriveApiInstance.shutdown(); + createDirectoryOnlineOneDriveApiInstance.releaseCurlEngine(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); return; @@ -4941,50 +4730,25 @@ class SyncEngine { } // Shutdown this API instance, as we will create API instances as required, when required - createDirectoryOnlineOneDriveApiInstance.shutdown(); + createDirectoryOnlineOneDriveApiInstance.releaseCurlEngine(); // Free object and memory object.destroy(createDirectoryOnlineOneDriveApiInstance); return; } else { - string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(createDirectoryOnlineOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to create a remote directory on OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within createDirectoryOnlineOneDriveApiInstance + + // If we get a 400 error, there is an issue creating this folder on Microsoft OneDrive for some reason + // If the error is not 400, re-try, else fail + if (exception.httpStatusCode != 400) { + // Attempt a re-try createDirectoryOnline(thisNewPathToCreate); } else { - // If we get a 400 error, there is an issue creating this folder on Microsoft OneDrive for some reason - // If the error is not 400, re-try, else fail - if (exception.httpStatusCode != 400) { - // Attempt a re-try - createDirectoryOnline(thisNewPathToCreate); - } else { - // We cant create this directory online - addLogEntry("This folder cannot be created online: " ~ buildNormalizedPath(absolutePath(thisNewPathToCreate)), ["debug"]); - } - } + // We cant create this directory online + addLogEntry("This folder cannot be created online: " ~ buildNormalizedPath(absolutePath(thisNewPathToCreate)), ["debug"]); + } } } @@ -5010,10 +4774,12 @@ class SyncEngine { // Add this path to businessSharedFoldersOnlineToSkip businessSharedFoldersOnlineToSkip ~= [thisNewPathToCreate]; // no save to database, no online create - // Shutdown this API instance, as we will create API instances as required, when required - createDirectoryOnlineOneDriveApiInstance.shutdown(); - // Free object and memory + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + createDirectoryOnlineOneDriveApiInstance.releaseCurlEngine(); object.destroy(createDirectoryOnlineOneDriveApiInstance); + createDirectoryOnlineOneDriveApiInstance = null; + return; } else { // As the 'onlinePathData' is potentially missing the actual correct parent folder id in the 'remoteItem' JSON response, we have to perform a further query to get the correct answer @@ -5035,10 +4801,11 @@ class SyncEngine { // Is the response a valid JSON object - validation checking done in saveItem saveItem(onlinePathData); - // Shutdown this API instance, as we will create API instances as required, when required - createDirectoryOnlineOneDriveApiInstance.shutdown(); - // Free object and memory + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + createDirectoryOnlineOneDriveApiInstance.releaseCurlEngine(); object.destroy(createDirectoryOnlineOneDriveApiInstance); + createDirectoryOnlineOneDriveApiInstance = null; + return; } else { // Normally this would throw an error, however we cant use throw new posixException() @@ -5049,10 +4816,12 @@ class SyncEngine { addLogEntry("Skipping creating this directory online due to 'case-insensitive match': " ~ thisNewPathToCreate); // Add this path to posixViolationPaths posixViolationPaths ~= [thisNewPathToCreate]; - // Shutdown this API instance, as we will create API instances as required, when required - createDirectoryOnlineOneDriveApiInstance.shutdown(); - // Free object and memory + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + createDirectoryOnlineOneDriveApiInstance.releaseCurlEngine(); object.destroy(createDirectoryOnlineOneDriveApiInstance); + createDirectoryOnlineOneDriveApiInstance = null; + return; } } else { @@ -5060,10 +4829,11 @@ class SyncEngine { addLogEntry("ERROR: There was an error performing this operation on Microsoft OneDrive"); addLogEntry("ERROR: Increase logging verbosity to assist determining why."); addLogEntry("Skipping: " ~ buildNormalizedPath(absolutePath(thisNewPathToCreate))); - // Shutdown this API instance, as we will create API instances as required, when required - createDirectoryOnlineOneDriveApiInstance.shutdown(); - // Free object and memory + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + createDirectoryOnlineOneDriveApiInstance.releaseCurlEngine(); object.destroy(createDirectoryOnlineOneDriveApiInstance); + createDirectoryOnlineOneDriveApiInstance = null; + return; } } @@ -5132,6 +4902,8 @@ class SyncEngine { driveDetailsCache cachedOnlineDriveData; ulong calculatedSpaceOnlinePostUpload; + OneDriveApi checkFileOneDriveApiInstance; + // Check the database for the parent path of fileToUpload Item parentItem; // What parent path to use? @@ -5250,12 +5022,6 @@ class SyncEngine { // Do we have space available or is space available being restricted (so we make the blind assumption that there is space available) if (spaceAvailableOnline) { // We need to check that this new local file does not exist on OneDrive - - // Create a new API Instance for this thread and initialise it - OneDriveApi checkFileOneDriveApiInstance; - checkFileOneDriveApiInstance = new OneDriveApi(appConfig); - checkFileOneDriveApiInstance.initialise(); - JSONValue fileDetailsFromOneDrive; // https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file @@ -5267,6 +5033,11 @@ class SyncEngine { // Does this 'file' already exist on OneDrive? try { + + // Create a new API Instance for this thread and initialise it + checkFileOneDriveApiInstance = new OneDriveApi(appConfig); + checkFileOneDriveApiInstance.initialise(); + if (parentItem.driveId == appConfig.defaultDriveId) { // getPathDetailsByDriveId is only reliable when the driveId is our driveId fileDetailsFromOneDrive = checkFileOneDriveApiInstance.getPathDetailsByDriveId(parentItem.driveId, fileToUpload); @@ -5281,6 +5052,11 @@ class SyncEngine { } } + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + checkFileOneDriveApiInstance.releaseCurlEngine(); + object.destroy(checkFileOneDriveApiInstance); + checkFileOneDriveApiInstance = null; + // Portable Operating System Interface (POSIX) testing of JSON response from OneDrive API if (hasName(fileDetailsFromOneDrive)) { performPosixTest(baseName(fileToUpload), fileDetailsFromOneDrive["name"].str); @@ -5342,57 +5118,43 @@ class SyncEngine { } } } catch (OneDriveException exception) { + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + checkFileOneDriveApiInstance.releaseCurlEngine(); + object.destroy(checkFileOneDriveApiInstance); + checkFileOneDriveApiInstance = null; + // If we get a 404 .. the file is not online .. this is what we want .. file does not exist online if (exception.httpStatusCode == 404) { // The file has been checked, client side filtering checked, does not exist online - we need to upload it addLogEntry("fileDetailsFromOneDrive = checkFileOneDriveApiInstance.getPathDetailsByDriveId(parentItem.driveId, fileToUpload); generated a 404 - file does not exist online - must upload it", ["debug"]); uploadFailed = performNewFileUpload(parentItem, fileToUpload, thisFileSize); } else { - + // some other error string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(checkFileOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to validate file details on OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - uploadNewFile(fileToUpload); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - } + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); } } catch (posixException e) { + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + checkFileOneDriveApiInstance.releaseCurlEngine(); + object.destroy(checkFileOneDriveApiInstance); + checkFileOneDriveApiInstance = null; + + // Display POSIX error message displayPosixErrorMessage(e.msg); uploadFailed = true; } catch (jsonResponseException e) { + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + checkFileOneDriveApiInstance.releaseCurlEngine(); + object.destroy(checkFileOneDriveApiInstance); + checkFileOneDriveApiInstance = null; + + // Display JSON error message addLogEntry(e.msg, ["debug"]); uploadFailed = true; } - - // Operations in this thread are done / complete - either upload was done or it failed - checkFileOneDriveApiInstance.shutdown(); - // Free object and memory - object.destroy(checkFileOneDriveApiInstance); } else { // skip file upload - insufficent space to upload addLogEntry("Skipping uploading this new file as it exceeds the available free space on OneDrive: " ~ fileToUpload); @@ -5465,90 +5227,47 @@ class SyncEngine { if ((thisFileSize == 0) || (useSimpleUpload)) { try { - // Initialise API + // Initialise API for simple upload uploadFileOneDriveApiInstance = new OneDriveApi(appConfig); uploadFileOneDriveApiInstance.initialise(); - + // Attempt to upload the zero byte file using simpleUpload for all account types uploadResponse = uploadFileOneDriveApiInstance.simpleUpload(fileToUpload, parentItem.driveId, parentItem.id, baseName(fileToUpload)); uploadFailed = false; addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... done."); - // If the API instance is still valid, shut it down - if (uploadFileOneDriveApiInstance !is null) { - // Shutdown this API instance, as we will create API instances as required, when required - uploadFileOneDriveApiInstance.shutdown(); - // Free object and memory - object.destroy(uploadFileOneDriveApiInstance); - } + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + uploadFileOneDriveApiInstance.releaseCurlEngine(); + object.destroy(uploadFileOneDriveApiInstance); + uploadFileOneDriveApiInstance = null; + } catch (OneDriveException exception) { // An error was responded with - what was it string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to upload a new file to OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - performNewFileUpload(parentItem, fileToUpload, thisFileSize); - - // If the API instance is still valid, shut it down - if (uploadFileOneDriveApiInstance !is null) { - // Shutdown this API instance, as we will create API instances as required, when required - uploadFileOneDriveApiInstance.shutdown(); - // Free object and memory - object.destroy(uploadFileOneDriveApiInstance); - } - - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - - // If the API instance is still valid, shut it down - if (uploadFileOneDriveApiInstance !is null) { - // Shutdown this API instance, as we will create API instances as required, when required - uploadFileOneDriveApiInstance.shutdown(); - // Free object and memory - object.destroy(uploadFileOneDriveApiInstance); - } - } + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + uploadFileOneDriveApiInstance.releaseCurlEngine(); + object.destroy(uploadFileOneDriveApiInstance); + uploadFileOneDriveApiInstance = null; } catch (FileException e) { // display the error message addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); displayFileSystemErrorMessage(e.msg, getFunctionName!({})); - // If the API instance is still valid, shut it down - if (uploadFileOneDriveApiInstance !is null) { - // Shutdown this API instance, as we will create API instances as required, when required - uploadFileOneDriveApiInstance.shutdown(); - // Free object and memory - object.destroy(uploadFileOneDriveApiInstance); - } + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + uploadFileOneDriveApiInstance.releaseCurlEngine(); + object.destroy(uploadFileOneDriveApiInstance); + uploadFileOneDriveApiInstance = null; } } else { - - // Initialise API + // Initialise API for session upload uploadFileOneDriveApiInstance = new OneDriveApi(appConfig); uploadFileOneDriveApiInstance.initialise(); @@ -5568,38 +5287,12 @@ class SyncEngine { // An error was responded with - what was it string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to create an upload session on OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - performNewFileUpload(parentItem, fileToUpload, thisFileSize); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - } - + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } catch (FileException e) { // display the error message addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); @@ -5629,7 +5322,6 @@ class SyncEngine { if (sessionDataValid) { // We have a valid Upload Session Data we can use - try { // Try and perform the upload session uploadResponse = performSessionFileUpload(uploadFileOneDriveApiInstance, thisFileSize, uploadSessionData, threadUploadSessionFilePath); @@ -5644,37 +5336,12 @@ class SyncEngine { } catch (OneDriveException exception) { string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(uploadFileOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to upload a new file via a session to OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - performNewFileUpload(parentItem, fileToUpload, thisFileSize); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - } + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } } else { // No Upload URL or nextExpectedRanges or localPath .. not a valid JSON we can use @@ -5686,13 +5353,10 @@ class SyncEngine { addLogEntry("Uploading new file: " ~ fileToUpload ~ " ... failed."); } - // If the API instance is still valid, shut it down - if (uploadFileOneDriveApiInstance !is null) { - // Shutdown this API instance, as we will create API instances as required, when required - uploadFileOneDriveApiInstance.shutdown(); - // Free object and memory - object.destroy(uploadFileOneDriveApiInstance); - } + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + uploadFileOneDriveApiInstance.releaseCurlEngine(); + object.destroy(uploadFileOneDriveApiInstance); + uploadFileOneDriveApiInstance = null; } } else { // We are in a --dry-run scenario @@ -5875,39 +5539,14 @@ class SyncEngine { // 503 - Service Unavailable // 504 - Gateway Timeout - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle 'HTTP request returned status code 429 (Too Many Requests)' first - addLogEntry("Fragment upload failed - received throttle request uploadResponse from OneDrive", ["debug"]); - - if (exception.httpStatusCode == 429) { - auto retryAfterValue = activeOneDriveApiInstance.getRetryAfterValue(); - addLogEntry("Using Retry-After Value = " ~ to!string(retryAfterValue), ["debug"]); - - // Sleep thread as per request - addLogEntry(); - addLogEntry("Thread sleeping due to 'HTTP request returned status code 429' - The request has been throttled"); - addLogEntry("Sleeping for " ~ to!string(retryAfterValue) ~ " seconds"); - Thread.sleep(dur!"seconds"(retryAfterValue)); - addLogEntry("Retrying fragment upload"); - } else { - // Handle 408, 503 and 504 - auto errorArray = splitLines(exception.msg); - auto retryAfterValue = 30; - addLogEntry(); - addLogEntry("Thread sleeping due to '" ~ to!string(errorArray[0]) ~ "' - retrying applicable request in 30 seconds"); - addLogEntry("Sleeping for " ~ to!string(retryAfterValue) ~ " seconds"); - Thread.sleep(dur!"seconds"(retryAfterValue)); - addLogEntry("Retrying fragment upload"); - } - } else { - // insert a new line as well, so that the below error is inserted on the console in the right location - addLogEntry("Fragment upload failed - received an exception response from OneDrive API", ["verbose"]); - // display what the error is - displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); - // retry fragment upload in case error is transient - addLogEntry("Retrying fragment upload", ["verbose"]); - } + + // insert a new line as well, so that the below error is inserted on the console in the right location + addLogEntry("Fragment upload failed - received an exception response from OneDrive API", ["verbose"]); + // display what the error is + displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + // retry fragment upload in case error is transient + addLogEntry("Retrying fragment upload", ["verbose"]); + try { uploadResponse = activeOneDriveApiInstance.uploadFragment( @@ -5979,6 +5618,8 @@ class SyncEngine { // Delete an item on OneDrive void uploadDeletedItem(Item itemToDelete, string path) { + + OneDriveApi uploadDeletedItemOneDriveApiInstance; // Are we in a situation where we HAVE to keep the data online - do not delete the remote object if (noRemoteDelete) { @@ -6028,26 +5669,32 @@ class SyncEngine { // We are not in a dry run scenario addLogEntry("itemToDelete: " ~ to!string(itemToDelete), ["debug"]); - // Create new OneDrive API Instance - OneDriveApi uploadDeletedItemOneDriveApiInstance; - uploadDeletedItemOneDriveApiInstance = new OneDriveApi(appConfig); - uploadDeletedItemOneDriveApiInstance.initialise(); - // what item are we trying to delete? addLogEntry("Attempting to delete this single item id: " ~ itemToDelete.id ~ " from drive: " ~ itemToDelete.driveId, ["debug"]); try { - // perform the delete via the default OneDrive API instance + // Create new OneDrive API Instance + uploadDeletedItemOneDriveApiInstance = new OneDriveApi(appConfig); + uploadDeletedItemOneDriveApiInstance.initialise(); + + // Perform the delete via the default OneDrive API instance uploadDeletedItemOneDriveApiInstance.deleteById(itemToDelete.driveId, itemToDelete.id); - // Shutdown API - uploadDeletedItemOneDriveApiInstance.shutdown(); - // Free object and memory + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + uploadDeletedItemOneDriveApiInstance.releaseCurlEngine(); object.destroy(uploadDeletedItemOneDriveApiInstance); + uploadDeletedItemOneDriveApiInstance = null; + } catch (OneDriveException e) { if (e.httpStatusCode == 404) { // item.id, item.eTag could not be found on the specified driveId addLogEntry("OneDrive reported: The resource could not be found to be deleted.", ["verbose"]); } + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + uploadDeletedItemOneDriveApiInstance.releaseCurlEngine(); + object.destroy(uploadDeletedItemOneDriveApiInstance); + uploadDeletedItemOneDriveApiInstance = null; } // Delete the reference in the local database @@ -6106,10 +5753,11 @@ class SyncEngine { // Perform the delete via the default OneDrive API instance performReverseDeletionOneDriveApiInstance.deleteById(itemToDelete.driveId, itemToDelete.id, itemToDelete.eTag); - // Shutdown this API instance, as we will create API instances as required, when required - performReverseDeletionOneDriveApiInstance.shutdown(); - // Free object and memory + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + performReverseDeletionOneDriveApiInstance.releaseCurlEngine(); object.destroy(performReverseDeletionOneDriveApiInstance); + performReverseDeletionOneDriveApiInstance = null; } // Create a fake OneDrive response suitable for use with saveItem @@ -6299,7 +5947,7 @@ class SyncEngine { } // Print the fileDownloadFailures and fileUploadFailures arrays if they are not empty - void displaySyncFailures() { + void displaySyncFailures_old() { // Were there any file download failures? if (!fileDownloadFailures.empty) { @@ -6360,6 +6008,36 @@ class SyncEngine { } } + // Print the fileDownloadFailures and fileUploadFailures arrays if they are not empty + void displaySyncFailures() { + bool logFailures(string[] failures, string operation) { + if (failures.empty) return false; + + addLogEntry(); + addLogEntry("Failed items to " ~ operation ~ " to/from OneDrive: " ~ to!string(failures.length)); + + foreach (failedFile; failures) { + addLogEntry("Failed to " ~ operation ~ ": " ~ failedFile, ["info", "notify"]); + + foreach (searchDriveId; onlineDriveDetails.keys) { + Item dbItem; + if (itemDB.selectByPath(failedFile, searchDriveId, dbItem)) { + addLogEntry("ERROR: Failed " ~ operation ~ " path found in database, must delete this item from the database .. it should not be in there if the file failed to " ~ operation); + itemDB.deleteById(dbItem.driveId, dbItem.id); + if (dbItem.remoteDriveId != null) { + itemDB.deleteById(dbItem.remoteDriveId, dbItem.remoteId); + } + } + } + } + return true; + } + + bool downloadFailuresLogged = logFailures(fileDownloadFailures, "download"); + bool uploadFailuresLogged = logFailures(fileUploadFailures, "upload"); + syncFailures = downloadFailuresLogged || uploadFailuresLogged; + } + // Generate a /delta compatible response - for use when we cant actually use /delta // This is required when the application is configured to use National Azure AD deployments as these do not support /delta queries // The same technique can also be used when we are using --single-directory. The parent objects up to the single directory target can be added, @@ -6378,6 +6056,7 @@ class SyncEngine { JSONValue topLevelChildren; JSONValue[] childrenData; string nextLink; + OneDriveApi generateDeltaResponseOneDriveApiInstance; // Was a path to query passed in? if (pathToQuery.empty) { @@ -6386,10 +6065,10 @@ class SyncEngine { } // Create new OneDrive API Instance - OneDriveApi generateDeltaResponseOneDriveApiInstance; generateDeltaResponseOneDriveApiInstance = new OneDriveApi(appConfig); generateDeltaResponseOneDriveApiInstance.initialise(); + // Is this a --single-directory invocation? if (!singleDirectoryScope) { // In a --resync scenario, there is no DB data to query, so we have to query the OneDrive API here to get relevant details try { @@ -6408,10 +6087,12 @@ class SyncEngine { } catch (OneDriveException e) { // Display error message displayOneDriveErrorMessage(e.msg, getFunctionName!({})); - // Must exit here - generateDeltaResponseOneDriveApiInstance.shutdown(); - // Free object and memory + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + generateDeltaResponseOneDriveApiInstance.releaseCurlEngine(); object.destroy(generateDeltaResponseOneDriveApiInstance); + generateDeltaResponseOneDriveApiInstance = null; + // Must force exit here, allow logging to be done forceExit(); } @@ -6445,36 +6126,10 @@ class SyncEngine { addLogEntry("driveData = generateDeltaResponseOneDriveApiInstance.getPathDetailsById(searchItem.driveId, searchItem.id) generated a OneDriveException", ["debug"]); string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to query path details on OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - generateDeltaResponse(pathToQuery); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - } + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); } // Was a valid JSON response for 'driveData' provided? @@ -6498,36 +6153,11 @@ class SyncEngine { addLogEntry("rootData = onedrive.getDriveIdRoot(searchItem.driveId) generated a OneDriveException", ["debug"]); string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to query drive root details on OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Query: rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId)"); - rootData = generateDeltaResponseOneDriveApiInstance.getDriveIdRoot(searchItem.driveId); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - } + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } // Add driveData JSON data to array addLogEntry("Adding OneDrive root details for processing", ["verbose"]); @@ -6541,7 +6171,7 @@ class SyncEngine { // driveData is an invalid JSON object writeln("CODING TO DO: The query of OneDrive API to getPathDetailsById generated an invalid JSON response - thus we cant build our own /delta simulated response ... how to handle?"); // Must exit here - generateDeltaResponseOneDriveApiInstance.shutdown(); + generateDeltaResponseOneDriveApiInstance.releaseCurlEngine(); // Free object and memory object.destroy(generateDeltaResponseOneDriveApiInstance); // Must force exit here, allow logging to be done @@ -6562,36 +6192,11 @@ class SyncEngine { addLogEntry("nextLink: " ~ nextLink, ["debug"]); string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(generateDeltaResponseOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry topLevelChildren = generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink)", ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to query OneDrive top level drive children on OneDrive - retrying applicable request in 30 seconds"); - addLogEntry("generateDeltaResponseOneDriveApiInstance.listChildren(searchItem.driveId, searchItem.id, nextLink) previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - generateDeltaResponse(pathToQuery); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - } + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } // process top level children @@ -6653,10 +6258,10 @@ class SyncEngine { "value": JSONValue(childrenData.array) ]; - // Shutdown API - generateDeltaResponseOneDriveApiInstance.shutdown(); - // Free object and memory + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + generateDeltaResponseOneDriveApiInstance.releaseCurlEngine(); object.destroy(generateDeltaResponseOneDriveApiInstance); + generateDeltaResponseOneDriveApiInstance = null; // Return the generated JSON response return selfGeneratedDeltaResponse; @@ -6680,9 +6285,8 @@ class SyncEngine { try { thisLevelChildren = queryThisLevelChildren(driveId, idToQuery, nextLink, queryChildrenOneDriveApiInstance); } catch (OneDriveException exception) { - + // MAY NEED FUTURE WORK HERE .. YET TO TRIGGER THIS writeln("CODING TO DO: EXCEPTION HANDLING NEEDED: thisLevelChildren = queryThisLevelChildren(driveId, idToQuery, nextLink, queryChildrenOneDriveApiInstance)"); - } if (appConfig.verbosityCount == 0) { @@ -6748,10 +6352,10 @@ class SyncEngine { } } - // Shutdown this API instance, as we will create API instances as required, when required - queryChildrenOneDriveApiInstance.shutdown(); - // Free object and memory + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + queryChildrenOneDriveApiInstance.releaseCurlEngine(); object.destroy(queryChildrenOneDriveApiInstance); + queryChildrenOneDriveApiInstance = null; // return response return thisLevelChildrenData; @@ -6778,36 +6382,11 @@ class SyncEngine { addLogEntry("nextLink: " ~ nextLink, ["debug"]); string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(queryChildrenOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to query OneDrive drive item children - retrying applicable request in 30 seconds"); - addLogEntry("thisLevelChildren = queryChildrenOneDriveApiInstance.listChildren(driveId, idToQuery, nextLink) previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - queryThisLevelChildren(driveId, idToQuery, nextLink, queryChildrenOneDriveApiInstance); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - } + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } // return response @@ -6864,36 +6443,11 @@ class SyncEngine { } catch (OneDriveException exception) { string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(queryOneDriveForSpecificPath); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to query path on OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - queryOneDriveForSpecificPathAndCreateIfMissing(thisNewPathToSearch, createPathIfMissing); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - } + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } } else { // Ensure we have a valid driveId to search here @@ -6948,36 +6502,11 @@ class SyncEngine { } else { string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(queryOneDriveForSpecificPath); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry " ~ thisFunctionName, ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to query path on OneDrive - retrying applicable request in 30 seconds"); - addLogEntry(thisFunctionName ~ " previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - queryOneDriveForSpecificPathAndCreateIfMissing(thisNewPathToSearch, createPathIfMissing); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - } + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); + } } catch (jsonResponseException e) { addLogEntry(e.msg, ["debug"]); @@ -7089,10 +6618,10 @@ class SyncEngine { } } - // Shutdown this API instance, as we will create API instances as required, when required - queryOneDriveForSpecificPath.shutdown(); - // Free object and memory + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + queryOneDriveForSpecificPath.releaseCurlEngine(); object.destroy(queryOneDriveForSpecificPath); + queryOneDriveForSpecificPath = null; // Output our search results addLogEntry("queryOneDriveForSpecificPathAndCreateIfMissing.getPathDetailsAPIResponse = " ~ to!string(getPathDetailsAPIResponse), ["debug"]); @@ -7275,10 +6804,11 @@ class SyncEngine { // Try the online move for (int i = 0; i < 3; i++) { try { - response = movePathOnlineApiInstance.updateById(oldItem.driveId, oldItem.id, data, oldItem.eTag); + response = movePathOnlineApiInstance.updateById(oldItem.driveId, oldItem.id, data, eTag); isMoveSuccess = true; break; } catch (OneDriveException e) { + // Handle a 412 - A precondition provided in the request (such as an if-match header) does not match the resource's current state. if (e.httpStatusCode == 412) { // OneDrive threw a 412 error, most likely: ETag does not match current item's value // Retry without eTag @@ -7287,7 +6817,7 @@ class SyncEngine { eTag = null; // Retry to move the file but without the eTag, via the for() loop } else if (e.httpStatusCode == 409) { - // Destination item already exists, delete it first + // Destination item already exists and is a conflict, delete it first addLogEntry("Moved local item overwrote an existing item - deleting old online item"); uploadDeletedItem(newItem, newPath); } else @@ -7295,10 +6825,10 @@ class SyncEngine { } } - // Shutdown this API instance, as we will create API instances as required, when required - movePathOnlineApiInstance.shutdown(); - // Free object and memory + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + movePathOnlineApiInstance.releaseCurlEngine(); object.destroy(movePathOnlineApiInstance); + movePathOnlineApiInstance = null; // save the move response from OneDrive in the database // Is the response a valid JSON object - validation checking done in saveItem @@ -7408,8 +6938,15 @@ class SyncEngine { // Forbidden - most likely authentication scope needs to be updated if (e.httpStatusCode == 403) { addLogEntry("ERROR: Authentication scope needs to be updated. Use --reauth and re-authenticate client."); + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + querySharePointLibraryNameApiInstance.releaseCurlEngine(); + object.destroy(querySharePointLibraryNameApiInstance); + querySharePointLibraryNameApiInstance = null; + return; } + // Requested resource cannot be found if (e.httpStatusCode == 404) { string siteSearchUrl; @@ -7421,42 +6958,26 @@ class SyncEngine { // log the error addLogEntry("ERROR: Your OneDrive Account and Authentication Scope cannot access this OneDrive API: " ~ siteSearchUrl); addLogEntry("ERROR: To resolve, please discuss this issue with whomever supports your OneDrive and SharePoint environment."); + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + querySharePointLibraryNameApiInstance.releaseCurlEngine(); + object.destroy(querySharePointLibraryNameApiInstance); + querySharePointLibraryNameApiInstance = null; + return; } - // HTTP request returned status code 429 (Too Many Requests) - if (e.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(querySharePointLibraryNameApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to query OneDrive drive children", ["debug"]); - } - // HTTP request returned status code 504 (Gateway Timeout) or 429 retry - if ((e.httpStatusCode == 429) || (e.httpStatusCode == 504)) { - // re-try the specific changes queries - if (e.httpStatusCode == 504) { - addLogEntry("OneDrive returned a 'HTTP 504 - Gateway Timeout' when attempting to query Sharepoint Sites - retrying applicable request"); - addLogEntry("siteQuery = onedrive.o365SiteSearch(nextLink) previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429 and 504 - try { - addLogEntry("Retrying Query: siteQuery = onedrive.o365SiteSearch(nextLink)", ["debug"]); - siteQuery = querySharePointLibraryNameApiInstance.o365SiteSearch(nextLink); - addLogEntry("Query 'siteQuery = onedrive.o365SiteSearch(nextLink)' performed successfully on re-try", ["debug"]); - } catch (OneDriveException e) { - // display what the error is - addLogEntry("Query Error: siteQuery = onedrive.o365SiteSearch(nextLink) on re-try after delay", ["debug"]); - // error was not a 504 this time - displayOneDriveErrorMessage(e.msg, getFunctionName!({})); - return; - } - } else { - // display what the error is - displayOneDriveErrorMessage(e.msg, getFunctionName!({})); - return; - } + + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + querySharePointLibraryNameApiInstance.releaseCurlEngine(); + object.destroy(querySharePointLibraryNameApiInstance); + querySharePointLibraryNameApiInstance = null; + + return; } // is siteQuery a valid JSON object & contain data we can use? @@ -7481,6 +7002,12 @@ class SyncEngine { addLogEntry("ERROR: Query of OneDrive for Office Site ID failed"); // display what the error is displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + querySharePointLibraryNameApiInstance.releaseCurlEngine(); + object.destroy(querySharePointLibraryNameApiInstance); + querySharePointLibraryNameApiInstance = null; + return; } @@ -7503,6 +7030,12 @@ class SyncEngine { // not a valid JSON object addLogEntry("ERROR: There was an error performing this operation on Microsoft OneDrive"); addLogEntry("ERROR: Increase logging verbosity to assist determining why."); + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + querySharePointLibraryNameApiInstance.releaseCurlEngine(); + object.destroy(querySharePointLibraryNameApiInstance); + querySharePointLibraryNameApiInstance = null; + return; } } @@ -7552,6 +7085,12 @@ class SyncEngine { // not a valid JSON object addLogEntry("ERROR: There was an error performing this operation on Microsoft OneDrive"); addLogEntry("ERROR: Increase logging verbosity to assist determining why."); + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + querySharePointLibraryNameApiInstance.releaseCurlEngine(); + object.destroy(querySharePointLibraryNameApiInstance); + querySharePointLibraryNameApiInstance = null; + return; } @@ -7566,7 +7105,6 @@ class SyncEngine { // Was the intended target found? if(!found) { - // Was the search a wildcard? if (sharepointLibraryNameToQuery != "*") { // Only print this out if the search was not a wildcard @@ -7582,10 +7120,10 @@ class SyncEngine { } } - // Shutdown this API instance, as we will create API instances as required, when required - querySharePointLibraryNameApiInstance.shutdown(); - // Free object and memory + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + querySharePointLibraryNameApiInstance.releaseCurlEngine(); object.destroy(querySharePointLibraryNameApiInstance); + querySharePointLibraryNameApiInstance = null; } // Query the sync status of the client and the local system @@ -7693,6 +7231,12 @@ class SyncEngine { } else break; } + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + getDeltaQueryOneDriveApiInstance.releaseCurlEngine(); + object.destroy(getDeltaQueryOneDriveApiInstance); + getDeltaQueryOneDriveApiInstance = null; + // Needed after printing out '....' when fetching changes from OneDrive API if (appConfig.verbosityCount == 0) addLogEntry("\n", ["consoleOnlyNoNewLine"]); @@ -7761,6 +7305,8 @@ class SyncEngine { // Query OneDrive for file details of a given path, returning either the 'webURL' or 'lastModifiedBy' JSON facet void queryOneDriveForFileDetails(string inputFilePath, string runtimePath, string outputType) { + OneDriveApi queryOneDriveForFileDetailsApiInstance; + // Calculate the full local file path string fullLocalFilePath = buildNormalizedPath(buildPath(runtimePath, inputFilePath)); @@ -7779,15 +7325,23 @@ class SyncEngine { JSONValue fileDetailsFromOneDrive; // Create a new API Instance for this thread and initialise it - OneDriveApi queryOneDriveForFileDetailsApiInstance; queryOneDriveForFileDetailsApiInstance = new OneDriveApi(appConfig); queryOneDriveForFileDetailsApiInstance.initialise(); try { fileDetailsFromOneDrive = queryOneDriveForFileDetailsApiInstance.getPathDetailsById(dbItem.driveId, dbItem.id); + + // Dont cleanup here as if we are creating a shareable file link (below) it is still needed + } catch (OneDriveException exception) { // display what the error is displayOneDriveErrorMessage(exception.msg, getFunctionName!({})); + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + queryOneDriveForFileDetailsApiInstance.releaseCurlEngine(); + object.destroy(queryOneDriveForFileDetailsApiInstance); + queryOneDriveForFileDetailsApiInstance = null; + return; } @@ -7867,10 +7421,10 @@ class SyncEngine { } } - // Shutdown this API instance, as we will create API instances as required, when required access - queryOneDriveForFileDetailsApiInstance.shutdown(); - // Free object and memory + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + queryOneDriveForFileDetailsApiInstance.releaseCurlEngine(); object.destroy(queryOneDriveForFileDetailsApiInstance); + queryOneDriveForFileDetailsApiInstance = null; } } @@ -7890,6 +7444,7 @@ class SyncEngine { // This function is similar to getRemainingFreeSpace() but is different in data being analysed and output method JSONValue currentDriveQuota; string driveId; + OneDriveApi getCurrentDriveQuotaApiInstance; if (appConfig.getValueString("drive_id").length) { driveId = appConfig.getValueString("drive_id"); @@ -7899,17 +7454,22 @@ class SyncEngine { try { // Create a new OneDrive API instance - OneDriveApi getCurrentDriveQuotaApiInstance; getCurrentDriveQuotaApiInstance = new OneDriveApi(appConfig); getCurrentDriveQuotaApiInstance.initialise(); addLogEntry("Seeking available quota for this drive id: " ~ driveId, ["debug"]); currentDriveQuota = getCurrentDriveQuotaApiInstance.getDriveQuota(driveId); - // Shut this API instance down - getCurrentDriveQuotaApiInstance.shutdown(); - // Free object and memory + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + getCurrentDriveQuotaApiInstance.releaseCurlEngine(); object.destroy(getCurrentDriveQuotaApiInstance); + getCurrentDriveQuotaApiInstance = null; + } catch (OneDriveException e) { addLogEntry("currentDriveQuota = onedrive.getDriveQuota(driveId) generated a OneDriveException", ["debug"]); + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + getCurrentDriveQuotaApiInstance.releaseCurlEngine(); + object.destroy(getCurrentDriveQuotaApiInstance); + getCurrentDriveQuotaApiInstance = null; } // validate that currentDriveQuota is a JSON value @@ -7959,7 +7519,6 @@ class SyncEngine { } else { writeln("Microsoft OneDrive quota information is being restricted for this Drive ID: ", driveId); } - } } @@ -8025,7 +7584,6 @@ class SyncEngine { // At this point we should have an array of JSON items to resume uploading if (count(jsonItemsToResumeUpload) > 0) { // there are valid items to resume upload - // Lets deal with all the JSON items that need to be reumed for upload in a batch process size_t batchSize = to!int(appConfig.getValueLong("threads")); ulong batchCount = (jsonItemsToResumeUpload.length + batchSize - 1) / batchSize; @@ -8038,9 +7596,12 @@ class SyncEngine { } } + // A resume session upload file need to be valid to be used + // This function validates this data bool validateUploadSessionFileData(string sessionFilePath) { JSONValue sessionFileData; + OneDriveApi validateUploadSessionFileDataApiInstance; // Try and read the text from the session file as a JSON array try { @@ -8088,24 +7649,31 @@ class SyncEngine { if ("uploadUrl" in sessionFileData) { JSONValue response; - // Create a new OneDrive API instance - OneDriveApi validateUploadSessionFileDataApiInstance; - validateUploadSessionFileDataApiInstance = new OneDriveApi(appConfig); - validateUploadSessionFileDataApiInstance.initialise(); - try { + // Create a new OneDrive API instance + validateUploadSessionFileDataApiInstance = new OneDriveApi(appConfig); + validateUploadSessionFileDataApiInstance.initialise(); + + // Request upload status response = validateUploadSessionFileDataApiInstance.requestUploadStatus(sessionFileData["uploadUrl"].str); + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + validateUploadSessionFileDataApiInstance.releaseCurlEngine(); + object.destroy(validateUploadSessionFileDataApiInstance); + validateUploadSessionFileDataApiInstance = null; + } catch (OneDriveException e) { // handle any onedrive error response as invalid addLogEntry("SESSION-RESUME: Invalid response when using uploadUrl in: " ~ sessionFilePath, ["debug"]); + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + validateUploadSessionFileDataApiInstance.releaseCurlEngine(); + object.destroy(validateUploadSessionFileDataApiInstance); + validateUploadSessionFileDataApiInstance = null; + return false; } - // Shutdown this API instance, as we will create API instances as required, when required - validateUploadSessionFileDataApiInstance.shutdown(); - // Free object and memory - object.destroy(validateUploadSessionFileDataApiInstance); - // Do we have a valid response from OneDrive? if (response.type() == JSONType.object) { // Valid JSON object was returned @@ -8140,6 +7708,7 @@ class SyncEngine { return true; } + // Resume all resumable session uploads in parrallel void resumeSessionUploadsInParallel(JSONValue[] array) { // This function recieved an array of 16 JSON items to resume upload foreach (i, jsonItemToResume; processPool.parallel(array)) { @@ -8147,6 +7716,8 @@ class SyncEngine { JSONValue uploadResponse; OneDriveApi uploadFileOneDriveApiInstance; + + // Create a new API instance uploadFileOneDriveApiInstance = new OneDriveApi(appConfig); uploadFileOneDriveApiInstance.initialise(); @@ -8161,6 +7732,11 @@ class SyncEngine { writeln("CODING TO DO: Handle an exception when performing a resume session upload"); } + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + uploadFileOneDriveApiInstance.releaseCurlEngine(); + object.destroy(uploadFileOneDriveApiInstance); + uploadFileOneDriveApiInstance = null; + // Was the response from the OneDrive API a valid JSON item? if (uploadResponse.type() == JSONType.object) { // A valid JSON object was returned - session resumption upload sucessful @@ -8190,11 +7766,6 @@ class SyncEngine { // No valid response was returned addLogEntry("CODING TO DO: what to do when session upload resumption JSON data is not valid ... nothing ? error message ?"); } - - // Shutdown this API instance, as we will create API instances as required, when required - uploadFileOneDriveApiInstance.shutdown(); - // Free object and memory - object.destroy(uploadFileOneDriveApiInstance); } } @@ -8239,7 +7810,6 @@ class SyncEngine { addLogEntry("onlineDriveDetails: " ~ to!string(onlineDriveDetails), ["debug"]); } - // Return a specific 'driveId' details from 'onlineDriveDetails' driveDetailsCache getDriveDetails(string driveId) { auto ptr = driveId in onlineDriveDetails; @@ -8278,36 +7848,10 @@ class SyncEngine { addLogEntry("nextLink: " ~ nextLink, ["debug"]); string thisFunctionName = getFunctionName!({}); - // HTTP request returned status code 408,429,503,504 - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 429) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // Handle the 429 - if (exception.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests). We need to leverage the response Retry-After HTTP header to ensure minimum delay until the throttle is removed. - handleOneDriveThrottleRequest(checkFileOneDriveApiInstance); - addLogEntry("Retrying original request that generated the OneDrive HTTP 429 Response Code (Too Many Requests) - attempting to retry thisLevelChildren = checkFileOneDriveApiInstance.listChildren(parentItemDriveId, parentItemId, nextLink)", ["debug"]); - } - // re-try the specific changes queries - if ((exception.httpStatusCode == 408) || (exception.httpStatusCode == 503) || (exception.httpStatusCode == 504)) { - // 408 - Request Time Out - // 503 - Service Unavailable - // 504 - Gateway Timeout - // Transient error - try again in 30 seconds - auto errorArray = splitLines(exception.msg); - addLogEntry(to!string(errorArray[0]) ~ " when attempting to query OneDrive top level drive children on OneDrive - retrying applicable request in 30 seconds"); - addLogEntry("checkFileOneDriveApiInstance.listChildren(parentItemDriveId, parentItemId, nextLink) previously threw an error - retrying", ["debug"]); - - // The server, while acting as a proxy, did not receive a timely response from the upstream server it needed to access in attempting to complete the request. - addLogEntry("Thread sleeping for 30 seconds as the server did not receive a timely response from the upstream server it needed to access in attempting to complete the request", ["debug"]); - Thread.sleep(dur!"seconds"(30)); - } - // re-try original request - retried for 429, 503, 504 - but loop back calling this function - addLogEntry("Retrying Function: " ~ thisFunctionName, ["debug"]); - searchDriveItemForFile(parentItemDriveId, parentItemId, fileToUpload); - } else { - // Default operation if not 408,429,503,504 errors - // display what the error is - displayOneDriveErrorMessage(exception.msg, thisFunctionName); - } + // Default operation if not 408,429,503,504 errors + // - 408,429,503,504 errors are handled as a retry within oneDriveApiInstance + // Display what the error is + displayOneDriveErrorMessage(exception.msg, thisFunctionName); } // process thisLevelChildren response @@ -8316,9 +7860,12 @@ class SyncEngine { if ((child["name"].str == searchName) && (("file" in child) != null)) { // Found the matching file, return its JSON representation // Operations in this thread are done / complete - checkFileOneDriveApiInstance.shutdown(); - // Free object and memory + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + checkFileOneDriveApiInstance.releaseCurlEngine(); object.destroy(checkFileOneDriveApiInstance); + checkFileOneDriveApiInstance = null; + // Return child return child; } @@ -8333,10 +7880,11 @@ class SyncEngine { } else break; } - // Operations in this thread are done / complete - checkFileOneDriveApiInstance.shutdown(); - // Free object and memory + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + checkFileOneDriveApiInstance.releaseCurlEngine(); object.destroy(checkFileOneDriveApiInstance); + checkFileOneDriveApiInstance = null; + // return an empty JSON item return onedriveJSONItem; } @@ -8461,14 +8009,22 @@ class SyncEngine { try { sharedWithMeItems = sharedWithMeOneDriveApiInstance.getSharedWithMe(); - } catch (OneDriveException e) { + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + sharedWithMeOneDriveApiInstance.releaseCurlEngine(); + object.destroy(sharedWithMeOneDriveApiInstance); + sharedWithMeOneDriveApiInstance = null; + + } catch (OneDriveException e) { // Display error message displayOneDriveErrorMessage(e.msg, getFunctionName!({})); - // Must exit here - sharedWithMeOneDriveApiInstance.shutdown(); - // Free object and memory + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + sharedWithMeOneDriveApiInstance.releaseCurlEngine(); object.destroy(sharedWithMeOneDriveApiInstance); + sharedWithMeOneDriveApiInstance = null; + + return; } if (sharedWithMeItems.type() == JSONType.object) { @@ -8536,11 +8092,6 @@ class SyncEngine { addLogEntry(); } } - - // Shutdown API access - sharedWithMeOneDriveApiInstance.shutdown(); - // Free object and memory - object.destroy(sharedWithMeOneDriveApiInstance); } // Query all the OneDrive Business Shared Objects to sync only Shared Files @@ -8556,10 +8107,19 @@ class SyncEngine { try { sharedWithMeItems = sharedWithMeOneDriveApiInstance.getSharedWithMe(); + + // We cant shutdown the API instance here, as we re-use it below + } catch (OneDriveException e) { + // Display error message + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); - // Add eventual API error handling here + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + sharedWithMeOneDriveApiInstance.releaseCurlEngine(); + object.destroy(sharedWithMeOneDriveApiInstance); + sharedWithMeOneDriveApiInstance = null; + return; } // Valid JSON response @@ -8759,5 +8319,10 @@ class SyncEngine { } } } + + // OneDrive API Instance Cleanup - Shutdown API, free curl object and memory + sharedWithMeOneDriveApiInstance.releaseCurlEngine(); + object.destroy(sharedWithMeOneDriveApiInstance); + sharedWithMeOneDriveApiInstance = null; } } \ No newline at end of file diff --git a/src/util.d b/src/util.d index 01720fe04..650b2a1a5 100644 --- a/src/util.d +++ b/src/util.d @@ -42,6 +42,7 @@ import curlEngine; // module variables shared string deviceName; +ulong previousRSS; static this() { deviceName = Socket.hostName; @@ -201,42 +202,58 @@ Regex!char wild2regex(const(char)[] pattern) { return regex(str, "i"); } -// Test Internet access to Microsoft OneDrive +// Test Internet access to Microsoft OneDrive using a simple HTTP HEAD request bool testInternetReachability(ApplicationConfig appConfig) { - CurlEngine curlEngine; - bool result = false; + auto http = HTTP(); + http.url = "https://login.microsoftonline.com"; + + // Configure timeouts based on application configuration + http.dnsTimeout = dur!"seconds"(appConfig.getValueLong("dns_timeout")); + http.connectTimeout = dur!"seconds"(appConfig.getValueLong("connect_timeout")); + http.dataTimeout = dur!"seconds"(appConfig.getValueLong("data_timeout")); + http.operationTimeout = dur!"seconds"(appConfig.getValueLong("operation_timeout")); + + // Set IP protocol version + http.handle.set(CurlOption.ipresolve, appConfig.getValueLong("ip_protocol_version")); + + // Set HTTP method to HEAD for minimal data transfer + http.method = HTTP.Method.head; + + // Execute the request and handle exceptions try { - // Use preconfigured object with all the correct http values assigned - curlEngine = CurlEngine.get(); - curlEngine.initialise(appConfig.getValueLong("dns_timeout"), appConfig.getValueLong("connect_timeout"), appConfig.getValueLong("data_timeout"), appConfig.getValueLong("operation_timeout"), appConfig.defaultMaxRedirects, appConfig.getValueBool("debug_https"), appConfig.getValueString("user_agent"), appConfig.getValueBool("force_http_11"), appConfig.getValueLong("rate_limit"), appConfig.getValueLong("ip_protocol_version")); - - // Configure the remaining items required - // URL to use - // HTTP connection test method - - curlEngine.connect(HTTP.Method.head, "https://login.microsoftonline.com"); - addLogEntry("Attempting to contact Microsoft OneDrive Login Service", ["debug"]); - curlEngine.http.perform(); - addLogEntry("Shutting down HTTP engine as successfully reached OneDrive Login Service", ["debug"]); - result = true; - } catch (SocketException e) { - addLogEntry("HTTP Socket Issue", ["debug"]); - addLogEntry("Cannot connect to Microsoft OneDrive Login Service - Socket Issue"); - displayOneDriveErrorMessage(e.msg, getFunctionName!({})); - } catch (CurlException e) { - addLogEntry("No Network Connection", ["debug"]); - addLogEntry("Cannot connect to Microsoft OneDrive Login Service - Network Connection Issue"); - displayOneDriveErrorMessage(e.msg, getFunctionName!({})); - } + addLogEntry("Attempting to contact Microsoft OneDrive Login Service"); + http.perform(); - // Shutdown engine - curlEngine.http.shutdown(); - curlEngine.releaseAll(); - object.destroy(curlEngine); - curlEngine = null; + // Check response for HTTP status code + if (http.statusLine.code >= 200 && http.statusLine.code < 400) { + addLogEntry("Successfully reached Microsoft OneDrive Login Service"); + } else { + addLogEntry("Failed to reach Microsoft OneDrive Login Service. HTTP status code: " ~ to!string(http.statusLine.code)); + throw new Exception("HTTP Request Failed with Status Code: " ~ to!string(http.statusLine.code)); + } - // Return test result - return result; + http.shutdown(); + object.destroy(http); + return true; + } catch (SocketException e) { + addLogEntry("Cannot connect to Microsoft OneDrive Service - Socket Issue: " ~ e.msg); + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + http.shutdown(); + object.destroy(http); + return false; + } catch (CurlException e) { + addLogEntry("Cannot connect to Microsoft OneDrive Service - Network Connection Issue: " ~ e.msg); + displayOneDriveErrorMessage(e.msg, getFunctionName!({})); + http.shutdown(); + object.destroy(http); + return false; + } catch (Exception e) { + addLogEntry("Unexpected error occurred: " ~ e.toString()); + displayOneDriveErrorMessage(e.toString(), getFunctionName!({})); + http.shutdown(); + object.destroy(http); + return false; + } } // Retry Internet access test to Microsoft OneDrive @@ -489,7 +506,6 @@ bool isValidUTF16(string path) { return true; } - // Does the path contain any HTML URL encoded items (e.g., '%20' for space) bool containsURLEncodedItems(string path) { // Check for null or empty string @@ -636,7 +652,7 @@ void displayFileSystemErrorMessage(string message, string callingFunction) { addLogEntry(" Error Message: " ~ errorMessage); // Log the calling function - addLogEntry(" Calling Function: " ~ callingFunction, ["verbose"]); + addLogEntry(" Calling Function: " ~ callingFunction); try { // Safely check for disk space @@ -659,32 +675,75 @@ void displayPosixErrorMessage(string message) { addLogEntry(" Error Message: " ~ message); } +// Display the Error Message +void displayGeneralErrorMessage(Exception e, string callingFunction=__FUNCTION__, int lineno=__LINE__) { + addLogEntry(); // used rather than writeln + addLogEntry("ERROR: Encounter " ~ e.classinfo.name ~ ":"); + addLogEntry(" Error Message: " ~ e.msg); + addLogEntry(" Calling Function: " ~ callingFunction); + addLogEntry(" Line number: " ~ to!string(lineno)); +} + // Get the function name that is being called to assist with identifying where an error is being generated string getFunctionName(alias func)() { return __traits(identifier, __traits(parent, func)) ~ "()\n"; } +JSONValue fetchOnlineURLContent(string url) { + // Function variables + char[] content; + JSONValue onlineContent; + + // Setup HTTP request + HTTP http = HTTP(); + + // Create an HTTP object within a scope to ensure cleanup + scope(exit) { + http.shutdown(); + object.destroy(http); + } + + // Configure the URL to access + http.url = url; + // HTTP the connection method + http.method = HTTP.Method.get; + + // Data receive handler + http.onReceive = (ubyte[] data) { + content ~= data; // Append data as it's received + return data.length; + }; + + // Perform HTTP request + http.perform(); + + // Parse Content + onlineContent = parseJSON(to!string(content)); + + // Ensure resources are cleaned up + http.shutdown(); + object.destroy(http); + + // Return onlineResponse + return onlineContent; +} + // Get the latest release version from GitHub JSONValue getLatestReleaseDetails() { - // Import curl just for this function - import std.net.curl; - char[] content; JSONValue githubLatest; JSONValue versionDetails; string latestTag; string publishedDate; // Query GitHub for the 'latest' release details - try { - content = get("https://api.github.com/repos/abraunegg/onedrive/releases/latest"); - githubLatest = content.parseJSON(); + try { + githubLatest = fetchOnlineURLContent("https://api.github.com/repos/abraunegg/onedrive/releases/latest"); } catch (CurlException e) { addLogEntry("CurlException: Unable to query GitHub for latest release - " ~ e.msg, ["debug"]); } catch (JSONException e) { addLogEntry("JSONException: Unable to parse GitHub JSON response - " ~ e.msg, ["debug"]); } - // githubLatest has to be a valid JSON object if (githubLatest.type() == JSONType.object){ // use the returned tag_name @@ -726,9 +785,6 @@ JSONValue getLatestReleaseDetails() { // Get the release details from the 'current' running version JSONValue getCurrentVersionDetails(string thisVersion) { - // Import curl just for this function - import std.net.curl; - char[] content; JSONValue githubDetails; JSONValue versionDetails; string versionTag = "v" ~ thisVersion; @@ -736,9 +792,8 @@ JSONValue getCurrentVersionDetails(string thisVersion) { // Query GitHub for the release details to match the running version try { - content = get("https://api.github.com/repos/abraunegg/onedrive/releases"); - githubDetails = content.parseJSON(); - } catch (CurlException e) { + githubDetails = fetchOnlineURLContent("https://api.github.com/repos/abraunegg/onedrive/releases"); + } catch (CurlException e) { addLogEntry("CurlException: Unable to query GitHub for release details - " ~ e.msg, ["debug"]); return parseJSON(`{"Error": "CurlException", "message": "` ~ e.msg ~ `"}`); } catch (JSONException e) { @@ -1023,29 +1078,51 @@ string generateAlphanumericString(size_t length = 16) { return to!string(randomString); } +// Display internal memory stats pre garbage collection void displayMemoryUsagePreGC() { // Display memory usage - writeln(); - writeln("Memory Usage pre GC (KB)"); - writeln("------------------------"); + addLogEntry(); + addLogEntry("Memory Usage PRE Garbage Collection (KB)"); + addLogEntry("-----------------------------------------------------"); writeMemoryStats(); - writeln(); + addLogEntry(); } +// Display internal memory stats post garbage collection + RSS (actual memory being used) void displayMemoryUsagePostGC() { // Display memory usage - writeln(); - writeln("Memory Usage post GC (KB)"); - writeln("-------------------------"); + addLogEntry("Memory Usage POST Garbage Collection (KB)"); + addLogEntry("-----------------------------------------------------"); writeMemoryStats(); - writeln(); + + // Query the actual Resident Set Size (RSS) for the PID + pid_t pid = getCurrentPID(); + ulong rss = getRSS(pid); + addLogEntry("current Resident Set Size (RSS) = " ~ to!string(rss)); // actual memory in RAM used by the process - this needs to remain stable, already in KB + + // Is there a previous value + if (previousRSS != 0) { + addLogEntry("previous Resident Set Size (RSS) = " ~ to!string(previousRSS)); // actual memory in RAM used by the process - this needs to remain stable, already in KB + // Increase or decrease in RSS + if (rss > previousRSS) { + addLogEntry("difference in Resident Set Size (RSS) = +" ~ to!string((rss - previousRSS))); // Difference in actual memory used + } else { + addLogEntry("difference in Resident Set Size (RSS) = -" ~ to!string((previousRSS - rss))); // Difference in actual memory used + } + } + + // Update previous RSS with new value + previousRSS = rss; + + // Closout + addLogEntry(); } +// Write internal memory stats void writeMemoryStats() { - // write memory stats - writeln("memory usedSize = ", (GC.stats.usedSize/1024)); - writeln("memory freeSize = ", (GC.stats.freeSize/1024)); - writeln("memory allocatedInCurrentThread = ", (GC.stats.allocatedInCurrentThread/1024)); + addLogEntry("current memory usedSize = " ~ to!string((GC.stats.usedSize/1024))); // number of used bytes on the GC heap (might only get updated after a collection) + addLogEntry("current memory freeSize = " ~ to!string((GC.stats.freeSize/1024))); // number of free bytes on the GC heap (might only get updated after a collection) + addLogEntry("current memory allocatedInCurrentThread = " ~ to!string((GC.stats.allocatedInCurrentThread/1024))); // number of bytes allocated for current thread since program start } // Return the username of the UID running the 'onedrive' process @@ -1116,9 +1193,56 @@ int calc_eta(size_t counter, size_t iterations, ulong start_time) { } } +// Force Exit void forceExit() { // Allow logging to flush and complete Thread.sleep(dur!("msecs")(500)); // Force Exit exit(EXIT_FAILURE); -} \ No newline at end of file +} + +// Get the current PID of the application +pid_t getCurrentPID() { + // The '/proc/self' is a symlink to the current process's proc directory + string path = "/proc/self/stat"; + + // Read the content of the stat file + string content; + try { + content = readText(path); + } catch (Exception e) { + writeln("Failed to read stat file: ", e.msg); + return 0; + } + + // The first value in the stat file is the PID + auto parts = split(content); + return to!pid_t(parts[0]); // Convert the first part to pid_t +} + +// Access the Resident Set Size (RSS) based on the PID of the running application +ulong getRSS(pid_t pid) { + // Construct the path to the statm file for the given PID + string path = format("/proc/%s/statm", to!string(pid)); + + // Read the content of the file + string content; + try { + content = readText(path); + } catch (Exception e) { + writeln("Failed to read statm file: ", e.msg); + return 0; + } + + // Split the content and get the RSS (second value) + auto stats = split(content); + if (stats.length < 2) { + writeln("Unexpected format in statm file."); + return 0; + } + + // RSS is in pages, convert it to kilobytes + ulong rssPages = to!ulong(stats[1]); + ulong rssKilobytes = rssPages * sysconf(_SC_PAGESIZE) / 1024; + return rssKilobytes; +} diff --git a/src/webhook.d b/src/webhook.d index 065e6fb64..cb39eeda5 100644 --- a/src/webhook.d +++ b/src/webhook.d @@ -78,8 +78,10 @@ class OneDriveWebhook { } catch (OneDriveException e) { logSubscriptionError(e); } - oneDriveApiInstance.shutdown(); - object.destroy(oneDriveApiInstance); + // Release API instance back to the pool + oneDriveApiInstance.releaseCurlEngine(); + object.destroy(oneDriveApiInstance); + oneDriveApiInstance = null; } private static void handle(shared OneDriveWebhook _this, Cgi cgi) {