From f06e8160d028ee7f24dff67ad4c2e411a0f204f8 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Wed, 18 Sep 2024 12:58:16 +1000 Subject: [PATCH] Fix issue #2810 & #2813 * Add isValidUTCDateTime function to validate timestamp as received from OneDrive API to ensure it is valid * Use new function before attempting to call SysTime.fromISOExtString to ensure this call will be successful --- src/itemdb.d | 39 +++++++++++++++++++++++++++---- src/sync.d | 66 ++++++++++++++++++++++++++++++++++++++++++++++------ src/util.d | 20 ++++++++++++++++ 3 files changed, 114 insertions(+), 11 deletions(-) diff --git a/src/itemdb.d b/src/itemdb.d index 8b91b70af..6fb2d771e 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -63,6 +63,7 @@ Item makeDatabaseItem(JSONValue driveItem) { item.mtime = SysTime(0); } else { // Item is not in a deleted state + string lastModifiedTimestamp; // Resolve 'Key not found: fileSystemInfo' when then item is a remote item // https://github.com/abraunegg/onedrive/issues/11 if (isItemRemote(driveItem)) { @@ -72,17 +73,47 @@ Item makeDatabaseItem(JSONValue driveItem) { // See: https://github.com/abraunegg/onedrive/issues/1533 if ("fileSystemInfo" in driveItem["remoteItem"]) { // 'fileSystemInfo' is in 'remoteItem' which will be the majority of cases - item.mtime = SysTime.fromISOExtString(driveItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str); + lastModifiedTimestamp = strip(driveItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str); + // is lastModifiedTimestamp valid? + if (isValidUTCDateTime(lastModifiedTimestamp)) { + // string is a valid timestamp + item.mtime = SysTime.fromISOExtString(lastModifiedTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ lastModifiedTimestamp); + // Set mtime to SysTime(0) to ensure we have a value + item.mtime = SysTime(0); + } + } else { // is a remote item, but 'fileSystemInfo' is missing from 'remoteItem' - if ("fileSystemInfo" in driveItem) { - item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str); + lastModifiedTimestamp = strip(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str); + // is lastModifiedTimestamp valid? + if (isValidUTCDateTime(lastModifiedTimestamp)) { + // string is a valid timestamp + item.mtime = SysTime.fromISOExtString(lastModifiedTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ lastModifiedTimestamp); + // Set mtime to SysTime(0) to ensure we have a value + item.mtime = SysTime(0); } } } else { // Does fileSystemInfo exist at all ? if ("fileSystemInfo" in driveItem) { - item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str); + // fileSystemInfo exists + lastModifiedTimestamp = strip(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str); + // is lastModifiedTimestamp valid? + if (isValidUTCDateTime(lastModifiedTimestamp)) { + // string is a valid timestamp + item.mtime = SysTime.fromISOExtString(lastModifiedTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ lastModifiedTimestamp); + // Set mtime to SysTime(0) to ensure we have a value + item.mtime = SysTime(0); + } } } } diff --git a/src/sync.d b/src/sync.d index fd3f555ca..8d1f047c2 100644 --- a/src/sync.d +++ b/src/sync.d @@ -2488,12 +2488,33 @@ class SyncEngine { try { // get the mtime from the JSON data SysTime itemModifiedTime; + string lastModifiedTimestamp; if (isItemRemote(onedriveJSONItem)) { // remote file item - itemModifiedTime = SysTime.fromISOExtString(onedriveJSONItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str); + lastModifiedTimestamp = strip(onedriveJSONItem["remoteItem"]["fileSystemInfo"]["lastModifiedDateTime"].str); + // is lastModifiedTimestamp valid? + if (isValidUTCDateTime(lastModifiedTimestamp)) { + // string is a valid timestamp + itemModifiedTime = SysTime.fromISOExtString(lastModifiedTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ lastModifiedTimestamp); + // Set mtime to SysTime(0) + itemModifiedTime = SysTime(0); + } } else { // not a remote item - itemModifiedTime = SysTime.fromISOExtString(onedriveJSONItem["fileSystemInfo"]["lastModifiedDateTime"].str); + lastModifiedTimestamp = strip(onedriveJSONItem["fileSystemInfo"]["lastModifiedDateTime"].str); + // is lastModifiedTimestamp valid? + if (isValidUTCDateTime(lastModifiedTimestamp)) { + // string is a valid timestamp + itemModifiedTime = SysTime.fromISOExtString(lastModifiedTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ lastModifiedTimestamp); + // Set mtime to SysTime(0) + itemModifiedTime = SysTime(0); + } } // set the correct time on the downloaded file @@ -4499,7 +4520,7 @@ class SyncEngine { if (appConfig.accountType == "personal") { addLogEntry("ERROR: OneDrive account currently has zero space available. Please free up some space online or purchase additional capacity."); } else { // Assuming 'business' or 'sharedLibrary' - addLogEntry("WARNING: OneDrive quota information is being restricted or providing a zero value. Please fix by speaking to your OneDrive / Office 365 Administrator."); + addLogEntry("WARNING: OneDrive quota information is being restricted or providing a zero value. Please fix by speaking to your OneDrive / Office 365 Administrator." , ["verbose"]); } } } else { @@ -4508,10 +4529,10 @@ class SyncEngine { // what sort of account type is this? if (appConfig.accountType == "personal") { - addLogEntry("ERROR: OneDrive quota information is missing. Your OneDrive account potentially has zero space available. Please free up some space online."); + addLogEntry("ERROR: OneDrive quota information is missing. Your OneDrive account potentially has zero space available. Please free up some space online.", ["verbose"]); } else { // quota details not available - addLogEntry("WARNING: OneDrive quota information is being restricted. Please fix by speaking to your OneDrive / Office 365 Administrator."); + addLogEntry("WARNING: OneDrive quota information is being restricted. Please fix by speaking to your OneDrive / Office 365 Administrator.", ["verbose"]); } } } else { @@ -8343,7 +8364,22 @@ class SyncEngine { // Check the session data for expirationDateTime if ("expirationDateTime" in sessionFileData) { - auto expiration = SysTime.fromISOExtString(sessionFileData["expirationDateTime"].str); + addLogEntry("expirationDateTime: " ~ sessionFileData["expirationDateTime"].str); + SysTime expiration; + string expirationTimestamp; + expirationTimestamp = strip(sessionFileData["expirationDateTime"].str); + + // is expirationTimestamp valid? + if (isValidUTCDateTime(expirationTimestamp)) { + // string is a valid timestamp + expiration = SysTime.fromISOExtString(expirationTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ expirationTimestamp); + return false; + } + + // valid timestamp if (expiration < Clock.currTime()) { addLogEntry("The upload session has expired for: " ~ sessionFilePath, ["verbose"]); return false; @@ -8656,6 +8692,7 @@ class SyncEngine { // New DB Tie Item to detail the 'root' of the Shared Folder Item tieDBItem; + string lastModifiedTimestamp; tieDBItem.name = "root"; // Get the right parentReference details @@ -8680,8 +8717,23 @@ class SyncEngine { } } + // set the item type tieDBItem.type = ItemType.dir; - tieDBItem.mtime = SysTime.fromISOExtString(onedriveJSONItem["fileSystemInfo"]["lastModifiedDateTime"].str); + + // get the lastModifiedDateTime + lastModifiedTimestamp = strip(onedriveJSONItem["fileSystemInfo"]["lastModifiedDateTime"].str); + // is lastModifiedTimestamp valid? + if (isValidUTCDateTime(lastModifiedTimestamp)) { + // string is a valid timestamp + tieDBItem.mtime = SysTime.fromISOExtString(lastModifiedTimestamp); + } else { + // invalid timestamp from JSON file + addLogEntry("WARNING: Invalid timestamp provided by the Microsoft OneDrive API: " ~ lastModifiedTimestamp); + // Set mtime to SysTime(0) + tieDBItem.mtime = SysTime(0); + } + + // ensure there is no parentId tieDBItem.parentId = null; // Add this DB Tie parent record to the local database diff --git a/src/util.d b/src/util.d index 4cb272e15..ad6541d2c 100644 --- a/src/util.d +++ b/src/util.d @@ -512,6 +512,25 @@ bool isValidUTF16(string path) { return true; } +// Validate that the provided string is a valid date time stamp in UTC format +bool isValidUTCDateTime(string dateTimeString) { + // Regular expression for validating the datetime format + auto pattern = regex(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$"); + + // First, check if the string matches the pattern + if (!match(dateTimeString, pattern)) { + return false; + } + + // Attempt to parse the string into a DateTime object + try { + auto dt = SysTime.fromISOExtString(dateTimeString); + return true; + } catch (TimeException) { + return false; + } +} + // Does the path contain any HTML URL encoded items (e.g., '%20' for space) bool containsURLEncodedItems(string path) { // Check for null or empty string @@ -1292,6 +1311,7 @@ extern(C) nothrow @nogc @system void exitScopeSignalHandler(int signo) { } } +// Return the compiler details string compilerDetails() { version(DigitalMars) enum compiler = "DMD"; else version(LDC) enum compiler = "LDC";