Skip to content

Commit

Permalink
Remove sha1 use and cleanup defunct remaining crc32 use (#2424)
Browse files Browse the repository at this point in the history 
* Remove sha1 from being used by the client as this is being depreciated by Microsoft in July 2023 - https://devblogs.microsoft.com/microsoft365dev/deprecation-of-sha1hash-on-onedrive-personal/
* Complete the removal of crc32 as this is also no longer present for a long time, but some code elements still existed
* Only compute quickXorHash, not quickXorHash and sha256Hash as computing sha256Hash is CPU expensive
* Update cache database stored items to only store quickXorHash and sha256Hash values (remove crc32 and sha1)
  • Loading branch information
@abraunegg
abraunegg authored Jun 19, 2023
1 parent c9fe8ad commit 06420c9
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 73 deletions.
38 changes: 17 additions & 21 deletions src/itemdb.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,8 @@ struct Item {
string cTag;
SysTime mtime;
string parentId;
string crc32Hash;
string sha1Hash;
string quickXorHash;
string sha256Hash;
string remoteDriveId;
string remoteId;
string syncStatus;
Expand All @@ -34,7 +33,7 @@ struct Item {
final class ItemDatabase
{
// increment this for every change in the db schema
immutable int itemDatabaseVersion = 10;
immutable int itemDatabaseVersion = 11;

Database db;
string insertItemStmt;
Expand Down Expand Up @@ -100,12 +99,12 @@ final class ItemDatabase
db.exec("PRAGMA locking_mode = EXCLUSIVE");

insertItemStmt = "
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId, syncStatus)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, quickXorHash, sha256Hash, remoteDriveId, remoteId, syncStatus)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)
";
updateItemStmt = "
UPDATE item
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, crc32Hash = ?9, sha1Hash = ?10, quickXorHash = ?11, remoteDriveId = ?12, remoteId = ?13, syncStatus = ?14
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, quickXorHash = ?9, sha256Hash = ?10, remoteDriveId = ?11, remoteId = ?12, syncStatus = ?13
WHERE driveId = ?1 AND id = ?2
";
selectItemByIdStmt = "
Expand Down Expand Up @@ -136,9 +135,8 @@ final class ItemDatabase
cTag TEXT,
mtime TEXT NOT NULL,
parentId TEXT,
crc32Hash TEXT,
sha1Hash TEXT,
quickXorHash TEXT,
sha256Hash TEXT,
remoteDriveId TEXT,
remoteId TEXT,
deltaLink TEXT,
Expand Down Expand Up @@ -321,19 +319,18 @@ final class ItemDatabase
bind(6, cTag);
bind(7, mtime.toISOExtString());
bind(8, parentId);
bind(9, crc32Hash);
bind(10, sha1Hash);
bind(11, quickXorHash);
bind(12, remoteDriveId);
bind(13, remoteId);
bind(14, syncStatus);
bind(9, quickXorHash);
bind(10, sha256Hash);
bind(11, remoteDriveId);
bind(12, remoteId);
bind(13, syncStatus);
}
}

private Item buildItem(Statement.Result result)
{
assert(!result.empty, "The result must not be empty");
assert(result.front.length == 15, "The result must have 15 columns");
assert(result.front.length == 14, "The result must have 14 columns");
Item item = {
driveId: result.front[0].dup,
id: result.front[1].dup,
Expand All @@ -342,12 +339,11 @@ final class ItemDatabase
cTag: result.front[5].dup,
mtime: SysTime.fromISOExtString(result.front[6]),
parentId: result.front[7].dup,
crc32Hash: result.front[8].dup,
sha1Hash: result.front[9].dup,
quickXorHash: result.front[10].dup,
remoteDriveId: result.front[11].dup,
remoteId: result.front[12].dup,
syncStatus: result.front[14].dup
quickXorHash: result.front[8].dup,
sha256Hash: result.front[9].dup,
remoteDriveId: result.front[10].dup,
remoteId: result.front[11].dup,
syncStatus: result.front[12].dup
};
switch (result.front[3]) {
case "file": item.type = ItemType.file; break;
Expand Down
65 changes: 35 additions & 30 deletions src/sync.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,9 @@ private bool hasQuickXorHash(const ref JSONValue item)
return ("quickXorHash" in item["file"]["hashes"]) != null;
}

private bool hasSha1Hash(const ref JSONValue item)
private bool hasSHA256Hash(const ref JSONValue item)
{
return ("sha1Hash" in item["file"]["hashes"]) != null;
return ("sha256Hash" in item["file"]["hashes"]) != null;
}

private bool isDotFile(const(string) path)
Expand Down Expand Up @@ -173,16 +173,24 @@ private Item makeItem(const ref JSONValue driveItem)

// extract the file hash
if (isItemFile(driveItem) && ("hashes" in driveItem["file"])) {
if ("crc32Hash" in driveItem["file"]["hashes"]) {
item.crc32Hash = driveItem["file"]["hashes"]["crc32Hash"].str;
} else if ("sha1Hash" in driveItem["file"]["hashes"]) {
item.sha1Hash = driveItem["file"]["hashes"]["sha1Hash"].str;
} else if ("quickXorHash" in driveItem["file"]["hashes"]) {
// Get quickXorHash
if ("quickXorHash" in driveItem["file"]["hashes"]) {
item.quickXorHash = driveItem["file"]["hashes"]["quickXorHash"].str;
} else {
log.vlog("The file does not have any hash");
log.vdebug("quickXorHash is missing");
}
}
// sha256Hash
if ("sha256Hash" in driveItem["file"]["hashes"]) {
item.sha256Hash = driveItem["file"]["hashes"]["sha256Hash"].str;
} else {
log.vdebug("sha256Hash is missing");
}
// No hashes ..
if ((item.quickXorHash.empty) && (item.sha256Hash.empty) ) {
// Odd .. no hash ......
log.error("ERROR: OneDrive API inconsistency - the file does not have any hash");
}
}

if (isItemRemote(driveItem)) {
item.remoteDriveId = driveItem["remoteItem"]["parentReference"]["driveId"].str;
Expand All @@ -201,13 +209,11 @@ private Item makeItem(const ref JSONValue driveItem)

private bool testFileHash(const(string) path, const ref Item item)
{
// Try and compute the file hash
if (item.crc32Hash) {
if (item.crc32Hash == computeCrc32(path)) return true;
} else if (item.sha1Hash) {
if (item.sha1Hash == computeSha1Hash(path)) return true;
} else if (item.quickXorHash) {
// Generate QuickXORHash first before others
if (item.quickXorHash) {
if (item.quickXorHash == computeQuickXorHash(path)) return true;
} else if (item.sha256Hash) {
if (item.sha256Hash == computeSHA256Hash(path)) return true;
}
return false;
}
Expand Down Expand Up @@ -3018,11 +3024,11 @@ final class SyncEngine
OneDriveFileHash = fileDetails["file"]["hashes"]["quickXorHash"].str;
}
}
// Check for Sha1Hash
if (hasSha1Hash(fileDetails)) {
// Use the configured sha1Hash as reported by OneDrive
if (fileDetails["file"]["hashes"]["sha1Hash"].str != "") {
OneDriveFileHash = fileDetails["file"]["hashes"]["sha1Hash"].str;
// Check for sha256Hash
if (hasSHA256Hash(fileDetails)) {
// Use the configured sha256Hash as reported by OneDrive
if (fileDetails["file"]["hashes"]["sha256Hash"].str != "") {
OneDriveFileHash = fileDetails["file"]["hashes"]["sha256Hash"].str;
}
}
} else {
Expand Down Expand Up @@ -3152,9 +3158,8 @@ final class SyncEngine
// A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on
// does the file hash OneDrive reports match what we have locally?
string quickXorHash = computeQuickXorHash(path);
string sha1Hash = computeSha1Hash(path);

if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash) || (OneDriveFileHash == sha1Hash)) {
if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash)) {
// downloaded matches either size or hash
log.vdebug("Downloaded file matches reported size and or reported file hash");
try {
Expand All @@ -3173,7 +3178,7 @@ final class SyncEngine
log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why.");
}
// hash error?
if ((OneDriveFileHash != quickXorHash) || (OneDriveFileHash != sha1Hash)) {
if (OneDriveFileHash != quickXorHash) {
// downloaded file hash does not match
log.vdebug("Actual file hash: ", OneDriveFileHash);
log.vdebug("OneDrive API reported hash: ", quickXorHash);
Expand Down Expand Up @@ -6769,16 +6774,16 @@ final class SyncEngine

// real id / eTag / cTag are different format for personal / business account
auto sha1 = new SHA1Digest();
ubyte[] hash1 = sha1.digest(path);
ubyte[] fakedOneDriveItemValues = sha1.digest(path);

JSONValue fakeResponse;

if (isDir(path)) {
// path is a directory
fakeResponse = [
"id": JSONValue(toHexString(hash1)),
"cTag": JSONValue(toHexString(hash1)),
"eTag": JSONValue(toHexString(hash1)),
"id": JSONValue(toHexString(fakedOneDriveItemValues)),
"cTag": JSONValue(toHexString(fakedOneDriveItemValues)),
"eTag": JSONValue(toHexString(fakedOneDriveItemValues)),
"fileSystemInfo": JSONValue([
"createdDateTime": mtime.toISOExtString(),
"lastModifiedDateTime": mtime.toISOExtString()
Expand All @@ -6797,9 +6802,9 @@ final class SyncEngine
string quickXorHash = computeQuickXorHash(path);

fakeResponse = [
"id": JSONValue(toHexString(hash1)),
"cTag": JSONValue(toHexString(hash1)),
"eTag": JSONValue(toHexString(hash1)),
"id": JSONValue(toHexString(fakedOneDriveItemValues)),
"cTag": JSONValue(toHexString(fakedOneDriveItemValues)),
"eTag": JSONValue(toHexString(fakedOneDriveItemValues)),
"fileSystemInfo": JSONValue([
"createdDateTime": mtime.toISOExtString(),
"lastModifiedDateTime": mtime.toISOExtString()
Expand Down
32 changes: 10 additions & 22 deletions src/util.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,28 +48,6 @@ void safeRemove(const(char)[] path)
if (exists(path)) remove(path);
}

// returns the crc32 hex string of a file
string computeCrc32(string path)
{
CRC32 crc;
auto file = File(path, "rb");
foreach (ubyte[] data; chunks(file, 4096)) {
crc.put(data);
}
return crc.finish().toHexString().dup;
}

// returns the sha1 hash hex string of a file
string computeSha1Hash(string path)
{
SHA1 sha;
auto file = File(path, "rb");
foreach (ubyte[] data; chunks(file, 4096)) {
sha.put(data);
}
return sha.finish().toHexString().dup;
}

// returns the quickXorHash base64 string of a file
string computeQuickXorHash(string path)
{
Expand All @@ -81,6 +59,16 @@ string computeQuickXorHash(string path)
return Base64.encode(qxor.finish());
}

// returns the SHA256 hex string of a file
string computeSHA256Hash(string path) {
SHA256 sha256;
auto file = File(path, "rb");
foreach (ubyte[] data; chunks(file, 4096)) {
sha256.put(data);
}
return sha256.finish().toHexString().dup;
}

// converts wildcards (*, ?) to regex
Regex!char wild2regex(const(char)[] pattern)
{
Expand Down

0 comments on commit 06420c9

Please sign in to comment.