From 5eab0c9f90dec3228a6cb34608b724282e3d9e68 Mon Sep 17 00:00:00 2001 From: Christopher Rokita Date: Wed, 3 Mar 2021 17:22:43 -0500 Subject: [PATCH 1/3] Add CRC checksums to the file manager --- lib/js/src/manager/file/FileManager.js | 7 +++++++ lib/js/src/manager/file/filetypes/SdlFile.js | 4 ++-- lib/js/src/util/_FileUtils.js | 16 ++++++++++++++++ lib/node/src/util/_FileUtils.js | 16 ++++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/lib/js/src/manager/file/FileManager.js b/lib/js/src/manager/file/FileManager.js index e12a07e6..25256d10 100644 --- a/lib/js/src/manager/file/FileManager.js +++ b/lib/js/src/manager/file/FileManager.js @@ -77,6 +77,13 @@ class FileManager extends _FileManagerBase { if (file.getType() !== null && file.getType() !== undefined) { putFile.setFileType(file.getType()); } + + // set the crc value if putfile data exists + if (putFile.getFileData() !== null && putFile.getFileData() !== undefined) { + // use the file data to generate a crc32 checksum from it + putFile.setCrc(_FileUtils.createCrc(putFile.getFileData())); + } + putFile.setPersistentFile(file.isPersistent()); return putFile; } diff --git a/lib/js/src/manager/file/filetypes/SdlFile.js b/lib/js/src/manager/file/filetypes/SdlFile.js index 5f47210e..c3d20055 100644 --- a/lib/js/src/manager/file/filetypes/SdlFile.js +++ b/lib/js/src/manager/file/filetypes/SdlFile.js @@ -90,7 +90,7 @@ class SdlFile { /** * Sets the byte array that represents the content of the file - * @param {String} data - a byte array representing the data of the file + * @param {Uint8Array} data - a byte array representing the data of the file * @returns {SdlFile} - A reference to this instance to support method chaining. */ setFileData (data) { @@ -100,7 +100,7 @@ class SdlFile { /** * Gets the byte array that represents the content of the file - * @returns {String} - a byte array representing the data of the file + * @returns {Uint8Array} - a byte array representing the data of the file */ getFileData () { return this._fileData; diff --git a/lib/js/src/util/_FileUtils.js b/lib/js/src/util/_FileUtils.js index 0d161cff..80bb940a 100644 --- a/lib/js/src/util/_FileUtils.js +++ b/lib/js/src/util/_FileUtils.js @@ -59,7 +59,23 @@ class _FileUtils { } return null; } + + /** + * Generates a CRC32 checksum for a file data string + * @param {Uint8Array} fileData - A byte array of the file contents. + * @returns {Number} - The checksum + */ + static createCrc (fileData) { + let crc = 0 ^ (-1); + + for (let index = 0; index < fileData.length; index++) { + crc = (crc >>> 8) ^ _FileUtils.CRC_TABLE[(crc ^ fileData[index]) & 0xFF]; + } + + return (crc ^ (-1)) >>> 0; + } } +_FileUtils.CRC_TABLE = [0, 1996959894, -301047508, -1727442502, 124634137, 1886057615, -379345611, -1637575261, 249268274, 2044508324, -522852066, -1747789432, 162941995, 2125561021, -407360249, -1866523247, 498536548, 1789927666, -205950648, -2067906082, 450548861, 1843258603, -187386543, -2083289657, 325883990, 1684777152, -43845254, -1973040660, 335633487, 1661365465, -99664541, -1928851979, 997073096, 1281953886, -715111964, -1570279054, 1006888145, 1258607687, -770865667, -1526024853, 901097722, 1119000684, -608450090, -1396901568, 853044451, 1172266101, -589951537, -1412350631, 651767980, 1373503546, -925412992, -1076862698, 565507253, 1454621731, -809855591, -1195530993, 671266974, 1594198024, -972236366, -1324619484, 795835527, 1483230225, -1050600021, -1234817731, 1994146192, 31158534, -1731059524, -271249366, 1907459465, 112637215, -1614814043, -390540237, 2013776290, 251722036, -1777751922, -519137256, 2137656763, 141376813, -1855689577, -429695999, 1802195444, 476864866, -2056965928, -228458418, 1812370925, 453092731, -2113342271, -183516073, 1706088902, 314042704, -1950435094, -54949764, 1658658271, 366619977, -1932296973, -69972891, 1303535960, 984961486, -1547960204, -725929758, 1256170817, 1037604311, -1529756563, -740887301, 1131014506, 879679996, -1385723834, -631195440, 1141124467, 855842277, -1442165665, -586318647, 1342533948, 654459306, -1106571248, -921952122, 1466479909, 544179635, -1184443383, -832445281, 1591671054, 702138776, -1328506846, -942167884, 1504918807, 783551873, -1212326853, -1061524307, -306674912, -1698712650, 62317068, 1957810842, -355121351, -1647151185, 81470997, 1943803523, -480048366, -1805370492, 225274430, 2053790376, -468791541, -1828061283, 167816743, 2097651377, -267414716, -2029476910, 503444072, 1762050814, -144550051, -2140837941, 426522225, 1852507879, -19653770, -1982649376, 282753626, 1742555852, -105259153, -1900089351, 397917763, 1622183637, -690576408, -1580100738, 953729732, 1340076626, -776247311, -1497606297, 1068828381, 1219638859, -670225446, -1358292148, 906185462, 1090812512, -547295293, -1469587627, 829329135, 1181335161, -882789492, -1134132454, 628085408, 1382605366, -871598187, -1156888829, 570562233, 1426400815, -977650754, -1296233688, 733239954, 1555261956, -1026031705, -1244606671, 752459403, 1541320221, -1687895376, -328994266, 1969922972, 40735498, -1677130071, -351390145, 1913087877, 83908371, -1782625662, -491226604, 2075208622, 213261112, -1831694693, -438977011, 2094854071, 198958881, -2032938284, -237706686, 1759359992, 534414190, -2118248755, -155638181, 1873836001, 414664567, -2012718362, -15766928, 1711684554, 285281116, -1889165569, -127750551, 1634467795, 376229701, -1609899400, -686959890, 1308918612, 956543938, -1486412191, -799009033, 1231636301, 1047427035, -1362007478, -640263460, 1088359270, 936918000, -1447252397, -558129467, 1202900863, 817233897, -1111625188, -893730166, 1404277552, 615818150, -1160759803, -841546093, 1423857449, 601450431, -1285129682, -1000256840, 1567103746, 711928724, -1274298825, -1022587231, 1510334235, 755167117]; export { _FileUtils }; \ No newline at end of file diff --git a/lib/node/src/util/_FileUtils.js b/lib/node/src/util/_FileUtils.js index dbd66536..417e9477 100644 --- a/lib/node/src/util/_FileUtils.js +++ b/lib/node/src/util/_FileUtils.js @@ -62,7 +62,23 @@ class _FileUtils { }); }); } + + /** + * Generates a CRC32 checksum for a file data string + * @param {Uint8Array} fileData - A byte array of the file contents. + * @returns {Number} - The checksum + */ + static createCrc (fileData) { + let crc = 0 ^ (-1); + + for (let index = 0; index < fileData.length; index++) { + crc = (crc >>> 8) ^ _FileUtils.CRC_TABLE[(crc ^ fileData[index]) & 0xFF]; + } + + return (crc ^ (-1)) >>> 0; + } } +_FileUtils.CRC_TABLE = [0, 1996959894, -301047508, -1727442502, 124634137, 1886057615, -379345611, -1637575261, 249268274, 2044508324, -522852066, -1747789432, 162941995, 2125561021, -407360249, -1866523247, 498536548, 1789927666, -205950648, -2067906082, 450548861, 1843258603, -187386543, -2083289657, 325883990, 1684777152, -43845254, -1973040660, 335633487, 1661365465, -99664541, -1928851979, 997073096, 1281953886, -715111964, -1570279054, 1006888145, 1258607687, -770865667, -1526024853, 901097722, 1119000684, -608450090, -1396901568, 853044451, 1172266101, -589951537, -1412350631, 651767980, 1373503546, -925412992, -1076862698, 565507253, 1454621731, -809855591, -1195530993, 671266974, 1594198024, -972236366, -1324619484, 795835527, 1483230225, -1050600021, -1234817731, 1994146192, 31158534, -1731059524, -271249366, 1907459465, 112637215, -1614814043, -390540237, 2013776290, 251722036, -1777751922, -519137256, 2137656763, 141376813, -1855689577, -429695999, 1802195444, 476864866, -2056965928, -228458418, 1812370925, 453092731, -2113342271, -183516073, 1706088902, 314042704, -1950435094, -54949764, 1658658271, 366619977, -1932296973, -69972891, 1303535960, 984961486, -1547960204, -725929758, 1256170817, 1037604311, -1529756563, -740887301, 1131014506, 879679996, -1385723834, -631195440, 1141124467, 855842277, -1442165665, -586318647, 1342533948, 654459306, -1106571248, -921952122, 1466479909, 544179635, -1184443383, -832445281, 1591671054, 702138776, -1328506846, -942167884, 1504918807, 783551873, -1212326853, -1061524307, -306674912, -1698712650, 62317068, 1957810842, -355121351, -1647151185, 81470997, 1943803523, -480048366, -1805370492, 225274430, 2053790376, -468791541, -1828061283, 167816743, 2097651377, -267414716, -2029476910, 503444072, 1762050814, -144550051, -2140837941, 426522225, 1852507879, -19653770, -1982649376, 282753626, 1742555852, -105259153, -1900089351, 397917763, 1622183637, -690576408, -1580100738, 953729732, 1340076626, -776247311, -1497606297, 1068828381, 1219638859, -670225446, -1358292148, 906185462, 1090812512, -547295293, -1469587627, 829329135, 1181335161, -882789492, -1134132454, 628085408, 1382605366, -871598187, -1156888829, 570562233, 1426400815, -977650754, -1296233688, 733239954, 1555261956, -1026031705, -1244606671, 752459403, 1541320221, -1687895376, -328994266, 1969922972, 40735498, -1677130071, -351390145, 1913087877, 83908371, -1782625662, -491226604, 2075208622, 213261112, -1831694693, -438977011, 2094854071, 198958881, -2032938284, -237706686, 1759359992, 534414190, -2118248755, -155638181, 1873836001, 414664567, -2012718362, -15766928, 1711684554, 285281116, -1889165569, -127750551, 1634467795, 376229701, -1609899400, -686959890, 1308918612, 956543938, -1486412191, -799009033, 1231636301, 1047427035, -1362007478, -640263460, 1088359270, 936918000, -1447252397, -558129467, 1202900863, 817233897, -1111625188, -893730166, 1404277552, 615818150, -1160759803, -841546093, 1423857449, 601450431, -1285129682, -1000256840, 1567103746, 711928724, -1274298825, -1022587231, 1510334235, 755167117]; export { _FileUtils }; \ No newline at end of file From 38bdf60a1896ce648aadef6bb5abb7a2f626adcb Mon Sep 17 00:00:00 2001 From: Christopher Rokita Date: Thu, 4 Mar 2021 10:16:37 -0500 Subject: [PATCH 2/3] Add unit tests --- tests/managers/file/FileManagerTests.js | 53 +++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/managers/file/FileManagerTests.js b/tests/managers/file/FileManagerTests.js index 008e4f7b..2b6e7774 100644 --- a/tests/managers/file/FileManagerTests.js +++ b/tests/managers/file/FileManagerTests.js @@ -468,5 +468,58 @@ module.exports = function (appClient) { Validator.assertTrue(fileManager.hasUploadedFile(file)); sdlManager.removeRpcListener(SDL.rpc.enums.FunctionID.ListFiles, expectSuccess); }); + + it('testCrcUndefinedVersion4', async function () { + const stub = sinon.stub(lifecycleManager, 'getSdlMsgVersion') + .callsFake(() => { + return new SDL.rpc.structs.SdlMsgVersion() + .setMajorVersion(4) + .setMinorVersion(5); + }); + + const file = new SDL.manager.file.filetypes.SdlFile() + .setName('hello') + .setFilePath('./test_icon_1.png') + .setType(SDL.rpc.enums.FileType.GRAPHIC_PNG) + .setPersistent(true); + + const putFile = await fileManager._createPutFile(file); + + Validator.assertNull(putFile.getCrc()); + stub.restore(); + }); + + it('testCrcAssignmentFilePath', async function () { + const file = new SDL.manager.file.filetypes.SdlFile() + .setName('hello') + .setFilePath('./test_icon_1.png') + .setType(SDL.rpc.enums.FileType.GRAPHIC_PNG) + .setPersistent(true); + + const putFile = await fileManager._createPutFile(file); + + Validator.assertNotNullUndefined(putFile.getCrc()); + }); + + it('testCorrectCrcValue', async function () { + // set the file data to an input with an output that we expect to know + const file1 = new SDL.manager.file.filetypes.SdlFile() + .setName('hello') + .setType(SDL.rpc.enums.FileType.GRAPHIC_PNG) + .setFileData(SDL.util._JsonRpcMarshaller._encode('abcde')) + .setPersistent(true); + + const putFile1 = await fileManager._createPutFile(file1); + Validator.assertEquals(putFile1.getCrc(), 2240272485); + + const file2 = new SDL.manager.file.filetypes.SdlFile() + .setName('hello') + .setType(SDL.rpc.enums.FileType.GRAPHIC_PNG) + .setFileData(SDL.util._JsonRpcMarshaller._encode('it is Wednesday my dudes')) + .setPersistent(true); + + const putFile2 = await fileManager._createPutFile(file2); + Validator.assertEquals(putFile2.getCrc(), 801050498); + }); }); }; \ No newline at end of file From 70065ba81fba417504e889a3f2af45687050de9a Mon Sep 17 00:00:00 2001 From: Christopher Rokita Date: Thu, 4 Mar 2021 16:24:19 -0500 Subject: [PATCH 3/3] RPC version check before setting crc --- lib/js/src/manager/file/FileManager.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/js/src/manager/file/FileManager.js b/lib/js/src/manager/file/FileManager.js index 25256d10..9dec42d6 100644 --- a/lib/js/src/manager/file/FileManager.js +++ b/lib/js/src/manager/file/FileManager.js @@ -78,10 +78,13 @@ class FileManager extends _FileManagerBase { putFile.setFileType(file.getType()); } - // set the crc value if putfile data exists - if (putFile.getFileData() !== null && putFile.getFileData() !== undefined) { - // use the file data to generate a crc32 checksum from it - putFile.setCrc(_FileUtils.createCrc(putFile.getFileData())); + // putfile crc introduced in 5.0.0 + if (this._lifecycleManager.getSdlMsgVersion().getMajorVersion() >= 5) { + // set the crc value if putfile data exists + if (putFile.getFileData() !== null && putFile.getFileData() !== undefined) { + // use the file data to generate a crc32 checksum from it + putFile.setCrc(_FileUtils.createCrc(putFile.getFileData())); + } } putFile.setPersistentFile(file.isPersistent());