Fix HalModuleParser.parsePrefix() when parsing valid compressed binaries

lib/HalModuleParser.js

@@ -22,10 +22,9 @@ var fs = require('fs');
 var when = require('when');
 var pipeline = require('when/pipeline');
 var utilities = require('./utilities.js');
-var buffers = require('h5.buffers');
-var moduleInfo = require('./ModuleInfo');
+var ModuleInfo = require('./ModuleInfo');
 
-var MIN_MODULE_SUFFIX_SIZE = moduleInfo.MIN_MODULE_SUFFIX_SIZE;
+var MIN_MODULE_SUFFIX_SIZE = ModuleInfo.MIN_MODULE_SUFFIX_SIZE;
 
 //var BufferStream = require('./BufferStream.js');
 
@@ -231,81 +230,155 @@ HalModuleParser.prototype = {
 	},
 
 	/*
-	 * tries to determine where we should be looking in the file.
+	 * Validates whether the module binary has a valid
+	 * module prefix at a certain offset.
 	 *
-	 * This changes based on whether or not we're in the bootloader,
-	 * and we attempt to validate the result based on the assumption
-	 * that firmware addresses tend to start with hex '80' since
-	 * they're stored after
-	 * 0x0800xxxx, etc.
+	 * @param {Object} fileBuffer
+	 * @param {number} offset
+	 * @returns {Object} result
+	 * @returns {number} result.score A score from 0 to N, where 0 indicates invalid and N the highest level of confidence
+	 * @returns {boolean} result.passed Set to true if module prefix completely passed all checks
+	 * @private
+	 */
+	_validateModulePrefix: function(fileBuffer, offset = 0) {
+		const result = { score: 0, passed: false };
+		const prefix = fileBuffer.slice(offset, offset + ModuleInfo.MODULE_PREFIX_SIZE);
+		if (prefix.length < ModuleInfo.MODULE_PREFIX_SIZE) {
+			return result;
+		}
+
+		// Parse module prefix
+		const prefixInfo = this._parsePrefix(prefix);
+		// Start address should be < end address
+		let startAddr = prefixInfo.moduleStartAddy;
+		if (typeof startAddr === 'string') {
+			startAddr = Number.parseInt(startAddr, 16);
+		}
+		let endAddr = prefixInfo.moduleEndAddy;
+		if (typeof endAddr === 'string') {
+			endAddr = Number.parseInt(endAddr, 16);
+		}
+		if (endAddr <= startAddr) {
+			return result;
+		}
+		const expectedSize = endAddr - startAddr + 4 /* CRC-32 */;
+		if (fileBuffer.length < expectedSize) {
+			return result;
+		}
+		// Module function should not be empty
+		if (prefixInfo.moduleFunction === ModuleInfo.FunctionType.NONE) {
+			return result;
+		}
+		// Empty dependencies should have index and version set to zero
+		if (prefixInfo.depModuleFunction === ModuleInfo.FunctionType.NONE) {
+			if (prefixInfo.depModuleIndex !== 0 || prefixInfo.depModuleVersion !== 0) {
+				return result;
+			}
+		}
+		// Empty dependencies should have index and version set to zero
+		if (prefixInfo.dep2ModuleFunction === ModuleInfo.FunctionType.NONE) {
+			if (prefixInfo.dep2ModuleIndex !== 0 || prefixInfo.dep2ModuleVersion !== 0) {
+				return result;
+			}
+		}
+
+		// If we got here, we have a seemingly valid module prefix
+		result.score++;
+		// XXX: make sure to update this if adding other sanity checks
+		const maxScore = 7;
+		// Increase score if platform ID is known
+		if (Object.values(ModuleInfo.Platform).includes(prefixInfo.platformID)) {
+			result.score++;
+		}
+		// Increase score if module function is one of the known module function types
+		if (Object.values(ModuleInfo.FunctionType).includes(prefixInfo.moduleFunction)) {
+			result.score++;
+		}
+		// Increase score if first dependency module function is one of the known module function types
+		if (Object.values(ModuleInfo.FunctionType).includes(prefixInfo.depModuleFunction)) {
+			result.score++;
+		}
+		// Increase score if second dependency module function is one of the known module function types
+		if (Object.values(ModuleInfo.FunctionType).includes(prefixInfo.dep2ModuleFunction)) {
+			result.score++;
+		}
+		// Module size according to the module prefix exactly matches
+		// This is normally true for regular modules unless they are padded for certain reasons
+		// or the passed buffer contains multiple modules (e.g. in case of combined modules)
+		if (fileBuffer.length === expectedSize) {
+			result.score++;
+		}
+		// Module flags are known
+		const allFlags = Object.values(ModuleInfo.Flags).reduce((a, b) => (a | b), 0);
+		if ((prefixInfo.moduleFlags & (~allFlags)) === 0) {
+			result.score++;
+		}
+		result.passed = (result.score >= maxScore);
+
+		return result;
+	},
+
+	/*
+	 * Finds the offset in the provided buffer where a valid module
+	 * prefix is located.
+	 *
+	 * Depending on the module type, hardware platform requirements
+	 * or other factors, module prefix may be present at a certain
+	 * non-zero offset in the module binary.
 	 *
 	 * @param fileBuffer
 	 * @returns {number}
 	 * @private
 	 */
-	_divineModulePrefixOffset: function(fileBuffer) {
-		// try no offset
-		var r = new buffers.BufferReader(fileBuffer);
-		var userModuleStartAddy = r.shiftUInt32(true).toString(16);
-
-		// start over
-		r = new buffers.BufferReader(fileBuffer);
-		// skip 0x184 for system modules to skip the vector table
-		r.skip(388);
-
-		var sysModuleStartAddy = r.shiftUInt32(true).toString(16);
-
-		// start over, test for Mesh modular firmware which has a different offset than Photon/Electron modular firmware
-		r = new buffers.BufferReader(fileBuffer);
-		// skip 0x200 for Mesh firmware
-		r.skip(0x200);
-		var meshModule = this._parsePrefix(r);
-
-		// start over, test for Core monolithic firmware which has a different offset than modular firmware
-		r = new buffers.BufferReader(fileBuffer);
-		// skip 0x10C for Core firmware
-		r.skip(0x10C);
-
-
-		var coreModuleStartAddy = r.shiftUInt32(true).toString(16);
-
-		// start over, test for bluz system part which has a different offset than Photon
-		r = new buffers.BufferReader(fileBuffer);
-		// skip 0xc0 for bluz system modules to skip the vector table
-		r.skip(192);
-		var bluzModuleStartAddy = r.shiftUInt32(true);
-		// also check for the platform ID since the address is pretty nebulous to check for
-		r = new buffers.BufferReader(fileBuffer);
-		r.skip(192+12);
-		var bluzModulesPlatformID = r.shiftUInt16(true);
-
-		//just system modules have the offset at the beginning.
-		// system module addresses always tend to start at 0x2xxxxxxx
-		// while user modules tend to start around 0x8xxxxxxx
-
-		// but any valid address right now should start with 80... something, since they're small / in the realm
-		// of the bootloader....  we'll need some extra sanity checks somehow later if hardware changes dramatically.
-		var mightBeUser = (userModuleStartAddy.indexOf("80") === 0) || (userModuleStartAddy === 'd4000');
-		var mightBeSystem = (sysModuleStartAddy.indexOf("80") === 0);
-		var isCore = ((userModuleStartAddy.indexOf("20") === 0) && (coreModuleStartAddy === "8005000"));
-		var mightBeBluz = (bluzModulesPlatformID == 103 && bluzModuleStartAddy < 262144);
-		var isGen3 = (meshModule.platformID >= 12 && meshModule.platformID <= 30) && (meshModule.moduleStartAddy === '30000' || meshModule.moduleStartAddy === 'f4000') &&
-			(userModuleStartAddy.indexOf("20") == 0);   // stack is located in ram at 0x2000000
-
-		if (isCore) {
-			return 0x10C;
+	_findModulePrefixOffset: function(fileBuffer) {
+		// This is simply an optimization to try a few preferred locations first
+		// 0x184 is the size of Gen 2 MCU (STM32F2) interrupt vector table
+		// 0x200 is the size of Gen 3 MCU (nRF52840) interrupt vector table
+		// FIXME: Some of the deprecated platforms should be removed
+		const preferredOffsets = [
+			// Module prefix is normally found right at the beginning of the binary
+			// at offset 0, unless there is a hardware requirement for something to be placed
+			// there instead (e.g. interrupt vector table)
+			0,
+			0x200, // Gen 3
+			0x184, // Gen 2
+			0xC0, // Bluez
+			0x10C, // Core
+		];
+
+		let likelyOffset = 0;
+		let likelyScore = 0;
+		for (let offset of preferredOffsets) {
+			const { score, passed } = this._validateModulePrefix(fileBuffer, offset);
+			if (passed) {
+				return offset;
+			}
+			if (score > likelyScore) {
+				likelyOffset = offset;
+				likelyScore = score;
+			}
 		}
-		else if (!mightBeUser && mightBeSystem) {
-			return 388;
-		} else if (mightBeUser && !mightBeSystem) {
-			return 0;
-		} else if (mightBeBluz) {
-			return 192;
-		} else if (isGen3) {
-			return 0x200;
-		} else {
-			return 0;
+
+		// Fallback to trying all (reasonable) offsets with 4 byte alignment
+		const maxModulePrefixOffset = 8 * 1024; // Anything above 8KB is unreasonable
+		const endAddress = Math.min(maxModulePrefixOffset,
+				fileBuffer.length - ModuleInfo.MIN_MODULE_SUFFIX_SIZE - ModuleInfo.MODULE_PREFIX_SIZE - 4 /* CRC-32 */);
+		for (let offset = 4; offset <= endAddress; offset += 4) {
+			// Just an optimization
+			if (preferredOffsets.includes(offset)) {
+				continue;
+			}
+			const { score, passed } = this._validateModulePrefix(fileBuffer, offset);
+			if (passed) {
+				return offset;
+			}
+			if (score > likelyScore) {
+				likelyOffset = offset;
+				likelyScore = score;
+			}
 		}
+
+		return likelyOffset;
 	},
 
 	/*
@@ -321,71 +394,83 @@ HalModuleParser.prototype = {
 	 *	moduleIndex: number,
 	 *	depModuleFunction: number,
 	 *	depModuleIndex: number,
-	 *	depModuleVersion: number
+	 *	depModuleVersion: number,
+	 *	dep2ModuleFunction: number,
+	 *	dep2ModuleIndex: number,
+	 *	dep2ModuleVersion: number
 	 * }
 	 * @param fileBuffer
 	 * @returns {{moduleStartAddy: string, moduleEndAddy: string, moduleFlags: number, moduleVersion: number, platformID: number, moduleFunction: number, moduleIndex: number, depModuleFunction: number, depModuleIndex: number, depModuleVersion: number}}
 	 * @private
 	 */
 	_readPrefix: function(fileBuffer) {
-		var prefixOffset = this._divineModulePrefixOffset(fileBuffer);
-		var r = new buffers.BufferReader(fileBuffer);
+		const prefixOffset = this._findModulePrefixOffset(fileBuffer);
 
-		// skip to system module offset, or stay at user module no offset
-		r.skip(prefixOffset);
-
-		var prefixInfo = this._parsePrefix(r);
+		const prefixInfo = this._parsePrefix(fileBuffer.slice(prefixOffset));
 		prefixInfo.prefixOffset = prefixOffset;
 		return prefixInfo;
 	},
 
 	_parsePrefix: function(r) {
-		var isLittleEndian = true;
 		//offset 0, 4-bytes: module start address
-		var moduleStartAddy = r.shiftUInt32(isLittleEndian).toString(16);
+		let offs = 0;
+		var moduleStartAddy = r.readUInt32LE(offs).toString(16);
+		offs += 4;
 
 		//offset 4, 4-bytes: module end address
-		var moduleEndAddy = r.shiftUInt32(isLittleEndian).toString(16);
+		var moduleEndAddy = r.readUInt32LE(offs).toString(16);
+		offs += 4;
 
 		// reserved (MCU target on Gen 3)
-		var reserved = r.shiftUInt8();
+		var reserved = r.readUInt8(offs);
+		offs += 1;
 
 		// offset 9, 1-byte: module flags (module_info_flags_t)
-		var moduleFlags = r.shiftUInt8();
+		var moduleFlags = r.readUInt8(offs);
+		offs += 1;
 
 		// offset 10, 2-bytes: module version (this is not the same as
 		// product version, it relates to the module export functions)
-		var moduleVersion = r.shiftUInt16(isLittleEndian);
+		var moduleVersion = r.readUInt16LE(offs);
+		offs += 2;
 
 		// offset 12, 2-bytes: Platform ID (6 for Photon)
-		var platformID = r.shiftUInt16(isLittleEndian);
+		var platformID = r.readUInt16LE(offs);
+		offs += 2;
 
 		// offset 14, 1-byte: module function (5 for user firmware)
-		var moduleFunction = r.shiftUInt8();
+		var moduleFunction = r.readUInt8(offs);
+		offs += 1;
 
 		// offset 15, 1-byte: module index (1 for user firmware)
-		var moduleIndex = r.shiftUInt8();
+		var moduleIndex = r.readUInt8(offs);
+		offs += 1;
 
 		// offset 16, 1-byte: dependency module function (usually system, 4)
-		var depModuleFunction = r.shiftUInt8(isLittleEndian);
+		var depModuleFunction = r.readUInt8(offs);
+		offs += 1;
 
 		// offset 17, 1-byte: dependency module index (usually 2, so
 		// dependency is system-part2)
-		var depModuleIndex = r.shiftUInt8(isLittleEndian);
+		var depModuleIndex = r.readUInt8(offs);
+		offs += 1;
 
 		// offset 18, 1-byte: minimum version of system dependency
-		var depModuleVersion = r.shiftUInt16(isLittleEndian);
+		var depModuleVersion = r.readUInt16LE(offs);
+		offs += 2;
 
 		// offset 20, 1-byte: dependency module function (usually system, 4)
-		var dep2ModuleFunction = r.shiftUInt8(isLittleEndian);
+		var dep2ModuleFunction = r.readUInt8(offs);
+		offs += 1;
 
 		// offset 21, 1-byte: dependency module index (usually 2, so
 		// dependency is system-part2)
-		var dep2ModuleIndex = r.shiftUInt8(isLittleEndian);
+		var dep2ModuleIndex = r.readUInt8(offs);
+		offs += 1;
 
 		// offset 22, 2-byte: minimum version of system dependency
-		var dep2ModuleVersion = r.shiftUInt16(isLittleEndian);
-
+		var dep2ModuleVersion = r.readUInt16LE(offs);
+		offs += 2;
 
 		return {
 			moduleStartAddy: moduleStartAddy,

package.json

@@ -8,7 +8,6 @@
   },
   "dependencies": {
     "buffer-crc32": "^0.2.5",
-    "h5.buffers": "^0.1.1",
     "when": "^3.7.3",
     "xtend": "^4.0.2"
   },