From b4e19ef5dc8bcba179c872b52cf8bc6e45ded523 Mon Sep 17 00:00:00 2001 From: Phoenix The Fallen Date: Tue, 23 Mar 2021 19:44:55 +0300 Subject: [PATCH 1/3] Move most functions out of closure --- lib/decoder.js | 355 ++++++++++++++++++++++++------------------------- 1 file changed, 177 insertions(+), 178 deletions(-) diff --git a/lib/decoder.js b/lib/decoder.js index a64130b..b342202 100644 --- a/lib/decoder.js +++ b/lib/decoder.js @@ -38,6 +38,7 @@ function isValidDataSize (dataLength, bufLength, headerLength) { } module.exports = function buildDecode (decodingTypes, options) { + const context = { decodingTypes, options, decode } return decode function decode (buf) { @@ -46,225 +47,223 @@ module.exports = function buildDecode (decodingTypes, options) { buf = bl().append(buf) } - const result = tryDecode(buf, 0) + const result = tryDecode(buf, 0, context) // Handle worst case ASAP and keep code flat if (!result) throw new IncompleteBufferError() buf.consume(result[1]) return result[0] } +} - function tryDecode (buf, initialOffset) { - if (buf.length <= initialOffset) return null - - const bufLength = buf.length - initialOffset - let offset = initialOffset +function decodeArray (buf, initialOffset, length, headerLength, context) { + let offset = initialOffset + const result = [] + let i = 0 - const first = buf.readUInt8(offset) - offset += 1 + while (i++ < length) { + const decodeResult = tryDecode(buf, offset, context) + if (!decodeResult) return null - const size = SIZES[first] || -1 - if (bufLength < size) return null + result.push(decodeResult[0]) + offset += decodeResult[1] + } + return [result, headerLength + offset - initialOffset] +} - const inRange = (start, end) => first >= start && first <= end +function decodeMap (buf, offset, length, headerLength, context) { + const _temp = decodeArray(buf, offset, 2 * length, headerLength, context) + if (!_temp) return null + const [result, consumedBytes] = _temp - if (first < 0x80) return [first, 1] // 7-bits positive ints - if ((first & 0xf0) === 0x80) { - const length = first & 0x0f - const headerSize = offset - initialOffset - // we have a map with less than 15 elements - return decodeMap(buf, offset, length, headerSize, options) - } - if ((first & 0xf0) === 0x90) { - const length = first & 0x0f - const headerSize = offset - initialOffset - // we have an array with less than 15 elements - return decodeArray(buf, offset, length, headerSize) - } + let isPlainObject = !context.options.preferMap - if ((first & 0xe0) === 0xa0) { - // fixstr up to 31 bytes - const length = first & 0x1f - if (!isValidDataSize(length, bufLength, 1)) return null - const result = buf.toString('utf8', offset, offset + length) - return [result, length + 1] - } - if (inRange(0xc0, 0xc3)) return decodeConstants(first) - if (inRange(0xc4, 0xc6)) { - const length = buf.readUIntBE(offset, size - 1) - offset += size - 1 - - if (!isValidDataSize(length, bufLength, size)) return null - const result = buf.slice(offset, offset + length) - return [result, size + length] + if (isPlainObject) { + for (let i = 0; i < 2 * length; i += 2) { + if (typeof result[i] !== 'string') { + isPlainObject = false + break + } } - if (inRange(0xc7, 0xc9)) { - const length = buf.readUIntBE(offset, size - 2) - offset += size - 2 + } - const type = buf.readInt8(offset) - offset += 1 + if (isPlainObject) { + const object = {} + for (let i = 0; i < 2 * length; i += 2) { + const key = result[i] + const val = result[i + 1] - if (!isValidDataSize(length, bufLength, size)) return null - return decodeExt(buf, offset, type, length, size) - } - if (inRange(0xca, 0xcb)) return decodeFloat(buf, offset, size - 1) - if (inRange(0xcc, 0xcf)) return decodeUnsignedInt(buf, offset, size - 1) - if (inRange(0xd0, 0xd3)) return decodeSigned(buf, offset, size - 1) - if (inRange(0xd4, 0xd8)) { - const type = buf.readInt8(offset) // Signed - offset += 1 - return decodeExt(buf, offset, type, size - 2, 2) - } + if (key === '__proto__') { + if (context.options.protoAction === 'error') { + throw new SyntaxError('Object contains forbidden prototype property') + } - if (inRange(0xd9, 0xdb)) { - const length = buf.readUIntBE(offset, size - 1) - offset += size - 1 + if (context.options.protoAction === 'remove') { + continue + } + } - if (!isValidDataSize(length, bufLength, size)) return null - const result = buf.toString('utf8', offset, offset + length) - return [result, size + length] + object[key] = val } - if (inRange(0xdc, 0xdd)) { - const length = buf.readUIntBE(offset, size - 1) - offset += size - 1 - return decodeArray(buf, offset, length, size) - } - if (inRange(0xde, 0xdf)) { - let length - switch (first) { - case 0xde: - // maps up to 2^16 elements - 2 bytes - length = buf.readUInt16BE(offset) - offset += 2 - // console.log(offset - initialOffset) - return decodeMap(buf, offset, length, 3, options) - - case 0xdf: - length = buf.readUInt32BE(offset) - offset += 4 - return decodeMap(buf, offset, length, 5, options) - } + return [object, consumedBytes] + } else { + const mapping = new Map() + for (let i = 0; i < 2 * length; i += 2) { + const key = result[i] + const val = result[i + 1] + mapping.set(key, val) } - if (first >= 0xe0) return [first - 0x100, 1] // 5 bits negative ints - - throw new Error('not implemented yet') + return [mapping, consumedBytes] } +} - function decodeArray (buf, initialOffset, length, headerLength) { - let offset = initialOffset - const result = [] - let i = 0 +function tryDecode (buf, initialOffset, context) { + if (buf.length <= initialOffset) return null - while (i++ < length) { - const decodeResult = tryDecode(buf, offset) - if (!decodeResult) return null + const bufLength = buf.length - initialOffset + let offset = initialOffset - result.push(decodeResult[0]) - offset += decodeResult[1] - } - return [result, headerLength + offset - initialOffset] - } + const first = buf.readUInt8(offset) + offset += 1 - function decodeMap (buf, offset, length, headerLength, options) { - const _temp = decodeArray(buf, offset, 2 * length, headerLength) - if (!_temp) return null - const [result, consumedBytes] = _temp + const size = SIZES[first] || -1 + if (bufLength < size) return null - let isPlainObject = !options.preferMap + if (first < 0x80) return [first, 1] // 7-bits positive ints + if ((first & 0xf0) === 0x80) { + const length = first & 0x0f + const headerSize = offset - initialOffset + // we have a map with less than 15 elements + return decodeMap(buf, offset, length, headerSize, context) + } + if ((first & 0xf0) === 0x90) { + const length = first & 0x0f + const headerSize = offset - initialOffset + // we have an array with less than 15 elements + return decodeArray(buf, offset, length, headerSize) + } - if (isPlainObject) { - for (let i = 0; i < 2 * length; i += 2) { - if (typeof result[i] !== 'string') { - isPlainObject = false - break - } - } - } + if ((first & 0xe0) === 0xa0) { + // fixstr up to 31 bytes + const length = first & 0x1f + if (!isValidDataSize(length, bufLength, 1)) return null + const result = buf.toString('utf8', offset, offset + length) + return [result, length + 1] + } + if (first >= 0xc0 && first <= 0xc3) return decodeConstants(first) + if (first >= 0xc4 && first <= 0xc6) { + const length = buf.readUIntBE(offset, size - 1) + offset += size - 1 + + if (!isValidDataSize(length, bufLength, size)) return null + const result = buf.slice(offset, offset + length) + return [result, size + length] + } + if (first >= 0xc7 && first <= 0xc9) { + const length = buf.readUIntBE(offset, size - 2) + offset += size - 2 - if (isPlainObject) { - const object = {} - for (let i = 0; i < 2 * length; i += 2) { - const key = result[i] - const val = result[i + 1] + const type = buf.readInt8(offset) + offset += 1 - if (key === '__proto__') { - if (options.protoAction === 'error') { - throw new SyntaxError('Object contains forbidden prototype property') - } + if (!isValidDataSize(length, bufLength, size)) return null + return decodeExt(buf, offset, type, length, size, context) + } + if (first >= 0xca && first <= 0xcb) return decodeFloat(buf, offset, size - 1) + if (first >= 0xcc && first <= 0xcf) return decodeUnsignedInt(buf, offset, size - 1) + if (first >= 0xd0 && first <= 0xd3) return decodeSigned(buf, offset, size - 1) + if (first >= 0xd4 && first <= 0xd8) { + const type = buf.readInt8(offset) // Signed + offset += 1 + return decodeExt(buf, offset, type, size - 2, 2, context) + } - if (options.protoAction === 'remove') { - continue - } - } + if (first >= 0xd9 && first <= 0xdb) { + const length = buf.readUIntBE(offset, size - 1) + offset += size - 1 - object[key] = val - } - return [object, consumedBytes] - } else { - const mapping = new Map() - for (let i = 0; i < 2 * length; i += 2) { - const key = result[i] - const val = result[i + 1] - mapping.set(key, val) - } - return [mapping, consumedBytes] + if (!isValidDataSize(length, bufLength, size)) return null + const result = buf.toString('utf8', offset, offset + length) + return [result, size + length] + } + if (first >= 0xdc && first <= 0xdd) { + const length = buf.readUIntBE(offset, size - 1) + offset += size - 1 + return decodeArray(buf, offset, length, size) + } + if (first >= 0xde && first <= 0xdf) { + let length + switch (first) { + case 0xde: + // maps up to 2^16 elements - 2 bytes + length = buf.readUInt16BE(offset) + offset += 2 + // console.log(offset - initialOffset) + return decodeMap(buf, offset, length, 3, context) + + case 0xdf: + length = buf.readUInt32BE(offset) + offset += 4 + return decodeMap(buf, offset, length, 5, context) } } + if (first >= 0xe0) return [first - 0x100, 1] // 5 bits negative ints - function readInt64BE (buf, offset) { - var negate = (buf[offset] & 0x80) == 0x80; // eslint-disable-line + throw new Error('not implemented yet') +} - if (negate) { - let carry = 1 - for (let i = offset + 7; i >= offset; i--) { - const v = (buf[i] ^ 0xff) + carry - buf[i] = v & 0xff - carry = v >> 8 - } - } +function decodeSigned (buf, offset, size) { + let result + if (size === 1) result = buf.readInt8(offset) + if (size === 2) result = buf.readInt16BE(offset) + if (size === 4) result = buf.readInt32BE(offset) + if (size === 8) result = readInt64BE(buf.slice(offset, offset + 8), 0) + return [result, size + 1] +} - const hi = buf.readUInt32BE(offset + 0) - const lo = buf.readUInt32BE(offset + 4) - return (hi * 4294967296 + lo) * (negate ? -1 : +1) - } +function decodeExt (buf, offset, type, size, headerSize, context) { + const toDecode = buf.slice(offset, offset + size) - function decodeUnsignedInt (buf, offset, size) { - const maxOffset = offset + size - let result = 0 - while (offset < maxOffset) { result += buf.readUInt8(offset++) * Math.pow(256, maxOffset - offset) } - return [result, size + 1] - } + const decode = context.decodingTypes.get(type) + if (!decode) throw new Error('unable to find ext type ' + type) - function decodeConstants (first) { - if (first === 0xc0) return [null, 1] - if (first === 0xc2) return [false, 1] - if (first === 0xc3) return [true, 1] - } + const value = decode(toDecode) + return [value, headerSize + size] +} - function decodeSigned (buf, offset, size) { - let result - if (size === 1) result = buf.readInt8(offset) - if (size === 2) result = buf.readInt16BE(offset) - if (size === 4) result = buf.readInt32BE(offset) - if (size === 8) result = readInt64BE(buf.slice(offset, offset + 8), 0) - return [result, size + 1] - } +function decodeUnsignedInt (buf, offset, size) { + const maxOffset = offset + size + let result = 0 + while (offset < maxOffset) { result += buf.readUInt8(offset++) * Math.pow(256, maxOffset - offset) } + return [result, size + 1] +} - function decodeFloat (buf, offset, size) { - let result - if (size === 4) result = buf.readFloatBE(offset) - if (size === 8) result = buf.readDoubleBE(offset) - return [result, size + 1] - } +function decodeConstants (first) { + if (first === 0xc0) return [null, 1] + if (first === 0xc2) return [false, 1] + if (first === 0xc3) return [true, 1] +} - function decodeExt (buf, offset, type, size, headerSize) { - const toDecode = buf.slice(offset, offset + size) +function decodeFloat (buf, offset, size) { + let result + if (size === 4) result = buf.readFloatBE(offset) + if (size === 8) result = buf.readDoubleBE(offset) + return [result, size + 1] +} - const decode = decodingTypes.get(type) - if (!decode) throw new Error('unable to find ext type ' + type) +function readInt64BE (buf, offset) { + var negate = (buf[offset] & 0x80) == 0x80; // eslint-disable-line - const value = decode(toDecode) - return [value, headerSize + size] + if (negate) { + let carry = 1 + for (let i = offset + 7; i >= offset; i--) { + const v = (buf[i] ^ 0xff) + carry + buf[i] = v & 0xff + carry = v >> 8 + } } + + const hi = buf.readUInt32BE(offset + 0) + const lo = buf.readUInt32BE(offset + 4) + return (hi * 4294967296 + lo) * (negate ? -1 : +1) } From f90c539f41dbea6980c6a84b9eb713e8205351a4 Mon Sep 17 00:00:00 2001 From: Phoenix The Fallen Date: Tue, 23 Mar 2021 20:29:11 +0300 Subject: [PATCH 2/3] Fix regression --- lib/decoder.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/decoder.js b/lib/decoder.js index b342202..ecb0ff7 100644 --- a/lib/decoder.js +++ b/lib/decoder.js @@ -140,7 +140,7 @@ function tryDecode (buf, initialOffset, context) { const length = first & 0x0f const headerSize = offset - initialOffset // we have an array with less than 15 elements - return decodeArray(buf, offset, length, headerSize) + return decodeArray(buf, offset, length, headerSize, context) } if ((first & 0xe0) === 0xa0) { @@ -189,7 +189,7 @@ function tryDecode (buf, initialOffset, context) { if (first >= 0xdc && first <= 0xdd) { const length = buf.readUIntBE(offset, size - 1) offset += size - 1 - return decodeArray(buf, offset, length, size) + return decodeArray(buf, offset, length, size, context) } if (first >= 0xde && first <= 0xdf) { let length From 11176c39a8766c6a0733acad6c251fa8ee4d55cb Mon Sep 17 00:00:00 2001 From: Phoenix The Fallen Date: Wed, 24 Mar 2021 17:43:25 +0300 Subject: [PATCH 3/3] Add possible regression test --- test/nested-containers.js | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 test/nested-containers.js diff --git a/test/nested-containers.js b/test/nested-containers.js new file mode 100644 index 0000000..0ffcbd3 --- /dev/null +++ b/test/nested-containers.js @@ -0,0 +1,44 @@ +'use strict' + +const test = require('tape').test +const msgpack = require('../') + +test('encode/decode nested containers (map/array)', function (t) { + const encoder = msgpack() + + function doEncodeDecode (value) { + return encoder.decode(encoder.encode(value)) + } + + function preserveTest (A, message = 'works') { + const B = doEncodeDecode(A) + t.deepEqual(A, B, message) + } + + preserveTest({ + hello: 'world', + digit: 111, + array: [1, 2, 3, 4, 'string', { hello: 'world' }] + }) + + preserveTest([ + [ + { + hello: 'world', + array: [1, 2, 3, 4, 'string', { hello: 'world' }] + }, + { + digit: 111 + } + ], + [ + { + hello: 'world', + digit: 111, + array: [1, 2, 3, 4, 'string', { hello: 'world' }] + } + ] + ]) + + t.end() +})