From 165e3d524f6585e5913a45a4e7b99f3033964c2a Mon Sep 17 00:00:00 2001 From: Brandon Dahler Date: Sat, 5 Sep 2015 16:24:49 -0500 Subject: [PATCH] Issue #16 - xxHash Deviations xxHash calculated incorrect valuex when in 64-bit mode and the input satisfies: length % 8 == 4. xxHash calculated incorrect values when in either mode and the input length was greater than or equal to 2^31 characters in length. Both of these issues cause breaking changes to any current users of xxHash given the value(s) could satisfy those constraints. --- src/SolutionInfo.cs | 2 +- .../IHashFunctionTests_HashFunctionBase.cs | 5 +++- .../IHashFunctionTests_HashFunctionBase.json | 14 +++++---- src/xxHash/xxHash.cs | 30 +++++++++---------- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index 18d6eaa..d66f328 100755 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -10,5 +10,5 @@ [assembly: CLSCompliant(false)] #if !CUSTOM_VERSIONING -[assembly: AssemblyVersion("1.6.*")] +[assembly: AssemblyVersion("1.7.*")] #endif \ No newline at end of file diff --git a/src/Test/IHashFunctionTests_HashFunctionBase.cs b/src/Test/IHashFunctionTests_HashFunctionBase.cs index 2437146..3f865b9 100755 --- a/src/Test/IHashFunctionTests_HashFunctionBase.cs +++ b/src/Test/IHashFunctionTests_HashFunctionBase.cs @@ -1,7 +1,6 @@ //! Automatically generated from IHashFunctionTests_HashFunctionBase.tt //! Direct modifications to this file will be lost. -using Moq; using System; using System.Collections.Generic; using System.Data.HashFunction.Test.Mocks; @@ -9,6 +8,8 @@ using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; +using Moq; +using MoreLinq; using Xunit; namespace System.Data.HashFunction.Test @@ -2088,9 +2089,11 @@ protected override IEnumerable KnownValues new KnownValue(32, TestConstants.Empty, 0x02cc5d05), new KnownValue(32, TestConstants.FooBar, 0xeda34aaf), new KnownValue(32, TestConstants.LoremIpsum, 0x92ea46ac), + new KnownValue(32, TestConstants.LoremIpsum.Take(4), 0x0df3e9ea), new KnownValue(64, TestConstants.Empty, 0xef46db3751d8e999), new KnownValue(64, TestConstants.FooBar, 0xa2aa05ed9085aaf9), new KnownValue(64, TestConstants.LoremIpsum, 0xaf35642971419cbe), + new KnownValue(64, TestConstants.LoremIpsum.Take(4), 0x103460bb4a599cab), }; } } diff --git a/src/Test/IHashFunctionTests_HashFunctionBase.json b/src/Test/IHashFunctionTests_HashFunctionBase.json index 85d76ef..96da099 100755 --- a/src/Test/IHashFunctionTests_HashFunctionBase.json +++ b/src/Test/IHashFunctionTests_HashFunctionBase.json @@ -1416,12 +1416,14 @@ "Async": true, "ConstructionParameters": [ "hashSize" ], "KnownValues": [ - { "Bits": 32, "TestValue": "TestConstants.Empty", "ExpectedValue": "0x02cc5d05" }, - { "Bits": 32, "TestValue": "TestConstants.FooBar", "ExpectedValue": "0xeda34aaf" }, - { "Bits": 32, "TestValue": "TestConstants.LoremIpsum", "ExpectedValue": "0x92ea46ac" }, - { "Bits": 64, "TestValue": "TestConstants.Empty", "ExpectedValue": "0xef46db3751d8e999" }, - { "Bits": 64, "TestValue": "TestConstants.FooBar", "ExpectedValue": "0xa2aa05ed9085aaf9" }, - { "Bits": 64, "TestValue": "TestConstants.LoremIpsum", "ExpectedValue": "0xaf35642971419cbe" } + { "Bits": 32, "TestValue": "TestConstants.Empty", "ExpectedValue": "0x02cc5d05" }, + { "Bits": 32, "TestValue": "TestConstants.FooBar", "ExpectedValue": "0xeda34aaf" }, + { "Bits": 32, "TestValue": "TestConstants.LoremIpsum", "ExpectedValue": "0x92ea46ac" }, + { "Bits": 32, "TestValue": "TestConstants.LoremIpsum.Take(4)", "ExpectedValue": "0x0df3e9ea" }, + { "Bits": 64, "TestValue": "TestConstants.Empty", "ExpectedValue": "0xef46db3751d8e999" }, + { "Bits": 64, "TestValue": "TestConstants.FooBar", "ExpectedValue": "0xa2aa05ed9085aaf9" }, + { "Bits": 64, "TestValue": "TestConstants.LoremIpsum", "ExpectedValue": "0xaf35642971419cbe" }, + { "Bits": 64, "TestValue": "TestConstants.LoremIpsum.Take(4)", "ExpectedValue": "0x103460bb4a599cab" } ] }, diff --git a/src/xxHash/xxHash.cs b/src/xxHash/xxHash.cs index 0b137e9..d746ea7 100755 --- a/src/xxHash/xxHash.cs +++ b/src/xxHash/xxHash.cs @@ -130,7 +130,7 @@ protected override byte[] ComputeHashInternal(UnifiedData data) { var h = ((UInt32) InitVal) + _primes32[4]; - int dataCount = 0; + ulong dataCount = 0; byte[] remainder = null; @@ -153,13 +153,13 @@ protected override byte[] ComputeHashInternal(UnifiedData data) } } - dataCount += length; + dataCount += (ulong)length; }, (remainderData, position, length) => { remainder = new byte[length]; Array.Copy(remainderData, position, remainder, 0, length); - dataCount += length; + dataCount += (ulong)length; }); @@ -173,7 +173,7 @@ protected override byte[] ComputeHashInternal(UnifiedData data) { var h = InitVal + _primes64[4]; - int dataCount = 0; + ulong dataCount = 0; byte[] remainder = null; var initValues = new[] { @@ -197,13 +197,13 @@ protected override byte[] ComputeHashInternal(UnifiedData data) } } - dataCount += length; + dataCount += (ulong) length; }, (remainderData, position, length) => { remainder = new byte[length]; Array.Copy(remainderData, position, remainder, 0, length); - dataCount += length; + dataCount += (ulong) length; }); @@ -230,7 +230,7 @@ protected override async Task ComputeHashAsyncInternal(UnifiedData data) { var h = ((UInt32) InitVal) + _primes32[4]; - int dataCount = 0; + ulong dataCount = 0; byte[] remainder = null; @@ -253,13 +253,13 @@ await data.ForEachGroupAsync(16, } } - dataCount += length; + dataCount += (ulong) length; }, (remainderData, position, length) => { remainder = new byte[length]; Array.Copy(remainderData, position, remainder, 0, length); - dataCount += length; + dataCount += (ulong) length; }).ConfigureAwait(false); PostProcess(ref h, initValues, dataCount, remainder); @@ -272,7 +272,7 @@ await data.ForEachGroupAsync(16, { var h = InitVal + _primes64[4]; - int dataCount = 0; + ulong dataCount = 0; byte[] remainder = null; var initValues = new[] { @@ -295,13 +295,13 @@ await data.ForEachGroupAsync(32, } } - dataCount += length; + dataCount += (ulong) length; }, (remainderData, position, length) => { remainder = new byte[length]; Array.Copy(remainderData, position, remainder, 0, length); - dataCount += remainder.Length; + dataCount += (ulong) remainder.Length; }).ConfigureAwait(false); @@ -320,7 +320,7 @@ await data.ForEachGroupAsync(32, #if !NET40 [MethodImpl(MethodImplOptions.AggressiveInlining)] #endif - private static void PostProcess(ref UInt32 h, UInt32[] initValues, int dataCount, byte[] remainder) + private static void PostProcess(ref UInt32 h, UInt32[] initValues, ulong dataCount, byte[] remainder) { if (dataCount >= 16) { @@ -361,7 +361,7 @@ private static void PostProcess(ref UInt32 h, UInt32[] initValues, int dataCount #if !NET40 [MethodImpl(MethodImplOptions.AggressiveInlining)] #endif - private static void PostProcess(ref UInt64 h, UInt64[] initValues, int dataCount, byte[] remainder) + private static void PostProcess(ref UInt64 h, UInt64[] initValues, ulong dataCount, byte[] remainder) { if (dataCount >= 32) { @@ -395,7 +395,7 @@ private static void PostProcess(ref UInt64 h, UInt64[] initValues, int dataCount // Process a 4-byte chunk if it exists - if ((remainder.Length % 8) > 4) + if ((remainder.Length % 8) >= 4) { h ^= ((UInt64) BitConverter.ToUInt32(remainder, remainder.Length - (remainder.Length % 8))) * _primes64[0]; h = (h.RotateLeft(23) * _primes64[1]) + _primes64[2];