Skip to content
This repository was archived by the owner on Oct 12, 2022. It is now read-only.

Commit 2fba8d3

Browse files
n8shn8sh
committed
Fix Issue 21006 - core.internal.hash.bytesHash: in 64-bit builds use a 64-bit-oriented algorithm
1 parent 104ac71 commit 2fba8d3

File tree

1 file changed

+120
-14
lines changed

1 file changed

+120
-14
lines changed

‎src/core/internal/hash.d

+120-14
Original file line numberDiff line numberDiff line change
@@ -623,14 +623,21 @@ size_t bytesHash()(scope const(void)* buf, size_t len, size_t seed)
623623

624624
private template bytesHashAlignedBy(AlignType)
625625
{
626-
alias bytesHashAlignedBy = bytesHash!(AlignType.alignof >= uint.alignof);
626+
static if (size_t.sizeof == 4)
627+
alias bytesHashAlignedBy = bytesHash32!(
628+
AlignType.alignof >= uint.alignof ? uint.alignof :
629+
ubyte.alignof);
630+
else
631+
alias bytesHashAlignedBy = bytesHash64!(
632+
AlignType.alignof >= ulong.alignof ? ulong.alignof :
633+
AlignType.alignof >= uint.alignof ? uint.alignof :
634+
ubyte.alignof);
627635
}
628636

629637
private template bytesHashWithExactSizeAndAlignment(SizeAndAlignType)
630638
{
631-
static if (SizeAndAlignType.alignof < uint.alignof
632-
? SizeAndAlignType.sizeof <= 12
633-
: SizeAndAlignType.sizeof <= 10)
639+
static if (SizeAndAlignType.sizeof <= 3 || size_t.sizeof <= 4 &&
640+
SizeAndAlignType.sizeof <= (SizeAndAlignType.alignof < uint.alignof ? 12 : 10))
634641
alias bytesHashWithExactSizeAndAlignment = smallBytesHash;
635642
else
636643
alias bytesHashWithExactSizeAndAlignment = bytesHashAlignedBy!SizeAndAlignType;
@@ -666,13 +673,36 @@ private uint get32bits()(scope const(ubyte)* x) @nogc nothrow pure @system
666673
}
667674
}
668675

669-
/+
670-
Params:
671-
dataKnownToBeAligned = whether the data is known at compile time to be uint-aligned.
672-
+/
676+
private ulong get64bits()(scope const(ubyte)* x) @nogc nothrow pure @system
677+
{
678+
version (BigEndian)
679+
{
680+
return ((cast(ulong) x[0]) << 56) |
681+
((cast(ulong) x[1]) << 48) |
682+
((cast(ulong) x[2]) << 40) |
683+
((cast(ulong) x[3]) << 32) |
684+
((cast(ulong) x[4]) << 24) |
685+
((cast(ulong) x[5]) << 16) |
686+
((cast(ulong) x[6]) << 8) |
687+
(cast(ulong) x[7]); }
688+
else
689+
{
690+
return ((cast(ulong) x[7]) << 56) |
691+
((cast(ulong) x[6]) << 48) |
692+
((cast(ulong) x[5]) << 40) |
693+
((cast(ulong) x[4]) << 32) |
694+
((cast(ulong) x[3]) << 24) |
695+
((cast(ulong) x[2]) << 16) |
696+
((cast(ulong) x[1]) << 8) |
697+
(cast(ulong) x[0]);
698+
}
699+
}
700+
701+
static if (size_t.sizeof == 4)
673702
@nogc nothrow pure @trusted
674-
private size_t bytesHash(bool dataKnownToBeAligned)(scope const(ubyte)[] bytes, size_t seed)
703+
private uint bytesHash32(uint alignment)(scope const(ubyte)[] bytes, size_t seed)
675704
{
705+
// MurmurHash3_x86_32.
676706
auto len = bytes.length;
677707
auto data = bytes.ptr;
678708
auto nblocks = len / 4;
@@ -688,10 +718,12 @@ private size_t bytesHash(bool dataKnownToBeAligned)(scope const(ubyte)[] bytes,
688718
auto end_data = data+nblocks*uint.sizeof;
689719
for (; data!=end_data; data += uint.sizeof)
690720
{
691-
static if (dataKnownToBeAligned)
721+
static if (alignment == uint.alignof)
692722
uint k1 = __ctfe ? get32bits(data) : *(cast(const uint*) data);
693-
else
723+
else static if (alignment == ubyte.alignof)
694724
uint k1 = get32bits(data);
725+
else
726+
static assert(0, "Do not create unnecessary template instantiations.");
695727
k1 *= c1;
696728
k1 = (k1 << 15) | (k1 >> (32 - 15));
697729
k1 *= c2;
@@ -725,6 +757,74 @@ private size_t bytesHash(bool dataKnownToBeAligned)(scope const(ubyte)[] bytes,
725757
return h1;
726758
}
727759

760+
static if (size_t.sizeof == 8)
761+
@nogc nothrow pure @trusted
762+
private ulong bytesHash64(uint alignment)(scope const ubyte[] bytes, ulong seed)
763+
{
764+
// MurmurHash3_x86_32 modified to be 64-bit using constants from MurmurHash3_x64_128.
765+
alias h1 = seed;
766+
767+
enum ulong c1 = 0x87c37b91114253d5;
768+
enum ulong c2 = 0x4cf5ad432745937f;
769+
enum uint c3 = 0x52dce729;
770+
771+
const(ubyte)* data = bytes.ptr;
772+
//----------
773+
// body
774+
for (const end_data = bytes.ptr + (bytes.length & ~7);
775+
data !is end_data;
776+
data += 8)
777+
{
778+
static if (alignment == ulong.alignof)
779+
auto k1 = __ctfe ? get64bits(data) : *cast(ulong*) data;
780+
else static if (alignment == uint.alignof)
781+
{
782+
version (BigEndian)
783+
auto k1 = __ctfe ? get64bits(data) : (((cast(ulong) *cast(uint*) data) << 32) | *cast(uint*) (data + 4));
784+
else
785+
auto k1 = __ctfe ? get64bits(data) : (((cast(ulong) *cast(uint*) (data + 4)) << 32) | *cast(uint*) data);
786+
}
787+
else static if (alignment == ubyte.alignof)
788+
auto k1 = get64bits(data);
789+
else
790+
static assert(0, "Do not create unnecessary template instantiations.");
791+
792+
k1 *= c1;
793+
k1 = (k1 << 31) | (k1 >> (64 - 31));
794+
k1 *= c2;
795+
796+
h1 ^= k1;
797+
h1 = (h1 << 27) | (h1 >> (64 - 27));
798+
h1 = h1*5+c3;
799+
}
800+
801+
//----------
802+
// tail
803+
ulong k1 = 0;
804+
805+
switch (bytes.length & 7)
806+
{
807+
case 7: k1 ^= (cast(ulong) data[6]) << 48; goto case;
808+
case 6: k1 ^= (cast(ulong) data[5]) << 40; goto case;
809+
case 5: k1 ^= (cast(ulong) data[4]) << 32; goto case;
810+
case 4: k1 ^= (cast(ulong) data[3]) << 24; goto case;
811+
case 3: k1 ^= (cast(ulong) data[2]) << 16; goto case;
812+
case 2: k1 ^= (cast(ulong) data[1]) << 8; goto case;
813+
case 1: k1 ^= (cast(ulong) data[0]);
814+
k1 *= c1; k1 = (k1 << 31) | (k1 >> (64 - 31)); k1 *= c2; h1 ^= k1;
815+
goto default;
816+
default:
817+
}
818+
819+
//----------
820+
// finalization
821+
h1 ^= bytes.length;
822+
// Force all bits of the hash block to avalanche.
823+
h1 = (h1 ^ (h1 >> 33)) * 0xff51afd7ed558ccd;
824+
h1 = (h1 ^ (h1 >> 33)) * 0xc4ceb9fe1a85ec53;
825+
return h1 ^= h1 >> 33;
826+
}
827+
728828
// Check that bytesHash works with CTFE
729829
pure nothrow @system @nogc unittest
730830
{
@@ -742,15 +842,21 @@ pure nothrow @system @nogc unittest
742842
{
743843
const ubyte[7] a = [99, 4, 3, 2, 1, 5, 88];
744844
const uint[2] b = [0x04_03_02_01, 0x05_ff_ff_ff];
845+
const ulong[1] c = [0x04_03_02_01_05_ff_ff_ff];
745846
}
746847
else
747848
{
748849
const ubyte[7] a = [99, 1, 2, 3, 4, 5, 88];
749850
const uint[2] b = [0x04_03_02_01, 0xff_ff_ff_05];
851+
const ulong[1] c = [0xff_ff_ff_05_04_03_02_01];
750852
}
751853
// It is okay to change the below values if you make a change
752854
// that you expect to change the result of bytesHash.
753-
assert(bytesHash(&a[1], a.length - 2, 0) == 2727459272);
754-
assert(bytesHash(&b, 5, 0) == 2727459272);
755-
assert(bytesHashAlignedBy!uint((cast(const ubyte*) &b)[0 .. 5], 0) == 2727459272);
855+
enum expectedResult = size_t.sizeof == 4 ? 2727459272U : 10677742034643552556UL;
856+
assert(bytesHash(&a[1], a.length - 2, 0) == expectedResult);
857+
assert(bytesHash(&b, 5, 0) == expectedResult);
858+
assert(bytesHashAlignedBy!uint((cast(const ubyte*) &b)[0 .. 5], 0) == expectedResult);
859+
assert(bytesHashAlignedBy!ulong((cast(const ubyte*) &c)[0 .. 5], 0) == expectedResult);
860+
assert(bytesHashAlignedBy!ulong((cast(const ubyte*) &c)[0 .. c.sizeof], 0) ==
861+
(size_t.sizeof == 4 ? 2948526704 : 7625915911016357963));
756862
}

0 commit comments

Comments
 (0)