Skip to content

Commit

Permalink
runtime: refactor cpu feature detection for 386 & amd64
Browse files Browse the repository at this point in the history 
Changes all cpu features to be detected and stored in bools in rt0_go.

Updates: #15403

Change-Id: I5a9961cdec789b331d09c44d86beb53833d5dc3e
Reviewed-on: https://go-review.googlesource.com/41950
Run-TryBot: Martin Möhrmann <moehrmann@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ilya Tocar <ilya.tocar@intel.com>
Reviewed-by: Keith Randall <khr@golang.org>
  • Loading branch information
@martisch
martisch committed May 1, 2017
1 parent 1f85d3a commit 5a6c580
Show file tree
Hide file tree
Showing 9 changed files with 243 additions and 94 deletions.
6 changes: 3 additions & 3 deletions src/runtime/alg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,9 +283,9 @@ func alginit() {
// Install aes hash algorithm if we have the instructions we need
if (GOARCH == "386" || GOARCH == "amd64") &&
GOOS != "nacl" &&
cpuid_ecx&(1<<25) != 0 && // aes (aesenc)
cpuid_ecx&(1<<9) != 0 && // sse3 (pshufb)
cpuid_ecx&(1<<19) != 0 { // sse4.1 (pinsr{d,q})
support_aes && // AESENC
support_ssse3 && // PSHUFB
support_sse41 { // PINSR{D,Q}
useAeshash = true
algarray[alg_MEM32].hash = aeshash32
algarray[alg_MEM64].hash = aeshash64
Expand Down
79 changes: 68 additions & 11 deletions src/runtime/asm_386.s
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,24 +75,81 @@ notintel:
MOVL $1, AX
CPUID
MOVL CX, DI // Move to global variable clobbers CX when generating PIC
MOVL AX, runtime·cpuid_eax(SB)
MOVL AX, runtime·processorVersionInfo(SB)
MOVL DI, runtime·cpuid_ecx(SB)
MOVL DX, runtime·cpuid_edx(SB)

// Check for MMX support
TESTL $(1<<23), DX // MMX
JZ bad_proc
TESTL $(1<<23), DX // MMX
JZ bad_proc

TESTL $(1<<26), DX // SSE2
SETNE runtime·support_sse2(SB)

TESTL $(1<<9), DI // SSSE3
SETNE runtime·support_ssse3(SB)

TESTL $(1<<19), DI // SSE4.1
SETNE runtime·support_sse41(SB)

TESTL $(1<<20), DI // SSE4.2
SETNE runtime·support_sse42(SB)

TESTL $(1<<23), DI // POPCNT
SETNE runtime·support_popcnt(SB)

TESTL $(1<<25), DI // AES
SETNE runtime·support_aes(SB)

TESTL $(1<<27), DI // OSXSAVE
SETNE runtime·support_osxsave(SB)

// If OS support for XMM and YMM is not present
// support_avx will be set back to false later.
TESTL $(1<<28), DI // AVX
SETNE runtime·support_avx(SB)

eax7:
// Load EAX=7/ECX=0 cpuid flags
CMPL SI, $7
JLT nocpuinfo
JLT osavx
MOVL $7, AX
MOVL $0, CX
CPUID
MOVL BX, runtime·cpuid_ebx7(SB)

nocpuinfo:
TESTL $(1<<3), BX // BMI1
SETNE runtime·support_bmi1(SB)

// If OS support for XMM and YMM is not present
// support_avx2 will be set back to false later.
TESTL $(1<<5), BX
SETNE runtime·support_avx2(SB)

TESTL $(1<<8), BX // BMI2
SETNE runtime·support_bmi2(SB)

TESTL $(1<<9), BX // ERMS
SETNE runtime·support_erms(SB)

osavx:
// nacl does not support XGETBV to test
// for XMM and YMM OS support.
#ifndef GOOS_nacl
CMPB runtime·support_osxsave(SB), $1
JNE noavx
MOVL $0, CX
// For XGETBV, OSXSAVE bit is required and sufficient
XGETBV
ANDL $6, AX
CMPL AX, $6 // Check for OS support of XMM and YMM registers.
JE nocpuinfo
#endif
noavx:
MOVB $0, runtime·support_avx(SB)
MOVB $0, runtime·support_avx2(SB)

nocpuinfo:
// if there is an _cgo_init, call it to let it
// initialize and to set up GS. if not,
// we set up GS ourselves.
Expand Down Expand Up @@ -803,8 +860,8 @@ TEXT runtime·getcallerpc(SB),NOSPLIT,$4-8

// func cputicks() int64
TEXT runtime·cputicks(SB),NOSPLIT,$0-8
TESTL $0x4000000, runtime·cpuid_edx(SB) // no sse2, no mfence
JEQ done
CMPB runtime·support_sse2(SB), $1
JNE done
CMPB runtime·lfenceBeforeRdtsc(SB), $1
JNE mfence
BYTE $0x0f; BYTE $0xae; BYTE $0xe8 // LFENCE
Expand Down Expand Up @@ -1311,8 +1368,8 @@ TEXT runtime·memeqbody(SB),NOSPLIT,$0-0
hugeloop:
CMPL BX, $64
JB bigloop
TESTL $0x4000000, runtime·cpuid_edx(SB) // check for sse2
JE bigloop
CMPB runtime·support_sse2(SB), $1
JNE bigloop
MOVOU (SI), X0
MOVOU (DI), X1
MOVOU 16(SI), X2
Expand Down Expand Up @@ -1455,8 +1512,8 @@ TEXT runtime·cmpbody(SB),NOSPLIT,$0-0
JEQ allsame
CMPL BP, $4
JB small
TESTL $0x4000000, runtime·cpuid_edx(SB) // check for sse2
JE mediumloop
CMPB runtime·support_sse2(SB), $1
JNE mediumloop
largeloop:
CMPL BP, $16
JB mediumloop
Expand Down
112 changes: 62 additions & 50 deletions src/runtime/asm_amd64.s
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$0
MOVQ SP, (g_stack+stack_hi)(DI)

// find out information about the processor we're on
MOVQ $0, AX
MOVL $0, AX
CPUID
MOVQ AX, SI
CMPQ AX, $0
MOVL AX, SI
CMPL AX, $0
JE nocpuinfo

// Figure out how to serialize RDTSC.
Expand All @@ -46,62 +46,75 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$0
notintel:

// Load EAX=1 cpuid flags
MOVQ $1, AX
MOVL $1, AX
CPUID
MOVL AX, runtime·cpuid_eax(SB)
MOVL AX, runtime·processorVersionInfo(SB)
MOVL CX, runtime·cpuid_ecx(SB)
MOVL DX, runtime·cpuid_edx(SB)

TESTL $(1<<26), DX // SSE2
SETNE runtime·support_sse2(SB)

TESTL $(1<<9), CX // SSSE3
SETNE runtime·support_ssse3(SB)

TESTL $(1<<19), CX // SSE4.1
SETNE runtime·support_sse41(SB)

TESTL $(1<<20), CX // SSE4.2
SETNE runtime·support_sse42(SB)

TESTL $(1<<23), CX // POPCNT
SETNE runtime·support_popcnt(SB)

TESTL $(1<<25), CX // AES
SETNE runtime·support_aes(SB)

TESTL $(1<<27), CX // OSXSAVE
SETNE runtime·support_osxsave(SB)

// If OS support for XMM and YMM is not present
// support_avx will be set back to false later.
TESTL $(1<<28), CX // AVX
SETNE runtime·support_avx(SB)

eax7:
// Load EAX=7/ECX=0 cpuid flags
CMPQ SI, $7
JLT no7
CMPL SI, $7
JLT osavx
MOVL $7, AX
MOVL $0, CX
CPUID
MOVL BX, runtime·cpuid_ebx7(SB)
no7:
// Detect AVX and AVX2 as per 14.7.1 Detection of AVX2 chapter of [1]
// [1] 64-ia-32-architectures-software-developer-manual-325462.pdf
// http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-manual-325462.pdf
MOVL runtime·cpuid_ecx(SB), CX
ANDL $0x18000000, CX // check for OSXSAVE and AVX bits
CMPL CX, $0x18000000
JNE noavx
MOVL $0, CX

TESTL $(1<<3), BX // BMI1
SETNE runtime·support_bmi1(SB)

// If OS support for XMM and YMM is not present
// support_avx2 will be set back to false later.
TESTL $(1<<5), BX
SETNE runtime·support_avx2(SB)

TESTL $(1<<8), BX // BMI2
SETNE runtime·support_bmi2(SB)

TESTL $(1<<9), BX // ERMS
SETNE runtime·support_erms(SB)

osavx:
CMPB runtime·support_osxsave(SB), $1
JNE noavx
MOVL $0, CX
// For XGETBV, OSXSAVE bit is required and sufficient
XGETBV
ANDL $6, AX
CMPL AX, $6 // Check for OS support of YMM registers
JNE noavx
MOVB $1, runtime·support_avx(SB)
TESTL $(1<<5), runtime·cpuid_ebx7(SB) // check for AVX2 bit
JEQ noavx2
MOVB $1, runtime·support_avx2(SB)
JMP testbmi1
ANDL $6, AX
CMPL AX, $6 // Check for OS support of XMM and YMM registers.
JE nocpuinfo
noavx:
MOVB $0, runtime·support_avx(SB)
noavx2:
MOVB $0, runtime·support_avx2(SB)
testbmi1:
// Detect BMI1 and BMI2 extensions as per
// 5.1.16.1 Detection of VEX-encoded GPR Instructions,
// LZCNT and TZCNT, PREFETCHW chapter of [1]
MOVB $0, runtime·support_bmi1(SB)
TESTL $(1<<3), runtime·cpuid_ebx7(SB) // check for BMI1 bit
JEQ testbmi2
MOVB $1, runtime·support_bmi1(SB)
testbmi2:
MOVB $0, runtime·support_bmi2(SB)
TESTL $(1<<8), runtime·cpuid_ebx7(SB) // check for BMI2 bit
JEQ testpopcnt
MOVB $1, runtime·support_bmi2(SB)
testpopcnt:
MOVB $0, runtime·support_popcnt(SB)
TESTL $(1<<23), runtime·cpuid_ecx(SB) // check for POPCNT bit
JEQ nocpuinfo
MOVB $1, runtime·support_popcnt(SB)
nocpuinfo:

MOVB $0, runtime·support_avx(SB)
MOVB $0, runtime·support_avx2(SB)

nocpuinfo:
// if there is an _cgo_init, call it.
MOVQ _cgo_init(SB), AX
TESTQ AX, AX
Expand Down Expand Up @@ -1942,9 +1955,8 @@ success_avx2:
VZEROUPPER
JMP success
sse42:
MOVL runtime·cpuid_ecx(SB), CX
ANDL $0x100000, CX
JZ no_sse42
CMPB runtime·support_sse42(SB), $1
JNE no_sse42
CMPQ AX, $12
// PCMPESTRI is slower than normal compare,
// so using it makes sense only if we advance 4+ bytes per compare
Expand Down
80 changes: 74 additions & 6 deletions src/runtime/asm_amd64p32.s
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$0
MOVL SP, (g_stack+stack_hi)(DI)

// find out information about the processor we're on
MOVQ $0, AX
MOVL $0, AX
CPUID
CMPQ AX, $0
CMPL AX, $0
JE nocpuinfo

CMPL BX, $0x756E6547 // "Genu"
Expand All @@ -42,13 +42,81 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$0
MOVB $1, runtime·isIntel(SB)
notintel:

MOVQ $1, AX
// Load EAX=1 cpuid flags
MOVL $1, AX
CPUID
MOVL AX, runtime·cpuid_eax(SB)
MOVL AX, runtime·processorVersionInfo(SB)
MOVL CX, runtime·cpuid_ecx(SB)
MOVL DX, runtime·cpuid_edx(SB)
nocpuinfo:


TESTL $(1<<26), DX // SSE2
SETNE runtime·support_sse2(SB)

TESTL $(1<<9), CX // SSSE3
SETNE runtime·support_ssse3(SB)

TESTL $(1<<19), CX // SSE4.1
SETNE runtime·support_sse41(SB)

TESTL $(1<<20), CX // SSE4.2
SETNE runtime·support_sse42(SB)

TESTL $(1<<23), CX // POPCNT
SETNE runtime·support_popcnt(SB)

TESTL $(1<<25), CX // AES
SETNE runtime·support_aes(SB)

TESTL $(1<<27), CX // OSXSAVE
SETNE runtime·support_osxsave(SB)

// If OS support for XMM and YMM is not present
// support_avx will be set back to false later.
TESTL $(1<<28), CX // AVX
SETNE runtime·support_avx(SB)

eax7:
// Load EAX=7/ECX=0 cpuid flags
CMPL SI, $7
JLT osavx
MOVL $7, AX
MOVL $0, CX
CPUID
MOVL BX, runtime·cpuid_ebx7(SB)

TESTL $(1<<3), BX // BMI1
SETNE runtime·support_bmi1(SB)

// If OS support for XMM and YMM is not present
// support_avx2 will be set back to false later.
TESTL $(1<<5), BX
SETNE runtime·support_avx2(SB)

TESTL $(1<<8), BX // BMI2
SETNE runtime·support_bmi2(SB)

TESTL $(1<<9), BX // ERMS
SETNE runtime·support_erms(SB)

osavx:
// nacl does not support XGETBV to test
// for XMM and YMM OS support.
#ifndef GOOS_nacl
CMPB runtime·support_osxsave(SB), $1
JNE noavx
MOVL $0, CX
// For XGETBV, OSXSAVE bit is required and sufficient
XGETBV
ANDL $6, AX
CMPL AX, $6 // Check for OS support of XMM and YMM registers.
JE nocpuinfo
#endif
noavx:
MOVB $0, runtime·support_avx(SB)
MOVB $0, runtime·support_avx2(SB)

nocpuinfo:

needtls:
LEAL runtime·m0+m_tls(SB), DI
CALL runtime·settls(SB)
Expand Down
10 changes: 5 additions & 5 deletions src/runtime/cpuflags_amd64.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,13 @@ var useAVXmemmove bool

func init() {
// Let's remove stepping and reserved fields
processorVersionInfo := cpuid_eax & 0x0FFF3FF0
processor := processorVersionInfo & 0x0FFF3FF0

isIntelBridgeFamily := isIntel &&
(processorVersionInfo == 0x206A0 ||
processorVersionInfo == 0x206D0 ||
processorVersionInfo == 0x306A0 ||
processorVersionInfo == 0x306E0)
processor == 0x206A0 ||
processor == 0x206D0 ||
processor == 0x306A0 ||
processor == 0x306E0

useAVXmemmove = support_avx && !isIntelBridgeFamily
}
Loading

0 comments on commit 5a6c580

Please sign in to comment.