icp: fix all !ENDBR objtool warnings in x86 Asm code

Currently, only Blake3 x86 Asm code has signs of being ENDBR-aware. At least, under certain conditions it includes some header file and uses some custom macro from there. Linux has its own NOENDBR since several releases ago. It's defined in the same <asm/linkage.h>, so currently <sys/asm_linkage.h> already is provided with it. Let's unify those two into one %ENDBR macro. At first, check if it's present already. If so -- use Linux kernel version. Otherwise, try to go that second way and use %_CET_ENDBR from <cet.h> if available. If no, fall back to just empty definition. This fixes a couple more 'relocations to !ENDBR' across the module. And now that we always have the latest/actual ENDBR definition, use it at the entrance of the few corresponding functions that objtool still complains about. This matches the way how it's used in the upstream x86 core Asm code. Reviewed-by: Attila Fülöp <attila@fueloep.org> Reviewed-by: Tino Reichardt <milky-zfs@mcmilk.de> Reviewed-by: Richard Yao <richard.yao@alumni.stonybrook.edu> Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Signed-off-by: Alexander Lobakin <alobakin@pm.me> Closes openzfs#14035
andrewc12 · Nov 11, 2022 · 2b2b10e · 2b2b10e
1 parent 99e3f1c
commit 2b2b10e
Show file tree

Hide file tree

Showing 10 changed files with 41 additions and 50 deletions.
diff --git a/module/icp/asm-x86_64/aes/aes_amd64.S b/module/icp/asm-x86_64/aes/aes_amd64.S
@@ -704,6 +704,7 @@ enc_tab:
 
 
 ENTRY_NP(aes_encrypt_amd64)
+	ENDBR
 #ifdef	GLADMAN_INTERFACE
 	// Original interface
 	sub	$[4*8], %rsp	// gnu/linux/opensolaris binary interface
@@ -809,6 +810,7 @@ dec_tab:
 
 
 ENTRY_NP(aes_decrypt_amd64)
+	ENDBR
 #ifdef	GLADMAN_INTERFACE
 	// Original interface
 	sub	$[4*8], %rsp	// gnu/linux/opensolaris binary interface

diff --git a/module/icp/asm-x86_64/blake3/blake3_avx2.S b/module/icp/asm-x86_64/blake3/blake3_avx2.S
@@ -30,24 +30,14 @@
 #define _ASM
 #include <sys/asm_linkage.h>
 
-#if defined(__ELF__) && defined(__CET__) && defined(__has_include)
-#if __has_include(<cet.h>)
-#include <cet.h>
-#endif
-#endif
-
-#if !defined(_CET_ENDBR)
-#define _CET_ENDBR
-#endif
-
 .intel_syntax noprefix
 .global zfs_blake3_hash_many_avx2
 .text
 
 .type zfs_blake3_hash_many_avx2,@function
 .p2align  6
 zfs_blake3_hash_many_avx2:
-        _CET_ENDBR
+        ENDBR
         push    r15
         push    r14
         push    r13

diff --git a/module/icp/asm-x86_64/blake3/blake3_avx512.S b/module/icp/asm-x86_64/blake3/blake3_avx512.S
@@ -30,16 +30,6 @@
 #define _ASM
 #include <sys/asm_linkage.h>
 
-#if defined(__ELF__) && defined(__CET__) && defined(__has_include)
-#if __has_include(<cet.h>)
-#include <cet.h>
-#endif
-#endif
-
-#if !defined(_CET_ENDBR)
-#define _CET_ENDBR
-#endif
-
 .intel_syntax noprefix
 .global zfs_blake3_hash_many_avx512
 .global zfs_blake3_compress_in_place_avx512
@@ -52,7 +42,7 @@
 
 .p2align  6
 zfs_blake3_hash_many_avx512:
-        _CET_ENDBR
+        ENDBR
         push    r15
         push    r14
         push    r13
@@ -2409,7 +2399,7 @@ zfs_blake3_hash_many_avx512:
         jmp     4b
 .p2align 6
 zfs_blake3_compress_in_place_avx512:
-        _CET_ENDBR
+        ENDBR
         vmovdqu xmm0, xmmword ptr [rdi]
         vmovdqu xmm1, xmmword ptr [rdi+0x10]
         movzx   eax, r8b
@@ -2491,7 +2481,7 @@ zfs_blake3_compress_in_place_avx512:
 
 .p2align 6
 zfs_blake3_compress_xof_avx512:
-        _CET_ENDBR
+        ENDBR
         vmovdqu xmm0, xmmword ptr [rdi]
         vmovdqu xmm1, xmmword ptr [rdi+0x10]
         movzx   eax, r8b

diff --git a/module/icp/asm-x86_64/blake3/blake3_sse2.S b/module/icp/asm-x86_64/blake3/blake3_sse2.S
@@ -30,16 +30,6 @@
 #define _ASM
 #include <sys/asm_linkage.h>
 
-#if defined(__ELF__) && defined(__CET__) && defined(__has_include)
-#if __has_include(<cet.h>)
-#include <cet.h>
-#endif
-#endif
-
-#if !defined(_CET_ENDBR)
-#define _CET_ENDBR
-#endif
-
 .intel_syntax noprefix
 .global zfs_blake3_hash_many_sse2
 .global zfs_blake3_compress_in_place_sse2
@@ -52,7 +42,7 @@
 
         .p2align  6
 zfs_blake3_hash_many_sse2:
-        _CET_ENDBR
+        ENDBR
         push    r15
         push    r14
         push    r13
@@ -2050,7 +2040,7 @@ zfs_blake3_hash_many_sse2:
 
 .p2align 6
 zfs_blake3_compress_in_place_sse2:
-        _CET_ENDBR
+        ENDBR
         movups  xmm0, xmmword ptr [rdi]
         movups  xmm1, xmmword ptr [rdi+0x10]
         movaps  xmm2, xmmword ptr [BLAKE3_IV+rip]
@@ -2161,7 +2151,7 @@ zfs_blake3_compress_in_place_sse2:
 
 .p2align 6
 zfs_blake3_compress_xof_sse2:
-        _CET_ENDBR
+        ENDBR
         movups  xmm0, xmmword ptr [rdi]
         movups  xmm1, xmmword ptr [rdi+0x10]
         movaps  xmm2, xmmword ptr [BLAKE3_IV+rip]

diff --git a/module/icp/asm-x86_64/blake3/blake3_sse41.S b/module/icp/asm-x86_64/blake3/blake3_sse41.S
@@ -30,16 +30,6 @@
 #define _ASM
 #include <sys/asm_linkage.h>
 
-#if defined(__ELF__) && defined(__CET__) && defined(__has_include)
-#if __has_include(<cet.h>)
-#include <cet.h>
-#endif
-#endif
-
-#if !defined(_CET_ENDBR)
-#define _CET_ENDBR
-#endif
-
 .intel_syntax noprefix
 .global zfs_blake3_compress_in_place_sse41
 .global zfs_blake3_compress_xof_sse41
@@ -52,7 +42,7 @@
 
 .p2align  6
 zfs_blake3_hash_many_sse41:
-        _CET_ENDBR
+        ENDBR
         push    r15
         push    r14
         push    r13
@@ -1812,7 +1802,7 @@ zfs_blake3_hash_many_sse41:
         jmp     4b
 .p2align 6
 zfs_blake3_compress_in_place_sse41:
-        _CET_ENDBR
+        ENDBR
         movups  xmm0, xmmword ptr [rdi]
         movups  xmm1, xmmword ptr [rdi+0x10]
         movaps  xmm2, xmmword ptr [BLAKE3_IV+rip]
@@ -1911,7 +1901,7 @@ zfs_blake3_compress_in_place_sse41:
         RET
 .p2align 6
 zfs_blake3_compress_xof_sse41:
-        _CET_ENDBR
+        ENDBR
         movups  xmm0, xmmword ptr [rdi]
         movups  xmm1, xmmword ptr [rdi+0x10]
         movaps  xmm2, xmmword ptr [BLAKE3_IV+rip]

diff --git a/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S b/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S
@@ -59,6 +59,7 @@
 .align	32
 _aesni_ctr32_ghash_6x:
 .cfi_startproc
+	ENDBR
 	vmovdqu	32(%r11),%xmm2
 	subq	$6,%rdx
 	vpxor	%xmm4,%xmm4,%xmm4
@@ -375,6 +376,7 @@ _aesni_ctr32_ghash_6x:
 .align	32
 _aesni_ctr32_ghash_no_movbe_6x:
 .cfi_startproc
+	ENDBR
 	vmovdqu	32(%r11),%xmm2
 	subq	$6,%rdx
 	vpxor	%xmm4,%xmm4,%xmm4
@@ -703,6 +705,7 @@ _aesni_ctr32_ghash_no_movbe_6x:
 .align	32
 aesni_gcm_decrypt:
 .cfi_startproc
+	ENDBR
 	xorq	%r10,%r10
 	cmpq	$0x60,%rdx
 	jb	.Lgcm_dec_abort
@@ -820,6 +823,7 @@ aesni_gcm_decrypt:
 .align	32
 _aesni_ctr32_6x:
 .cfi_startproc
+	ENDBR
 	vmovdqu	0-128(%rcx),%xmm4
 	vmovdqu	32(%r11),%xmm2
 	leaq	-2(%rbp),%r13	// ICP uses 10,12,14 not 9,11,13 for rounds.
@@ -914,6 +918,7 @@ _aesni_ctr32_6x:
 .align	32
 aesni_gcm_encrypt:
 .cfi_startproc
+	ENDBR
 	xorq	%r10,%r10
 	cmpq	$288,%rdx
 	jb	.Lgcm_enc_abort

diff --git a/module/icp/asm-x86_64/modes/ghash-x86_64.S b/module/icp/asm-x86_64/modes/ghash-x86_64.S
@@ -107,6 +107,7 @@
 .align	16
 gcm_gmult_clmul:
 .cfi_startproc
+	ENDBR
 .L_gmult_clmul:
 	movdqu	(%rdi),%xmm0
 	movdqa	.Lbswap_mask(%rip),%xmm5
@@ -161,6 +162,7 @@ gcm_gmult_clmul:
 .align	32
 gcm_init_htab_avx:
 .cfi_startproc
+	ENDBR
 	vzeroupper
 
 	vmovdqu	(%rsi),%xmm2
@@ -274,6 +276,7 @@ gcm_init_htab_avx:
 .align	32
 gcm_gmult_avx:
 .cfi_startproc
+	ENDBR
 	jmp	.L_gmult_clmul
 .cfi_endproc
 .size	gcm_gmult_avx,.-gcm_gmult_avx
@@ -282,6 +285,7 @@ gcm_gmult_avx:
 .align	32
 gcm_ghash_avx:
 .cfi_startproc
+	ENDBR
 	vzeroupper
 
 	vmovdqu	(%rdi),%xmm10

diff --git a/module/icp/asm-x86_64/sha2/sha256_impl.S b/module/icp/asm-x86_64/sha2/sha256_impl.S
@@ -84,6 +84,7 @@ SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num)
 
 ENTRY_NP(SHA256TransformBlocks)
 .cfi_startproc
+	ENDBR
 	movq	%rsp, %rax
 .cfi_def_cfa_register %rax
 	push	%rbx

diff --git a/module/icp/asm-x86_64/sha2/sha512_impl.S b/module/icp/asm-x86_64/sha2/sha512_impl.S
@@ -85,6 +85,7 @@ SHA512TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num)
 
 ENTRY_NP(SHA512TransformBlocks)
 .cfi_startproc
+	ENDBR
 	movq	%rsp, %rax
 .cfi_def_cfa_register %rax
 	push	%rbx

diff --git a/module/icp/include/sys/ia32/asm_linkage.h b/module/icp/include/sys/ia32/asm_linkage.h
@@ -34,6 +34,24 @@
 #include <linux/linkage.h>
 #endif
 
+#ifndef ENDBR
+#if defined(__ELF__) && defined(__CET__) && defined(__has_include)
+/* CSTYLED */
+#if __has_include(<cet.h>)
+
+#include <cet.h>
+
+#ifdef _CET_ENDBR
+#define	ENDBR	_CET_ENDBR
+#endif /* _CET_ENDBR */
+
+#endif /* <cet.h> */
+#endif /* __ELF__ && __CET__ && __has_include */
+#endif /* !ENDBR */
+
+#ifndef ENDBR
+#define	ENDBR
+#endif
 #ifndef RET
 #define	RET	ret
 #endif