diff --git a/module/icp/asm-x86_64/aes/aes_amd64.S b/module/icp/asm-x86_64/aes/aes_amd64.S index f546e8933be1..a0525dd464f5 100644 --- a/module/icp/asm-x86_64/aes/aes_amd64.S +++ b/module/icp/asm-x86_64/aes/aes_amd64.S @@ -704,6 +704,7 @@ enc_tab: ENTRY_NP(aes_encrypt_amd64) + ENDBR #ifdef GLADMAN_INTERFACE // Original interface sub $[4*8], %rsp // gnu/linux/opensolaris binary interface @@ -809,6 +810,7 @@ dec_tab: ENTRY_NP(aes_decrypt_amd64) + ENDBR #ifdef GLADMAN_INTERFACE // Original interface sub $[4*8], %rsp // gnu/linux/opensolaris binary interface diff --git a/module/icp/asm-x86_64/blake3/blake3_avx2.S b/module/icp/asm-x86_64/blake3/blake3_avx2.S index f4d9cb766d46..cb08430b81ed 100644 --- a/module/icp/asm-x86_64/blake3/blake3_avx2.S +++ b/module/icp/asm-x86_64/blake3/blake3_avx2.S @@ -30,16 +30,6 @@ #define _ASM #include -#if defined(__ELF__) && defined(__CET__) && defined(__has_include) -#if __has_include() -#include -#endif -#endif - -#if !defined(_CET_ENDBR) -#define _CET_ENDBR -#endif - .intel_syntax noprefix .global zfs_blake3_hash_many_avx2 .text @@ -47,7 +37,7 @@ .type zfs_blake3_hash_many_avx2,@function .p2align 6 zfs_blake3_hash_many_avx2: - _CET_ENDBR + ENDBR push r15 push r14 push r13 diff --git a/module/icp/asm-x86_64/blake3/blake3_avx512.S b/module/icp/asm-x86_64/blake3/blake3_avx512.S index 71b5715c88c1..960406ea2c01 100644 --- a/module/icp/asm-x86_64/blake3/blake3_avx512.S +++ b/module/icp/asm-x86_64/blake3/blake3_avx512.S @@ -30,16 +30,6 @@ #define _ASM #include -#if defined(__ELF__) && defined(__CET__) && defined(__has_include) -#if __has_include() -#include -#endif -#endif - -#if !defined(_CET_ENDBR) -#define _CET_ENDBR -#endif - .intel_syntax noprefix .global zfs_blake3_hash_many_avx512 .global zfs_blake3_compress_in_place_avx512 @@ -52,7 +42,7 @@ .p2align 6 zfs_blake3_hash_many_avx512: - _CET_ENDBR + ENDBR push r15 push r14 push r13 @@ -2409,7 +2399,7 @@ zfs_blake3_hash_many_avx512: jmp 4b .p2align 6 zfs_blake3_compress_in_place_avx512: - _CET_ENDBR + ENDBR vmovdqu xmm0, xmmword ptr [rdi] vmovdqu xmm1, xmmword ptr [rdi+0x10] movzx eax, r8b @@ -2491,7 +2481,7 @@ zfs_blake3_compress_in_place_avx512: .p2align 6 zfs_blake3_compress_xof_avx512: - _CET_ENDBR + ENDBR vmovdqu xmm0, xmmword ptr [rdi] vmovdqu xmm1, xmmword ptr [rdi+0x10] movzx eax, r8b diff --git a/module/icp/asm-x86_64/blake3/blake3_sse2.S b/module/icp/asm-x86_64/blake3/blake3_sse2.S index 20689a7dcef5..c4290aaa8faf 100644 --- a/module/icp/asm-x86_64/blake3/blake3_sse2.S +++ b/module/icp/asm-x86_64/blake3/blake3_sse2.S @@ -30,16 +30,6 @@ #define _ASM #include -#if defined(__ELF__) && defined(__CET__) && defined(__has_include) -#if __has_include() -#include -#endif -#endif - -#if !defined(_CET_ENDBR) -#define _CET_ENDBR -#endif - .intel_syntax noprefix .global zfs_blake3_hash_many_sse2 .global zfs_blake3_compress_in_place_sse2 @@ -52,7 +42,7 @@ .p2align 6 zfs_blake3_hash_many_sse2: - _CET_ENDBR + ENDBR push r15 push r14 push r13 @@ -2050,7 +2040,7 @@ zfs_blake3_hash_many_sse2: .p2align 6 zfs_blake3_compress_in_place_sse2: - _CET_ENDBR + ENDBR movups xmm0, xmmword ptr [rdi] movups xmm1, xmmword ptr [rdi+0x10] movaps xmm2, xmmword ptr [BLAKE3_IV+rip] @@ -2161,7 +2151,7 @@ zfs_blake3_compress_in_place_sse2: .p2align 6 zfs_blake3_compress_xof_sse2: - _CET_ENDBR + ENDBR movups xmm0, xmmword ptr [rdi] movups xmm1, xmmword ptr [rdi+0x10] movaps xmm2, xmmword ptr [BLAKE3_IV+rip] diff --git a/module/icp/asm-x86_64/blake3/blake3_sse41.S b/module/icp/asm-x86_64/blake3/blake3_sse41.S index c5975a4f0877..45b90cc9ed89 100644 --- a/module/icp/asm-x86_64/blake3/blake3_sse41.S +++ b/module/icp/asm-x86_64/blake3/blake3_sse41.S @@ -30,16 +30,6 @@ #define _ASM #include -#if defined(__ELF__) && defined(__CET__) && defined(__has_include) -#if __has_include() -#include -#endif -#endif - -#if !defined(_CET_ENDBR) -#define _CET_ENDBR -#endif - .intel_syntax noprefix .global zfs_blake3_compress_in_place_sse41 .global zfs_blake3_compress_xof_sse41 @@ -52,7 +42,7 @@ .p2align 6 zfs_blake3_hash_many_sse41: - _CET_ENDBR + ENDBR push r15 push r14 push r13 @@ -1812,7 +1802,7 @@ zfs_blake3_hash_many_sse41: jmp 4b .p2align 6 zfs_blake3_compress_in_place_sse41: - _CET_ENDBR + ENDBR movups xmm0, xmmword ptr [rdi] movups xmm1, xmmword ptr [rdi+0x10] movaps xmm2, xmmword ptr [BLAKE3_IV+rip] @@ -1911,7 +1901,7 @@ zfs_blake3_compress_in_place_sse41: RET .p2align 6 zfs_blake3_compress_xof_sse41: - _CET_ENDBR + ENDBR movups xmm0, xmmword ptr [rdi] movups xmm1, xmmword ptr [rdi+0x10] movaps xmm2, xmmword ptr [BLAKE3_IV+rip] diff --git a/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S b/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S index 6da43ee00597..cf17b3768712 100644 --- a/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S +++ b/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S @@ -59,6 +59,7 @@ .align 32 _aesni_ctr32_ghash_6x: .cfi_startproc + ENDBR vmovdqu 32(%r11),%xmm2 subq $6,%rdx vpxor %xmm4,%xmm4,%xmm4 @@ -375,6 +376,7 @@ _aesni_ctr32_ghash_6x: .align 32 _aesni_ctr32_ghash_no_movbe_6x: .cfi_startproc + ENDBR vmovdqu 32(%r11),%xmm2 subq $6,%rdx vpxor %xmm4,%xmm4,%xmm4 @@ -703,6 +705,7 @@ _aesni_ctr32_ghash_no_movbe_6x: .align 32 aesni_gcm_decrypt: .cfi_startproc + ENDBR xorq %r10,%r10 cmpq $0x60,%rdx jb .Lgcm_dec_abort @@ -820,6 +823,7 @@ aesni_gcm_decrypt: .align 32 _aesni_ctr32_6x: .cfi_startproc + ENDBR vmovdqu 0-128(%rcx),%xmm4 vmovdqu 32(%r11),%xmm2 leaq -2(%rbp),%r13 // ICP uses 10,12,14 not 9,11,13 for rounds. @@ -914,6 +918,7 @@ _aesni_ctr32_6x: .align 32 aesni_gcm_encrypt: .cfi_startproc + ENDBR xorq %r10,%r10 cmpq $288,%rdx jb .Lgcm_enc_abort diff --git a/module/icp/asm-x86_64/modes/ghash-x86_64.S b/module/icp/asm-x86_64/modes/ghash-x86_64.S index d7cdaeb368d7..bf3724a23eae 100644 --- a/module/icp/asm-x86_64/modes/ghash-x86_64.S +++ b/module/icp/asm-x86_64/modes/ghash-x86_64.S @@ -107,6 +107,7 @@ .align 16 gcm_gmult_clmul: .cfi_startproc + ENDBR .L_gmult_clmul: movdqu (%rdi),%xmm0 movdqa .Lbswap_mask(%rip),%xmm5 @@ -161,6 +162,7 @@ gcm_gmult_clmul: .align 32 gcm_init_htab_avx: .cfi_startproc + ENDBR vzeroupper vmovdqu (%rsi),%xmm2 @@ -274,6 +276,7 @@ gcm_init_htab_avx: .align 32 gcm_gmult_avx: .cfi_startproc + ENDBR jmp .L_gmult_clmul .cfi_endproc .size gcm_gmult_avx,.-gcm_gmult_avx @@ -282,6 +285,7 @@ gcm_gmult_avx: .align 32 gcm_ghash_avx: .cfi_startproc + ENDBR vzeroupper vmovdqu (%rdi),%xmm10 diff --git a/module/icp/asm-x86_64/sha2/sha256_impl.S b/module/icp/asm-x86_64/sha2/sha256_impl.S index 1391bd59a017..60d34b4a3be0 100644 --- a/module/icp/asm-x86_64/sha2/sha256_impl.S +++ b/module/icp/asm-x86_64/sha2/sha256_impl.S @@ -84,6 +84,7 @@ SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num) ENTRY_NP(SHA256TransformBlocks) .cfi_startproc + ENDBR movq %rsp, %rax .cfi_def_cfa_register %rax push %rbx diff --git a/module/icp/asm-x86_64/sha2/sha512_impl.S b/module/icp/asm-x86_64/sha2/sha512_impl.S index e61e96957bc6..ed7fb362a1ac 100644 --- a/module/icp/asm-x86_64/sha2/sha512_impl.S +++ b/module/icp/asm-x86_64/sha2/sha512_impl.S @@ -85,6 +85,7 @@ SHA512TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num) ENTRY_NP(SHA512TransformBlocks) .cfi_startproc + ENDBR movq %rsp, %rax .cfi_def_cfa_register %rax push %rbx diff --git a/module/icp/include/sys/ia32/asm_linkage.h b/module/icp/include/sys/ia32/asm_linkage.h index f0aa2dc184c7..e3e769ffd858 100644 --- a/module/icp/include/sys/ia32/asm_linkage.h +++ b/module/icp/include/sys/ia32/asm_linkage.h @@ -34,6 +34,24 @@ #include #endif +#ifndef ENDBR +#if defined(__ELF__) && defined(__CET__) && defined(__has_include) +/* CSTYLED */ +#if __has_include() + +#include + +#ifdef _CET_ENDBR +#define ENDBR _CET_ENDBR +#endif /* _CET_ENDBR */ + +#endif /* */ +#endif /* __ELF__ && __CET__ && __has_include */ +#endif /* !ENDBR */ + +#ifndef ENDBR +#define ENDBR +#endif #ifndef RET #define RET ret #endif