Linux 5.16 compat: don't use XSTATE_XSAVE to save FPU state

config/kernel-fpu.m4

@@ -1,9 +1,12 @@
-dnl # 
+dnl #
 dnl # Handle differences in kernel FPU code.
 dnl #
 dnl # Kernel
 dnl # 5.16:	XCR code put into asm/fpu/xcr.h
-dnl # 		HAVE_KERNEL_FPU_XCR_HEADER
+dnl #		HAVE_KERNEL_FPU_XCR_HEADER
+dnl #
+dnl #		XSTATE_XSAVE and XSTATE_XRESTORE aren't accessible any more
+dnl #		HAVE_KERNEL_FPU_XSAVE_INTERNAL
 dnl #
 dnl # 5.0:	Wrappers have been introduced to save/restore the FPU state.
 dnl #		This change was made to the 4.19.38 and 4.14.120 LTS kernels.
@@ -107,6 +110,36 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FPU], [
 		struct fxregs_state *fxr __attribute__ ((unused)) = &st->fxsave;
 		struct xregs_state *xr __attribute__ ((unused)) = &st->xsave;
 	])
+
+	ZFS_LINUX_TEST_SRC([fpu_xsave_internal], [
+		#include <linux/sched.h>
+		#if defined(__x86_64) || defined(__x86_64__) || \
+		    defined(__i386) || defined(__i386__)
+		#if !defined(__x86)
+		#define __x86
+		#endif
+		#endif
+
+		#if !defined(__x86)
+		#error Unsupported architecture
+		#endif
+
+		#include <linux/types.h>
+		#ifdef HAVE_KERNEL_FPU_API_HEADER
+		#include <asm/fpu/api.h>
+		#include <asm/fpu/internal.h>
+		#else
+		#include <asm/i387.h>
+		#include <asm/xcr.h>
+		#endif
+
+	],[
+		struct fpu *fpu = &current->thread.fpu;
+		union fpregs_state *st = &fpu->fpstate->regs;
+		struct fregs_state *fr __attribute__ ((unused)) = &st->fsave;
+		struct fxregs_state *fxr __attribute__ ((unused)) = &st->fxsave;
+		struct xregs_state *xr __attribute__ ((unused)) = &st->xsave;
+	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_FPU], [
@@ -139,7 +172,13 @@ AC_DEFUN([ZFS_AC_KERNEL_FPU], [
 				AC_DEFINE(HAVE_KERNEL_FPU_INTERNAL, 1,
 				    [kernel fpu internal])
 			],[
+				ZFS_LINUX_TEST_RESULT([fpu_xsave_internal], [
+				    AC_MSG_RESULT(internal with internal XSAVE)
+				    AC_DEFINE(HAVE_KERNEL_FPU_XSAVE_INTERNAL, 1,
+					[kernel fpu and XSAVE internal])
+			    ],[
 				AC_MSG_RESULT(unavailable)
+			    ])
 			])
 		])
 	])

config/toolchain-simd.m4

@@ -24,6 +24,9 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD], [
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AES
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_PCLMULQDQ
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE
+			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE
+			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT
+			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES
 			;;
 	esac
 ])
@@ -422,3 +425,66 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE], [
 		AC_MSG_RESULT([no])
 	])
 ])
+
+dnl #
+dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE], [
+	AC_MSG_CHECKING([whether host toolchain supports XSAVE])
+
+	AC_LINK_IFELSE([AC_LANG_SOURCE([
+	[
+		void main()
+		{
+		  char b[4096] __attribute__ ((aligned (64)));
+		  __asm__ __volatile__("xsave %[b]\n" : : [b] "m" (*b) : "memory");
+		}
+	]])], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE([HAVE_XSAVE], 1, [Define if host toolchain supports XSAVE])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])
+
+dnl #
+dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT], [
+	AC_MSG_CHECKING([whether host toolchain supports XSAVEOPT])
+
+	AC_LINK_IFELSE([AC_LANG_SOURCE([
+	[
+		void main()
+		{
+		  char b[4096] __attribute__ ((aligned (64)));
+		  __asm__ __volatile__("xsaveopt %[b]\n" : : [b] "m" (*b) : "memory");
+		}
+	]])], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE([HAVE_XSAVEOPT], 1, [Define if host toolchain supports XSAVEOPT])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])
+
+dnl #
+dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES], [
+	AC_MSG_CHECKING([whether host toolchain supports XSAVES])
+
+	AC_LINK_IFELSE([AC_LANG_SOURCE([
+	[
+		void main()
+		{
+		  char b[4096] __attribute__ ((aligned (64)));
+		  __asm__ __volatile__("xsaves %[b]\n" : : [b] "m" (*b) : "memory");
+		}
+	]])], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE([HAVE_XSAVES], 1, [Define if host toolchain supports XSAVES])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])

configure.ac

@@ -383,6 +383,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/tests/functional/rootpool/Makefile
 	tests/zfs-tests/tests/functional/rsend/Makefile
 	tests/zfs-tests/tests/functional/scrub_mirror/Makefile
+	tests/zfs-tests/tests/functional/simd/Makefile
 	tests/zfs-tests/tests/functional/slog/Makefile
 	tests/zfs-tests/tests/functional/snapshot/Makefile
 	tests/zfs-tests/tests/functional/snapused/Makefile

include/os/linux/kernel/linux/simd_x86.h

@@ -136,9 +136,27 @@
  * When the kernel_fpu_* symbols are unavailable then provide our own
  * versions which allow the FPU to be safely used.
  */
+#if defined(HAVE_KERNEL_FPU_INTERNAL) || defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL)
+
+#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL)
+/*
+ * Some sanity checks.
+ * HAVE_KERNEL_FPU_INTERNAL and HAVE_KERNEL_FPU_XSAVE_INTERNAL are exclusive.
+ */
 #if defined(HAVE_KERNEL_FPU_INTERNAL)
+#error "HAVE_KERNEL_FPU_INTERNAL and HAVE_KERNEL_FPU_XSAVE_INTERNAL defined"
+#endif
+/*
+ * For kernels >= 5.16 we have to use inline assembly with the XSAVE{,OPT,S}
+ * instructions, so we need the toolchain to support at least XSAVE.
+ */
+#if !defined(HAVE_XSAVE)
+#error "Toolchain needs to support the XSAVE assembler instruction"
+#endif
+#endif
 
 #include <linux/mm.h>
+#include <linux/slab.h>
 
 extern union fpregs_state **zfs_kfpu_fpregs;
 
@@ -191,7 +209,9 @@ kfpu_init(void)
 }
 
 #define	kfpu_allowed()		1
+#if defined(HAVE_KERNEL_FPU_INTERNAL)
 #define	ex_handler_fprestore	ex_handler_default
+#endif
 
 /*
  * FPU save and restore instructions.
@@ -206,6 +226,7 @@ kfpu_init(void)
 #define	kfpu_fxsr_clean(rval)	__asm("fnclex; emms; fildl %P[addr]" \
 				    : : [addr] "m" (rval));
 
+#if defined(HAVE_KERNEL_FPU_INTERNAL)
 static inline void
 kfpu_save_xsave(struct xregs_state *addr, uint64_t mask)
 {
@@ -217,6 +238,21 @@ kfpu_save_xsave(struct xregs_state *addr, uint64_t mask)
 	XSTATE_XSAVE(addr, low, hi, err);
 	WARN_ON_ONCE(err);
 }
+#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */
+
+#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL)
+#define	kfpu_do_xsave(instruction, addr, mask)			\
+{								\
+	uint32_t low, hi;					\
+								\
+	low = mask;						\
+	hi = (uint64_t)(mask) >> 32;				\
+	__asm(instruction " %[dst]\n\t"				\
+	    :							\
+	    : [dst] "m" (*(addr)), "a" (low), "d" (hi)		\
+	    : "memory");					\
+}
+#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */
 
 static inline void
 kfpu_save_fxsr(struct fxregs_state *addr)
@@ -233,6 +269,7 @@ kfpu_save_fsave(struct fregs_state *addr)
 	kfpu_fnsave(addr);
 }
 
+#if defined(HAVE_KERNEL_FPU_INTERNAL)
 static inline void
 kfpu_begin(void)
 {
@@ -250,7 +287,6 @@ kfpu_begin(void)
 	 * FPU state to be correctly preserved and restored.
 	 */
 	union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()];
-
 	if (static_cpu_has(X86_FEATURE_XSAVE)) {
 		kfpu_save_xsave(&state->xsave, ~0);
 	} else if (static_cpu_has(X86_FEATURE_FXSR)) {
@@ -259,7 +295,50 @@ kfpu_begin(void)
 		kfpu_save_fsave(&state->fsave);
 	}
 }
+#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */
+
+#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL)
+static inline void
+kfpu_begin(void)
+{
+	/*
+	 * Preemption and interrupts must be disabled for the critical
+	 * region where the FPU state is being modified.
+	 */
+	preempt_disable();
+	local_irq_disable();
+
+	/*
+	 * The current FPU registers need to be preserved by kfpu_begin()
+	 * and restored by kfpu_end().  They are stored in a dedicated
+	 * per-cpu variable, not in the task struct, this allows any user
+	 * FPU state to be correctly preserved and restored.
+	 */
+	union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()];
+#if defined(HAVE_XSAVES)
+	if (static_cpu_has(X86_FEATURE_XSAVES)) {
+		kfpu_do_xsave("xsaves", &state->xsave, ~0);
+		goto out;
+	}
+#endif
+#if defined(HAVE_XSAVEOPT)
+	if (static_cpu_has(X86_FEATURE_XSAVEOPT)) {
+		kfpu_do_xsave("xsaveopt", &state->xsave, ~0);
+		goto out;
+	}
+#endif
+	if (static_cpu_has(X86_FEATURE_XSAVE)) {
+		kfpu_do_xsave("xsave", &state->xsave, ~0);
+	} else if (static_cpu_has(X86_FEATURE_FXSR)) {
+		kfpu_save_fxsr(&state->fxsave);
+	} else {
+		kfpu_save_fsave(&state->fsave);
+	}
+out:
+}
+#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */
 
+#if defined(HAVE_KERNEL_FPU_INTERNAL)
 static inline void
 kfpu_restore_xsave(struct xregs_state *addr, uint64_t mask)
 {
@@ -269,6 +348,21 @@ kfpu_restore_xsave(struct xregs_state *addr, uint64_t mask)
 	hi = mask >> 32;
 	XSTATE_XRESTORE(addr, low, hi);
 }
+#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */
+
+#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL)
+#define	kfpu_do_xrstor(instruction, addr, mask)			\
+{								\
+	uint32_t low, hi;					\
+								\
+	low = mask;						\
+	hi = (uint64_t)(mask) >> 32;				\
+	__asm(instruction " %[src]"				\
+	    :							\
+	    : [src] "m" (*(addr)), "a" (low), "d" (hi)		\
+	    : "memory");					\
+}
+#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */
 
 static inline void
 kfpu_restore_fxsr(struct fxregs_state *addr)
@@ -294,6 +388,7 @@ kfpu_restore_fsave(struct fregs_state *addr)
 	kfpu_frstor(addr);
 }
 
+#if defined(HAVE_KERNEL_FPU_INTERNAL)
 static inline void
 kfpu_end(void)
 {
@@ -310,6 +405,32 @@ kfpu_end(void)
 	local_irq_enable();
 	preempt_enable();
 }
+#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */
+
+#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL)
+static inline void
+kfpu_end(void)
+{
+	union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()];
+#if defined(HAVE_XSAVES)
+	if (static_cpu_has(X86_FEATURE_XSAVES)) {
+		kfpu_do_xrstor("xrstors", &state->xsave, ~0);
+		goto out;
+	}
+#endif
+	if (static_cpu_has(X86_FEATURE_XSAVE)) {
+		kfpu_do_xrstor("xrstor", &state->xsave, ~0);
+	} else if (static_cpu_has(X86_FEATURE_FXSR)) {
+		kfpu_save_fxsr(&state->fxsave);
+	} else {
+		kfpu_save_fsave(&state->fsave);
+	}
+out:
+	local_irq_enable();
+	preempt_enable();
+
+}
+#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */
 
 #else
 
@@ -322,7 +443,7 @@ kfpu_end(void)
 #define	kfpu_init()		0
 #define	kfpu_fini()		((void) 0)
 
-#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */
+#endif /* defined(HAVE_KERNEL_FPU_INTERNAL || HAVE_KERNEL_FPU_XSAVE_INTERNAL) */
 #endif /* defined(KERNEL_EXPORTS_X86_FPU) */
 
 /*

module/zcommon/zfs_prop.c

@@ -993,10 +993,10 @@ zfs_prop_align_right(zfs_prop_t prop)
 
 #include <sys/simd.h>
 
-#if defined(HAVE_KERNEL_FPU_INTERNAL)
+#if defined(HAVE_KERNEL_FPU_INTERNAL) || defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL)
 union fpregs_state **zfs_kfpu_fpregs;
 EXPORT_SYMBOL(zfs_kfpu_fpregs);
-#endif /* HAVE_KERNEL_FPU_INTERNAL */
+#endif /* HAVE_KERNEL_FPU_INTERNAL || HAVE_KERNEL_FPU_XSAVE_INTERNAL */
 
 static int __init
 zcommon_init(void)

tests/runfiles/linux.run

@@ -147,6 +147,12 @@ tags = ['functional', 'projectquota']
 tests = ['send_realloc_dnode_size', 'send_encrypted_files']
 tags = ['functional', 'rsend']
 
+[tests/functional/simd:Linux]
+pre =
+post =
+tests = ['simd_supported']
+tags = ['functional', 'simd']
+
 [tests/functional/snapshot:Linux]
 tests = ['snapshot_015_pos', 'snapshot_016_pos']
 tags = ['functional', 'snapshot']

tests/zfs-tests/tests/functional/Makefile.am

@@ -89,5 +89,6 @@ SUBDIRS = \
 
 if BUILD_LINUX
 SUBDIRS += \
+	simd \
 	tmpfile
 endif

tests/zfs-tests/tests/functional/simd/Makefile.am

@@ -0,0 +1,2 @@
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/simd
+dist_pkgdata_SCRIPTS = simd_supported.ksh