Add BLAKE3 hash support and chksum benchmarking to OpenZFS

AUTHORS

@@ -285,6 +285,7 @@ CONTRIBUTORS:
     Tim Connors <tconnors@rather.puzzling.org>
     Tim Crawford <tcrawford@datto.com>
     Tim Haley <Tim.Haley@Sun.COM>
+    Tino Reichardt <milky-zfs@mcmilk.de>
     Tobin Harding <me@tobin.cc>
     Tom Caputi <tcaputi@datto.com>
     Tom Matthews <tom@axiom-partners.com>

cmd/ztest.c

@@ -121,6 +121,7 @@
 #include <sys/zfeature.h>
 #include <sys/dsl_userhold.h>
 #include <sys/abd.h>
+#include <sys/blake3.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -417,6 +418,7 @@ ztest_func_t ztest_device_removal;
 ztest_func_t ztest_spa_checkpoint_create_discard;
 ztest_func_t ztest_initialize;
 ztest_func_t ztest_trim;
+ztest_func_t ztest_blake3;
 ztest_func_t ztest_fletcher;
 ztest_func_t ztest_fletcher_incr;
 ztest_func_t ztest_verify_dnode_bt;
@@ -470,6 +472,7 @@ ztest_info_t ztest_info[] = {
 	ZTI_INIT(ztest_spa_checkpoint_create_discard, 1, &zopt_rarely),
 	ZTI_INIT(ztest_initialize, 1, &zopt_sometimes),
 	ZTI_INIT(ztest_trim, 1, &zopt_sometimes),
+	ZTI_INIT(ztest_blake3, 1, &zopt_rarely),
 	ZTI_INIT(ztest_fletcher, 1, &zopt_rarely),
 	ZTI_INIT(ztest_fletcher_incr, 1, &zopt_rarely),
 	ZTI_INIT(ztest_verify_dnode_bt, 1, &zopt_sometimes),
@@ -6373,6 +6376,92 @@ ztest_reguid(ztest_ds_t *zd, uint64_t id)
 	VERIFY3U(load, ==, spa_load_guid(spa));
 }
 
+void
+ztest_blake3(ztest_ds_t *zd, uint64_t id)
+{
+	(void) zd, (void) id;
+	hrtime_t end = gethrtime() + NANOSEC;
+	zio_cksum_salt_t salt;
+	void *salt_ptr = &salt.zcs_bytes;
+	struct abd *abd_data, *abd_meta;
+	void *buf, *templ;
+	int i, *ptr;
+	uint32_t size;
+	BLAKE3_CTX ctx;
+
+	size = ztest_random_blocksize();
+	buf = umem_alloc(size, UMEM_NOFAIL);
+	abd_data = abd_alloc(size, B_FALSE);
+	abd_meta = abd_alloc(size, B_TRUE);
+
+	for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++)
+		*ptr = ztest_random(UINT_MAX);
+	memset(salt_ptr, 'A', 32);
+
+	abd_copy_from_buf_off(abd_data, buf, 0, size);
+	abd_copy_from_buf_off(abd_meta, buf, 0, size);
+
+	while (gethrtime() <= end) {
+		int run_count = 100;
+		zio_cksum_t zc_ref1, zc_ref2;
+		zio_cksum_t zc_res1, zc_res2;
+
+		void *ref1 = &zc_ref1;
+		void *ref2 = &zc_ref2;
+		void *res1 = &zc_res1;
+		void *res2 = &zc_res2;
+
+		/* BLAKE3_KEY_LEN = 32 */
+		VERIFY0(blake3_set_impl_name("generic"));
+		templ = abd_checksum_blake3_tmpl_init(&salt);
+		Blake3_InitKeyed(&ctx, salt_ptr);
+		Blake3_Update(&ctx, buf, size);
+		Blake3_Final(&ctx, ref1);
+		zc_ref2 = zc_ref1;
+		ZIO_CHECKSUM_BSWAP(&zc_ref2);
+		abd_checksum_blake3_tmpl_free(templ);
+
+		VERIFY0(blake3_set_impl_name("cycle"));
+		while (run_count-- > 0) {
+
+			/* Test current implementation */
+			Blake3_InitKeyed(&ctx, salt_ptr);
+			Blake3_Update(&ctx, buf, size);
+			Blake3_Final(&ctx, res1);
+			zc_res2 = zc_res1;
+			ZIO_CHECKSUM_BSWAP(&zc_res2);
+
+			VERIFY0(memcmp(ref1, res1, 32));
+			VERIFY0(memcmp(ref2, res2, 32));
+
+			/* Test ABD - data */
+			templ = abd_checksum_blake3_tmpl_init(&salt);
+			abd_checksum_blake3_native(abd_data, size,
+			    templ, &zc_res1);
+			abd_checksum_blake3_byteswap(abd_data, size,
+			    templ, &zc_res2);
+
+			VERIFY0(memcmp(ref1, res1, 32));
+			VERIFY0(memcmp(ref2, res2, 32));
+
+			/* Test ABD - metadata */
+			abd_checksum_blake3_native(abd_meta, size,
+			    templ, &zc_res1);
+			abd_checksum_blake3_byteswap(abd_meta, size,
+			    templ, &zc_res2);
+			abd_checksum_blake3_tmpl_free(templ);
+
+			VERIFY0(memcmp(ref1, res1, 32));
+			VERIFY0(memcmp(ref2, res2, 32));
+
+		}
+	}
+
+	abd_free(abd_data);
+	abd_free(abd_meta);
+	umem_free(buf, size);
+}
+
 void
 ztest_fletcher(ztest_ds_t *zd, uint64_t id)
 {

config/always-arch.m4

@@ -30,6 +30,8 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_ARCH], [
 		;;
 	esac
 
+	AM_CONDITIONAL([TARGET_CPU_AARCH64], test $TARGET_CPU = aarch64)
 	AM_CONDITIONAL([TARGET_CPU_X86_64], test $TARGET_CPU = x86_64)
 	AM_CONDITIONAL([TARGET_CPU_POWERPC], test $TARGET_CPU = powerpc)
+	AM_CONDITIONAL([TARGET_CPU_SPARC64], test $TARGET_CPU = sparc64)
 ])

include/Makefile.am

@@ -23,6 +23,7 @@ COMMON_H = \
 	sys/avl.h \
 	sys/avl_impl.h \
 	sys/bitops.h \
+	sys/blake3.h \
 	sys/blkptr.h \
 	sys/bplist.h \
 	sys/bpobj.h \
@@ -117,6 +118,7 @@ COMMON_H = \
 	sys/zfeature.h \
 	sys/zfs_acl.h \
 	sys/zfs_bootenv.h \
+	sys/zfs_chksum.h \
 	sys/zfs_context.h \
 	sys/zfs_debug.h \
 	sys/zfs_delay.h \

include/os/freebsd/spl/sys/ccompile.h

@@ -74,10 +74,12 @@ extern "C" {
 
 #ifndef LOCORE
 #ifndef HAVE_RPC_TYPES
+#ifndef _KERNEL
 typedef int bool_t;
 typedef int enum_t;
 #endif
 #endif
+#endif
 
 #ifndef __cplusplus
 #define	__init

include/os/linux/kernel/linux/simd_powerpc.h

@@ -57,25 +57,45 @@
 #include <sys/types.h>
 #include <linux/version.h>
 
-#define	kfpu_allowed()		1
-#define	kfpu_begin()					\
-	{						\
-		preempt_disable();			\
-		enable_kernel_altivec();		\
-	}
+#define	kfpu_allowed()			1
+
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)
 #define	kfpu_end()				\
 	{					\
+		disable_kernel_vsx();		\
 		disable_kernel_altivec();	\
 		preempt_enable();		\
 	}
+#define	kfpu_begin()				\
+	{					\
+		preempt_disable();		\
+		enable_kernel_altivec();	\
+		enable_kernel_vsx();		\
+	}
 #else
-/* seems that before 4.5 no-one bothered disabling ... */
+/* seems that before 4.5 no-one bothered */
+#define	kfpu_begin()
 #define	kfpu_end()		preempt_enable()
 #endif
 #define	kfpu_init()		0
 #define	kfpu_fini()		((void) 0)
 
+static inline boolean_t
+zfs_vsx_available(void)
+{
+	boolean_t res;
+#if defined(__powerpc64__)
+	u64 msr;
+#else
+	u32 msr;
+#endif
+	kfpu_begin();
+	__asm volatile("mfmsr %0" : "=r"(msr));
+	res = (msr & 0x800000) != 0;
+	kfpu_end();
+	return (res);
+}
+
 /*
  * Check if AltiVec instruction set is available
  */

include/sys/blake3.h

@@ -0,0 +1,120 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Based on BLAKE3 v1.3.1, https://github.com/BLAKE3-team/BLAKE3
+ * Copyright (c) 2019-2020 Samuel Neves and Jack O'Connor
+ * Copyright (c) 2021 Tino Reichardt <milky-zfs@mcmilk.de>
+ */
+
+#ifndef BLAKE3_H
+#define	BLAKE3_H
+
+#ifdef  _KERNEL
+#include <sys/types.h>
+#else
+#include <stdint.h>
+#include <stdlib.h>
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define	BLAKE3_KEY_LEN		32
+#define	BLAKE3_OUT_LEN		32
+#define	BLAKE3_MAX_DEPTH	54
+#define	BLAKE3_BLOCK_LEN	64
+#define	BLAKE3_CHUNK_LEN	1024
+
+/*
+ * This struct is a private implementation detail.
+ * It has to be here because it's part of BLAKE3_CTX below.
+ */
+typedef struct {
+	uint32_t cv[8];
+	uint64_t chunk_counter;
+	uint8_t buf[BLAKE3_BLOCK_LEN];
+	uint8_t buf_len;
+	uint8_t blocks_compressed;
+	uint8_t flags;
+} blake3_chunk_state_t;
+
+typedef struct {
+	uint32_t key[8];
+	blake3_chunk_state_t chunk;
+	uint8_t cv_stack_len;
+
+	/*
+	 * The stack size is MAX_DEPTH + 1 because we do lazy merging. For
+	 * example, with 7 chunks, we have 3 entries in the stack. Adding an
+	 * 8th chunk requires a 4th entry, rather than merging everything down
+	 * to 1, because we don't know whether more input is coming. This is
+	 * different from how the reference implementation does things.
+	 */
+	uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN];
+
+	/* const blake3_impl_ops_t *ops */
+	const void *ops;
+} BLAKE3_CTX;
+
+/* init the context for hash operation */
+void Blake3_Init(BLAKE3_CTX *ctx);
+
+/* init the context for a MAC and/or tree hash operation */
+void Blake3_InitKeyed(BLAKE3_CTX *ctx, const uint8_t key[BLAKE3_KEY_LEN]);
+
+/* process the input bytes */
+void Blake3_Update(BLAKE3_CTX *ctx, const void *input, size_t input_len);
+
+/* finalize the hash computation and output the result */
+void Blake3_Final(const BLAKE3_CTX *ctx, uint8_t *out);
+
+/* finalize the hash computation and output the result */
+void Blake3_FinalSeek(const BLAKE3_CTX *ctx, uint64_t seek, uint8_t *out,
+    size_t out_len);
+
+/* return number of supported implementations */
+extern int blake3_get_impl_count(void);
+
+/* return id of selected implementation */
+extern int blake3_get_impl_id(void);
+
+/* return name of selected implementation */
+extern const char *blake3_get_impl_name(void);
+
+/* setup id as fastest implementation */
+extern void blake3_set_impl_fastest(uint32_t id);
+
+/* set implementation by id */
+extern void blake3_set_impl_id(uint32_t id);
+
+/* set implementation by name */
+extern int blake3_set_impl_name(const char *name);
+
+/* set startup implementation */
+extern void blake3_setup_impl(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* BLAKE3_H */

include/sys/zfs_chksum.h

@@ -0,0 +1,48 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2021 Tino Reichardt <milky-zfs@mcmilk.de>
+ */
+
+#ifndef	_ZFS_CHKSUM_H
+#define	_ZFS_CHKSUM_H
+
+#ifdef  _KERNEL
+#include <sys/types.h>
+#else
+#include <stdint.h>
+#include <stdlib.h>
+#endif
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/* Benchmark the chksums of ZFS when the module is loading */
+void chksum_init(void);
+void chksum_fini(void);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _ZFS_CHKSUM_H */