Add --enable-asan and --enable-ubsan switches

`configure` now accepts `--enable-asan` and `--enable-ubsan` switches
which results in passing `-fsanitize=address`
and `-fsanitize=undefined`, respectively, to the compiler. Those
flags are enabled in GitHub workflows for ZTS and zloop. Errors
reported by both instrumentations are corrected, except for:

- Memory leak reporting is (temporarily) suppressed. The cost of
  fixing them is relatively high compared to the gains.

- Checksum computing functions in `module/zcommon/zfs_fletcher*`
  have UBSan errors suppressed. It is completely impractical
  to enforce 64-byte payload alignment there due to performance
  impact.

- There's no ASan heap poisoning in `module/zstd/lib/zstd.c`. A custom
  memory allocator is used there rendering that measure
  unfeasible.

- Memory leaks detection has to be suppressed for `cmd/zvol_id`.
  `zvol_id` is run by udev with the help of `ptrace(2)`. Tracing is
  incompatible with memory leaks detection.

Reviewed-by: Ahelenia Ziemiańska <nabijaczleweli@nabijaczleweli.xyz>
Reviewed-by: George Melikov <mail@gmelikov.ru>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: szubersk <szuberskidamian@gmail.com>
Closes openzfs#12928

.github/workflows/zfs-tests-functional.yml

@@ -32,7 +32,7 @@ jobs:
         sh autogen.sh
     - name: Configure
       run: |
-        ./configure --enable-debug --enable-debuginfo
+        ./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan
     - name: Make
       run: |
         make --no-print-directory -s pkg-utils pkg-kmod

.github/workflows/zfs-tests-sanity.yml

@@ -28,7 +28,7 @@ jobs:
         sh autogen.sh
     - name: Configure
       run: |
-        ./configure --enable-debug --enable-debuginfo
+        ./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan
     - name: Make
       run: |
         make --no-print-directory -s pkg-utils pkg-kmod

.github/workflows/zloop.yml

@@ -28,7 +28,7 @@ jobs:
         sh autogen.sh
     - name: Configure
       run: |
-        ./configure --enable-debug --enable-debuginfo
+        ./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan
     - name: Make
       run: |
         make --no-print-directory -s pkg-utils pkg-kmod

cmd/zdb/zdb.c

@@ -8659,7 +8659,7 @@ main(int argc, char **argv)
 			dump_opt[c] += verbose;
 	}
 
-	libspl_assert_ok = (dump_opt['A'] == 1) || (dump_opt['A'] > 2);
+	libspl_set_assert_ok((dump_opt['A'] == 1) || (dump_opt['A'] > 2));
 	zfs_recover = (dump_opt['A'] > 1);
 
 	argc -= optind;

cmd/zfs/zfs_main.c

@@ -575,8 +575,9 @@ usage(boolean_t requested)
 
 		(void) fprintf(fp, gettext("\nSizes are specified in bytes "
 		    "with standard units such as K, M, G, etc.\n"));
-		(void) fprintf(fp, gettext("\nUser-defined properties can "
-		    "be specified by using a name containing a colon (:).\n"));
+		(void) fprintf(fp, "%s", gettext("\nUser-defined properties "
+		    "can be specified by using a name containing a colon "
+		    "(:).\n"));
 		(void) fprintf(fp, gettext("\nThe {user|group|project}"
 		    "[obj]{used|quota}@ properties must be appended with\n"
 		    "a user|group|project specifier of one of these forms:\n"

cmd/zpool/zpool_main.c

@@ -1760,17 +1760,19 @@ zpool_do_create(int argc, char **argv)
 			    "feature@%s", feat->fi_uname);
 
 			if (!nvlist_lookup_string(props, propname, &propval)) {
-				if (strcmp(propval, ZFS_FEATURE_DISABLED) == 0)
+				if (strcmp(propval,
+				    ZFS_FEATURE_DISABLED) == 0) {
 					(void) nvlist_remove_all(props,
 					    propname);
-				if (strcmp(propval,
+				} else if (strcmp(propval,
 				    ZFS_FEATURE_ENABLED) == 0 &&
-				    !requested_features[i])
+				    !requested_features[i]) {
 					(void) fprintf(stderr, gettext(
 					    "Warning: feature \"%s\" enabled "
 					    "but is not in specified "
 					    "'compatibility' feature set.\n"),
 					    feat->fi_uname);
+				}
 			} else if (
 			    enable_pool_features &&
 			    feat->fi_zfs_mod_supported &&
@@ -9315,7 +9317,7 @@ zpool_do_upgrade(int argc, char **argv)
 		}
 	}
 
-	(void) printf(gettext("This system supports ZFS pool feature "
+	(void) printf("%s", gettext("This system supports ZFS pool feature "
 	    "flags.\n\n"));
 	if (showversions) {
 		int i;

cmd/ztest/ztest.c

@@ -676,6 +676,8 @@ fatal(int do_perror, char *message, ...)
 
 	(void) fflush(stdout);
 	buf = umem_alloc(FATAL_MSG_SZ, UMEM_NOFAIL);
+	if (buf == NULL)
+		goto out;
 
 	va_start(args, message);
 	(void) sprintf(buf, "ztest: ");
@@ -689,6 +691,7 @@ fatal(int do_perror, char *message, ...)
 	(void) fprintf(stderr, "%s\n", buf);
 	fatal_msg = buf;			/* to ease debugging */
 
+out:
 	if (ztest_dump_core)
 		abort();
 	else

cmd/zvol_id/zvol_id_main.c

@@ -35,6 +35,21 @@
 #include <sys/zfs_znode.h>
 #include <sys/fs/zfs.h>
 
+#if defined(ZFS_ASAN_ENABLED)
+/*
+ * zvol_id is invoked by udev with the help of ptrace()
+ * making sanitized binary with leak detection croak
+ * because of tracing mechanisms collision
+ */
+extern const char *__asan_default_options(void);
+
+const char *__asan_default_options(void) {
+	return ("abort_on_error=true:halt_on_error=true:"
+		"allocator_may_return_null=true:disable_coredump=false:"
+		"detect_stack_use_after_return=true:detect_leaks=false");
+}
+#endif
+
 static int
 ioctl_get_msg(char *var, int fd)
 {

config/Rules.am

@@ -29,6 +29,7 @@ AM_CFLAGS += $(NO_OMIT_FRAME_POINTER)
 AM_CFLAGS += $(IMPLICIT_FALLTHROUGH)
 AM_CFLAGS += $(DEBUG_CFLAGS)
 AM_CFLAGS += $(ASAN_CFLAGS)
+AM_CFLAGS += $(UBSAN_CFLAGS)
 AM_CFLAGS += $(CODE_COVERAGE_CFLAGS) $(NO_FORMAT_ZERO_LENGTH)
 if BUILD_FREEBSD
 AM_CFLAGS += -fPIC -Werror -Wno-unknown-pragmas -Wno-enum-conversion
@@ -58,8 +59,17 @@ AM_CPPFLAGS += -D"__xpg_basename(...)=__xpg_basename(__VA_ARGS__) __attribute__(
 AM_CPPFLAGS += -D"basename(...)=basename(__VA_ARGS__) __attribute__((deprecated(\"basename(3) is underspecified. Use zfs_basename() instead!\")))"
 AM_CPPFLAGS += -D"dirname(...)=dirname(__VA_ARGS__) __attribute__((deprecated(\"dirname(3) is underspecified. Use zfs_dirnamelen() instead!\")))"
 
+if ASAN_ENABLED
+AM_CPPFLAGS += -DZFS_ASAN_ENABLED
+endif
+
+if UBSAN_ENABLED
+AM_CPPFLAGS += -DZFS_UBSAN_ENABLED
+endif
+
 AM_LDFLAGS  = $(DEBUG_LDFLAGS)
 AM_LDFLAGS += $(ASAN_LDFLAGS)
+AM_LDFLAGS += $(UBSAN_LDFLAGS)
 
 if BUILD_FREEBSD
 AM_LDFLAGS += -fstack-protector-strong -shared

config/Substfiles.am

@@ -17,7 +17,9 @@ subst_sed_cmd = \
 	-e 's|@DEFAULT_INIT_NFS_SERVER[@]|$(DEFAULT_INIT_NFS_SERVER)|g' \
 	-e 's|@DEFAULT_INIT_SHELL[@]|$(DEFAULT_INIT_SHELL)|g' \
 	-e 's|@LIBFETCH_DYNAMIC[@]|$(LIBFETCH_DYNAMIC)|g' \
-	-e 's|@LIBFETCH_SONAME[@]|$(LIBFETCH_SONAME)|g'
+	-e 's|@LIBFETCH_SONAME[@]|$(LIBFETCH_SONAME)|g' \
+	-e 's|@ASAN_ENABLED[@]|$(ASAN_ENABLED)|g' \
+	-e 's|@UBSAN_ENABLED[@]|$(UBSAN_ENABLED)|g'
 
 SUBSTFILES =
 CLEANFILES = $(SUBSTFILES)

config/always-compiler-options.m4

@@ -45,6 +45,53 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_ASAN], [
 	AC_SUBST([ASAN_ZFS])
 ])
 
+dnl #
+dnl # Enabled -fsanitize=undefined if supported by gcc.
+dnl #
+dnl # LDFLAGS needs -fsanitize=undefined at all times so libraries compiled with
+dnl # it will be linked successfully. CFLAGS will vary by binary being built.
+dnl #
+dnl # The UBSAN_OPTIONS environment variable can be used to further control
+dnl # the behavior of binaries and libraries build with -fsanitize=undefined.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_UBSAN], [
+	AC_MSG_CHECKING([whether to build with -fsanitize=undefined support])
+	AC_ARG_ENABLE([ubsan],
+		[AS_HELP_STRING([--enable-ubsan],
+		[Enable -fsanitize=undefined support  @<:@default=no@:>@])],
+		[],
+		[enable_ubsan=no])
+
+	AM_CONDITIONAL([UBSAN_ENABLED], [test x$enable_ubsan = xyes])
+	AC_SUBST([UBSAN_ENABLED], [$enable_ubsan])
+	AC_MSG_RESULT($enable_ubsan)
+
+	AS_IF([ test "$enable_ubsan" = "yes" ], [
+		AC_MSG_CHECKING([whether $CC supports -fsanitize=undefined])
+		saved_cflags="$CFLAGS"
+		CFLAGS="$CFLAGS -Werror -fsanitize=undefined"
+		AC_LINK_IFELSE([
+			AC_LANG_SOURCE([[ int main() { return 0; } ]])
+		], [
+			UBSAN_CFLAGS="-fsanitize=undefined"
+			UBSAN_LDFLAGS="-fsanitize=undefined"
+			UBSAN_ZFS="_with_ubsan"
+			AC_MSG_RESULT([yes])
+		], [
+			AC_MSG_ERROR([$CC does not support -fsanitize=undefined])
+		])
+		CFLAGS="$saved_cflags"
+	], [
+		UBSAN_CFLAGS=""
+		UBSAN_LDFLAGS=""
+		UBSAN_ZFS="_without_ubsan"
+	])
+
+	AC_SUBST([UBSAN_CFLAGS])
+	AC_SUBST([UBSAN_LDFLAGS])
+	AC_SUBST([UBSAN_ZFS])
+])
+
 dnl #
 dnl # Check if gcc supports -Wframe-larger-than=<size> option.
 dnl #

config/zfs-build.m4

@@ -218,6 +218,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_OMIT_FRAME_POINTER
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_IPA_SRA
 	ZFS_AC_CONFIG_ALWAYS_CC_ASAN
+	ZFS_AC_CONFIG_ALWAYS_CC_UBSAN
 	ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD
 	ZFS_AC_CONFIG_ALWAYS_SYSTEM
 	ZFS_AC_CONFIG_ALWAYS_ARCH
@@ -323,6 +324,7 @@ AC_DEFUN([ZFS_AC_RPM], [
 	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(DEBUG_KMEM_ZFS) 1"'
 	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(DEBUG_KMEM_TRACKING_ZFS) 1"'
 	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(ASAN_ZFS) 1"'
+	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(UBSAN_ZFS) 1"'
 
 	RPM_DEFINE_UTIL=' --define "_initconfdir $(initconfdir)"'
 

include/zfs_fletcher.h

@@ -160,4 +160,20 @@ _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_aarch64_neon_ops;
 }
 #endif
 
+#if	defined(ZFS_UBSAN_ENABLED)
+#if	defined(__has_attribute)
+#if	__has_attribute(no_sanitize_undefined)
+#define	ZFS_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined))
+#elif	__has_attribute(no_sanitize)
+#define	ZFS_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined")))
+#else
+#error	"Compiler has to support attribute "
+	"`no_sanitize_undefined` or `no_sanitize(\"undefined\")`"
+	"when compiling with UBSan enabled"
+#endif	/* __has_attribute(no_sanitize_undefined) */
+#endif	/* defined(__has_attribute) */
+#else
+#define	ZFS_NO_SANITIZE_UNDEFINED
+#endif	/* defined(ZFS_UBSAN_ENABLED) */
+
 #endif	/* _ZFS_FLETCHER_H */

lib/libnvpair/libnvpair.abi

@@ -78,6 +78,7 @@
     <elf-symbol name='fnvpair_value_uint64' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='fnvpair_value_uint8' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='libspl_assertf' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='libspl_set_assert_ok' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='nv_alloc_fini' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='nv_alloc_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='nv_alloc_reset' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -230,7 +231,6 @@
     <elf-symbol name='nvpair_value_uint8_array' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
   </elf-function-symbols>
   <elf-variable-symbols>
-    <elf-symbol name='libspl_assert_ok' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='nv_alloc_nosleep' size='8' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='nv_fixed_ops' size='8' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
   </elf-variable-symbols>
@@ -2793,7 +2793,10 @@
     </function-type>
   </abi-instr>
   <abi-instr address-size='64' path='assert.c' language='LANG_C99'>
-    <var-decl name='libspl_assert_ok' type-id='95e97e5e' mangled-name='libspl_assert_ok' visibility='default' elf-symbol-id='libspl_assert_ok'/>
+    <function-decl name='libspl_set_assert_ok' mangled-name='libspl_set_assert_ok' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libspl_set_assert_ok'>
+      <parameter type-id='f58c8277' name='val'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
     <function-decl name='libspl_assertf' mangled-name='libspl_assertf' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libspl_assertf'>
       <parameter type-id='80f4b756' name='file'/>
       <parameter type-id='80f4b756' name='func'/>

lib/libspl/assert.c

@@ -25,7 +25,13 @@
 
 #include <assert.h>
 
-int libspl_assert_ok = 0;
+static boolean_t libspl_assert_ok = B_FALSE;
+
+void
+libspl_set_assert_ok(boolean_t val)
+{
+	libspl_assert_ok = val;
+}
 
 /* printf version of libspl_assert */
 void

lib/libspl/include/assert.h

@@ -32,9 +32,10 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdarg.h>
+#include <sys/types.h>
 
 /* Set to non-zero to avoid abort()ing on an assertion failure */
-extern int libspl_assert_ok;
+extern void libspl_set_assert_ok(boolean_t val);
 
 /* printf version of libspl_assert */
 extern void libspl_assertf(const char *file, const char *func, int line,

lib/libuutil/libuutil.abi

@@ -152,6 +152,7 @@
     <elf-symbol name='getmntany' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='getzoneid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='libspl_assertf' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='libspl_set_assert_ok' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='list_create' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='list_destroy' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='list_head' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -255,7 +256,6 @@
     <elf-symbol name='uu_zalloc' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
   </elf-function-symbols>
   <elf-variable-symbols>
-    <elf-symbol name='libspl_assert_ok' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='uu_exit_fatal_value' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='uu_exit_ok_value' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='uu_exit_usage_value' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -365,7 +365,10 @@
     </function-type>
   </abi-instr>
   <abi-instr address-size='64' path='assert.c' language='LANG_C99'>
-    <var-decl name='libspl_assert_ok' type-id='95e97e5e' mangled-name='libspl_assert_ok' visibility='default' elf-symbol-id='libspl_assert_ok'/>
+    <function-decl name='libspl_set_assert_ok' mangled-name='libspl_set_assert_ok' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libspl_set_assert_ok'>
+      <parameter type-id='f58c8277' name='val'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
     <function-decl name='libspl_assertf' mangled-name='libspl_assertf' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libspl_assertf'>
       <parameter type-id='80f4b756' name='file'/>
       <parameter type-id='80f4b756' name='func'/>

lib/libuutil/uu_pname.c

@@ -88,7 +88,8 @@ uu_warn_internal(int err, const char *format, va_list alist)
 	if (pname != NULL)
 		(void) fprintf(stderr, "%s: ", pname);
 
-	(void) vfprintf(stderr, format, alist);
+	if (format != NULL)
+		(void) vfprintf(stderr, format, alist);
 
 	if (strrchr(format, '\n') == NULL)
 		(void) fprintf(stderr, ": %s\n", strerror(err));

lib/libzfs/libzfs.abi

@@ -191,6 +191,7 @@
     <elf-symbol name='is_mounted' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='is_mpath_whole_disk' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='libspl_assertf' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='libspl_set_assert_ok' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='libzfs_add_handle' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='libzfs_envvar_is_set' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='libzfs_errno' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -593,7 +594,6 @@
     <elf-symbol name='fletcher_4_ssse3_ops' size='64' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='fletcher_4_superscalar4_ops' size='64' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='fletcher_4_superscalar_ops' size='64' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
-    <elf-symbol name='libspl_assert_ok' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='libzfs_config_ops' size='16' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='spa_feature_table' size='1904' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfeature_checks_disable' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -906,7 +906,10 @@
     <var-decl name='smb_shares' type-id='a3e5c654' visibility='default'/>
   </abi-instr>
   <abi-instr address-size='64' path='assert.c' language='LANG_C99'>
-    <var-decl name='libspl_assert_ok' type-id='95e97e5e' mangled-name='libspl_assert_ok' visibility='default' elf-symbol-id='libspl_assert_ok'/>
+    <function-decl name='libspl_set_assert_ok' mangled-name='libspl_set_assert_ok' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libspl_set_assert_ok'>
+      <parameter type-id='f58c8277' name='val'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
     <function-decl name='libspl_assertf' mangled-name='libspl_assertf' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libspl_assertf'>
       <parameter type-id='80f4b756' name='file'/>
       <parameter type-id='80f4b756' name='func'/>

lib/libzfs/libzfs_mount.c

@@ -1611,7 +1611,8 @@ zpool_disable_datasets(zpool_handle_t *zhp, boolean_t force)
 	 * At this point, we have the entire list of filesystems, so sort it by
 	 * mountpoint.
 	 */
-	qsort(sets, used, sizeof (struct sets_s), mountpoint_compare);
+	if (used != 0)
+		qsort(sets, used, sizeof (struct sets_s), mountpoint_compare);
 
 	/*
 	 * Walk through and first unshare everything.