ZTS: Fix zvol_misc_volmode test

Changing volmode may need to remove minors, which could be open, so
call udev_wait() before we "zfs set volmode=<value>".  This ensures
no udev process has the zvol open (i.e. blkid) and the kernel
zvol_remove_minor_impl() function won't skip removing the in use
device.

Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Melikov <mail@gmelikov.ru>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes openzfs#13075

tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode.ksh

@@ -43,8 +43,6 @@
 # 7. Verify "volmode" behaves correctly at import time
 # 8. Verify "volmode" behaves accordingly to zvol_inhibit_dev (Linux only)
 #
-# NOTE: changing volmode may need to remove minors, which could be open, so call
-#       block_device_wait() before we "zfs set volmode=<value>".
 
 verify_runnable "global"
 
@@ -93,6 +91,21 @@ function test_io # dev
 	log_must dd if=/dev/zero of=$dev count=1
 }
 
+#
+# Changing volmode may need to remove minors, which could be open, so call
+# udev_wait() before we "zfs set volmode=<value>".  This ensures no udev
+# process has the zvol open (i.e. blkid) and the zvol_remove_minor_impl()
+# function won't skip removing the in use device.
+#
+function set_volmode # value ds
+{
+	typeset value="$1"
+	typeset ds="$2"
+
+	is_linux && udev_wait
+	log_must zfs set volmode="$value" "$ds"
+}
+
 log_assert "Verify that ZFS volume property 'volmode' works as intended"
 log_onexit cleanup
 
@@ -119,20 +132,20 @@ do
 done
 
 # 2. Verify "volmode=none" hides ZVOL device nodes
-log_must zfs set volmode=none $ZVOL
+set_volmode none $ZVOL
 blockdev_missing $ZDEV
 log_must_busy zfs destroy $ZVOL
 blockdev_missing $ZDEV
 
 # 3. Verify "volmode=full" exposes a fully functional device
 log_must zfs create -V $VOLSIZE -s $ZVOL
 blockdev_exists $ZDEV
-log_must zfs set volmode=full $ZVOL
+set_volmode full $ZVOL
 blockdev_exists $ZDEV
 test_io $ZDEV
 log_must verify_partition $ZDEV
 # 3.1 Verify "volmode=geom" is an alias for "volmode=full"
-log_must zfs set volmode=geom $ZVOL
+set_volmode geom $ZVOL
 blockdev_exists $ZDEV
 if [[ "$(get_prop 'volmode' $ZVOL)" != "full" ]]; then
 	log_fail " Volmode value 'geom' is not an alias for 'full'"
@@ -143,7 +156,7 @@ blockdev_missing $ZDEV
 # 4. Verify "volmode=dev" hides partition info on the device
 log_must zfs create -V $VOLSIZE -s $ZVOL
 blockdev_exists $ZDEV
-log_must zfs set volmode=dev $ZVOL
+set_volmode dev $ZVOL
 blockdev_exists $ZDEV
 test_io $ZDEV
 log_mustnot verify_partition $ZDEV
@@ -155,7 +168,7 @@ blockdev_missing $ZDEV
 sysctl_volmode 1
 log_must zfs create -V $VOLSIZE -s $ZVOL
 blockdev_exists $ZDEV
-log_must zfs set volmode=default $ZVOL
+set_volmode default $ZVOL
 blockdev_exists $ZDEV
 log_must verify_partition $ZDEV
 log_must_busy zfs destroy $ZVOL
@@ -164,7 +177,7 @@ blockdev_missing $ZDEV
 sysctl_volmode 2
 log_must zfs create -V $VOLSIZE -s $ZVOL
 blockdev_exists $ZDEV
-log_must zfs set volmode=default $ZVOL
+set_volmode default $ZVOL
 blockdev_exists $ZDEV
 log_mustnot verify_partition $ZDEV
 log_must_busy zfs destroy $ZVOL
@@ -173,33 +186,33 @@ blockdev_missing $ZDEV
 sysctl_volmode 3
 log_must zfs create -V $VOLSIZE -s $ZVOL
 blockdev_missing $ZDEV
-log_must zfs set volmode=default $ZVOL
+set_volmode default $ZVOL
 blockdev_missing $ZDEV
 
 # 6. Verify "volmode" property is inherited correctly
 log_must zfs inherit volmode $ZVOL
 blockdev_missing $ZDEV
 # 6.1 Check volmode=full case
-log_must zfs set volmode=full $TESTPOOL
+set_volmode full $TESTPOOL
 verify_inherited 'volmode' 'full' $ZVOL $TESTPOOL
 blockdev_exists $ZDEV
 # 6.2 Check volmode=none case
-log_must zfs set volmode=none $TESTPOOL
+set_volmode none $TESTPOOL
 verify_inherited 'volmode' 'none' $ZVOL $TESTPOOL
 blockdev_missing $ZDEV
 # 6.3 Check volmode=dev case
-log_must zfs set volmode=dev $TESTPOOL
+set_volmode dev $TESTPOOL
 verify_inherited 'volmode' 'dev' $ZVOL $TESTPOOL
 blockdev_exists $ZDEV
 # 6.4 Check volmode=default case
 sysctl_volmode 1
-log_must zfs set volmode=default $TESTPOOL
+set_volmode default $TESTPOOL
 verify_inherited 'volmode' 'default' $ZVOL $TESTPOOL
 blockdev_exists $ZDEV
 # 6.5 Check inheritance on multiple levels
 log_must zfs inherit volmode $SUBZVOL
-log_must zfs set volmode=none $VOLFS
-log_must zfs set volmode=full $TESTPOOL
+set_volmode none $VOLFS
+set_volmode full $TESTPOOL
 verify_inherited 'volmode' 'none' $SUBZVOL $VOLFS
 blockdev_missing $SUBZDEV
 blockdev_exists $ZDEV
@@ -223,23 +236,23 @@ if is_linux; then
 	sysctl_volmode 1
 	log_must zfs create -V $VOLSIZE -s $ZVOL
 	blockdev_missing $ZDEV
-	log_must zfs set volmode=full $ZVOL
+	set_volmode full $ZVOL
 	blockdev_missing $ZDEV
 	log_must_busy zfs destroy $ZVOL
 	blockdev_missing $ZDEV
 	# 7.1 Verify device nodes not are not created with "volmode=dev"
 	sysctl_volmode 2
 	log_must zfs create -V $VOLSIZE -s $ZVOL
 	blockdev_missing $ZDEV
-	log_must zfs set volmode=dev $ZVOL
+	set_volmode dev $ZVOL
 	blockdev_missing $ZDEV
 	log_must_busy zfs destroy $ZVOL
 	blockdev_missing $ZDEV
 	# 7.1 Verify device nodes not are not created with "volmode=none"
 	sysctl_volmode 3
 	log_must zfs create -V $VOLSIZE -s $ZVOL
 	blockdev_missing $ZDEV
-	log_must zfs set volmode=none $ZVOL
+	set_volmode none $ZVOL
 	blockdev_missing $ZDEV
 fi