Add scrub after resilver zed script

* Add a zed script to kick off a scrub after a resilver.  The script is
disabled by default.

* Add a optional $PATH (-P) option to zed to allow it to use a custom
$PATH for its zedlets.  This is needed when you're running zed under
the ZTS in a local workspace.

Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Closes: #4662

cmd/zed/Makefile.am

@@ -66,7 +66,8 @@ dist_zedexec_SCRIPTS = \
 	zed.d/statechange-notify.sh \
 	zed.d/vdev_clear-led.sh \
 	zed.d/vdev_attach-led.sh \
-	zed.d/pool_import-led.sh
+	zed.d/pool_import-led.sh \
+	zed.d/resilver_finish-start-scrub.sh
 
 zedconfdefaults = \
 	all-syslog.sh \
@@ -77,7 +78,8 @@ zedconfdefaults = \
 	statechange-notify.sh \
 	vdev_clear-led.sh \
 	vdev_attach-led.sh \
-	pool_import-led.sh
+	pool_import-led.sh \
+	resilver_finish-start-scrub.sh
 
 install-data-hook:
 	$(MKDIR_P) "$(DESTDIR)$(zedconfdir)"

cmd/zed/zed.d/resilver_finish-start-scrub.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+# resilver_finish-start-scrub.sh
+# Run a scrub after a resilver
+#
+# Exit codes:
+# 1: Internal error
+# 2: Script wasn't enabled in zed.rc
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+[ "${ZED_SCRUB_AFTER_RESILVER}" = "1" ] || exit 2
+[ -n "${ZEVENT_POOL}" ] || exit 1
+[ -n "${ZEVENT_SUBCLASS}" ] || exit 1
+zed_check_cmd "${ZPOOL}" || exit 1
+
+zed_log_msg "Starting scrub after resilver on ${ZEVENT_POOL}"
+"${ZPOOL}" scrub "${ZEVENT_POOL}"

cmd/zed/zed.d/zed.rc

@@ -86,6 +86,9 @@
 #
 ZED_USE_ENCLOSURE_LEDS=1
 
+##
+# Run a scrub after every resilver
+#ZED_SCRUB_AFTER_RESILVER=1
 
 ##
 # The syslog priority (e.g., specified as a "facility.level" pair).

cmd/zed/zed_conf.c

@@ -155,6 +155,8 @@ _zed_conf_display_help(const char *prog, int got_err)
 	    "Run daemon in the foreground.");
 	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-M",
 	    "Lock all pages in memory.");
+	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-P",
+	    "$PATH for ZED to use (only used by ZTS).");
 	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-Z",
 	    "Zero state file.");
 	fprintf(fp, "\n");
@@ -247,7 +249,7 @@ _zed_conf_parse_path(char **resultp, const char *path)
 void
 zed_conf_parse_opts(struct zed_conf *zcp, int argc, char **argv)
 {
-	const char * const opts = ":hLVc:d:p:s:vfFMZ";
+	const char * const opts = ":hLVc:d:p:P:s:vfFMZ";
 	int opt;
 
 	if (!zcp || !argv || !argv[0])
@@ -275,6 +277,9 @@ zed_conf_parse_opts(struct zed_conf *zcp, int argc, char **argv)
 		case 'p':
 			_zed_conf_parse_path(&zcp->pid_file, optarg);
 			break;
+		case 'P':
+			_zed_conf_parse_path(&zcp->path, optarg);
+			break;
 		case 's':
 			_zed_conf_parse_path(&zcp->state_file, optarg);
 			break;

cmd/zed/zed_conf.h

@@ -37,6 +37,7 @@ struct zed_conf {
 	int		state_fd;		/* fd to state file */
 	libzfs_handle_t	*zfs_hdl;		/* handle to libzfs */
 	int		zevent_fd;		/* fd for access to zevents */
+	char		*path;		/* custom $PATH for zedlets to use */
 };
 
 struct zed_conf *zed_conf_create(void);

cmd/zed/zed_event.c

@@ -733,12 +733,14 @@ _zed_event_add_nvpair(uint64_t eid, zed_strings_t *zsp, nvpair_t *nvp)
 
 /*
  * Restrict various environment variables to safe and sane values
- * when constructing the environment for the child process.
+ * when constructing the environment for the child process, unless
+ * we're running with a custom $PATH (like under the ZFS test suite).
  *
  * Reference: Secure Programming Cookbook by Viega & Messier, Section 1.1.
  */
 static void
-_zed_event_add_env_restrict(uint64_t eid, zed_strings_t *zsp)
+_zed_event_add_env_restrict(uint64_t eid, zed_strings_t *zsp,
+    const char *path)
 {
 	const char *env_restrict[][2] = {
 		{ "IFS",		" \t\n" },
@@ -753,11 +755,35 @@ _zed_event_add_env_restrict(uint64_t eid, zed_strings_t *zsp)
 		{ "ZFS_RELEASE",	ZFS_META_RELEASE },
 		{ NULL,			NULL }
 	};
+
+	/*
+	 * If we have a custom $PATH, use the default ZFS binary locations
+	 * instead of the hard-coded ones.
+	 */
+	const char *env_path[][2] = {
+		{ "IFS",		" \t\n" },
+		{ "PATH",		NULL }, /* $PATH copied in later on */
+		{ "ZDB",		"zdb" },
+		{ "ZED",		"zed" },
+		{ "ZFS",		"zfs" },
+		{ "ZINJECT",		"zinject" },
+		{ "ZPOOL",		"zpool" },
+		{ "ZFS_ALIAS",		ZFS_META_ALIAS },
+		{ "ZFS_VERSION",	ZFS_META_VERSION },
+		{ "ZFS_RELEASE",	ZFS_META_RELEASE },
+		{ NULL,			NULL }
+	};
 	const char *(*pa)[2];
 
 	assert(zsp != NULL);
 
-	for (pa = env_restrict; *(*pa); pa++) {
+	pa = path != NULL ? env_path : env_restrict;
+
+	for (; *(*pa); pa++) {
+		/* Use our custom $PATH if we have one */
+		if (path != NULL && strcmp((*pa)[0], "PATH") == 0)
+			(*pa)[1] = path;
+
 		_zed_event_add_var(eid, zsp, NULL, (*pa)[0], "%s", (*pa)[1]);
 	}
 }
@@ -902,7 +928,7 @@ zed_event_service(struct zed_conf *zcp)
 		while ((nvp = nvlist_next_nvpair(nvl, nvp)))
 			_zed_event_add_nvpair(eid, zsp, nvp);
 
-		_zed_event_add_env_restrict(eid, zsp);
+		_zed_event_add_env_restrict(eid, zsp, zcp->path);
 		_zed_event_add_env_preserve(eid, zsp);
 
 		_zed_event_add_var(eid, zsp, ZED_VAR_PREFIX, "PID",

man/man8/zed.8.in

@@ -27,6 +27,7 @@ ZED \- ZFS Event Daemon
 [\fB\-L\fR]
 [\fB\-M\fR]
 [\fB\-p\fR \fIpidfile\fR]
+[\fB\-P\fR \fIpath\fR]
 [\fB\-s\fR \fIstatefile\fR]
 [\fB\-v\fR]
 [\fB\-V\fR]
@@ -78,9 +79,16 @@ Read the enabled ZEDLETs from the specified directory.
 .BI \-p\  pidfile
 Write the daemon's process ID to the specified file.
 .TP
+.BI \-P\  path
+Custom $PATH for zedlets to use.  Normally zedlets run in a locked-down
+environment, with hardcoded paths to the ZFS commands ($ZFS, $ZPOOL, $ZED, ...),
+and a hardcoded $PATH.  This is done for security reasons.  However, the
+ZFS test suite uses a custom PATH for its ZFS commands, and passes it to zed
+with -P.  In short, -P is only to be used by the ZFS test suite; never use
+it in production!
+.TP
 .BI \-s\  statefile
 Write the daemon's state to the specified file.
-
 .SH ZEVENTS
 .PP
 A zevent is comprised of a list of nvpairs (name/value pairs).  Each zevent

tests/runfiles/linux.run

@@ -471,7 +471,8 @@ tags = ['functional', 'exec']
 
 [tests/functional/fault]
 tests = ['auto_online_001_pos', 'auto_replace_001_pos', 'auto_spare_001_pos',
-    'auto_spare_002_pos', 'auto_spare_ashift', 'auto_spare_multiple']
+    'auto_spare_002_pos', 'auto_spare_ashift', 'auto_spare_multiple',
+    'scrub_after_resilver']
 tags = ['functional', 'fault']
 
 [tests/functional/features/async_destroy]

tests/zfs-tests/include/libtest.shlib

@@ -3053,9 +3053,32 @@ function wait_replacing #pool
 	done
 }
 
+#
+# Wait for a pool to be scrubbed
+#
+# $1 pool name
+# $2 number of seconds to wait (optional)
+#
+# Returns true when pool has been scrubbed, or false if there's a timeout or if
+# no scrub was done.
+#
+function wait_scrubbed
+{
+	typeset pool=${1:-$TESTPOOL}
+	typeset iter=${2:-10}
+	for i in {1..$iter} ; do
+		if is_pool_scrubbed $pool ; then
+			return 0
+		fi
+		sleep 1
+	done
+	return 1
+}
+
 #
 # Setup custom environment for the ZED.
 #
+# $1 Optional zedlet script to copy into our zedlet test directory.
 function zed_setup
 {
 	if ! is_linux; then
@@ -3073,6 +3096,7 @@ function zed_setup
 	if [[ -e $VDEVID_CONF_ETC ]]; then
 		log_fail "Must not have $VDEVID_CONF_ETC file present on system"
 	fi
+	EXTRA_ZEDLET="$1"
 
 	# Create a symlink for /etc/zfs/vdev_id.conf file.
 	log_must ln -s $VDEVID_CONF $VDEVID_CONF_ETC
@@ -3081,6 +3105,9 @@ function zed_setup
 	# add additional ZEDLETs as needed for their specific test.
 	log_must cp ${ZEDLET_ETC_DIR}/zed.rc $ZEDLET_DIR
 	log_must cp ${ZEDLET_ETC_DIR}/zed-functions.sh $ZEDLET_DIR
+	if [[ -n "$EXTRA_ZEDLET" ]] ; then
+		log_must cp ${ZEDLET_ETC_DIR}/$EXTRA_ZEDLET $ZEDLET_DIR
+	fi
 
 	# Customize the zed.rc file to enable the full debug log.
 	log_must sed -i '/\#ZED_DEBUG_LOG=.*/d' $ZEDLET_DIR/zed.rc
@@ -3097,17 +3124,22 @@ function zed_setup
 #
 # Cleanup custom ZED environment.
 #
+# $1 Optional zedlet script(s) to remove from our zedlet test directory.
 function zed_cleanup
 {
 	if ! is_linux; then
 		return
 	fi
+	EXTRA_ZEDLET="$1"
 
 	log_must rm -f ${ZEDLET_DIR}/zed.rc
 	log_must rm -f ${ZEDLET_DIR}/zed-functions.sh
 	log_must rm -f ${ZEDLET_DIR}/all-syslog.sh
 	log_must rm -f ${ZEDLET_DIR}/all-debug.sh
 	log_must rm -f ${ZEDLET_DIR}/state
+	if [[ -n "$EXTRA_ZEDLET" ]] ; then
+		log_must rm -f ${ZEDLET_DIR}/$EXTRA_ZEDLET
+	fi
 	log_must rm -f $ZED_LOG
 	log_must rm -f $ZED_DEBUG_LOG
 	log_must rm -f $VDEVID_CONF_ETC
@@ -3139,7 +3171,7 @@ function zed_start
 	# run ZED in the background and redirect foreground logging
 	# output to $ZED_LOG.
 	log_must truncate -s 0 $ZED_DEBUG_LOG
-	log_must eval "zed -vF -d $ZEDLET_DIR -p $ZEDLET_DIR/zed.pid" \
+	log_must eval "zed -vF -d $ZEDLET_DIR -p $ZEDLET_DIR/zed.pid -P $PATH" \
 	    "-s $ZEDLET_DIR/state 2>$ZED_LOG &"
 
 	return 0

tests/zfs-tests/tests/functional/fault/Makefile.am

@@ -8,4 +8,5 @@ dist_pkgdata_SCRIPTS = \
 	auto_spare_001_pos.ksh \
 	auto_spare_002_pos.ksh \
 	auto_spare_ashift.ksh \
-	auto_spare_multiple.ksh
+	auto_spare_multiple.ksh \
+	scrub_after_resilver.ksh

tests/zfs-tests/tests/functional/fault/cleanup.ksh

@@ -31,6 +31,6 @@ verify_runnable "global"
 cleanup_devices $DISKS
 
 zed_stop
-zed_cleanup
+zed_cleanup resilver_finish-start-scrub.sh
 
 log_pass

tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh

@@ -0,0 +1,65 @@
+#!/bin/ksh -p
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
+# All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/fault/fault.cfg
+
+#
+# DESCRIPTION:
+# Test the scrub after resilver zedlet
+#
+# STRATEGY:
+# 1. Create a mirrored pool
+# 2. Fault a disk
+# 3. Replace the disk, starting a resilver
+# 4. Verify that a scrub happens after the resilver finishes
+#
+
+log_assert "Testing the scrub after resilver zedlet"
+
+# Backup our zed.rc
+zedrc_backup="$(mktemp)"
+log_must cp $ZEDLET_DIR/zed.rc $zedrc_backup
+
+# Enable ZED_SCRUB_AFTER_RESILVER
+eval "sed -i 's/\#ZED_SCRUB_AFTER_RESILVER/ZED_SCRUB_AFTER_RESILVER/g' $ZEDLET_DIR/zed.rc"
+
+function cleanup
+{
+	# Restore our zed.rc
+	log_must mv $zedrc_backup $ZEDLET_DIR/zed.rc
+	default_cleanup_noexit
+}
+
+log_onexit cleanup
+
+verify_disk_count "$DISKS" 3
+default_mirror_setup_noexit $DISK1 $DISK2
+
+log_must zpool offline -f $TESTPOOL $DISK1
+
+# Write to our degraded pool so we have some data to resilver
+log_must mkfile 16M $TESTDIR/file1
+
+# Replace the failed disks, forcing a resilver
+log_must zpool replace $TESTPOOL $DISK1 $DISK3
+
+# Wait for the resilver to finish, and then the subsequent scrub to finish.
+# Waiting for the scrub has the effect of waiting for both.  Timeout after 10
+# seconds if nothing is happening.
+log_must wait_scrubbed $TESTPOOL 10
+log_pass "Successfully ran the scrub after resilver zedlet"

tests/zfs-tests/tests/functional/fault/setup.ksh

@@ -28,7 +28,7 @@
 
 verify_runnable "global"
 
-zed_setup
+zed_setup resilver_finish-start-scrub.sh
 zed_start
 
 log_pass