ZED should handle spares using configured ashift

If the zpool 'ashift' property is set then ZED should use its value
when kicking in a hotspare; with this change 512e disks can be used
as spares for VDEVs that were created with ashift=9, even if ZFS
natively detects them as 4K block devices.

Also introduce an additional auto_spare test case which verifies that
in the face of multiple device failures an appropiate number of spares
are kicked in.

Signed-off-by: loli10K <ezomori.nozomu@gmail.com>
Requires-builders: style test

TEST

@@ -5,25 +5,25 @@
 #TEST_PREPARE_SHARES="yes"
 
 ### SPLAT
-#TEST_SPLAT_SKIP="yes"
+TEST_SPLAT_SKIP="yes"
 #TEST_SPLAT_OPTIONS="-acvx"
 
 ### ztest
-#TEST_ZTEST_SKIP="yes"
+TEST_ZTEST_SKIP="yes"
 #TEST_ZTEST_TIMEOUT=1800
 #TEST_ZTEST_DIR="/var/tmp/"
 #TEST_ZTEST_OPTIONS="-V"
 #TEST_ZTEST_CORE_DIR="/mnt/zloop"
 
 ### zimport
-#TEST_ZIMPORT_SKIP="yes"
+TEST_ZIMPORT_SKIP="yes"
 #TEST_ZIMPORT_DIR="/var/tmp/zimport"
 #TEST_ZIMPORT_VERSIONS="master installed"
 #TEST_ZIMPORT_POOLS="zol-0.6.1 zol-0.6.2 master installed"
 #TEST_ZIMPORT_OPTIONS="-c"
 
 ### xfstests
-#TEST_XFSTESTS_SKIP="yes"
+TEST_XFSTESTS_SKIP="yes"
 #TEST_XFSTESTS_URL="https://github.com/behlendorf/xfstests/archive/"
 #TEST_XFSTESTS_VER="zfs.tar.gz"
 #TEST_XFSTESTS_POOL="tank"
@@ -36,13 +36,14 @@
 #TEST_ZFSTESTS_DIR="/mnt/"
 #TEST_ZFSTESTS_DISKS="vdb vdc vdd"
 #TEST_ZFSTESTS_DISKSIZE="8G"
-#TEST_ZFSTESTS_ITERS="1"
+TEST_ZFSTESTS_ITERS="10"
 #TEST_ZFSTESTS_OPTIONS="-vx"
 #TEST_ZFSTESTS_RUNFILE="linux.run"
+TEST_ZFSTESTS_RUNFILE="issue-2562.run"
 #TEST_ZFSTESTS_TAGS="functional"
 
 ### zfsstress
-#TEST_ZFSSTRESS_SKIP="yes"
+TEST_ZFSSTRESS_SKIP="yes"
 #TEST_ZFSSTRESS_URL="https://github.com/nedbass/zfsstress/archive/"
 #TEST_ZFSSTRESS_VER="master.tar.gz"
 #TEST_ZFSSTRESS_RUNTIME=300

cmd/zed/agents/zfs_retire.c

@@ -176,6 +176,8 @@ replace_with_spare(fmd_hdl_t *hdl, zpool_handle_t *zhp, nvlist_t *vdev)
 	nvlist_t **spares;
 	uint_t s, nspares;
 	char *dev_name;
+	zprop_source_t source;
+	int ashift;
 
 	config = zpool_get_config(zhp, NULL);
 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
@@ -189,6 +191,11 @@ replace_with_spare(fmd_hdl_t *hdl, zpool_handle_t *zhp, nvlist_t *vdev)
 	    &spares, &nspares) != 0)
 		return;
 
+	/*
+	 * lookup "ashift" pool property, we may need it for the replacement
+	 */
+	ashift = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &source);
+
 	replacement = fmd_nvl_alloc(hdl, FMD_SLEEP);
 
 	(void) nvlist_add_string(replacement, ZPOOL_CONFIG_TYPE,
@@ -207,6 +214,11 @@ replace_with_spare(fmd_hdl_t *hdl, zpool_handle_t *zhp, nvlist_t *vdev)
 		    &spare_name) != 0)
 			continue;
 
+		/* if set, add the "ashift" pool property to the spare nvlist */
+		if (source != ZPROP_SRC_DEFAULT)
+			(void) nvlist_add_uint64(spares[s],
+			    ZPOOL_CONFIG_ASHIFT, ashift);
+
 		(void) nvlist_add_nvlist_array(replacement,
 		    ZPOOL_CONFIG_CHILDREN, &spares[s], 1);
 

tests/runfiles/issue-2562.run

@@ -0,0 +1,26 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+[DEFAULT]
+pre = setup
+quiet = False
+pre_user = root
+user = root
+timeout = 600
+post_user = root
+post = cleanup
+outputdir = /var/tmp/test_results
+
+[tests/functional/fault]
+tests = ['auto_online_001_pos', 'auto_replace_001_pos', 'auto_spare_001_pos',
+    'auto_spare_002_pos', 'auto_spare_ashift', 'auto_spare_multiple']
+tags = ['functional', 'fault']
+

tests/runfiles/linux.run

@@ -464,7 +464,7 @@ tags = ['functional', 'exec']
 
 [tests/functional/fault]
 tests = ['auto_online_001_pos', 'auto_replace_001_pos', 'auto_spare_001_pos',
-    'auto_spare_002_pos.ksh']
+    'auto_spare_002_pos', 'auto_spare_ashift', 'auto_spare_multiple']
 tags = ['functional', 'fault']
 
 [tests/functional/features/async_destroy]

tests/zfs-tests/include/blkdev.shlib

@@ -353,16 +353,35 @@ function insert_disk #disk scsi_host
 
 #
 # Load scsi_debug module with specified parameters
+# $blksz can be either one of: < 512b | 512e | 4Kn >
 #
-function load_scsi_debug # dev_size_mb add_host num_tgts max_luns
+function load_scsi_debug # dev_size_mb add_host num_tgts max_luns blksz
 {
 	typeset devsize=$1
 	typeset hosts=$2
 	typeset tgts=$3
 	typeset luns=$4
+	typeset blksz=$5
 
 	[[ -z $devsize ]] || [[ -z $hosts ]] || [[ -z $tgts ]] || \
-	    [[ -z $luns ]] && log_fail "Arguments invalid or missing"
+	    [[ -z $luns ]] || [[ -z $blksz ]] && \
+	    log_fail "Arguments invalid or missing"
+
+	case "$5" in
+		'512b')
+			typeset sector=512
+			typeset blkexp=0
+		;;
+		'512e')
+			typeset sector=512
+			typeset blkexp=3
+		;;
+		'4Kn')
+			typeset sector=4096
+			typeset blkexp=0
+		;;
+		*) log_fail "Unsupported blksz value: $5" ;;
+	esac
 
 	if is_linux; then
 		modprobe -n scsi_debug
@@ -375,7 +394,8 @@ function load_scsi_debug # dev_size_mb add_host num_tgts max_luns
 			log_fail "scsi_debug module already installed"
 		else
 			log_must modprobe scsi_debug dev_size_mb=$devsize \
-			    add_host=$hosts num_tgts=$tgts max_luns=$luns
+			    add_host=$hosts num_tgts=$tgts max_luns=$luns \
+			    sector_size=$sector physblk_exp=$blkexp
 			block_device_wait
 			lsscsi | egrep scsi_debug > /dev/null
 			if (($? == 1)); then
@@ -385,6 +405,16 @@ function load_scsi_debug # dev_size_mb add_host num_tgts max_luns
 	fi
 }
 
+#
+# Unload scsi_debug module, if needed.
+#
+function unload_scsi_debug
+{
+	if lsmod | grep scsi_debug >/dev/null; then
+		log_must modprobe -r scsi_debug
+	fi
+}
+
 #
 # Get scsi_debug device name.
 # Returns basename of scsi_debug device (for example "sdb").

tests/zfs-tests/include/libtest.shlib

@@ -3158,10 +3158,11 @@ function zed_stop
 	if [[ -f ${ZEDLET_DIR}/zed.pid ]]; then
 		zedpid=$(cat ${ZEDLET_DIR}/zed.pid)
 		kill $zedpid
-		wait $zedpid
+		while ps -p $zedpid > /dev/null; do
+			sleep 1
+		done
 		rm -f ${ZEDLET_DIR}/zed.pid
 	fi
-
 	return 0
 }
 

tests/zfs-tests/tests/functional/cli_root/zpool_reopen/cleanup.ksh

@@ -27,7 +27,7 @@ if is_linux; then
 	for SDDEVICE in $(get_debug_device); do
 		unplug $SDDEVICE
 	done
-	modprobe -r scsi_debug
+	unload_scsi_debug
 fi
 
 log_pass

tests/zfs-tests/tests/functional/cli_root/zpool_reopen/setup.ksh

@@ -22,7 +22,7 @@ verify_runnable "global"
 
 # Create scsi_debug devices for the reopen tests
 if is_linux; then
-	load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS
+	load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS '512b'
 else
 	log_unsupported "scsi debug module unsupported"
 fi

tests/zfs-tests/tests/functional/fault/Makefile.am

@@ -6,4 +6,6 @@ dist_pkgdata_SCRIPTS = \
 	auto_online_001_pos.ksh \
 	auto_replace_001_pos.ksh \
 	auto_spare_001_pos.ksh \
-	auto_spare_002_pos.ksh
+	auto_spare_002_pos.ksh \
+	auto_spare_ashift.ksh \
+	auto_spare_multiple.ksh

tests/zfs-tests/tests/functional/fault/auto_online_001_pos.ksh

@@ -54,9 +54,8 @@ fi
 
 function cleanup
 {
-	#online last disk before fail
-	insert_disk $offline_disk $host
-	poolexists $TESTPOOL && destroy_pool $TESTPOOL
+	destroy_pool $TESTPOOL
+	unload_scsi_debug
 }
 
 log_assert "Testing automated auto-online FMA test"
@@ -65,8 +64,8 @@ log_onexit cleanup
 
 # If using the default loop devices, need a scsi_debug device for auto-online
 if is_loop_device $DISK1; then
-	SD=$(lsscsi | nawk '/scsi_debug/ {print $6; exit}')
-	SDDEVICE=$(echo $SD | nawk -F / '{print $3}')
+	load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS '512b'
+	SDDEVICE=$(get_debug_device)
 	SDDEVICE_ID=$(get_persistent_disk_name $SDDEVICE)
 	autoonline_disks="$SDDEVICE"
 else

tests/zfs-tests/tests/functional/fault/auto_replace_001_pos.ksh

@@ -57,27 +57,23 @@ fi
 
 function setup
 {
-	lsmod | egrep scsi_debug > /dev/null
-	if (($? == 1)); then
-		load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS
-	fi
+	load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS '512b'
+	SD=$(get_debug_device)
+	SDDEVICE_ID=$(get_persistent_disk_name $SD)
 	# Register vdev_id alias rule for scsi_debug device to create a
 	# persistent path
-	SD=$(lsscsi | nawk '/scsi_debug/ {print $6; exit}' \
-	    | nawk -F / '{print $3}')
-	SDDEVICE_ID=$(get_persistent_disk_name $SD)
 	log_must eval "echo "alias scsidebug /dev/disk/by-id/$SDDEVICE_ID" \
 	    >> $VDEVID_CONF"
 	block_device_wait
-
-	SDDEVICE=$(udevadm info -q all -n $DEV_DSKDIR/$SD | egrep ID_VDEV \
-	    | nawk '{print $2; exit}' | nawk -F = '{print $2; exit}')
+	SDDEVICE=$(udevadm info -q all -n $DEV_DSKDIR/$SD \
+	    | awk -F'=' '/ID_VDEV=/{print $2; exit}')
 	[[ -z $SDDEVICE ]] && log_fail "vdev rule was not registered properly"
 }
 
 function cleanup
 {
-	poolexists $TESTPOOL && destroy_pool $TESTPOOL
+	destroy_pool $TESTPOOL
+	unload_scsi_debug
 }
 
 log_assert "Testing automated auto-replace FMA test"
@@ -112,7 +108,7 @@ log_must zpool export -F $TESTPOOL
 # Offline disk
 remove_disk $SD
 block_device_wait
-log_must modprobe -r scsi_debug
+unload_scsi_debug
 
 # Reimport pool with drive missing
 log_must zpool import $TESTPOOL

tests/zfs-tests/tests/functional/fault/auto_spare_001_pos.ksh

@@ -42,7 +42,7 @@ verify_runnable "both"
 function cleanup
 {
 	log_must zinject -c all
-	poolexists $TESTPOOL && destroy_pool $TESTPOOL
+	destroy_pool $TESTPOOL
 	rm -f $VDEV_FILES $SPARE_FILE
 }
 

tests/zfs-tests/tests/functional/fault/auto_spare_002_pos.ksh

@@ -42,7 +42,7 @@ verify_runnable "both"
 function cleanup
 {
 	log_must zinject -c all
-	poolexists $TESTPOOL && destroy_pool $TESTPOOL
+	destroy_pool $TESTPOOL
 	rm -f $VDEV_FILES $SPARE_FILE
 }