Add zpool events tests

* events_001_pos - Verify the expected events are generated when
  invoking the various zpool sub-commands.  These events must
  appear in `zpool event` and be consumed by the ZED.

* events_002_pos - Verify the ZED consumes events on stat which
  were generated while it wasn't running.  Additionally, check
  that events are only processed once.

Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>

configure.ac

@@ -168,6 +168,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/cmd/threadsappend/Makefile
 	tests/zfs-tests/cmd/xattrtest/Makefile
 	tests/zfs-tests/include/Makefile
+	tests/zfs-tests/include/default.cfg
 	tests/zfs-tests/tests/Makefile
 	tests/zfs-tests/tests/functional/Makefile
 	tests/zfs-tests/tests/functional/acl/Makefile
@@ -234,6 +235,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/tests/functional/ctime/Makefile
 	tests/zfs-tests/tests/functional/delegate/Makefile
 	tests/zfs-tests/tests/functional/devices/Makefile
+	tests/zfs-tests/tests/functional/events/Makefile
 	tests/zfs-tests/tests/functional/exec/Makefile
 	tests/zfs-tests/tests/functional/fault/Makefile
 	tests/zfs-tests/tests/functional/features/async_destroy/Makefile

scripts/zfs-tests.sh

@@ -401,6 +401,8 @@ constrain_path
 # Check if ksh exists
 #
 [ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
+[ -e "$STF_SUITE/include/default.cfg" ] || fail \
+    "Missing $STF_SUITE/include/default.cfg file."
 
 #
 # Verify the ZFS module stack if loaded.

tests/runfiles/linux.run

@@ -344,6 +344,9 @@ tests = ['zfs_allow_001_pos', 'zfs_allow_002_pos',
 [tests/functional/devices]
 tests = ['devices_001_pos', 'devices_002_neg', 'devices_003_pos']
 
+[tests/functional/events]
+tests = ['events_001_pos', 'events_002_pos']
+
 [tests/functional/exec]
 tests = ['exec_001_pos', 'exec_002_neg']
 

tests/zfs-tests/include/.gitignore

@@ -0,0 +1 @@
+/default.cfg

tests/zfs-tests/include/Makefile.am

@@ -6,3 +6,8 @@ dist_pkgdata_SCRIPTS = \
 	math.shlib \
 	properties.shlib \
 	zpool_script.shlib
+
+EXTRA_DIST=default.cfg.in
+
+distclean-local::
+	-$(RM) $(dist_pkgdata_SCRIPTS)

tests/zfs-tests/include/default.cfg → tests/zfs-tests/include/default.cfg.in

@@ -33,8 +33,11 @@
 . $STF_SUITE/include/libtest.shlib
 
 # ZFS Directories
-export ZEDLETDIR=${ZEDLETDIR:-/etc/zfs/zed.d}
-export ZPOOLSCRIPTDIR=${ZPOOLSCRIPTDIR:-/etc/zfs/zpool.d}
+export prefix=@prefix@
+export exec_prefix=@exec_prefix@
+export ZEDLET_ETC_DIR=${ZEDLET_ETC_DIR:-@sysconfdir@/zfs/zed.d}
+export ZEDLET_LIBEXEC_DIR=${ZEDLET_LIBEXEC_DIR:-@libexecdir@/zfs/zed.d}
+export ZPOOL_SCRIPT_DIR=${ZPOOL_SCRIPT_DIR:-@sysconfdir@/zfs/zpool.d}
 
 # Define run length constants
 export RT_LONG="3"

tests/zfs-tests/include/libtest.shlib

@@ -3280,11 +3280,16 @@ function zed_setup
 
 	# Setup minimal ZED configuration.  Individual test cases should
 	# add additional ZEDLETs as needed for their specific test.
-	log_must cp ${ZEDLETDIR}/zed.rc $ZEDLET_DIR
-	log_must cp ${ZEDLETDIR}/zed-functions.sh $ZEDLET_DIR
-	log_must cp ${ZEDLETDIR}/all-syslog.sh $ZEDLET_DIR
+	log_must cp ${ZEDLET_ETC_DIR}/zed.rc $ZEDLET_DIR
+	log_must cp ${ZEDLET_ETC_DIR}/zed-functions.sh $ZEDLET_DIR
 
-	log_must zpool events -c
+	# Customize the zed.rc file to enable the full debug log.
+	log_must sed -i '/\#ZED_DEBUG_LOG=.*/d' $ZEDLET_DIR/zed.rc
+	echo "ZED_DEBUG_LOG=$ZEDLET_DIR/zed.debug.log" >>$ZEDLET_DIR/zed.rc
+
+	log_must cp ${ZEDLET_LIBEXEC_DIR}/all-syslog.sh $ZEDLET_DIR
+	log_must cp ${ZEDLET_LIBEXEC_DIR}/all-debug.sh $ZEDLET_DIR
+	log_must touch $ZEDLET_DIR/zed.debug.log
 }
 
 #
@@ -3299,8 +3304,10 @@ function zed_cleanup
 	log_must rm -f ${ZEDLET_DIR}/zed.rc
 	log_must rm -f ${ZEDLET_DIR}/zed-functions.sh
 	log_must rm -f ${ZEDLET_DIR}/all-syslog.sh
+	log_must rm -f ${ZEDLET_DIR}/all-debug.sh
 	log_must rm -f ${ZEDLET_DIR}/zed.pid
 	log_must rm -f ${ZEDLET_DIR}/zedlog
+	log_must rm -f ${ZEDLET_DIR}/zed.debug.log
 	log_must rm -f ${ZEDLET_DIR}/state
 	log_must rm -f $VDEVID_CONF_ETC
 	log_must rm -f $VDEVID_CONF
@@ -3332,6 +3339,8 @@ function zed_start
 	# output to zedlog
 	log_must eval "zed -vF -d $ZEDLET_DIR -p $ZEDLET_DIR/zed.pid" \
 	    "-s $ZEDLET_DIR/state 2>${ZEDLET_DIR}/zedlog &"
+
+	return 0
 }
 
 #
@@ -3343,10 +3352,13 @@ function zed_stop
 		return
 	fi
 
+	log_note "Stopping ZED"
 	if [[ -f ${ZEDLET_DIR}/zed.pid ]]; then
 		zedpid=$(cat ${ZEDLET_DIR}/zed.pid)
 		log_must kill $zedpid
 	fi
+
+	return 0
 }
 
 #

tests/zfs-tests/tests/functional/Makefile.am

@@ -14,6 +14,7 @@ SUBDIRS = \
 	ctime \
 	delegate \
 	devices \
+	events \
 	exec \
 	fault \
 	features \

tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_003_pos.ksh

@@ -53,10 +53,10 @@ else
 	testpool="${TESTPOOL%%/*}"
 fi
 
-files="$(ls $ZPOOLSCRIPTDIR)"
+files="$(ls $ZPOOL_SCRIPT_DIR)"
 scripts=""
 for i in $files ; do
-	if [ ! -x "$ZPOOLSCRIPTDIR/$i" ] ; then
+	if [ ! -x "$ZPOOL_SCRIPT_DIR/$i" ] ; then
 		# Skip non-executables
 		continue
 	fi

tests/zfs-tests/tests/functional/cli_user/zpool_iostat/zpool_iostat_005_pos.ksh

@@ -53,10 +53,10 @@ else
 	testpool=${TESTPOOL%%/*}
 fi
 
-files="$(ls $ZPOOLSCRIPTDIR)"
+files="$(ls $ZPOOL_SCRIPT_DIR)"
 scripts=""
 for i in $files ; do
-	if [ ! -x "$ZPOOLSCRIPTDIR/$i" ] ; then
+	if [ ! -x "$ZPOOL_SCRIPT_DIR/$i" ] ; then
 		# Skip non-executables
 		continue
 	fi

tests/zfs-tests/tests/functional/events/Makefile.am

@@ -0,0 +1,8 @@
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/events
+dist_pkgdata_SCRIPTS = \
+	setup.ksh \
+	cleanup.ksh \
+	events.cfg \
+	events_common.kshlib \
+	events_001_pos.ksh \
+	events_002_pos.ksh

tests/zfs-tests/tests/functional/events/cleanup.ksh

@@ -0,0 +1,31 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+zed_cleanup
+
+default_cleanup

tests/zfs-tests/tests/functional/events/events.cfg

@@ -0,0 +1,38 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+export MPOOL=mpool.$$
+
+VDEV1=$TEST_BASE_DIR/vdev1
+VDEV2=$TEST_BASE_DIR/vdev2
+VDEV3=$TEST_BASE_DIR/vdev3
+VDEV4=$TEST_BASE_DIR/vdev4
+
+export TMP_EVENTS=/tmp/tmp_events.$$
+export TMP_EVENTS_FULL=/tmp/tmp_events_full.$$
+export TMP_EVENT_FULL=/tmp/tmp_event_full.$$
+export TMP_EVENTS_ZED=/tmp/tmp_events_zed.$$
+export TMP_EVENT_ZED=/tmp/tmp_event_zed.$$

tests/zfs-tests/tests/functional/events/events_001_pos.ksh

@@ -0,0 +1,134 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+# DESCRIPTION:
+# Verify zpool events command logs events.
+#
+# STRATEGY:
+# 1. Execute zpool sub-commands on a pool.
+# 2. Verify the expected events are logged in 'zpool events'.
+# 3. Verify the expected events are logged by the ZED.
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/events/events_common.kshlib
+
+verify_runnable "both"
+
+function cleanup
+{
+	if poolexists $MPOOL; then
+		destroy_pool $MPOOL
+	fi
+
+	for file in $VDEV1 $VDEV2 $VDEV3 $VDEV4; do
+		[[ -f $file ]] && rm -f $file
+	done
+
+	log_must zed_stop
+}
+
+log_assert "Verify zpool sub-commands generate expected events"
+log_onexit cleanup
+
+log_must truncate -s $MINVDEVSIZE $VDEV1 $VDEV2 $VDEV3 $VDEV4
+log_must zed_start
+
+# Create a mirrored pool with two devices.
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.config_sync" \
+    -e "sysevent.fs.zfs.pool_create" \
+    "zpool create $MPOOL mirror $VDEV1 $VDEV2"
+
+# Add a cache device then remove it.
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.config_sync" \
+    -e "sysevent.fs.zfs.vdev_add" \
+    "zpool add -f $MPOOL spare $VDEV3"
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.vdev_remove_aux" \
+    "zpool remove $MPOOL $VDEV3"
+
+# Add a log device then remove it.
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.config_sync" \
+    -e "sysevent.fs.zfs.vdev_add" \
+    "zpool add -f $MPOOL log $VDEV3"
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.vdev_remove_dev" \
+    "zpool remove $MPOOL $VDEV3"
+
+# Offline then online a device.
+run_and_verify -p "$MPOOL"\
+    -e "resource.fs.zfs.statechange" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool offline $MPOOL $VDEV1"
+run_and_verify -p "$MPOOL" \
+    -e "resource.fs.zfs.statechange" \
+    -e "sysevent.fs.zfs.vdev_online" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool online $MPOOL $VDEV1"
+
+# Attach then detach a device from the mirror.
+run_and_verify -p "$MPOOL" \
+     -e "sysevent.fs.zfs.vdev_attach" \
+    "zpool attach $MPOOL $VDEV1 $VDEV4"
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.vdev_remove" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool detach $MPOOL $VDEV4"
+
+# Replace a device
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.vdev_attach" \
+    -e "sysevent.fs.zfs.resilver_start" \
+    -e "sysevent.fs.zfs.resilver_finish" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool replace -f $MPOOL $VDEV1 $VDEV4"
+
+# Scrub a pool.
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.scrub_start" \
+    -e "sysevent.fs.zfs.scrub_finish" \
+    "zpool scrub $MPOOL"
+
+# Export then import a pool (may change to a pool_export event)
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.pool_destroy" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool export $MPOOL"
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.pool_import" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool import -d $TEST_BASE_DIR $MPOOL"
+
+# Destroy the pool
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.pool_destroy" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool destroy $MPOOL"
+
+log_pass "Verify zpool sub-commands generate expected events"