From 913ae452184edf34c175aad9efc41a8deb9292c2 Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Thu, 16 Dec 2021 16:22:15 -0500 Subject: [PATCH 001/109] FreeBSD: Provide correct file generation number va_seq was actually a thin veil over va_gen, so z_gen is a more appropriate value than z_seq to populate the field with. Drop the unnecessary compat obfuscation and provide the correct file generation number. Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Signed-off-by: Ryan Moeller Closes #12851 --- include/os/freebsd/spl/sys/vnode.h | 1 - module/os/freebsd/zfs/zfs_ctldir.c | 2 +- module/os/freebsd/zfs/zfs_vnops_os.c | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/include/os/freebsd/spl/sys/vnode.h b/include/os/freebsd/spl/sys/vnode.h index 3bc8a18eeb72..bcfc177a4751 100644 --- a/include/os/freebsd/spl/sys/vnode.h +++ b/include/os/freebsd/spl/sys/vnode.h @@ -132,7 +132,6 @@ vn_flush_cached_data(vnode_t *vp, boolean_t sync) /* TODO: This field needs conversion! */ #define va_nblocks va_bytes #define va_blksize va_blocksize -#define va_seq va_gen #define MAXOFFSET_T OFF_MAX #define EXCL 0 diff --git a/module/os/freebsd/zfs/zfs_ctldir.c b/module/os/freebsd/zfs/zfs_ctldir.c index cde40e87698b..3a5c9f8caf0a 100644 --- a/module/os/freebsd/zfs/zfs_ctldir.c +++ b/module/os/freebsd/zfs/zfs_ctldir.c @@ -496,7 +496,7 @@ zfsctl_common_getattr(vnode_t *vp, vattr_t *vap) */ vap->va_blksize = 0; vap->va_nblocks = 0; - vap->va_seq = 0; + vap->va_gen = 0; vn_fsid(vp, vap); vap->va_mode = zfsctl_ctldir_mode; vap->va_type = VDIR; diff --git a/module/os/freebsd/zfs/zfs_vnops_os.c b/module/os/freebsd/zfs/zfs_vnops_os.c index 8e48f78b7311..31ce860b442c 100644 --- a/module/os/freebsd/zfs/zfs_vnops_os.c +++ b/module/os/freebsd/zfs/zfs_vnops_os.c @@ -2065,7 +2065,7 @@ zfs_getattr(vnode_t *vp, vattr_t *vap, int flags, cred_t *cr) vap->va_size = zp->z_size; if (vp->v_type == VBLK || vp->v_type == VCHR) vap->va_rdev = zfs_cmpldev(rdev); - vap->va_seq = zp->z_seq; + vap->va_gen = zp->z_gen; vap->va_flags = 0; /* FreeBSD: Reset chflags(2) flags. */ vap->va_filerev = zp->z_seq; From 786abf53211d25102e9463e1ec10b6032f189db2 Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Wed, 22 Dec 2021 20:07:13 -0500 Subject: [PATCH 002/109] Reduce number of arc_prune threads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On FreeBSD vnode reclamation is single-threaded, protected by single global lock. Linux seems to be able to use a thread per mount point, but at this time it creates more harm than good. Reduce number of threads to 1, adding tunable in case somebody wants to try more. Reviewed-by: Ryan Moeller Reviewed-by: Brian Behlendorf Reviewed-by: Chris Dunlop Reviewed-by: Ahelenia Ziemiańska Signed-off-by: Alexander Motin Closes #12896 Issue #9966 --- man/man4/zfs.4 | 7 +++++++ module/zfs/arc.c | 13 ++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/man/man4/zfs.4 b/man/man4/zfs.4 index 20b24d898d84..657afc6169b6 100644 --- a/man/man4/zfs.4 +++ b/man/man4/zfs.4 @@ -671,6 +671,13 @@ Minimum time "prescient prefetched" blocks are locked in the ARC. These blocks are meant to be prefetched fairly aggressively ahead of the code that may use them. . +.It Sy zfs_arc_prune_task_threads Ns = Ns Sy 1 Pq int +Number of arc_prune threads. +.Fx +does not need more than one. +Linux may theoretically use one per mount point up to number of CPUs, +but that was not proven to be useful. +. .It Sy zfs_max_missing_tvds Ns = Ns Sy 0 Pq int Number of missing top-level vdevs which will be allowed during pool import (only in read-only mode). diff --git a/module/zfs/arc.c b/module/zfs/arc.c index 875986de99f0..9f22f877d20d 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -463,6 +463,11 @@ int zfs_arc_meta_strategy = ARC_STRATEGY_META_BALANCED; int zfs_arc_meta_adjust_restarts = 4096; int zfs_arc_lotsfree_percent = 10; +/* + * Number of arc_prune threads + */ +static int zfs_arc_prune_task_threads = 1; + /* The 6 states: */ arc_state_t ARC_anon; arc_state_t ARC_mru; @@ -7972,9 +7977,8 @@ arc_init(void) offsetof(arc_prune_t, p_node)); mutex_init(&arc_prune_mtx, NULL, MUTEX_DEFAULT, NULL); - arc_prune_taskq = taskq_create("arc_prune", 100, defclsyspri, - boot_ncpus, INT_MAX, TASKQ_PREPOPULATE | TASKQ_DYNAMIC | - TASKQ_THREADS_CPU_PCT); + arc_prune_taskq = taskq_create("arc_prune", zfs_arc_prune_task_threads, + defclsyspri, 100, INT_MAX, TASKQ_PREPOPULATE | TASKQ_DYNAMIC); arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED, sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); @@ -11106,4 +11110,7 @@ ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, eviction_pct, INT, ZMOD_RW, ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, evict_batch_limit, INT, ZMOD_RW, "The number of headers to evict per sublist before moving to the next"); + +ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, prune_task_threads, INT, ZMOD_RW, + "Number of arc_prune threads"); /* END CSTYLED */ From 4b2bac5fe9545b8a38b53a9b256ab14cc6085b9f Mon Sep 17 00:00:00 2001 From: Alan Somers Date: Fri, 17 Dec 2021 10:50:12 -0700 Subject: [PATCH 003/109] FreeBSD: Update argument types for VOP_READDIR A recent commit to FreeBSD changed the type of vop_readdir_args.a_cookies to a uint64_t**. There is no functional impact to ZFS because ZFS only uses 32-bit cookies, which will be zero-extended to 64-bits by the existing code. https://github.com/freebsd/freebsd-src/commit/b214fcceacad6b842545150664bd2695c1c2b34f Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Signed-off-by: Alan Somers Closes #12874 --- module/os/freebsd/zfs/zfs_vnops_os.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/module/os/freebsd/zfs/zfs_vnops_os.c b/module/os/freebsd/zfs/zfs_vnops_os.c index 31ce860b442c..6d3dac75484f 100644 --- a/module/os/freebsd/zfs/zfs_vnops_os.c +++ b/module/os/freebsd/zfs/zfs_vnops_os.c @@ -113,6 +113,12 @@ VFS_SMR_DECLARE; #define VNCHECKREF(vp) #endif +#if __FreeBSD_version >= 1400045 +typedef uint64_t cookie_t; +#else +typedef ulong_t cookie_t; +#endif + /* * Programming rules. * @@ -1679,7 +1685,7 @@ zfs_rmdir(znode_t *dzp, const char *name, znode_t *cwd, cred_t *cr, int flags) /* ARGSUSED */ static int zfs_readdir(vnode_t *vp, zfs_uio_t *uio, cred_t *cr, int *eofp, - int *ncookies, ulong_t **cookies) + int *ncookies, cookie_t **cookies) { znode_t *zp = VTOZ(vp); iovec_t *iovp; @@ -1701,7 +1707,7 @@ zfs_readdir(vnode_t *vp, zfs_uio_t *uio, cred_t *cr, int *eofp, boolean_t check_sysattrs; uint8_t type; int ncooks; - ulong_t *cooks = NULL; + cookie_t *cooks = NULL; int flags = 0; ZFS_ENTER(zfsvfs); @@ -1778,7 +1784,7 @@ zfs_readdir(vnode_t *vp, zfs_uio_t *uio, cred_t *cr, int *eofp, */ ncooks = zfs_uio_resid(uio) / (sizeof (struct dirent) - sizeof (((struct dirent *)NULL)->d_name) + 1); - cooks = malloc(ncooks * sizeof (ulong_t), M_TEMP, M_WAITOK); + cooks = malloc(ncooks * sizeof (*cooks), M_TEMP, M_WAITOK); *cookies = cooks; *ncookies = ncooks; } @@ -4732,7 +4738,7 @@ struct vop_readdir_args { struct ucred *a_cred; int *a_eofflag; int *a_ncookies; - ulong_t **a_cookies; + cookie_t **a_cookies; }; #endif From 9ec630ff2cd7563409b804455cc843815d00ae94 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Fri, 17 Dec 2021 09:52:13 -0800 Subject: [PATCH 004/109] Fix zvol_open() lock inversion When restructuring the zvol_open() logic for the Linux 5.13 kernel a lock inversion was accidentally introduced. In the updated code the spa_namespace_lock is now taken before the zv_suspend_lock allowing the following scenario to occur: down_read <=== waiting for zv_suspend_lock zvol_open <=== holds spa_namespace_lock __blkdev_get blkdev_get_by_dev blkdev_open ... mutex_lock <== waiting for spa_namespace_lock spa_open_common spa_open dsl_pool_hold dmu_objset_hold_flags dmu_objset_hold dsl_prop_get dsl_prop_get_integer zvol_create_minor dmu_recv_end zfs_ioc_recv_impl <=== holds zv_suspend_lock via zvol_suspend() zfs_ioc_recv ... This commit resolves the issue by moving the acquisition of the spa_namespace_lock back to after the zv_suspend_lock which restores the original ordering. Additionally, as part of this change the error exit paths were simplified where possible. Reviewed-by: Tony Hutter Reviewed-by: Rich Ercolani Signed-off-by: Brian Behlendorf Closes #12863 --- module/os/linux/zfs/zvol_os.c | 121 ++++++++++++++++------------------ 1 file changed, 58 insertions(+), 63 deletions(-) diff --git a/module/os/linux/zfs/zvol_os.c b/module/os/linux/zfs/zvol_os.c index 44caadd587f7..69479b3f7988 100644 --- a/module/os/linux/zfs/zvol_os.c +++ b/module/os/linux/zfs/zvol_os.c @@ -496,8 +496,7 @@ zvol_open(struct block_device *bdev, fmode_t flag) { zvol_state_t *zv; int error = 0; - boolean_t drop_suspend = B_TRUE; - boolean_t drop_namespace = B_FALSE; + boolean_t drop_suspend = B_FALSE; #ifndef HAVE_BLKDEV_GET_ERESTARTSYS hrtime_t timeout = MSEC2NSEC(zvol_open_timeout_ms); hrtime_t start = gethrtime(); @@ -517,7 +516,36 @@ zvol_open(struct block_device *bdev, fmode_t flag) return (SET_ERROR(-ENXIO)); } - if (zv->zv_open_count == 0 && !mutex_owned(&spa_namespace_lock)) { + mutex_enter(&zv->zv_state_lock); + /* + * Make sure zvol is not suspended during first open + * (hold zv_suspend_lock) and respect proper lock acquisition + * ordering - zv_suspend_lock before zv_state_lock + */ + if (zv->zv_open_count == 0) { + if (!rw_tryenter(&zv->zv_suspend_lock, RW_READER)) { + mutex_exit(&zv->zv_state_lock); + rw_enter(&zv->zv_suspend_lock, RW_READER); + mutex_enter(&zv->zv_state_lock); + /* check to see if zv_suspend_lock is needed */ + if (zv->zv_open_count != 0) { + rw_exit(&zv->zv_suspend_lock); + } else { + drop_suspend = B_TRUE; + } + } else { + drop_suspend = B_TRUE; + } + } + rw_exit(&zvol_state_lock); + + ASSERT(MUTEX_HELD(&zv->zv_state_lock)); + + if (zv->zv_open_count == 0) { + boolean_t drop_namespace = B_FALSE; + + ASSERT(RW_READ_HELD(&zv->zv_suspend_lock)); + /* * In all other call paths the spa_namespace_lock is taken * before the bdev->bd_mutex lock. However, on open(2) @@ -542,84 +570,51 @@ zvol_open(struct block_device *bdev, fmode_t flag) * the kernel so the only option is to return the error for * the caller to handle it. */ - if (!mutex_tryenter(&spa_namespace_lock)) { - rw_exit(&zvol_state_lock); + if (!mutex_owned(&spa_namespace_lock)) { + if (!mutex_tryenter(&spa_namespace_lock)) { + mutex_exit(&zv->zv_state_lock); + rw_exit(&zv->zv_suspend_lock); #ifdef HAVE_BLKDEV_GET_ERESTARTSYS - schedule(); - return (SET_ERROR(-ERESTARTSYS)); -#else - if ((gethrtime() - start) > timeout) + schedule(); return (SET_ERROR(-ERESTARTSYS)); +#else + if ((gethrtime() - start) > timeout) + return (SET_ERROR(-ERESTARTSYS)); - schedule_timeout(MSEC_TO_TICK(10)); - goto retry; + schedule_timeout(MSEC_TO_TICK(10)); + goto retry; #endif - } else { - drop_namespace = B_TRUE; - } - } - - mutex_enter(&zv->zv_state_lock); - /* - * make sure zvol is not suspended during first open - * (hold zv_suspend_lock) and respect proper lock acquisition - * ordering - zv_suspend_lock before zv_state_lock - */ - if (zv->zv_open_count == 0) { - if (!rw_tryenter(&zv->zv_suspend_lock, RW_READER)) { - mutex_exit(&zv->zv_state_lock); - rw_enter(&zv->zv_suspend_lock, RW_READER); - mutex_enter(&zv->zv_state_lock); - /* check to see if zv_suspend_lock is needed */ - if (zv->zv_open_count != 0) { - rw_exit(&zv->zv_suspend_lock); - drop_suspend = B_FALSE; + } else { + drop_namespace = B_TRUE; } } - } else { - drop_suspend = B_FALSE; - } - rw_exit(&zvol_state_lock); - - ASSERT(MUTEX_HELD(&zv->zv_state_lock)); - if (zv->zv_open_count == 0) { - ASSERT(RW_READ_HELD(&zv->zv_suspend_lock)); error = -zvol_first_open(zv, !(flag & FMODE_WRITE)); - if (error) - goto out_mutex; - } - if ((flag & FMODE_WRITE) && (zv->zv_flags & ZVOL_RDONLY)) { - error = -EROFS; - goto out_open_count; + if (drop_namespace) + mutex_exit(&spa_namespace_lock); } - zv->zv_open_count++; - - mutex_exit(&zv->zv_state_lock); - if (drop_namespace) - mutex_exit(&spa_namespace_lock); - if (drop_suspend) - rw_exit(&zv->zv_suspend_lock); - - zfs_check_media_change(bdev); - - return (0); + if (error == 0) { + if ((flag & FMODE_WRITE) && (zv->zv_flags & ZVOL_RDONLY)) { + if (zv->zv_open_count == 0) + zvol_last_close(zv); -out_open_count: - if (zv->zv_open_count == 0) - zvol_last_close(zv); + error = SET_ERROR(-EROFS); + } else { + zv->zv_open_count++; + } + } -out_mutex: mutex_exit(&zv->zv_state_lock); - if (drop_namespace) - mutex_exit(&spa_namespace_lock); if (drop_suspend) rw_exit(&zv->zv_suspend_lock); - return (SET_ERROR(error)); + if (error == 0) + zfs_check_media_change(bdev); + + return (error); } static void From a1a52a356ba7c41c7318a893e0aafd9faacca939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Mon, 17 May 2021 18:03:26 +0200 Subject: [PATCH 005/109] freebsd/libshare: nfs: don't send SIGHUP to all processes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pidfile_open() sets *pidptr to -1 if the process currently holding the lock is between pidfile_open() and pidfile_write(), the subsequent kill(mountdpid) would potentially SIGHUP all non-system processes except init: just sleep for half a millisecond and try again in that case Reviewed-by: Don Brady Reviewed-by: Brian Behlendorf Reviewed-by: John Kennedy Signed-off-by: Ahelenia Ziemiańska Closes #12067 --- lib/libshare/os/freebsd/nfs.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/libshare/os/freebsd/nfs.c b/lib/libshare/os/freebsd/nfs.c index 56df3e66643c..97092bdc0fdf 100644 --- a/lib/libshare/os/freebsd/nfs.c +++ b/lib/libshare/os/freebsd/nfs.c @@ -423,9 +423,10 @@ nfs_commit_shares(void) struct pidfh *pfh; pid_t mountdpid; +start: pfh = pidfile_open(_PATH_MOUNTDPID, 0600, &mountdpid); if (pfh != NULL) { - /* Mountd is not running. */ + /* mountd(8) is not running. */ pidfile_remove(pfh); return (SA_OK); } @@ -433,6 +434,11 @@ nfs_commit_shares(void) /* Cannot open pidfile for some reason. */ return (SA_SYSTEM_ERR); } + if (mountdpid == -1) { + /* mountd(8) exists, but didn't write the PID yet */ + usleep(500); + goto start; + } /* We have mountd(8) PID in mountdpid variable. */ kill(mountdpid, SIGHUP); return (SA_OK); From 1833de81033e0bac0f68d1d3dc5f35454efc019a Mon Sep 17 00:00:00 2001 From: Philipp Riederer Date: Mon, 20 Dec 2021 19:50:46 +0100 Subject: [PATCH 006/109] Fix error propagation from lzc_send_redacted Any error from lzc_send_redacted is overwritten by the error of send_conclusion_record; skip writing the conclusion record if there was an earlier error. Reviewed-by: Paul Dagnelie Reviewed-by: Brian Behlendorf Signed-off-by: Philipp Riederer Closes #12766 --- lib/libzfs/libzfs_sendrecv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index 86ff8c91a91e..3e2eb5d9116b 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -2532,7 +2532,7 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags, "progress thread exited nonzero"))); } - if (flags->props || flags->holds || flags->backup) { + if (err == 0 && (flags->props || flags->holds || flags->backup)) { /* Write the final end record. */ err = send_conclusion_record(fd, NULL); if (err != 0) From 14bf91a0431e6b410ec9cf029172dfc5a99eb58f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Matu=C5=A1ka?= Date: Mon, 20 Dec 2021 23:28:43 +0100 Subject: [PATCH 007/109] FreeBSD: fix world build after 143476ce8 Do not redefine the fallthrough macro when building with libcpp. Reviewed-by: Ryan Moeller Reviewed-by: Brian Behlendorf Signed-off-by: Martin Matuska Closes #12880 --- lib/libspl/include/sys/feature_tests.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/libspl/include/sys/feature_tests.h b/lib/libspl/include/sys/feature_tests.h index a36fd7b8cffb..c9564b2c3269 100644 --- a/lib/libspl/include/sys/feature_tests.h +++ b/lib/libspl/include/sys/feature_tests.h @@ -30,7 +30,7 @@ #define ____cacheline_aligned #define __NORETURN __attribute__((__noreturn__)) -#if !defined(fallthrough) +#if !defined(fallthrough) && !defined(_LIBCPP_VERSION) #if defined(HAVE_IMPLICIT_FALLTHROUGH) #define fallthrough __attribute__((__fallthrough__)) #else From 5d8c081193d4fdfdd58a0edf4dc87aa18f75ed33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Thu, 23 Dec 2021 20:39:29 +0100 Subject: [PATCH 008/109] FreeBSD: fix unpropagated error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When performing I/O on FreeBSD using a file based vdev ensure all errors encountered when reading/writing are propagated through the zio pipeline. Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Reviewed-by: Ryan Moeller Signed-off-by: Ahelenia Ziemiańska Closes #12904 --- module/os/freebsd/zfs/vdev_file.c | 1 + 1 file changed, 1 insertion(+) diff --git a/module/os/freebsd/zfs/vdev_file.c b/module/os/freebsd/zfs/vdev_file.c index fc04a7476154..742ac9786d72 100644 --- a/module/os/freebsd/zfs/vdev_file.c +++ b/module/os/freebsd/zfs/vdev_file.c @@ -234,6 +234,7 @@ vdev_file_io_strategy(void *arg) err = zfs_file_pwrite(vf->vf_file, buf, size, off, &resid); abd_return_buf(zio->io_abd, buf, size); } + zio->io_error = err; if (resid != 0 && zio->io_error == 0) zio->io_error = ENOSPC; From 6575defc527ff78d2754f0d95815ea995724c2b2 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Tue, 4 Jan 2022 16:46:32 -0800 Subject: [PATCH 009/109] Verify dRAID empty sectors Verify that all empty sectors are zero filled before using them to calculate parity. Failure to do so can result in incorrect parity columns being generated and written to disk if the contents of an empty sector are non-zero. This was possible because the checksum only protects the data portions of the buffer, not the empty sector padding. This issue has been addressed by updating raidz_parity_verify() to check that all dRAID empty sectors are zero filled. Any sectors which are non-zero will be fixed, repair IO issued, and a checksum error logged. They can then be safely used to verify the parity. This specific type of damage is unlikely to occur since it requires a disk to have silently returned bad data, for an empty sector, while performing a scrub. However, if a pool were to have been damaged in this way, scrubbing the pool with this change applied will repair both the empty sector and parity columns as long as the data checksum is valid. Checksum errors will be reported in the `zpool status` output for any repairs which are made. Reviewed-by: Tony Hutter Reviewed-by: Mark Maybee Reviewed-by: Brian Atkinson Signed-off-by: Brian Behlendorf Closes #12857 --- include/sys/vdev_draid.h | 1 + include/sys/vdev_raidz.h | 2 + module/zfs/vdev_draid.c | 47 +++++++++++++++++++ module/zfs/vdev_raidz.c | 16 +++++-- .../zpool_events/zpool_events_errors.ksh | 11 +++-- 5 files changed, 67 insertions(+), 10 deletions(-) diff --git a/include/sys/vdev_draid.h b/include/sys/vdev_draid.h index 52ce4ba16105..dd334acbacf1 100644 --- a/include/sys/vdev_draid.h +++ b/include/sys/vdev_draid.h @@ -96,6 +96,7 @@ extern boolean_t vdev_draid_readable(vdev_t *, uint64_t); extern boolean_t vdev_draid_missing(vdev_t *, uint64_t, uint64_t, uint64_t); extern uint64_t vdev_draid_asize_to_psize(vdev_t *, uint64_t); extern void vdev_draid_map_alloc_empty(zio_t *, struct raidz_row *); +extern int vdev_draid_map_verify_empty(zio_t *, struct raidz_row *); extern nvlist_t *vdev_draid_read_config_spare(vdev_t *); /* Functions for dRAID distributed spares. */ diff --git a/include/sys/vdev_raidz.h b/include/sys/vdev_raidz.h index ee597eb0dbb3..c7cf0af6d945 100644 --- a/include/sys/vdev_raidz.h +++ b/include/sys/vdev_raidz.h @@ -32,6 +32,7 @@ extern "C" { #endif struct zio; +struct raidz_col; struct raidz_row; struct raidz_map; #if !defined(_KERNEL) @@ -49,6 +50,7 @@ void vdev_raidz_generate_parity(struct raidz_map *); void vdev_raidz_reconstruct(struct raidz_map *, const int *, int); void vdev_raidz_child_done(zio_t *); void vdev_raidz_io_done(zio_t *); +void vdev_raidz_checksum_error(zio_t *, struct raidz_col *, abd_t *); extern const zio_vsd_ops_t vdev_raidz_vsd_ops; diff --git a/module/zfs/vdev_draid.c b/module/zfs/vdev_draid.c index b8f82d52e8f0..2d83c1ac977c 100644 --- a/module/zfs/vdev_draid.c +++ b/module/zfs/vdev_draid.c @@ -841,6 +841,53 @@ vdev_draid_map_alloc_empty(zio_t *zio, raidz_row_t *rr) ASSERT3U(skip_off, ==, rr->rr_nempty * skip_size); } +/* + * Verify that all empty sectors are zero filled before using them to + * calculate parity. Otherwise, silent corruption in an empty sector will + * result in bad parity being generated. That bad parity will then be + * considered authoritative and overwrite the good parity on disk. This + * is possible because the checksum is only calculated over the data, + * thus it cannot be used to detect damage in empty sectors. + */ +int +vdev_draid_map_verify_empty(zio_t *zio, raidz_row_t *rr) +{ + uint64_t skip_size = 1ULL << zio->io_vd->vdev_top->vdev_ashift; + uint64_t parity_size = rr->rr_col[0].rc_size; + uint64_t skip_off = parity_size - skip_size; + uint64_t empty_off = 0; + int ret = 0; + + ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ); + ASSERT3P(rr->rr_abd_empty, !=, NULL); + ASSERT3U(rr->rr_bigcols, >, 0); + + void *zero_buf = kmem_zalloc(skip_size, KM_SLEEP); + + for (int c = rr->rr_bigcols; c < rr->rr_cols; c++) { + raidz_col_t *rc = &rr->rr_col[c]; + + ASSERT3P(rc->rc_abd, !=, NULL); + ASSERT3U(rc->rc_size, ==, parity_size); + + if (abd_cmp_buf_off(rc->rc_abd, zero_buf, skip_off, + skip_size) != 0) { + vdev_raidz_checksum_error(zio, rc, rc->rc_abd); + abd_zero_off(rc->rc_abd, skip_off, skip_size); + rc->rc_error = SET_ERROR(ECKSUM); + ret++; + } + + empty_off += skip_size; + } + + ASSERT3U(empty_off, ==, abd_get_size(rr->rr_abd_empty)); + + kmem_free(zero_buf, skip_size); + + return (ret); +} + /* * Given a logical address within a dRAID configuration, return the physical * address on the first drive in the group that this address maps to diff --git a/module/zfs/vdev_raidz.c b/module/zfs/vdev_raidz.c index 1feebf7089b4..9a7cf665643c 100644 --- a/module/zfs/vdev_raidz.c +++ b/module/zfs/vdev_raidz.c @@ -1654,8 +1654,8 @@ vdev_raidz_io_start(zio_t *zio) /* * Report a checksum error for a child of a RAID-Z device. */ -static void -raidz_checksum_error(zio_t *zio, raidz_col_t *rc, abd_t *bad_data) +void +vdev_raidz_checksum_error(zio_t *zio, raidz_col_t *rc, abd_t *bad_data) { vdev_t *vd = zio->io_vd->vdev_child[rc->rc_devidx]; @@ -1725,6 +1725,13 @@ raidz_parity_verify(zio_t *zio, raidz_row_t *rr) abd_copy(orig[c], rc->rc_abd, rc->rc_size); } + /* + * Verify any empty sectors are zero filled to ensure the parity + * is calculated correctly even if these non-data sectors are damaged. + */ + if (rr->rr_nempty && rr->rr_abd_empty != NULL) + ret += vdev_draid_map_verify_empty(zio, rr); + /* * Regenerates parity even for !tried||rc_error!=0 columns. This * isn't harmful but it does have the side effect of fixing stuff @@ -1739,7 +1746,7 @@ raidz_parity_verify(zio_t *zio, raidz_row_t *rr) continue; if (abd_cmp(orig[c], rc->rc_abd) != 0) { - raidz_checksum_error(zio, rc, orig[c]); + vdev_raidz_checksum_error(zio, rc, orig[c]); rc->rc_error = SET_ERROR(ECKSUM); ret++; } @@ -1799,7 +1806,6 @@ vdev_raidz_io_done_verified(zio_t *zio, raidz_row_t *rr) (zio->io_flags & ZIO_FLAG_RESILVER)) { int n = raidz_parity_verify(zio, rr); unexpected_errors += n; - ASSERT3U(parity_errors + n, <=, rr->rr_firstdatacol); } if (zio->io_error == 0 && spa_writeable(zio->io_spa) && @@ -1925,7 +1931,7 @@ raidz_reconstruct(zio_t *zio, int *ltgts, int ntgts, int nparity) */ if (rc->rc_error == 0 && c >= rr->rr_firstdatacol) { - raidz_checksum_error(zio, + vdev_raidz_checksum_error(zio, rc, rc->rc_orig_data); rc->rc_error = SET_ERROR(ECKSUM); diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_events/zpool_events_errors.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_events/zpool_events_errors.ksh index 4645e245c973..a6833f167c66 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_events/zpool_events_errors.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_events/zpool_events_errors.ksh @@ -28,11 +28,12 @@ # in zpool status. # # STRATEGY: -# 1. Create a raidz or mirror pool +# 1. Create a mirror, raidz, or draid pool # 2. Inject read/write IO errors or checksum errors # 3. Verify the number of errors in zpool status match the corresponding # number of error events. -# 4. Repeat for all combinations of raidz/mirror and io/checksum errors. +# 4. Repeat for all combinations of mirror/raidz/draid and io/checksum +# errors. # . $STF_SUITE/include/libtest.shlib @@ -74,7 +75,7 @@ log_must mkdir -p $MOUNTDIR # Run error test on a specific type of pool # -# $1: pool - raidz, mirror +# $1: pool - mirror, raidz, draid # $2: test type - corrupt (checksum error), io # $3: read, write function do_test @@ -142,8 +143,8 @@ function do_test log_must zpool destroy $POOL } -# Test all types of errors on mirror and raidz pools -for pooltype in mirror raidz ; do +# Test all types of errors on mirror, raidz, and draid pools +for pooltype in mirror raidz draid; do do_test $pooltype corrupt read do_test $pooltype io read do_test $pooltype io write From 1259dc6e6a8387c4302f73077f6dbeb3f2e4be13 Mon Sep 17 00:00:00 2001 From: chrisrd Date: Wed, 5 Jan 2022 12:07:33 +1100 Subject: [PATCH 010/109] zfs_prune: reset sc.nr_to_scan sc.nr_to_scan is an input to super_cache_clean (via shrinker->scan_objects), used to set the number of objects to scan in the various caches. However super_cache_scan also modifies sc.nr_to_scan, so when used in a loop we need to reset sc.nr_to_scan back to our desired nr_to_scan for the next iteration. Issue discovered and solution suggested by Tenzin Lhakhang @tlhakhan. Reviewed-by: Brian Behlendorf Signed-off-by: Chris Dunlop Issue #12433 Closes #12908 --- module/os/linux/zfs/zfs_vfsops.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/module/os/linux/zfs/zfs_vfsops.c b/module/os/linux/zfs/zfs_vfsops.c index 5d672af0e8aa..469b82efeeb7 100644 --- a/module/os/linux/zfs/zfs_vfsops.c +++ b/module/os/linux/zfs/zfs_vfsops.c @@ -1259,6 +1259,11 @@ zfs_prune(struct super_block *sb, unsigned long nr_to_scan, int *objects) *objects = 0; for_each_online_node(sc.nid) { *objects += (*shrinker->scan_objects)(shrinker, &sc); + /* + * reset sc.nr_to_scan, modified by + * scan_objects == super_cache_scan + */ + sc.nr_to_scan = nr_to_scan; } } else { *objects = (*shrinker->scan_objects)(shrinker, &sc); From 36a91d6ceff48dee36939aa93e60abbec1665c57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Thu, 6 Jan 2022 20:15:08 +0100 Subject: [PATCH 011/109] FreeBSD: vfsops: use setgen for error case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix from https://github.com/openzfs/zfs/pull/12844#discussion_r774179413 Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Reviewed-by: Ryan Moeller Signed-off-by: Ahelenia Ziemiańska Closes #12905 --- module/os/freebsd/zfs/zfs_vfsops.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/module/os/freebsd/zfs/zfs_vfsops.c b/module/os/freebsd/zfs/zfs_vfsops.c index c534309351e9..cdd762dcbcbf 100644 --- a/module/os/freebsd/zfs/zfs_vfsops.c +++ b/module/os/freebsd/zfs/zfs_vfsops.c @@ -1801,6 +1801,7 @@ zfs_fhtovp(vfs_t *vfsp, fid_t *fidp, int flags, vnode_t **vpp) vnode_t *dvp; uint64_t object = 0; uint64_t fid_gen = 0; + uint64_t setgen = 0; uint64_t gen_mask; uint64_t zp_gen; int i, err; @@ -1816,7 +1817,6 @@ zfs_fhtovp(vfs_t *vfsp, fid_t *fidp, int flags, vnode_t **vpp) if (zfsvfs->z_parent == zfsvfs && fidp->fid_len == LONG_FID_LEN) { zfid_long_t *zlfid = (zfid_long_t *)fidp; uint64_t objsetid = 0; - uint64_t setgen = 0; for (i = 0; i < sizeof (zlfid->zf_setid); i++) objsetid |= ((uint64_t)zlfid->zf_setid[i]) << (8 * i); @@ -1845,6 +1845,12 @@ zfs_fhtovp(vfs_t *vfsp, fid_t *fidp, int flags, vnode_t **vpp) return (SET_ERROR(EINVAL)); } + if (fidp->fid_len == LONG_FID_LEN && (fid_gen > 1 || setgen != 0)) { + dprintf("snapdir fid: fid_gen (%llu) and setgen (%llu)\n", + (u_longlong_t)fid_gen, (u_longlong_t)setgen); + return (SET_ERROR(EINVAL)); + } + /* * A zero fid_gen means we are in .zfs or the .zfs/snapshot * directory tree. If the object == zfsvfs->z_shares_dir, then From c9c9d634aa6f4aeaf867b534a967f2dc73bfa25a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Thu, 13 Jan 2022 20:07:54 +0100 Subject: [PATCH 012/109] linux: libzfs: mount: fix uninitialised flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They're later |=d with constants, but never reset Caught by valgrind while investigating https://github.com/openzfs/zfs/pull/12928#issuecomment-1007496550 Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #12954 --- lib/libzfs/os/linux/libzfs_mount_os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/libzfs/os/linux/libzfs_mount_os.c b/lib/libzfs/os/linux/libzfs_mount_os.c index 21d64053862e..b800e069e707 100644 --- a/lib/libzfs/os/linux/libzfs_mount_os.c +++ b/lib/libzfs/os/linux/libzfs_mount_os.c @@ -327,7 +327,7 @@ do_mount(zfs_handle_t *zhp, const char *mntpt, char *opts, int flags) if (!libzfs_envvar_is_set("ZFS_MOUNT_HELPER")) { char badopt[MNT_LINE_MAX] = {0}; - unsigned long mntflags = flags, zfsflags; + unsigned long mntflags = flags, zfsflags = 0; char myopts[MNT_LINE_MAX] = {0}; if (zfs_parse_mount_options(opts, &mntflags, From f4def7ec6c9a003dd6c85a8a78531bf5eb5996e0 Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Fri, 14 Jan 2022 15:38:32 -0500 Subject: [PATCH 013/109] FreeBSD: Fix leaked strings in libspl mnttab The FreeBSD implementations of various libspl functions for getting mounted device information were found to leak several strings which were being allocated in statfs2mnttab but never freed. The Solaris getmntany(3C) and related interfaces are expected to return strings residing in static buffers that need to be copied rather than freed by the caller. Use static thread-local storage to stash the mnttab structure strings from FreeBSD's statfs info rather than strings allocated on the heap by strdup(3). While here, remove some stray commented out lines. Reviewed-by: Alexander Motin Reviewed-by: Rich Ercolani Signed-off-by: Ryan Moeller Closes #12961 --- lib/libspl/os/freebsd/mnttab.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/libspl/os/freebsd/mnttab.c b/lib/libspl/os/freebsd/mnttab.c index bd3e3e4e3eef..d830257fbd16 100644 --- a/lib/libspl/os/freebsd/mnttab.c +++ b/lib/libspl/os/freebsd/mnttab.c @@ -91,16 +91,28 @@ optadd(char *mntopts, size_t size, const char *opt) strlcat(mntopts, opt, size); } +static __thread char gfstypename[MFSNAMELEN]; +static __thread char gmntfromname[MNAMELEN]; +static __thread char gmntonname[MNAMELEN]; +static __thread char gmntopts[MNTMAXSTR]; + void statfs2mnttab(struct statfs *sfs, struct mnttab *mp) { - static char mntopts[MNTMAXSTR]; long flags; - mntopts[0] = '\0'; + strlcpy(gfstypename, sfs->f_fstypename, sizeof (gfstypename)); + mp->mnt_fstype = gfstypename; + + strlcpy(gmntfromname, sfs->f_mntfromname, sizeof (gmntfromname)); + mp->mnt_special = gmntfromname; + + strlcpy(gmntonname, sfs->f_mntonname, sizeof (gmntonname)); + mp->mnt_mountp = gmntonname; flags = sfs->f_flags; -#define OPTADD(opt) optadd(mntopts, sizeof (mntopts), (opt)) + gmntopts[0] = '\0'; +#define OPTADD(opt) optadd(gmntopts, sizeof (gmntopts), (opt)) if (flags & MNT_RDONLY) OPTADD(MNTOPT_RO); else @@ -121,10 +133,7 @@ statfs2mnttab(struct statfs *sfs, struct mnttab *mp) else OPTADD(MNTOPT_EXEC); #undef OPTADD - mp->mnt_special = strdup(sfs->f_mntfromname); - mp->mnt_mountp = strdup(sfs->f_mntonname); - mp->mnt_fstype = strdup(sfs->f_fstypename); - mp->mnt_mntopts = strdup(mntopts); + mp->mnt_mntopts = gmntopts; } static struct statfs *gsfs = NULL; @@ -166,7 +175,6 @@ statfs_init(void) int getmntany(FILE *fd __unused, struct mnttab *mgetp, struct mnttab *mrefp) { - // struct statfs *sfs; int i, error; error = statfs_init(); @@ -195,7 +203,6 @@ getmntany(FILE *fd __unused, struct mnttab *mgetp, struct mnttab *mrefp) int getmntent(FILE *fp, struct mnttab *mp) { - // struct statfs *sfs; int error, nfs; nfs = (int)lseek(fileno(fp), 0, SEEK_CUR); From 1828b68a0b27c9ae195316722df8967ea1b29ab2 Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Thu, 23 Dec 2021 19:03:23 +0000 Subject: [PATCH 014/109] FreeBSD: Fix zvol_*_open() locking These are the changes for FreeBSD corresponding to the changes made for Linux in #12863, see that PR for details. Changes from #12863 are applied for zvol_geom_open and zvol_cdev_open on FreeBSD. This also adds a check for the zvol dying which we had in zvol_geom_open but was missing in zvol_cdev_open. The check causes the open to fail early with ENXIO when we are in the middle of changing volmode. Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Signed-off-by: Ryan Moeller Closes #12934 --- module/os/freebsd/zfs/zvol_os.c | 90 +++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 31 deletions(-) diff --git a/module/os/freebsd/zfs/zvol_os.c b/module/os/freebsd/zfs/zvol_os.c index 450369192569..ffd96d159e36 100644 --- a/module/os/freebsd/zfs/zvol_os.c +++ b/module/os/freebsd/zfs/zvol_os.c @@ -210,7 +210,6 @@ zvol_geom_open(struct g_provider *pp, int flag, int count) zvol_state_t *zv; int err = 0; boolean_t drop_suspend = B_FALSE; - boolean_t drop_namespace = B_FALSE; if (!zpool_on_zvol && tsd_get(zfs_geom_probe_vdev_key) != NULL) { /* @@ -226,6 +225,12 @@ zvol_geom_open(struct g_provider *pp, int flag, int count) retry: rw_enter(&zvol_state_lock, ZVOL_RW_READER); + /* + * Obtain a copy of private under zvol_state_lock to make sure either + * the result of zvol free code setting private to NULL is observed, + * or the zv is protected from being freed because of the positive + * zv_open_count. + */ zv = pp->private; if (zv == NULL) { rw_exit(&zvol_state_lock); @@ -233,18 +238,6 @@ zvol_geom_open(struct g_provider *pp, int flag, int count) goto out_locked; } - if (zv->zv_open_count == 0 && !mutex_owned(&spa_namespace_lock)) { - /* - * We need to guarantee that the namespace lock is held - * to avoid spurious failures in zvol_first_open. - */ - drop_namespace = B_TRUE; - if (!mutex_tryenter(&spa_namespace_lock)) { - rw_exit(&zvol_state_lock); - mutex_enter(&spa_namespace_lock); - goto retry; - } - } mutex_enter(&zv->zv_state_lock); if (zv->zv_zso->zso_dying) { rw_exit(&zvol_state_lock); @@ -276,8 +269,27 @@ zvol_geom_open(struct g_provider *pp, int flag, int count) ASSERT(MUTEX_HELD(&zv->zv_state_lock)); if (zv->zv_open_count == 0) { + boolean_t drop_namespace = B_FALSE; + ASSERT(ZVOL_RW_READ_HELD(&zv->zv_suspend_lock)); + + /* + * Take spa_namespace_lock to prevent lock inversion when + * zvols from one pool are opened as vdevs in another. + */ + if (!mutex_owned(&spa_namespace_lock)) { + if (!mutex_tryenter(&spa_namespace_lock)) { + mutex_exit(&zv->zv_state_lock); + rw_exit(&zv->zv_suspend_lock); + kern_yield(PRI_USER); + goto retry; + } else { + drop_namespace = B_TRUE; + } + } err = zvol_first_open(zv, !(flag & FWRITE)); + if (drop_namespace) + mutex_exit(&spa_namespace_lock); if (err) goto out_zv_locked; pp->mediasize = zv->zv_volsize; @@ -285,6 +297,8 @@ zvol_geom_open(struct g_provider *pp, int flag, int count) pp->stripesize = zv->zv_volblocksize; } + ASSERT(MUTEX_HELD(&zv->zv_state_lock)); + /* * Check for a bad on-disk format version now since we * lied about owning the dataset readonly before. @@ -317,8 +331,6 @@ zvol_geom_open(struct g_provider *pp, int flag, int count) out_zv_locked: mutex_exit(&zv->zv_state_lock); out_locked: - if (drop_namespace) - mutex_exit(&spa_namespace_lock); if (drop_suspend) rw_exit(&zv->zv_suspend_lock); return (err); @@ -859,10 +871,15 @@ zvol_cdev_open(struct cdev *dev, int flags, int fmt, struct thread *td) struct zvol_state_dev *zsd; int err = 0; boolean_t drop_suspend = B_FALSE; - boolean_t drop_namespace = B_FALSE; retry: rw_enter(&zvol_state_lock, ZVOL_RW_READER); + /* + * Obtain a copy of si_drv2 under zvol_state_lock to make sure either + * the result of zvol free code setting si_drv2 to NULL is observed, + * or the zv is protected from being freed because of the positive + * zv_open_count. + */ zv = dev->si_drv2; if (zv == NULL) { rw_exit(&zvol_state_lock); @@ -870,20 +887,12 @@ zvol_cdev_open(struct cdev *dev, int flags, int fmt, struct thread *td) goto out_locked; } - if (zv->zv_open_count == 0 && !mutex_owned(&spa_namespace_lock)) { - /* - * We need to guarantee that the namespace lock is held - * to avoid spurious failures in zvol_first_open. - */ - drop_namespace = B_TRUE; - if (!mutex_tryenter(&spa_namespace_lock)) { - rw_exit(&zvol_state_lock); - mutex_enter(&spa_namespace_lock); - goto retry; - } - } mutex_enter(&zv->zv_state_lock); - + if (zv->zv_zso->zso_dying) { + rw_exit(&zvol_state_lock); + err = SET_ERROR(ENXIO); + goto out_zv_locked; + } ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_DEV); /* @@ -909,12 +918,33 @@ zvol_cdev_open(struct cdev *dev, int flags, int fmt, struct thread *td) ASSERT(MUTEX_HELD(&zv->zv_state_lock)); if (zv->zv_open_count == 0) { + boolean_t drop_namespace = B_FALSE; + ASSERT(ZVOL_RW_READ_HELD(&zv->zv_suspend_lock)); + + /* + * Take spa_namespace_lock to prevent lock inversion when + * zvols from one pool are opened as vdevs in another. + */ + if (!mutex_owned(&spa_namespace_lock)) { + if (!mutex_tryenter(&spa_namespace_lock)) { + rw_exit(&zvol_state_lock); + mutex_enter(&spa_namespace_lock); + kern_yield(PRI_USER); + goto retry; + } else { + drop_namespace = B_TRUE; + } + } err = zvol_first_open(zv, !(flags & FWRITE)); + if (drop_namespace) + mutex_exit(&spa_namespace_lock); if (err) goto out_zv_locked; } + ASSERT(MUTEX_HELD(&zv->zv_state_lock)); + if ((flags & FWRITE) && (zv->zv_flags & ZVOL_RDONLY)) { err = SET_ERROR(EROFS); goto out_opened; @@ -949,8 +979,6 @@ zvol_cdev_open(struct cdev *dev, int flags, int fmt, struct thread *td) out_zv_locked: mutex_exit(&zv->zv_state_lock); out_locked: - if (drop_namespace) - mutex_exit(&spa_namespace_lock); if (drop_suspend) rw_exit(&zv->zv_suspend_lock); return (err); From af1630c88361ea84eac9bea92949383fee4c6761 Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Wed, 26 Jan 2022 14:23:39 -0500 Subject: [PATCH 015/109] FreeBSD: Fix zvol_cdev_open locking First open locking changes were correctly applied to zvol_geom_open but incorrectly applied to zvol_cdev_open, causing spa_namespace_lock to be held indefinitely. Make the first open locking in zvol_cdev_open match zvol_geom_open. Reviewed-by: Alexander Motin Reviewed-by: Brian Behlendorf Signed-off-by: Ryan Moeller Closes #13016 --- module/os/freebsd/zfs/zvol_os.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module/os/freebsd/zfs/zvol_os.c b/module/os/freebsd/zfs/zvol_os.c index ffd96d159e36..0b347e7f98fc 100644 --- a/module/os/freebsd/zfs/zvol_os.c +++ b/module/os/freebsd/zfs/zvol_os.c @@ -928,8 +928,8 @@ zvol_cdev_open(struct cdev *dev, int flags, int fmt, struct thread *td) */ if (!mutex_owned(&spa_namespace_lock)) { if (!mutex_tryenter(&spa_namespace_lock)) { - rw_exit(&zvol_state_lock); - mutex_enter(&spa_namespace_lock); + mutex_exit(&zv->zv_state_lock); + rw_exit(&zv->zv_suspend_lock); kern_yield(PRI_USER); goto retry; } else { From ddb5a7a182bd7e6fdbee121411132ab2d8f487df Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Tue, 27 Jul 2021 13:47:27 +0000 Subject: [PATCH 016/109] libzfs_sendrecv: Avoid extra avl_find avl_add does avl_find internally, then avl_insert. We're already doing the avl_find, so using avl_insert directly avoids repeating the search. Reviewed-by: Brian Behlendorf Signed-off-by: Ryan Moeller Closes #12967 --- lib/libzfs/libzfs_sendrecv.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index 3e2eb5d9116b..13ac21df933a 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -206,8 +206,9 @@ fsavl_create(nvlist_t *fss) * Note: if there are multiple snaps with the * same GUID, we ignore all but one. */ - if (avl_find(fsavl, fn, NULL) == NULL) - avl_add(fsavl, fn); + avl_index_t where = 0; + if (avl_find(fsavl, fn, &where) == NULL) + avl_insert(fsavl, fn, where); else free(fn); } From 4aceda0497a44706a1361cd2db34266e38a48e8b Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Tue, 27 Jul 2021 13:48:42 +0000 Subject: [PATCH 017/109] libzfs_sendrecv: Fix leaked holds nvlist There is no need to allocate a holds nvlist. lzc_get_holds does that for us. Reviewed-by: Brian Behlendorf Signed-off-by: Ryan Moeller Closes #12967 --- lib/libzfs/libzfs_sendrecv.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index 13ac21df933a..b67c9b30c84a 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -351,12 +351,11 @@ send_iterate_snap(zfs_handle_t *zhp, void *arg) fnvlist_add_nvlist(sd->snapprops, snapname, nv); fnvlist_free(nv); if (sd->holds) { - nvlist_t *holds = fnvlist_alloc(); - int err = lzc_get_holds(zhp->zfs_name, &holds); - if (err == 0) { + nvlist_t *holds; + if (lzc_get_holds(zhp->zfs_name, &holds) == 0) { fnvlist_add_nvlist(sd->snapholds, snapname, holds); + fnvlist_free(holds); } - fnvlist_free(holds); } zfs_close(zhp); From 5303fc4c9565803eeee07237d7c867c3cebe2a0a Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Fri, 21 Jan 2022 13:28:13 -0500 Subject: [PATCH 018/109] Avoid memory allocations in the ARC eviction thread When the eviction thread goes to shrink an ARC state, it allocates a set of marker buffers used to hold its place in the state's sublists. This can be problematic in low memory conditions, since 1) the allocation can be substantial, as we allocate NCPU markers; 2) on at least FreeBSD, page reclamation can block in arc_wait_for_eviction() In particular, in stress tests it's possible to hit a deadlock on FreeBSD when the number of free pages is very low, wherein the system is waiting for the page daemon to reclaim memory, the page daemon is waiting for the ARC eviction thread to finish, and the ARC eviction thread is blocked waiting for more memory. Try to reduce the likelihood of such deadlocks by pre-allocating markers for the eviction thread at ARC initialization time. When evicting buffers from an ARC state, check to see if the current thread is the ARC eviction thread, and use the pre-allocated markers for that purpose rather than dynamically allocating them. Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Reviewed-by: George Amanakis Signed-off-by: Mark Johnston Closes #12985 --- include/sys/zthr.h | 1 + module/zfs/arc.c | 138 ++++++++++++++++++++++++++++----------------- module/zfs/zthr.c | 6 ++ 3 files changed, 92 insertions(+), 53 deletions(-) diff --git a/include/sys/zthr.h b/include/sys/zthr.h index 19be89eeebe5..4881b4572641 100644 --- a/include/sys/zthr.h +++ b/include/sys/zthr.h @@ -38,6 +38,7 @@ extern void zthr_resume(zthr_t *t); extern void zthr_wait_cycle_done(zthr_t *t); extern boolean_t zthr_iscancelled(zthr_t *t); +extern boolean_t zthr_iscurthread(zthr_t *t); extern boolean_t zthr_has_waiters(zthr_t *t); #endif /* _SYS_ZTHR_H */ diff --git a/module/zfs/arc.c b/module/zfs/arc.c index 9f22f877d20d..6dcd2d1deef8 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -328,6 +328,8 @@ static zthr_t *arc_reap_zthr; * arc_evict(), which improves arc_is_overflowing(). */ static zthr_t *arc_evict_zthr; +static arc_buf_hdr_t **arc_state_evict_markers; +static int arc_state_evict_marker_count; static kmutex_t arc_evict_lock; static boolean_t arc_evict_needed = B_FALSE; @@ -4154,6 +4156,38 @@ arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker, return (bytes_evicted); } +/* + * Allocate an array of buffer headers used as placeholders during arc state + * eviction. + */ +static arc_buf_hdr_t ** +arc_state_alloc_markers(int count) +{ + arc_buf_hdr_t **markers; + + markers = kmem_zalloc(sizeof (*markers) * count, KM_SLEEP); + for (int i = 0; i < count; i++) { + markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP); + + /* + * A b_spa of 0 is used to indicate that this header is + * a marker. This fact is used in arc_evict_type() and + * arc_evict_state_impl(). + */ + markers[i]->b_spa = 0; + + } + return (markers); +} + +static void +arc_state_free_markers(arc_buf_hdr_t **markers, int count) +{ + for (int i = 0; i < count; i++) + kmem_cache_free(hdr_full_cache, markers[i]); + kmem_free(markers, sizeof (*markers) * count); +} + /* * Evict buffers from the given arc state, until we've removed the * specified number of bytes. Move the removed buffers to the @@ -4185,19 +4219,15 @@ arc_evict_state(arc_state_t *state, uint64_t spa, uint64_t bytes, * pick up where we left off for each individual sublist, rather * than starting from the tail each time. */ - markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP); + if (zthr_iscurthread(arc_evict_zthr)) { + markers = arc_state_evict_markers; + ASSERT3S(num_sublists, <=, arc_state_evict_marker_count); + } else { + markers = arc_state_alloc_markers(num_sublists); + } for (int i = 0; i < num_sublists; i++) { multilist_sublist_t *mls; - markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP); - - /* - * A b_spa of 0 is used to indicate that this header is - * a marker. This fact is used in arc_evict_type() and - * arc_evict_state_impl(). - */ - markers[i]->b_spa = 0; - mls = multilist_sublist_lock(ml, i); multilist_sublist_insert_tail(mls, markers[i]); multilist_sublist_unlock(mls); @@ -4279,10 +4309,9 @@ arc_evict_state(arc_state_t *state, uint64_t spa, uint64_t bytes, multilist_sublist_t *mls = multilist_sublist_lock(ml, i); multilist_sublist_remove(mls, markers[i]); multilist_sublist_unlock(mls); - - kmem_cache_free(hdr_full_cache, markers[i]); } - kmem_free(markers, sizeof (*markers) * num_sublists); + if (markers != arc_state_evict_markers) + arc_state_free_markers(markers, num_sublists); return (total_evicted); } @@ -7599,53 +7628,52 @@ arc_tuning_update(boolean_t verbose) WARN_IF_TUNING_IGNORED(zfs_arc_sys_free, arc_sys_free, verbose); } +static void +arc_state_multilist_init(multilist_t *ml, + multilist_sublist_index_func_t *index_func, int *maxcountp) +{ + multilist_create(ml, sizeof (arc_buf_hdr_t), + offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), index_func); + *maxcountp = MAX(*maxcountp, multilist_get_num_sublists(ml)); +} + static void arc_state_init(void) { - multilist_create(&arc_mru->arcs_list[ARC_BUFC_METADATA], - sizeof (arc_buf_hdr_t), - offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - arc_state_multilist_index_func); - multilist_create(&arc_mru->arcs_list[ARC_BUFC_DATA], - sizeof (arc_buf_hdr_t), - offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - arc_state_multilist_index_func); - multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA], - sizeof (arc_buf_hdr_t), - offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - arc_state_multilist_index_func); - multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA], - sizeof (arc_buf_hdr_t), - offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - arc_state_multilist_index_func); - multilist_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA], - sizeof (arc_buf_hdr_t), - offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - arc_state_multilist_index_func); - multilist_create(&arc_mfu->arcs_list[ARC_BUFC_DATA], - sizeof (arc_buf_hdr_t), - offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - arc_state_multilist_index_func); - multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA], - sizeof (arc_buf_hdr_t), - offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - arc_state_multilist_index_func); - multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA], - sizeof (arc_buf_hdr_t), - offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - arc_state_multilist_index_func); + int num_sublists = 0; + + arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_METADATA], + arc_state_multilist_index_func, &num_sublists); + arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_DATA], + arc_state_multilist_index_func, &num_sublists); + arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA], + arc_state_multilist_index_func, &num_sublists); + arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA], + arc_state_multilist_index_func, &num_sublists); + arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_METADATA], + arc_state_multilist_index_func, &num_sublists); + arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_DATA], + arc_state_multilist_index_func, &num_sublists); + arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA], + arc_state_multilist_index_func, &num_sublists); + arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA], + arc_state_multilist_index_func, &num_sublists); + /* * L2 headers should never be on the L2 state list since they don't * have L1 headers allocated. Special index function asserts that. */ - multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA], - sizeof (arc_buf_hdr_t), - offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - arc_state_l2c_multilist_index_func); - multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA], - sizeof (arc_buf_hdr_t), - offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - arc_state_l2c_multilist_index_func); + arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA], + arc_state_l2c_multilist_index_func, &num_sublists); + arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_DATA], + arc_state_l2c_multilist_index_func, &num_sublists); + + /* + * Keep track of the number of markers needed to reclaim buffers from + * any ARC state. The markers will be pre-allocated so as to minimize + * the number of memory allocations performed by the eviction thread. + */ + arc_state_evict_marker_count = num_sublists; zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]); zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]); @@ -7989,6 +8017,8 @@ arc_init(void) kstat_install(arc_ksp); } + arc_state_evict_markers = + arc_state_alloc_markers(arc_state_evict_marker_count); arc_evict_zthr = zthr_create("arc_evict", arc_evict_cb_check, arc_evict_cb, NULL, defclsyspri); arc_reap_zthr = zthr_create_timer("arc_reap", @@ -8056,6 +8086,8 @@ arc_fini(void) (void) zthr_cancel(arc_evict_zthr); (void) zthr_cancel(arc_reap_zthr); + arc_state_free_markers(arc_state_evict_markers, + arc_state_evict_marker_count); mutex_destroy(&arc_evict_lock); list_destroy(&arc_evict_waiters); diff --git a/module/zfs/zthr.c b/module/zfs/zthr.c index 33fdda7b68d1..52ddffae7aaa 100644 --- a/module/zfs/zthr.c +++ b/module/zfs/zthr.c @@ -469,6 +469,12 @@ zthr_iscancelled(zthr_t *t) return (cancelled); } +boolean_t +zthr_iscurthread(zthr_t *t) +{ + return (t->zthr_thread == curthread); +} + /* * Wait for the zthr to finish its current function. Similar to * zthr_iscancelled, you can use zthr_has_waiters to have the zthr_func end From 0da15f9194bfd181f26207c9daac420e6a3f9609 Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Fri, 21 Jan 2022 14:54:05 -0500 Subject: [PATCH 019/109] Fix handling of errors from dmu_write_uio_dbuf() on FreeBSD FreeBSD's implementation of zfs_uio_fault_move() returns EFAULT when a page fault occurs while copying data in or out of user buffers. The VFS treats such errors specially and will retry the I/O operation (which may have made some partial progress). When the FreeBSD and Linux implementations of zfs_write() were merged, the handling of errors from dmu_write_uio_dbuf() changed such that EFAULT is not handled as a partial write. For example, when appending to a file, the z_size field of the znode is not updated after a partial write resulting in EFAULT. Restore the old handling of errors from dmu_write_uio_dbuf() to fix this. This should have no impact on Linux, which has special handling for EFAULT already. Reviewed-by: Andriy Gapon Reviewed-by: Ryan Moeller Signed-off-by: Mark Johnston Closes #12964 --- module/zfs/zfs_vnops.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/module/zfs/zfs_vnops.c b/module/zfs/zfs_vnops.c index 170e392abe93..54749810d45d 100644 --- a/module/zfs/zfs_vnops.c +++ b/module/zfs/zfs_vnops.c @@ -323,7 +323,7 @@ zfs_read(struct znode *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) int zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) { - int error = 0; + int error = 0, error1; ssize_t start_resid = zfs_uio_resid(uio); /* @@ -561,7 +561,11 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) continue; } #endif - if (error != 0) { + /* + * On FreeBSD, EFAULT should be propagated back to the + * VFS, which will handle faulting and will retry. + */ + if (error != 0 && error != EFAULT) { dmu_tx_commit(tx); break; } @@ -645,7 +649,7 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) while ((end_size = zp->z_size) < zfs_uio_offset(uio)) { (void) atomic_cas_64(&zp->z_size, end_size, zfs_uio_offset(uio)); - ASSERT(error == 0); + ASSERT(error == 0 || error == EFAULT); } /* * If we are replaying and eof is non zero then force @@ -655,7 +659,10 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) if (zfsvfs->z_replay && zfsvfs->z_replay_eof != 0) zp->z_size = zfsvfs->z_replay_eof; - error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); + error1 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); + if (error1 != 0) + /* Avoid clobbering EFAULT. */ + error = error1; zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes, ioflag, NULL, NULL); From b3e08539510a3372fe55369a41547021f3d55c86 Mon Sep 17 00:00:00 2001 From: Rich Ercolani Date: Fri, 14 Jan 2022 05:09:08 -0500 Subject: [PATCH 020/109] Linux 5.16 compat: Check slab.h for kvmalloc As it says on the tin - the folio work moved a bunch out of mm.h. Reviewed-by: Brian Behlendorf Reviewed-by: Coleman Kane Signed-off-by: Rich Ercolani Closes #12975 --- config/kernel-kmem.m4 | 1 + 1 file changed, 1 insertion(+) diff --git a/config/kernel-kmem.m4 b/config/kernel-kmem.m4 index 43f9e72f88d8..03c2a41fbdb2 100644 --- a/config/kernel-kmem.m4 +++ b/config/kernel-kmem.m4 @@ -64,6 +64,7 @@ dnl # AC_DEFUN([ZFS_AC_KERNEL_SRC_KVMALLOC], [ ZFS_LINUX_TEST_SRC([kvmalloc], [ #include + #include ],[ void *p __attribute__ ((unused)); From c31c1146b6d795f305f7e09a3de13bcdd66f2ac4 Mon Sep 17 00:00:00 2001 From: Rich Ercolani Date: Fri, 14 Jan 2022 05:08:33 -0500 Subject: [PATCH 021/109] Linux 5.16 compat: Added add_disk check for return add_disk went from void to must-check int return. Reviewed-by: Brian Behlendorf Reviewed-by: Coleman Kane Signed-off-by: Rich Ercolani Closes #12975 --- config/kernel-add-disk.m4 | 27 +++++++++++++++++++++++++++ config/kernel.m4 | 2 ++ module/os/linux/zfs/zvol_os.c | 4 ++++ 3 files changed, 33 insertions(+) create mode 100644 config/kernel-add-disk.m4 diff --git a/config/kernel-add-disk.m4 b/config/kernel-add-disk.m4 new file mode 100644 index 000000000000..3bc40f1e7d21 --- /dev/null +++ b/config/kernel-add-disk.m4 @@ -0,0 +1,27 @@ +dnl # +dnl # 5.16 API change +dnl # add_disk grew a must-check return code +dnl # +AC_DEFUN([ZFS_AC_KERNEL_SRC_ADD_DISK], [ + + ZFS_LINUX_TEST_SRC([add_disk_ret], [ + #include + ], [ + struct gendisk *disk = NULL; + int err = add_disk(disk); + err = err; + ]) + +]) + +AC_DEFUN([ZFS_AC_KERNEL_ADD_DISK], [ + AC_MSG_CHECKING([whether add_disk() returns int]) + ZFS_LINUX_TEST_RESULT([add_disk_ret], + [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_ADD_DISK_RET, 1, + [add_disk() returns int]) + ], [ + AC_MSG_RESULT(no) + ]) +]) diff --git a/config/kernel.m4 b/config/kernel.m4 index bdd3caed2b3d..b3ed966b7264 100644 --- a/config/kernel.m4 +++ b/config/kernel.m4 @@ -135,6 +135,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [ ZFS_AC_KERNEL_SRC_VFS_SET_PAGE_DIRTY_NOBUFFERS ZFS_AC_KERNEL_SRC_STANDALONE_LINUX_STDARG ZFS_AC_KERNEL_SRC_PAGEMAP_FOLIO_WAIT_BIT + ZFS_AC_KERNEL_SRC_ADD_DISK AC_MSG_CHECKING([for available kernel interfaces]) ZFS_LINUX_TEST_COMPILE_ALL([kabi]) @@ -243,6 +244,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [ ZFS_AC_KERNEL_VFS_SET_PAGE_DIRTY_NOBUFFERS ZFS_AC_KERNEL_STANDALONE_LINUX_STDARG ZFS_AC_KERNEL_PAGEMAP_FOLIO_WAIT_BIT + ZFS_AC_KERNEL_ADD_DISK ]) dnl # diff --git a/module/os/linux/zfs/zvol_os.c b/module/os/linux/zfs/zvol_os.c index 69479b3f7988..7a979eb91e7f 100644 --- a/module/os/linux/zfs/zvol_os.c +++ b/module/os/linux/zfs/zvol_os.c @@ -1110,7 +1110,11 @@ zvol_os_create_minor(const char *name) rw_enter(&zvol_state_lock, RW_WRITER); zvol_insert(zv); rw_exit(&zvol_state_lock); +#ifdef HAVE_ADD_DISK_RET + error = add_disk(zv->zv_zso->zvo_disk); +#else add_disk(zv->zv_zso->zvo_disk); +#endif } else { ida_simple_remove(&zvol_ida, idx); } From 70b7b1975d70c80f9294efbf6964cfed236cf59f Mon Sep 17 00:00:00 2001 From: Rich Ercolani Date: Fri, 14 Jan 2022 05:07:33 -0500 Subject: [PATCH 022/109] Linux 5.16 compat: Added mapping for iov_iter_fault_in_readable Linux decided to rename this for some reason. At some point, we should probably invert this mapping, but for now... Reviewed-by: Brian Behlendorf Reviewed-by: Coleman Kane Signed-off-by: Rich Ercolani Closes #12975 --- config/kernel-add-disk.m4 | 1 - config/kernel-vfs-iov_iter.m4 | 22 ++++++++++++++++++++-- include/os/linux/spl/sys/uio.h | 4 ++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/config/kernel-add-disk.m4 b/config/kernel-add-disk.m4 index 3bc40f1e7d21..5d1779eb4328 100644 --- a/config/kernel-add-disk.m4 +++ b/config/kernel-add-disk.m4 @@ -13,7 +13,6 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_ADD_DISK], [ ]) ]) - AC_DEFUN([ZFS_AC_KERNEL_ADD_DISK], [ AC_MSG_CHECKING([whether add_disk() returns int]) ZFS_LINUX_TEST_RESULT([add_disk_ret], diff --git a/config/kernel-vfs-iov_iter.m4 b/config/kernel-vfs-iov_iter.m4 index ecdda939f1cf..57f78745a24b 100644 --- a/config/kernel-vfs-iov_iter.m4 +++ b/config/kernel-vfs-iov_iter.m4 @@ -41,6 +41,17 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_IOV_ITER], [ error = iov_iter_fault_in_readable(&iter, size); ]) + ZFS_LINUX_TEST_SRC([fault_in_iov_iter_readable], [ + #include + #include + ],[ + struct iov_iter iter = { 0 }; + size_t size = 512; + int error __attribute__ ((unused)); + + error = fault_in_iov_iter_readable(&iter, size); + ]) + ZFS_LINUX_TEST_SRC([iov_iter_count], [ #include #include @@ -123,8 +134,15 @@ AC_DEFUN([ZFS_AC_KERNEL_VFS_IOV_ITER], [ AC_DEFINE(HAVE_IOV_ITER_FAULT_IN_READABLE, 1, [iov_iter_fault_in_readable() is available]) ],[ - AC_MSG_RESULT(no) - enable_vfs_iov_iter="no" + AC_MSG_CHECKING([whether fault_in_iov_iter_readable() is available]) + ZFS_LINUX_TEST_RESULT([fault_in_iov_iter_readable], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_FAULT_IN_IOV_ITER_READABLE, 1, + [fault_in_iov_iter_readable() is available]) + ],[ + AC_MSG_RESULT(no) + enable_vfs_iov_iter="no" + ]) ]) AC_MSG_CHECKING([whether iov_iter_count() is available]) diff --git a/include/os/linux/spl/sys/uio.h b/include/os/linux/spl/sys/uio.h index 66af2b0b534c..439eec986236 100644 --- a/include/os/linux/spl/sys/uio.h +++ b/include/os/linux/spl/sys/uio.h @@ -34,6 +34,10 @@ #include #include +#if defined(HAVE_VFS_IOV_ITER) && defined(HAVE_FAULT_IN_IOV_ITER_READABLE) +#define iov_iter_fault_in_readable(a, b) fault_in_iov_iter_readable(a, b) +#endif + typedef struct iovec iovec_t; typedef enum zfs_uio_rw { From 8ef01afbfcd84cc69ce4ff728bdda01b03dd1589 Mon Sep 17 00:00:00 2001 From: Rich Ercolani Date: Wed, 19 Jan 2022 00:12:15 -0500 Subject: [PATCH 023/109] Add support for FALLOC_FL_ZERO_RANGE For us, I think it's always just FALLOC_FL_PUNCH_HOLE with a fake mustache on. Reviewed-by: Brian Behlendorf Reviewed-by: Coleman Kane Signed-off-by: Rich Ercolani Closes #12975 --- config/kernel-fallocate.m4 | 17 +++++++++++++++++ module/os/linux/zfs/zpl_file.c | 9 +++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/config/kernel-fallocate.m4 b/config/kernel-fallocate.m4 index 7a8550f7e760..815602d3e2c6 100644 --- a/config/kernel-fallocate.m4 +++ b/config/kernel-fallocate.m4 @@ -3,6 +3,10 @@ dnl # Linux 2.6.38 - 3.x API dnl # The fallocate callback was moved from the inode_operations dnl # structure to the file_operations structure. dnl # +dnl # +dnl # Linux 3.15+ +dnl # fallocate learned a new flag, FALLOC_FL_ZERO_RANGE +dnl # AC_DEFUN([ZFS_AC_KERNEL_SRC_FALLOCATE], [ ZFS_LINUX_TEST_SRC([file_fallocate], [ #include @@ -15,12 +19,25 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FALLOCATE], [ .fallocate = test_fallocate, }; ], []) + ZFS_LINUX_TEST_SRC([falloc_fl_zero_range], [ + #include + ],[ + int flags __attribute__ ((unused)); + flags = FALLOC_FL_ZERO_RANGE; + ]) ]) AC_DEFUN([ZFS_AC_KERNEL_FALLOCATE], [ AC_MSG_CHECKING([whether fops->fallocate() exists]) ZFS_LINUX_TEST_RESULT([file_fallocate], [ AC_MSG_RESULT(yes) + AC_MSG_CHECKING([whether FALLOC_FL_ZERO_RANGE exists]) + ZFS_LINUX_TEST_RESULT([falloc_fl_zero_range], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_FALLOC_FL_ZERO_RANGE, 1, [FALLOC_FL_ZERO_RANGE is defined]) + ],[ + AC_MSG_RESULT(no) + ]) ],[ ZFS_LINUX_TEST_ERROR([file_fallocate]) ]) diff --git a/module/os/linux/zfs/zpl_file.c b/module/os/linux/zfs/zpl_file.c index 7e88eae33711..f1241c443797 100644 --- a/module/os/linux/zfs/zpl_file.c +++ b/module/os/linux/zfs/zpl_file.c @@ -745,7 +745,12 @@ zpl_fallocate_common(struct inode *ip, int mode, loff_t offset, loff_t len) fstrans_cookie_t cookie; int error = 0; - if ((mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) != 0) + int test_mode = FALLOC_FL_PUNCH_HOLE; +#ifdef HAVE_FALLOC_FL_ZERO_RANGE + test_mode |= FALLOC_FL_ZERO_RANGE; +#endif + + if ((mode & ~(FALLOC_FL_KEEP_SIZE | test_mode)) != 0) return (-EOPNOTSUPP); if (offset < 0 || len <= 0) @@ -756,7 +761,7 @@ zpl_fallocate_common(struct inode *ip, int mode, loff_t offset, loff_t len) crhold(cr); cookie = spl_fstrans_mark(); - if (mode & FALLOC_FL_PUNCH_HOLE) { + if (mode & (test_mode)) { flock64_t bf; if (offset > olen) From 2ce06d93a84ef11c5834d8d037225d74fe1213f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Wed, 19 Jan 2022 21:10:49 +0100 Subject: [PATCH 024/109] Linux 5.17 compat: detect complete_and_exit() rename MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linux 5.17 sees a rename from complete_and_exit() to kthread complete_and_exit() Upstream commit cead18552660702a4a46f58e65188fe5f36e9dfe ("exit: Rename complete_and_exit to kthread_complete_and_exit") Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #12989 --- config/kernel-kthread.m4 | 27 +++++++++++++++++++++++++++ config/kernel.m4 | 2 ++ module/os/linux/spl/spl-thread.c | 2 +- 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 config/kernel-kthread.m4 diff --git a/config/kernel-kthread.m4 b/config/kernel-kthread.m4 new file mode 100644 index 000000000000..6459f831e799 --- /dev/null +++ b/config/kernel-kthread.m4 @@ -0,0 +1,27 @@ +AC_DEFUN([ZFS_AC_KERNEL_KTHREAD], [ + dnl # + dnl # 5.17 API, + dnl # cead18552660702a4a46f58e65188fe5f36e9dfe ("exit: Rename complete_and_exit to kthread_complete_and_exit") + dnl # + dnl # Also moves the definition from include/linux/kernel.h to include/linux/kthread.h + dnl # + AC_MSG_CHECKING([whether kthread_complete_and_exit() is available]) + ZFS_LINUX_TEST_RESULT([kthread_complete_and_exit], [ + AC_MSG_RESULT(yes) + AC_DEFINE(SPL_KTHREAD_COMPLETE_AND_EXIT, kthread_complete_and_exit, [kthread_complete_and_exit() available]) + ], [ + AC_MSG_RESULT(no) + AC_DEFINE(SPL_KTHREAD_COMPLETE_AND_EXIT, complete_and_exit, [using complete_and_exit() instead]) + ]) +]) + +AC_DEFUN([ZFS_AC_KERNEL_SRC_KTHREAD], [ + ZFS_LINUX_TEST_SRC([kthread_complete_and_exit], [ + #include + ], [ + struct completion *completion = NULL; + long code = 0; + + kthread_complete_and_exit(completion, code); + ]) +]) diff --git a/config/kernel.m4 b/config/kernel.m4 index b3ed966b7264..1adec975c0bf 100644 --- a/config/kernel.m4 +++ b/config/kernel.m4 @@ -136,6 +136,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [ ZFS_AC_KERNEL_SRC_STANDALONE_LINUX_STDARG ZFS_AC_KERNEL_SRC_PAGEMAP_FOLIO_WAIT_BIT ZFS_AC_KERNEL_SRC_ADD_DISK + ZFS_AC_KERNEL_SRC_KTHREAD AC_MSG_CHECKING([for available kernel interfaces]) ZFS_LINUX_TEST_COMPILE_ALL([kabi]) @@ -245,6 +246,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [ ZFS_AC_KERNEL_STANDALONE_LINUX_STDARG ZFS_AC_KERNEL_PAGEMAP_FOLIO_WAIT_BIT ZFS_AC_KERNEL_ADD_DISK + ZFS_AC_KERNEL_KTHREAD ]) dnl # diff --git a/module/os/linux/spl/spl-thread.c b/module/os/linux/spl/spl-thread.c index 834c527117a3..0ba4c18c1659 100644 --- a/module/os/linux/spl/spl-thread.c +++ b/module/os/linux/spl/spl-thread.c @@ -66,7 +66,7 @@ void __thread_exit(void) { tsd_exit(); - complete_and_exit(NULL, 0); + SPL_KTHREAD_COMPLETE_AND_EXIT(NULL, 0); /* Unreachable */ } EXPORT_SYMBOL(__thread_exit); From f42c12602913c7e725f647e48b12e6d0283756a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Wed, 19 Jan 2022 21:33:50 +0100 Subject: [PATCH 025/109] Linux 5.17 compat: dequeue_signal() takes a 4th argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linux 5.17's dequeue_signal() takes an additional enum pid_type * output argument Upstream commit 5768d8906bc23d512b1a736c1e198aa833a6daa4 ("signal: Requeue signals in the appropriate queue") Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #12989 --- config/kernel-kthread.m4 | 45 ++++++++++++++++++++++++++++++-- module/os/linux/spl/spl-thread.c | 5 ++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/config/kernel-kthread.m4 b/config/kernel-kthread.m4 index 6459f831e799..f5b824d7947a 100644 --- a/config/kernel-kthread.m4 +++ b/config/kernel-kthread.m4 @@ -1,4 +1,4 @@ -AC_DEFUN([ZFS_AC_KERNEL_KTHREAD], [ +AC_DEFUN([ZFS_AC_KERNEL_KTHREAD_COMPLETE_AND_EXIT], [ dnl # dnl # 5.17 API, dnl # cead18552660702a4a46f58e65188fe5f36e9dfe ("exit: Rename complete_and_exit to kthread_complete_and_exit") @@ -15,7 +15,24 @@ AC_DEFUN([ZFS_AC_KERNEL_KTHREAD], [ ]) ]) -AC_DEFUN([ZFS_AC_KERNEL_SRC_KTHREAD], [ +AC_DEFUN([ZFS_AC_KERNEL_KTHREAD_DEQUEUE_SIGNAL_4ARG], [ + dnl # + dnl # 5.17 API: enum pid_type * as new 4th dequeue_signal() argument, + dnl # 5768d8906bc23d512b1a736c1e198aa833a6daa4 ("signal: Requeue signals in the appropriate queue") + dnl # + dnl # int dequeue_signal(struct task_struct *task, sigset_t *mask, kernel_siginfo_t *info); + dnl # int dequeue_signal(struct task_struct *task, sigset_t *mask, kernel_siginfo_t *info, enum pid_type *type); + dnl # + AC_MSG_CHECKING([whether dequeue_signal() takes 4 arguments]) + ZFS_LINUX_TEST_RESULT([kthread_dequeue_signal], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_DEQUEUE_SIGNAL_4ARG, 1, [dequeue_signal() takes 4 arguments]) + ], [ + AC_MSG_RESULT(no) + ]) +]) + +AC_DEFUN([ZFS_AC_KERNEL_SRC_KTHREAD_COMPLETE_AND_EXIT], [ ZFS_LINUX_TEST_SRC([kthread_complete_and_exit], [ #include ], [ @@ -25,3 +42,27 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_KTHREAD], [ kthread_complete_and_exit(completion, code); ]) ]) + +AC_DEFUN([ZFS_AC_KERNEL_SRC_KTHREAD_DEQUEUE_SIGNAL_4ARG], [ + ZFS_LINUX_TEST_SRC([kthread_dequeue_signal], [ + #include + ], [ + struct task_struct *task = NULL; + sigset_t *mask = NULL; + kernel_siginfo_t *info = NULL; + enum pid_type *type = NULL; + int error __attribute__ ((unused)); + + error = dequeue_signal(task, mask, info, type); + ]) +]) + +AC_DEFUN([ZFS_AC_KERNEL_KTHREAD], [ + ZFS_AC_KERNEL_KTHREAD_COMPLETE_AND_EXIT + ZFS_AC_KERNEL_KTHREAD_DEQUEUE_SIGNAL_4ARG +]) + +AC_DEFUN([ZFS_AC_KERNEL_SRC_KTHREAD], [ + ZFS_AC_KERNEL_SRC_KTHREAD_COMPLETE_AND_EXIT + ZFS_AC_KERNEL_SRC_KTHREAD_DEQUEUE_SIGNAL_4ARG +]) diff --git a/module/os/linux/spl/spl-thread.c b/module/os/linux/spl/spl-thread.c index 0ba4c18c1659..16d2ca1b133b 100644 --- a/module/os/linux/spl/spl-thread.c +++ b/module/os/linux/spl/spl-thread.c @@ -188,7 +188,12 @@ issig(int why) spin_lock_irq(&task->sighand->siglock); int ret; +#ifdef HAVE_DEQUEUE_SIGNAL_4ARG + enum pid_type __type; + if ((ret = dequeue_signal(task, &set, &__info, &__type)) != 0) { +#else if ((ret = dequeue_signal(task, &set, &__info)) != 0) { +#endif #ifdef HAVE_SIGNAL_STOP spin_unlock_irq(&task->sighand->siglock); kernel_signal_stop(); From 4f6599416afcbf88b536ceb147a3f100712b3c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Mon, 24 Jan 2022 20:11:07 +0100 Subject: [PATCH 026/109] Linux 5.17 compat: PDE_DATA() renamed to pde_data() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream commit 359745d78351c6f5442435f81549f0207ece28aa ("proc: remove PDE_DATA() completely") Link: https://lore.kernel.org/all/20211124081956.87711-2-songmuchun@bytedance.com/T/#u Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #13004 Closes #12989 --- config/kernel-pde-data.m4 | 16 +++++++++------- module/os/linux/spl/spl-kstat.c | 2 +- module/os/linux/spl/spl-procfs-list.c | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/config/kernel-pde-data.m4 b/config/kernel-pde-data.m4 index f866d77a11df..4fc665dfbe2e 100644 --- a/config/kernel-pde-data.m4 +++ b/config/kernel-pde-data.m4 @@ -1,20 +1,22 @@ dnl # -dnl # 3.10 API change, -dnl # PDE is replaced by PDE_DATA +dnl # 5.17 API: PDE_DATA() renamed to pde_data(), +dnl # 359745d78351c6f5442435f81549f0207ece28aa ("proc: remove PDE_DATA() completely") dnl # AC_DEFUN([ZFS_AC_KERNEL_SRC_PDE_DATA], [ ZFS_LINUX_TEST_SRC([pde_data], [ #include ], [ - PDE_DATA(NULL); + pde_data(NULL); ]) ]) AC_DEFUN([ZFS_AC_KERNEL_PDE_DATA], [ - AC_MSG_CHECKING([whether PDE_DATA() is available]) - ZFS_LINUX_TEST_RESULT_SYMBOL([pde_data], [PDE_DATA], [], [ + AC_MSG_CHECKING([whether pde_data() is lowercase]) + ZFS_LINUX_TEST_RESULT([pde_data], [ AC_MSG_RESULT(yes) - ],[ - ZFS_LINUX_TEST_ERROR([PDE_DATA]) + AC_DEFINE(SPL_PDE_DATA, pde_data, [pde_data() is pde_data()]) + ], [ + AC_MSG_RESULT(no) + AC_DEFINE(SPL_PDE_DATA, PDE_DATA, [pde_data() is PDE_DATA()]) ]) ]) diff --git a/module/os/linux/spl/spl-kstat.c b/module/os/linux/spl/spl-kstat.c index 0c46708326d8..b5666e78842b 100644 --- a/module/os/linux/spl/spl-kstat.c +++ b/module/os/linux/spl/spl-kstat.c @@ -418,7 +418,7 @@ proc_kstat_open(struct inode *inode, struct file *filp) return (rc); f = filp->private_data; - f->private = PDE_DATA(inode); + f->private = SPL_PDE_DATA(inode); return (0); } diff --git a/module/os/linux/spl/spl-procfs-list.c b/module/os/linux/spl/spl-procfs-list.c index cae13228c62c..2cba1a3022e0 100644 --- a/module/os/linux/spl/spl-procfs-list.c +++ b/module/os/linux/spl/spl-procfs-list.c @@ -175,7 +175,7 @@ procfs_list_open(struct inode *inode, struct file *filp) struct seq_file *f = filp->private_data; procfs_list_cursor_t *cursor = f->private; - cursor->procfs_list = PDE_DATA(inode); + cursor->procfs_list = SPL_PDE_DATA(inode); cursor->cached_node = NULL; cursor->cached_pos = 0; From 1009e609920f83d803224d1345332fb82dc85009 Mon Sep 17 00:00:00 2001 From: Finix1979 Date: Wed, 26 Jan 2022 05:12:49 +0800 Subject: [PATCH 027/109] Linux <4.8 compat: submit_bio() rw arg When using the two argument version of submit_bio() in kernel's prior to 4.8 the first argument should be specified. It's used by block dump to report the bio direction. Reviewed-by: Brian Behlendorf Signed-off-by: Finix Yan Closes #13006 --- module/os/linux/zfs/vdev_disk.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/os/linux/zfs/vdev_disk.c b/module/os/linux/zfs/vdev_disk.c index a432a736453c..581a790865b7 100644 --- a/module/os/linux/zfs/vdev_disk.c +++ b/module/os/linux/zfs/vdev_disk.c @@ -446,7 +446,7 @@ vdev_submit_bio_impl(struct bio *bio) #ifdef HAVE_1ARG_SUBMIT_BIO (void) submit_bio(bio); #else - (void) submit_bio(0, bio); + (void) submit_bio(bio_data_dir(bio), bio); #endif } From e257bd481bb77181e475bf35292c1a4b0cb2c57a Mon Sep 17 00:00:00 2001 From: George Amanakis Date: Fri, 21 Jan 2022 20:41:17 +0100 Subject: [PATCH 028/109] Introduce a flag to skip comparing the local mac when raw sending Raw receiving a snapshot back to the originating dataset is currently impossible because of user accounting being present in the originating dataset. One solution would be resetting user accounting when raw receiving on the receiving dataset. However, to recalculate it we would have to dirty all dnodes, which may not be preferable on big datasets. Instead, we rely on the os_phys flag OBJSET_FLAG_USERACCOUNTING_COMPLETE to indicate that user accounting is incomplete when raw receiving. Thus, on the next mount of the receiving dataset the local mac protecting user accounting is zeroed out. The flag is then cleared when user accounting of the raw received snapshot is calculated. Reviewed-by: Brian Behlendorf Signed-off-by: George Amanakis Closes #12981 Closes #10523 Closes #11221 Closes #11294 Closes #12594 Issue #11300 --- module/os/freebsd/zfs/zio_crypt.c | 17 ++- module/os/linux/zfs/zio_crypt.c | 17 ++- module/zfs/dnode_sync.c | 9 +- module/zfs/dsl_crypt.c | 9 +- tests/runfiles/common.run | 3 +- .../tests/functional/userquota/Makefile.am | 3 +- .../userquota/userspace_send_encrypted.ksh | 119 ++++++++++++++++++ 7 files changed, 163 insertions(+), 14 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/userquota/userspace_send_encrypted.ksh diff --git a/module/os/freebsd/zfs/zio_crypt.c b/module/os/freebsd/zfs/zio_crypt.c index aeb42b304e73..e4ab78705832 100644 --- a/module/os/freebsd/zfs/zio_crypt.c +++ b/module/os/freebsd/zfs/zio_crypt.c @@ -1076,11 +1076,26 @@ zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN); + /* + * This is necessary here as we check next whether + * OBJSET_FLAG_USERACCOUNTING_COMPLETE is set in order to + * decide if the local_mac should be zeroed out. That flag will always + * be set by dmu_objset_id_quota_upgrade_cb() and + * dmu_objset_userspace_upgrade_cb() if useraccounting has been + * completed. + */ + intval = osp->os_flags; + if (should_bswap) + intval = BSWAP_64(intval); + boolean_t uacct_incomplete = + !(intval & OBJSET_FLAG_USERACCOUNTING_COMPLETE); + /* * The local MAC protects the user, group and project accounting. * If these objects are not present, the local MAC is zeroed out. */ - if ((datalen >= OBJSET_PHYS_SIZE_V3 && + if (uacct_incomplete || + (datalen >= OBJSET_PHYS_SIZE_V3 && osp->os_userused_dnode.dn_type == DMU_OT_NONE && osp->os_groupused_dnode.dn_type == DMU_OT_NONE && osp->os_projectused_dnode.dn_type == DMU_OT_NONE) || diff --git a/module/os/linux/zfs/zio_crypt.c b/module/os/linux/zfs/zio_crypt.c index 52e62f4d1da4..13e156ac3b6a 100644 --- a/module/os/linux/zfs/zio_crypt.c +++ b/module/os/linux/zfs/zio_crypt.c @@ -1203,11 +1203,26 @@ zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN); + /* + * This is necessary here as we check next whether + * OBJSET_FLAG_USERACCOUNTING_COMPLETE is set in order to + * decide if the local_mac should be zeroed out. That flag will always + * be set by dmu_objset_id_quota_upgrade_cb() and + * dmu_objset_userspace_upgrade_cb() if useraccounting has been + * completed. + */ + intval = osp->os_flags; + if (should_bswap) + intval = BSWAP_64(intval); + boolean_t uacct_incomplete = + !(intval & OBJSET_FLAG_USERACCOUNTING_COMPLETE); + /* * The local MAC protects the user, group and project accounting. * If these objects are not present, the local MAC is zeroed out. */ - if ((datalen >= OBJSET_PHYS_SIZE_V3 && + if (uacct_incomplete || + (datalen >= OBJSET_PHYS_SIZE_V3 && osp->os_userused_dnode.dn_type == DMU_OT_NONE && osp->os_groupused_dnode.dn_type == DMU_OT_NONE && osp->os_projectused_dnode.dn_type == DMU_OT_NONE) || diff --git a/module/zfs/dnode_sync.c b/module/zfs/dnode_sync.c index dd37e3af7ed5..f574130e5049 100644 --- a/module/zfs/dnode_sync.c +++ b/module/zfs/dnode_sync.c @@ -655,8 +655,13 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx) DNODE_FLAG_USEROBJUSED_ACCOUNTED; mutex_exit(&dn->dn_mtx); dmu_objset_userquota_get_ids(dn, B_FALSE, tx); - } else { - /* Once we account for it, we should always account for it */ + } else if (!(os->os_encrypted && dmu_objset_is_receiving(os))) { + /* + * Once we account for it, we should always account for it, + * except for the case of a raw receive. We will not be able + * to account for it until the receiving dataset has been + * mounted. + */ ASSERT(!(dn->dn_phys->dn_flags & DNODE_FLAG_USERUSED_ACCOUNTED)); ASSERT(!(dn->dn_phys->dn_flags & diff --git a/module/zfs/dsl_crypt.c b/module/zfs/dsl_crypt.c index 26d4c2fe7e33..1ea184de338c 100644 --- a/module/zfs/dsl_crypt.c +++ b/module/zfs/dsl_crypt.c @@ -2007,14 +2007,6 @@ dsl_crypto_recv_raw_objset_check(dsl_dataset_t *ds, dsl_dataset_t *fromds, if (ret != 0) return (ret); - /* - * Useraccounting is not portable and must be done with the keys loaded. - * Therefore, whenever we do any kind of receive the useraccounting - * must not be present. - */ - ASSERT0(os->os_flags & OBJSET_FLAG_USERACCOUNTING_COMPLETE); - ASSERT0(os->os_flags & OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE); - mdn = DMU_META_DNODE(os); /* @@ -2106,6 +2098,7 @@ dsl_crypto_recv_raw_objset_sync(dsl_dataset_t *ds, dmu_objset_type_t ostype, arc_release(os->os_phys_buf, &os->os_phys_buf); bcopy(portable_mac, os->os_phys->os_portable_mac, ZIO_OBJSET_MAC_LEN); bzero(os->os_phys->os_local_mac, ZIO_OBJSET_MAC_LEN); + os->os_flags &= ~OBJSET_FLAG_USERACCOUNTING_COMPLETE; os->os_next_write_raw[tx->tx_txg & TXG_MASK] = B_TRUE; /* set metadnode compression and checksum */ diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 97ee7b8ae241..bc32dfc68346 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -889,7 +889,8 @@ tests = [ 'userquota_004_pos', 'userquota_005_neg', 'userquota_006_pos', 'userquota_007_pos', 'userquota_008_pos', 'userquota_009_pos', 'userquota_010_pos', 'userquota_011_pos', 'userquota_012_neg', - 'userspace_001_pos', 'userspace_002_pos', 'userspace_encrypted'] + 'userspace_001_pos', 'userspace_002_pos', 'userspace_encrypted', + 'userspace_send_encrypted'] tags = ['functional', 'userquota'] [tests/functional/vdev_zaps] diff --git a/tests/zfs-tests/tests/functional/userquota/Makefile.am b/tests/zfs-tests/tests/functional/userquota/Makefile.am index 9100e4adadca..2c94d3e1521c 100644 --- a/tests/zfs-tests/tests/functional/userquota/Makefile.am +++ b/tests/zfs-tests/tests/functional/userquota/Makefile.am @@ -21,7 +21,8 @@ dist_pkgdata_SCRIPTS = \ userspace_001_pos.ksh \ userspace_002_pos.ksh \ userspace_003_pos.ksh \ - userspace_encrypted.ksh + userspace_encrypted.ksh \ + userspace_send_encrypted.ksh dist_pkgdata_DATA = \ userquota.cfg \ diff --git a/tests/zfs-tests/tests/functional/userquota/userspace_send_encrypted.ksh b/tests/zfs-tests/tests/functional/userquota/userspace_send_encrypted.ksh new file mode 100755 index 000000000000..e9ef0c4262e7 --- /dev/null +++ b/tests/zfs-tests/tests/functional/userquota/userspace_send_encrypted.ksh @@ -0,0 +1,119 @@ +#!/bin/ksh -p +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright 2021, George Amanakis . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/userquota/userquota_common.kshlib + +# +# DESCRIPTION: +# Sending raw encrypted datasets back to the source dataset succeeds. +# +# +# STRATEGY: +# 1. Create encrypted source dataset, set userquota and write a file +# 2. Create base snapshot +# 3. Write new file, snapshot, get userspace +# 4. Raw send both snapshots +# 5. Destroy latest snapshot at source and rollback +# 6. Unmount, unload key from source +# 7. Raw send latest snapshot back to source +# 8. Mount both source and target datasets +# 9. Verify encrypted datasets support 'zfs userspace' and 'zfs groupspace' +# and the accounting is done correctly +# + +function cleanup +{ + destroy_pool $POOLNAME + rm -f $FILEDEV +} + +log_onexit cleanup + +FILEDEV="$TEST_BASE_DIR/userspace_encrypted" +POOLNAME="testpool$$" +ENC_SOURCE="$POOLNAME/source" +ENC_TARGET="$POOLNAME/target" + +log_assert "Sending raw encrypted datasets back to the source dataset succeeds." + +# Setup pool and create source +truncate -s 200m $FILEDEV +log_must zpool create -o feature@encryption=enabled $POOLNAME \ + $FILEDEV +log_must eval "echo 'password' | zfs create -o encryption=on" \ + "-o keyformat=passphrase -o keylocation=prompt " \ + "$ENC_SOURCE" + +# Set user quota and write file +log_must zfs set userquota@$QUSER1=50m $ENC_SOURCE +mkmount_writable $ENC_SOURCE +mntpnt=$(get_prop mountpoint $ENC_SOURCE) +log_must user_run $QUSER1 mkfile 10m /$mntpnt/file1 +sync + +# Snapshot +log_must zfs snap $ENC_SOURCE@base + +# Write new file, snapshot, get userspace +log_must user_run $QUSER1 mkfile 20m /$mntpnt/file2 +log_must zfs snap $ENC_SOURCE@s1 + +# Raw send both snapshots +log_must eval "zfs send -w $ENC_SOURCE@base | zfs recv " \ + "$ENC_TARGET" +log_must eval "zfs send -w -i @base $ENC_SOURCE@s1 | zfs recv " \ + "$ENC_TARGET" + +# Destroy latest snapshot at source and rollback +log_must zfs destroy $ENC_SOURCE@s1 +log_must zfs rollback $ENC_SOURCE@base +rollback_uspace=$(zfs userspace -Hp $ENC_SOURCE | \ + awk "/$QUSER1/"' {printf "%d\n", $4 / 1024 / 1024}') + +# Unmount, unload key +log_must zfs umount $ENC_SOURCE +log_must zfs unload-key -a + +# Raw send latest snapshot back to source +log_must eval "zfs send -w -i @base $ENC_TARGET@s1 | zfs recv " \ + "$ENC_SOURCE" + +# Mount encrypted datasets and verify they support 'zfs userspace' and +# 'zfs groupspace' and the accounting is done correctly +log_must eval "echo 'password' | zfs load-key $ENC_SOURCE" +log_must eval "echo 'password' | zfs load-key $ENC_TARGET" +log_must zfs mount $ENC_SOURCE +log_must zfs mount $ENC_TARGET +sync + +sleep 5 + +src_uspace=$(zfs userspace -Hp $ENC_SOURCE | \ + awk "/$QUSER1/"' {printf "%d\n", $4 / 1024 / 1024}') +tgt_uspace=$(zfs userspace -Hp $ENC_TARGET | \ + awk "/$QUSER1/"' {printf "%d\n", $4 / 1024 / 1024}') +log_must test "$src_uspace" -eq "$tgt_uspace" +log_must test "$rollback_uspace" -ne "$src_uspace" + +src_uquota=$(zfs userspace -Hp $ENC_SOURCE | awk "/$QUSER1/"' {print $5}') +tgt_uquota=$(zfs userspace -Hp $ENC_TARGET | awk "/$QUSER1/"' {print $5}') +log_must test "$src_uquota" -eq "$tgt_uquota" + +# Cleanup +cleanup + +log_pass "Sending raw encrypted datasets back to the source dataset succeeds." From e56dffe4b5cd00df2d46b75a80d3d17c8fe70eec Mon Sep 17 00:00:00 2001 From: shodanshok Date: Wed, 22 Dec 2021 01:24:05 +0100 Subject: [PATCH 029/109] zed: send notification email by default Reviewed-by: Brian Behlendorf Reviewed-by: George Melikov Signed-off-by: Gionatan Danti Closes #12806 --- cmd/zed/zed.d/zed.rc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/zed/zed.d/zed.rc b/cmd/zed/zed.d/zed.rc index 1c278b2ef96e..b9bbb86aab12 100644 --- a/cmd/zed/zed.d/zed.rc +++ b/cmd/zed/zed.d/zed.rc @@ -13,9 +13,9 @@ # Email address of the zpool administrator for receipt of notifications; # multiple addresses can be specified if they are delimited by whitespace. # Email will only be sent if ZED_EMAIL_ADDR is defined. -# Disabled by default; uncomment to enable. +# Enabled by default; comment to disable. # -#ZED_EMAIL_ADDR="root" +ZED_EMAIL_ADDR="root" ## # Name or path of executable responsible for sending notifications via email; From 44bb2fcf38683820ccf18a2ff01668ad6cb55425 Mon Sep 17 00:00:00 2001 From: Phil Kauffman Date: Thu, 3 Feb 2022 16:31:57 -0600 Subject: [PATCH 030/109] zed-functions.sh: escape newline to produce valid json This was discovered when using Discords Slack compatible webhook. Slack webhooks works without the escape, however Discord rightly refuses the POST as it contains invalid JSON. https://discord.com/developers/docs/resources/webhook#execute-slackcompatible-webhook Valid (while escaping the newline: ``` + msg_json='{"text": "*ZFS scrub_finish error for test on quartz*\nZFS has detected a data error:\n\n eid: 124\n class: scrub_finish\n host: quartz\n time: \n error: \n objid: :\n pool: test\n"}' ``` Invalid (no escape): ``` + msg_json='{"text": "*ZFS scrub_finish error for test on quartz* ZFS has detected a data error:\n\n eid: 124\n class: scrub_finish\n host: quartz\n time: \n error: \n objid: :\n pool: test\n"}' ``` The new line gets rendered and not sent inside the JSON as intended. ``` ++ curl -X POST https://discord.com/api/webhooks/{webhook.id}/{webhook.token}/slack --header 'Content-Type: application/json' --data-binary '{"text": "*ZFS scrub_finish error for test on quartz* ZFS has detected a data error:\n\n eid: 124\n class: scrub_finish\n host: quartz\n time: \n error: \n objid: :\n pool: test\n"}' + msg_out='{"message": "Cannot send an empty message", "code": 50006}' ``` Test method: `root@quartz:/etc/zfs/zed.d# export ZED_ZEDLET_DIR=/etc/zfs/zed.d; export ZEVENT_EID=124; export ZEVENT_SUBCLASS=scrub_finish; export ZEVENT_POOL=test; export ZED_NOTIFY_DATA=1; bash -x ./data-notify.sh` Reviewed-by: Damian Szuberski Reviewed-by: Brian Behlendorf Signed-off-by: Philip Kauffman Closes #13049 --- cmd/zed/zed.d/zed-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/zed/zed.d/zed-functions.sh b/cmd/zed/zed.d/zed-functions.sh index d1ebf7dbcc16..80c9fe61854e 100644 --- a/cmd/zed/zed.d/zed-functions.sh +++ b/cmd/zed/zed.d/zed-functions.sh @@ -413,7 +413,7 @@ zed_notify_slack_webhook() # Construct the JSON message for posting. # - msg_json="$(printf '{"text": "*%s*\n%s"}' "${subject}" "${msg_body}" )" + msg_json="$(printf '{"text": "*%s*\\n%s"}' "${subject}" "${msg_body}" )" # Send the POST request and check for errors. # From 46135048098d8e4784610b88e4db1b9979127361 Mon Sep 17 00:00:00 2001 From: Scott Colby Date: Thu, 13 May 2021 13:02:24 -0400 Subject: [PATCH 031/109] zed: Add Pushover notifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add zed_notify_pushover to zed-functions.sh, along with the necessary configuration variables in zed.rc. Reviewed-by: Brian Behlendorf Reviewed-by: Ahelenia Ziemiańska Signed-off-by: Scott Colby Closes #12012 --- cmd/zed/zed.d/zed-functions.sh | 82 ++++++++++++++++++++++++++++++++++ cmd/zed/zed.d/zed.rc | 17 +++++++ 2 files changed, 99 insertions(+) diff --git a/cmd/zed/zed.d/zed-functions.sh b/cmd/zed/zed.d/zed-functions.sh index 80c9fe61854e..9044922ac346 100644 --- a/cmd/zed/zed.d/zed-functions.sh +++ b/cmd/zed/zed.d/zed-functions.sh @@ -202,6 +202,10 @@ zed_notify() [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + zed_notify_pushover "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + [ "${num_success}" -gt 0 ] && return 0 [ "${num_failure}" -gt 0 ] && return 1 return 2 @@ -433,6 +437,84 @@ zed_notify_slack_webhook() return 0 } +# zed_notify_pushover (subject, pathname) +# +# Send a notification via Pushover . +# The access token (ZED_PUSHOVER_TOKEN) identifies this client to the +# Pushover server. The user token (ZED_PUSHOVER_USER) defines the user or +# group to which the notification will be sent. +# +# Requires curl and sed executables to be installed in the standard PATH. +# +# References +# https://pushover.net/api +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_PUSHOVER_TOKEN +# ZED_PUSHOVER_USER +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_pushover() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_out + local msg_err + local url="https://api.pushover.net/1/messages.json" + + [ -n "${ZED_PUSHOVER_TOKEN}" ] && [ -n "${ZED_PUSHOVER_USER}" ] || return 2 + + if [ ! -r "${pathname}" ]; then + zed_log_err "pushover cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "curl" "sed" || return 1 + + # Read the message body in. + # + msg_body="$(cat "${pathname}")" + + if [ -z "${msg_body}" ] + then + msg_body=$subject + subject="" + fi + + # Send the POST request and check for errors. + # + msg_out="$( \ + curl \ + --form-string "token=${ZED_PUSHOVER_TOKEN}" \ + --form-string "user=${ZED_PUSHOVER_USER}" \ + --form-string "message=${msg_body}" \ + --form-string "title=${subject}" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "pushover \"${msg_err}"\" + return 1 + fi + return 0 +} + + # zed_rate_limit (tag, [interval]) # # Check whether an event of a given type [tag] has already occurred within the diff --git a/cmd/zed/zed.d/zed.rc b/cmd/zed/zed.d/zed.rc index b9bbb86aab12..9ac77f929c73 100644 --- a/cmd/zed/zed.d/zed.rc +++ b/cmd/zed/zed.d/zed.rc @@ -82,6 +82,23 @@ ZED_EMAIL_ADDR="root" # #ZED_SLACK_WEBHOOK_URL="" +## +# Pushover token. +# This defines the application from which the notification will be sent. +# +# Disabled by default; uncomment to enable. +# ZED_PUSHOVER_USER, below, must also be configured. +# +#ZED_PUSHOVER_TOKEN="" + +## +# Pushover user key. +# This defines which user or group will receive Pushover notifications. +# +# Disabled by default; uncomment to enable. +# ZED_PUSHOVER_TOKEN, above, must also be configured. +#ZED_PUSHOVER_USER="" + ## # Default directory for zed state files. # From f31b45176c8ae9a511bf6b50e83795b42a38c8c7 Mon Sep 17 00:00:00 2001 From: Jorgen Lundman Date: Fri, 10 Sep 2021 02:44:21 +0900 Subject: [PATCH 032/109] Upstream: Add snapshot and zvol events For kernel to send snapshot mount/unmount events to zed. For kernel to send symlink creates/removes on zvol plumbing. (/dev/run/dsk/zvol/$pool/$zvol -> /dev/diskX) If zed misses the ENODEV, all errors after are EINVAL. Treat any error as kernel module failure. Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf Signed-off-by: Jorgen Lundman Closes #12416 --- cmd/zed/agents/zfs_retire.c | 1 + cmd/zed/zed.c | 2 +- cmd/zed/zed_conf.c | 1 + cmd/zed/zed_exec.c | 2 ++ include/sys/fm/fs/zfs.h | 9 +++++++ include/sys/spa.h | 2 ++ include/sys/zio.h | 2 ++ module/zfs/zfs_fm.c | 52 +++++++++++++++++++++++++++++++++++++ 8 files changed, 70 insertions(+), 1 deletion(-) diff --git a/cmd/zed/agents/zfs_retire.c b/cmd/zed/agents/zfs_retire.c index 1c4cc885b5e5..f4063bea7378 100644 --- a/cmd/zed/agents/zfs_retire.c +++ b/cmd/zed/agents/zfs_retire.c @@ -40,6 +40,7 @@ #include #include #include +#include #include "zfs_agents.h" #include "fmd_api.h" diff --git a/cmd/zed/zed.c b/cmd/zed/zed.c index 0aa03fded468..e45176c00bf2 100644 --- a/cmd/zed/zed.c +++ b/cmd/zed/zed.c @@ -291,7 +291,7 @@ main(int argc, char *argv[]) rv = zed_event_service(&zcp); /* ENODEV: When kernel module is unloaded (osx) */ - if (rv == ENODEV) + if (rv != 0) break; } diff --git a/cmd/zed/zed_conf.c b/cmd/zed/zed_conf.c index 2cf2311dbb42..59935102f123 100644 --- a/cmd/zed/zed_conf.c +++ b/cmd/zed/zed_conf.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include diff --git a/cmd/zed/zed_exec.c b/cmd/zed/zed_exec.c index 1eecfa0a92c4..03dcd03aceb7 100644 --- a/cmd/zed/zed_exec.c +++ b/cmd/zed/zed_exec.c @@ -26,6 +26,8 @@ #include #include #include +#include + #include "zed_exec.h" #include "zed_log.h" #include "zed_strings.h" diff --git a/include/sys/fm/fs/zfs.h b/include/sys/fm/fs/zfs.h index 6491606d328b..cd080c8ee667 100644 --- a/include/sys/fm/fs/zfs.h +++ b/include/sys/fm/fs/zfs.h @@ -110,6 +110,10 @@ extern "C" { #define FM_EREPORT_PAYLOAD_ZFS_BAD_CLEARED_BITS "bad_cleared_bits" #define FM_EREPORT_PAYLOAD_ZFS_BAD_SET_HISTOGRAM "bad_set_histogram" #define FM_EREPORT_PAYLOAD_ZFS_BAD_CLEARED_HISTOGRAM "bad_cleared_histogram" +#define FM_EREPORT_PAYLOAD_ZFS_SNAPSHOT_NAME "snapshot_name" +#define FM_EREPORT_PAYLOAD_ZFS_DEVICE_NAME "device_name" +#define FM_EREPORT_PAYLOAD_ZFS_RAW_DEVICE_NAME "raw_name" +#define FM_EREPORT_PAYLOAD_ZFS_VOLUME "volume" #define FM_EREPORT_FAILMODE_WAIT "wait" #define FM_EREPORT_FAILMODE_CONTINUE "continue" @@ -119,6 +123,11 @@ extern "C" { #define FM_RESOURCE_AUTOREPLACE "autoreplace" #define FM_RESOURCE_STATECHANGE "statechange" +#define FM_RESOURCE_ZFS_SNAPSHOT_MOUNT "snapshot_mount" +#define FM_RESOURCE_ZFS_SNAPSHOT_UNMOUNT "snapshot_unmount" +#define FM_RESOURCE_ZVOL_CREATE_SYMLINK "zvol_create" +#define FM_RESOURCE_ZVOL_REMOVE_SYMLINK "zvol_remove" + #ifdef __cplusplus } #endif diff --git a/include/sys/spa.h b/include/sys/spa.h index 532926e12487..f168015abffc 100644 --- a/include/sys/spa.h +++ b/include/sys/spa.h @@ -1189,6 +1189,8 @@ extern void spa_configfile_set(spa_t *, nvlist_t *, boolean_t); /* asynchronous event notification */ extern void spa_event_notify(spa_t *spa, vdev_t *vdev, nvlist_t *hist_nvl, const char *name); +extern void zfs_ereport_zvol_post(const char *subclass, const char *name, + const char *device_name, const char *raw_name); /* waiting for pool activities to complete */ extern int spa_wait(const char *pool, zpool_wait_activity_t activity, diff --git a/include/sys/zio.h b/include/sys/zio.h index 97bd1a26ad40..e46455ea990b 100644 --- a/include/sys/zio.h +++ b/include/sys/zio.h @@ -690,6 +690,8 @@ extern int zfs_ereport_post_checksum(spa_t *spa, vdev_t *vd, struct zio_bad_cksum *info); void zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr); +extern void zfs_ereport_snapshot_post(const char *subclass, spa_t *spa, + const char *name); /* Called from spa_sync(), but primarily an injection handler */ extern void spa_handle_ignored_writes(spa_t *spa); diff --git a/module/zfs/zfs_fm.c b/module/zfs/zfs_fm.c index 60e631567a89..007f31b4e7b3 100644 --- a/module/zfs/zfs_fm.c +++ b/module/zfs/zfs_fm.c @@ -1444,6 +1444,58 @@ zfs_ereport_fini(void) mutex_destroy(&recent_events_lock); } +void +zfs_ereport_snapshot_post(const char *subclass, spa_t *spa, const char *name) +{ + nvlist_t *aux; + + aux = fm_nvlist_create(NULL); + nvlist_add_string(aux, FM_EREPORT_PAYLOAD_ZFS_SNAPSHOT_NAME, name); + + zfs_post_common(spa, NULL, FM_RSRC_CLASS, subclass, aux); + fm_nvlist_destroy(aux, FM_NVA_FREE); +} + +/* + * Post when a event when a zvol is created or removed + * + * This is currently only used by macOS, since it uses the event to create + * symlinks between the volume name (mypool/myvol) and the actual /dev + * device (/dev/disk3). For example: + * + * /var/run/zfs/dsk/mypool/myvol -> /dev/disk3 + * + * name: The full name of the zvol ("mypool/myvol") + * dev_name: The full /dev name for the zvol ("/dev/disk3") + * raw_name: The raw /dev name for the zvol ("/dev/rdisk3") + */ +void +zfs_ereport_zvol_post(const char *subclass, const char *name, + const char *dev_name, const char *raw_name) +{ + nvlist_t *aux; + char *r; + + boolean_t locked = mutex_owned(&spa_namespace_lock); + if (!locked) mutex_enter(&spa_namespace_lock); + spa_t *spa = spa_lookup(name); + if (!locked) mutex_exit(&spa_namespace_lock); + + if (spa == NULL) + return; + + aux = fm_nvlist_create(NULL); + nvlist_add_string(aux, FM_EREPORT_PAYLOAD_ZFS_DEVICE_NAME, dev_name); + nvlist_add_string(aux, FM_EREPORT_PAYLOAD_ZFS_RAW_DEVICE_NAME, + raw_name); + r = strchr(name, '/'); + if (r && r[1]) + nvlist_add_string(aux, FM_EREPORT_PAYLOAD_ZFS_VOLUME, &r[1]); + + zfs_post_common(spa, NULL, FM_RSRC_CLASS, subclass, aux); + fm_nvlist_destroy(aux, FM_NVA_FREE); +} + EXPORT_SYMBOL(zfs_ereport_post); EXPORT_SYMBOL(zfs_ereport_is_valid); EXPORT_SYMBOL(zfs_ereport_post_checksum); From 487bb776234b9f1d947879e306fc33402ab7509a Mon Sep 17 00:00:00 2001 From: Francesco Mazzoli Date: Sat, 16 Oct 2021 00:57:23 +0200 Subject: [PATCH 033/109] Notify on UNAVAIL statechange `UNAVAIL` is maybe not quite as concerning as `DEGRADED`, but still an event of notice, in my opinion. For example it is triggered when a drive goes missing. Reviewed-by: Don Brady Reviewed-by: Brian Behlendorf Signed-off-by: Francesco Mazzoli Closes #12629 Closes #12630 --- cmd/zed/zed.d/statechange-notify.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/zed/zed.d/statechange-notify.sh b/cmd/zed/zed.d/statechange-notify.sh index 76f09061c5b5..ab11dfbc99d5 100755 --- a/cmd/zed/zed.d/statechange-notify.sh +++ b/cmd/zed/zed.d/statechange-notify.sh @@ -15,7 +15,7 @@ # Send notification in response to a fault induced statechange # # ZEVENT_SUBCLASS: 'statechange' -# ZEVENT_VDEV_STATE_STR: 'DEGRADED', 'FAULTED' or 'REMOVED' +# ZEVENT_VDEV_STATE_STR: 'DEGRADED', 'FAULTED', 'REMOVED', or 'UNAVAIL' # # Exit codes: # 0: notification sent @@ -31,7 +31,8 @@ if [ "${ZEVENT_VDEV_STATE_STR}" != "FAULTED" ] \ && [ "${ZEVENT_VDEV_STATE_STR}" != "DEGRADED" ] \ - && [ "${ZEVENT_VDEV_STATE_STR}" != "REMOVED" ]; then + && [ "${ZEVENT_VDEV_STATE_STR}" != "REMOVED" ] \ + && [ "${ZEVENT_VDEV_STATE_STR}" != "UNAVAIL" ]; then exit 3 fi From d76917b2ecd681cea8911ce11375842cab312c1a Mon Sep 17 00:00:00 2001 From: ogelpre Date: Sun, 12 Dec 2021 20:17:14 +0100 Subject: [PATCH 034/109] Add init script to load keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add new init scripts which allow automatic loading of keys if keylocation property is set to a URI. Reviewed-by: Ahelenia Ziemiańska Reviewed-by: Brian Behlendorf Signed-off-by: Benedikt Neuffer Closes #11659 Closes #11662 --- etc/default/zfs.in | 8 ++- etc/init.d/.gitignore | 1 + etc/init.d/Makefile.am | 2 +- etc/init.d/README.md | 11 ++-- etc/init.d/zfs-load-key.in | 131 +++++++++++++++++++++++++++++++++++++ etc/zfs/zfs-functions.in | 9 ++- rpm/generic/zfs.spec.in | 2 + 7 files changed, 155 insertions(+), 9 deletions(-) create mode 100755 etc/init.d/zfs-load-key.in diff --git a/etc/default/zfs.in b/etc/default/zfs.in index 3b6e5486dd33..77cc604d8f4f 100644 --- a/etc/default/zfs.in +++ b/etc/default/zfs.in @@ -1,4 +1,4 @@ -# ZoL userland configuration. +# OpenZFS userland configuration. # NOTE: This file is intended for sysv init and initramfs. # Changing some of these settings may not make any difference on @@ -9,6 +9,12 @@ # To enable a boolean setting, set it to yes, on, true, or 1. # Anything else will be interpreted as unset. +# Run `zfs load-key` during system start? +ZFS_LOAD_KEY='yes' + +# Run `zfs unload-key` during system stop? +ZFS_UNLOAD_KEY='no' + # Run `zfs mount -a` during system start? ZFS_MOUNT='yes' diff --git a/etc/init.d/.gitignore b/etc/init.d/.gitignore index 43a673d55343..b3402f831806 100644 --- a/etc/init.d/.gitignore +++ b/etc/init.d/.gitignore @@ -1,4 +1,5 @@ zfs-import +zfs-load-key zfs-mount zfs-share zfs-zed diff --git a/etc/init.d/Makefile.am b/etc/init.d/Makefile.am index f93af1fd77e0..658623fda4aa 100644 --- a/etc/init.d/Makefile.am +++ b/etc/init.d/Makefile.am @@ -3,7 +3,7 @@ include $(top_srcdir)/config/Shellcheck.am EXTRA_DIST += README.md -init_SCRIPTS = zfs-import zfs-mount zfs-share zfs-zed +init_SCRIPTS = zfs-import zfs-load-key zfs-mount zfs-share zfs-zed SUBSTFILES += $(init_SCRIPTS) diff --git a/etc/init.d/README.md b/etc/init.d/README.md index c14b01937db2..f417b24c5923 100644 --- a/etc/init.d/README.md +++ b/etc/init.d/README.md @@ -42,14 +42,16 @@ INSTALLING INIT SCRIPT LINKS To setup the init script links in /etc/rc?.d manually on a Debian GNU/Linux (or derived) system, run the following commands (the order is important!): - update-rc.d zfs-import start 07 S . stop 07 0 1 6 . - update-rc.d zfs-mount start 02 2 3 4 5 . stop 06 0 1 6 . - update-rc.d zfs-zed start 07 2 3 4 5 . stop 08 0 1 6 . - update-rc.d zfs-share start 27 2 3 4 5 . stop 05 0 1 6 . + update-rc.d zfs-import start 07 S . stop 07 0 1 6 . + update-rc.d zfs-load-key start 02 2 3 4 5 . stop 06 0 1 6 . + update-rc.d zfs-mount start 02 2 3 4 5 . stop 06 0 1 6 . + update-rc.d zfs-zed start 07 2 3 4 5 . stop 08 0 1 6 . + update-rc.d zfs-share start 27 2 3 4 5 . stop 05 0 1 6 . To do the same on RedHat, Fedora and/or CentOS: chkconfig zfs-import + chkconfig zfs-load-key chkconfig zfs-mount chkconfig zfs-zed chkconfig zfs-share @@ -57,6 +59,7 @@ INSTALLING INIT SCRIPT LINKS On Gentoo: rc-update add zfs-import boot + rc-update add zfs-load-key boot rc-update add zfs-mount boot rc-update add zfs-zed default rc-update add zfs-share default diff --git a/etc/init.d/zfs-load-key.in b/etc/init.d/zfs-load-key.in new file mode 100755 index 000000000000..2f8deffdc809 --- /dev/null +++ b/etc/init.d/zfs-load-key.in @@ -0,0 +1,131 @@ +#!@DEFAULT_INIT_SHELL@ +# +# zfs-load-key This script will load/unload the zfs filesystems keys. +# +# chkconfig: 2345 06 99 +# description: This script will load or unload the zfs filesystems keys during +# system boot/shutdown. Only filesystems with key path set +# in keylocation property. See the zfs(8) man page for details. +# probe: true +# +### BEGIN INIT INFO +# Provides: zfs-load-key +# Required-Start: $local_fs zfs-import +# Required-Stop: $local_fs zfs-import +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# X-Start-Before: zfs-mount +# X-Stop-After: zfs-zed +# Short-Description: Load ZFS keys for filesystems and volumes +# Description: Run the `zfs load-key` or `zfs unload-key` commands. +### END INIT INFO +# +# Released under the 2-clause BSD license. +# +# This script is based on debian/zfsutils.zfs.init from the +# Debian GNU/kFreeBSD zfsutils 8.1-3 package, written by Aurelien Jarno. + +# Source the common init script +. @sysconfdir@/zfs/zfs-functions + +# ---------------------------------------------------- + +do_depend() +{ + # bootmisc will log to /var which may be a different zfs than root. + before bootmisc logger zfs-mount + + after zfs-import sysfs + keyword -lxc -openvz -prefix -vserver +} + +# Load keys for all datasets/filesystems +do_load_keys() +{ + zfs_log_begin_msg "Load ZFS filesystem(s) keys" + + "$ZFS" list -Ho name,encryptionroot,keystatus,keylocation | + while IFS=" " read -r name encryptionroot keystatus keylocation; do + if [ "$encryptionroot" != "-" ] && + [ "$name" = "$encryptionroot" ] && + [ "$keystatus" = "unavailable" ] && + [ "$keylocation" != "prompt" ] && + [ "$keylocation" != "none" ] + then + zfs_action "Load key for $encryptionroot" \ + "$ZFS" load-key "$encryptionroot" + fi + done + + zfs_log_end_msg 0 + + return 0 +} + +# Unload keys for all datasets/filesystems +do_unload_keys() +{ + zfs_log_begin_msg "Unload ZFS filesystem(s) key" + + "$ZFS" list -Ho name,encryptionroot,keystatus | sed '1!G;h;$!d' | + while IFS=" " read -r name encryptionroot keystatus; do + if [ "$encryptionroot" != "-" ] && + [ "$name" = "$encryptionroot" ] && + [ "$keystatus" = "available" ] + then + zfs_action "Unload key for $encryptionroot" \ + "$ZFS" unload-key "$encryptionroot" + fi + done + + zfs_log_end_msg 0 + + return 0 +} + +do_start() +{ + check_boolean "$ZFS_LOAD_KEY" || exit 0 + + check_module_loaded "zfs" || exit 0 + + do_load_keys +} + +do_stop() +{ + check_boolean "$ZFS_UNLOAD_KEY" || exit 0 + + check_module_loaded "zfs" || exit 0 + + do_unload_keys +} + +# ---------------------------------------------------- + +if [ ! -e /sbin/openrc-run ] +then + case "$1" in + start) + do_start + ;; + stop) + do_stop + ;; + force-reload|condrestart|reload|restart|status) + # no-op + ;; + *) + [ -n "$1" ] && echo "Error: Unknown command $1." + echo "Usage: $0 {start|stop}" + exit 3 + ;; + esac + + exit $? +else + # Create wrapper functions since Gentoo don't use the case part. + depend() { do_depend; } + start() { do_start; } + stop() { do_stop; } +fi diff --git a/etc/zfs/zfs-functions.in b/etc/zfs/zfs-functions.in index 2fb065afdb9a..10fb5b19a829 100644 --- a/etc/zfs/zfs-functions.in +++ b/etc/zfs/zfs-functions.in @@ -1,5 +1,5 @@ -# This is a script with common functions etc used by zfs-import, zfs-mount, -# zfs-share and zfs-zed. +# This is a script with common functions etc used by zfs-import, zfs-load-key, +# zfs-mount, zfs-share and zfs-zed. # # It is _NOT_ to be called independently # @@ -92,6 +92,8 @@ ZPOOL="@sbindir@/zpool" ZPOOL_CACHE="@sysconfdir@/zfs/zpool.cache" # Sensible defaults +ZFS_LOAD_KEY='yes' +ZFS_UNLOAD_KEY='no' ZFS_MOUNT='yes' ZFS_UNMOUNT='yes' ZFS_SHARE='yes' @@ -104,7 +106,8 @@ fi # ---------------------------------------------------- -export ZFS ZED ZPOOL ZPOOL_CACHE ZFS_MOUNT ZFS_UNMOUNT ZFS_SHARE ZFS_UNSHARE +export ZFS ZED ZPOOL ZPOOL_CACHE ZFS_LOAD_KEY ZFS_UNLOAD_KEY ZFS_MOUNT ZFS_UNMOUNT \ + ZFS_SHARE ZFS_UNSHARE zfs_action() { diff --git a/rpm/generic/zfs.spec.in b/rpm/generic/zfs.spec.in index 4a37ae8ce1d5..25beadce7c1f 100644 --- a/rpm/generic/zfs.spec.in +++ b/rpm/generic/zfs.spec.in @@ -424,6 +424,7 @@ fi %else if [ -x /sbin/chkconfig ]; then /sbin/chkconfig --add zfs-import + /sbin/chkconfig --add zfs-load-key /sbin/chkconfig --add zfs-mount /sbin/chkconfig --add zfs-share /sbin/chkconfig --add zfs-zed @@ -454,6 +455,7 @@ fi %else if [ "$1" = "0" -o "$1" = "remove" ] && [ -x /sbin/chkconfig ]; then /sbin/chkconfig --del zfs-import + /sbin/chkconfig --del zfs-load-key /sbin/chkconfig --del zfs-mount /sbin/chkconfig --del zfs-share /sbin/chkconfig --del zfs-zed From f471a0a0a7d8c303857828edbe997df2f0f3072c Mon Sep 17 00:00:00 2001 From: Georgy Yakovlev <168902+gyakovlev@users.noreply.github.com> Date: Thu, 16 Dec 2021 11:47:22 -0800 Subject: [PATCH 035/109] systemd: add weekly and monthly scrub timers Timers can be enabled as follows: systemctl enable zfs-scrub-weekly@rpool.timer --now systemctl enable zfs-scrub-monthly@datapool.timer --now Each timer will pull in zfs-scrub@${poolname}.service, which is not schedule-specific. Added PERIODIC SCRUB section to zpool-scrub.8. Reviewed-by: Richard Laager Reviewed-by: Brian Behlendorf Signed-off-by: Georgy Yakovlev Closes #12193 --- etc/systemd/system/.gitignore | 1 + etc/systemd/system/Makefile.am | 5 ++++- .../system/zfs-scrub-monthly@.timer.in | 12 +++++++++++ etc/systemd/system/zfs-scrub-weekly@.timer.in | 12 +++++++++++ etc/systemd/system/zfs-scrub@.service.in | 14 +++++++++++++ man/man8/zpool-scrub.8 | 21 +++++++++++++++++++ 6 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 etc/systemd/system/zfs-scrub-monthly@.timer.in create mode 100644 etc/systemd/system/zfs-scrub-weekly@.timer.in create mode 100644 etc/systemd/system/zfs-scrub@.service.in diff --git a/etc/systemd/system/.gitignore b/etc/systemd/system/.gitignore index efada54ad932..4813c65a25a8 100644 --- a/etc/systemd/system/.gitignore +++ b/etc/systemd/system/.gitignore @@ -1,3 +1,4 @@ *.service *.target *.preset +*.timer diff --git a/etc/systemd/system/Makefile.am b/etc/systemd/system/Makefile.am index c374a52ac7db..5e65e1db420c 100644 --- a/etc/systemd/system/Makefile.am +++ b/etc/systemd/system/Makefile.am @@ -12,7 +12,10 @@ systemdunit_DATA = \ zfs-volume-wait.service \ zfs-import.target \ zfs-volumes.target \ - zfs.target + zfs.target \ + zfs-scrub-monthly@.timer \ + zfs-scrub-weekly@.timer \ + zfs-scrub@.service SUBSTFILES += $(systemdpreset_DATA) $(systemdunit_DATA) diff --git a/etc/systemd/system/zfs-scrub-monthly@.timer.in b/etc/systemd/system/zfs-scrub-monthly@.timer.in new file mode 100644 index 000000000000..903068468278 --- /dev/null +++ b/etc/systemd/system/zfs-scrub-monthly@.timer.in @@ -0,0 +1,12 @@ +[Unit] +Description=Monthly zpool scrub timer for %i +Documentation=man:zpool-scrub(8) + +[Timer] +OnCalendar=monthly +Persistent=true +RandomizedDelaySec=1h +Unit=zfs-scrub@%i.service + +[Install] +WantedBy=timers.target diff --git a/etc/systemd/system/zfs-scrub-weekly@.timer.in b/etc/systemd/system/zfs-scrub-weekly@.timer.in new file mode 100644 index 000000000000..ede699500599 --- /dev/null +++ b/etc/systemd/system/zfs-scrub-weekly@.timer.in @@ -0,0 +1,12 @@ +[Unit] +Description=Weekly zpool scrub timer for %i +Documentation=man:zpool-scrub(8) + +[Timer] +OnCalendar=weekly +Persistent=true +RandomizedDelaySec=1h +Unit=zfs-scrub@%i.service + +[Install] +WantedBy=timers.target diff --git a/etc/systemd/system/zfs-scrub@.service.in b/etc/systemd/system/zfs-scrub@.service.in new file mode 100644 index 000000000000..bebe91d746ae --- /dev/null +++ b/etc/systemd/system/zfs-scrub@.service.in @@ -0,0 +1,14 @@ +[Unit] +Description=zpool scrub on %i +Documentation=man:zpool-scrub(8) +Requires=zfs.target +After=zfs.target +ConditionACPower=true +ConditionPathIsDirectory=/sys/module/zfs + +[Service] +ExecStart=/bin/sh -c '\ +if @sbindir@/zpool status %i | grep "scrub in progress"; then\ +exec @sbindir@/zpool wait -t scrub %i;\ +else exec @sbindir@/zpool scrub -w %i; fi' +ExecStop=-/bin/sh -c '@sbindir@/zpool scrub -p %i 2>/dev/null || true' diff --git a/man/man8/zpool-scrub.8 b/man/man8/zpool-scrub.8 index 768f71539290..69ae825b6158 100644 --- a/man/man8/zpool-scrub.8 +++ b/man/man8/zpool-scrub.8 @@ -116,8 +116,29 @@ scanned at 100M/s, and 68.4M of that file data has been scrubbed sequentially at 10.0M/s. .El .El +.Sh PERIODIC SCRUB +On machines using systemd, scrub timers can be enabled on per-pool basis. +.Nm weekly +and +.Nm monthly +timer units are provided. +.Bl -tag -width Ds +.It Xo +.Xc +.Nm systemctl +.Cm enable +.Cm zfs-scrub-\fIweekly\fB@\fIrpool\fB.timer +.Cm --now +.It Xo +.Xc +.Nm systemctl +.Cm enable +.Cm zfs-scrub-\fImonthly\fB@\fIotherpool\fB.timer +.Cm --now +.El . .Sh SEE ALSO +.Xr systemd.timer 5 , .Xr zpool-iostat 8 , .Xr zpool-resilver 8 , .Xr zpool-status 8 From 29e05d5345d456e4434a6ffc16483a0fc0ba0cbd Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Fri, 17 Dec 2021 12:37:21 -0800 Subject: [PATCH 036/109] ZTS: Fix zpool_reopen_[1-5] on Fedora 35 The zpool_reopen_[1-5] tests are failing Fedora 35 with: zpool_reopen_001_pos.ksh[64]: log_must[67]: log_pos[270]: wait_for_resilver_end[98]: wait_for_action: line 71: func: is read only Renaming 'func' -> 'funct' fixes the issue. Reviewed-by: George Melikov Reviewed-by: Brian Behlendorf Signed-off-by: Tony Hutter Closes #12871 --- .../tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib b/tests/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib index 075ad85e9f96..3d142fdf70ca 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib @@ -68,11 +68,11 @@ function wait_for_action #pool timeout function { typeset pool=$1 typeset -i timeout=$2 - typeset func=$3 + typeset funct=$3 while [ $timeout -gt 0 ]; do (( --timeout )) - if ( $func $pool ); then + if ( $funct $pool ); then return 0 fi sleep 1 From d4794c820486dd8475101e727d7da7d24da73472 Mon Sep 17 00:00:00 2001 From: Rich Ercolani <214141+rincebrain@users.noreply.github.com> Date: Fri, 17 Dec 2021 15:39:10 -0500 Subject: [PATCH 037/109] ZTS: Avoid piping send directly to /dev/null Unfortunately, #11445 means while we fail gracefully now, we still fail, unless people want to implement a complex workaround just to support /dev/null. So let's just use the cheap workaround in a test for now. Reviewed-by: Brian Behlendorf Reviewed-by: John Kennedy Signed-off-by: Rich Ercolani Closes #12872 --- tests/zfs-tests/tests/functional/history/history_006_neg.ksh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/zfs-tests/tests/functional/history/history_006_neg.ksh b/tests/zfs-tests/tests/functional/history/history_006_neg.ksh index 19b7114faf5b..c3a5e092d02d 100755 --- a/tests/zfs-tests/tests/functional/history/history_006_neg.ksh +++ b/tests/zfs-tests/tests/functional/history/history_006_neg.ksh @@ -75,7 +75,9 @@ if ! is_linux; then log_must zfs share $fs log_must zfs unshare $fs fi -log_must zfs send -i $snap1 $snap2 > /dev/null +# https://github.com/openzfs/zfs/issues/11445 +set -o pipefail +log_must zfs send -i $snap1 $snap2 | cat > /dev/null log_must zfs holds $snap1 log_must eval "zpool history $TESTPOOL > $NEW_HISTORY" From fe8b0a33d4ad903683404cfe2b1f38f3247fc0dd Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Fri, 17 Dec 2021 12:40:34 -0800 Subject: [PATCH 038/109] ZTS: alloc_class.ksh must wait for the process to exit The alloc_class_* tests may fail on Linux with an EBUSY error if `zfs destroy` is run before the `dd` process has had a chance to terminate. Wait on the pid after the `kill -9` to make sure. When testing I didn't observe any failures for the alloc_class tests. Remove them from the exceptions list, the CI was used to verify the tests pass on all platforms. Reviewed-by: John Kennedy Reviewed-by: Rich Ercolani Signed-off-by: Brian Behlendorf Closes #12873 --- tests/test-runner/bin/zts-report.py.in | 5 ----- .../tests/functional/alloc_class/alloc_class.kshlib | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index c3a679675280..210a08a4ec11 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -280,11 +280,6 @@ if sys.platform.startswith('freebsd'): }) elif sys.platform.startswith('linux'): maybe.update({ - 'alloc_class/alloc_class_009_pos': ['FAIL', known_reason], - 'alloc_class/alloc_class_010_pos': ['FAIL', known_reason], - 'alloc_class/alloc_class_011_neg': ['FAIL', known_reason], - 'alloc_class/alloc_class_012_pos': ['FAIL', known_reason], - 'alloc_class/alloc_class_013_pos': ['FAIL', '11888'], 'cli_root/zfs_rename/zfs_rename_002_pos': ['FAIL', known_reason], 'cli_root/zpool_expand/zpool_expand_001_pos': ['FAIL', known_reason], 'cli_root/zpool_expand/zpool_expand_005_pos': ['FAIL', known_reason], diff --git a/tests/zfs-tests/tests/functional/alloc_class/alloc_class.kshlib b/tests/zfs-tests/tests/functional/alloc_class/alloc_class.kshlib index 4c64cff69643..e204f43b3bcd 100644 --- a/tests/zfs-tests/tests/functional/alloc_class/alloc_class.kshlib +++ b/tests/zfs-tests/tests/functional/alloc_class/alloc_class.kshlib @@ -62,6 +62,7 @@ function display_status ((ret |= $?)) kill -9 $pid + wait $pid 2> /dev/null return $ret } From a35125e3d5a176075882aab2d4b657387cf064f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Salvador=20Rufo?= Date: Fri, 1 Oct 2021 19:44:34 +0200 Subject: [PATCH 039/109] Proper support for DESTDIR and INSTALL_MOD_PATH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The environment variables DESTDIR and INSTALL_MOD_PATH must be mutually exclusive. https://www.gnu.org/prep/standards/html_node/DESTDIR.html https://www.kernel.org/doc/Documentation/kbuild/modules.txt This issue was discussed in this Buildroot thread: https://lists.buildroot.org/pipermail/buildroot/2021-August/621350.html I saw this behavior in other different projects, as: - Yocto Project: https://www.yoctoproject.org/pipermail/meta-freescale/2013-August/004307.html - Google IA Coral: https://coral.googlesource.com/linux-imx-debian/+/refs/heads/master/debian/rules For the above reasons, INSTALL_MOD_PATH will be set as DESTDIR by default. Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf Signed-off-by: José Luis Salvador Rufo Signed-off-by: Romain Naour Closes #12577 --- module/Makefile.in | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/module/Makefile.in b/module/Makefile.in index 05c673231cc5..b15ab9109725 100644 --- a/module/Makefile.in +++ b/module/Makefile.in @@ -1,6 +1,7 @@ include Kbuild INSTALL_MOD_DIR ?= extra +INSTALL_MOD_PATH ?= $(DESTDIR) SUBDIR_TARGETS = icp lua zstd @@ -81,15 +82,15 @@ clean: clean-@ac_system@ modules_install-Linux: @# Install the kernel modules $(MAKE) -C @LINUX_OBJ@ M=`pwd` modules_install \ - INSTALL_MOD_PATH=$(DESTDIR)$(INSTALL_MOD_PATH) \ + INSTALL_MOD_PATH=$(INSTALL_MOD_PATH) \ INSTALL_MOD_DIR=$(INSTALL_MOD_DIR) \ KERNELRELEASE=@LINUX_VERSION@ @# Remove extraneous build products when packaging - kmoddir=$(DESTDIR)$(INSTALL_MOD_PATH)/lib/modules/@LINUX_VERSION@; \ + kmoddir=$(INSTALL_MOD_PATH)/lib/modules/@LINUX_VERSION@; \ if [ -n "$(DESTDIR)" ]; then \ find $$kmoddir -name 'modules.*' | xargs $(RM); \ fi - sysmap=$(DESTDIR)$(INSTALL_MOD_PATH)/boot/System.map-@LINUX_VERSION@; \ + sysmap=$(INSTALL_MOD_PATH)/boot/System.map-@LINUX_VERSION@; \ if [ -f $$sysmap ]; then \ depmod -ae -F $$sysmap @LINUX_VERSION@; \ fi @@ -102,7 +103,7 @@ modules_install: modules_install-@ac_system@ modules_uninstall-Linux: @# Uninstall the kernel modules - kmoddir=$(DESTDIR)$(INSTALL_MOD_PATH)/lib/modules/@LINUX_VERSION@; \ + kmoddir=$(INSTALL_MOD_PATH)/lib/modules/@LINUX_VERSION@; \ for objdir in $(ZFS_MODULES); do \ $(RM) -R $$kmoddir/$(INSTALL_MOD_DIR)/$$objdir; \ done From 2e3b3e3a2ed7a299c29ce0460823e865f6226c45 Mon Sep 17 00:00:00 2001 From: Rich Ercolani <214141+rincebrain@users.noreply.github.com> Date: Fri, 17 Dec 2021 15:43:13 -0500 Subject: [PATCH 040/109] Workaround Debian's fake System.map behavior MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Debian ships fake System.map files by default, leading to the invocation of depmod with them to flood you with errors about missing symbols. Let's notice and not do that. Reviewed-by: Ahelenia Ziemiańska Reviewed-by: Brian Behlendorf Signed-off-by: Rich Ercolani Closes #12862 --- module/Makefile.in | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/module/Makefile.in b/module/Makefile.in index b15ab9109725..b47ee73b87cd 100644 --- a/module/Makefile.in +++ b/module/Makefile.in @@ -90,7 +90,15 @@ modules_install-Linux: if [ -n "$(DESTDIR)" ]; then \ find $$kmoddir -name 'modules.*' | xargs $(RM); \ fi + @# Debian ships tiny fake System.map files that are + @# syntactically valid but just say + @# "if you want system.map go install this package" + @# Naturally, depmod is less than amused by this. + @# So if we find it missing or with one of these present, + @# we check for the alternate path for the System.map sysmap=$(INSTALL_MOD_PATH)/boot/System.map-@LINUX_VERSION@; \ + { [ -f "$$sysmap" ] && [ $$(wc -l < "$$sysmap") -ge 100 ]; } || \ + sysmap=$(INSTALL_MOD_PATH)/usr/lib/debug/boot/System.map-@LINUX_VERSION@; \ if [ -f $$sysmap ]; then \ depmod -ae -F $$sysmap @LINUX_VERSION@; \ fi From 687de107b7db267f8170e5cf5ff5d3ba461d2080 Mon Sep 17 00:00:00 2001 From: Rich Ercolani <214141+rincebrain@users.noreply.github.com> Date: Mon, 24 Jan 2022 18:54:52 -0500 Subject: [PATCH 041/109] Add explicit timeout to test step If we die from timeout of the whole GH action run, we don't run the collect step afterward, which can make it hard to investigate the timeout. If we timeout first in the test action, though, it qualifies as failure, and collects appropriately. (330 minutes seems like an acceptable tradeoff between the 6h timeout by default on the action and the 4h and change "functional" usually takes.) Reviewed-by: George Melikov Reviewed-by: Brian Behlendorf Signed-off-by: Rich Ercolani Closes #12999 --- .github/workflows/zfs-tests-functional.yml | 1 + .github/workflows/zfs-tests-sanity.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/zfs-tests-functional.yml b/.github/workflows/zfs-tests-functional.yml index cad763287ea9..53b5e890a872 100644 --- a/.github/workflows/zfs-tests-functional.yml +++ b/.github/workflows/zfs-tests-functional.yml @@ -64,6 +64,7 @@ jobs: - name: Tests run: | /usr/share/zfs/zfs-tests.sh -vR -s 3G + timeout-minutes: 330 - name: Prepare artifacts if: failure() run: | diff --git a/.github/workflows/zfs-tests-sanity.yml b/.github/workflows/zfs-tests-sanity.yml index 78187212bb26..6f7ee1347748 100644 --- a/.github/workflows/zfs-tests-sanity.yml +++ b/.github/workflows/zfs-tests-sanity.yml @@ -60,6 +60,7 @@ jobs: - name: Tests run: | /usr/share/zfs/zfs-tests.sh -vR -s 3G -r sanity + timeout-minutes: 330 - name: Prepare artifacts if: failure() run: | From 4d4f0d1a05f4a6f5d27b67b0876c19219b04d7a5 Mon Sep 17 00:00:00 2001 From: "D. Ebdrup" Date: Sat, 30 Oct 2021 01:30:44 +0200 Subject: [PATCH 042/109] zfsprops.7: Add note about comma-separation This change primarily seeks to make implicit documentation explicit, as it is not outright stated that options should be comma-separated, nor is there a reason given for it. Reviewed-by: Brian Behlendorf Signed-off-by: Daniel Ebdrup Jensen Closes #12579 --- man/man7/zfsprops.7 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/man/man7/zfsprops.7 b/man/man7/zfsprops.7 index 387e51b0b5ee..73e7ed972f81 100644 --- a/man/man7/zfsprops.7 +++ b/man/man7/zfsprops.7 @@ -1623,6 +1623,11 @@ If the property is set to the dataset is shared using the default options: .Dl sec=sys,rw,crossmnt,no_subtree_check .Pp +Please note that the options are comma-separated, unlike those found in +.Xr exports 5 . +This is done to negate the need for quoting, as well as to make parsing +with scripts easier. +.Pp See .Xr exports 5 for the meaning of the default options. From 9b185de6fa9f1b3a7614448fe0116ed370ec7e2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Fri, 30 Apr 2021 16:31:22 +0200 Subject: [PATCH 043/109] ZTS: cli_root/zfs_load-key: add separate key files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Issue: #11956 Closes #11976 --- .../zfs-tests/tests/functional/cli_root/zfs_load-key/HEXKEY | 1 + .../tests/functional/cli_root/zfs_load-key/Makefile.am | 5 ++++- .../tests/functional/cli_root/zfs_load-key/PASSPHRASE | 1 + .../zfs-tests/tests/functional/cli_root/zfs_load-key/RAWKEY | 1 + .../tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg | 3 +++ 5 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 tests/zfs-tests/tests/functional/cli_root/zfs_load-key/HEXKEY create mode 100644 tests/zfs-tests/tests/functional/cli_root/zfs_load-key/PASSPHRASE create mode 100644 tests/zfs-tests/tests/functional/cli_root/zfs_load-key/RAWKEY diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/HEXKEY b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/HEXKEY new file mode 100644 index 000000000000..95ed1c051a21 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/HEXKEY @@ -0,0 +1 @@ +000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile.am b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile.am index 06b4239a6d96..03c2916387ef 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile.am +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile.am @@ -11,4 +11,7 @@ dist_pkgdata_SCRIPTS = \ dist_pkgdata_DATA = \ zfs_load-key.cfg \ - zfs_load-key_common.kshlib + zfs_load-key_common.kshlib \ + PASSPHRASE \ + HEXKEY \ + RAWKEY diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/PASSPHRASE b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/PASSPHRASE new file mode 100644 index 000000000000..f3097ab13082 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/PASSPHRASE @@ -0,0 +1 @@ +password diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/RAWKEY b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/RAWKEY new file mode 100644 index 000000000000..f2d4cbf581ce --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/RAWKEY @@ -0,0 +1 @@ +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ No newline at end of file diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg index 90d9f63f1dba..2f01aac7c0a2 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg @@ -17,6 +17,9 @@ # Copyright (c) 2017 Datto, Inc. All rights reserved. # +# $PASSPHRASE, $HEXKEY, and $RAWKEY must be kept in sync +# with the corresponding files in this directory + export PASSPHRASE="password" export PASSPHRASE1="password1" export PASSPHRASE2="password2" From 9cbc2ed20f710326d16e8fe7357999eaa3f90142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Thu, 13 May 2021 06:21:35 +0200 Subject: [PATCH 044/109] libzfs: add keylocation=https://, backed by fetch(3) or libcurl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for http and https to the keylocation properly to allow encryption keys to be fetched from the specified URL. Reviewed-by: Brian Behlendorf Reviewed-by: Ryan Moeller Signed-off-by: Ahelenia Ziemiańska Issue #9543 Closes #9947 Closes #11956 --- .github/workflows/zfs-tests-functional.yml | 3 +- .github/workflows/zfs-tests-sanity.yml | 3 +- config/Substfiles.am | 4 +- config/user-libfetch.m4 | 71 +++++++ config/user.m4 | 1 + contrib/dracut/90zfs/module-setup.sh.in | 5 + .../dracut/90zfs/zfs-env-bootfs.service.in | 2 +- contrib/dracut/90zfs/zfs-load-key.sh.in | 7 +- .../90zfs/zfs-rollback-bootfs.service.in | 2 +- .../90zfs/zfs-snapshot-bootfs.service.in | 2 +- contrib/initramfs/hooks/zfs.in | 7 + contrib/initramfs/scripts/zfs | 11 +- include/libzfs_impl.h | 2 + lib/libzfs/Makefile.am | 2 +- lib/libzfs/libzfs.abi | 73 +++++-- lib/libzfs/libzfs_crypto.c | 186 ++++++++++++++++++ lib/libzfs/libzfs_util.c | 8 + man/man7/zfsprops.7 | 20 +- module/zcommon/zfs_prop.c | 6 +- tests/runfiles/common.run | 3 +- tests/runfiles/sanity.run | 3 +- .../cli_root/zfs_load-key/Makefile.am | 1 + .../cli_root/zfs_load-key/cleanup.ksh | 2 + .../cli_root/zfs_load-key/setup.ksh | 5 +- .../cli_root/zfs_load-key/zfs_load-key.cfg | 28 +++ .../zfs_load-key/zfs_load-key_all.ksh | 9 + .../zfs_load-key/zfs_load-key_common.kshlib | 63 ++++++ .../zfs_load-key/zfs_load-key_https.ksh | 78 ++++++++ .../zfs_load-key/zfs_load-key_location.ksh | 5 + .../zfs_load-key/zfs_load-key_recursive.ksh | 6 + .../cli_root/zfs_set/zfs_set_keylocation.ksh | 22 +-- 31 files changed, 585 insertions(+), 55 deletions(-) create mode 100644 config/user-libfetch.m4 create mode 100755 tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_https.ksh diff --git a/.github/workflows/zfs-tests-functional.yml b/.github/workflows/zfs-tests-functional.yml index 53b5e890a872..adcbcb15681a 100644 --- a/.github/workflows/zfs-tests-functional.yml +++ b/.github/workflows/zfs-tests-functional.yml @@ -26,7 +26,8 @@ jobs: xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \ libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \ libpam0g-dev pamtester python-dev python-setuptools python-cffi \ - python3 python3-dev python3-setuptools python3-cffi python3-packaging + python3 python3-dev python3-setuptools python3-cffi python3-packaging \ + libcurl4-openssl-dev - name: Autogen.sh run: | sh autogen.sh diff --git a/.github/workflows/zfs-tests-sanity.yml b/.github/workflows/zfs-tests-sanity.yml index 6f7ee1347748..c1e257dd1572 100644 --- a/.github/workflows/zfs-tests-sanity.yml +++ b/.github/workflows/zfs-tests-sanity.yml @@ -22,7 +22,8 @@ jobs: xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \ libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \ libpam0g-dev pamtester python-dev python-setuptools python-cffi \ - python3 python3-dev python3-setuptools python3-cffi python3-packaging + python3 python3-dev python3-setuptools python3-cffi python3-packaging \ + libcurl4-openssl-dev - name: Autogen.sh run: | sh autogen.sh diff --git a/config/Substfiles.am b/config/Substfiles.am index 63697bfa2b6a..911903e10e69 100644 --- a/config/Substfiles.am +++ b/config/Substfiles.am @@ -15,7 +15,9 @@ subst_sed_cmd = \ -e 's|@PYTHON[@]|$(PYTHON)|g' \ -e 's|@PYTHON_SHEBANG[@]|$(PYTHON_SHEBANG)|g' \ -e 's|@DEFAULT_INIT_NFS_SERVER[@]|$(DEFAULT_INIT_NFS_SERVER)|g' \ - -e 's|@DEFAULT_INIT_SHELL[@]|$(DEFAULT_INIT_SHELL)|g' + -e 's|@DEFAULT_INIT_SHELL[@]|$(DEFAULT_INIT_SHELL)|g' \ + -e 's|@LIBFETCH_DYNAMIC[@]|$(LIBFETCH_DYNAMIC)|g' \ + -e 's|@LIBFETCH_SONAME[@]|$(LIBFETCH_SONAME)|g' SUBSTFILES = CLEANFILES = $(SUBSTFILES) diff --git a/config/user-libfetch.m4 b/config/user-libfetch.m4 new file mode 100644 index 000000000000..f5149fc1a5d7 --- /dev/null +++ b/config/user-libfetch.m4 @@ -0,0 +1,71 @@ +dnl # +dnl # Check for a libfetch - either fetch(3) or libcurl. +dnl # +dnl # There are two configuration dimensions: +dnl # * fetch(3) vs libcurl +dnl # * static vs dynamic +dnl # +dnl # fetch(3) is only dynamic. +dnl # We use sover 6, which first appeared in FreeBSD 8.0-RELEASE. +dnl # +dnl # libcurl development packages include curl-config(1) – we want: +dnl # * HTTPS support +dnl # * version at least 7.16 (October 2006), for sover 4 +dnl # * to decide if it's static or not +dnl # +AC_DEFUN([ZFS_AC_CONFIG_USER_LIBFETCH], [ + AC_MSG_CHECKING([for libfetch]) + LIBFETCH_LIBS= + LIBFETCH_IS_FETCH=0 + LIBFETCH_IS_LIBCURL=0 + LIBFETCH_DYNAMIC=0 + LIBFETCH_SONAME= + have_libfetch= + + saved_libs="$LIBS" + LIBS="$LIBS -lfetch" + AC_LINK_IFELSE([AC_LANG_PROGRAM([[ + #include + #include + #include + ]], [fetchGetURL("", "");])], [ + have_libfetch=1 + LIBFETCH_IS_FETCH=1 + LIBFETCH_DYNAMIC=1 + LIBFETCH_SONAME='"libfetch.so.6"' + LIBFETCH_LIBS="-ldl" + AC_MSG_RESULT([fetch(3)]) + ], []) + LIBS="$saved_libs" + + if test -z "$have_libfetch"; then + if curl-config --protocols 2>/dev/null | grep -q HTTPS && + test "$(printf "%u" "0x$(curl-config --vernum)")" -ge "$(printf "%u" "0x071000")"; then + have_libfetch=1 + LIBFETCH_IS_LIBCURL=1 + if test "$(curl-config --built-shared)" = "yes"; then + LIBFETCH_DYNAMIC=1 + LIBFETCH_SONAME='"libcurl.so.4"' + LIBFETCH_LIBS="-ldl" + AC_MSG_RESULT([libcurl]) + else + LIBFETCH_LIBS="$(curl-config --libs)" + AC_MSG_RESULT([libcurl (static)]) + fi + + CCFLAGS="$CCFLAGS $(curl-config --cflags)" + fi + fi + + if test -z "$have_libfetch"; then + AC_MSG_RESULT([none]) + fi + + AC_SUBST([LIBFETCH_LIBS]) + AC_SUBST([LIBFETCH_DYNAMIC]) + AC_SUBST([LIBFETCH_SONAME]) + AC_DEFINE_UNQUOTED([LIBFETCH_IS_FETCH], [$LIBFETCH_IS_FETCH], [libfetch is fetch(3)]) + AC_DEFINE_UNQUOTED([LIBFETCH_IS_LIBCURL], [$LIBFETCH_IS_LIBCURL], [libfetch is libcurl]) + AC_DEFINE_UNQUOTED([LIBFETCH_DYNAMIC], [$LIBFETCH_DYNAMIC], [whether the chosen libfetch is to be loaded at run-time]) + AC_DEFINE_UNQUOTED([LIBFETCH_SONAME], [$LIBFETCH_SONAME], [soname of chosen libfetch]) +]) diff --git a/config/user.m4 b/config/user.m4 index e799faffb61c..670820b37715 100644 --- a/config/user.m4 +++ b/config/user.m4 @@ -22,6 +22,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER], [ ZFS_AC_CONFIG_USER_LIBCRYPTO ZFS_AC_CONFIG_USER_LIBAIO ZFS_AC_CONFIG_USER_LIBATOMIC + ZFS_AC_CONFIG_USER_LIBFETCH ZFS_AC_CONFIG_USER_CLOCK_GETTIME ZFS_AC_CONFIG_USER_PAM ZFS_AC_CONFIG_USER_RUNSTATEDIR diff --git a/contrib/dracut/90zfs/module-setup.sh.in b/contrib/dracut/90zfs/module-setup.sh.in index 213b48a25f5a..8c2951dd3523 100755 --- a/contrib/dracut/90zfs/module-setup.sh.in +++ b/contrib/dracut/90zfs/module-setup.sh.in @@ -60,6 +60,11 @@ install() { # Fallback: Guess the path and include all matches dracut_install /usr/lib*/gcc/**/libgcc_s.so* fi + if [ @LIBFETCH_DYNAMIC@ != 0 ]; then + for d in $libdirs; do + [ -e "$d"/@LIBFETCH_SONAME@ ] && dracut_install "$d"/@LIBFETCH_SONAME@ + done + fi dracut_install @mounthelperdir@/mount.zfs dracut_install @udevdir@/vdev_id dracut_install awk diff --git a/contrib/dracut/90zfs/zfs-env-bootfs.service.in b/contrib/dracut/90zfs/zfs-env-bootfs.service.in index 2bc43482c187..e143cb5ec1ed 100644 --- a/contrib/dracut/90zfs/zfs-env-bootfs.service.in +++ b/contrib/dracut/90zfs/zfs-env-bootfs.service.in @@ -8,7 +8,7 @@ Before=zfs-import.target [Service] Type=oneshot -ExecStart=/bin/sh -c "systemctl set-environment BOOTFS=$(@sbindir@/zpool list -H -o bootfs | grep -m1 -v '^-$')" +ExecStart=/bin/sh -c "exec systemctl set-environment BOOTFS=$(@sbindir@/zpool list -H -o bootfs | grep -m1 -v '^-$')" [Install] WantedBy=zfs-import.target diff --git a/contrib/dracut/90zfs/zfs-load-key.sh.in b/contrib/dracut/90zfs/zfs-load-key.sh.in index 5eb28fee3fe3..f15118ad02ca 100755 --- a/contrib/dracut/90zfs/zfs-load-key.sh.in +++ b/contrib/dracut/90zfs/zfs-load-key.sh.in @@ -43,13 +43,14 @@ if [ "$(zpool list -H -o feature@encryption "${BOOTFS%%/*}")" = 'active' ]; then KEYLOCATION="$(zfs get -H -o value keylocation "${ENCRYPTIONROOT}")" if ! [ "${KEYLOCATION}" = "prompt" ]; then + if ! [ "${KEYLOCATION#http}" = "${KEYLOCATION}" ]; then + systemctl start network-online.target + fi zfs load-key "${ENCRYPTIONROOT}" else # decrypt them - TRY_COUNT=5 - while [ $TRY_COUNT -gt 0 ]; do + for _ in 1 2 3 4 5; do systemd-ask-password "Encrypted ZFS password for ${BOOTFS}" --no-tty | zfs load-key "${ENCRYPTIONROOT}" && break - TRY_COUNT=$((TRY_COUNT - 1)) done fi fi diff --git a/contrib/dracut/90zfs/zfs-rollback-bootfs.service.in b/contrib/dracut/90zfs/zfs-rollback-bootfs.service.in index 0d45f71eadce..bdc246943208 100644 --- a/contrib/dracut/90zfs/zfs-rollback-bootfs.service.in +++ b/contrib/dracut/90zfs/zfs-rollback-bootfs.service.in @@ -10,5 +10,5 @@ ConditionKernelCommandLine=bootfs.rollback # ${BOOTFS} should have been set by zfs-env-bootfs.service Type=oneshot ExecStartPre=/bin/sh -c 'test -n "${BOOTFS}"' -ExecStart=/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.rollback)"; @sbindir@/zfs rollback -Rf "${BOOTFS}@${SNAPNAME:-%v}"' +ExecStart=/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.rollback)"; exec @sbindir@/zfs rollback -Rf "${BOOTFS}@${SNAPNAME:-%v}"' RemainAfterExit=yes diff --git a/contrib/dracut/90zfs/zfs-snapshot-bootfs.service.in b/contrib/dracut/90zfs/zfs-snapshot-bootfs.service.in index 11513ba27b01..6ea13850c3a7 100644 --- a/contrib/dracut/90zfs/zfs-snapshot-bootfs.service.in +++ b/contrib/dracut/90zfs/zfs-snapshot-bootfs.service.in @@ -10,5 +10,5 @@ ConditionKernelCommandLine=bootfs.snapshot # ${BOOTFS} should have been set by zfs-env-bootfs.service Type=oneshot ExecStartPre=/bin/sh -c 'test -n "${BOOTFS}"' -ExecStart=-/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.snapshot)"; @sbindir@/zfs snapshot "${BOOTFS}@${SNAPNAME:-%v}"' +ExecStart=-/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.snapshot)"; exec @sbindir@/zfs snapshot "${BOOTFS}@${SNAPNAME:-%v}"' RemainAfterExit=yes diff --git a/contrib/initramfs/hooks/zfs.in b/contrib/initramfs/hooks/zfs.in index 32331b264da8..bdf169fd96c2 100755 --- a/contrib/initramfs/hooks/zfs.in +++ b/contrib/initramfs/hooks/zfs.in @@ -30,6 +30,13 @@ find /lib/ -type f -name "libgcc_s.so.[1-9]" | while read -r libgcc; do copy_exec "$libgcc" done +# shellcheck disable=SC2050 +if [ @LIBFETCH_DYNAMIC@ != 0 ]; then + find /lib/ -name @LIBFETCH_SONAME@ | while read -r libfetch; do + copy_exec "$libfetch" + done +fi + copy_file config "/etc/hostid" copy_file cache "@sysconfdir@/zfs/zpool.cache" copy_file config "@initconfdir@/zfs" diff --git a/contrib/initramfs/scripts/zfs b/contrib/initramfs/scripts/zfs index 82eceaedb56e..306e6e157e62 100644 --- a/contrib/initramfs/scripts/zfs +++ b/contrib/initramfs/scripts/zfs @@ -403,28 +403,25 @@ decrypt_fs() KEYSTATUS="$(get_fs_value "${ENCRYPTIONROOT}" keystatus)" # Continue only if the key needs to be loaded [ "$KEYSTATUS" = "unavailable" ] || return 0 - TRY_COUNT=3 - # If key is stored in a file, do not prompt + # Do not prompt if key is stored noninteractively, if ! [ "${KEYLOCATION}" = "prompt" ]; then $ZFS load-key "${ENCRYPTIONROOT}" # Prompt with plymouth, if active - elif [ -e /bin/plymouth ] && /bin/plymouth --ping 2>/dev/null; then + elif /bin/plymouth --ping 2>/dev/null; then echo "plymouth" > /run/zfs_console_askpwd_cmd - while [ $TRY_COUNT -gt 0 ]; do + for _ in 1 2 3; do plymouth ask-for-password --prompt "Encrypted ZFS password for ${ENCRYPTIONROOT}" | \ $ZFS load-key "${ENCRYPTIONROOT}" && break - TRY_COUNT=$((TRY_COUNT - 1)) done # Prompt with systemd, if active elif [ -e /run/systemd/system ]; then echo "systemd-ask-password" > /run/zfs_console_askpwd_cmd - while [ $TRY_COUNT -gt 0 ]; do + for _ in 1 2 3; do systemd-ask-password "Encrypted ZFS password for ${ENCRYPTIONROOT}" --no-tty | \ $ZFS load-key "${ENCRYPTIONROOT}" && break - TRY_COUNT=$((TRY_COUNT - 1)) done # Prompt with ZFS tty, otherwise diff --git a/include/libzfs_impl.h b/include/libzfs_impl.h index dfb63285c1fd..96b11dad137c 100644 --- a/include/libzfs_impl.h +++ b/include/libzfs_impl.h @@ -72,6 +72,8 @@ struct libzfs_handle { boolean_t libzfs_prop_debug; regex_t libzfs_urire; uint64_t libzfs_max_nvlist; + void *libfetch; + char *libfetch_load_error; }; struct zfs_handle { diff --git a/lib/libzfs/Makefile.am b/lib/libzfs/Makefile.am index 1a7698b4760e..31267fd9a5e9 100644 --- a/lib/libzfs/Makefile.am +++ b/lib/libzfs/Makefile.am @@ -75,7 +75,7 @@ libzfs_la_LIBADD = \ $(abs_top_builddir)/lib/libnvpair/libnvpair.la \ $(abs_top_builddir)/lib/libuutil/libuutil.la -libzfs_la_LIBADD += -lm $(LIBCRYPTO_LIBS) $(ZLIB_LIBS) $(LTLIBINTL) +libzfs_la_LIBADD += -lm $(LIBCRYPTO_LIBS) $(ZLIB_LIBS) $(LIBFETCH_LIBS) $(LTLIBINTL) libzfs_la_LDFLAGS = -pthread diff --git a/lib/libzfs/libzfs.abi b/lib/libzfs/libzfs.abi index 82f8b7dc87f8..14e03ee28ffe 100644 --- a/lib/libzfs/libzfs.abi +++ b/lib/libzfs/libzfs.abi @@ -551,10 +551,6 @@ - - - - @@ -1373,7 +1369,7 @@ - + @@ -1434,6 +1430,12 @@ + + + + + + @@ -3190,6 +3192,19 @@ + + + + + + + + + + + + + @@ -3231,6 +3246,11 @@ + + + + + @@ -3243,6 +3263,12 @@ + + + + + + @@ -3262,6 +3288,10 @@ + + + + @@ -3285,6 +3315,10 @@ + + + + @@ -3317,6 +3351,10 @@ + + + + @@ -4425,12 +4463,6 @@ - - - - - - @@ -4452,10 +4484,6 @@ - - - - @@ -4620,11 +4648,6 @@ - - - - - @@ -6705,6 +6728,12 @@ + + + + + + @@ -7235,6 +7264,10 @@ + + + + diff --git a/lib/libzfs/libzfs_crypto.c b/lib/libzfs/libzfs_crypto.c index e31d4ce44bfb..644dd26859f1 100644 --- a/lib/libzfs/libzfs_crypto.c +++ b/lib/libzfs/libzfs_crypto.c @@ -26,6 +26,16 @@ #include #include #include +#if LIBFETCH_DYNAMIC +#include +#endif +#if LIBFETCH_IS_FETCH +#include +#include +#include +#elif LIBFETCH_IS_LIBCURL +#include +#endif #include #include "libzfs_impl.h" #include "zfeature_common.h" @@ -59,9 +69,13 @@ static int caught_interrupt; static int get_key_material_file(libzfs_handle_t *, const char *, const char *, zfs_keyformat_t, boolean_t, uint8_t **, size_t *); +static int get_key_material_https(libzfs_handle_t *, const char *, const char *, + zfs_keyformat_t, boolean_t, uint8_t **, size_t *); static zfs_uri_handler_t uri_handlers[] = { { "file", get_key_material_file }, + { "https", get_key_material_https }, + { "http", get_key_material_https }, { NULL, NULL } }; @@ -483,6 +497,178 @@ get_key_material_file(libzfs_handle_t *hdl, const char *uri, return (ret); } +static int +get_key_material_https(libzfs_handle_t *hdl, const char *uri, + const char *fsname, zfs_keyformat_t keyformat, boolean_t newkey, + uint8_t **restrict buf, size_t *restrict len_out) +{ + int ret = 0; + FILE *key = NULL; + boolean_t is_http = strncmp(uri, "http:", strlen("http:")) == 0; + + if (strlen(uri) < (is_http ? 7 : 8)) { + ret = EINVAL; + goto end; + } + +#if LIBFETCH_DYNAMIC +#define LOAD_FUNCTION(func) \ + __typeof__(func) *func = dlsym(hdl->libfetch, #func); + + if (hdl->libfetch == NULL) + hdl->libfetch = dlopen(LIBFETCH_SONAME, RTLD_LAZY); + + if (hdl->libfetch == NULL) { + hdl->libfetch = (void *)-1; + char *err = dlerror(); + if (err) + hdl->libfetch_load_error = strdup(err); + } + + if (hdl->libfetch == (void *)-1) { + ret = ENOSYS; + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "Couldn't load %s: %s"), + LIBFETCH_SONAME, hdl->libfetch_load_error ?: "(?)"); + goto end; + } + + boolean_t ok; +#if LIBFETCH_IS_FETCH + LOAD_FUNCTION(fetchGetURL); + char *fetchLastErrString = dlsym(hdl->libfetch, "fetchLastErrString"); + + ok = fetchGetURL && fetchLastErrString; +#elif LIBFETCH_IS_LIBCURL + LOAD_FUNCTION(curl_easy_init); + LOAD_FUNCTION(curl_easy_setopt); + LOAD_FUNCTION(curl_easy_perform); + LOAD_FUNCTION(curl_easy_cleanup); + LOAD_FUNCTION(curl_easy_strerror); + LOAD_FUNCTION(curl_easy_getinfo); + + ok = curl_easy_init && curl_easy_setopt && curl_easy_perform && + curl_easy_cleanup && curl_easy_strerror && curl_easy_getinfo; +#endif + if (!ok) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "keylocation=%s back-end %s missing symbols."), + is_http ? "http://" : "https://", LIBFETCH_SONAME); + ret = ENOSYS; + goto end; + } +#endif + +#if LIBFETCH_IS_FETCH + key = fetchGetURL(uri, ""); + if (key == NULL) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "Couldn't GET %s: %s"), + uri, fetchLastErrString); + ret = ENETDOWN; + } +#elif LIBFETCH_IS_LIBCURL + CURL *curl = curl_easy_init(); + if (curl == NULL) { + ret = ENOTSUP; + goto end; + } + + int kfd = -1; +#ifdef O_TMPFILE + kfd = open(getenv("TMPDIR") ?: "/tmp", + O_RDWR | O_TMPFILE | O_EXCL | O_CLOEXEC, 0600); + if (kfd != -1) + goto kfdok; +#endif + + char *path; + if (asprintf(&path, + "%s/libzfs-XXXXXXXX.https", getenv("TMPDIR") ?: "/tmp") == -1) { + ret = ENOMEM; + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "%s"), + strerror(ret)); + goto end; + } + + kfd = mkostemps(path, strlen(".https"), O_CLOEXEC); + if (kfd == -1) { + ret = errno; + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "Couldn't create temporary file %s: %s"), + path, strerror(ret)); + free(path); + goto end; + } + (void) unlink(path); + free(path); + +kfdok: + if ((key = fdopen(kfd, "r+")) == NULL) { + ret = errno; + free(path); + (void) close(kfd); + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "Couldn't reopen temporary file: %s"), strerror(ret)); + goto end; + } + + char errbuf[CURL_ERROR_SIZE] = ""; + char *cainfo = getenv("SSL_CA_CERT_FILE"); /* matches fetch(3) */ + char *capath = getenv("SSL_CA_CERT_PATH"); /* matches fetch(3) */ + char *clcert = getenv("SSL_CLIENT_CERT_FILE"); /* matches fetch(3) */ + char *clkey = getenv("SSL_CLIENT_KEY_FILE"); /* matches fetch(3) */ + (void) curl_easy_setopt(curl, CURLOPT_URL, uri); + (void) curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + (void) curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, 30000L); + (void) curl_easy_setopt(curl, CURLOPT_WRITEDATA, key); + (void) curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); + if (cainfo != NULL) + (void) curl_easy_setopt(curl, CURLOPT_CAINFO, cainfo); + if (capath != NULL) + (void) curl_easy_setopt(curl, CURLOPT_CAPATH, capath); + if (clcert != NULL) + (void) curl_easy_setopt(curl, CURLOPT_SSLCERT, clcert); + if (clkey != NULL) + (void) curl_easy_setopt(curl, CURLOPT_SSLKEY, clkey); + + CURLcode res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "Failed to connect to %s: %s"), + uri, strlen(errbuf) ? errbuf : curl_easy_strerror(res)); + ret = ENETDOWN; + } else { + long resp = 200; + (void) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp); + + if (resp < 200 || resp >= 300) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "Couldn't GET %s: %ld"), + uri, resp); + ret = ENOENT; + } else + rewind(key); + } + + curl_easy_cleanup(curl); +#else + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "No keylocation=%s back-end."), is_http ? "http://" : "https://"); + ret = ENOSYS; +#endif + +end: + if (ret == 0) + ret = get_key_material_raw(key, keyformat, buf, len_out); + + if (key != NULL) + fclose(key); + + return (ret); +} + /* * Attempts to fetch key material, no matter where it might live. The key * material is allocated and returned in km_out. *can_retry_out will be set diff --git a/lib/libzfs/libzfs_util.c b/lib/libzfs/libzfs_util.c index 6e57d8e42563..7dd38bb3d838 100644 --- a/lib/libzfs/libzfs_util.c +++ b/lib/libzfs/libzfs_util.c @@ -44,6 +44,9 @@ #include #include #include +#if LIBFETCH_DYNAMIC +#include +#endif #include #include #include @@ -1101,6 +1104,11 @@ libzfs_fini(libzfs_handle_t *hdl) libzfs_core_fini(); regfree(&hdl->libzfs_urire); fletcher_4_fini(); +#if LIBFETCH_DYNAMIC + if (hdl->libfetch != (void *)-1 && hdl->libfetch != NULL) + (void) dlclose(hdl->libfetch); + free(hdl->libfetch_load_error); +#endif free(hdl); } diff --git a/man/man7/zfsprops.7 b/man/man7/zfsprops.7 index 73e7ed972f81..8b79603f4945 100644 --- a/man/man7/zfsprops.7 +++ b/man/man7/zfsprops.7 @@ -1099,7 +1099,7 @@ Even though the encryption suite cannot be changed after dataset creation, the keyformat can be with .Nm zfs Cm change-key . .It Xo -.Sy keylocation Ns = Ns Sy prompt Ns | Ns Sy file:// Ns Em +.Sy keylocation Ns = Ns Sy prompt Ns | Ns Sy file:// Ns Em Ns | Ns Sy https:// Ns Em
| Ns Sy http:// Ns Em
.Xc Controls where the user's encryption key will be loaded from by default for commands such as @@ -1126,6 +1126,24 @@ but users should be careful not to place keys which should be kept secret on the command line. If a file URI is selected, the key will be loaded from the specified absolute file path. +If an HTTPS or HTTP URL is selected, it will be GETted using +.Xr fetch 3 , +libcurl, or nothing, depending on compile-time configuration and run-time +availability. +The +.Ev SSL_CA_CERT_FILE +environment variable can be set to set the location +of the concatenated certificate store. +The +.Ev SSL_CA_CERT_PATH +environment variable can be set to override the location +of the directory containing the certificate authority bundle. +The +.Ev SSL_CLIENT_CERT_FILE +and +.Ev SSL_CLIENT_KEY_FILE +environment variables can be set to configure the path +to the client certificate and its key. .It Sy pbkdf2iters Ns = Ns Ar iterations Controls the number of PBKDF2 iterations that a .Sy passphrase diff --git a/module/zcommon/zfs_prop.c b/module/zcommon/zfs_prop.c index 402d749c1aeb..d17321990809 100644 --- a/module/zcommon/zfs_prop.c +++ b/module/zcommon/zfs_prop.c @@ -583,7 +583,7 @@ zfs_prop_init(void) "ENCROOT"); zprop_register_string(ZFS_PROP_KEYLOCATION, "keylocation", "none", PROP_DEFAULT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, - "prompt | ", "KEYLOCATION"); + "prompt | | | ", "KEYLOCATION"); zprop_register_string(ZFS_PROP_REDACT_SNAPS, "redact_snaps", NULL, PROP_READONLY, ZFS_TYPE_DATASET | ZFS_TYPE_BOOKMARK, "[,...]", @@ -936,6 +936,10 @@ zfs_prop_valid_keylocation(const char *str, boolean_t encrypted) return (B_TRUE); else if (strlen(str) > 8 && strncmp("file:///", str, 8) == 0) return (B_TRUE); + else if (strlen(str) > 8 && strncmp("https://", str, 8) == 0) + return (B_TRUE); + else if (strlen(str) > 7 && strncmp("http://", str, 7) == 0) + return (B_TRUE); return (B_FALSE); } diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index bc32dfc68346..d487b7f895b3 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -198,7 +198,8 @@ tags = ['functional', 'cli_root', 'zfs_inherit'] [tests/functional/cli_root/zfs_load-key] tests = ['zfs_load-key', 'zfs_load-key_all', 'zfs_load-key_file', - 'zfs_load-key_location', 'zfs_load-key_noop', 'zfs_load-key_recursive'] + 'zfs_load-key_https', 'zfs_load-key_location', 'zfs_load-key_noop', + 'zfs_load-key_recursive'] tags = ['functional', 'cli_root', 'zfs_load-key'] [tests/functional/cli_root/zfs_mount] diff --git a/tests/runfiles/sanity.run b/tests/runfiles/sanity.run index 3f4eb302f1e0..fb39fa54b9d5 100644 --- a/tests/runfiles/sanity.run +++ b/tests/runfiles/sanity.run @@ -146,7 +146,8 @@ tags = ['functional', 'cli_root', 'zfs_inherit'] [tests/functional/cli_root/zfs_load-key] tests = ['zfs_load-key', 'zfs_load-key_all', 'zfs_load-key_file', - 'zfs_load-key_location', 'zfs_load-key_noop', 'zfs_load-key_recursive'] + 'zfs_load-key_https', 'zfs_load-key_location', 'zfs_load-key_noop', + 'zfs_load-key_recursive'] tags = ['functional', 'cli_root', 'zfs_load-key'] [tests/functional/cli_root/zfs_mount] diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile.am b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile.am index 03c2916387ef..7dfec435ce7f 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile.am +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile.am @@ -5,6 +5,7 @@ dist_pkgdata_SCRIPTS = \ zfs_load-key.ksh \ zfs_load-key_all.ksh \ zfs_load-key_file.ksh \ + zfs_load-key_https.ksh \ zfs_load-key_location.ksh \ zfs_load-key_noop.ksh \ zfs_load-key_recursive.ksh diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/cleanup.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/cleanup.ksh index 79cd6e9f908e..d397bcf4e9f0 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/cleanup.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/cleanup.ksh @@ -26,5 +26,7 @@ # . $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib +cleanup_https default_cleanup diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/setup.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/setup.ksh index 6a9af3bc28c3..6cc5528ce5d7 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/setup.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/setup.ksh @@ -26,7 +26,10 @@ # . $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib DISK=${DISKS%% *} -default_setup $DISK +default_setup_noexit $DISK +setup_https +log_pass diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg index 2f01aac7c0a2..cc1e3b330543 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg @@ -27,3 +27,31 @@ export HEXKEY="000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F" export HEXKEY1="201F1E1D1C1B1A191817161514131211100F0E0D0C0B0A090807060504030201" export RAWKEY="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" export RAWKEY1="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +export SSL_CA_CERT_FILE="/$TESTPOOL/snakeoil.crt" +export HTTPS_PORT_FILE="/$TESTPOOL/snakeoil.port" +export HTTPS_HOSTNAME="localhost" +export HTTPS_PORT= +export HTTPS_BASE_URL= + +function get_https_port +{ + if [ -z "$HTTPS_PORT" ]; then + read -r HTTPS_PORT < "$HTTPS_PORT_FILE" || return + fi + + echo "$HTTPS_PORT" +} + +function get_https_base_url +{ + if [ -z "$HTTPS_BASE_URL" ]; then + HTTPS_BASE_URL="https://$HTTPS_HOSTNAME:$(get_https_port)" || { + typeset ret=$? + HTTPS_BASE_URL= + return $ret + } + fi + + echo "$HTTPS_BASE_URL" +} diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_all.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_all.ksh index 5e330eb0deb9..3c18e4538d34 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_all.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_all.ksh @@ -38,6 +38,7 @@ verify_runnable "both" function cleanup { datasetexists $TESTPOOL/$TESTFS1 && destroy_dataset $TESTPOOL/$TESTFS1 + datasetexists $TESTPOOL/$TESTFS2 && destroy_dataset $TESTPOOL/$TESTFS2 datasetexists $TESTPOOL/zvol && destroy_dataset $TESTPOOL/zvol poolexists $TESTPOOL1 && log_must destroy_pool $TESTPOOL1 } @@ -49,6 +50,9 @@ log_must eval "echo $PASSPHRASE1 > /$TESTPOOL/pkey" log_must zfs create -o encryption=on -o keyformat=passphrase \ -o keylocation=file:///$TESTPOOL/pkey $TESTPOOL/$TESTFS1 +log_must zfs create -o encryption=on -o keyformat=passphrase \ + -o keylocation=$(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS2 + log_must zfs create -V 64M -o encryption=on -o keyformat=passphrase \ -o keylocation=file:///$TESTPOOL/pkey $TESTPOOL/zvol @@ -59,6 +63,9 @@ log_must zpool create -O encryption=on -O keyformat=passphrase \ log_must zfs unmount $TESTPOOL/$TESTFS1 log_must_busy zfs unload-key $TESTPOOL/$TESTFS1 +log_must zfs unmount $TESTPOOL/$TESTFS2 +log_must_busy zfs unload-key $TESTPOOL/$TESTFS2 + log_must_busy zfs unload-key $TESTPOOL/zvol log_must zfs unmount $TESTPOOL1 @@ -69,8 +76,10 @@ log_must zfs load-key -a log_must key_available $TESTPOOL1 log_must key_available $TESTPOOL/zvol log_must key_available $TESTPOOL/$TESTFS1 +log_must key_available $TESTPOOL/$TESTFS2 log_must zfs mount $TESTPOOL1 log_must zfs mount $TESTPOOL/$TESTFS1 +log_must zfs mount $TESTPOOL/$TESTFS2 log_pass "'zfs load-key -a' loads keys for all datasets" diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib index d9066f9cbf57..f7461437c615 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib @@ -99,3 +99,66 @@ function verify_origin return 0 } + +function setup_https +{ + log_must openssl req -x509 -newkey rsa:4096 -sha256 -days 1 -nodes -keyout "/$TESTPOOL/snakeoil.key" -out "$SSL_CA_CERT_FILE" -subj "/CN=$HTTPS_HOSTNAME" + + python3 -uc " +import http.server, ssl, sys, os, time, random + +sys.stdin.close() + +httpd, err, port = None, None, None +for i in range(1, 100): + port = random.randint(0xC000, 0xFFFF) # ephemeral range + try: + httpd = http.server.HTTPServer(('$HTTPS_HOSTNAME', port), http.server.SimpleHTTPRequestHandler) + break + except: + err = sys.exc_info()[1] + time.sleep(i / 100) +if not httpd: + raise err + +with open('$HTTPS_PORT_FILE', 'w') as portf: + print(port, file=portf) + +httpd.socket = ssl.wrap_socket(httpd.socket, server_side=True, keyfile='/$TESTPOOL/snakeoil.key', certfile='$SSL_CA_CERT_FILE', ssl_version=ssl.PROTOCOL_TLS) + +os.chdir('$STF_SUITE/tests/functional/cli_root/zfs_load-key') + +with open('/$TESTPOOL/snakeoil.pid', 'w') as pidf: + if os.fork() != 0: + os._exit(0) + print(os.getpid(), file=pidf) + +sys.stdout.close() +sys.stderr.close() +try: + sys.stdout = sys.stderr = open('/tmp/ZTS-snakeoil.log', 'w', buffering=1) # line +except: + sys.stdout = sys.stderr = open('/dev/null', 'w') + +print('{} start on {}'.format(os.getpid(), port)) +httpd.serve_forever() +" || log_fail + + typeset https_pid= + for d in $(seq 0 0.1 5); do + read -r https_pid 2>/dev/null < "/$TESTPOOL/snakeoil.pid" && [ -n "$https_pid" ] && break + sleep "$d" + done + [ -z "$https_pid" ] && log_fail "Couldn't start HTTPS server" + log_note "Started HTTPS server as $https_pid on port $(get_https_port)" +} + +function cleanup_https +{ + typeset https_pid= + read -r https_pid 2>/dev/null < "/$TESTPOOL/snakeoil.pid" || return 0 + + log_must kill "$https_pid" + cat /tmp/ZTS-snakeoil.log + rm -f "/$TESTPOOL/snakeoil.pid" "/tmp/ZTS-snakeoil.log" +} diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_https.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_https.ksh new file mode 100755 index 000000000000..cac9c4140322 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_https.ksh @@ -0,0 +1,78 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib + +# +# DESCRIPTION: +# 'zfs load-key' should load a dataset's key from an https:// URL, +# but fail to do so if the domain doesn't exist or the file 404s. +# +# STRATEGY: +# 1. Try to create a dataset pointing to an RFC6761-guaranteed unresolvable domain, +# one to the sshd port (which will be either unoccupied (ECONNREFUSED) +# or have sshd on it ("wrong version number")). +# and one pointing to an URL that will always 404. +# 2. Create encrypted datasets with keylocation=https://address +# 3. Unmount the datasets and unload their keys +# 4. Attempt to load the keys +# 5. Verify the keys are loaded +# 6. Attempt to mount the datasets +# + +verify_runnable "both" + +function cleanup +{ + for fs in "$TESTFS1" "$TESTFS2" "$TESTFS3"; do + datasetexists $TESTPOOL/$fs && \ + log_must zfs destroy $TESTPOOL/$fs + done +} +log_onexit cleanup + +log_assert "'zfs load-key' should load a key from a file" + +log_mustnot zfs create -o encryption=on -o keyformat=passphrase \ + -o keylocation=https://invalid./where-ever $TESTPOOL/$TESTFS1 + +log_mustnot zfs create -o encryption=on -o keyformat=passphrase \ + -o keylocation=https://$HTTPS_HOSTNAME:22 $TESTPOOL/$TESTFS1 + +log_mustnot zfs create -o encryption=on -o keyformat=passphrase \ + -o keylocation=$(get_https_base_url)/ENOENT $TESTPOOL/$TESTFS1 + +log_must zfs create -o encryption=on -o keyformat=passphrase \ + -o keylocation=$(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS1 + +log_must zfs create -o encryption=on -o keyformat=hex \ + -o keylocation=$(get_https_base_url)/HEXKEY $TESTPOOL/$TESTFS2 + +log_must zfs create -o encryption=on -o keyformat=raw \ + -o keylocation=$(get_https_base_url)/RAWKEY $TESTPOOL/$TESTFS3 + +for fs in "$TESTFS1" "$TESTFS2" "$TESTFS3"; do + log_must zfs unmount $TESTPOOL/$fs + log_must zfs unload-key $TESTPOOL/$fs +done +for fs in "$TESTFS1" "$TESTFS2" "$TESTFS3"; do + log_must zfs load-key $TESTPOOL/$fs + log_must key_available $TESTPOOL/$fs + log_must zfs mount $TESTPOOL/$fs +done + +log_pass "'zfs load-key' loads a key from a file" diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_location.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_location.ksh index 8538143cb62e..11f16e45ad3f 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_location.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_location.ksh @@ -70,4 +70,9 @@ log_must eval "echo $PASSPHRASE | zfs load-key -L prompt $TESTPOOL/$TESTFS1" log_must key_available $TESTPOOL/$TESTFS1 log_must verify_keylocation $TESTPOOL/$TESTFS1 "file://$key_location" +log_must zfs unload-key $TESTPOOL/$TESTFS1 +log_must zfs load-key -L $(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS1 +log_must key_available $TESTPOOL/$TESTFS1 +log_must verify_keylocation $TESTPOOL/$TESTFS1 "file://$key_location" + log_pass "'zfs load-key -L' overrides keylocation with provided value" diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_recursive.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_recursive.ksh index 54c390f2737f..c0b5553e39c9 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_recursive.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_recursive.ksh @@ -52,15 +52,21 @@ log_must zfs create -o encryption=on -o keyformat=passphrase \ log_must zfs create -o keyformat=passphrase \ -o keylocation=file:///$TESTPOOL/pkey $TESTPOOL/$TESTFS1/child +log_must zfs create -o keyformat=passphrase \ + -o keylocation=$(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS1/child/child + log_must zfs unmount $TESTPOOL/$TESTFS1 +log_must zfs unload-key $TESTPOOL/$TESTFS1/child/child log_must zfs unload-key $TESTPOOL/$TESTFS1/child log_must zfs unload-key $TESTPOOL/$TESTFS1 log_must zfs load-key -r $TESTPOOL log_must key_available $TESTPOOL/$TESTFS1 log_must key_available $TESTPOOL/$TESTFS1/child +log_must key_available $TESTPOOL/$TESTFS1/child/child log_must zfs mount $TESTPOOL/$TESTFS1 log_must zfs mount $TESTPOOL/$TESTFS1/child +log_must zfs mount $TESTPOOL/$TESTFS1/child/child log_pass "'zfs load-key -r' recursively loads keys" diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_set/zfs_set_keylocation.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_set/zfs_set_keylocation.ksh index d8a2fcbdf9b3..9791339479d7 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_set/zfs_set_keylocation.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_set/zfs_set_keylocation.ksh @@ -50,8 +50,8 @@ function cleanup } log_onexit cleanup -log_assert "Key location can only be 'prompt' or a file path for encryption" \ - "roots, and 'none' for unencrypted volumes" +log_assert "Key location can only be 'prompt', 'file://', or 'https://'" \ + "for encryption roots, and 'none' for unencrypted volumes" log_must eval "echo $PASSPHRASE > /$TESTPOOL/pkey" @@ -65,19 +65,15 @@ log_must zfs create -o encryption=on -o keyformat=passphrase \ -o keylocation=file:///$TESTPOOL/pkey $TESTPOOL/$TESTFS1 log_mustnot zfs set keylocation=none $TESTPOOL/$TESTFS1 -if true; then - log_mustnot zfs set keylocation=/$TESTPOOL/pkey $TESTPOOL/$TESTFS1 -else - ### SOON: ### - # file:///$TESTPOOL/pkey and /$TESTPOOL/pkey are equivalent on FreeBSD - # thanks to libfetch. Eventually we want to make the other platforms - # work this way as well, either by porting libfetch or by other means. - log_must zfs set keylocation=/$TESTPOOL/pkey $TESTPOOL/$TESTFS1 -fi +log_mustnot zfs set keylocation=/$TESTPOOL/pkey $TESTPOOL/$TESTFS1 log_must zfs set keylocation=file:///$TESTPOOL/pkey $TESTPOOL/$TESTFS1 log_must verify_keylocation $TESTPOOL/$TESTFS1 "file:///$TESTPOOL/pkey" +setup_https +log_must zfs set keylocation=$(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS1 +log_must verify_keylocation $TESTPOOL/$TESTFS1 "$(get_https_base_url)/PASSPHRASE" + log_must zfs set keylocation=prompt $TESTPOOL/$TESTFS1 log_must verify_keylocation $TESTPOOL/$TESTFS1 "prompt" @@ -98,5 +94,5 @@ log_mustnot zfs set keylocation=/$TESTPOOL/pkey $TESTPOOL/$TESTFS1/child log_must verify_keylocation $TESTPOOL/$TESTFS1/child "none" -log_pass "Key location can only be 'prompt' or a file path for encryption" \ - "roots, and 'none' for unencrypted volumes" +log_pass "Key location can only be 'prompt', 'file://', or 'https://'" \ + "for encryption roots, and 'none' for unencrypted volumes" From f9baf968b8199095489d9bffb8f31549af8bf9d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Sun, 23 May 2021 22:09:35 +0200 Subject: [PATCH 045/109] dracut: 90zfs: zfs-load-key: wait for key to appear for up to 10 seconds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also reduce password retries to 3 to match i-t Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #12065 Closes #12108 --- contrib/dracut/90zfs/zfs-load-key.sh.in | 36 ++++++++++++++++++------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/contrib/dracut/90zfs/zfs-load-key.sh.in b/contrib/dracut/90zfs/zfs-load-key.sh.in index f15118ad02ca..2138ff943c64 100755 --- a/contrib/dracut/90zfs/zfs-load-key.sh.in +++ b/contrib/dracut/90zfs/zfs-load-key.sh.in @@ -42,16 +42,32 @@ if [ "$(zpool list -H -o feature@encryption "${BOOTFS%%/*}")" = 'active' ]; then [ "$KEYSTATUS" = "unavailable" ] || exit 0 KEYLOCATION="$(zfs get -H -o value keylocation "${ENCRYPTIONROOT}")" - if ! [ "${KEYLOCATION}" = "prompt" ]; then - if ! [ "${KEYLOCATION#http}" = "${KEYLOCATION}" ]; then + case "${KEYLOCATION%%://*}" in + prompt) + for _ in 1 2 3; do + systemd-ask-password "Encrypted ZFS password for ${BOOTFS}" --no-tty | zfs load-key "${ENCRYPTIONROOT}" && break + done + ;; + http*) systemctl start network-online.target - fi - zfs load-key "${ENCRYPTIONROOT}" - else - # decrypt them - for _ in 1 2 3 4 5; do - systemd-ask-password "Encrypted ZFS password for ${BOOTFS}" --no-tty | zfs load-key "${ENCRYPTIONROOT}" && break - done - fi + zfs load-key "${ENCRYPTIONROOT}" + ;; + file) + KEYFILE="${KEYLOCATION#file://}" + [ -r "${KEYFILE}" ] || udevadm settle + [ -r "${KEYFILE}" ] || { + info "Waiting for key ${KEYFILE} for ${ENCRYPTIONROOT}..." + for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do + sleep 0.5s + [ -r "${KEYFILE}" ] && break + done + } + [ -r "${KEYFILE}" ] || warn "Key ${KEYFILE} for ${ENCRYPTIONROOT} hasn't appeared. Trying anyway." + zfs load-key "${ENCRYPTIONROOT}" + ;; + *) + zfs load-key "${ENCRYPTIONROOT}" + ;; + esac fi fi From 19a4bf445f415f3574ffdd1c7e511017e5c295b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Fri, 17 Dec 2021 21:44:23 +0100 Subject: [PATCH 046/109] contrib/initrd: systemd-ask-password --no-tty before argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In systemd 249 (sid), sd-a-p processes its arguments in getopt + mode, so "systemd-ask-password zupa --no-tty" prompts for "zupa --no-tty", not "zupa" not on the tty, as expected (bullseye, 247). Ref: https://github.com/systemd/systemd/commit/4b1c842d95bfd6ab352ade1a4655f9e512f35185 Ref: https://github.com/systemd/systemd/pull/19806 Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #12870 --- contrib/dracut/90zfs/zfs-load-key.sh.in | 2 +- contrib/initramfs/scripts/zfs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/dracut/90zfs/zfs-load-key.sh.in b/contrib/dracut/90zfs/zfs-load-key.sh.in index 2138ff943c64..c974b3d9ec4c 100755 --- a/contrib/dracut/90zfs/zfs-load-key.sh.in +++ b/contrib/dracut/90zfs/zfs-load-key.sh.in @@ -45,7 +45,7 @@ if [ "$(zpool list -H -o feature@encryption "${BOOTFS%%/*}")" = 'active' ]; then case "${KEYLOCATION%%://*}" in prompt) for _ in 1 2 3; do - systemd-ask-password "Encrypted ZFS password for ${BOOTFS}" --no-tty | zfs load-key "${ENCRYPTIONROOT}" && break + systemd-ask-password --no-tty "Encrypted ZFS password for ${BOOTFS}" | zfs load-key "${ENCRYPTIONROOT}" && break done ;; http*) diff --git a/contrib/initramfs/scripts/zfs b/contrib/initramfs/scripts/zfs index 306e6e157e62..6e39f139d845 100644 --- a/contrib/initramfs/scripts/zfs +++ b/contrib/initramfs/scripts/zfs @@ -420,7 +420,7 @@ decrypt_fs() elif [ -e /run/systemd/system ]; then echo "systemd-ask-password" > /run/zfs_console_askpwd_cmd for _ in 1 2 3; do - systemd-ask-password "Encrypted ZFS password for ${ENCRYPTIONROOT}" --no-tty | \ + systemd-ask-password --no-tty "Encrypted ZFS password for ${ENCRYPTIONROOT}" | \ $ZFS load-key "${ENCRYPTIONROOT}" && break done From efbed102f0c7510f13674b52d4ede5fb0ef71cea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Mon, 17 May 2021 18:09:46 +0200 Subject: [PATCH 047/109] zfs-share.8: document -l flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Description stolen from zfs-mount.8 Reviewed-by: Don Brady Reviewed-by: Brian Behlendorf Reviewed-by: John Kennedy Signed-off-by: Ahelenia Ziemiańska Closes #12067 --- man/man8/zfs-share.8 | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/man/man8/zfs-share.8 b/man/man8/zfs-share.8 index e30d538814ca..89121ead0958 100644 --- a/man/man8/zfs-share.8 +++ b/man/man8/zfs-share.8 @@ -29,7 +29,7 @@ .\" Copyright 2018 Nexenta Systems, Inc. .\" Copyright 2019 Joyent, Inc. .\" -.Dd June 30, 2019 +.Dd May 17, 2021 .Dt ZFS-SHARE 8 .Os . @@ -39,6 +39,7 @@ .Sh SYNOPSIS .Nm zfs .Cm share +.Op Fl l .Fl a Ns | Ns Ar filesystem .Nm zfs .Cm unshare @@ -49,10 +50,19 @@ .It Xo .Nm zfs .Cm share +.Op Fl l .Fl a Ns | Ns Ar filesystem .Xc Shares available ZFS file systems. .Bl -tag -width "-a" +.It Fl l +Load keys for encrypted filesystems as they are being mounted. +This is equivalent to executing +.Nm zfs Cm load-key +on each encryption root before mounting it. +Note that if a filesystem has +.Sy keylocation Ns = Ns Sy prompt , +this will cause the terminal to interactively block after asking for the key. .It Fl a Share all available ZFS file systems. Invoked automatically as part of the boot process. From be01ee8629dc3387509b2cb2d1ff8a56fc815655 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Fri, 17 Dec 2021 23:20:10 +0000 Subject: [PATCH 048/109] ZTS: rsend_007_pos failures The rsend_007_pos test reliably fails on Linux in the cleanup function. This is caused by an unmount error when attempting to recursively destroy the newly received datasets. Invoking `df` prior to the `zfs destroy` interestingly avoids the unmont error. Why this should matter is unclear and should be investigated. However, this minor tweak may allow us to remove the ZTS rsend exceptions. The subsequent rsend_010_pos and rsend_011_pos failures were a result of this initial failure. The other "maybe" failures I was unable to reproduce and have not been recently observed in the master branch. Reviewed-by: Tony Nguyen Reviewed-by: Ryan Moeller Signed-off-by: Brian Behlendorf Closes #5665 Closes #6086 Closes #6087 Closes #6446 Closes #12876 --- tests/test-runner/bin/zts-report.py.in | 9 --------- tests/zfs-tests/tests/functional/rsend/rsend.kshlib | 1 + 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index 210a08a4ec11..328a4048889f 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -242,12 +242,6 @@ maybe = { 'removal/removal_condense_export': ['FAIL', known_reason], 'reservation/reservation_008_pos': ['FAIL', '7741'], 'reservation/reservation_018_pos': ['FAIL', '5642'], - 'rsend/rsend_019_pos': ['FAIL', '6086'], - 'rsend/rsend_020_pos': ['FAIL', '6446'], - 'rsend/rsend_021_pos': ['FAIL', '6446'], - 'rsend/rsend_024_pos': ['FAIL', '5665'], - 'rsend/send-c_volume': ['FAIL', '6087'], - 'rsend/send_partial_dataset': ['FAIL', known_reason], 'snapshot/clone_001_pos': ['FAIL', known_reason], 'snapshot/snapshot_009_pos': ['FAIL', '7961'], 'snapshot/snapshot_010_pos': ['FAIL', '7961'], @@ -292,9 +286,6 @@ elif sys.platform.startswith('linux'): 'mmp/mmp_exported_import': ['FAIL', known_reason], 'mmp/mmp_inactive_import': ['FAIL', known_reason], 'refreserv/refreserv_raidz': ['FAIL', known_reason], - 'rsend/rsend_007_pos': ['FAIL', known_reason], - 'rsend/rsend_010_pos': ['FAIL', known_reason], - 'rsend/rsend_011_pos': ['FAIL', known_reason], 'snapshot/rollback_003_pos': ['FAIL', known_reason], }) diff --git a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib index d06bd39b4d49..8cd35e4afdfe 100644 --- a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib +++ b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib @@ -121,6 +121,7 @@ function cleanup_pool log_must rm -rf $BACKDIR/* if is_global_zone ; then + log_must df >/dev/null log_must_busy zfs destroy -Rf $pool else typeset list=$(zfs list -H -r -t all -o name $pool) From 1fb5566a254ee42df097f0aab6c9973673a98a3b Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Fri, 17 Dec 2021 23:29:23 +0000 Subject: [PATCH 049/109] ZTS: speed up rsend tests With some minor tweaks several of rsend tests can be sped up considerably without significantly reducing test coverage. * send-c_verify_ratio: ~120s -> ~60s * send_realloc_*_files: ~330s -> ~65s For the send_realloc* tests this also has the advantage of removing (most of) the linux/freebsd conditional logic. Note that for this test more passes, and thus more incremental send/recvs, are preferable to a larger number of files. Total run time of the rsend test group was reduced from roughly 20 to 11 minutes in an environment similar to what's used by the CI. Reviewed-by: Tony Nguyen Reviewed-by: Ryan Moeller Signed-off-by: Brian Behlendorf Closes #12876 --- .../tests/functional/rsend/send-c_verify_ratio.ksh | 2 +- .../tests/functional/rsend/send_realloc_encrypted_files.ksh | 6 +----- .../zfs-tests/tests/functional/rsend/send_realloc_files.ksh | 6 +----- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh index b7d978624f2b..845349a95873 100755 --- a/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh +++ b/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh @@ -36,7 +36,7 @@ log_assert "Verify send -c streams are compressed" log_onexit cleanup_pool $POOL2 typeset sendfs=$POOL2/$FS -typeset megs=128 +typeset megs=64 for prop in "${compress_prop_vals[@]}"; do for compressible in 'yes' 'no'; do diff --git a/tests/zfs-tests/tests/functional/rsend/send_realloc_encrypted_files.ksh b/tests/zfs-tests/tests/functional/rsend/send_realloc_encrypted_files.ksh index a653f8b3f15e..361f6b375ea1 100755 --- a/tests/zfs-tests/tests/functional/rsend/send_realloc_encrypted_files.ksh +++ b/tests/zfs-tests/tests/functional/rsend/send_realloc_encrypted_files.ksh @@ -72,12 +72,8 @@ if is_kmemleak; then # to avoid timeout due to reduced performance. nr_files=100 passes=2 -elif is_freebsd; then - # Use fewer files and passes on FreeBSD to avoid timeout. - nr_files=500 - passes=2 else - nr_files=1000 + nr_files=300 passes=3 fi diff --git a/tests/zfs-tests/tests/functional/rsend/send_realloc_files.ksh b/tests/zfs-tests/tests/functional/rsend/send_realloc_files.ksh index 083a2bec9daa..187a899a23c6 100755 --- a/tests/zfs-tests/tests/functional/rsend/send_realloc_files.ksh +++ b/tests/zfs-tests/tests/functional/rsend/send_realloc_files.ksh @@ -64,12 +64,8 @@ if is_kmemleak; then # to avoid timeout due to reduced performance. nr_files=100 passes=2 -elif is_freebsd; then - # Use fewer passes and files on FreeBSD to avoid timeout. - nr_files=500 - passes=2 else - nr_files=1000 + nr_files=300 passes=3 fi From f22ebf8fa666a4fbecfde4edb3a6ccd0a00db387 Mon Sep 17 00:00:00 2001 From: Georgy Yakovlev <168902+gyakovlev@users.noreply.github.com> Date: Tue, 21 Dec 2021 16:44:18 -0800 Subject: [PATCH 050/109] zfs-test/mmap_seek: fix build on musl The build on musl needs linux/fs.h for SEEK_DATA and friends, and sys/sysmacros.h for P2ROUNDUP. Add the needed headers. Reviewed-by: Brian Behlendorf Signed-off-by: Georgy Yakovlev Closes #12891 --- tests/zfs-tests/cmd/mmap_seek/mmap_seek.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/zfs-tests/cmd/mmap_seek/mmap_seek.c b/tests/zfs-tests/cmd/mmap_seek/mmap_seek.c index f476e1dba9a4..bb36527aafee 100644 --- a/tests/zfs-tests/cmd/mmap_seek/mmap_seek.c +++ b/tests/zfs-tests/cmd/mmap_seek/mmap_seek.c @@ -29,7 +29,11 @@ #include #include #include +#include #include +#ifdef __linux__ +#include +#endif static void seek_data(int fd, off_t offset, off_t expected) From 74bba85423f1b1dc20da289af2e39094c35919ea Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 22 Dec 2021 09:37:27 -0800 Subject: [PATCH 051/109] ZTS: Fix refreserv_raidz.ksh The rerefreserv_raidz test was failing on Linux because the sync being issued doesn't guarantee a pool sync. Switch to using the sync_pool function and remove the ZTS exception for Linux. Reviewed-by: George Melikov Reviewed-by: John Kennedy Signed-off-by: Brian Behlendorf Closes #12897 --- tests/test-runner/bin/zts-report.py.in | 1 - tests/zfs-tests/tests/functional/refreserv/refreserv_raidz.ksh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index 328a4048889f..3fab375e27fb 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -285,7 +285,6 @@ elif sys.platform.startswith('linux'): 'mmp/mmp_active_import': ['FAIL', known_reason], 'mmp/mmp_exported_import': ['FAIL', known_reason], 'mmp/mmp_inactive_import': ['FAIL', known_reason], - 'refreserv/refreserv_raidz': ['FAIL', known_reason], 'snapshot/rollback_003_pos': ['FAIL', known_reason], }) diff --git a/tests/zfs-tests/tests/functional/refreserv/refreserv_raidz.ksh b/tests/zfs-tests/tests/functional/refreserv/refreserv_raidz.ksh index 22891ef1d513..056c7916d990 100755 --- a/tests/zfs-tests/tests/functional/refreserv/refreserv_raidz.ksh +++ b/tests/zfs-tests/tests/functional/refreserv/refreserv_raidz.ksh @@ -110,7 +110,7 @@ for parity in 1 2 3; do block_device_wait "/dev/zvol/$vol" log_must dd if=/dev/zero of=/dev/zvol/$vol \ bs=1024k count=$volsize - sync + sync_pool $TESTPOOL ref=$(zfs get -Hpo value referenced "$vol") refres=$(zfs get -Hpo value refreservation "$vol") From ed064ed59640892c462d1126e2883a068d71243d Mon Sep 17 00:00:00 2001 From: Ka Ho Ng Date: Sun, 22 Aug 2021 23:22:07 +0800 Subject: [PATCH 052/109] ZTS: Enable punch-hole tests on FreeBSD Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Reviewed-by: Ryan Moeller Signed-off-by: Ka Ho Ng Sponsored-by: The FreeBSD Foundation Closes #12458 --- tests/runfiles/common.run | 4 +++ tests/runfiles/linux.run | 2 +- tests/test-runner/bin/zts-report.py.in | 8 ++++++ tests/zfs-tests/include/libtest.shlib | 20 ++++++++++++++ .../fallocate/fallocate_punch-hole.ksh | 27 ++++++++++++++----- 5 files changed, 53 insertions(+), 8 deletions(-) diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index d487b7f895b3..f4d8eb59f088 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -603,6 +603,10 @@ tags = ['functional', 'delegate'] tests = ['exec_001_pos', 'exec_002_neg'] tags = ['functional', 'exec'] +[tests/functional/fallocate] +tests = ['fallocate_punch-hole'] +tags = ['functional', 'fallocate'] + [tests/functional/features/async_destroy] tests = ['async_destroy_001_pos'] tags = ['functional', 'features', 'async_destroy'] diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 642ed824d462..b7831c3acfd1 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -94,7 +94,7 @@ tests = ['events_001_pos', 'events_002_pos', 'zed_rc_filter', 'zed_fd_spill'] tags = ['functional', 'events'] [tests/functional/fallocate:Linux] -tests = ['fallocate_prealloc', 'fallocate_punch-hole'] +tests = ['fallocate_prealloc'] tags = ['functional', 'fallocate'] [tests/functional/fault:Linux] diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index 3fab375e27fb..60c19fabc74f 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -127,6 +127,13 @@ fio_reason = 'Fio v2.3 or newer required' # trim_reason = 'DISKS must support discard (TRIM/UNMAP)' +# +# Some tests on FreeBSD require the fspacectl(2) system call and the +# truncate(1) utility supporting the -d option. The system call was first +# introduced in FreeBSD version 1400032. +# +fspacectl_reason = 'fspacectl(2) and truncate -d support required' + # # Some tests are not applicable to a platform or need to be updated to operate # in the manor required by the platform. Any tests which are skipped for this @@ -225,6 +232,7 @@ maybe = { 'cli_root/zpool_trim/setup': ['SKIP', trim_reason], 'cli_root/zpool_upgrade/zpool_upgrade_004_pos': ['FAIL', '6141'], 'delegate/setup': ['SKIP', exec_reason], + 'fallocate/fallocate_punch-hole': ['SKIP', fspacectl_reason], 'history/history_004_pos': ['FAIL', '7026'], 'history/history_005_neg': ['FAIL', '6680'], 'history/history_006_neg': ['FAIL', '5657'], diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib index 9391c050b776..773d134e3a2c 100644 --- a/tests/zfs-tests/include/libtest.shlib +++ b/tests/zfs-tests/include/libtest.shlib @@ -27,6 +27,7 @@ # Copyright (c) 2017, Lawrence Livermore National Security LLC. # Copyright (c) 2017, Datto Inc. All rights reserved. # Copyright (c) 2017, Open-E Inc. All rights reserved. +# Copyright (c) 2021, The FreeBSD Foundation. # Use is subject to license terms. # @@ -4218,6 +4219,25 @@ function get_arcstat # stat esac } +function punch_hole # offset length file +{ + typeset offset=$1 + typeset length=$2 + typeset file=$3 + + case $(uname) in + FreeBSD) + truncate -d -o $offset -l $length "$file" + ;; + Linux) + fallocate --punch-hole --offset $offset --length $length "$file" + ;; + *) + false + ;; + esac +} + # # Wait for the specified arcstat to reach non-zero quiescence. # If echo is 1 echo the value after reaching quiescence, otherwise diff --git a/tests/zfs-tests/tests/functional/fallocate/fallocate_punch-hole.ksh b/tests/zfs-tests/tests/functional/fallocate/fallocate_punch-hole.ksh index 6a8faa48704d..ed83561bd556 100755 --- a/tests/zfs-tests/tests/functional/fallocate/fallocate_punch-hole.ksh +++ b/tests/zfs-tests/tests/functional/fallocate/fallocate_punch-hole.ksh @@ -22,13 +22,14 @@ # # Copyright (c) 2020 by Lawrence Livermore National Security, LLC. +# Copyright (c) 2021 by The FreeBSD Foundation. # . $STF_SUITE/include/libtest.shlib # # DESCRIPTION: -# Test `fallocate --punch-hole` +# Test hole-punching functionality # # STRATEGY: # 1. Create a dense file @@ -37,6 +38,20 @@ verify_runnable "global" +# +# Prior to __FreeBSD_version 1400032 there are no mechanism to punch hole in a +# file on FreeBSD. truncate -d support is required to call fspacectl(2) on +# behalf of the script. +# +if is_freebsd; then + if [[ $(uname -K) -lt 1400032 ]]; then + log_unsupported "Requires fspacectl(2) support on FreeBSD" + fi + if truncate -d 2>&1 | grep "illegal option" > /dev/null; then + log_unsupported "Requires truncate(1) -d support on FreeBSD" + fi +fi + FILE=$TESTDIR/$TESTFILE0 BLKSZ=$(get_prop recordsize $TESTPOOL) @@ -74,23 +89,21 @@ log_must file_write -o create -f $FILE -b $BLKSZ -c 8 log_must check_disk_size $((131072 * 8)) # Punch a hole for the first full block. -log_must fallocate --punch-hole --offset 0 --length $BLKSZ $FILE +log_must punch_hole 0 $BLKSZ $FILE log_must check_disk_size $((131072 * 7)) # Partially punch a hole in the second block. -log_must fallocate --punch-hole --offset $BLKSZ --length $((BLKSZ / 2)) $FILE +log_must punch_hole $BLKSZ $((BLKSZ / 2)) $FILE log_must check_disk_size $((131072 * 7)) # Punch a hole which overlaps the third and forth block. -log_must fallocate --punch-hole --offset $(((BLKSZ * 2) + (BLKSZ / 2))) \ - --length $((BLKSZ)) $FILE +log_must punch_hole $(((BLKSZ * 2) + (BLKSZ / 2))) $((BLKSZ)) $FILE log_must check_disk_size $((131072 * 7)) # Punch a hole from the fifth block past the end of file. The apparent # file size should not change since --keep-size is implied. apparent_size=$(stat_size $FILE) -log_must fallocate --punch-hole --offset $((BLKSZ * 4)) \ - --length $((BLKSZ * 10)) $FILE +log_must punch_hole $((BLKSZ * 4)) $((BLKSZ * 10)) $FILE log_must check_disk_size $((131072 * 4)) log_must check_apparent_size $apparent_size From fc3230a78116490f7a18e4110b374b0ccad5a198 Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Fri, 1 Oct 2021 11:36:02 -0400 Subject: [PATCH 053/109] ZTS: Minimize udev_wait in zvol_misc tests The zvol_misc tests, in particular zvol_misc_volmode, make use of a common udev_wait function to wait for zvol devices in /dev to quiesce on Linux. On other platforms this function currently only sleeps for one second before returning. This is insufficient, and zvol_misc_volmode has been flaky on FreeBSD as a result. Replace udev_wait with block_device_wait, passing through the optional device parameter where possible. Rearrange a few checks to strengthen the verifications we are making and avoid unnecessarily sleeping. We must keep udev_wait in a couple places to pass in Github CI workflows. Remove zvol_misc_volmode from the maybe failing tests on FreeBSD in zts-report.py. Reviewed-by: Brian Behlendorf Reviewed-by: John Kennedy Signed-off-by: Ryan Moeller Closes #12583 --- tests/test-runner/bin/zts-report.py.in | 1 - tests/zfs-tests/include/blkdev.shlib | 2 +- .../zvol/zvol_misc/zvol_misc_common.kshlib | 11 ++--- .../zvol/zvol_misc/zvol_misc_rename_inuse.ksh | 4 +- .../zvol/zvol_misc/zvol_misc_snapdev.ksh | 2 +- .../zvol/zvol_misc/zvol_misc_volmode.ksh | 46 ++++++++++--------- .../zvol/zvol_misc/zvol_misc_zil.ksh | 4 +- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index 60c19fabc74f..c4f01b8718fe 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -278,7 +278,6 @@ if sys.platform.startswith('freebsd'): 'delegate/zfs_allow_003_pos': ['FAIL', known_reason], 'inheritance/inherit_001_pos': ['FAIL', '11829'], 'resilver/resilver_restart_001': ['FAIL', known_reason], - 'zvol/zvol_misc/zvol_misc_volmode': ['FAIL', known_reason], }) elif sys.platform.startswith('linux'): maybe.update({ diff --git a/tests/zfs-tests/include/blkdev.shlib b/tests/zfs-tests/include/blkdev.shlib index bcba8ee759c9..bf70952904bb 100644 --- a/tests/zfs-tests/include/blkdev.shlib +++ b/tests/zfs-tests/include/blkdev.shlib @@ -73,7 +73,7 @@ function scan_scsi_hosts function block_device_wait { if is_linux; then - udevadm trigger $* + udevadm trigger $* 2>/dev/null typeset start=$SECONDS udevadm settle typeset elapsed=$((SECONDS - start)) diff --git a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib index 3ee09a151b12..b69d2ce02913 100644 --- a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib +++ b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib @@ -37,8 +37,6 @@ # function udev_wait { - sleep 1 - is_linux || return 0 udevadm trigger --action=change udevadm settle for i in {1..3}; do @@ -58,7 +56,6 @@ function udev_wait # function udev_cleanup { - is_linux || return 0 log_note "Pruning broken ZVOL symlinks ..." udevadm settle @@ -79,7 +76,8 @@ function blockdev_exists # device # because there are other commands (zfs snap, zfs inherit, zfs destroy) # that can affect device nodes for i in {1..3}; do - udev_wait + is_linux && udev_wait + block_device_wait "$device" is_disk_device "$device" && return 0 done log_fail "$device does not exist as a block device" @@ -96,8 +94,9 @@ function blockdev_missing # device # because there are other commands (zfs snap, zfs inherit, zfs destroy) # that can affect device nodes for i in {1..3}; do - udev_wait - [[ ! -e "$device" ]] && return 0 + is_linux && udev_wait + block_device_wait + is_disk_device "$device" || return 0 done log_fail "$device exists when not expected" } diff --git a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_rename_inuse.ksh b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_rename_inuse.ksh index c4ea23c48d16..e9b7bbc4c4a5 100755 --- a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_rename_inuse.ksh +++ b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_rename_inuse.ksh @@ -39,7 +39,7 @@ function cleanup for ds in "$SENDFS" "$ZVOL" "$ZVOL-renamed"; do destroy_dataset "$ds" '-rf' done - udev_wait + block_device_wait } log_assert "Verify 'zfs rename' works on a ZVOL already in use as block device" @@ -54,7 +54,7 @@ SENDFS="$TESTPOOL/sendfs.$$" log_must zfs create -V $VOLSIZE "$ZVOL" # 2. Create a filesystem on the ZVOL device and mount it -udev_wait +block_device_wait "$ZDEV" log_must eval "new_fs $ZDEV >/dev/null 2>&1" log_must mkdir "$MNTPFS" log_must mount "$ZDEV" "$MNTPFS" diff --git a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_snapdev.ksh b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_snapdev.ksh index 1fd87aefdffc..9d3fb05da4b0 100755 --- a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_snapdev.ksh +++ b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_snapdev.ksh @@ -47,7 +47,7 @@ function cleanup datasetexists $ZVOL && destroy_dataset $ZVOL -r log_must zfs inherit snapdev $TESTPOOL block_device_wait - udev_cleanup + is_linux && udev_cleanup } log_assert "Verify that ZFS volume property 'snapdev' works as expected." diff --git a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode.ksh b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode.ksh index 322a31e07d89..264ec131ad6b 100755 --- a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode.ksh +++ b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode.ksh @@ -44,19 +44,18 @@ # 8. Verify "volmode" behaves accordingly to zvol_inhibit_dev (Linux only) # # NOTE: changing volmode may need to remove minors, which could be open, so call -# udev_wait() before we "zfs set volmode=". +# block_device_wait() before we "zfs set volmode=". verify_runnable "global" function cleanup { - datasetexists $VOLFS && log_must_busy zfs destroy -r $VOLFS - datasetexists $ZVOL && log_must_busy zfs destroy -r $ZVOL - log_must zfs inherit volmode $TESTPOOL - udev_wait + datasetexists $VOLFS && destroy_dataset $VOLFS -r + datasetexists $ZVOL && destroy_dataset $ZVOL -r + zfs inherit volmode $TESTPOOL sysctl_inhibit_dev 0 sysctl_volmode 1 - udev_cleanup + is_linux && udev_cleanup } # @@ -90,8 +89,8 @@ function test_io # dev { typeset dev=$1 - log_must dd if=/dev/zero of=$dev count=1 log_must dd if=$dev of=/dev/null count=1 + log_must dd if=/dev/zero of=$dev count=1 } log_assert "Verify that ZFS volume property 'volmode' works as intended" @@ -99,14 +98,14 @@ log_onexit cleanup VOLFS="$TESTPOOL/volfs" ZVOL="$TESTPOOL/vol" -ZDEV="${ZVOL_DEVDIR}/$ZVOL" +ZDEV="$ZVOL_DEVDIR/$ZVOL" SUBZVOL="$VOLFS/subvol" -SUBZDEV="${ZVOL_DEVDIR}/$SUBZVOL" +SUBZDEV="$ZVOL_DEVDIR/$SUBZVOL" +# 0. Verify basic ZVOL functionality log_must zfs create -o mountpoint=none $VOLFS log_must zfs create -V $VOLSIZE -s $SUBZVOL log_must zfs create -V $VOLSIZE -s $ZVOL -udev_wait blockdev_exists $ZDEV blockdev_exists $SUBZDEV test_io $ZDEV @@ -123,62 +122,63 @@ done log_must zfs set volmode=none $ZVOL blockdev_missing $ZDEV log_must_busy zfs destroy $ZVOL +blockdev_missing $ZDEV # 3. Verify "volmode=full" exposes a fully functional device log_must zfs create -V $VOLSIZE -s $ZVOL -udev_wait +blockdev_exists $ZDEV log_must zfs set volmode=full $ZVOL blockdev_exists $ZDEV test_io $ZDEV log_must verify_partition $ZDEV -udev_wait # 3.1 Verify "volmode=geom" is an alias for "volmode=full" log_must zfs set volmode=geom $ZVOL blockdev_exists $ZDEV if [[ "$(get_prop 'volmode' $ZVOL)" != "full" ]]; then log_fail " Volmode value 'geom' is not an alias for 'full'" fi -udev_wait log_must_busy zfs destroy $ZVOL +blockdev_missing $ZDEV # 4. Verify "volmode=dev" hides partition info on the device log_must zfs create -V $VOLSIZE -s $ZVOL -udev_wait +blockdev_exists $ZDEV log_must zfs set volmode=dev $ZVOL blockdev_exists $ZDEV test_io $ZDEV log_mustnot verify_partition $ZDEV -udev_wait log_must_busy zfs destroy $ZVOL +blockdev_missing $ZDEV # 5. Verify "volmode=default" behaves accordingly to "volmode" module parameter # 5.1 Verify sysctl "volmode=full" sysctl_volmode 1 log_must zfs create -V $VOLSIZE -s $ZVOL -udev_wait +blockdev_exists $ZDEV log_must zfs set volmode=default $ZVOL blockdev_exists $ZDEV log_must verify_partition $ZDEV -udev_wait log_must_busy zfs destroy $ZVOL +blockdev_missing $ZDEV # 5.2 Verify sysctl "volmode=dev" sysctl_volmode 2 log_must zfs create -V $VOLSIZE -s $ZVOL -udev_wait +blockdev_exists $ZDEV log_must zfs set volmode=default $ZVOL blockdev_exists $ZDEV log_mustnot verify_partition $ZDEV -udev_wait log_must_busy zfs destroy $ZVOL +blockdev_missing $ZDEV # 5.2 Verify sysctl "volmode=none" sysctl_volmode 3 log_must zfs create -V $VOLSIZE -s $ZVOL -udev_wait +blockdev_missing $ZDEV log_must zfs set volmode=default $ZVOL blockdev_missing $ZDEV # 6. Verify "volmode" property is inherited correctly log_must zfs inherit volmode $ZVOL +blockdev_missing $ZDEV # 6.1 Check volmode=full case log_must zfs set volmode=full $TESTPOOL verify_inherited 'volmode' 'full' $ZVOL $TESTPOOL @@ -198,9 +198,7 @@ verify_inherited 'volmode' 'default' $ZVOL $TESTPOOL blockdev_exists $ZDEV # 6.5 Check inheritance on multiple levels log_must zfs inherit volmode $SUBZVOL -udev_wait log_must zfs set volmode=none $VOLFS -udev_wait log_must zfs set volmode=full $TESTPOOL verify_inherited 'volmode' 'none' $SUBZVOL $VOLFS blockdev_missing $SUBZDEV @@ -215,6 +213,8 @@ blockdev_exists $ZDEV blockdev_missing $SUBZDEV log_must_busy zfs destroy $ZVOL log_must_busy zfs destroy $SUBZVOL +blockdev_missing $ZDEV +blockdev_missing $SUBZDEV # 8. Verify "volmode" behaves accordingly to zvol_inhibit_dev (Linux only) if is_linux; then @@ -226,6 +226,7 @@ if is_linux; then log_must zfs set volmode=full $ZVOL blockdev_missing $ZDEV log_must_busy zfs destroy $ZVOL + blockdev_missing $ZDEV # 7.1 Verify device nodes not are not created with "volmode=dev" sysctl_volmode 2 log_must zfs create -V $VOLSIZE -s $ZVOL @@ -233,6 +234,7 @@ if is_linux; then log_must zfs set volmode=dev $ZVOL blockdev_missing $ZDEV log_must_busy zfs destroy $ZVOL + blockdev_missing $ZDEV # 7.1 Verify device nodes not are not created with "volmode=none" sysctl_volmode 3 log_must zfs create -V $VOLSIZE -s $ZVOL diff --git a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_zil.ksh b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_zil.ksh index b8989f478727..a393606d0fa9 100755 --- a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_zil.ksh +++ b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_zil.ksh @@ -42,8 +42,8 @@ verify_runnable "global" function cleanup { - datasetexists $ZVOL && log_must_busy zfs destroy $ZVOL - udev_wait + datasetexists $ZVOL && destroy_dataset $ZVOL + block_device_wait } log_assert "Verify ZIL functionality on ZVOLs" From 4fea6a6737b86a960ba6d2944db6e25a4c272ac5 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Mon, 11 Oct 2021 10:52:32 -0700 Subject: [PATCH 054/109] ZTS: Add known exceptions Add the following test failures to the exception list for FreeBSD to ensure we notice new unexpected failures. pool_checkpoint/checkpoint_big_rewind pool_checkpoint/checkpoint_indirect And the following for Linux. zvol/zvol_misc/zvol_misc_snapdev Reviewed-by: George Melikov Signed-off-by: Brian Behlendorf Issue #12621 Issue #12622 Issue #12623 Closes #12624 --- tests/test-runner/bin/zts-report.py.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index c4f01b8718fe..19548b3842db 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -278,6 +278,8 @@ if sys.platform.startswith('freebsd'): 'delegate/zfs_allow_003_pos': ['FAIL', known_reason], 'inheritance/inherit_001_pos': ['FAIL', '11829'], 'resilver/resilver_restart_001': ['FAIL', known_reason], + 'pool_checkpoint/checkpoint_big_rewind': ['FAIL', '12622'], + 'pool_checkpoint/checkpoint_indirect': ['FAIL', '12623'], }) elif sys.platform.startswith('linux'): maybe.update({ @@ -293,6 +295,7 @@ elif sys.platform.startswith('linux'): 'mmp/mmp_exported_import': ['FAIL', known_reason], 'mmp/mmp_inactive_import': ['FAIL', known_reason], 'snapshot/rollback_003_pos': ['FAIL', known_reason], + 'zvol/zvol_misc/zvol_misc_snapdev': ['FAIL', '12621'], }) From 4730c3f249a9ff6bff15ee574aace78f9426317b Mon Sep 17 00:00:00 2001 From: Rich Ercolani <214141+rincebrain@users.noreply.github.com> Date: Mon, 8 Nov 2021 21:01:19 -0500 Subject: [PATCH 055/109] Exclude zvol_misc_volmode for now It keeps failing, on changes which aren't related at all. So until someone runs down why, I'd like it to stop being the sole reason for CI failures. Reviewed-by: John Kennedy Reviewed-by: Brian Behlendorf Reviewed-by: George Melikov Signed-off-by: Rich Ercolani Closes #12733 --- tests/test-runner/bin/zts-report.py.in | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index 19548b3842db..430143121eac 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -296,6 +296,7 @@ elif sys.platform.startswith('linux'): 'mmp/mmp_inactive_import': ['FAIL', known_reason], 'snapshot/rollback_003_pos': ['FAIL', known_reason], 'zvol/zvol_misc/zvol_misc_snapdev': ['FAIL', '12621'], + 'zvol/zvol_misc/zvol_misc_volmode': ['FAIL', known_reason], }) From 306cccca274d2af19bbfa76452249359db00a832 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Mon, 22 Jun 2020 09:44:49 -0700 Subject: [PATCH 056/109] Update zts-report.py with additional tests The following test cases may still occasionally fail and are being added to the "maybe" list for Linux until they can be updated to be entirely reliable. cli_root/zfs_rename/zfs_rename_002_pos.ksh cli_root/zpool_reopen/zpool_reopen_003_pos.ksh refreserv/refreserv_raidz These 6 tests consistently fail only on Fedora 31+, the failures are related to the kernel rescanning the partition table on loopback devices which is no longer reliable unless partprobe is used. In order to enable the Fedora bot by default they are also being added to the list until the tests can be updated. Any significant regression in functionality covered by these tests will still be detected by the FreeBSD builders. alloc_class/alloc_class_009_pos alloc_class/alloc_class_010_pos cli_root/zpool_expand/zpool_expand_001_pos cli_root/zpool_expand/zpool_expand_005_pos rsend/rsend_007_pos rsend/rsend_010_pos rsend/rsend_011_pos snapshot/rollback_003_pos Signed-off-by: Brian Behlendorf Closes #10489 --- tests/test-runner/bin/zts-report.py.in | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index 430143121eac..414e266568dc 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -328,6 +328,20 @@ if os.environ.get('CI') == 'true': maybe.update({ 'events/events_002_pos': ['FAIL', '11546'], }) +elif sys.platform.startswith('linux'): + maybe.update({ + 'alloc_class/alloc_class_009_pos': ['FAIL', known_reason], + 'alloc_class/alloc_class_010_pos': ['FAIL', known_reason], + 'cli_root/zfs_rename/zfs_rename_002_pos': ['FAIL', known_reason], + 'cli_root/zpool_expand/zpool_expand_001_pos': ['FAIL', known_reason], + 'cli_root/zpool_expand/zpool_expand_005_pos': ['FAIL', known_reason], + 'cli_root/zpool_reopen/zpool_reopen_003_pos': ['FAIL', known_reason], + 'refreserv/refreserv_raidz': ['FAIL', known_reason], + 'rsend/rsend_007_pos': ['FAIL', known_reason], + 'rsend/rsend_010_pos': ['FAIL', known_reason], + 'rsend/rsend_011_pos': ['FAIL', known_reason], + 'snapshot/rollback_003_pos': ['FAIL', known_reason], + }) def usage(s): From c454e46336c31d3fe2c6491c57b1e71963ca7ed6 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 22 Dec 2021 11:05:07 -0800 Subject: [PATCH 057/109] ZTS: Fix rollback_003_pos.ksh Under Linux when rolling back a mounted filesystem negative dentries may not be dropped from the cache. This can result in an ENOENT being incorrectly returned on first access. Issuing a `df` before the unmount results in the negative dentries being invalidated and side steps the issue. This is solely a workaround for the test case on Linux and not correct behavior. The core issue of invalidating negative dentries needs to be handled with a kernel side change. This is being tracked as issue #6143. Reviewed-by: George Melikov Reviewed-by: John Kennedy Signed-off-by: Brian Behlendorf Closes #12898 Issue #6143 --- tests/test-runner/bin/zts-report.py.in | 1 - .../tests/functional/rsend/rsend.kshlib | 7 +++++++ .../functional/snapshot/rollback_003_pos.ksh | 17 ++++++++++------- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index 414e266568dc..d306e04760c9 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -294,7 +294,6 @@ elif sys.platform.startswith('linux'): 'mmp/mmp_active_import': ['FAIL', known_reason], 'mmp/mmp_exported_import': ['FAIL', known_reason], 'mmp/mmp_inactive_import': ['FAIL', known_reason], - 'snapshot/rollback_003_pos': ['FAIL', known_reason], 'zvol/zvol_misc/zvol_misc_snapdev': ['FAIL', '12621'], 'zvol/zvol_misc/zvol_misc_volmode': ['FAIL', known_reason], }) diff --git a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib index 8cd35e4afdfe..516d41263294 100644 --- a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib +++ b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib @@ -121,6 +121,13 @@ function cleanup_pool log_must rm -rf $BACKDIR/* if is_global_zone ; then + # + # Linux: Issuing a `df` seems to properly force any negative + # dcache entries to be invalidated preventing failures when + # accessing the mount point. Additional investigation required. + # + # https://github.com/openzfs/zfs/issues/6143 + # log_must df >/dev/null log_must_busy zfs destroy -Rf $pool else diff --git a/tests/zfs-tests/tests/functional/snapshot/rollback_003_pos.ksh b/tests/zfs-tests/tests/functional/snapshot/rollback_003_pos.ksh index 59e7c110ddd7..036e71410c6d 100755 --- a/tests/zfs-tests/tests/functional/snapshot/rollback_003_pos.ksh +++ b/tests/zfs-tests/tests/functional/snapshot/rollback_003_pos.ksh @@ -92,6 +92,15 @@ log_note "Verify rollback of multiple nested file systems succeeds." log_must zfs snapshot $TESTPOOL/$TESTFILE@$TESTSNAP log_must zfs snapshot $SNAPPOOL.1 +# +# Linux: Issuing a `df` seems to properly force any negative dcache entries to +# be invalidated preventing failures when accessing the mount point. Additional +# investigation required. +# +# https://github.com/openzfs/zfs/issues/6143 +# +log_must df >/dev/null + export __ZFS_POOL_RESTRICT="$TESTPOOL" log_must zfs unmount -a log_must zfs mount -a @@ -100,12 +109,6 @@ unset __ZFS_POOL_RESTRICT log_must touch /$TESTPOOL/$TESTFILE/$TESTFILE.1 log_must zfs rollback $SNAPPOOL.1 - -# -# Workaround for issue #6143. Issuing a `df` seems to properly force any -# negative dcache entries to be invalidated preventing subsequent failures -# when accessing the mount point. Additional investigation required. -# -log_must df +log_must df >/dev/null log_pass "Rollbacks succeed when nested file systems are present." From 8285e1b09d7ba27825dec1fcc9c482683232ff67 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Thu, 23 Dec 2021 09:21:40 -0800 Subject: [PATCH 058/109] ZTS: Fix enospc_002_pos.ksh again This is a follow up commit for e03a41a60 which aimed to resolve this same test failure. The core "problem" here is that it takes very little space to perform a clone/snapshot/bookmark, which means if we want these commands to reliably fail the pool must truely have exhausted all free space. This commit increases the number of fill iterations to try and consume every block which we can. This still can't guarantee the clone/snapshot/bookmark will fail, but it significantly improves the odds. The exception was kept since it's still not a sure thing. Reviewed-by: George Melikov Reviewed-by: John Kennedy Reviewed-by: Igor Kozhukhov Signed-off-by: Brian Behlendorf Closes #12903 --- tests/zfs-tests/tests/functional/no_space/enospc_002_pos.ksh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/zfs-tests/tests/functional/no_space/enospc_002_pos.ksh b/tests/zfs-tests/tests/functional/no_space/enospc_002_pos.ksh index db6ee6ba7ca5..6239d491e6eb 100755 --- a/tests/zfs-tests/tests/functional/no_space/enospc_002_pos.ksh +++ b/tests/zfs-tests/tests/functional/no_space/enospc_002_pos.ksh @@ -49,7 +49,7 @@ log_must zfs snapshot $TESTPOOL/$TESTFS@snap # log_note "Writing files until ENOSPC." -for i in $(seq 30); do +for i in $(seq 100); do file_write -o create -f $TESTDIR/file.$i -b $BLOCKSZ \ -c $NUM_WRITES -d $DATA ret=$? From 5987838a3f7cc8a3adcce7055750cf657e1efdbf Mon Sep 17 00:00:00 2001 From: chrisrd Date: Fri, 7 Jan 2022 06:00:01 +1100 Subject: [PATCH 059/109] man: speling Fix spelling. Reviewed-by: Rich Ercolani Reviewed-by: Matthew Ahrens Reviewed-by: Brian Behlendorf Signed-off-by: Chris Dunlop Closes #12911 --- man/man7/zfsprops.7 | 8 ++++---- man/man8/zfs-load-key.8 | 2 +- man/man8/zgenhostid.8 | 2 +- man/man8/zpool-attach.8 | 2 +- man/man8/zpool-events.8 | 4 ++-- man/man8/zpool-replace.8 | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/man/man7/zfsprops.7 b/man/man7/zfsprops.7 index 8b79603f4945..b753ec321ad0 100644 --- a/man/man7/zfsprops.7 +++ b/man/man7/zfsprops.7 @@ -909,7 +909,7 @@ ratio; for example, 8kB blocks on disks with 4kB disk sectors must compress to 1 or less of their original size. .It Xo .Sy context Ns = Ns Sy none Ns | Ns -.Ar SELinux-User : Ns Ar SElinux-Role : Ns Ar Selinux-Type : Ns Ar Sensitivity-Level +.Ar SELinux-User : Ns Ar SELinux-Role : Ns Ar SELinux-Type : Ns Ar Sensitivity-Level .Xc This flag sets the SELinux context for all files in the file system under a mount point for that file system. @@ -918,7 +918,7 @@ See for more information. .It Xo .Sy fscontext Ns = Ns Sy none Ns | Ns -.Ar SELinux-User : Ns Ar SElinux-Role : Ns Ar Selinux-Type : Ns Ar Sensitivity-Level +.Ar SELinux-User : Ns Ar SELinux-Role : Ns Ar SELinux-Type : Ns Ar Sensitivity-Level .Xc This flag sets the SELinux context for the file system file system being mounted. @@ -927,7 +927,7 @@ See for more information. .It Xo .Sy defcontext Ns = Ns Sy none Ns | Ns -.Ar SELinux-User : Ns Ar SElinux-Role : Ns Ar Selinux-Type : Ns Ar Sensitivity-Level +.Ar SELinux-User : Ns Ar SELinux-Role : Ns Ar SELinux-Type : Ns Ar Sensitivity-Level .Xc This flag sets the SELinux default context for unlabeled files. See @@ -935,7 +935,7 @@ See for more information. .It Xo .Sy rootcontext Ns = Ns Sy none Ns | Ns -.Ar SELinux-User : Ns Ar SElinux-Role : Ns Ar Selinux-Type : Ns Ar Sensitivity-Level +.Ar SELinux-User : Ns Ar SELinux-Role : Ns Ar SELinux-Type : Ns Ar Sensitivity-Level .Xc This flag sets the SELinux context for the root inode of the file system. See diff --git a/man/man8/zfs-load-key.8 b/man/man8/zfs-load-key.8 index ed89b65d7159..b12a79e0150a 100644 --- a/man/man8/zfs-load-key.8 +++ b/man/man8/zfs-load-key.8 @@ -200,7 +200,7 @@ if supported by your hardware, otherwise .Bl -tag -width "-r" .It Fl l Ensures the key is loaded before attempting to change the key. -This is effectively equivalent to runnin +This is effectively equivalent to running .Nm zfs Cm load-key Ar filesystem ; Nm zfs Cm change-key Ar filesystem .It Fl o Ar property Ns = Ns Ar value Allows the user to set encryption key properties diff --git a/man/man8/zgenhostid.8 b/man/man8/zgenhostid.8 index 0dcebef73c31..e157578cf0bb 100644 --- a/man/man8/zgenhostid.8 +++ b/man/man8/zgenhostid.8 @@ -80,7 +80,7 @@ be an 8-digit-long hexadecimal number, optionally prefixed by .Dl # Nm Qq $ Ns Pq Nm hostid .It Record a custom hostid Po Ar 0xdeadbeef Pc in Pa /etc/hostid .Dl # Nm Ar deadbeef -.It Record a custom hostid Po Ar 0x01234567 Pc in Pa /tmp/hostid No and ovewrite the file if it exists +.It Record a custom hostid Po Ar 0x01234567 Pc in Pa /tmp/hostid No and overwrite the file if it exists .Dl # Nm Fl f o Ar /tmp/hostid 0x01234567 .El . diff --git a/man/man8/zpool-attach.8 b/man/man8/zpool-attach.8 index 19d8f6ac07ac..9dfa35a107d3 100644 --- a/man/man8/zpool-attach.8 +++ b/man/man8/zpool-attach.8 @@ -79,7 +79,7 @@ The only property supported at the moment is The .Ar new_device is reconstructed sequentially to restore redundancy as quickly as possible. -Checksums are not verfied during sequential reconstruction so a scrub is +Checksums are not verified during sequential reconstruction so a scrub is started when the resilver completes. Sequential reconstruction is not supported for raidz configurations. .It Fl w diff --git a/man/man8/zpool-events.8 b/man/man8/zpool-events.8 index ab1d6ea56213..55f7babae2bd 100644 --- a/man/man8/zpool-events.8 +++ b/man/man8/zpool-events.8 @@ -68,7 +68,7 @@ Print the entire payload for each event. .El . .Sh EVENTS -Theese are the different event subclasses. +These are the different event subclasses. The full event name would be .Sy ereport.fs.zfs.\& Ns Em SUBCLASS , but only the last part is listed here. @@ -136,7 +136,7 @@ Issued when the ashift alignment requirement has increased. .It Sy vdev.remove Issued when a vdev is detached from a mirror (or a spare detached from a vdev where it have been used to replace a failed drive - only works if -the original drive have been readded). +the original drive have been re-added). .It Sy vdev.clear Issued when clearing device errors in a pool. Such as running diff --git a/man/man8/zpool-replace.8 b/man/man8/zpool-replace.8 index 2b2875ed4292..cc61fa3ea37e 100644 --- a/man/man8/zpool-replace.8 +++ b/man/man8/zpool-replace.8 @@ -85,7 +85,7 @@ The only property supported at the moment is The .Ar new-device is reconstructed sequentially to restore redundancy as quickly as possible. -Checksums are not verfied during sequential reconstruction so a scrub is +Checksums are not verified during sequential reconstruction so a scrub is started when the resilver completes. Sequential reconstruction is not supported for raidz configurations. .It Fl w From f601ee1e437095294966c71dc47b179ab30e7844 Mon Sep 17 00:00:00 2001 From: Ryan Hirasaki <4690732+RyanHir@users.noreply.github.com> Date: Thu, 6 Jan 2022 18:25:01 -0600 Subject: [PATCH 060/109] README: Update OpenZFS website url This change is to first replace the OpenZFS website in the README to point to openzfs.org as this is what open-zfs.org redirects to. Along with replacing the URL, the protocol is also upgraded from http to https. These changes should prevent web browsers such as Firefox from complaining about visiting a http site, if the proper security settings are enabled, when it will still end up on a https page after the redirect. Reviewed-by: George Melikov Reviewed-by: Brian Behlendorf Signed-off-by: Ryan Hirasaki Closes #12939 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d666df7af309..331889560950 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This repository contains the code for running OpenZFS on Linux and FreeBSD. * [Documentation](https://openzfs.github.io/openzfs-docs/) - for using and developing this repo * [ZoL Site](https://zfsonlinux.org) - Linux release info & links * [Mailing lists](https://openzfs.github.io/openzfs-docs/Project%20and%20Community/Mailing%20Lists.html) - * [OpenZFS site](http://open-zfs.org/) - for conference videos and info on other platforms (illumos, OSX, Windows, etc) + * [OpenZFS site](https://openzfs.org/) - for conference videos and info on other platforms (illumos, OSX, Windows, etc) # Installation From 7f4f461bcf8c9217f4efebb3b728c957f6249d48 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Fri, 14 Jan 2022 15:30:07 -0800 Subject: [PATCH 061/109] Clarify `failmode=wait` documentation Nowhere in the description of the failmode property does it clearly state how to bring a suspended pool back online. Add a few words to property description and the zpool-clear(8) man page. Reviewed-by: George Melikov Signed-off-by: Brian Behlendorf Closes #12907 Closes #9395 --- man/man7/zpoolprops.7 | 3 ++- man/man8/zpool-clear.8 | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/man/man7/zpoolprops.7 b/man/man7/zpoolprops.7 index 513f02e0314f..543920a1d58e 100644 --- a/man/man7/zpoolprops.7 +++ b/man/man7/zpoolprops.7 @@ -337,7 +337,8 @@ The behavior of such an event is determined as follows: .Bl -tag -width "continue" .It Sy wait Blocks all I/O access until the device connectivity is recovered and the errors -are cleared. +are cleared with +.Nm zpool Cm clear . This is the default behavior. .It Sy continue Returns diff --git a/man/man8/zpool-clear.8 b/man/man8/zpool-clear.8 index 6e41566ca6da..0b256b28bd21 100644 --- a/man/man8/zpool-clear.8 +++ b/man/man8/zpool-clear.8 @@ -44,9 +44,12 @@ Clears device errors in a pool. If no arguments are specified, all device errors within the pool are cleared. If one or more devices is specified, only those errors associated with the specified device or devices are cleared. -If +.Pp +If the pool was suspended it will be brought back online provided the +devices can be accessed. +Pools with .Sy multihost -is enabled and the pool has been suspended, this will not resume I/O. +enabled which have been suspended cannot be resumed. While the pool was suspended, it may have been imported on another host, and resuming I/O could result in pool damage. . From 5753e7a7c503d7f903048163b370efa750209ee2 Mon Sep 17 00:00:00 2001 From: ColMelvin Date: Tue, 25 Jan 2022 15:14:43 -0600 Subject: [PATCH 062/109] RPM: Add missing BuildRequires for PAM component When the optional PAM binaries are included in a build, ./configure will look for security/pam_modules.h and - if it doesn't find it - recommend the user install `libpam0g-dev`. On Red Hat systems, `pam-devel` is the package that supplies this requirement; `libpam0g-dev` does not exist. By encoding this requirement in the spec file, we give packagers more appropriate (and timely) recommendations for completing the build. Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf Signed-off-by: Chris Lindee Closes #13001 --- rpm/generic/zfs.spec.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rpm/generic/zfs.spec.in b/rpm/generic/zfs.spec.in index 25beadce7c1f..0e9cdcc31f32 100644 --- a/rpm/generic/zfs.spec.in +++ b/rpm/generic/zfs.spec.in @@ -148,10 +148,15 @@ BuildRequires: ncompress BuildRequires: libtirpc-devel %endif +%if %{with pam} +BuildRequires: pam-devel +%endif + Requires: openssl %if 0%{?_systemd} BuildRequires: systemd %endif + %endif %if 0%{?_systemd} From 72a82f312f9fd0abd5b15f1d847e448a56353ec7 Mon Sep 17 00:00:00 2001 From: George Amanakis Date: Thu, 3 Feb 2022 23:28:19 +0100 Subject: [PATCH 063/109] Report dnodes with faulty bonuslen In files created/modified before 4254acb there may be a corruption of xattrs which is not reported during scrub and normal send/receive. It manifests only as an error when raw sending/receiving. This happens because currently only the raw receive path checks for discrepancies between the dnode bonus length and the spill pointer flag. In case we encounter a dnode whose bonus length is greater than the predicted one, we should report an error. Modify in this regard dnode_sync() with an assertion at the end, dump_dnode() to error out, dsl_scan_recurse() to report errors during a scrub, and zstream to report a warning when dumping. Also added a test to verify spill blocks are sent correctly in a raw send. Reviewed-by: Brian Behlendorf Signed-off-by: George Amanakis Closes #12720 Closes #13014 --- cmd/zstream/zstream_dump.c | 8 + module/zfs/dmu_send.c | 2 + module/zfs/dnode_sync.c | 2 + module/zfs/dsl_scan.c | 13 ++ tests/runfiles/common.run | 3 +- .../tests/functional/rsend/Makefile.am | 1 + .../functional/rsend/send_raw_spill_block.ksh | 161 ++++++++++++++++++ 7 files changed, 189 insertions(+), 1 deletion(-) create mode 100755 tests/zfs-tests/tests/functional/rsend/send_raw_spill_block.ksh diff --git a/cmd/zstream/zstream_dump.c b/cmd/zstream/zstream_dump.c index 45cf7b97a147..af69a1494f28 100644 --- a/cmd/zstream/zstream_dump.c +++ b/cmd/zstream/zstream_dump.c @@ -461,6 +461,14 @@ zstream_do_dump(int argc, char *argv[]) BSWAP_64(drro->drr_maxblkid); } + if (drro->drr_bonuslen > drro->drr_raw_bonuslen) { + (void) fprintf(stderr, + "Warning: Object %llu has bonuslen = " + "%u > raw_bonuslen = %u\n\n", + (u_longlong_t)drro->drr_object, + drro->drr_bonuslen, drro->drr_raw_bonuslen); + } + payload_size = DRR_OBJECT_PAYLOAD_SIZE(drro); if (verbose) { diff --git a/module/zfs/dmu_send.c b/module/zfs/dmu_send.c index 2f2fd4c3d6c8..58a72a68a65c 100644 --- a/module/zfs/dmu_send.c +++ b/module/zfs/dmu_send.c @@ -763,6 +763,8 @@ dump_dnode(dmu_send_cookie_t *dscp, const blkptr_t *bp, uint64_t object, * to send it. */ if (bonuslen != 0) { + if (drro->drr_bonuslen > DN_MAX_BONUS_LEN(dnp)) + return (SET_ERROR(EINVAL)); drro->drr_raw_bonuslen = DN_MAX_BONUS_LEN(dnp); bonuslen = drro->drr_raw_bonuslen; } diff --git a/module/zfs/dnode_sync.c b/module/zfs/dnode_sync.c index f574130e5049..12ab4bea145f 100644 --- a/module/zfs/dnode_sync.c +++ b/module/zfs/dnode_sync.c @@ -854,6 +854,8 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx) dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg); } + ASSERT3U(dnp->dn_bonuslen, <=, DN_MAX_BONUS_LEN(dnp)); + /* * Although we have dropped our reference to the dnode, it * can't be evicted until its written, and we haven't yet diff --git a/module/zfs/dsl_scan.c b/module/zfs/dsl_scan.c index 62ee9bb9ab6c..765998417153 100644 --- a/module/zfs/dsl_scan.c +++ b/module/zfs/dsl_scan.c @@ -1821,6 +1821,19 @@ dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype, ASSERT(!BP_IS_REDACTED(bp)); + /* + * There is an unlikely case of encountering dnodes with contradicting + * dn_bonuslen and DNODE_FLAG_SPILL_BLKPTR flag before in files created + * or modified before commit 4254acb was merged. As it is not possible + * to know which of the two is correct, report an error. + */ + if (dnp != NULL && + dnp->dn_bonuslen > DN_MAX_BONUS_LEN(dnp)) { + scn->scn_phys.scn_errors++; + spa_log_error(dp->dp_spa, zb); + return (SET_ERROR(EINVAL)); + } + if (BP_GET_LEVEL(bp) > 0) { arc_flags_t flags = ARC_FLAG_WAIT; int i; diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index f4d8eb59f088..98d91eae19ab 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -831,7 +831,8 @@ tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos', 'send_freeobjects', 'send_realloc_files', 'send_realloc_encrypted_files', 'send_spill_block', 'send_holds', 'send_hole_birth', 'send_mixed_raw', 'send-wR_encrypted_zvol', - 'send_partial_dataset', 'send_invalid', 'send_doall'] + 'send_partial_dataset', 'send_invalid', 'send_doall', + 'send_raw_spill_block'] tags = ['functional', 'rsend'] [tests/functional/scrub_mirror] diff --git a/tests/zfs-tests/tests/functional/rsend/Makefile.am b/tests/zfs-tests/tests/functional/rsend/Makefile.am index 94bdd2674517..ba1bcf24a501 100644 --- a/tests/zfs-tests/tests/functional/rsend/Makefile.am +++ b/tests/zfs-tests/tests/functional/rsend/Makefile.am @@ -49,6 +49,7 @@ dist_pkgdata_SCRIPTS = \ send_realloc_files.ksh \ send_realloc_encrypted_files.ksh \ send_spill_block.ksh \ + send_raw_spill_block.ksh \ send_holds.ksh \ send_hole_birth.ksh \ send_invalid.ksh \ diff --git a/tests/zfs-tests/tests/functional/rsend/send_raw_spill_block.ksh b/tests/zfs-tests/tests/functional/rsend/send_raw_spill_block.ksh new file mode 100755 index 000000000000..8d7451ae62e2 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send_raw_spill_block.ksh @@ -0,0 +1,161 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2021, George Amanakis. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify spill blocks are correctly preserved in raw sends. +# +# Strategy: +# 1) Create a set of files each containing some file data in an +# encrypted filesystem. +# 2) Add enough xattrs to the file to require a spill block. +# 3) Snapshot and raw send these files to a new dataset. +# 4) Modify the files and spill blocks in a variety of ways. +# 5) Send the changes using an raw incremental send stream. +# 6) Verify that all the xattrs (and thus the spill block) were +# preserved when receiving the incremental stream. +# + +verify_runnable "both" + +log_assert "Verify spill blocks are correctly preserved in raw sends" + +function cleanup +{ + rm -f $BACKDIR/fs@* + destroy_dataset $POOL/fs "-rR" + destroy_dataset $POOL/newfs "-rR" +} + +attrvalue="abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + +log_onexit cleanup + +log_must eval "echo 'password' | zfs create -o encryption=on" \ + "-o keyformat=passphrase -o keylocation=prompt " \ + "$POOL/fs" +log_must zfs set xattr=sa $POOL/fs +log_must zfs set dnodesize=legacy $POOL/fs +log_must zfs set recordsize=128k $POOL/fs + +# Create 40 files each with a spill block containing xattrs. Each file +# will be modified in a different way to validate the incremental receive. +for i in {1..40}; do + file="/$POOL/fs/file$i" + + log_must mkfile 16384 $file + for j in {1..20}; do + log_must set_xattr "testattr$j" "$attrvalue" $file + done +done + +# Snapshot the pool and send it to the new dataset. +log_must zfs snapshot $POOL/fs@snap1 +log_must eval "zfs send -w $POOL/fs@snap1 >$BACKDIR/fs@snap1" +log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs@snap1" + +# +# Modify file[1-6]'s contents but not the spill blocks. +# +# file1 - Increase record size; single block +# file2 - Increase record size; multiple blocks +# file3 - Truncate file to zero size; single block +# file4 - Truncate file to smaller size; single block +# file5 - Truncate file to much larger size; add holes +# file6 - Truncate file to embedded size; embedded data +# +log_must mkfile 32768 /$POOL/fs/file1 +log_must mkfile 1048576 /$POOL/fs/file2 +log_must truncate -s 0 /$POOL/fs/file3 +log_must truncate -s 8192 /$POOL/fs/file4 +log_must truncate -s 1073741824 /$POOL/fs/file5 +log_must truncate -s 50 /$POOL/fs/file6 + +# +# Modify file[11-16]'s contents and their spill blocks. +# +# file11 - Increase record size; single block +# file12 - Increase record size; multiple blocks +# file13 - Truncate file to zero size; single block +# file14 - Truncate file to smaller size; single block +# file15 - Truncate file to much larger size; add holes +# file16 - Truncate file to embedded size; embedded data +# +log_must mkfile 32768 /$POOL/fs/file11 +log_must mkfile 1048576 /$POOL/fs/file12 +log_must truncate -s 0 /$POOL/fs/file13 +log_must truncate -s 8192 /$POOL/fs/file14 +log_must truncate -s 1073741824 /$POOL/fs/file15 +log_must truncate -s 50 /$POOL/fs/file16 + +for i in {11..20}; do + log_must rm_xattr testattr1 /$POOL/fs/file$i +done + +# +# Modify file[21-26]'s contents and remove their spill blocks. +# +# file21 - Increase record size; single block +# file22 - Increase record size; multiple blocks +# file23 - Truncate file to zero size; single block +# file24 - Truncate file to smaller size; single block +# file25 - Truncate file to much larger size; add holes +# file26 - Truncate file to embedded size; embedded data +# +log_must mkfile 32768 /$POOL/fs/file21 +log_must mkfile 1048576 /$POOL/fs/file22 +log_must truncate -s 0 /$POOL/fs/file23 +log_must truncate -s 8192 /$POOL/fs/file24 +log_must truncate -s 1073741824 /$POOL/fs/file25 +log_must truncate -s 50 /$POOL/fs/file26 + +for i in {21..30}; do + for j in {1..20}; do + log_must rm_xattr testattr$j /$POOL/fs/file$i + done +done + +# +# Modify file[31-40]'s spill blocks but not the file contents. +# +for i in {31..40}; do + file="/$POOL/fs/file$i" + log_must rm_xattr testattr$(((RANDOM % 20) + 1)) $file + log_must set_xattr testattr$(((RANDOM % 20) + 1)) "$attrvalue" $file +done + +# Calculate the expected recursive checksum for the source. +expected_cksum=$(recursive_cksum /$POOL/fs) + +# Snapshot the pool and send the incremental snapshot. +log_must zfs snapshot $POOL/fs@snap2 +log_must eval "zfs send -w -i $POOL/fs@snap1 $POOL/fs@snap2 >$BACKDIR/fs@snap2" +log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs@snap2" +log_must eval "echo 'password' | zfs load-key $POOL/newfs" +log_must zfs mount $POOL/newfs + +# Validate the received copy using the received recursive checksum. +actual_cksum=$(recursive_cksum /$POOL/newfs) +if [[ "$expected_cksum" != "$actual_cksum" ]]; then + log_fail "Checksums differ ($expected_cksum != $actual_cksum)" +fi + +log_pass "Verify spill blocks are correctly preserved in raw sends" From 9221ff1888c9d8b80ffd9c45b44231ad19c0e9fb Mon Sep 17 00:00:00 2001 From: Akash B Date: Fri, 4 Feb 2022 03:59:29 +0530 Subject: [PATCH 064/109] Add enumerated vdev names to 'zpool iostat -v' and 'zpool list -v' This commit adds enumerated names to disambiguate between the different vdevs. Previously only 'zpool status' showed enumerated vdev names, now 'zpool list -v' and 'zpool iostat -v' also shows the enumerated vdev names. Reviewed-by: Brian Behlendorf Reviewed-by: Dipak Ghosh Signed-off-by: Akash B Closes #12510 Closes #13031 --- cmd/zpool/zpool_main.c | 8 ++++---- tests/zfs-tests/include/libtest.shlib | 2 +- .../tests/functional/redundancy/redundancy.kshlib | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c index 9cf0163ab327..b93a6196beea 100644 --- a/cmd/zpool/zpool_main.c +++ b/cmd/zpool/zpool_main.c @@ -4825,7 +4825,7 @@ print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv, continue; vname = zpool_vdev_name(g_zfs, zhp, newchild[c], - cb->cb_name_flags); + cb->cb_name_flags | VDEV_NAME_TYPE_ID); ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL, newchild[c], cb, depth + 2); free(vname); @@ -4868,7 +4868,7 @@ print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv, } vname = zpool_vdev_name(g_zfs, zhp, newchild[c], - cb->cb_name_flags); + cb->cb_name_flags | VDEV_NAME_TYPE_ID); ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL, newchild[c], cb, depth + 2); free(vname); @@ -6182,7 +6182,7 @@ print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv, continue; vname = zpool_vdev_name(g_zfs, zhp, child[c], - cb->cb_name_flags); + cb->cb_name_flags | VDEV_NAME_TYPE_ID); print_list_stats(zhp, vname, child[c], cb, depth + 2, B_FALSE); free(vname); } @@ -6216,7 +6216,7 @@ print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv, printed = B_TRUE; } vname = zpool_vdev_name(g_zfs, zhp, child[c], - cb->cb_name_flags); + cb->cb_name_flags | VDEV_NAME_TYPE_ID); print_list_stats(zhp, vname, child[c], cb, depth + 2, B_FALSE); free(vname); diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib index 773d134e3a2c..8a41b960f8d9 100644 --- a/tests/zfs-tests/include/libtest.shlib +++ b/tests/zfs-tests/include/libtest.shlib @@ -2145,7 +2145,7 @@ function get_disklist # pool disklist=$(zpool iostat -v $1 | nawk '(NR >4) {print $1}' | \ grep -v "\-\-\-\-\-" | \ - egrep -v -e "^(mirror|raidz[1-3]|spare|log|cache|special|dedup)$") + egrep -v -e "^(mirror|raidz[1-3]|draid[1-3]|spare|log|cache|special|dedup)|\-[0-9]$") echo $disklist } diff --git a/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib b/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib index baee8269b1e1..9888034667a7 100644 --- a/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib +++ b/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib @@ -243,7 +243,7 @@ function get_vdevs #pool cnt typeset -i cnt=$2 typeset all_devs=$(zpool iostat -v $pool | awk '{print $1}'| \ - egrep -v "^pool$|^capacity$|^mirror$|^raidz1$|^raidz2$|^raidz3$|^draid1.*|^draid2.*|^draid3.*|---" | \ + egrep -v "^pool$|^capacity$|^mirror\-[0-9]$|^raidz[1-3]\-[0-9]$|^draid[1-3].*\-[0-9]$|---" | \ egrep -v "/old$|^$pool$") typeset -i i=0 typeset vdevs From 3e27b589cff711a8c3cca659678d48361ff4e33b Mon Sep 17 00:00:00 2001 From: Pawel Jakub Dawidek Date: Thu, 3 Feb 2022 14:37:57 -0800 Subject: [PATCH 065/109] Fix clearing set-uid and set-gid bits on a file when replying a write POSIX requires that set-uid and set-gid bits to be removed when an unprivileged user writes to a file and ZFS does that during normal operation. The problem arrises when the write is stored in the ZIL and replayed. During replay we have no access to original credentials of the process doing the write, so zfs_write() will be performed with the root credentials. When root is doing the write set-uid and set-gid bits are not removed from the file. To correct that, log a separate TX_SETATTR entry that removed those bits on first write to such file. Idea from: Christian Schwarz Add test for ZIL replay of setuid/setgid clearing. Improve various edge cases when clearing setid bits: - The setid bits can be readded during a single write, so make sure to check for them on every chunk write. - Log TX_SETATTR record at most once per transaction group (if the setid bits are keep coming back). - Move zfs_log_setattr() outside of zp->z_acl_lock. Reviewed-by: Dan McDonald Reviewed-by: Brian Behlendorf Co-authored-by: Christian Schwarz Signed-off-by: Pawel Jakub Dawidek Closes #13027 --- module/zfs/zfs_vnops.c | 105 +++++++++--- tests/runfiles/common.run | 2 +- .../tests/functional/suid/Makefile.am | 1 + .../functional/suid/suid_write_to_file.c | 150 +++++++++--------- .../functional/suid/suid_write_to_none.ksh | 2 +- .../functional/suid/suid_write_to_sgid.ksh | 2 +- .../functional/suid/suid_write_to_suid.ksh | 2 +- .../suid/suid_write_to_suid_sgid.ksh | 2 +- .../functional/suid/suid_write_zil_replay.ksh | 99 ++++++++++++ 9 files changed, 261 insertions(+), 104 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/suid/suid_write_zil_replay.ksh diff --git a/module/zfs/zfs_vnops.c b/module/zfs/zfs_vnops.c index 54749810d45d..1b32d3876724 100644 --- a/module/zfs/zfs_vnops.c +++ b/module/zfs/zfs_vnops.c @@ -300,6 +300,62 @@ zfs_read(struct znode *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) return (error); } +static void +zfs_clear_setid_bits_if_necessary(zfsvfs_t *zfsvfs, znode_t *zp, cred_t *cr, + uint64_t *clear_setid_bits_txgp, dmu_tx_t *tx) +{ + zilog_t *zilog = zfsvfs->z_log; + const uint64_t uid = KUID_TO_SUID(ZTOUID(zp)); + + ASSERT(clear_setid_bits_txgp != NULL); + ASSERT(tx != NULL); + + /* + * Clear Set-UID/Set-GID bits on successful write if not + * privileged and at least one of the execute bits is set. + * + * It would be nice to do this after all writes have + * been done, but that would still expose the ISUID/ISGID + * to another app after the partial write is committed. + * + * Note: we don't call zfs_fuid_map_id() here because + * user 0 is not an ephemeral uid. + */ + mutex_enter(&zp->z_acl_lock); + if ((zp->z_mode & (S_IXUSR | (S_IXUSR >> 3) | (S_IXUSR >> 6))) != 0 && + (zp->z_mode & (S_ISUID | S_ISGID)) != 0 && + secpolicy_vnode_setid_retain(zp, cr, + ((zp->z_mode & S_ISUID) != 0 && uid == 0)) != 0) { + uint64_t newmode; + + zp->z_mode &= ~(S_ISUID | S_ISGID); + newmode = zp->z_mode; + (void) sa_update(zp->z_sa_hdl, SA_ZPL_MODE(zfsvfs), + (void *)&newmode, sizeof (uint64_t), tx); + + mutex_exit(&zp->z_acl_lock); + + /* + * Make sure SUID/SGID bits will be removed when we replay the + * log. If the setid bits are keep coming back, don't log more + * than one TX_SETATTR per transaction group. + */ + if (*clear_setid_bits_txgp != dmu_tx_get_txg(tx)) { + vattr_t va; + + bzero(&va, sizeof (va)); + va.va_mask = AT_MODE; + va.va_nodeid = zp->z_id; + va.va_mode = newmode; + zfs_log_setattr(zilog, tx, TX_SETATTR, zp, &va, AT_MODE, + NULL); + *clear_setid_bits_txgp = dmu_tx_get_txg(tx); + } + } else { + mutex_exit(&zp->z_acl_lock); + } +} + /* * Write the bytes to a file. * @@ -325,6 +381,7 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) { int error = 0, error1; ssize_t start_resid = zfs_uio_resid(uio); + uint64_t clear_setid_bits_txg = 0; /* * Fasttrack empty write @@ -503,6 +560,11 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) break; } + /* + * NB: We must call zfs_clear_setid_bits_if_necessary before + * committing the transaction! + */ + /* * If rangelock_enter() over-locked we grow the blocksize * and then reduce the lock range. This will only happen @@ -544,6 +606,8 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) zfs_uio_fault_disable(uio, B_FALSE); #ifdef __linux__ if (error == EFAULT) { + zfs_clear_setid_bits_if_necessary(zfsvfs, zp, + cr, &clear_setid_bits_txg, tx); dmu_tx_commit(tx); /* * Account for partial writes before @@ -566,6 +630,8 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) * VFS, which will handle faulting and will retry. */ if (error != 0 && error != EFAULT) { + zfs_clear_setid_bits_if_necessary(zfsvfs, zp, + cr, &clear_setid_bits_txg, tx); dmu_tx_commit(tx); break; } @@ -590,6 +656,13 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) error = dmu_assign_arcbuf_by_dbuf( sa_get_db(zp->z_sa_hdl), woff, abuf, tx); if (error != 0) { + /* + * XXX This might not be necessary if + * dmu_assign_arcbuf_by_dbuf is guaranteed + * to be atomic. + */ + zfs_clear_setid_bits_if_necessary(zfsvfs, zp, + cr, &clear_setid_bits_txg, tx); dmu_return_arcbuf(abuf); dmu_tx_commit(tx); break; @@ -615,30 +688,8 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) break; } - /* - * Clear Set-UID/Set-GID bits on successful write if not - * privileged and at least one of the execute bits is set. - * - * It would be nice to do this after all writes have - * been done, but that would still expose the ISUID/ISGID - * to another app after the partial write is committed. - * - * Note: we don't call zfs_fuid_map_id() here because - * user 0 is not an ephemeral uid. - */ - mutex_enter(&zp->z_acl_lock); - if ((zp->z_mode & (S_IXUSR | (S_IXUSR >> 3) | - (S_IXUSR >> 6))) != 0 && - (zp->z_mode & (S_ISUID | S_ISGID)) != 0 && - secpolicy_vnode_setid_retain(zp, cr, - ((zp->z_mode & S_ISUID) != 0 && uid == 0)) != 0) { - uint64_t newmode; - zp->z_mode &= ~(S_ISUID | S_ISGID); - newmode = zp->z_mode; - (void) sa_update(zp->z_sa_hdl, SA_ZPL_MODE(zfsvfs), - (void *)&newmode, sizeof (uint64_t), tx); - } - mutex_exit(&zp->z_acl_lock); + zfs_clear_setid_bits_if_necessary(zfsvfs, zp, cr, + &clear_setid_bits_txg, tx); zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime); @@ -664,8 +715,14 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) /* Avoid clobbering EFAULT. */ error = error1; + /* + * NB: During replay, the TX_SETATTR record logged by + * zfs_clear_setid_bits_if_necessary must precede any of + * the TX_WRITE records logged here. + */ zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes, ioflag, NULL, NULL); + dmu_tx_commit(tx); if (error != 0) diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 98d91eae19ab..18995511ed58 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -869,7 +869,7 @@ tags = ['functional', 'sparse'] [tests/functional/suid] tests = ['suid_write_to_suid', 'suid_write_to_sgid', 'suid_write_to_suid_sgid', - 'suid_write_to_none'] + 'suid_write_to_none', 'suid_write_zil_replay'] tags = ['functional', 'suid'] [tests/functional/threadsappend] diff --git a/tests/zfs-tests/tests/functional/suid/Makefile.am b/tests/zfs-tests/tests/functional/suid/Makefile.am index 594d2b77ca8e..0145c1205fb3 100644 --- a/tests/zfs-tests/tests/functional/suid/Makefile.am +++ b/tests/zfs-tests/tests/functional/suid/Makefile.am @@ -7,6 +7,7 @@ dist_pkgdata_SCRIPTS = \ suid_write_to_sgid.ksh \ suid_write_to_suid_sgid.ksh \ suid_write_to_none.ksh \ + suid_write_zil_replay.ksh \ cleanup.ksh \ setup.ksh diff --git a/tests/zfs-tests/tests/functional/suid/suid_write_to_file.c b/tests/zfs-tests/tests/functional/suid/suid_write_to_file.c index 571dc553bec2..f3febb903b59 100644 --- a/tests/zfs-tests/tests/functional/suid/suid_write_to_file.c +++ b/tests/zfs-tests/tests/functional/suid/suid_write_to_file.c @@ -29,86 +29,16 @@ #include #include #include - -static void -test_stat_mode(mode_t extra) -{ - struct stat st; - int i, fd; - char fpath[1024]; - char *penv[] = {"TESTDIR", "TESTFILE0"}; - char buf[] = "test"; - mode_t res; - mode_t mode = 0777 | extra; - - /* - * Get the environment variable values. - */ - for (i = 0; i < sizeof (penv) / sizeof (char *); i++) { - if ((penv[i] = getenv(penv[i])) == NULL) { - fprintf(stderr, "getenv(penv[%d])\n", i); - exit(1); - } - } - - umask(0); - if (stat(penv[0], &st) == -1 && mkdir(penv[0], mode) == -1) { - perror("mkdir"); - exit(2); - } - - snprintf(fpath, sizeof (fpath), "%s/%s", penv[0], penv[1]); - unlink(fpath); - if (stat(fpath, &st) == 0) { - fprintf(stderr, "%s exists\n", fpath); - exit(3); - } - - fd = creat(fpath, mode); - if (fd == -1) { - perror("creat"); - exit(4); - } - close(fd); - - if (setuid(65534) == -1) { - perror("setuid"); - exit(5); - } - - fd = open(fpath, O_RDWR); - if (fd == -1) { - perror("open"); - exit(6); - } - - if (write(fd, buf, sizeof (buf)) == -1) { - perror("write"); - exit(7); - } - close(fd); - - if (stat(fpath, &st) == -1) { - perror("stat"); - exit(8); - } - unlink(fpath); - - /* Verify SUID/SGID are dropped */ - res = st.st_mode & (0777 | S_ISUID | S_ISGID); - if (res != (mode & 0777)) { - fprintf(stderr, "stat(2) %o\n", res); - exit(9); - } -} +#include int main(int argc, char *argv[]) { - const char *name; + const char *name, *phase; mode_t extra; + struct stat st; - if (argc < 2) { + if (argc < 3) { fprintf(stderr, "Invalid argc\n"); exit(1); } @@ -127,7 +57,77 @@ main(int argc, char *argv[]) exit(1); } - test_stat_mode(extra); + const char *testdir = getenv("TESTDIR"); + if (!testdir) { + fprintf(stderr, "getenv(TESTDIR)\n"); + exit(1); + } + + umask(0); + if (stat(testdir, &st) == -1 && mkdir(testdir, 0777) == -1) { + perror("mkdir"); + exit(2); + } + + char fpath[1024]; + snprintf(fpath, sizeof (fpath), "%s/%s", testdir, name); + + + phase = argv[2]; + if (strcmp(phase, "PRECRASH") == 0) { + + /* clean up last run */ + unlink(fpath); + if (stat(fpath, &st) == 0) { + fprintf(stderr, "%s exists\n", fpath); + exit(3); + } + + int fd; + + fd = creat(fpath, 0777 | extra); + if (fd == -1) { + perror("creat"); + exit(4); + } + close(fd); + + if (setuid(65534) == -1) { + perror("setuid"); + exit(5); + } + + fd = open(fpath, O_RDWR); + if (fd == -1) { + perror("open"); + exit(6); + } + + const char buf[] = "test"; + if (write(fd, buf, sizeof (buf)) == -1) { + perror("write"); + exit(7); + } + close(fd); + + } else if (strcmp(phase, "REPLAY") == 0) { + /* created in PRECRASH run */ + } else { + fprintf(stderr, "Invalid phase %s\n", phase); + exit(1); + } + + if (stat(fpath, &st) == -1) { + perror("stat"); + exit(8); + } + + /* Verify SUID/SGID are dropped */ + mode_t res = st.st_mode & (0777 | S_ISUID | S_ISGID); + if (res != 0777) { + fprintf(stderr, "stat(2) %o\n", res); + exit(9); + } return (0); } diff --git a/tests/zfs-tests/tests/functional/suid/suid_write_to_none.ksh b/tests/zfs-tests/tests/functional/suid/suid_write_to_none.ksh index dd01978619f9..470350f960cf 100755 --- a/tests/zfs-tests/tests/functional/suid/suid_write_to_none.ksh +++ b/tests/zfs-tests/tests/functional/suid/suid_write_to_none.ksh @@ -47,6 +47,6 @@ function cleanup log_onexit cleanup log_note "Verify write(2) to regular file by non-owner" -log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "NONE" +log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "NONE" "PRECRASH" log_pass "Verify write(2) to regular file by non-owner passed" diff --git a/tests/zfs-tests/tests/functional/suid/suid_write_to_sgid.ksh b/tests/zfs-tests/tests/functional/suid/suid_write_to_sgid.ksh index 49ae2bd1b31e..3c95a402658e 100755 --- a/tests/zfs-tests/tests/functional/suid/suid_write_to_sgid.ksh +++ b/tests/zfs-tests/tests/functional/suid/suid_write_to_sgid.ksh @@ -47,6 +47,6 @@ function cleanup log_onexit cleanup log_note "Verify write(2) to SGID file by non-owner" -log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "SGID" +log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "SGID" "PRECRASH" log_pass "Verify write(2) to SGID file by non-owner passed" diff --git a/tests/zfs-tests/tests/functional/suid/suid_write_to_suid.ksh b/tests/zfs-tests/tests/functional/suid/suid_write_to_suid.ksh index 3983aad2e51d..4183cbeefc20 100755 --- a/tests/zfs-tests/tests/functional/suid/suid_write_to_suid.ksh +++ b/tests/zfs-tests/tests/functional/suid/suid_write_to_suid.ksh @@ -47,6 +47,6 @@ function cleanup log_onexit cleanup log_note "Verify write(2) to SUID file by non-owner" -log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "SUID" +log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "SUID" "PRECRASH" log_pass "Verify write(2) to SUID file by non-owner passed" diff --git a/tests/zfs-tests/tests/functional/suid/suid_write_to_suid_sgid.ksh b/tests/zfs-tests/tests/functional/suid/suid_write_to_suid_sgid.ksh index a058c7e7d4bc..f7a08a55fc4b 100755 --- a/tests/zfs-tests/tests/functional/suid/suid_write_to_suid_sgid.ksh +++ b/tests/zfs-tests/tests/functional/suid/suid_write_to_suid_sgid.ksh @@ -47,6 +47,6 @@ function cleanup log_onexit cleanup log_note "Verify write(2) to SUID/SGID file by non-owner" -log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "SUID_SGID" +log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "SUID_SGID" "PRECRASH" log_pass "Verify write(2) to SUID/SGID file by non-owner passed" diff --git a/tests/zfs-tests/tests/functional/suid/suid_write_zil_replay.ksh b/tests/zfs-tests/tests/functional/suid/suid_write_zil_replay.ksh new file mode 100755 index 000000000000..81f431f6b68b --- /dev/null +++ b/tests/zfs-tests/tests/functional/suid/suid_write_zil_replay.ksh @@ -0,0 +1,99 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +. $STF_SUITE/tests/functional/slog/slog.kshlib + +verify_runnable "global" + +function cleanup_fs +{ + cleanup +} + +log_assert "Verify ZIL replay results in correct SUID/SGID bits for unprivileged write to SUID/SGID files" +log_onexit cleanup_fs +log_must setup + +# +# 1. Create a file system (TESTFS) +# +log_must zpool destroy "$TESTPOOL" +log_must zpool create $TESTPOOL $VDEV log mirror $LDEV +log_must zfs set compression=on $TESTPOOL +log_must zfs create -o mountpoint="$TESTDIR" $TESTPOOL/$TESTFS + +# Make all the writes from suid_write_to_file.c sync +log_must zfs set sync=always "$TESTPOOL/$TESTFS" + +# +# This dd command works around an issue where ZIL records aren't created +# after freezing the pool unless a ZIL header already exists. Create a file +# synchronously to force ZFS to write one out. +# +log_must dd if=/dev/zero of=$TESTDIR/sync \ + conv=fdatasync,fsync bs=1 count=1 + +# +# 2. Freeze TESTFS +# +log_must zpool freeze $TESTPOOL + +# +# 3. Unprivileged write to a setuid file +# +log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "NONE" "PRECRASH" +log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "SUID" "PRECRASH" +log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "SGID" "PRECRASH" +log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "SUID_SGID" "PRECRASH" + +# +# 4. Unmount filesystem and export the pool +# +# At this stage TESTFS is empty again and frozen, the intent log contains +# a complete set of deltas to replay. +# +log_must zfs unmount $TESTPOOL/$TESTFS + +log_note "List transactions to replay:" +log_must zdb -iv $TESTPOOL/$TESTFS + +log_must zpool export $TESTPOOL + +# +# 5. Remount TESTFS +# +# Import the pool to unfreeze it and claim log blocks. It has to be +# `zpool import -f` because we can't write a frozen pool's labels! +# +log_must zpool import -f -d $VDIR $TESTPOOL + +log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "NONE" "REPLAY" +log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "SUID" "REPLAY" +log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "SGID" "REPLAY" +log_must $STF_SUITE/tests/functional/suid/suid_write_to_file "SUID_SGID" "REPLAY" + +log_pass From cd0e2380496e40924822b3fed71300ed48ab68d2 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Fri, 4 Feb 2022 09:36:46 -0800 Subject: [PATCH 066/109] ZTS: Update enospc_002_pos test case The on-disk cost of creating a snapshot or bookmark is sufficiently low that it is difficult to make it reliably fail even when the pool is "full". In order to avoid false positives remove these two checks from the test case. Reviewed-by: George Melikov Reviewed-by: John Kennedy Signed-off-by: Brian Behlendorf Closes #13060 --- tests/test-runner/bin/zts-report.py.in | 1 - tests/zfs-tests/tests/functional/no_space/enospc_002_pos.ksh | 3 --- 2 files changed, 4 deletions(-) diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index d306e04760c9..c1aacd54f6c5 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -242,7 +242,6 @@ maybe = { 'largest_pool/largest_pool_001_pos': ['FAIL', known_reason], 'mmp/mmp_on_uberblocks': ['FAIL', known_reason], 'pyzfs/pyzfs_unittest': ['SKIP', python_deps_reason], - 'no_space/enospc_002_pos': ['FAIL', enospc_reason], 'pool_checkpoint/checkpoint_discard_busy': ['FAIL', '11946'], 'projectquota/setup': ['SKIP', exec_reason], 'redundancy/redundancy_004_neg': ['FAIL', '7290'], diff --git a/tests/zfs-tests/tests/functional/no_space/enospc_002_pos.ksh b/tests/zfs-tests/tests/functional/no_space/enospc_002_pos.ksh index 6239d491e6eb..ffd14f4c550d 100755 --- a/tests/zfs-tests/tests/functional/no_space/enospc_002_pos.ksh +++ b/tests/zfs-tests/tests/functional/no_space/enospc_002_pos.ksh @@ -61,9 +61,6 @@ done log_mustnot_expect space zfs create $TESTPOOL/$TESTFS/subfs log_mustnot_expect space zfs clone $TESTPOOL/$TESTFS@snap $TESTPOOL/clone -log_mustnot_expect space zfs snapshot $TESTPOOL/$TESTFS@snap2 -log_mustnot_expect space zfs bookmark \ - $TESTPOOL/$TESTFS@snap $TESTPOOL/$TESTFS#bookmark log_must zfs send $TESTPOOL/$TESTFS@snap > $TEST_BASE_DIR/stream.$$ log_mustnot_expect space zfs receive $TESTPOOL/$TESTFS/recvd < $TEST_BASE_DIR/stream.$$ From d6b7903032507b0aace4780e2010a1cee24be246 Mon Sep 17 00:00:00 2001 From: Jorgen Lundman Date: Thu, 2 Sep 2021 22:51:29 +0900 Subject: [PATCH 067/109] autoconf: allow Release to contain hyphen To avoid clashing with tags and releases, we'll use "zfs-macOS". Meta: 1 Name: zfs-macOS Reviewed-by: John Kennedy Reviewed-by: Brian Behlendorf Reviewed-by: Tony Hutter Signed-off-by: Jorgen Lundman Closes #12437 --- config/zfs-meta.m4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/zfs-meta.m4 b/config/zfs-meta.m4 index b3c1befaac5d..1c9d246124d1 100644 --- a/config/zfs-meta.m4 +++ b/config/zfs-meta.m4 @@ -73,14 +73,14 @@ AC_DEFUN([ZFS_AC_META], [ if test ! -f ".nogitrelease" && git rev-parse --git-dir > /dev/null 2>&1; then _match="${ZFS_META_NAME}-${ZFS_META_VERSION}" _alias=$(git describe --match=${_match} 2>/dev/null) - _release=$(echo ${_alias}|cut -f3- -d'-'|sed 's/-/_/g') + _release=$(echo ${_alias}|sed "s/${ZFS_META_NAME}//"|cut -f3- -d'-'|sed 's/-/_/g') if test -n "${_release}"; then ZFS_META_RELEASE=${_release} _zfs_ac_meta_type="git describe" else _match="${ZFS_META_NAME}-${ZFS_META_VERSION}-${ZFS_META_RELEASE}" _alias=$(git describe --match=${_match} 2>/dev/null) - _release=$(echo ${_alias}|cut -f3- -d'-'|sed 's/-/_/g') + _release=$(echo ${_alias}|sed 's/${ZFS_META_NAME}//'|cut -f3- -d'-'|sed 's/-/_/g') if test -n "${_release}"; then ZFS_META_RELEASE=${_release} _zfs_ac_meta_type="git describe" From 745a7f78da95d1969613cd55fcb33df1b46c1e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Thu, 11 Nov 2021 21:27:37 +0100 Subject: [PATCH 068/109] Remove basename(1). Clean up/shorten some coreutils pipelines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Basenames that remain, in cmd/zed/zed.d/statechange-led.sh: dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')") vdev=$(basename "$ZEVENT_VDEV_PATH") I don't wanna interfere with #11988 scripts/zfs-tests.sh: SINGLETESTFILE=$(basename "$SINGLETEST") tests/zfs-tests/tests/functional/cli_user/zfs_list/zfs_list.kshlib: ACTUAL=$(basename $dataset) ACTUAL=$(basename $dataset) tests/zfs-tests/tests/functional/cli_user/zpool_iostat/ zpool_iostat_-c_homedir.ksh: typeset USER_SCRIPT=$(basename "$USER_SCRIPT_FULL") tests/zfs-tests/tests/functional/cli_user/zpool_iostat/ zpool_iostat_-c_searchpath.ksh: typeset CMD_1=$(basename "$SCRIPT_1") typeset CMD_2=$(basename "$SCRIPT_2") tests/zfs-tests/tests/functional/cli_user/zpool_status/ zpool_status_-c_homedir.ksh: typeset USER_SCRIPT=$(basename "$USER_SCRIPT_FULL") tests/zfs-tests/tests/functional/cli_user/zpool_status/ zpool_status_-c_searchpath.ksh typeset CMD_1=$(basename "$SCRIPT_1") typeset CMD_2=$(basename "$SCRIPT_2") tests/zfs-tests/tests/functional/migration/migration.cfg: export BNAME=`basename $TESTFILE` tests/zfs-tests/tests/perf/perf.shlib: typeset logbase="$(get_perf_output_dir)/$(basename \ tests/zfs-tests/tests/perf/perf.shlib: typeset logbase="$(get_perf_output_dir)/$(basename \ These are potentially Of Directories, where basename is actually useful Reviewed-by: Brian Behlendorf Reviewed-by: John Kennedy Signed-off-by: Ahelenia Ziemiańska Closes #12652 --- cmd/vdev_id/vdev_id | 11 +++++++---- cmd/zed/zed.d/all-syslog.sh | 2 +- cmd/zed/zed.d/generic-notify.sh | 2 +- cmd/zed/zed.d/zed-functions.sh | 6 +++--- cmd/zpool/zpool.d/dm-deps | 10 ++++------ cmd/zpool/zpool.d/iostat | 6 +++--- cmd/zpool/zpool.d/lsblk | 2 +- cmd/zpool/zpool.d/media | 15 ++++++--------- cmd/zpool/zpool.d/ses | 2 +- config/always-python.m4 | 2 +- config/always-pyzfs.m4 | 2 +- config/zfs-meta.m4 | 4 ++-- contrib/bpftrace/zfs-trace.sh | 3 ++- .../02zfsexpandknowledge/module-setup.sh.in | 16 ++++++---------- contrib/dracut/90zfs/module-setup.sh.in | 2 +- contrib/dracut/90zfs/parse-zfs.sh.in | 2 +- contrib/dracut/90zfs/zfs-generator.sh.in | 2 +- contrib/initramfs/scripts/zfs | 5 ++--- etc/init.d/zfs-import.in | 3 +-- etc/zfs/zfs-functions.in | 14 ++++++-------- rpm/generic/zfs-dkms.spec.in | 2 +- scripts/kmodtool | 2 +- scripts/zfs-tests.sh | 4 ++-- scripts/zfs.sh | 11 +++++++---- scripts/zimport.sh | 2 +- tests/zfs-tests/include/libtest.shlib | 8 +++----- .../reservation/reservation_021_neg.ksh | 2 +- 27 files changed, 67 insertions(+), 75 deletions(-) diff --git a/cmd/vdev_id/vdev_id b/cmd/vdev_id/vdev_id index 8cc4399a5668..7b5aab141997 100755 --- a/cmd/vdev_id/vdev_id +++ b/cmd/vdev_id/vdev_id @@ -596,7 +596,9 @@ enclosure_handler () { # DEVPATH=/sys/devices/pci0000:00/0000:00:03.0/0000:05:00.0/host0/subsystem/devices/0:0:0:0/scsi_generic/sg0 # Get the enclosure ID ("0:0:0:0") - ENC=$(basename $(readlink -m "/sys/$DEVPATH/../..")) + ENC="${DEVPATH%/*}" + ENC="${ENC%/*}" + ENC="${ENC##*/}" if [ ! -d "/sys/class/enclosure/$ENC" ] ; then # Not an enclosure, bail out return @@ -616,10 +618,11 @@ enclosure_handler () { # The PCI directory is two directories up from the port directory # /sys/devices/pci0000:00/0000:00:03.0/0000:05:00.0 - PCI_ID_LONG=$(basename $(readlink -m "/sys/$PORT_DIR/../..")) + PCI_ID_LONG="$(readlink -m "/sys/$PORT_DIR/../..")" + PCI_ID_LONG="${PCI_ID_LONG##*/}" # Strip down the PCI address from 0000:05:00.0 to 05:00.0 - PCI_ID=$(echo "$PCI_ID_LONG" | sed -r 's/^[0-9]+://g') + PCI_ID="${PCI_ID_LONG#[0-9]*:}" # Name our device according to vdev_id.conf (like "L0" or "U1"). NAME=$(awk "/channel/{if (\$1 == \"channel\" && \$2 == \"$PCI_ID\" && \ @@ -674,7 +677,7 @@ alias_handler () { link=$(echo "$link" | sed 's/p[0-9][0-9]*$//') fi # Check both the fully qualified and the base name of link. - for l in $link $(basename "$link") ; do + for l in $link ${link##*/} ; do if [ ! -z "$l" ]; then alias=$(awk -v var="$l" '($1 == "alias") && \ ($3 == var) \ diff --git a/cmd/zed/zed.d/all-syslog.sh b/cmd/zed/zed.d/all-syslog.sh index b07cf0f295ad..ea108c47b779 100755 --- a/cmd/zed/zed.d/all-syslog.sh +++ b/cmd/zed/zed.d/all-syslog.sh @@ -21,7 +21,7 @@ if [ "${ZED_SYSLOG_DISPLAY_GUIDS}" = "1" ]; then [ -n "${ZEVENT_VDEV_GUID}" ] && msg="${msg} vdev_guid=${ZEVENT_VDEV_GUID}" else [ -n "${ZEVENT_POOL}" ] && msg="${msg} pool='${ZEVENT_POOL}'" - [ -n "${ZEVENT_VDEV_PATH}" ] && msg="${msg} vdev=$(basename "${ZEVENT_VDEV_PATH}")" + [ -n "${ZEVENT_VDEV_PATH}" ] && msg="${msg} vdev=${ZEVENT_VDEV_PATH##*/}" fi # log pool state if state is anything other than 'ACTIVE' diff --git a/cmd/zed/zed.d/generic-notify.sh b/cmd/zed/zed.d/generic-notify.sh index 1db26980c1a0..9cf657e39970 100755 --- a/cmd/zed/zed.d/generic-notify.sh +++ b/cmd/zed/zed.d/generic-notify.sh @@ -23,7 +23,7 @@ # Rate-limit the notification based in part on the filename. # -rate_limit_tag="${ZEVENT_POOL};${ZEVENT_SUBCLASS};$(basename -- "$0")" +rate_limit_tag="${ZEVENT_POOL};${ZEVENT_SUBCLASS};${0##*/}" rate_limit_interval="${ZED_NOTIFY_INTERVAL_SECS}" zed_rate_limit "${rate_limit_tag}" "${rate_limit_interval}" || exit 3 diff --git a/cmd/zed/zed.d/zed-functions.sh b/cmd/zed/zed.d/zed-functions.sh index 9044922ac346..290f9150b43f 100644 --- a/cmd/zed/zed.d/zed-functions.sh +++ b/cmd/zed/zed.d/zed-functions.sh @@ -77,7 +77,7 @@ zed_log_msg() zed_log_err() { logger -p "${ZED_SYSLOG_PRIORITY}" -t "${ZED_SYSLOG_TAG}" -- "error:" \ - "$(basename -- "$0"):""${ZEVENT_EID:+" eid=${ZEVENT_EID}:"}" "$@" + "${0##*/}:""${ZEVENT_EID:+" eid=${ZEVENT_EID}:"}" "$@" } @@ -258,7 +258,7 @@ zed_notify_email() [ -n "${subject}" ] || return 1 if [ ! -r "${pathname}" ]; then zed_log_err \ - "$(basename "${ZED_EMAIL_PROG}") cannot read \"${pathname}\"" + "${ZED_EMAIL_PROG##*/} cannot read \"${pathname}\"" return 1 fi @@ -270,7 +270,7 @@ zed_notify_email() eval ${ZED_EMAIL_PROG} ${ZED_EMAIL_OPTS} < "${pathname}" >/dev/null 2>&1 rv=$? if [ "${rv}" -ne 0 ]; then - zed_log_err "$(basename "${ZED_EMAIL_PROG}") exit=${rv}" + zed_log_err "${ZED_EMAIL_PROG##*/} exit=${rv}" return 1 fi return 0 diff --git a/cmd/zpool/zpool.d/dm-deps b/cmd/zpool/zpool.d/dm-deps index ee39514e4d92..42af6a8d63cd 100755 --- a/cmd/zpool/zpool.d/dm-deps +++ b/cmd/zpool/zpool.d/dm-deps @@ -16,14 +16,12 @@ if [ -L "$dev" ] ; then dev=$(readlink "$dev") fi -dev=$(basename "$dev") +dev="${dev##*/}" val="" if [ -d "/sys/class/block/$dev/slaves" ] ; then - # ls -C: output in columns, no newlines - val=$(ls -C "/sys/class/block/$dev/slaves") - - # ls -C will print two spaces between files; change to one space. - val=$(echo "$val" | sed -r 's/[[:blank:]]+/ /g') + # ls -C: output in columns, no newlines, two spaces (change to one) + # shellcheck disable=SC2012 + val=$(ls -C "/sys/class/block/$dev/slaves" | tr -s '[:space:]' ' ') fi echo "dm-deps=$val" diff --git a/cmd/zpool/zpool.d/iostat b/cmd/zpool/zpool.d/iostat index 41a3acfae7a4..19be475e9b27 100755 --- a/cmd/zpool/zpool.d/iostat +++ b/cmd/zpool/zpool.d/iostat @@ -9,7 +9,7 @@ iostat: Show iostat values since boot (summary page). iostat-1s: Do a single 1-second iostat sample and show values. iostat-10s: Do a single 10-second iostat sample and show values." -script=$(basename "$0") +script="${0##*/}" if [ "$1" = "-h" ] ; then echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- exit @@ -42,7 +42,7 @@ else ${brief:+"-y"} \ ${interval:+"$interval"} \ ${interval:+"1"} \ - "$VDEV_UPATH" | awk NF | tail -n 2) + "$VDEV_UPATH" | grep -v '^$' | tail -n 2) fi @@ -61,7 +61,7 @@ fi cols=$(echo "$out" | head -n 1) # Get the values and tab separate them to make them cut-able. -vals=$(echo "$out" | tail -n 1 | sed -r 's/[[:blank:]]+/\t/g') +vals=$(echo "$out" | tail -n 1 | tr -s '[:space:]' '\t') i=0 for col in $cols ; do diff --git a/cmd/zpool/zpool.d/lsblk b/cmd/zpool/zpool.d/lsblk index 1cdef40494fe..919783a1c1bf 100755 --- a/cmd/zpool/zpool.d/lsblk +++ b/cmd/zpool/zpool.d/lsblk @@ -48,7 +48,7 @@ size: Show the disk capacity. vendor: Show the disk vendor. lsblk: Show the disk size, vendor, and model number." -script=$(basename "$0") +script="${0##*/}" if [ "$1" = "-h" ] ; then echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- diff --git a/cmd/zpool/zpool.d/media b/cmd/zpool/zpool.d/media index 5683cdc3c023..660f78b743fc 100755 --- a/cmd/zpool/zpool.d/media +++ b/cmd/zpool/zpool.d/media @@ -9,15 +9,12 @@ if [ "$1" = "-h" ] ; then fi if [ -b "$VDEV_UPATH" ]; then - device=$(basename "$VDEV_UPATH") - val=$(cat "/sys/block/$device/queue/rotational" 2>/dev/null) - if [ "$val" = "0" ]; then - MEDIA="ssd" - fi - - if [ "$val" = "1" ]; then - MEDIA="hdd" - fi + device="${VDEV_UPATH##*/}" + read -r val 2>/dev/null < "/sys/block/$device/queue/rotational" + case "$val" in + 0) MEDIA="ssd" ;; + 1) MEDIA="hdd" ;; + esac vpd_pg83="/sys/block/$device/device/vpd_pg83" if [ -f "$vpd_pg83" ]; then diff --git a/cmd/zpool/zpool.d/ses b/cmd/zpool/zpool.d/ses index b1836d676528..b51fe31894ab 100755 --- a/cmd/zpool/zpool.d/ses +++ b/cmd/zpool/zpool.d/ses @@ -11,7 +11,7 @@ fault_led: Show value of the disk enclosure slot fault LED. locate_led: Show value of the disk enclosure slot locate LED. ses: Show disk's enc, enc device, slot, and fault/locate LED values." -script=$(basename "$0") +script="${0##*/}" if [ "$1" = "-h" ] ; then echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- exit diff --git a/config/always-python.m4 b/config/always-python.m4 index 76b06fcd8488..5f47df424c27 100644 --- a/config/always-python.m4 +++ b/config/always-python.m4 @@ -28,7 +28,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYTHON], [ dnl # AM_PATH_PYTHON([], [], [:]) AS_IF([test -z "$PYTHON_VERSION"], [ - PYTHON_VERSION=$(basename $PYTHON | tr -cd 0-9.) + PYTHON_VERSION=$(echo ${PYTHON##*/} | tr -cd 0-9.) ]) PYTHON_MINOR=${PYTHON_VERSION#*\.} diff --git a/config/always-pyzfs.m4 b/config/always-pyzfs.m4 index fa39fd88519c..00e5d0e2cbbd 100644 --- a/config/always-pyzfs.m4 +++ b/config/always-pyzfs.m4 @@ -6,7 +6,7 @@ dnl # https://www.gnu.org/software/autoconf-archive/ax_python_module.html dnl # Required by ZFS_AC_CONFIG_ALWAYS_PYZFS. dnl # AC_DEFUN([ZFS_AC_PYTHON_MODULE], [ - PYTHON_NAME=$(basename $PYTHON) + PYTHON_NAME=${PYTHON##*/} AC_MSG_CHECKING([for $PYTHON_NAME module: $1]) AS_IF([$PYTHON -c "import $1" 2>/dev/null], [ AC_MSG_RESULT(yes) diff --git a/config/zfs-meta.m4 b/config/zfs-meta.m4 index 1c9d246124d1..20064a0fb595 100644 --- a/config/zfs-meta.m4 +++ b/config/zfs-meta.m4 @@ -73,14 +73,14 @@ AC_DEFUN([ZFS_AC_META], [ if test ! -f ".nogitrelease" && git rev-parse --git-dir > /dev/null 2>&1; then _match="${ZFS_META_NAME}-${ZFS_META_VERSION}" _alias=$(git describe --match=${_match} 2>/dev/null) - _release=$(echo ${_alias}|sed "s/${ZFS_META_NAME}//"|cut -f3- -d'-'|sed 's/-/_/g') + _release=$(echo ${_alias}|sed "s/${ZFS_META_NAME}//"|cut -f3- -d'-'|tr - _) if test -n "${_release}"; then ZFS_META_RELEASE=${_release} _zfs_ac_meta_type="git describe" else _match="${ZFS_META_NAME}-${ZFS_META_VERSION}-${ZFS_META_RELEASE}" _alias=$(git describe --match=${_match} 2>/dev/null) - _release=$(echo ${_alias}|sed 's/${ZFS_META_NAME}//'|cut -f3- -d'-'|sed 's/-/_/g') + _release=$(echo ${_alias}|sed 's/${ZFS_META_NAME}//'|cut -f3- -d'-'|tr - _) if test -n "${_release}"; then ZFS_META_RELEASE=${_release} _zfs_ac_meta_type="git describe" diff --git a/contrib/bpftrace/zfs-trace.sh b/contrib/bpftrace/zfs-trace.sh index 54f66f3ba3fd..0165335c474b 100755 --- a/contrib/bpftrace/zfs-trace.sh +++ b/contrib/bpftrace/zfs-trace.sh @@ -1,6 +1,7 @@ #!/bin/sh -ZVER=$(cut -f 1 -d '-' /sys/module/zfs/version) +read -r ZVER < /sys/module/zfs/version +ZVER="${ZVER%%-*}" KVER=$(uname -r) exec bpftrace \ diff --git a/contrib/dracut/02zfsexpandknowledge/module-setup.sh.in b/contrib/dracut/02zfsexpandknowledge/module-setup.sh.in index d21ab74cc0d0..a161fbf6f113 100755 --- a/contrib/dracut/02zfsexpandknowledge/module-setup.sh.in +++ b/contrib/dracut/02zfsexpandknowledge/module-setup.sh.in @@ -2,8 +2,8 @@ get_devtype() { local typ - typ=$(udevadm info --query=property --name="$1" | grep "^ID_FS_TYPE=" | sed 's|^ID_FS_TYPE=||') - if [ "$typ" = "" ] ; then + typ=$(udevadm info --query=property --name="$1" | sed -n 's|^ID_FS_TYPE=||p') + if [ -z "$typ" ] ; then typ=$(blkid -c /dev/null "$1" -o value -s TYPE) fi echo "$typ" @@ -36,7 +36,6 @@ find_zfs_block_devices() { local dev local mp local fstype - local pool local _ numfields="$(awk '{print NF; exit}' /proc/self/mountinfo)" if [ "$numfields" = "10" ] ; then @@ -47,10 +46,7 @@ find_zfs_block_devices() { # shellcheck disable=SC2086 while read -r ${fields?} ; do [ "$fstype" = "zfs" ] || continue - if [ "$mp" = "$1" ]; then - pool=$(echo "$dev" | cut -d / -f 1) - get_pool_devices "$pool" - fi + [ "$mp" = "$1" ] && get_pool_devices "${dev%%/*}" done < /proc/self/mountinfo } @@ -100,9 +96,9 @@ if [ -n "$hostonly" ]; then majmin=$(get_maj_min "$dev") if [ -d "/sys/dev/block/$majmin/slaves" ] ; then for _depdev in "/sys/dev/block/$majmin/slaves"/*; do - [[ -f $_depdev/dev ]] || continue - _depdev=/dev/$(basename "$_depdev") - _depdevname=$(udevadm info --query=property --name="$_depdev" | grep "^DEVNAME=" | sed 's|^DEVNAME=||') + [ -f "$_depdev/dev" ] || continue + _depdev="/dev/${_depdev##*/}" + _depdevname=$(udevadm info --query=property --name="$_depdev" | sed -n 's|^DEVNAME=||p') _depdevtype=$(get_devtype "$_depdevname") dinfo "zfsexpandknowledge: underlying block device backing ZFS dataset $mp: ${_depdevname//$'\n'/ }" array_contains "$_depdevname" "${host_devs[@]}" || host_devs+=("$_depdevname") diff --git a/contrib/dracut/90zfs/module-setup.sh.in b/contrib/dracut/90zfs/module-setup.sh.in index 8c2951dd3523..bee53522772f 100755 --- a/contrib/dracut/90zfs/module-setup.sh.in +++ b/contrib/dracut/90zfs/module-setup.sh.in @@ -68,8 +68,8 @@ install() { dracut_install @mounthelperdir@/mount.zfs dracut_install @udevdir@/vdev_id dracut_install awk - dracut_install basename dracut_install cut + dracut_install tr dracut_install head dracut_install @udevdir@/zvol_id inst_hook cmdline 95 "${moddir}/parse-zfs.sh" diff --git a/contrib/dracut/90zfs/parse-zfs.sh.in b/contrib/dracut/90zfs/parse-zfs.sh.in index fe786a880699..0f92f5c80cce 100755 --- a/contrib/dracut/90zfs/parse-zfs.sh.in +++ b/contrib/dracut/90zfs/parse-zfs.sh.in @@ -43,7 +43,7 @@ case "${root}" in root="${root#FILESYSTEM=}" root="zfs:${root#ZFS=}" # switch + with spaces because kernel cmdline does not allow us to quote parameters - root=$(printf '%s\n' "$root" | sed "s/+/ /g") + root=$(echo "$root" | tr '+' ' ') rootok=1 wait_for_zfs=1 diff --git a/contrib/dracut/90zfs/zfs-generator.sh.in b/contrib/dracut/90zfs/zfs-generator.sh.in index b57c64c688b1..e50b9530c4f0 100755 --- a/contrib/dracut/90zfs/zfs-generator.sh.in +++ b/contrib/dracut/90zfs/zfs-generator.sh.in @@ -89,7 +89,7 @@ else _zfs_generator_cb() { dset="${1}" mpnt="${2}" - unit="sysroot$(echo "$mpnt" | sed 's;/;-;g').mount" + unit="sysroot$(echo "$mpnt" | tr '/' '-').mount" { echo "[Unit]" diff --git a/contrib/initramfs/scripts/zfs b/contrib/initramfs/scripts/zfs index 6e39f139d845..814547b6fa0c 100644 --- a/contrib/initramfs/scripts/zfs +++ b/contrib/initramfs/scripts/zfs @@ -105,8 +105,7 @@ find_rootfs() find_pools() { pools=$("$@" 2> /dev/null | \ - grep -E "pool:|^[a-zA-Z0-9]" | \ - sed 's@.*: @@' | \ + sed -Ee '/pool:|^[a-zA-Z0-9]/!d' -e 's@.*: @@' | \ tr '\n' ';') echo "${pools%%;}" # Return without the last ';'. @@ -428,7 +427,7 @@ decrypt_fs() else # Temporarily setting "printk" to "7" allows the prompt to appear even when the "quiet" kernel option has been used echo "load-key" > /run/zfs_console_askpwd_cmd - storeprintk="$(awk '{print $1}' /proc/sys/kernel/printk)" + read -r storeprintk _ < /proc/sys/kernel/printk echo 7 > /proc/sys/kernel/printk $ZFS load-key "${ENCRYPTIONROOT}" echo "$storeprintk" > /proc/sys/kernel/printk diff --git a/etc/init.d/zfs-import.in b/etc/init.d/zfs-import.in index e4bc7b8339fc..130174f74d06 100755 --- a/etc/init.d/zfs-import.in +++ b/etc/init.d/zfs-import.in @@ -57,8 +57,7 @@ find_pools() local pools pools=$("$@" 2> /dev/null | \ - grep -E "pool:|^[a-zA-Z0-9]" | \ - sed 's@.*: @@' | \ + sed -Ee '/pool:|^[a-zA-Z0-9]/!d' -e 's@.*: @@' | \ sort | \ tr '\n' ';') diff --git a/etc/zfs/zfs-functions.in b/etc/zfs/zfs-functions.in index 10fb5b19a829..30441dc35d4b 100644 --- a/etc/zfs/zfs-functions.in +++ b/etc/zfs/zfs-functions.in @@ -348,7 +348,7 @@ read_mtab() # Unset all MTAB_* variables # shellcheck disable=SC2046 - unset $(env | grep ^MTAB_ | sed 's,=.*,,') + unset $(env | sed -e '/^MTAB_/!d' -e 's,=.*,,') while read -r fs mntpnt fstype opts rest; do if echo "$fs $mntpnt $fstype $opts" | grep -qE "$match"; then @@ -363,9 +363,8 @@ read_mtab() fs=$(/bin/echo "$fs" | sed 's,\\0,\\00,') # Remove 'unwanted' characters. - mntpnt=$(printf '%b\n' "$mntpnt" | sed -e 's,/,,g' \ - -e 's,-,,g' -e 's,\.,,g' -e 's, ,,g') - fs=$(printf '%b\n' "$fs") + mntpnt=$(printf '%b' "$mntpnt" | tr -d '/. -') + fs=$(printf '%b' "$fs") # Set the variable. eval export "MTAB_$mntpnt=\"$fs\"" @@ -377,8 +376,7 @@ in_mtab() { local mntpnt="$1" # Remove 'unwanted' characters. - mntpnt=$(printf '%b\n' "$mntpnt" | sed -e 's,/,,g' \ - -e 's,-,,g' -e 's,\.,,g' -e 's, ,,g') + mntpnt=$(printf '%b' "$mntpnt" | tr -d '/. -') local var var="$(eval echo "MTAB_$mntpnt")" @@ -394,7 +392,7 @@ read_fstab() # Unset all FSTAB_* variables # shellcheck disable=SC2046 - unset $(env | grep ^FSTAB_ | sed 's,=.*,,') + unset $(env | sed -e '/^FSTAB_/!d' -e 's,=.*,,') i=0 while read -r fs mntpnt fstype opts; do @@ -404,7 +402,7 @@ read_fstab() if echo "$fs $mntpnt $fstype $opts" | grep -qE "$match"; then eval export "FSTAB_dev_$i=$fs" - fs=$(printf '%b\n' "$fs" | sed 's,/,_,g') + fs=$(printf '%b' "$fs" | tr '/' '_') eval export "FSTAB_$i=$mntpnt" i=$((i + 1)) diff --git a/rpm/generic/zfs-dkms.spec.in b/rpm/generic/zfs-dkms.spec.in index aab1d9399077..02be716aa964 100644 --- a/rpm/generic/zfs-dkms.spec.in +++ b/rpm/generic/zfs-dkms.spec.in @@ -86,7 +86,7 @@ exit 1 # Are we doing an upgrade? if [ "$1" = "1" -o "$1" = "upgrade" ] ; then # Yes we are. Are we upgrading to a new ZFS version? - NEWEST_VER=$(dkms status zfs | sed 's/,//g' | sort -r -V | awk '/installed/{print $2; exit}') + NEWEST_VER=$(dkms status zfs | tr -d , | sort -r -V | awk '/installed/{print $2; exit}') if [ "$NEWEST_VER" != "%{version}" ] ; then # Yes, it's a new ZFS version. We'll uninstall the old module # later on in this script. diff --git a/scripts/kmodtool b/scripts/kmodtool index 26bacf5991d2..b1021596997e 100755 --- a/scripts/kmodtool +++ b/scripts/kmodtool @@ -446,7 +446,7 @@ print_rpmtemplate () myprog_help () { - echo "Usage: $(basename ${0}) [OPTIONS]" + echo "Usage: ${0##*/} [OPTIONS]" echo $'\n'"Creates a template to be used during kmod building" echo $'\n'"Available options:" echo " --filterfile -- filter the results with grep --file " diff --git a/scripts/zfs-tests.sh b/scripts/zfs-tests.sh index 60499e09e249..f871a51d34c2 100755 --- a/scripts/zfs-tests.sh +++ b/scripts/zfs-tests.sh @@ -95,7 +95,7 @@ cleanup_freebsd_loopback() { cleanup_linux_loopback() { for TEST_LOOPBACK in ${LOOPBACKS}; do - LOOP_DEV=$(basename "$TEST_LOOPBACK") + LOOP_DEV="${TEST_LOOPBACK##*/}" DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \ grep "${LOOP_DEV}" | cut -f1) @@ -615,7 +615,7 @@ if [ -z "${DISKS}" ]; then TEST_LOOPBACK=$(sudo "${LOSETUP}" -f) sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" || fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}" - BASELOOPBACK=$(basename "$TEST_LOOPBACK") + BASELOOPBACK="${TEST_LOOPBACK##*/}" DISKS="$DISKS $BASELOOPBACK" LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK" fi diff --git a/scripts/zfs.sh b/scripts/zfs.sh index 7870b8930cab..940c83ffa28f 100755 --- a/scripts/zfs.sh +++ b/scripts/zfs.sh @@ -91,7 +91,8 @@ check_modules_linux() { for KMOD in $KMOD_SPL $KMOD_ZAVL $KMOD_ZNVPAIR $KMOD_ZUNICODE $KMOD_ZCOMMON \ $KMOD_ZLUA $KMOD_ZZSTD $KMOD_ICP $KMOD_ZFS; do - NAME=$(basename "$KMOD" .ko) + NAME="${KMOD##*/}" + NAME="${NAME%.ko}" if lsmod | grep -E -q "^${NAME}"; then LOADED_MODULES="$LOADED_MODULES\t$NAME\n" @@ -172,7 +173,8 @@ load_modules_linux() { unload_module_linux() { KMOD=$1 - NAME=$(basename "$KMOD" .ko) + NAME="${KMOD##*/}" + NAME="${NAME%.ko}" FILE=$(modinfo "$KMOD" | awk '/^filename:/ {print $2}') VERSION=$(modinfo "$KMOD" | awk '/^version:/ {print $2}') @@ -198,8 +200,9 @@ unload_modules_freebsd() { unload_modules_linux() { for KMOD in $KMOD_ZFS $KMOD_ICP $KMOD_ZZSTD $KMOD_ZLUA $KMOD_ZCOMMON \ $KMOD_ZUNICODE $KMOD_ZNVPAIR $KMOD_ZAVL $KMOD_SPL; do - NAME=$(basename "$KMOD" .ko) - USE_COUNT=$(lsmod | grep -E "^${NAME} " | awk '{print $3}') + NAME="${KMOD##*/}" + NAME="${NAME%.ko}" + USE_COUNT=$(lsmod | awk '/^'"${NAME}"'/ {print $3}') if [ "$USE_COUNT" = "0" ] ; then unload_module_linux "$KMOD" || return 1 diff --git a/scripts/zimport.sh b/scripts/zimport.sh index 0e9c01182b8b..03c766cf36c2 100755 --- a/scripts/zimport.sh +++ b/scripts/zimport.sh @@ -486,7 +486,7 @@ for TAG in $POOL_TAGS; do "$POOL_DIR_COPY" || \ fail "Failed to copy $POOL_DIR_PRISTINE to $POOL_DIR_COPY" POOL_NAME=$($ZPOOL_CMD import -d "$POOL_DIR_COPY" | \ - awk '/pool:/ { print $2; exit 0 }') + awk '/pool:/ { print $2; exit }') if ! $ZPOOL_CMD import -N -d "$POOL_DIR_COPY" "$POOL_NAME" &>/dev/null; then diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib index 8a41b960f8d9..dd43b02a6868 100644 --- a/tests/zfs-tests/include/libtest.shlib +++ b/tests/zfs-tests/include/libtest.shlib @@ -252,7 +252,7 @@ function unmounted function splitline { - echo $1 | sed "s/,/ /g" + echo $1 | tr ',' ' ' } function default_setup @@ -1092,9 +1092,7 @@ function get_endslice # case "$(uname)" in Linux) endcyl=$(parted -s $DEV_DSKDIR/$disk -- unit cyl print | \ - grep "part${slice}" | \ - awk '{print $3}' | \ - sed 's,cyl,,') + awk "/part${slice}/"' {sub(/cyl/, "", $3); print $3}') ((endcyl = (endcyl + 1))) ;; FreeBSD) @@ -1461,7 +1459,7 @@ function is_shared_smb if datasetnonexists "$fs" ; then return 1 else - fs=$(echo $fs | sed 's@/@_@g') + fs=$(echo $fs | tr / _) fi if is_linux; then diff --git a/tests/zfs-tests/tests/functional/reservation/reservation_021_neg.ksh b/tests/zfs-tests/tests/functional/reservation/reservation_021_neg.ksh index c99a82c5db71..07da7e96306e 100755 --- a/tests/zfs-tests/tests/functional/reservation/reservation_021_neg.ksh +++ b/tests/zfs-tests/tests/functional/reservation/reservation_021_neg.ksh @@ -34,7 +34,7 @@ verify_runnable "both" -fs=$TESTPOOL/$TESTFS/$(basename $0).$$ +fs=$TESTPOOL/$TESTFS/${0##*/}.$$ function cleanup { From 0cb2d8a60b37d788db6799f2c504191c72b40e0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Sat, 11 Dec 2021 02:41:23 +0100 Subject: [PATCH 069/109] contrib/initrd hooks: properly quote @LIBFETCH_SONAME@ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bullseye shellcheck picks these up as SC2140, and it's right! @LIBFETCH_SONAME@ is already quoted, so dracut had "$d/"libcurl.so.4"" and i-t had ""libcurl.so.4"" Partially reverts 34eef3e9a7a74d24a59d016051d547afc55dbaa0 (#12760), which broke this Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #12835 --- contrib/dracut/90zfs/module-setup.sh.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/dracut/90zfs/module-setup.sh.in b/contrib/dracut/90zfs/module-setup.sh.in index bee53522772f..c27f905bfa8c 100755 --- a/contrib/dracut/90zfs/module-setup.sh.in +++ b/contrib/dracut/90zfs/module-setup.sh.in @@ -60,9 +60,10 @@ install() { # Fallback: Guess the path and include all matches dracut_install /usr/lib*/gcc/**/libgcc_s.so* fi + # shellcheck disable=SC2050 if [ @LIBFETCH_DYNAMIC@ != 0 ]; then for d in $libdirs; do - [ -e "$d"/@LIBFETCH_SONAME@ ] && dracut_install "$d"/@LIBFETCH_SONAME@ + [ -e "$d/"@LIBFETCH_SONAME@ ] && dracut_install "$d/"@LIBFETCH_SONAME@ done fi dracut_install @mounthelperdir@/mount.zfs From 765be3600603656aedb20ad6b0ad2f2ed5261cb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Thu, 6 Jan 2022 20:26:40 +0100 Subject: [PATCH 070/109] libfetch: unquote @LIBFETCH_SONAME@ subst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @LIBFETCH_SONAME@ is no longer quoted. The C define still is. Ref: 153f7c9f72082d7ef5ee27fcbec1bcb94ba88151 Ref: https://github.com/openzfs/zfs/pull/12835#discussion_r776833743 Reviewed-by: Brian Behlendorf Reviewed-by: Damian Szuberski Signed-off-by: Ahelenia Ziemiańska Closes #12922 --- config/user-libfetch.m4 | 6 +++--- contrib/dracut/90zfs/module-setup.sh.in | 4 ++-- contrib/initramfs/hooks/zfs.in | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/config/user-libfetch.m4 b/config/user-libfetch.m4 index f5149fc1a5d7..d961c6ca77a1 100644 --- a/config/user-libfetch.m4 +++ b/config/user-libfetch.m4 @@ -32,7 +32,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_LIBFETCH], [ have_libfetch=1 LIBFETCH_IS_FETCH=1 LIBFETCH_DYNAMIC=1 - LIBFETCH_SONAME='"libfetch.so.6"' + LIBFETCH_SONAME="libfetch.so.6" LIBFETCH_LIBS="-ldl" AC_MSG_RESULT([fetch(3)]) ], []) @@ -45,7 +45,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_LIBFETCH], [ LIBFETCH_IS_LIBCURL=1 if test "$(curl-config --built-shared)" = "yes"; then LIBFETCH_DYNAMIC=1 - LIBFETCH_SONAME='"libcurl.so.4"' + LIBFETCH_SONAME="libcurl.so.4" LIBFETCH_LIBS="-ldl" AC_MSG_RESULT([libcurl]) else @@ -67,5 +67,5 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_LIBFETCH], [ AC_DEFINE_UNQUOTED([LIBFETCH_IS_FETCH], [$LIBFETCH_IS_FETCH], [libfetch is fetch(3)]) AC_DEFINE_UNQUOTED([LIBFETCH_IS_LIBCURL], [$LIBFETCH_IS_LIBCURL], [libfetch is libcurl]) AC_DEFINE_UNQUOTED([LIBFETCH_DYNAMIC], [$LIBFETCH_DYNAMIC], [whether the chosen libfetch is to be loaded at run-time]) - AC_DEFINE_UNQUOTED([LIBFETCH_SONAME], [$LIBFETCH_SONAME], [soname of chosen libfetch]) + AC_DEFINE_UNQUOTED([LIBFETCH_SONAME], ["$LIBFETCH_SONAME"], [soname of chosen libfetch]) ]) diff --git a/contrib/dracut/90zfs/module-setup.sh.in b/contrib/dracut/90zfs/module-setup.sh.in index c27f905bfa8c..fbf32b6588a9 100755 --- a/contrib/dracut/90zfs/module-setup.sh.in +++ b/contrib/dracut/90zfs/module-setup.sh.in @@ -61,9 +61,9 @@ install() { dracut_install /usr/lib*/gcc/**/libgcc_s.so* fi # shellcheck disable=SC2050 - if [ @LIBFETCH_DYNAMIC@ != 0 ]; then + if [ @LIBFETCH_DYNAMIC@ -gt 0 ]; then for d in $libdirs; do - [ -e "$d/"@LIBFETCH_SONAME@ ] && dracut_install "$d/"@LIBFETCH_SONAME@ + [ -e "$d/@LIBFETCH_SONAME@" ] && dracut_install "$d/@LIBFETCH_SONAME@" done fi dracut_install @mounthelperdir@/mount.zfs diff --git a/contrib/initramfs/hooks/zfs.in b/contrib/initramfs/hooks/zfs.in index bdf169fd96c2..546841e6c36a 100755 --- a/contrib/initramfs/hooks/zfs.in +++ b/contrib/initramfs/hooks/zfs.in @@ -31,8 +31,8 @@ find /lib/ -type f -name "libgcc_s.so.[1-9]" | while read -r libgcc; do done # shellcheck disable=SC2050 -if [ @LIBFETCH_DYNAMIC@ != 0 ]; then - find /lib/ -name @LIBFETCH_SONAME@ | while read -r libfetch; do +if [ @LIBFETCH_DYNAMIC@ -gt 0 ]; then + find /lib/ -name "@LIBFETCH_SONAME@" | while read -r libfetch; do copy_exec "$libfetch" done fi From a61915e08687b70654290df7088928f42f829aea Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 4 Feb 2022 19:33:34 +0100 Subject: [PATCH 071/109] dsl_dir_tempreserve_impl: remove unused `deferred` variable The following commit moved the users of `deferred` into function dsl_pool_unreserved_space: commit d2734cce68cf740e015312314415f9034c67851c Author: Serapheim Dimitropoulos Date: Fri Dec 16 14:11:29 2016 -0800 OpenZFS 9166 - zfs storage pool checkpoint Reviewed-by: Brian Behlendorf Reviewed-by: Ryan Moeller Reviewed-by: George Melikov Signed-off-by: Christian Schwarz Closes #13056 --- module/zfs/dsl_dir.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/module/zfs/dsl_dir.c b/module/zfs/dsl_dir.c index 84caace4dbab..1101341451a5 100644 --- a/module/zfs/dsl_dir.c +++ b/module/zfs/dsl_dir.c @@ -1320,7 +1320,6 @@ dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree, * we're very close to full, this will allow a steady trickle of * removes to get through. */ - uint64_t deferred = 0; if (dd->dd_parent == NULL) { uint64_t avail = dsl_pool_unreserved_space(dd->dd_pool, (netfree) ? @@ -1340,7 +1339,7 @@ dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree, */ if (used_on_disk + est_inflight >= quota) { if (est_inflight > 0 || used_on_disk < quota || - (retval == ENOSPC && used_on_disk < quota + deferred)) + (retval == ENOSPC && used_on_disk < quota)) retval = ERESTART; dprintf_dd(dd, "failing: used=%lluK inflight = %lluK " "quota=%lluK tr=%lluK err=%d\n", From bb271d67e85ad90cda640e382daf3a15f6e82d9e Mon Sep 17 00:00:00 2001 From: Damian Szuberski Date: Tue, 8 Feb 2022 19:53:23 +0100 Subject: [PATCH 072/109] `mount.zfs -o zfsutil` leverages `zfs_mount_at()` Using `zfs_mount_at()` gives opportunity to properly propagate mountopts from what's stored in a pool to the `mount(2)` syscall invocation. It fixes cases when mount options are set to incorrect values and rectification is impossible (e. g. Linux initrd boot sequence in #7947). Moved debug information printing after all variables are initialized - printed text reflects what is passed to `mount(2)`. Reviewed-by: Brian Behlendorf Signed-off-by: szubersk Issue #7947 Closes #13021 --- cmd/mount_zfs/mount_zfs.c | 47 ++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/cmd/mount_zfs/mount_zfs.c b/cmd/mount_zfs/mount_zfs.c index de3833698a2b..55968ac88ffa 100644 --- a/cmd/mount_zfs/mount_zfs.c +++ b/cmd/mount_zfs/mount_zfs.c @@ -246,13 +246,6 @@ main(int argc, char **argv) } } - if (verbose) - (void) fprintf(stdout, gettext("mount.zfs:\n" - " dataset: \"%s\"\n mountpoint: \"%s\"\n" - " mountflags: 0x%lx\n zfsflags: 0x%lx\n" - " mountopts: \"%s\"\n mtabopts: \"%s\"\n"), - dataset, mntpoint, mntflags, zfsflags, mntopts, mtabopt); - if (mntflags & MS_REMOUNT) { nomtab = 1; remount = 1; @@ -275,7 +268,10 @@ main(int argc, char **argv) return (MOUNT_USAGE); } - zfs_adjust_mount_options(zhp, mntpoint, mntopts, mtabopt); + if (!zfsutil || sloppy || + libzfs_envvar_is_set("ZFS_MOUNT_HELPER")) { + zfs_adjust_mount_options(zhp, mntpoint, mntopts, mtabopt); + } /* treat all snapshots as legacy mount points */ if (zfs_get_type(zhp) == ZFS_TYPE_SNAPSHOT) @@ -293,12 +289,11 @@ main(int argc, char **argv) if (zfs_version == 0) { fprintf(stderr, gettext("unable to fetch " "ZFS version for filesystem '%s'\n"), dataset); + zfs_close(zhp); + libzfs_fini(g_zfs); return (MOUNT_SYSERR); } - zfs_close(zhp); - libzfs_fini(g_zfs); - /* * Legacy mount points may only be mounted using 'mount', never using * 'zfs mount'. However, since 'zfs mount' actually invokes 'mount' @@ -316,6 +311,8 @@ main(int argc, char **argv) "Use 'zfs set mountpoint=%s' or 'mount -t zfs %s %s'.\n" "See zfs(8) for more information.\n"), dataset, mntpoint, dataset, mntpoint); + zfs_close(zhp); + libzfs_fini(g_zfs); return (MOUNT_USAGE); } @@ -326,14 +323,38 @@ main(int argc, char **argv) "Use 'zfs set mountpoint=%s' or 'zfs mount %s'.\n" "See zfs(8) for more information.\n"), dataset, "legacy", dataset); + zfs_close(zhp); + libzfs_fini(g_zfs); return (MOUNT_USAGE); } + if (verbose) + (void) fprintf(stdout, gettext("mount.zfs:\n" + " dataset: \"%s\"\n mountpoint: \"%s\"\n" + " mountflags: 0x%lx\n zfsflags: 0x%lx\n" + " mountopts: \"%s\"\n mtabopts: \"%s\"\n"), + dataset, mntpoint, mntflags, zfsflags, mntopts, mtabopt); + if (!fake) { - error = mount(dataset, mntpoint, MNTTYPE_ZFS, - mntflags, mntopts); + if (zfsutil && !sloppy && + !libzfs_envvar_is_set("ZFS_MOUNT_HELPER")) { + error = zfs_mount_at(zhp, mntopts, mntflags, mntpoint); + if (error) { + (void) fprintf(stderr, "zfs_mount_at() failed: " + "%s", libzfs_error_description(g_zfs)); + zfs_close(zhp); + libzfs_fini(g_zfs); + return (MOUNT_SYSERR); + } + } else { + error = mount(dataset, mntpoint, MNTTYPE_ZFS, + mntflags, mntopts); + } } + zfs_close(zhp); + libzfs_fini(g_zfs); + if (error) { switch (errno) { case ENOENT: From 3b52ccd7d7f5e59a1306b7e904e0e244248f94d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20F=C3=BCl=C3=B6p?= Date: Wed, 9 Feb 2022 21:50:10 +0100 Subject: [PATCH 073/109] Linux 5.16 compat: don't use XSTATE_XSAVE to save FPU state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linux 5.16 moved XSTATE_XSAVE and XSTATE_XRESTORE out of our reach, so add our own XSAVE{,OPT,S} code and use it for Linux 5.16. Please note that this differs from previous behavior in that it won't handle exceptions created by XSAVE an XRSTOR. This is sensible for three reasons. - Exceptions during XSAVE and XRSTOR can only occur if the feature is not supported or enabled or the memory operand isn't aligned on a 64 byte boundary. If this happens something else went terribly wrong, and it may be better to stop execution. - Previously we just printed a warning and didn't handle the fault, this is arguable for the above reason. - All other *SAVE instruction also don't handle exceptions, so this at least aligns behavior. Finally add a test to catch such a regression in the future. Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf Signed-off-by: Attila Fülöp Closes #13042 Closes #13059 --- config/kernel-fpu.m4 | 43 +++++- config/toolchain-simd.m4 | 66 +++++++++ configure.ac | 1 + include/os/linux/kernel/linux/simd_x86.h | 125 +++++++++++++++++- module/zcommon/zfs_prop.c | 4 +- tests/runfiles/linux.run | 6 + tests/zfs-tests/tests/functional/Makefile.am | 1 + .../tests/functional/simd/Makefile.am | 2 + .../tests/functional/simd/simd_supported.ksh | 58 ++++++++ 9 files changed, 300 insertions(+), 6 deletions(-) create mode 100644 tests/zfs-tests/tests/functional/simd/Makefile.am create mode 100755 tests/zfs-tests/tests/functional/simd/simd_supported.ksh diff --git a/config/kernel-fpu.m4 b/config/kernel-fpu.m4 index faa64f1ec46b..06280239ddf0 100644 --- a/config/kernel-fpu.m4 +++ b/config/kernel-fpu.m4 @@ -1,9 +1,12 @@ -dnl # +dnl # dnl # Handle differences in kernel FPU code. dnl # dnl # Kernel dnl # 5.16: XCR code put into asm/fpu/xcr.h -dnl # HAVE_KERNEL_FPU_XCR_HEADER +dnl # HAVE_KERNEL_FPU_XCR_HEADER +dnl # +dnl # XSTATE_XSAVE and XSTATE_XRESTORE aren't accessible any more +dnl # HAVE_KERNEL_FPU_XSAVE_INTERNAL dnl # dnl # 5.0: Wrappers have been introduced to save/restore the FPU state. dnl # This change was made to the 4.19.38 and 4.14.120 LTS kernels. @@ -107,6 +110,36 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FPU], [ struct fxregs_state *fxr __attribute__ ((unused)) = &st->fxsave; struct xregs_state *xr __attribute__ ((unused)) = &st->xsave; ]) + + ZFS_LINUX_TEST_SRC([fpu_xsave_internal], [ + #include + #if defined(__x86_64) || defined(__x86_64__) || \ + defined(__i386) || defined(__i386__) + #if !defined(__x86) + #define __x86 + #endif + #endif + + #if !defined(__x86) + #error Unsupported architecture + #endif + + #include + #ifdef HAVE_KERNEL_FPU_API_HEADER + #include + #include + #else + #include + #include + #endif + + ],[ + struct fpu *fpu = ¤t->thread.fpu; + union fpregs_state *st = &fpu->fpstate->regs; + struct fregs_state *fr __attribute__ ((unused)) = &st->fsave; + struct fxregs_state *fxr __attribute__ ((unused)) = &st->fxsave; + struct xregs_state *xr __attribute__ ((unused)) = &st->xsave; + ]) ]) AC_DEFUN([ZFS_AC_KERNEL_FPU], [ @@ -139,7 +172,13 @@ AC_DEFUN([ZFS_AC_KERNEL_FPU], [ AC_DEFINE(HAVE_KERNEL_FPU_INTERNAL, 1, [kernel fpu internal]) ],[ + ZFS_LINUX_TEST_RESULT([fpu_xsave_internal], [ + AC_MSG_RESULT(internal with internal XSAVE) + AC_DEFINE(HAVE_KERNEL_FPU_XSAVE_INTERNAL, 1, + [kernel fpu and XSAVE internal]) + ],[ AC_MSG_RESULT(unavailable) + ]) ]) ]) ]) diff --git a/config/toolchain-simd.m4 b/config/toolchain-simd.m4 index 1153cd6941a8..061576fd94e3 100644 --- a/config/toolchain-simd.m4 +++ b/config/toolchain-simd.m4 @@ -24,6 +24,9 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD], [ ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AES ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_PCLMULQDQ ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE + ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE + ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT + ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES ;; esac ]) @@ -422,3 +425,66 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE], [ AC_MSG_RESULT([no]) ]) ]) + +dnl # +dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE +dnl # +AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE], [ + AC_MSG_CHECKING([whether host toolchain supports XSAVE]) + + AC_LINK_IFELSE([AC_LANG_SOURCE([ + [ + void main() + { + char b[4096] __attribute__ ((aligned (64))); + __asm__ __volatile__("xsave %[b]\n" : : [b] "m" (*b) : "memory"); + } + ]])], [ + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_XSAVE], 1, [Define if host toolchain supports XSAVE]) + ], [ + AC_MSG_RESULT([no]) + ]) +]) + +dnl # +dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT +dnl # +AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT], [ + AC_MSG_CHECKING([whether host toolchain supports XSAVEOPT]) + + AC_LINK_IFELSE([AC_LANG_SOURCE([ + [ + void main() + { + char b[4096] __attribute__ ((aligned (64))); + __asm__ __volatile__("xsaveopt %[b]\n" : : [b] "m" (*b) : "memory"); + } + ]])], [ + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_XSAVEOPT], 1, [Define if host toolchain supports XSAVEOPT]) + ], [ + AC_MSG_RESULT([no]) + ]) +]) + +dnl # +dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES +dnl # +AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES], [ + AC_MSG_CHECKING([whether host toolchain supports XSAVES]) + + AC_LINK_IFELSE([AC_LANG_SOURCE([ + [ + void main() + { + char b[4096] __attribute__ ((aligned (64))); + __asm__ __volatile__("xsaves %[b]\n" : : [b] "m" (*b) : "memory"); + } + ]])], [ + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_XSAVES], 1, [Define if host toolchain supports XSAVES]) + ], [ + AC_MSG_RESULT([no]) + ]) +]) diff --git a/configure.ac b/configure.ac index ebc7b276a640..2671434afc72 100644 --- a/configure.ac +++ b/configure.ac @@ -383,6 +383,7 @@ AC_CONFIG_FILES([ tests/zfs-tests/tests/functional/rootpool/Makefile tests/zfs-tests/tests/functional/rsend/Makefile tests/zfs-tests/tests/functional/scrub_mirror/Makefile + tests/zfs-tests/tests/functional/simd/Makefile tests/zfs-tests/tests/functional/slog/Makefile tests/zfs-tests/tests/functional/snapshot/Makefile tests/zfs-tests/tests/functional/snapused/Makefile diff --git a/include/os/linux/kernel/linux/simd_x86.h b/include/os/linux/kernel/linux/simd_x86.h index f2ae0fcbc21a..7c806bf2c6fa 100644 --- a/include/os/linux/kernel/linux/simd_x86.h +++ b/include/os/linux/kernel/linux/simd_x86.h @@ -136,9 +136,27 @@ * When the kernel_fpu_* symbols are unavailable then provide our own * versions which allow the FPU to be safely used. */ +#if defined(HAVE_KERNEL_FPU_INTERNAL) || defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) + +#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) +/* + * Some sanity checks. + * HAVE_KERNEL_FPU_INTERNAL and HAVE_KERNEL_FPU_XSAVE_INTERNAL are exclusive. + */ #if defined(HAVE_KERNEL_FPU_INTERNAL) +#error "HAVE_KERNEL_FPU_INTERNAL and HAVE_KERNEL_FPU_XSAVE_INTERNAL defined" +#endif +/* + * For kernels >= 5.16 we have to use inline assembly with the XSAVE{,OPT,S} + * instructions, so we need the toolchain to support at least XSAVE. + */ +#if !defined(HAVE_XSAVE) +#error "Toolchain needs to support the XSAVE assembler instruction" +#endif +#endif #include +#include extern union fpregs_state **zfs_kfpu_fpregs; @@ -191,7 +209,9 @@ kfpu_init(void) } #define kfpu_allowed() 1 +#if defined(HAVE_KERNEL_FPU_INTERNAL) #define ex_handler_fprestore ex_handler_default +#endif /* * FPU save and restore instructions. @@ -206,6 +226,7 @@ kfpu_init(void) #define kfpu_fxsr_clean(rval) __asm("fnclex; emms; fildl %P[addr]" \ : : [addr] "m" (rval)); +#if defined(HAVE_KERNEL_FPU_INTERNAL) static inline void kfpu_save_xsave(struct xregs_state *addr, uint64_t mask) { @@ -217,6 +238,21 @@ kfpu_save_xsave(struct xregs_state *addr, uint64_t mask) XSTATE_XSAVE(addr, low, hi, err); WARN_ON_ONCE(err); } +#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */ + +#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) +#define kfpu_do_xsave(instruction, addr, mask) \ +{ \ + uint32_t low, hi; \ + \ + low = mask; \ + hi = (uint64_t)(mask) >> 32; \ + __asm(instruction " %[dst]\n\t" \ + : \ + : [dst] "m" (*(addr)), "a" (low), "d" (hi) \ + : "memory"); \ +} +#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */ static inline void kfpu_save_fxsr(struct fxregs_state *addr) @@ -233,6 +269,7 @@ kfpu_save_fsave(struct fregs_state *addr) kfpu_fnsave(addr); } +#if defined(HAVE_KERNEL_FPU_INTERNAL) static inline void kfpu_begin(void) { @@ -250,7 +287,6 @@ kfpu_begin(void) * FPU state to be correctly preserved and restored. */ union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()]; - if (static_cpu_has(X86_FEATURE_XSAVE)) { kfpu_save_xsave(&state->xsave, ~0); } else if (static_cpu_has(X86_FEATURE_FXSR)) { @@ -259,7 +295,50 @@ kfpu_begin(void) kfpu_save_fsave(&state->fsave); } } +#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */ + +#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) +static inline void +kfpu_begin(void) +{ + /* + * Preemption and interrupts must be disabled for the critical + * region where the FPU state is being modified. + */ + preempt_disable(); + local_irq_disable(); + + /* + * The current FPU registers need to be preserved by kfpu_begin() + * and restored by kfpu_end(). They are stored in a dedicated + * per-cpu variable, not in the task struct, this allows any user + * FPU state to be correctly preserved and restored. + */ + union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()]; +#if defined(HAVE_XSAVES) + if (static_cpu_has(X86_FEATURE_XSAVES)) { + kfpu_do_xsave("xsaves", &state->xsave, ~0); + goto out; + } +#endif +#if defined(HAVE_XSAVEOPT) + if (static_cpu_has(X86_FEATURE_XSAVEOPT)) { + kfpu_do_xsave("xsaveopt", &state->xsave, ~0); + goto out; + } +#endif + if (static_cpu_has(X86_FEATURE_XSAVE)) { + kfpu_do_xsave("xsave", &state->xsave, ~0); + } else if (static_cpu_has(X86_FEATURE_FXSR)) { + kfpu_save_fxsr(&state->fxsave); + } else { + kfpu_save_fsave(&state->fsave); + } +out: +} +#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */ +#if defined(HAVE_KERNEL_FPU_INTERNAL) static inline void kfpu_restore_xsave(struct xregs_state *addr, uint64_t mask) { @@ -269,6 +348,21 @@ kfpu_restore_xsave(struct xregs_state *addr, uint64_t mask) hi = mask >> 32; XSTATE_XRESTORE(addr, low, hi); } +#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */ + +#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) +#define kfpu_do_xrstor(instruction, addr, mask) \ +{ \ + uint32_t low, hi; \ + \ + low = mask; \ + hi = (uint64_t)(mask) >> 32; \ + __asm(instruction " %[src]" \ + : \ + : [src] "m" (*(addr)), "a" (low), "d" (hi) \ + : "memory"); \ +} +#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */ static inline void kfpu_restore_fxsr(struct fxregs_state *addr) @@ -294,6 +388,7 @@ kfpu_restore_fsave(struct fregs_state *addr) kfpu_frstor(addr); } +#if defined(HAVE_KERNEL_FPU_INTERNAL) static inline void kfpu_end(void) { @@ -310,6 +405,32 @@ kfpu_end(void) local_irq_enable(); preempt_enable(); } +#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */ + +#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) +static inline void +kfpu_end(void) +{ + union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()]; +#if defined(HAVE_XSAVES) + if (static_cpu_has(X86_FEATURE_XSAVES)) { + kfpu_do_xrstor("xrstors", &state->xsave, ~0); + goto out; + } +#endif + if (static_cpu_has(X86_FEATURE_XSAVE)) { + kfpu_do_xrstor("xrstor", &state->xsave, ~0); + } else if (static_cpu_has(X86_FEATURE_FXSR)) { + kfpu_save_fxsr(&state->fxsave); + } else { + kfpu_save_fsave(&state->fsave); + } +out: + local_irq_enable(); + preempt_enable(); + +} +#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */ #else @@ -322,7 +443,7 @@ kfpu_end(void) #define kfpu_init() 0 #define kfpu_fini() ((void) 0) -#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */ +#endif /* defined(HAVE_KERNEL_FPU_INTERNAL || HAVE_KERNEL_FPU_XSAVE_INTERNAL) */ #endif /* defined(KERNEL_EXPORTS_X86_FPU) */ /* diff --git a/module/zcommon/zfs_prop.c b/module/zcommon/zfs_prop.c index d17321990809..5f88bd02089d 100644 --- a/module/zcommon/zfs_prop.c +++ b/module/zcommon/zfs_prop.c @@ -996,10 +996,10 @@ zfs_prop_align_right(zfs_prop_t prop) #include -#if defined(HAVE_KERNEL_FPU_INTERNAL) +#if defined(HAVE_KERNEL_FPU_INTERNAL) || defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) union fpregs_state **zfs_kfpu_fpregs; EXPORT_SYMBOL(zfs_kfpu_fpregs); -#endif /* HAVE_KERNEL_FPU_INTERNAL */ +#endif /* HAVE_KERNEL_FPU_INTERNAL || HAVE_KERNEL_FPU_XSAVE_INTERNAL */ static int __init zcommon_init(void) diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index b7831c3acfd1..5a9fbe994611 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -151,6 +151,12 @@ tags = ['functional', 'projectquota'] tests = ['send_realloc_dnode_size', 'send_encrypted_files'] tags = ['functional', 'rsend'] +[tests/functional/simd:Linux] +pre = +post = +tests = ['simd_supported'] +tags = ['functional', 'simd'] + [tests/functional/snapshot:Linux] tests = ['snapshot_015_pos', 'snapshot_016_pos'] tags = ['functional', 'snapshot'] diff --git a/tests/zfs-tests/tests/functional/Makefile.am b/tests/zfs-tests/tests/functional/Makefile.am index 137cddd5f784..fd586ecee518 100644 --- a/tests/zfs-tests/tests/functional/Makefile.am +++ b/tests/zfs-tests/tests/functional/Makefile.am @@ -88,5 +88,6 @@ SUBDIRS = \ if BUILD_LINUX SUBDIRS += \ + simd \ tmpfile endif diff --git a/tests/zfs-tests/tests/functional/simd/Makefile.am b/tests/zfs-tests/tests/functional/simd/Makefile.am new file mode 100644 index 000000000000..bfc28868024a --- /dev/null +++ b/tests/zfs-tests/tests/functional/simd/Makefile.am @@ -0,0 +1,2 @@ +pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/simd +dist_pkgdata_SCRIPTS = simd_supported.ksh diff --git a/tests/zfs-tests/tests/functional/simd/simd_supported.ksh b/tests/zfs-tests/tests/functional/simd/simd_supported.ksh new file mode 100755 index 000000000000..d88bc582bf08 --- /dev/null +++ b/tests/zfs-tests/tests/functional/simd/simd_supported.ksh @@ -0,0 +1,58 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2022 by Attila Fülöp +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# Make sure we have SIMD support, so it will not go away without notice +# +# STRATEGY: +# 1. Test if we are running on a Linux x86 system with SSE support +# 2. If so, check if the zfs_fletcher_4_impl module parameter contains +# a sse implementation +# 3. If not fail the test, otherwise pass it + +log_note "Testing if we support SIMD instructions (Linux x86 only)" + +if !is_linux; then + log_unsupported "Not a Linux System" +fi + +case "$(uname -m)" in +i386|i686|x86_64) + typeset -R modparam="/sys/module/zcommon/parameters/zfs_fletcher_4_impl" + if cat /proc/cpuinfo | awk '/^flags/ {print; exit;}' | grep -q sse; then + log_must grep -q sse "$modparam" + log_pass "SIMD instructions supported" + else + log_unsupported "No FPU present" + fi + ;; +*) + log_unsupported "Not a x86 CPU" + ;; +esac From 77ae804f9eb162c123a62cea3bdefec7e7ca9459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Thu, 23 Dec 2021 21:57:44 +0100 Subject: [PATCH 074/109] module: Makefile: flatten subdir loop, use $PWD instead of `pwd` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Issue #12899 --- module/Makefile.in | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/module/Makefile.in b/module/Makefile.in index b47ee73b87cd..0dd9df501782 100644 --- a/module/Makefile.in +++ b/module/Makefile.in @@ -51,10 +51,8 @@ endif FMAKE = env -u MAKEFLAGS make $(FMAKEFLAGS) modules-Linux: - list='$(SUBDIR_TARGETS)'; for targetdir in $$list; do \ - $(MAKE) -C $$targetdir; \ - done - $(MAKE) -C @LINUX_OBJ@ M=`pwd` @KERNEL_MAKE@ CONFIG_ZFS=m modules + list='$(SUBDIR_TARGETS)'; for td in $$list; do $(MAKE) -C $$td; done + $(MAKE) -C @LINUX_OBJ@ M="$$PWD" @KERNEL_MAKE@ CONFIG_ZFS=m modules modules-FreeBSD: +$(FMAKE) @@ -67,7 +65,7 @@ modules: modules-@ac_system@ clean-Linux: @# Only cleanup the kernel build directories when CONFIG_KERNEL @# is defined. This indicates that kernel modules should be built. -@CONFIG_KERNEL_TRUE@ $(MAKE) -C @LINUX_OBJ@ M=`pwd` @KERNEL_MAKE@ clean +@CONFIG_KERNEL_TRUE@ $(MAKE) -C @LINUX_OBJ@ M="$$PWD" @KERNEL_MAKE@ clean if [ -f @LINUX_SYMBOLS@ ]; then $(RM) @LINUX_SYMBOLS@; fi if [ -f Module.markers ]; then $(RM) Module.markers; fi @@ -81,7 +79,7 @@ clean: clean-@ac_system@ modules_install-Linux: @# Install the kernel modules - $(MAKE) -C @LINUX_OBJ@ M=`pwd` modules_install \ + $(MAKE) -C @LINUX_OBJ@ M="$$PWD" modules_install \ INSTALL_MOD_PATH=$(INSTALL_MOD_PATH) \ INSTALL_MOD_DIR=$(INSTALL_MOD_DIR) \ KERNELRELEASE=@LINUX_VERSION@ From 52aae04c6a0a88d6af7b1cda27694414ae202822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Mon, 27 Dec 2021 02:21:38 +0100 Subject: [PATCH 075/109] module: Makefile: simplify clean and install jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #12979 --- module/Makefile.in | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/module/Makefile.in b/module/Makefile.in index 0dd9df501782..9cd075e6859c 100644 --- a/module/Makefile.in +++ b/module/Makefile.in @@ -67,10 +67,8 @@ clean-Linux: @# is defined. This indicates that kernel modules should be built. @CONFIG_KERNEL_TRUE@ $(MAKE) -C @LINUX_OBJ@ M="$$PWD" @KERNEL_MAKE@ clean - if [ -f @LINUX_SYMBOLS@ ]; then $(RM) @LINUX_SYMBOLS@; fi - if [ -f Module.markers ]; then $(RM) Module.markers; fi - - find . -name '*.ur-safe' -type f -print | xargs $(RM) + $(RM) @LINUX_SYMBOLS@ Module.markers + find . -name '*.ur-safe' -type f -delete clean-FreeBSD: +$(FMAKE) clean @@ -86,7 +84,7 @@ modules_install-Linux: @# Remove extraneous build products when packaging kmoddir=$(INSTALL_MOD_PATH)/lib/modules/@LINUX_VERSION@; \ if [ -n "$(DESTDIR)" ]; then \ - find $$kmoddir -name 'modules.*' | xargs $(RM); \ + find $$kmoddir -name 'modules.*' -delete; \ fi @# Debian ships tiny fake System.map files that are @# syntactically valid but just say From c7fcf0091731e904aa26c1375348107b5ada984b Mon Sep 17 00:00:00 2001 From: Peter Levine Date: Wed, 2 Feb 2022 00:44:59 -0500 Subject: [PATCH 076/109] Add support for $KERNEL_{CC,LD,LLVM} variables Currently, $(CC), $(LD), and $(LLVM) variables aren't passed to kbuild while building modules. This causes modules to build with the default GNU GCC toolchain and prevents experimenting with other toolchains such as CLANG/LLVM. It can also lead to build failure if the CFLAGS/LDFLAGS passed are incompatible with gcc/ld. Pass $KERNEL_CC, $KERNEL_LD, and $KERNEL_LLVM as $(CC), $(LD), and $(LLVM), respectively, to kbuild for each that is defined in the environment. This should take care of the majority of alternative toolchain use cases. Reviewed-by: Damian Szuberski Reviewed-by: Brian Behlendorf Signed-off-by: Peter Levine Closes #13046 --- config/kernel.m4 | 8 +++++++- module/Makefile.in | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/config/kernel.m4 b/config/kernel.m4 index 1adec975c0bf..b918f045341b 100644 --- a/config/kernel.m4 +++ b/config/kernel.m4 @@ -601,9 +601,15 @@ dnl # dnl # Used internally by ZFS_LINUX_TEST_{COMPILE,MODPOST} dnl # AC_DEFUN([ZFS_LINUX_COMPILE], [ + AC_ARG_VAR([KERNEL_CC], [C compiler for + building kernel modules]) + AC_ARG_VAR([KERNEL_LD], [Linker for + building kernel modules]) + AC_ARG_VAR([KERNEL_LLVM], [Binary option to + build kernel modules with LLVM/CLANG toolchain]) AC_TRY_COMMAND([ KBUILD_MODPOST_NOFINAL="$5" KBUILD_MODPOST_WARN="$6" - make modules -k -j$TEST_JOBS -C $LINUX_OBJ $ARCH_UM + make modules -k -j$TEST_JOBS ${KERNEL_CC:+CC=$KERNEL_CC} ${KERNEL_LD:+LD=$KERNEL_LD} ${KERNEL_LLVM:+LLVM=$KERNEL_LLVM} -C $LINUX_OBJ $ARCH_UM M=$PWD/$1 >$1/build.log 2>&1]) AS_IF([AC_TRY_COMMAND([$2])], [$3], [$4]) ]) diff --git a/module/Makefile.in b/module/Makefile.in index 9cd075e6859c..b6338430e812 100644 --- a/module/Makefile.in +++ b/module/Makefile.in @@ -52,7 +52,9 @@ FMAKE = env -u MAKEFLAGS make $(FMAKEFLAGS) modules-Linux: list='$(SUBDIR_TARGETS)'; for td in $$list; do $(MAKE) -C $$td; done - $(MAKE) -C @LINUX_OBJ@ M="$$PWD" @KERNEL_MAKE@ CONFIG_ZFS=m modules + $(MAKE) -C @LINUX_OBJ@ $(if @KERNEL_CC@,CC=@KERNEL_CC@) \ + $(if @KERNEL_LD@,LD=@KERNEL_LD@) $(if @KERNEL_LLVM@,LLVM=@KERNEL_LLVM@) \ + M="$$PWD" @KERNEL_MAKE@ CONFIG_ZFS=m modules modules-FreeBSD: +$(FMAKE) From 2681f8a5b858cf44b293ec3f6918dded60ac9c7f Mon Sep 17 00:00:00 2001 From: Damian Szuberski Date: Tue, 8 Feb 2022 17:17:37 -0500 Subject: [PATCH 077/109] Propagate KERNEL_* to *.spec Reviewed-by: Brian Behlendorf Authored-by: Damian Szuberski Signed-off-by: Peter Levine Closes #13046 --- config/zfs-build.m4 | 3 +++ rpm/generic/zfs-kmod.spec.in | 5 ++++- rpm/redhat/zfs-kmod.spec.in | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/config/zfs-build.m4 b/config/zfs-build.m4 index 27041c054c26..126f78476bd4 100644 --- a/config/zfs-build.m4 +++ b/config/zfs-build.m4 @@ -368,6 +368,9 @@ AC_DEFUN([ZFS_AC_RPM], [ RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernels $(LINUX_VERSION)"' RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "ksrc $(LINUX)"' RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kobj $(LINUX_OBJ)"' + RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_cc KERNEL_CC=$(KERNEL_CC)"' + RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_ld KERNEL_LD=$(KERNEL_LD)"' + RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_llvm KERNEL_LLVM=$(KERNEL_LLVM)"' ]) RPM_DEFINE_DKMS='' diff --git a/rpm/generic/zfs-kmod.spec.in b/rpm/generic/zfs-kmod.spec.in index 1692be1a72e6..53b1e1385159 100644 --- a/rpm/generic/zfs-kmod.spec.in +++ b/rpm/generic/zfs-kmod.spec.in @@ -140,7 +140,10 @@ for kernel_version in %{?kernel_versions}; do --with-linux=%{ksrc} \ --with-linux-obj=%{kobj} \ %{debug} \ - %{debuginfo} + %{debuginfo} \ + %{?kernel_cc} \ + %{?kernel_ld} \ + %{?kernel_llvm} make %{?_smp_mflags} cd .. done diff --git a/rpm/redhat/zfs-kmod.spec.in b/rpm/redhat/zfs-kmod.spec.in index eb93aeeb2efe..7b74fdc51f43 100644 --- a/rpm/redhat/zfs-kmod.spec.in +++ b/rpm/redhat/zfs-kmod.spec.in @@ -69,7 +69,10 @@ fi --with-linux=%{ksrc} \ --with-linux-obj=%{kobj} \ %{debug} \ - %{debuginfo} + %{debuginfo} \ + %{?kernel_cc} \ + %{?kernel_ld} \ + %{?kernel_llvm} make %{?_smp_mflags} %install From 5c19af07d4dfb92cb5ff3ceae9867224203b04eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20F=C3=BCl=C3=B6p?= Date: Wed, 9 Feb 2022 23:38:33 +0100 Subject: [PATCH 078/109] Receive checks should allow unencrypted child datasets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dmu_recv_begin_check() unconditionally sets the DS_HOLD_FLAG_DECRYPT flag before calling dsl_dataset_hold_flags(). If the key on the receiving side isn't loaded or the send stream contains embedded blocks, the receive check fails for a stream which is perfectly valid and could be received without any problem. This seems like a remnant of the initial design, where unencrypted datasets below encrypted ones weren't allowed. Add a condition to set `DS_HOLD_FLAG_DECRYPT` only for encrypted datasets, modify an existing test to detect this regression and add a test for raw replication streams. Reviewed-by: Brian Behlendorf Reviewed-by: George Amanakis Co-authored-by: George Amanakis Signed-off-by: Attila Fülöp Closes #13033 Closes #13076 --- module/zfs/dmu_recv.c | 10 ++- tests/runfiles/common.run | 3 +- .../cli_root/zfs_receive/Makefile.am | 3 +- .../zfs_receive_-wR-encrypted-mix.ksh | 75 +++++++++++++++++++ .../zfs_receive/zfs_receive_to_encrypted.ksh | 18 ++++- 5 files changed, 102 insertions(+), 7 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_-wR-encrypted-mix.ksh diff --git a/module/zfs/dmu_recv.c b/module/zfs/dmu_recv.c index 0ec46bdb4f47..99eeceeb4139 100644 --- a/module/zfs/dmu_recv.c +++ b/module/zfs/dmu_recv.c @@ -597,7 +597,15 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx) if (!(flags & DRR_FLAG_SPILL_BLOCK)) return (SET_ERROR(ZFS_ERR_SPILL_BLOCK_FLAG_MISSING)); } else { - dsflags |= DS_HOLD_FLAG_DECRYPT; + /* + * We support unencrypted datasets below encrypted ones now, + * so add the DS_HOLD_FLAG_DECRYPT flag only if we are dealing + * with a dataset we may encrypt. + */ + if (drba->drba_dcp != NULL && + drba->drba_dcp->cp_crypt != ZIO_CRYPT_OFF) { + dsflags |= DS_HOLD_FLAG_DECRYPT; + } } error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds); diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 18995511ed58..bad2df2f97de 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -234,7 +234,8 @@ tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos', 'zfs_receive_016_pos', 'receive-o-x_props_override', 'zfs_receive_from_encrypted', 'zfs_receive_to_encrypted', 'zfs_receive_raw', 'zfs_receive_raw_incremental', 'zfs_receive_-e', - 'zfs_receive_raw_-d', 'zfs_receive_from_zstd', 'zfs_receive_new_props'] + 'zfs_receive_raw_-d', 'zfs_receive_from_zstd', 'zfs_receive_new_props', + 'zfs_receive_-wR-encrypted-mix'] tags = ['functional', 'cli_root', 'zfs_receive'] [tests/functional/cli_root/zfs_rename] diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am index 2b281e71c210..773e9f5a5585 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am @@ -26,7 +26,8 @@ dist_pkgdata_SCRIPTS = \ zfs_receive_raw.ksh \ zfs_receive_raw_incremental.ksh \ zfs_receive_raw_-d.ksh \ - zfs_receive_-e.ksh + zfs_receive_-e.ksh \ + zfs_receive_-wR-encrypted-mix.ksh dist_pkgdata_DATA = \ zstd_test_data.txt diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_-wR-encrypted-mix.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_-wR-encrypted-mix.ksh new file mode 100755 index 000000000000..6e27130e0248 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_-wR-encrypted-mix.ksh @@ -0,0 +1,75 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2022 by Attila Fülöp +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# ZFS should receive a raw send of a mix of unencrypted and encrypted +# child datasets +# +# The layout of the datasets is: enc/unenc/enc/unenc +# +# STRATEGY: +# 1. Create the dataset hierarchy +# 2. Snapshot the dataset hierarchy +# 3. Send -Rw the dataset hierarchy and receive into a top-level dataset +# 4. Check the encryption property of the received datasets + +verify_runnable "both" + +function cleanup +{ + datasetexists "$TESTPOOL/$TESTFS1" && \ + destroy_dataset "$TESTPOOL/$TESTFS1" -r + + datasetexists "$TESTPOOL/$TESTFS2" && \ + destroy_dataset "$TESTPOOL/$TESTFS2" -r +} + +log_onexit cleanup + +log_assert "ZFS should receive a mix of un/encrypted childs" + +typeset src="$TESTPOOL/$TESTFS1" +typeset dst="$TESTPOOL/$TESTFS2" +typeset snap="snap" + +echo "password" | \ + create_dataset "$src" -o encryption=on -o keyformat=passphrase +create_dataset "$src/u" "-o encryption=off" +echo "password" | \ + create_dataset "$src/u/e" -o encryption=on -o keyformat=passphrase +create_dataset "$src/u/e/u" -o encryption=off + +log_must zfs snapshot -r "$src@$snap" +log_must eval "zfs send -Rw $src@$snap | zfs receive -u $dst" +log_must test "$(get_prop 'encryption' $dst)" != "off" +log_must test "$(get_prop 'encryption' $dst/u)" == "off" +log_must test "$(get_prop 'encryption' $dst/u/e)" != "off" +log_must test "$(get_prop 'encryption' $dst/u/e/u)" == "off" + +log_pass "ZFS can receive a mix of un/encrypted childs" diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_to_encrypted.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_to_encrypted.ksh index 526497401f28..5d76c220fc45 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_to_encrypted.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_to_encrypted.ksh @@ -59,19 +59,29 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \ log_note "Verifying ZFS will receive to an encrypted child" log_must eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS1/c1" +log_must test "$(get_prop 'encryption' $TESTPOOL/$TESTFS1/c1)" != "off" -log_note "Verifying 'send -p' will receive to an encrypted child" +# Unload the key, the following tests won't require it and we will test +# the receive checks as well. +log_must zfs unmount $TESTPOOL/$TESTFS1 +log_must zfs unload-key $TESTPOOL/$TESTFS1 + +log_note "Verifying 'send -p' will receive to an unencrypted child" log_must eval "zfs send -p $snap | zfs receive $TESTPOOL/$TESTFS1/c2" log_must test "$(get_prop 'encryption' $TESTPOOL/$TESTFS1/c2)" == "off" -log_note "Verifying 'send -R' will receive to an encrypted child" +# For completeness add the property override case. +log_note "Verifying recv -o encyption=off' will receive to an unencrypted child" +log_must eval "zfs send $snap | \ + zfs receive -o encryption=off $TESTPOOL/$TESTFS1/c2o" +log_must test "$(get_prop 'encryption' $TESTPOOL/$TESTFS1/c2o)" == "off" + +log_note "Verifying 'send -R' will receive to an unencrypted child" log_must eval "zfs send -R $snap | zfs receive $TESTPOOL/$TESTFS1/c3" log_must test "$(get_prop 'encryption' $TESTPOOL/$TESTFS1/c3)" == "off" log_note "Verifying ZFS will not receive to an encrypted child when the" \ "parent key is unloaded" -log_must zfs unmount $TESTPOOL/$TESTFS1 -log_must zfs unload-key $TESTPOOL/$TESTFS1 log_mustnot eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS1/c4" log_pass "ZFS can receive encrypted filesystems into child dataset" From bc99c809d5034e0cd0b2ed731dd1cdc88e959aa2 Mon Sep 17 00:00:00 2001 From: drowfx Date: Wed, 9 Feb 2022 23:41:42 +0100 Subject: [PATCH 079/109] Add dataset_kstats_update.. to mmap read/write paths This allows reads/writes caused by accesses to mmap files to be accounted correctly in the per-dataset kstats for both Linux and FreeBSD. Reviewed-by: Brian Behlendorf Reviewed-by: Ryan Moeller Signed-off-by: Matthias Blankertz Closes #12994 Closes #13044 --- module/os/freebsd/zfs/zfs_vnops_os.c | 6 ++++++ module/os/linux/zfs/zfs_vnops_os.c | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/module/os/freebsd/zfs/zfs_vnops_os.c b/module/os/freebsd/zfs/zfs_vnops_os.c index 6d3dac75484f..7bcf80bf5a94 100644 --- a/module/os/freebsd/zfs/zfs_vnops_os.c +++ b/module/os/freebsd/zfs/zfs_vnops_os.c @@ -4141,6 +4141,9 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind, if (lr != NULL) zfs_rangelock_exit(lr); ZFS_ACCESSTIME_STAMP(zfsvfs, zp); + + dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, count*PAGE_SIZE); + ZFS_EXIT(zfsvfs); if (error != 0) @@ -4318,6 +4321,9 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags, if ((flags & (zfs_vm_pagerput_sync | zfs_vm_pagerput_inval)) != 0 || zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) zil_commit(zfsvfs->z_log, zp->z_id); + + dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, len); + ZFS_EXIT(zfsvfs); return (rtvals[0]); } diff --git a/module/os/linux/zfs/zfs_vnops_os.c b/module/os/linux/zfs/zfs_vnops_os.c index a1a0e44bb31f..2958439ace82 100644 --- a/module/os/linux/zfs/zfs_vnops_os.c +++ b/module/os/linux/zfs/zfs_vnops_os.c @@ -3635,6 +3635,8 @@ zfs_putpage(struct inode *ip, struct page *pp, struct writeback_control *wbc) zil_commit(zfsvfs->z_log, zp->z_id); } + dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, pglen); + ZFS_EXIT(zfsvfs); return (err); } @@ -3830,6 +3832,8 @@ zfs_getpage(struct inode *ip, struct page *pl[], int nr_pages) err = zfs_fillpage(ip, pl, nr_pages); + dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, nr_pages*PAGESIZE); + ZFS_EXIT(zfsvfs); return (err); } From f03cf651ec72ac40e9ac4699fa9f57e52d5a45ae Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 9 Feb 2022 17:00:03 -0800 Subject: [PATCH 080/109] ZTS: Fix zvol_misc_volmode test Changing volmode may need to remove minors, which could be open, so call udev_wait() before we "zfs set volmode=". This ensures no udev process has the zvol open (i.e. blkid) and the kernel zvol_remove_minor_impl() function won't skip removing the in use device. Reviewed-by: John Kennedy Reviewed-by: George Melikov Signed-off-by: Brian Behlendorf Closes #13075 --- .../zvol/zvol_misc/zvol_misc_volmode.ksh | 49 ++++++++++++------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode.ksh b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode.ksh index 264ec131ad6b..af808dc30764 100755 --- a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode.ksh +++ b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode.ksh @@ -43,8 +43,6 @@ # 7. Verify "volmode" behaves correctly at import time # 8. Verify "volmode" behaves accordingly to zvol_inhibit_dev (Linux only) # -# NOTE: changing volmode may need to remove minors, which could be open, so call -# block_device_wait() before we "zfs set volmode=". verify_runnable "global" @@ -93,6 +91,21 @@ function test_io # dev log_must dd if=/dev/zero of=$dev count=1 } +# +# Changing volmode may need to remove minors, which could be open, so call +# udev_wait() before we "zfs set volmode=". This ensures no udev +# process has the zvol open (i.e. blkid) and the zvol_remove_minor_impl() +# function won't skip removing the in use device. +# +function set_volmode # value ds +{ + typeset value="$1" + typeset ds="$2" + + is_linux && udev_wait + log_must zfs set volmode="$value" "$ds" +} + log_assert "Verify that ZFS volume property 'volmode' works as intended" log_onexit cleanup @@ -119,7 +132,7 @@ do done # 2. Verify "volmode=none" hides ZVOL device nodes -log_must zfs set volmode=none $ZVOL +set_volmode none $ZVOL blockdev_missing $ZDEV log_must_busy zfs destroy $ZVOL blockdev_missing $ZDEV @@ -127,12 +140,12 @@ blockdev_missing $ZDEV # 3. Verify "volmode=full" exposes a fully functional device log_must zfs create -V $VOLSIZE -s $ZVOL blockdev_exists $ZDEV -log_must zfs set volmode=full $ZVOL +set_volmode full $ZVOL blockdev_exists $ZDEV test_io $ZDEV log_must verify_partition $ZDEV # 3.1 Verify "volmode=geom" is an alias for "volmode=full" -log_must zfs set volmode=geom $ZVOL +set_volmode geom $ZVOL blockdev_exists $ZDEV if [[ "$(get_prop 'volmode' $ZVOL)" != "full" ]]; then log_fail " Volmode value 'geom' is not an alias for 'full'" @@ -143,7 +156,7 @@ blockdev_missing $ZDEV # 4. Verify "volmode=dev" hides partition info on the device log_must zfs create -V $VOLSIZE -s $ZVOL blockdev_exists $ZDEV -log_must zfs set volmode=dev $ZVOL +set_volmode dev $ZVOL blockdev_exists $ZDEV test_io $ZDEV log_mustnot verify_partition $ZDEV @@ -155,7 +168,7 @@ blockdev_missing $ZDEV sysctl_volmode 1 log_must zfs create -V $VOLSIZE -s $ZVOL blockdev_exists $ZDEV -log_must zfs set volmode=default $ZVOL +set_volmode default $ZVOL blockdev_exists $ZDEV log_must verify_partition $ZDEV log_must_busy zfs destroy $ZVOL @@ -164,7 +177,7 @@ blockdev_missing $ZDEV sysctl_volmode 2 log_must zfs create -V $VOLSIZE -s $ZVOL blockdev_exists $ZDEV -log_must zfs set volmode=default $ZVOL +set_volmode default $ZVOL blockdev_exists $ZDEV log_mustnot verify_partition $ZDEV log_must_busy zfs destroy $ZVOL @@ -173,33 +186,33 @@ blockdev_missing $ZDEV sysctl_volmode 3 log_must zfs create -V $VOLSIZE -s $ZVOL blockdev_missing $ZDEV -log_must zfs set volmode=default $ZVOL +set_volmode default $ZVOL blockdev_missing $ZDEV # 6. Verify "volmode" property is inherited correctly log_must zfs inherit volmode $ZVOL blockdev_missing $ZDEV # 6.1 Check volmode=full case -log_must zfs set volmode=full $TESTPOOL +set_volmode full $TESTPOOL verify_inherited 'volmode' 'full' $ZVOL $TESTPOOL blockdev_exists $ZDEV # 6.2 Check volmode=none case -log_must zfs set volmode=none $TESTPOOL +set_volmode none $TESTPOOL verify_inherited 'volmode' 'none' $ZVOL $TESTPOOL blockdev_missing $ZDEV # 6.3 Check volmode=dev case -log_must zfs set volmode=dev $TESTPOOL +set_volmode dev $TESTPOOL verify_inherited 'volmode' 'dev' $ZVOL $TESTPOOL blockdev_exists $ZDEV # 6.4 Check volmode=default case sysctl_volmode 1 -log_must zfs set volmode=default $TESTPOOL +set_volmode default $TESTPOOL verify_inherited 'volmode' 'default' $ZVOL $TESTPOOL blockdev_exists $ZDEV # 6.5 Check inheritance on multiple levels log_must zfs inherit volmode $SUBZVOL -log_must zfs set volmode=none $VOLFS -log_must zfs set volmode=full $TESTPOOL +set_volmode none $VOLFS +set_volmode full $TESTPOOL verify_inherited 'volmode' 'none' $SUBZVOL $VOLFS blockdev_missing $SUBZDEV blockdev_exists $ZDEV @@ -223,7 +236,7 @@ if is_linux; then sysctl_volmode 1 log_must zfs create -V $VOLSIZE -s $ZVOL blockdev_missing $ZDEV - log_must zfs set volmode=full $ZVOL + set_volmode full $ZVOL blockdev_missing $ZDEV log_must_busy zfs destroy $ZVOL blockdev_missing $ZDEV @@ -231,7 +244,7 @@ if is_linux; then sysctl_volmode 2 log_must zfs create -V $VOLSIZE -s $ZVOL blockdev_missing $ZDEV - log_must zfs set volmode=dev $ZVOL + set_volmode dev $ZVOL blockdev_missing $ZDEV log_must_busy zfs destroy $ZVOL blockdev_missing $ZDEV @@ -239,7 +252,7 @@ if is_linux; then sysctl_volmode 3 log_must zfs create -V $VOLSIZE -s $ZVOL blockdev_missing $ZDEV - log_must zfs set volmode=none $ZVOL + set_volmode none $ZVOL blockdev_missing $ZDEV fi From d4e8dcf07ea1d3ad56ce42e00a91224500f4d3e4 Mon Sep 17 00:00:00 2001 From: Zhu Chuang Date: Sat, 12 Feb 2022 03:47:35 +0800 Subject: [PATCH 081/109] Correct a typo in zfs-receive.8 Should be `-o keyformat=passphrase` instead of `-o -keyformat=passphrase` Reviewed-by: Brian Behlendorf Signed-off-by: Chuang Zhu Closes #13072 --- man/man8/zfs-receive.8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/man8/zfs-receive.8 b/man/man8/zfs-receive.8 index d2cec42a8e71..1f4f91bca2ee 100644 --- a/man/man8/zfs-receive.8 +++ b/man/man8/zfs-receive.8 @@ -319,7 +319,7 @@ received as an encryption root, specify encryption properties in the same manner as is required for .Nm zfs Cm create . For instance: -.Dl # Nm zfs Cm send Pa tank/test@snap1 | Nm zfs Cm recv Fl o Sy encryption Ns = Ns Sy on Fl o keyformat=passphrase Fl o Sy keylocation Ns = Ns Pa file:///path/to/keyfile +.Dl # Nm zfs Cm send Pa tank/test@snap1 | Nm zfs Cm recv Fl o Sy encryption Ns = Ns Sy on Fl o Sy keyformat=passphrase Fl o Sy keylocation Ns = Ns Pa file:///path/to/keyfile .Pp Note that .Fl o Sy keylocation Ns = Ns Sy prompt From a38e7bc9224f706ba31f86c7a1b0e7509ce29108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Fri, 11 Feb 2022 20:49:07 +0100 Subject: [PATCH 082/109] zpoolprops.7: document leaked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's noted very scarcely in the code as it stands, indeed the only actual comment on this is /* * We have finished background destroying, but there is still * some space left in the dp_free_dir. Transfer this leaked * space to the dp_leak_dir. */ Introduced in fbeddd60b79690b6a6ececc9b00b6014d21405aa ("Illumos 4390 - I/O errors can corrupt space map when deleting fs/vol"), which explains, alongside the references, that this can only happen with a corrupted pool Reviewed-by: Brian Behlendorf Reviewed-by: George Melikov Signed-off-by: Ahelenia Ziemiańska Closes #13081 --- man/man7/zpoolprops.7 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/man/man7/zpoolprops.7 b/man/man7/zpoolprops.7 index 543920a1d58e..5bd1c2bf3128 100644 --- a/man/man7/zpoolprops.7 +++ b/man/man7/zpoolprops.7 @@ -101,6 +101,10 @@ Over time will decrease while .Sy free increases. +.It Sy leaked +Space not released while +.Sy freeing +due to corruption, now permanently leaked into the pool. .It Sy health The current health of the pool. Health can be one of From 11bd8cd0024359b6cbbcca2dbc770cb2bf55479c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Thu, 10 Feb 2022 19:43:08 +0100 Subject: [PATCH 083/109] zpool-import.8: newpool is Ar, not Sy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: George Melikov Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #13082 --- man/man8/zpool-import.8 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/man/man8/zpool-import.8 b/man/man8/zpool-import.8 index 518e3cf1d76a..4e60fff3c8a6 100644 --- a/man/man8/zpool-import.8 +++ b/man/man8/zpool-import.8 @@ -390,9 +390,9 @@ WARNING: This option can be extremely hazardous to the health of your pool and should only be used as a last resort. .It Fl t Used with -.Sy newpool . +.Ar newpool . Specifies that -.Sy newpool +.Ar newpool is temporary. Temporary pool names last until export. Ensures that the original pool name will be used From d24bdf4ee4347fe751b0299cd6b866c6b3453f4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Thu, 10 Feb 2022 19:43:25 +0100 Subject: [PATCH 084/109] zpool-import.8: WARNING should be emphasised MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: George Melikov Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #13082 --- man/man8/zpool-import.8 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/man/man8/zpool-import.8 b/man/man8/zpool-import.8 index 4e60fff3c8a6..39b0e17ef586 100644 --- a/man/man8/zpool-import.8 +++ b/man/man8/zpool-import.8 @@ -386,7 +386,8 @@ For more details about pool recovery mode, see the .Fl X option, above. -WARNING: This option can be extremely hazardous to the +.Em WARNING : +This option can be extremely hazardous to the health of your pool and should only be used as a last resort. .It Fl t Used with From f4c2b21823e7989be826f13ec7913f09320dc919 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Fri, 11 Feb 2022 14:31:45 -0800 Subject: [PATCH 085/109] Fix gcc warning in kfpu_begin() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Observed when building on CentOS 8 Stream. Remove the `out` label at the end of the function and instead return. linux/simd_x86.h: In function 'kfpu_begin': linux/simd_x86.h:337:1: error: label at end of compound statement Reviewed-by: Tony Hutter Reviewed-by: Attila Fülöp Signed-off-by: Brian Behlendorf Closes #13089 --- include/os/linux/kernel/linux/simd_x86.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/os/linux/kernel/linux/simd_x86.h b/include/os/linux/kernel/linux/simd_x86.h index 7c806bf2c6fa..6d4c7a09fe82 100644 --- a/include/os/linux/kernel/linux/simd_x86.h +++ b/include/os/linux/kernel/linux/simd_x86.h @@ -318,13 +318,13 @@ kfpu_begin(void) #if defined(HAVE_XSAVES) if (static_cpu_has(X86_FEATURE_XSAVES)) { kfpu_do_xsave("xsaves", &state->xsave, ~0); - goto out; + return; } #endif #if defined(HAVE_XSAVEOPT) if (static_cpu_has(X86_FEATURE_XSAVEOPT)) { kfpu_do_xsave("xsaveopt", &state->xsave, ~0); - goto out; + return; } #endif if (static_cpu_has(X86_FEATURE_XSAVE)) { @@ -334,7 +334,6 @@ kfpu_begin(void) } else { kfpu_save_fsave(&state->fsave); } -out: } #endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */ From 882bc4ad618d345a571ede6d85937a6e0cc87464 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Sun, 13 Feb 2022 14:22:00 -0800 Subject: [PATCH 086/109] ZTS: Fix zpool_expand_001_pos The dRAID section of the zpool_expand_001_pos test would reliably fail because the calculated expansion size assumed the dRAID top-level vdev was created with a distributed spare. Create the vdev as expected to resolve the test failure. This test case flaw was accidentally caused by changing the default number of dRAID distributed spares from one to zero while dRAID was being developed. Additionally, remove zpool_expand_005_pos from the list of possible faulty tests. It appears to be passing consistently in my testing. Reviewed by: George Melikov Signed-off-by: Brian Behlendorf Closes #13091 --- tests/test-runner/bin/zts-report.py.in | 2 -- .../functional/cli_root/zpool_expand/zpool_expand_001_pos.ksh | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index c1aacd54f6c5..6a8553d033a9 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -283,8 +283,6 @@ if sys.platform.startswith('freebsd'): elif sys.platform.startswith('linux'): maybe.update({ 'cli_root/zfs_rename/zfs_rename_002_pos': ['FAIL', known_reason], - 'cli_root/zpool_expand/zpool_expand_001_pos': ['FAIL', known_reason], - 'cli_root/zpool_expand/zpool_expand_005_pos': ['FAIL', known_reason], 'cli_root/zpool_reopen/zpool_reopen_003_pos': ['FAIL', known_reason], 'fault/auto_spare_shared': ['FAIL', '11889'], 'io/io_uring': ['SKIP', 'io_uring support required'], diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_expand/zpool_expand_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_expand/zpool_expand_001_pos.ksh index 922e35125e4a..6bbd46289f7c 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_expand/zpool_expand_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_expand/zpool_expand_001_pos.ksh @@ -72,7 +72,7 @@ log_onexit cleanup log_assert "zpool can be autoexpanded after set autoexpand=on on vdev expansion" -for type in " " mirror raidz draid; do +for type in " " mirror raidz draid:1s; do log_note "Setting up loopback, scsi_debug, and file vdevs" log_must truncate -s $org_size $FILE_LO DEV1=$(losetup -f) @@ -144,7 +144,7 @@ for type in " " mirror raidz draid; do if [[ $? -ne 0 ]] ; then log_fail "pool $TESTPOOL1 has not expanded" fi - elif [[ $type == "draid" ]]; then + elif [[ $type == "draid:1s" ]]; then typeset expansion_size=$((2*($exp_size-$org_size))) zpool history -il $TESTPOOL1 | \ grep "pool '$TESTPOOL1' size:" | \ From ccbe9efd6bb37a58b0d70ed0d5f757460968a865 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Sun, 13 Feb 2022 14:22:49 -0800 Subject: [PATCH 087/109] ZTS: Fix checkpoint_ro_rewind.ksh Related to commit 90b77a036. Retry the `zpool export` if the pool is "busy" indicating there is a process accessing the mount point. This can happen after an import and allowing it to be retried will avoid spurious test failures. Reviewed by: George Melikov Signed-off-by: Brian Behlendorf Closes #13092 --- .../tests/functional/pool_checkpoint/checkpoint_ro_rewind.ksh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/zfs-tests/tests/functional/pool_checkpoint/checkpoint_ro_rewind.ksh b/tests/zfs-tests/tests/functional/pool_checkpoint/checkpoint_ro_rewind.ksh index fd7416612b7c..f326bf0c25d2 100755 --- a/tests/zfs-tests/tests/functional/pool_checkpoint/checkpoint_ro_rewind.ksh +++ b/tests/zfs-tests/tests/functional/pool_checkpoint/checkpoint_ro_rewind.ksh @@ -44,12 +44,12 @@ populate_test_pool log_must zpool checkpoint $TESTPOOL test_change_state_after_checkpoint -log_must zpool export $TESTPOOL +log_must_busy zpool export $TESTPOOL log_must zpool import -o readonly=on --rewind-to-checkpoint $TESTPOOL test_verify_pre_checkpoint_state "ro-check" -log_must zpool export $TESTPOOL +log_must_busy zpool export $TESTPOOL log_must zpool import $TESTPOOL test_verify_post_checkpoint_state From 94a4b7ec3d79b1815f73250b83477773502d35a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Sun, 12 Dec 2021 16:06:44 +0100 Subject: [PATCH 088/109] module: zfs: fix unused, remove argsused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #12844 --- module/os/freebsd/zfs/vdev_file.c | 2 +- module/os/freebsd/zfs/zfs_racct.c | 4 +++ module/os/freebsd/zfs/zio_crypt.c | 5 ++- module/os/linux/zfs/vdev_file.c | 2 +- module/os/linux/zfs/zfs_racct.c | 2 ++ module/os/linux/zfs/zio_crypt.c | 1 + module/zfs/abd.c | 6 ++-- module/zfs/arc.c | 53 +++++++++++++++++-------------- module/zfs/bpobj.c | 4 +-- module/zfs/bptree.c | 2 +- module/zfs/dbuf.c | 23 ++++++++------ module/zfs/dmu.c | 9 +++--- module/zfs/dmu_diff.c | 2 +- module/zfs/dmu_objset.c | 10 +++--- module/zfs/dmu_redact.c | 2 +- module/zfs/dmu_send.c | 3 +- module/zfs/dmu_traverse.c | 2 +- module/zfs/dmu_zfetch.c | 1 + module/zfs/dnode.c | 11 +++---- module/zfs/dsl_bookmark.c | 4 +-- module/zfs/dsl_dataset.c | 4 +-- module/zfs/dsl_destroy.c | 4 +-- module/zfs/dsl_dir.c | 5 ++- module/zfs/dsl_pool.c | 4 +-- module/zfs/dsl_prop.c | 2 +- module/zfs/dsl_scan.c | 21 ++++++------ module/zfs/dsl_synctask.c | 2 +- module/zfs/fm.c | 4 +-- module/zfs/gzip.c | 2 +- module/zfs/lz4.c | 6 ++-- module/zfs/lzjb.c | 4 +-- module/zfs/metaslab.c | 16 ++++------ module/zfs/range_tree.c | 8 ++--- module/zfs/sa.c | 8 ++--- module/zfs/sha256.c | 5 ++- module/zfs/spa.c | 17 ++++++---- module/zfs/spa_checkpoint.c | 10 +++--- module/zfs/spa_errlog.c | 2 ++ module/zfs/spa_misc.c | 2 ++ module/zfs/vdev.c | 9 +++++- module/zfs/vdev_draid.c | 4 +-- module/zfs/vdev_indirect.c | 6 ++-- module/zfs/vdev_initialize.c | 4 ++- module/zfs/vdev_mirror.c | 2 ++ module/zfs/vdev_missing.c | 7 ++-- module/zfs/vdev_raidz.c | 8 +++-- module/zfs/vdev_removal.c | 4 +-- module/zfs/vdev_trim.c | 1 + module/zfs/zcp.c | 4 +-- module/zfs/zcp_synctask.c | 8 +++-- module/zfs/zfs_fm.c | 25 ++++++++++++--- module/zfs/zil.c | 21 ++++++------ module/zfs/zio.c | 10 ++++-- module/zfs/zio_checksum.c | 10 +++--- module/zfs/zio_compress.c | 4 ++- 55 files changed, 226 insertions(+), 175 deletions(-) diff --git a/module/os/freebsd/zfs/vdev_file.c b/module/os/freebsd/zfs/vdev_file.c index 742ac9786d72..2d926813637e 100644 --- a/module/os/freebsd/zfs/vdev_file.c +++ b/module/os/freebsd/zfs/vdev_file.c @@ -287,10 +287,10 @@ vdev_file_io_start(zio_t *zio) TQ_SLEEP), !=, 0); } -/* ARGSUSED */ static void vdev_file_io_done(zio_t *zio) { + (void) zio; } vdev_ops_t vdev_file_ops = { diff --git a/module/os/freebsd/zfs/zfs_racct.c b/module/os/freebsd/zfs/zfs_racct.c index b46cc046268e..883255bc1901 100644 --- a/module/os/freebsd/zfs/zfs_racct.c +++ b/module/os/freebsd/zfs/zfs_racct.c @@ -37,6 +37,8 @@ zfs_racct_read(uint64_t size, uint64_t iops) racct_add_force(curproc, RACCT_READIOPS, iops); PROC_UNLOCK(curproc); } +#else + (void) size; #endif /* RACCT */ } @@ -51,5 +53,7 @@ zfs_racct_write(uint64_t size, uint64_t iops) racct_add_force(curproc, RACCT_WRITEIOPS, iops); PROC_UNLOCK(curproc); } +#else + (void) size; #endif /* RACCT */ } diff --git a/module/os/freebsd/zfs/zio_crypt.c b/module/os/freebsd/zfs/zio_crypt.c index e4ab78705832..9e0ab52a2ea2 100644 --- a/module/os/freebsd/zfs/zio_crypt.c +++ b/module/os/freebsd/zfs/zio_crypt.c @@ -1033,7 +1033,6 @@ zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version, * and le_bswap indicates whether a byteswap is needed to get this block * into little endian format. */ -/* ARGSUSED */ int zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac) @@ -1259,13 +1258,13 @@ zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd, * It also means we'll only return one zfs_uio_t. */ -/* ARGSUSED */ static int zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, zfs_uio_t *puio, zfs_uio_t *out_uio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, boolean_t *no_crypt) { + (void) puio; uint8_t *aadbuf = zio_buf_alloc(datalen); uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp; iovec_t *dst_iovecs; @@ -1562,12 +1561,12 @@ zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, return (0); } -/* ARGSUSED */ static int zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, zfs_uio_t *puio, zfs_uio_t *out_uio, uint_t *enc_len) { + (void) puio; int ret; uint_t nr_plain = 1, nr_cipher = 2; iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL; diff --git a/module/os/linux/zfs/vdev_file.c b/module/os/linux/zfs/vdev_file.c index bf8a13ae6154..98338e604fa8 100644 --- a/module/os/linux/zfs/vdev_file.c +++ b/module/os/linux/zfs/vdev_file.c @@ -298,10 +298,10 @@ vdev_file_io_start(zio_t *zio) TQ_SLEEP), !=, TASKQID_INVALID); } -/* ARGSUSED */ static void vdev_file_io_done(zio_t *zio) { + (void) zio; } vdev_ops_t vdev_file_ops = { diff --git a/module/os/linux/zfs/zfs_racct.c b/module/os/linux/zfs/zfs_racct.c index 7897e0f9edc1..ce623ef9d185 100644 --- a/module/os/linux/zfs/zfs_racct.c +++ b/module/os/linux/zfs/zfs_racct.c @@ -28,9 +28,11 @@ void zfs_racct_read(uint64_t size, uint64_t iops) { + (void) size, (void) iops; } void zfs_racct_write(uint64_t size, uint64_t iops) { + (void) size, (void) iops; } diff --git a/module/os/linux/zfs/zio_crypt.c b/module/os/linux/zfs/zio_crypt.c index 13e156ac3b6a..a13fc2aa2546 100644 --- a/module/os/linux/zfs/zio_crypt.c +++ b/module/os/linux/zfs/zio_crypt.c @@ -1776,6 +1776,7 @@ zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len) { + (void) encrypt; int ret; uint_t nr_plain = 1, nr_cipher = 2; iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL; diff --git a/module/zfs/abd.c b/module/zfs/abd.c index 03a7b1e033b3..42bf3e3036f9 100644 --- a/module/zfs/abd.c +++ b/module/zfs/abd.c @@ -889,10 +889,10 @@ abd_copy_from_buf_off(abd_t *abd, const void *buf, size_t off, size_t size) &ba_ptr); } -/*ARGSUSED*/ static int abd_zero_off_cb(void *buf, size_t size, void *private) { + (void) private; (void) memset(buf, 0, size); return (0); } @@ -967,10 +967,10 @@ abd_iterate_func2(abd_t *dabd, abd_t *sabd, size_t doff, size_t soff, return (ret); } -/*ARGSUSED*/ static int abd_copy_off_cb(void *dbuf, void *sbuf, size_t size, void *private) { + (void) private; (void) memcpy(dbuf, sbuf, size); return (0); } @@ -985,10 +985,10 @@ abd_copy_off(abd_t *dabd, abd_t *sabd, size_t doff, size_t soff, size_t size) abd_copy_off_cb, NULL); } -/*ARGSUSED*/ static int abd_cmp_cb(void *bufa, void *bufb, size_t size, void *private) { + (void) private; return (memcmp(bufa, bufb, size)); } diff --git a/module/zfs/arc.c b/module/zfs/arc.c index 6dcd2d1deef8..9984c2d27534 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -1093,8 +1093,6 @@ static kmem_cache_t *buf_cache; static void buf_fini(void) { - int i; - #if defined(_KERNEL) /* * Large allocations which do not require contiguous pages @@ -1106,7 +1104,7 @@ buf_fini(void) kmem_free(buf_hash_table.ht_table, (buf_hash_table.ht_mask + 1) * sizeof (void *)); #endif - for (i = 0; i < BUF_LOCKS; i++) + for (int i = 0; i < BUF_LOCKS; i++) mutex_destroy(BUF_HASH_LOCK(i)); kmem_cache_destroy(hdr_full_cache); kmem_cache_destroy(hdr_full_crypt_cache); @@ -1118,10 +1116,10 @@ buf_fini(void) * Constructor callback - called when the cache is empty * and a new buf is requested. */ -/* ARGSUSED */ static int hdr_full_cons(void *vbuf, void *unused, int kmflag) { + (void) unused, (void) kmflag; arc_buf_hdr_t *hdr = vbuf; bzero(hdr, HDR_FULL_SIZE); @@ -1137,10 +1135,10 @@ hdr_full_cons(void *vbuf, void *unused, int kmflag) return (0); } -/* ARGSUSED */ static int hdr_full_crypt_cons(void *vbuf, void *unused, int kmflag) { + (void) unused; arc_buf_hdr_t *hdr = vbuf; hdr_full_cons(vbuf, unused, kmflag); @@ -1150,10 +1148,10 @@ hdr_full_crypt_cons(void *vbuf, void *unused, int kmflag) return (0); } -/* ARGSUSED */ static int hdr_l2only_cons(void *vbuf, void *unused, int kmflag) { + (void) unused, (void) kmflag; arc_buf_hdr_t *hdr = vbuf; bzero(hdr, HDR_L2ONLY_SIZE); @@ -1162,10 +1160,10 @@ hdr_l2only_cons(void *vbuf, void *unused, int kmflag) return (0); } -/* ARGSUSED */ static int buf_cons(void *vbuf, void *unused, int kmflag) { + (void) unused, (void) kmflag; arc_buf_t *buf = vbuf; bzero(buf, sizeof (arc_buf_t)); @@ -1179,10 +1177,10 @@ buf_cons(void *vbuf, void *unused, int kmflag) * Destructor callback - called when a cached buf is * no longer required. */ -/* ARGSUSED */ static void hdr_full_dest(void *vbuf, void *unused) { + (void) unused; arc_buf_hdr_t *hdr = vbuf; ASSERT(HDR_EMPTY(hdr)); @@ -1193,30 +1191,30 @@ hdr_full_dest(void *vbuf, void *unused) arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS); } -/* ARGSUSED */ static void hdr_full_crypt_dest(void *vbuf, void *unused) { + (void) unused; arc_buf_hdr_t *hdr = vbuf; hdr_full_dest(vbuf, unused); arc_space_return(sizeof (hdr->b_crypt_hdr), ARC_SPACE_HDRS); } -/* ARGSUSED */ static void hdr_l2only_dest(void *vbuf, void *unused) { - arc_buf_hdr_t *hdr __maybe_unused = vbuf; + (void) unused; + arc_buf_hdr_t *hdr = vbuf; ASSERT(HDR_EMPTY(hdr)); arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS); } -/* ARGSUSED */ static void buf_dest(void *vbuf, void *unused) { + (void) unused; arc_buf_t *buf = vbuf; mutex_destroy(&buf->b_evict_lock); @@ -1512,11 +1510,11 @@ arc_cksum_compute(arc_buf_t *buf) void arc_buf_sigsegv(int sig, siginfo_t *si, void *unused) { + (void) sig, (void) unused; panic("Got SIGSEGV at address: 0x%lx\n", (long)si->si_addr); } #endif -/* ARGSUSED */ static void arc_buf_unwatch(arc_buf_t *buf) { @@ -1525,10 +1523,11 @@ arc_buf_unwatch(arc_buf_t *buf) ASSERT0(mprotect(buf->b_data, arc_buf_size(buf), PROT_READ | PROT_WRITE)); } +#else + (void) buf; #endif } -/* ARGSUSED */ static void arc_buf_watch(arc_buf_t *buf) { @@ -1536,6 +1535,8 @@ arc_buf_watch(arc_buf_t *buf) if (arc_watch) ASSERT0(mprotect(buf->b_data, arc_buf_size(buf), PROT_READ)); +#else + (void) buf; #endif } @@ -1947,7 +1948,7 @@ arc_fill_hdr_crypt(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, spa_t *spa, * arc_buf_fill(). */ static void -arc_buf_untransform_in_place(arc_buf_t *buf, kmutex_t *hash_lock) +arc_buf_untransform_in_place(arc_buf_t *buf) { arc_buf_hdr_t *hdr = buf->b_hdr; @@ -2051,7 +2052,7 @@ arc_buf_fill(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb, if (hash_lock != NULL) mutex_enter(hash_lock); - arc_buf_untransform_in_place(buf, hash_lock); + arc_buf_untransform_in_place(buf); if (hash_lock != NULL) mutex_exit(hash_lock); @@ -2337,6 +2338,7 @@ remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag) void arc_buf_info(arc_buf_t *ab, arc_buf_info_t *abi, int state_index) { + (void) state_index; arc_buf_hdr_t *hdr = ab->b_hdr; l1arc_buf_hdr_t *l1hdr = NULL; l2arc_buf_hdr_t *l2hdr = NULL; @@ -4872,10 +4874,11 @@ arc_kmem_reap_soon(void) abd_cache_reap_now(); } -/* ARGSUSED */ static boolean_t arc_evict_cb_check(void *arg, zthr_t *zthr) { + (void) arg, (void) zthr; + #ifdef ZFS_DEBUG /* * This is necessary in order to keep the kstat information @@ -4915,10 +4918,11 @@ arc_evict_cb_check(void *arg, zthr_t *zthr) * Keep arc_size under arc_c by running arc_evict which evicts data * from the ARC. */ -/* ARGSUSED */ static void arc_evict_cb(void *arg, zthr_t *zthr) { + (void) arg, (void) zthr; + uint64_t evicted = 0; fstrans_cookie_t cookie = spl_fstrans_mark(); @@ -4955,10 +4959,11 @@ arc_evict_cb(void *arg, zthr_t *zthr) spl_fstrans_unmark(cookie); } -/* ARGSUSED */ static boolean_t arc_reap_cb_check(void *arg, zthr_t *zthr) { + (void) arg, (void) zthr; + int64_t free_memory = arc_available_memory(); static int reap_cb_check_counter = 0; @@ -5002,10 +5007,11 @@ arc_reap_cb_check(void *arg, zthr_t *zthr) * target size of the cache (arc_c), causing the arc_evict_cb() * to free more buffers. */ -/* ARGSUSED */ static void arc_reap_cb(void *arg, zthr_t *zthr) { + (void) arg, (void) zthr; + int64_t free_memory; fstrans_cookie_t cookie = spl_fstrans_mark(); @@ -5620,11 +5626,12 @@ arc_buf_access(arc_buf_t *buf) } /* a generic arc_read_done_func_t which you can use */ -/* ARGSUSED */ void arc_bcopy_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, arc_buf_t *buf, void *arg) { + (void) zio, (void) zb, (void) bp; + if (buf == NULL) return; @@ -5633,11 +5640,11 @@ arc_bcopy_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, } /* a generic arc_read_done_func_t */ -/* ARGSUSED */ void arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, arc_buf_t *buf, void *arg) { + (void) zb, (void) bp; arc_buf_t **bufp = arg; if (buf == NULL) { @@ -9687,10 +9694,10 @@ l2arc_hdr_limit_reached(void) * This thread feeds the L2ARC at regular intervals. This is the beating * heart of the L2ARC. */ -/* ARGSUSED */ static void l2arc_feed_thread(void *unused) { + (void) unused; callb_cpr_t cpr; l2arc_dev_t *dev; spa_t *spa; diff --git a/module/zfs/bpobj.c b/module/zfs/bpobj.c index e75ba5cccde6..68f534c6b197 100644 --- a/module/zfs/bpobj.c +++ b/module/zfs/bpobj.c @@ -860,10 +860,10 @@ struct space_range_arg { uint64_t uncomp; }; -/* ARGSUSED */ static int space_range_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, dmu_tx_t *tx) { + (void) bp_freed, (void) tx; struct space_range_arg *sra = arg; if (bp->blk_birth > sra->mintxg && bp->blk_birth <= sra->maxtxg) { @@ -932,11 +932,11 @@ bpobj_space_range(bpobj_t *bpo, uint64_t mintxg, uint64_t maxtxg, * bpobj are designated as free or allocated that information is not preserved * in bplists. */ -/* ARGSUSED */ int bplist_append_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, dmu_tx_t *tx) { + (void) bp_freed, (void) tx; bplist_t *bpl = arg; bplist_append(bpl, bp); return (0); diff --git a/module/zfs/bptree.c b/module/zfs/bptree.c index 1827a3c4e326..4e9a4825e262 100644 --- a/module/zfs/bptree.c +++ b/module/zfs/bptree.c @@ -147,11 +147,11 @@ bptree_add(objset_t *os, uint64_t obj, blkptr_t *bp, uint64_t birth_txg, dmu_buf_rele(db, FTAG); } -/* ARGSUSED */ static int bptree_visit_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) { + (void) zilog, (void) dnp; int err; struct bptree_args *ba = arg; diff --git a/module/zfs/dbuf.c b/module/zfs/dbuf.c index b21ea8d358ea..a547fbb03e67 100644 --- a/module/zfs/dbuf.c +++ b/module/zfs/dbuf.c @@ -279,10 +279,10 @@ static unsigned long dbuf_metadata_cache_target_bytes(void); uint_t dbuf_cache_hiwater_pct = 10; uint_t dbuf_cache_lowater_pct = 10; -/* ARGSUSED */ static int dbuf_cons(void *vdb, void *unused, int kmflag) { + (void) unused, (void) kmflag; dmu_buf_impl_t *db = vdb; bzero(db, sizeof (dmu_buf_impl_t)); @@ -295,10 +295,10 @@ dbuf_cons(void *vdb, void *unused, int kmflag) return (0); } -/* ARGSUSED */ static void dbuf_dest(void *vdb, void *unused) { + (void) unused; dmu_buf_impl_t *db = vdb; mutex_destroy(&db->db_mtx); rw_destroy(&db->db_rwlock); @@ -720,10 +720,10 @@ dbuf_evict_one(void) * of the dbuf cache is at or below the maximum size. Once the dbuf is aged * out of the cache it is destroyed and becomes eligible for arc eviction. */ -/* ARGSUSED */ static void dbuf_evict_thread(void *unused) { + (void) unused; callb_cpr_t cpr; CALLB_CPR_INIT(&cpr, &dbuf_evict_lock, callb_generic_cpr, FTAG); @@ -1276,6 +1276,7 @@ static void dbuf_read_done(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, arc_buf_t *buf, void *vdb) { + (void) zb, (void) bp; dmu_buf_impl_t *db = vdb; mutex_enter(&db->db_mtx); @@ -1369,7 +1370,7 @@ dbuf_handle_indirect_hole(dmu_buf_impl_t *db, dnode_t *dn) * was taken, ENOENT if no action was taken. */ static int -dbuf_read_hole(dmu_buf_impl_t *db, dnode_t *dn, uint32_t flags) +dbuf_read_hole(dmu_buf_impl_t *db, dnode_t *dn) { ASSERT(MUTEX_HELD(&db->db_mtx)); @@ -1487,7 +1488,7 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags, goto early_unlock; } - err = dbuf_read_hole(db, dn, flags); + err = dbuf_read_hole(db, dn); if (err == 0) goto early_unlock; @@ -2617,10 +2618,10 @@ dbuf_override_impl(dmu_buf_impl_t *db, const blkptr_t *bp, dmu_tx_t *tx) dl->dr_overridden_by.blk_birth = dr->dr_txg; } -/* ARGSUSED */ void dmu_buf_fill_done(dmu_buf_t *dbuf, dmu_tx_t *tx) { + (void) tx; dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; dbuf_states_t old_state; mutex_enter(&db->db_mtx); @@ -3137,6 +3138,7 @@ static void dbuf_issue_final_prefetch_done(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *iobp, arc_buf_t *abuf, void *private) { + (void) zio, (void) zb, (void) iobp; dbuf_prefetch_arg_t *dpa = private; dbuf_prefetch_fini(dpa, B_TRUE); @@ -3185,6 +3187,7 @@ static void dbuf_prefetch_indirect_done(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *iobp, arc_buf_t *abuf, void *private) { + (void) zb, (void) iobp; dbuf_prefetch_arg_t *dpa = private; ASSERT3S(dpa->dpa_zb.zb_level, <, dpa->dpa_curlevel); @@ -4451,10 +4454,10 @@ dbuf_sync_list(list_t *list, int level, dmu_tx_t *tx) } } -/* ARGSUSED */ static void dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) { + (void) buf; dmu_buf_impl_t *db = vdb; dnode_t *dn; blkptr_t *bp = zio->io_bp; @@ -4542,7 +4545,6 @@ dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) dmu_buf_unlock_parent(db, dblt, FTAG); } -/* ARGSUSED */ /* * This function gets called just prior to running through the compression * stage of the zio pipeline. If we're an indirect block comprised of only @@ -4553,6 +4555,7 @@ dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) static void dbuf_write_children_ready(zio_t *zio, arc_buf_t *buf, void *vdb) { + (void) zio, (void) buf; dmu_buf_impl_t *db = vdb; dnode_t *dn; blkptr_t *bp; @@ -4596,10 +4599,10 @@ dbuf_write_children_ready(zio_t *zio, arc_buf_t *buf, void *vdb) * so this callback allows us to retire dirty space gradually, as the physical * i/os complete. */ -/* ARGSUSED */ static void dbuf_write_physdone(zio_t *zio, arc_buf_t *buf, void *arg) { + (void) buf; dmu_buf_impl_t *db = arg; objset_t *os = db->db_objset; dsl_pool_t *dp = dmu_objset_pool(os); @@ -4618,10 +4621,10 @@ dbuf_write_physdone(zio_t *zio, arc_buf_t *buf, void *arg) dsl_pool_undirty_space(dp, delta, zio->io_txg); } -/* ARGSUSED */ static void dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) { + (void) buf; dmu_buf_impl_t *db = vdb; blkptr_t *bp_orig = &zio->io_bp_orig; blkptr_t *bp = db->db_blkptr; diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c index 8302d506146f..0c528f68ccf4 100644 --- a/module/zfs/dmu.c +++ b/module/zfs/dmu.c @@ -812,13 +812,14 @@ get_next_chunk(dnode_t *dn, uint64_t *start, uint64_t minimum, uint64_t *l1blks) * otherwise return false. * Used below in dmu_free_long_range_impl() to enable abort when unmounting */ -/*ARGSUSED*/ static boolean_t dmu_objset_zfs_unmounting(objset_t *os) { #ifdef _KERNEL if (dmu_objset_type(os) == DMU_OST_ZFS) return (zfs_get_vfs_flag_unmounted(os)); +#else + (void) os; #endif return (B_FALSE); } @@ -1502,10 +1503,10 @@ typedef struct { dmu_tx_t *dsa_tx; } dmu_sync_arg_t; -/* ARGSUSED */ static void dmu_sync_ready(zio_t *zio, arc_buf_t *buf, void *varg) { + (void) buf; dmu_sync_arg_t *dsa = varg; dmu_buf_t *db = dsa->dsa_zgd->zgd_db; blkptr_t *bp = zio->io_bp; @@ -1530,10 +1531,10 @@ dmu_sync_late_arrival_ready(zio_t *zio) dmu_sync_ready(zio, NULL, zio->io_private); } -/* ARGSUSED */ static void dmu_sync_done(zio_t *zio, arc_buf_t *buf, void *varg) { + (void) buf; dmu_sync_arg_t *dsa = varg; dbuf_dirty_record_t *dr = dsa->dsa_dr; dmu_buf_impl_t *db = dr->dr_dbuf; @@ -2274,10 +2275,10 @@ byteswap_uint16_array(void *vbuf, size_t size) buf[i] = BSWAP_16(buf[i]); } -/* ARGSUSED */ void byteswap_uint8_array(void *vbuf, size_t size) { + (void) vbuf, (void) size; } void diff --git a/module/zfs/dmu_diff.c b/module/zfs/dmu_diff.c index a573a2e1bd41..1382da267d8a 100644 --- a/module/zfs/dmu_diff.c +++ b/module/zfs/dmu_diff.c @@ -108,11 +108,11 @@ report_dnode(dmu_diffarg_t *da, uint64_t object, dnode_phys_t *dnp) (((uint64_t)dnp->dn_datablkszsec) << (SPA_MINBLOCKSHIFT + \ (level) * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT))) -/* ARGSUSED */ static int diff_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) { + (void) zilog; dmu_diffarg_t *da = arg; int err = 0; diff --git a/module/zfs/dmu_objset.c b/module/zfs/dmu_objset.c index af107fb8ad63..b9380890230c 100644 --- a/module/zfs/dmu_objset.c +++ b/module/zfs/dmu_objset.c @@ -721,9 +721,9 @@ static int dmu_objset_own_impl(dsl_dataset_t *ds, dmu_objset_type_t type, boolean_t readonly, boolean_t decrypt, void *tag, objset_t **osp) { - int err; + (void) tag; - err = dmu_objset_from_ds(ds, osp); + int err = dmu_objset_from_ds(ds, osp); if (err != 0) { return (err); } else if (type != DMU_OST_ANY && type != (*osp)->os_phys->os_type) { @@ -1127,7 +1127,6 @@ typedef struct dmu_objset_create_arg { dsl_crypto_params_t *doca_dcp; } dmu_objset_create_arg_t; -/*ARGSUSED*/ static int dmu_objset_create_check(void *arg, dmu_tx_t *tx) { @@ -1323,7 +1322,6 @@ typedef struct dmu_objset_clone_arg { proc_t *doca_proc; } dmu_objset_clone_arg_t; -/*ARGSUSED*/ static int dmu_objset_clone_check(void *arg, dmu_tx_t *tx) { @@ -1535,10 +1533,10 @@ dmu_objset_sync_dnodes(multilist_sublist_t *list, dmu_tx_t *tx) } } -/* ARGSUSED */ static void dmu_objset_write_ready(zio_t *zio, arc_buf_t *abuf, void *arg) { + (void) abuf; blkptr_t *bp = zio->io_bp; objset_t *os = arg; dnode_phys_t *dnp = &os->os_phys->os_meta_dnode; @@ -1566,10 +1564,10 @@ dmu_objset_write_ready(zio_t *zio, arc_buf_t *abuf, void *arg) rrw_exit(&os->os_dsl_dataset->ds_bp_rwlock, FTAG); } -/* ARGSUSED */ static void dmu_objset_write_done(zio_t *zio, arc_buf_t *abuf, void *arg) { + (void) abuf; blkptr_t *bp = zio->io_bp; blkptr_t *bp_orig = &zio->io_bp_orig; objset_t *os = arg; diff --git a/module/zfs/dmu_redact.c b/module/zfs/dmu_redact.c index fdbdf7d6e868..7efe423d35f0 100644 --- a/module/zfs/dmu_redact.c +++ b/module/zfs/dmu_redact.c @@ -249,11 +249,11 @@ zfs_get_deleteq(objset_t *os) * Third, if there is a deleted object, we need to create a redaction record for * all of the blocks in that object. */ -/*ARGSUSED*/ static int redact_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, const zbookmark_phys_t *zb, const struct dnode_phys *dnp, void *arg) { + (void) spa, (void) zilog; struct redact_thread_arg *rta = arg; struct redact_record *record; diff --git a/module/zfs/dmu_send.c b/module/zfs/dmu_send.c index 58a72a68a65c..390afba92680 100644 --- a/module/zfs/dmu_send.c +++ b/module/zfs/dmu_send.c @@ -1099,11 +1099,11 @@ range_alloc(enum type type, uint64_t object, uint64_t start_blkid, * This is the callback function to traverse_dataset that acts as a worker * thread for dmu_send_impl. */ -/*ARGSUSED*/ static int send_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, const zbookmark_phys_t *zb, const struct dnode_phys *dnp, void *arg) { + (void) zilog; struct send_thread_arg *sta = arg; struct send_range *record; @@ -2157,6 +2157,7 @@ setup_resume_points(struct dmu_send_params *dspp, struct send_merge_thread_arg *smt_arg, boolean_t resuming, objset_t *os, redaction_list_t *redact_rl, nvlist_t *nvl) { + (void) smt_arg; dsl_dataset_t *to_ds = dspp->to_ds; int err = 0; diff --git a/module/zfs/dmu_traverse.c b/module/zfs/dmu_traverse.c index 862c0bf404ad..2f1c2978b3be 100644 --- a/module/zfs/dmu_traverse.c +++ b/module/zfs/dmu_traverse.c @@ -560,11 +560,11 @@ traverse_dnode(traverse_data_t *td, const blkptr_t *bp, const dnode_phys_t *dnp, return (err); } -/* ARGSUSED */ static int traverse_prefetcher(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) { + (void) zilog, (void) dnp; prefetch_data_t *pfd = arg; int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE; arc_flags_t aflags = ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH | diff --git a/module/zfs/dmu_zfetch.c b/module/zfs/dmu_zfetch.c index a26b0d739921..7566663bd3ad 100644 --- a/module/zfs/dmu_zfetch.c +++ b/module/zfs/dmu_zfetch.c @@ -260,6 +260,7 @@ dmu_zfetch_stream_create(zfetch_t *zf, uint64_t blkid) static void dmu_zfetch_stream_done(void *arg, boolean_t io_issued) { + (void) io_issued; zstream_t *zs = arg; if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0) diff --git a/module/zfs/dnode.c b/module/zfs/dnode.c index 7044c1fc6342..8e55d5447975 100644 --- a/module/zfs/dnode.c +++ b/module/zfs/dnode.c @@ -108,12 +108,11 @@ dbuf_compare(const void *x1, const void *x2) return (TREE_PCMP(d1, d2)); } -/* ARGSUSED */ static int dnode_cons(void *arg, void *unused, int kmflag) { + (void) unused, (void) kmflag; dnode_t *dn = arg; - int i; rw_init(&dn->dn_struct_rwlock, NULL, RW_NOLOCKDEP, NULL); mutex_init(&dn->dn_mtx, NULL, MUTEX_DEFAULT, NULL); @@ -139,7 +138,7 @@ dnode_cons(void *arg, void *unused, int kmflag) bzero(&dn->dn_next_blksz[0], sizeof (dn->dn_next_blksz)); bzero(&dn->dn_next_maxblkid[0], sizeof (dn->dn_next_maxblkid)); - for (i = 0; i < TXG_SIZE; i++) { + for (int i = 0; i < TXG_SIZE; i++) { multilist_link_init(&dn->dn_dirty_link[i]); dn->dn_free_ranges[i] = NULL; list_create(&dn->dn_dirty_records[i], @@ -174,11 +173,10 @@ dnode_cons(void *arg, void *unused, int kmflag) return (0); } -/* ARGSUSED */ static void dnode_dest(void *arg, void *unused) { - int i; + (void) unused; dnode_t *dn = arg; rw_destroy(&dn->dn_struct_rwlock); @@ -190,7 +188,7 @@ dnode_dest(void *arg, void *unused) zfs_refcount_destroy(&dn->dn_tx_holds); ASSERT(!list_link_active(&dn->dn_link)); - for (i = 0; i < TXG_SIZE; i++) { + for (int i = 0; i < TXG_SIZE; i++) { ASSERT(!multilist_link_active(&dn->dn_dirty_link[i])); ASSERT3P(dn->dn_free_ranges[i], ==, NULL); list_destroy(&dn->dn_dirty_records[i]); @@ -889,7 +887,6 @@ dnode_move_impl(dnode_t *odn, dnode_t *ndn) odn->dn_moved = (uint8_t)-1; } -/*ARGSUSED*/ static kmem_cbrc_t dnode_move(void *buf, void *newbuf, size_t size, void *arg) { diff --git a/module/zfs/dsl_bookmark.c b/module/zfs/dsl_bookmark.c index bead7da2237f..b8e3523ffc2d 100644 --- a/module/zfs/dsl_bookmark.c +++ b/module/zfs/dsl_bookmark.c @@ -1203,7 +1203,6 @@ dsl_redaction_list_long_rele(redaction_list_t *rl, void *tag) (void) zfs_refcount_remove(&rl->rl_longholds, tag); } -/* ARGSUSED */ static void redaction_list_evict_sync(void *rlu) { @@ -1470,10 +1469,11 @@ dsl_bookmark_next_changed(dsl_dataset_t *head, dsl_dataset_t *origin, * Adjust the FBN of any bookmarks that reference this block, whose "next" * is the head dataset. */ -/* ARGSUSED */ void dsl_bookmark_block_killed(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx) { + (void) tx; + /* * Iterate over bookmarks whose "next" is the head dataset. */ diff --git a/module/zfs/dsl_dataset.c b/module/zfs/dsl_dataset.c index f99964511aa6..4623bcec4ea8 100644 --- a/module/zfs/dsl_dataset.c +++ b/module/zfs/dsl_dataset.c @@ -2943,11 +2943,11 @@ typedef struct dsl_dataset_rename_snapshot_arg { dmu_tx_t *ddrsa_tx; } dsl_dataset_rename_snapshot_arg_t; -/* ARGSUSED */ static int dsl_dataset_rename_snapshot_check_impl(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) { + (void) dp; dsl_dataset_rename_snapshot_arg_t *ddrsa = arg; int error; uint64_t val; @@ -4305,7 +4305,6 @@ typedef struct dsl_dataset_set_qr_arg { } dsl_dataset_set_qr_arg_t; -/* ARGSUSED */ static int dsl_dataset_set_refquota_check(void *arg, dmu_tx_t *tx) { @@ -4512,7 +4511,6 @@ typedef struct dsl_dataset_set_compression_arg { uint64_t ddsca_value; } dsl_dataset_set_compression_arg_t; -/* ARGSUSED */ static int dsl_dataset_set_compression_check(void *arg, dmu_tx_t *tx) { diff --git a/module/zfs/dsl_destroy.c b/module/zfs/dsl_destroy.c index a2748197f29d..b32929b3320c 100644 --- a/module/zfs/dsl_destroy.c +++ b/module/zfs/dsl_destroy.c @@ -699,11 +699,11 @@ struct killarg { dmu_tx_t *tx; }; -/* ARGSUSED */ static int kill_blkptr(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) { + (void) spa, (void) dnp; struct killarg *ka = arg; dmu_tx_t *tx = ka->tx; @@ -1246,10 +1246,10 @@ dsl_destroy_head(const char *name) * inconsistent datasets, even if we encounter an error trying to * process one of them. */ -/* ARGSUSED */ int dsl_destroy_inconsistent(const char *dsname, void *arg) { + (void) arg; objset_t *os; if (dmu_objset_hold(dsname, FTAG, &os) == 0) { diff --git a/module/zfs/dsl_dir.c b/module/zfs/dsl_dir.c index 1101341451a5..e4ff7d8e754f 100644 --- a/module/zfs/dsl_dir.c +++ b/module/zfs/dsl_dir.c @@ -764,6 +764,8 @@ dsl_enforce_ds_ss_limits(dsl_dir_t *dd, zfs_prop_t prop, */ if (secpolicy_zfs_proc(cr, proc) == 0) return (ENFORCE_NEVER); +#else + (void) proc; #endif if ((obj = dsl_dir_phys(dd)->dd_head_dataset_obj) == 0) @@ -1895,10 +1897,10 @@ typedef struct dsl_valid_rename_arg { int nest_delta; } dsl_valid_rename_arg_t; -/* ARGSUSED */ static int dsl_valid_rename(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) { + (void) dp; dsl_valid_rename_arg_t *dvra = arg; char namebuf[ZFS_MAX_DATASET_NAME_LEN]; @@ -2395,6 +2397,7 @@ dsl_dir_activity_in_progress(dsl_dir_t *dd, dsl_dataset_t *ds, * The delete queue is ZPL specific, and libzpool doesn't have * it. It doesn't make sense to wait for it. */ + (void) ds; *in_progress = B_FALSE; break; #endif diff --git a/module/zfs/dsl_pool.c b/module/zfs/dsl_pool.c index e66c136a9e02..9301641aad2d 100644 --- a/module/zfs/dsl_pool.c +++ b/module/zfs/dsl_pool.c @@ -455,8 +455,8 @@ dsl_pool_destroy_obsolete_bpobj(dsl_pool_t *dp, dmu_tx_t *tx) } dsl_pool_t * -dsl_pool_create(spa_t *spa, nvlist_t *zplprops, dsl_crypto_params_t *dcp, - uint64_t txg) +dsl_pool_create(spa_t *spa, nvlist_t *zplprops __attribute__((unused)), + dsl_crypto_params_t *dcp, uint64_t txg) { int err; dsl_pool_t *dp = dsl_pool_open_impl(spa, txg); diff --git a/module/zfs/dsl_prop.c b/module/zfs/dsl_prop.c index dfa04d7681be..0089edf8683a 100644 --- a/module/zfs/dsl_prop.c +++ b/module/zfs/dsl_prop.c @@ -504,10 +504,10 @@ dsl_prop_hascb(dsl_dataset_t *ds) return (!list_is_empty(&ds->ds_prop_cbs)); } -/* ARGSUSED */ static int dsl_prop_notify_all_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) { + (void) arg; dsl_dir_t *dd = ds->ds_dir; dsl_prop_record_t *pr; dsl_prop_cb_record_t *cbr; diff --git a/module/zfs/dsl_scan.c b/module/zfs/dsl_scan.c index 765998417153..fac658180e1f 100644 --- a/module/zfs/dsl_scan.c +++ b/module/zfs/dsl_scan.c @@ -700,10 +700,10 @@ dsl_scan_sync_state(dsl_scan_t *scn, dmu_tx_t *tx, state_sync_type_t sync_type) } } -/* ARGSUSED */ int dsl_scan_setup_check(void *arg, dmu_tx_t *tx) { + (void) arg; dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; vdev_t *rvd = scn->scn_dp->dp_spa->spa_root_vdev; @@ -854,7 +854,6 @@ dsl_scan(dsl_pool_t *dp, pool_scan_func_t func) dsl_scan_setup_sync, &func, 0, ZFS_SPACE_CHECK_EXTRA_RESERVED)); } -/* ARGSUSED */ static void dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx) { @@ -1001,10 +1000,10 @@ dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx) ASSERT(!dsl_scan_is_running(scn)); } -/* ARGSUSED */ static int dsl_scan_cancel_check(void *arg, dmu_tx_t *tx) { + (void) arg; dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; if (!dsl_scan_is_running(scn)) @@ -1012,10 +1011,10 @@ dsl_scan_cancel_check(void *arg, dmu_tx_t *tx) return (0); } -/* ARGSUSED */ static void dsl_scan_cancel_sync(void *arg, dmu_tx_t *tx) { + (void) arg; dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; dsl_scan_done(scn, B_FALSE, tx); @@ -1377,11 +1376,11 @@ typedef struct zil_scan_arg { zil_header_t *zsa_zh; } zil_scan_arg_t; -/* ARGSUSED */ static int dsl_scan_zil_block(zilog_t *zilog, const blkptr_t *bp, void *arg, uint64_t claim_txg) { + (void) zilog; zil_scan_arg_t *zsa = arg; dsl_pool_t *dp = zsa->zsa_dp; dsl_scan_t *scn = dp->dp_scan; @@ -1408,11 +1407,11 @@ dsl_scan_zil_block(zilog_t *zilog, const blkptr_t *bp, void *arg, return (0); } -/* ARGSUSED */ static int dsl_scan_zil_record(zilog_t *zilog, const lr_t *lrc, void *arg, uint64_t claim_txg) { + (void) zilog; if (lrc->lrc_txtype == TX_WRITE) { zil_scan_arg_t *zsa = arg; dsl_pool_t *dp = zsa->zsa_dp; @@ -1637,6 +1636,7 @@ static void dsl_scan_prefetch_cb(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, arc_buf_t *buf, void *private) { + (void) zio; scan_prefetch_ctx_t *spc = private; dsl_scan_t *scn = spc->spc_scn; spa_t *spa = scn->scn_dp->dp_spa; @@ -1696,7 +1696,6 @@ dsl_scan_prefetch_cb(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, scan_prefetch_ctx_rele(spc, scn); } -/* ARGSUSED */ static void dsl_scan_prefetch_thread(void *arg) { @@ -2336,7 +2335,6 @@ dsl_scan_ds_clone_swapped(dsl_dataset_t *ds1, dsl_dataset_t *ds2, dmu_tx_t *tx) dsl_scan_sync_state(scn, tx, SYNC_CACHED); } -/* ARGSUSED */ static int enqueue_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) { @@ -2520,10 +2518,10 @@ dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx) dsl_dataset_rele(ds, FTAG); } -/* ARGSUSED */ static int enqueue_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) { + (void) arg; dsl_dataset_t *ds; int err; dsl_scan_t *scn = dp->dp_scan; @@ -2559,16 +2557,15 @@ enqueue_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) return (0); } -/* ARGSUSED */ void dsl_scan_ddt_entry(dsl_scan_t *scn, enum zio_checksum checksum, ddt_entry_t *dde, dmu_tx_t *tx) { + (void) tx; const ddt_key_t *ddk = &dde->dde_key; ddt_phys_t *ddp = dde->dde_phys; blkptr_t bp; zbookmark_phys_t zb = { 0 }; - int p; if (!dsl_scan_is_running(scn)) return; @@ -2587,7 +2584,7 @@ dsl_scan_ddt_entry(dsl_scan_t *scn, enum zio_checksum checksum, if (scn->scn_done_txg != 0) return; - for (p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { + for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { if (ddp->ddp_phys_birth == 0 || ddp->ddp_phys_birth > scn->scn_phys.scn_max_txg) continue; diff --git a/module/zfs/dsl_synctask.c b/module/zfs/dsl_synctask.c index 148e8fff2437..9fc9d4011d11 100644 --- a/module/zfs/dsl_synctask.c +++ b/module/zfs/dsl_synctask.c @@ -32,10 +32,10 @@ #define DST_AVG_BLKSHIFT 14 -/* ARGSUSED */ static int dsl_null_checkfunc(void *arg, dmu_tx_t *tx) { + (void) arg, (void) tx; return (0); } diff --git a/module/zfs/fm.c b/module/zfs/fm.c index b8a1c7c8a5ca..7981568b8df7 100644 --- a/module/zfs/fm.c +++ b/module/zfs/fm.c @@ -483,17 +483,17 @@ zfs_zevent_destroy(zfs_zevent_t *ze) /* * Wrappers for FM nvlist allocators */ -/* ARGSUSED */ static void * i_fm_alloc(nv_alloc_t *nva, size_t size) { + (void) nva; return (kmem_zalloc(size, KM_SLEEP)); } -/* ARGSUSED */ static void i_fm_free(nv_alloc_t *nva, void *buf, size_t size) { + (void) nva; kmem_free(buf, size); } diff --git a/module/zfs/gzip.c b/module/zfs/gzip.c index e2c6e59969d6..48191241bd7d 100644 --- a/module/zfs/gzip.c +++ b/module/zfs/gzip.c @@ -83,10 +83,10 @@ gzip_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) return ((size_t)dstlen); } -/*ARGSUSED*/ int gzip_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { + (void) n; zlen_t dstlen = d_len; ASSERT(d_len >= s_len); diff --git a/module/zfs/lz4.c b/module/zfs/lz4.c index 4b46e694891a..eba8f01b5065 100644 --- a/module/zfs/lz4.c +++ b/module/zfs/lz4.c @@ -46,11 +46,11 @@ static int LZ4_compress64kCtx(void *ctx, const char *source, char *dest, static kmem_cache_t *lz4_cache; -/*ARGSUSED*/ size_t lz4_compress_zfs(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { + (void) n; uint32_t bufsiz; char *dest = d_start; @@ -74,11 +74,11 @@ lz4_compress_zfs(void *s_start, void *d_start, size_t s_len, return (bufsiz + sizeof (bufsiz)); } -/*ARGSUSED*/ int lz4_decompress_zfs(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { + (void) n; const char *src = s_start; uint32_t bufsiz = BE_IN32(src); @@ -463,7 +463,6 @@ LZ4_NbCommonBytes(register U32 val) /* Compression functions */ -/*ARGSUSED*/ static int LZ4_compressCtx(void *ctx, const char *source, char *dest, int isize, int osize) @@ -654,7 +653,6 @@ LZ4_compressCtx(void *ctx, const char *source, char *dest, int isize, HASHLOG64K)) #define LZ4_HASH64K_VALUE(p) LZ4_HASH64K_FUNCTION(A32(p)) -/*ARGSUSED*/ static int LZ4_compress64kCtx(void *ctx, const char *source, char *dest, int isize, int osize) diff --git a/module/zfs/lzjb.c b/module/zfs/lzjb.c index a478e64c5141..1c536b110318 100644 --- a/module/zfs/lzjb.c +++ b/module/zfs/lzjb.c @@ -45,10 +45,10 @@ #define OFFSET_MASK ((1 << (16 - MATCH_BITS)) - 1) #define LEMPEL_SIZE 1024 -/*ARGSUSED*/ size_t lzjb_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { + (void) n; uchar_t *src = s_start; uchar_t *dst = d_start; uchar_t *cpy; @@ -100,10 +100,10 @@ lzjb_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) return (dst - (uchar_t *)d_start); } -/*ARGSUSED*/ int lzjb_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { + (void) s_len, (void) n; uchar_t *src = s_start; uchar_t *dst = d_start; uchar_t *d_end = (uchar_t *)d_start + d_len; diff --git a/module/zfs/metaslab.c b/module/zfs/metaslab.c index df0d83327c0b..f367bea98267 100644 --- a/module/zfs/metaslab.c +++ b/module/zfs/metaslab.c @@ -1406,7 +1406,6 @@ metaslab_size_tree_full_load(range_tree_t *rt) * Create any block allocator specific components. The current allocators * rely on using both a size-ordered range_tree_t and an array of uint64_t's. */ -/* ARGSUSED */ static void metaslab_rt_create(range_tree_t *rt, void *arg) { @@ -1431,10 +1430,10 @@ metaslab_rt_create(range_tree_t *rt, void *arg) mrap->mra_floor_shift = metaslab_by_size_min_shift; } -/* ARGSUSED */ static void metaslab_rt_destroy(range_tree_t *rt, void *arg) { + (void) rt; metaslab_rt_arg_t *mrap = arg; zfs_btree_t *size_tree = mrap->mra_bt; @@ -1442,7 +1441,6 @@ metaslab_rt_destroy(range_tree_t *rt, void *arg) kmem_free(mrap, sizeof (*mrap)); } -/* ARGSUSED */ static void metaslab_rt_add(range_tree_t *rt, range_seg_t *rs, void *arg) { @@ -1456,7 +1454,6 @@ metaslab_rt_add(range_tree_t *rt, range_seg_t *rs, void *arg) zfs_btree_add(size_tree, rs); } -/* ARGSUSED */ static void metaslab_rt_remove(range_tree_t *rt, range_seg_t *rs, void *arg) { @@ -1470,7 +1467,6 @@ metaslab_rt_remove(range_tree_t *rt, range_seg_t *rs, void *arg) zfs_btree_remove(size_tree, rs); } -/* ARGSUSED */ static void metaslab_rt_vacate(range_tree_t *rt, void *arg) { @@ -2240,6 +2236,8 @@ metaslab_potentially_evict(metaslab_class_t *mc) inuse = spl_kmem_cache_inuse(zfs_btree_leaf_cache); } } +#else + (void) mc; #endif } @@ -4719,7 +4717,6 @@ metaslab_active_mask_verify(metaslab_t *msp) } } -/* ARGSUSED */ static uint64_t metaslab_group_alloc_normal(metaslab_group_t *mg, zio_alloc_list_t *zal, uint64_t asize, uint64_t txg, boolean_t want_unique, dva_t *dva, int d, @@ -5345,11 +5342,11 @@ metaslab_free_concrete(vdev_t *vd, uint64_t offset, uint64_t asize, mutex_exit(&msp->ms_lock); } -/* ARGSUSED */ void metaslab_free_impl_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset, uint64_t size, void *arg) { + (void) inner_offset; boolean_t *checkpoint = arg; ASSERT3P(checkpoint, !=, NULL); @@ -5715,11 +5712,11 @@ typedef struct metaslab_claim_cb_arg_t { int mcca_error; } metaslab_claim_cb_arg_t; -/* ARGSUSED */ static void metaslab_claim_impl_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset, uint64_t size, void *arg) { + (void) inner_offset; metaslab_claim_cb_arg_t *mcca_arg = arg; if (mcca_arg->mcca_error == 0) { @@ -5971,11 +5968,12 @@ metaslab_fastwrite_unmark(spa_t *spa, const blkptr_t *bp) spa_config_exit(spa, SCL_VDEV, FTAG); } -/* ARGSUSED */ static void metaslab_check_free_impl_cb(uint64_t inner, vdev_t *vd, uint64_t offset, uint64_t size, void *arg) { + (void) inner, (void) arg; + if (vd->vdev_ops == &vdev_indirect_ops) return; diff --git a/module/zfs/range_tree.c b/module/zfs/range_tree.c index 595918e5a742..67910f9ffde0 100644 --- a/module/zfs/range_tree.c +++ b/module/zfs/range_tree.c @@ -741,7 +741,6 @@ range_tree_is_empty(range_tree_t *rt) return (range_tree_space(rt) == 0); } -/* ARGSUSED */ void rt_btree_create(range_tree_t *rt, void *arg) { @@ -764,35 +763,34 @@ rt_btree_create(range_tree_t *rt, void *arg) zfs_btree_create(size_tree, rt->rt_btree_compare, size); } -/* ARGSUSED */ void rt_btree_destroy(range_tree_t *rt, void *arg) { + (void) rt; zfs_btree_t *size_tree = arg; ASSERT0(zfs_btree_numnodes(size_tree)); zfs_btree_destroy(size_tree); } -/* ARGSUSED */ void rt_btree_add(range_tree_t *rt, range_seg_t *rs, void *arg) { + (void) rt; zfs_btree_t *size_tree = arg; zfs_btree_add(size_tree, rs); } -/* ARGSUSED */ void rt_btree_remove(range_tree_t *rt, range_seg_t *rs, void *arg) { + (void) rt; zfs_btree_t *size_tree = arg; zfs_btree_remove(size_tree, rs); } -/* ARGSUSED */ void rt_btree_vacate(range_tree_t *rt, void *arg) { diff --git a/module/zfs/sa.c b/module/zfs/sa.c index 2604a7513ecf..b69b0c68f13c 100644 --- a/module/zfs/sa.c +++ b/module/zfs/sa.c @@ -212,20 +212,20 @@ sa_attr_type_t sa_dummy_zpl_layout[] = { 0 }; static int sa_legacy_attr_count = ARRAY_SIZE(sa_legacy_attrs); static kmem_cache_t *sa_cache = NULL; -/*ARGSUSED*/ static int sa_cache_constructor(void *buf, void *unused, int kmflag) { + (void) unused, (void) kmflag; sa_handle_t *hdl = buf; mutex_init(&hdl->sa_lock, NULL, MUTEX_DEFAULT, NULL); return (0); } -/*ARGSUSED*/ static void sa_cache_destructor(void *buf, void *unused) { + (void) unused; sa_handle_t *hdl = buf; mutex_destroy(&hdl->sa_lock); } @@ -1218,11 +1218,11 @@ sa_attr_iter(objset_t *os, sa_hdr_phys_t *hdr, dmu_object_type_t type, } } -/*ARGSUSED*/ static void sa_byteswap_cb(void *hdr, void *attr_addr, sa_attr_type_t attr, uint16_t length, int length_idx, boolean_t variable_length, void *userp) { + (void) hdr, (void) length_idx, (void) variable_length; sa_handle_t *hdl = userp; sa_os_t *sa = hdl->sa_os->os_sa; @@ -1309,10 +1309,10 @@ sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype) return (0); } -/*ARGSUSED*/ static void sa_evict_sync(void *dbu) { + (void) dbu; panic("evicting sa dbuf\n"); } diff --git a/module/zfs/sha256.c b/module/zfs/sha256.c index d297768eada5..c5b033cf019d 100644 --- a/module/zfs/sha256.c +++ b/module/zfs/sha256.c @@ -41,11 +41,11 @@ sha_incremental(void *buf, size_t size, void *arg) return (0); } -/*ARGSUSED*/ void abd_checksum_SHA256(abd_t *abd, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { + (void) ctx_template; int ret; SHA2_CTX ctx; zio_cksum_t tmp; @@ -78,11 +78,11 @@ abd_checksum_SHA256(abd_t *abd, uint64_t size, zcp->zc_word[3] = BE_64(tmp.zc_word[3]); } -/*ARGSUSED*/ void abd_checksum_SHA512_native(abd_t *abd, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { + (void) ctx_template; SHA2_CTX ctx; SHA2Init(SHA512_256, &ctx); @@ -90,7 +90,6 @@ abd_checksum_SHA512_native(abd_t *abd, uint64_t size, SHA2Final(zcp, &ctx); } -/*ARGSUSED*/ void abd_checksum_SHA512_byteswap(abd_t *abd, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) diff --git a/module/zfs/spa.c b/module/zfs/spa.c index a02fd198bed0..9bfaefb6c214 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -834,7 +834,6 @@ spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx) } } -/*ARGSUSED*/ static int spa_change_guid_check(void *arg, dmu_tx_t *tx) { @@ -2281,11 +2280,12 @@ int spa_load_verify_shift = 4; int spa_load_verify_metadata = B_TRUE; int spa_load_verify_data = B_TRUE; -/*ARGSUSED*/ static int spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) { + (void) zilog, (void) dnp; + if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp) || BP_IS_REDACTED(bp)) return (0); @@ -2317,10 +2317,11 @@ spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, return (0); } -/* ARGSUSED */ static int verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) { + (void) dp, (void) arg; + if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN) return (SET_ERROR(ENAMETOOLONG)); @@ -2450,10 +2451,10 @@ spa_livelist_delete_check(spa_t *spa) return (spa->spa_livelists_to_delete != 0); } -/* ARGSUSED */ static boolean_t spa_livelist_delete_cb_check(void *arg, zthr_t *z) { + (void) z; spa_t *spa = arg; return (spa_livelist_delete_check(spa)); } @@ -2545,7 +2546,6 @@ livelist_delete_sync(void *arg, dmu_tx_t *tx) * be freed. Then, call a synctask which performs the actual frees and updates * the pool-wide livelist data. */ -/* ARGSUSED */ static void spa_livelist_delete_cb(void *arg, zthr_t *z) { @@ -2791,7 +2791,6 @@ spa_livelist_condense_cb(void *arg, zthr_t *t) zfs_livelist_condense_zthr_cancel++; } -/* ARGSUSED */ /* * Check that there is something to condense but that a condense is not * already in progress and that condensing has not been cancelled. @@ -2799,6 +2798,7 @@ spa_livelist_condense_cb(void *arg, zthr_t *t) static boolean_t spa_livelist_condense_cb_check(void *arg, zthr_t *z) { + (void) z; spa_t *spa = arg; if ((spa->spa_to_condense.ds != NULL) && (spa->spa_to_condense.syncing == B_FALSE) && @@ -9474,6 +9474,7 @@ spa_upgrade(spa_t *spa, uint64_t version) boolean_t spa_has_spare(spa_t *spa, uint64_t guid) { + (void) spa; int i; uint64_t spareguid; spa_aux_vdev_t *sav = &spa->spa_spares; @@ -9826,6 +9827,8 @@ spa_event_create(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) ev = kmem_alloc(sizeof (sysevent_t), KM_SLEEP); ev->resource = resource; } +#else + (void) spa, (void) vd, (void) hist_nvl, (void) name; #endif return (ev); } @@ -9838,6 +9841,8 @@ spa_event_post(sysevent_t *ev) zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb); kmem_free(ev, sizeof (*ev)); } +#else + (void) ev; #endif } diff --git a/module/zfs/spa_checkpoint.c b/module/zfs/spa_checkpoint.c index 09f62996853d..ddcdb6801053 100644 --- a/module/zfs/spa_checkpoint.c +++ b/module/zfs/spa_checkpoint.c @@ -380,10 +380,10 @@ spa_checkpoint_discard_is_done(spa_t *spa) return (B_TRUE); } -/* ARGSUSED */ boolean_t spa_checkpoint_discard_thread_check(void *arg, zthr_t *zthr) { + (void) zthr; spa_t *spa = arg; if (!spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) @@ -450,10 +450,10 @@ spa_checkpoint_discard_thread(void *arg, zthr_t *zthr) } -/* ARGSUSED */ static int spa_checkpoint_check(void *arg, dmu_tx_t *tx) { + (void) arg; spa_t *spa = dmu_tx_pool(tx)->dp_spa; if (!spa_feature_is_enabled(spa, SPA_FEATURE_POOL_CHECKPOINT)) @@ -474,10 +474,10 @@ spa_checkpoint_check(void *arg, dmu_tx_t *tx) return (0); } -/* ARGSUSED */ static void spa_checkpoint_sync(void *arg, dmu_tx_t *tx) { + (void) arg; dsl_pool_t *dp = dmu_tx_pool(tx); spa_t *spa = dp->dp_spa; uberblock_t checkpoint = spa->spa_ubsync; @@ -571,10 +571,10 @@ spa_checkpoint(const char *pool) return (error); } -/* ARGSUSED */ static int spa_checkpoint_discard_check(void *arg, dmu_tx_t *tx) { + (void) arg; spa_t *spa = dmu_tx_pool(tx)->dp_spa; if (!spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) @@ -589,10 +589,10 @@ spa_checkpoint_discard_check(void *arg, dmu_tx_t *tx) return (0); } -/* ARGSUSED */ static void spa_checkpoint_discard_sync(void *arg, dmu_tx_t *tx) { + (void) arg; spa_t *spa = dmu_tx_pool(tx)->dp_spa; VERIFY0(zap_remove(spa_meta_objset(spa), DMU_POOL_DIRECTORY_OBJECT, diff --git a/module/zfs/spa_errlog.c b/module/zfs/spa_errlog.c index fa5120eb61b3..c6b28ea7d1b8 100644 --- a/module/zfs/spa_errlog.c +++ b/module/zfs/spa_errlog.c @@ -252,6 +252,8 @@ spa_get_errlog(spa_t *spa, void *uaddr, size_t *count) mutex_exit(&spa->spa_errlist_lock); mutex_exit(&spa->spa_errlog_lock); +#else + (void) spa, (void) uaddr, (void) count; #endif return (ret); diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c index 00515db027d0..bcf6360c25c3 100644 --- a/module/zfs/spa_misc.c +++ b/module/zfs/spa_misc.c @@ -497,6 +497,7 @@ spa_config_tryenter(spa_t *spa, int locks, void *tag, krw_t rw) void spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw) { + (void) tag; int wlocks_held = 0; ASSERT3U(SCL_LOCKS, <, sizeof (wlocks_held) * NBBY); @@ -530,6 +531,7 @@ spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw) void spa_config_exit(spa_t *spa, int locks, const void *tag) { + (void) tag; for (int i = SCL_LOCKS - 1; i >= 0; i--) { spa_config_lock_t *scl = &spa->spa_config_lock[i]; if (!(locks & (1 << i))) diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c index d659ec5bf333..57dcfe72339d 100644 --- a/module/zfs/vdev.c +++ b/module/zfs/vdev.c @@ -262,11 +262,12 @@ vdev_get_mg(vdev_t *vd, metaslab_class_t *mc) return (vd->vdev_mg); } -/* ARGSUSED */ void vdev_default_xlate(vdev_t *vd, const range_seg64_t *logical_rs, range_seg64_t *physical_rs, range_seg64_t *remain_rs) { + (void) vd, (void) remain_rs; + physical_rs->rs_start = logical_rs->rs_start; physical_rs->rs_end = logical_rs->rs_end; } @@ -1768,6 +1769,7 @@ vdev_uses_zvols(vdev_t *vd) static boolean_t vdev_default_open_children_func(vdev_t *vd) { + (void) vd; return (B_TRUE); } @@ -2849,6 +2851,8 @@ boolean_t vdev_default_need_resilver(vdev_t *vd, const dva_t *dva, size_t psize, uint64_t phys_birth) { + (void) dva, (void) psize; + /* Set by sequential resilver. */ if (phys_birth == TXG_UNKNOWN) return (B_TRUE); @@ -4291,6 +4295,8 @@ vdev_get_child_stat(vdev_t *cvd, vdev_stat_t *vs, vdev_stat_t *cvs) static void vdev_get_child_stat_ex(vdev_t *cvd, vdev_stat_ex_t *vsx, vdev_stat_ex_t *cvsx) { + (void) cvd; + int t, b; for (t = 0; t < ZIO_TYPES; t++) { for (b = 0; b < ARRAY_SIZE(vsx->vsx_disk_histo[0]); b++) @@ -4727,6 +4733,7 @@ void vdev_space_update(vdev_t *vd, int64_t alloc_delta, int64_t defer_delta, int64_t space_delta) { + (void) defer_delta; int64_t dspace_delta; spa_t *spa = vd->vdev_spa; vdev_t *rvd = spa->spa_root_vdev; diff --git a/module/zfs/vdev_draid.c b/module/zfs/vdev_draid.c index 2d83c1ac977c..db87e69f2057 100644 --- a/module/zfs/vdev_draid.c +++ b/module/zfs/vdev_draid.c @@ -2201,6 +2201,7 @@ vdev_draid_config_generate(vdev_t *vd, nvlist_t *nv) static int vdev_draid_init(spa_t *spa, nvlist_t *nv, void **tsd) { + (void) spa; uint64_t ndata, nparity, nspares, ngroups; int error; @@ -2429,7 +2430,6 @@ vdev_draid_spare_get_child(vdev_t *vd, uint64_t physical_offset) return (cvd); } -/* ARGSUSED */ static void vdev_draid_spare_close(vdev_t *vd) { @@ -2688,10 +2688,10 @@ vdev_draid_spare_io_start(zio_t *zio) zio_execute(zio); } -/* ARGSUSED */ static void vdev_draid_spare_io_done(zio_t *zio) { + (void) zio; } /* diff --git a/module/zfs/vdev_indirect.c b/module/zfs/vdev_indirect.c index 3237dc4021a0..b541058d94f8 100644 --- a/module/zfs/vdev_indirect.c +++ b/module/zfs/vdev_indirect.c @@ -637,16 +637,15 @@ spa_condense_indirect_generate_new_mapping(vdev_t *vd, } } -/* ARGSUSED */ static boolean_t spa_condense_indirect_thread_check(void *arg, zthr_t *zthr) { + (void) zthr; spa_t *spa = arg; return (spa->spa_condensing_indirect != NULL); } -/* ARGSUSED */ static void spa_condense_indirect_thread(void *arg, zthr_t *zthr) { @@ -941,13 +940,12 @@ vdev_obsolete_counts_are_precise(vdev_t *vd, boolean_t *are_precise) return (error); } -/* ARGSUSED */ static void vdev_indirect_close(vdev_t *vd) { + (void) vd; } -/* ARGSUSED */ static int vdev_indirect_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize, uint64_t *logical_ashift, uint64_t *physical_ashift) diff --git a/module/zfs/vdev_initialize.c b/module/zfs/vdev_initialize.c index e9156c32f384..6ffd0d618fdd 100644 --- a/module/zfs/vdev_initialize.c +++ b/module/zfs/vdev_initialize.c @@ -255,10 +255,11 @@ vdev_initialize_write(vdev_t *vd, uint64_t start, uint64_t size, abd_t *data) * divisible by sizeof (uint64_t), and buf must be 8-byte aligned. The ABD * allocation will guarantee these for us. */ -/* ARGSUSED */ static int vdev_initialize_block_fill(void *buf, size_t len, void *unused) { + (void) unused; + ASSERT0(len % sizeof (uint64_t)); #ifdef _ILP32 for (uint64_t i = 0; i < len; i += sizeof (uint32_t)) { @@ -624,6 +625,7 @@ vdev_initialize_stop_wait_impl(vdev_t *vd) void vdev_initialize_stop_wait(spa_t *spa, list_t *vd_list) { + (void) spa; vdev_t *vd; ASSERT(MUTEX_HELD(&spa_namespace_lock)); diff --git a/module/zfs/vdev_mirror.c b/module/zfs/vdev_mirror.c index 5eb331046953..45b744b2ec89 100644 --- a/module/zfs/vdev_mirror.c +++ b/module/zfs/vdev_mirror.c @@ -880,6 +880,8 @@ static uint64_t vdev_mirror_rebuild_asize(vdev_t *vd, uint64_t start, uint64_t asize, uint64_t max_segment) { + (void) start; + uint64_t psize = MIN(P2ROUNDUP(max_segment, 1 << vd->vdev_ashift), SPA_MAXBLOCKSIZE); diff --git a/module/zfs/vdev_missing.c b/module/zfs/vdev_missing.c index e9145fd012d7..505df23c1fb2 100644 --- a/module/zfs/vdev_missing.c +++ b/module/zfs/vdev_missing.c @@ -42,7 +42,6 @@ #include #include -/* ARGSUSED */ static int vdev_missing_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize, uint64_t *ashift, uint64_t *pshift) @@ -53,6 +52,7 @@ vdev_missing_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize, * VDEV_AUX_BAD_GUID_SUM. So we pretend to succeed, knowing that we * will fail the GUID sum check before ever trying to open the pool. */ + (void) vd; *psize = 0; *max_psize = 0; *ashift = 0; @@ -60,13 +60,12 @@ vdev_missing_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize, return (0); } -/* ARGSUSED */ static void vdev_missing_close(vdev_t *vd) { + (void) vd; } -/* ARGSUSED */ static void vdev_missing_io_start(zio_t *zio) { @@ -74,10 +73,10 @@ vdev_missing_io_start(zio_t *zio) zio_execute(zio); } -/* ARGSUSED */ static void vdev_missing_io_done(zio_t *zio) { + (void) zio; } vdev_ops_t vdev_missing_ops = { diff --git a/module/zfs/vdev_raidz.c b/module/zfs/vdev_raidz.c index 9a7cf665643c..b14e995e3ce1 100644 --- a/module/zfs/vdev_raidz.c +++ b/module/zfs/vdev_raidz.c @@ -545,10 +545,10 @@ vdev_raidz_generate_parity(raidz_map_t *rm) } } -/* ARGSUSED */ static int vdev_raidz_reconst_p_func(void *dbuf, void *sbuf, size_t size, void *private) { + (void) private; uint64_t *dst = dbuf; uint64_t *src = sbuf; int cnt = size / sizeof (src[0]); @@ -560,11 +560,11 @@ vdev_raidz_reconst_p_func(void *dbuf, void *sbuf, size_t size, void *private) return (0); } -/* ARGSUSED */ static int vdev_raidz_reconst_q_pre_func(void *dbuf, void *sbuf, size_t size, void *private) { + (void) private; uint64_t *dst = dbuf; uint64_t *src = sbuf; uint64_t mask; @@ -578,10 +578,10 @@ vdev_raidz_reconst_q_pre_func(void *dbuf, void *sbuf, size_t size, return (0); } -/* ARGSUSED */ static int vdev_raidz_reconst_q_pre_tail_func(void *buf, size_t size, void *private) { + (void) private; uint64_t *dst = buf; uint64_t mask; int cnt = size / sizeof (dst[0]); @@ -2404,6 +2404,8 @@ static void vdev_raidz_xlate(vdev_t *cvd, const range_seg64_t *logical_rs, range_seg64_t *physical_rs, range_seg64_t *remain_rs) { + (void) remain_rs; + vdev_t *raidvd = cvd->vdev_parent; ASSERT(raidvd->vdev_ops == &vdev_raidz_ops); diff --git a/module/zfs/vdev_removal.c b/module/zfs/vdev_removal.c index 8b0c76ff0a14..b51d862f6466 100644 --- a/module/zfs/vdev_removal.c +++ b/module/zfs/vdev_removal.c @@ -1623,10 +1623,10 @@ spa_vdev_remove_suspend(spa_t *spa) mutex_exit(&svr->svr_lock); } -/* ARGSUSED */ static int spa_vdev_remove_cancel_check(void *arg, dmu_tx_t *tx) { + (void) arg; spa_t *spa = dmu_tx_pool(tx)->dp_spa; if (spa->spa_vdev_removal == NULL) @@ -1638,10 +1638,10 @@ spa_vdev_remove_cancel_check(void *arg, dmu_tx_t *tx) * Cancel a removal by freeing all entries from the partial mapping * and marking the vdev as no longer being removing. */ -/* ARGSUSED */ static void spa_vdev_remove_cancel_sync(void *arg, dmu_tx_t *tx) { + (void) arg; spa_t *spa = dmu_tx_pool(tx)->dp_spa; spa_vdev_removal_t *svr = spa->spa_vdev_removal; vdev_t *vd = vdev_lookup_top(spa, svr->svr_vdev_id); diff --git a/module/zfs/vdev_trim.c b/module/zfs/vdev_trim.c index deea7fedd770..2bae33b2b532 100644 --- a/module/zfs/vdev_trim.c +++ b/module/zfs/vdev_trim.c @@ -1003,6 +1003,7 @@ vdev_trim_stop_wait_impl(vdev_t *vd) void vdev_trim_stop_wait(spa_t *spa, list_t *vd_list) { + (void) spa; vdev_t *vd; ASSERT(MUTEX_HELD(&spa_namespace_lock)); diff --git a/module/zfs/zcp.c b/module/zfs/zcp.c index f724b44baf1d..f200b928bc6d 100644 --- a/module/zfs/zcp.c +++ b/module/zfs/zcp.c @@ -769,10 +769,10 @@ zcp_lua_alloc(void *ud, void *ptr, size_t osize, size_t nsize) } } -/* ARGSUSED */ static void zcp_lua_counthook(lua_State *state, lua_Debug *ar) { + (void) ar; lua_getfield(state, LUA_REGISTRYINDEX, ZCP_RUN_INFO_KEY); zcp_run_info_t *ri = lua_touserdata(state, -1); @@ -974,10 +974,10 @@ zcp_pool_error(zcp_run_info_t *ri, const char *poolname) * The txg_wait_synced_sig will continue to wait for the txg to complete * after calling this callback. */ -/* ARGSUSED */ static void zcp_eval_sig(void *arg, dmu_tx_t *tx) { + (void) tx; zcp_run_info_t *ri = arg; ri->zri_canceled = B_TRUE; diff --git a/module/zfs/zcp_synctask.c b/module/zfs/zcp_synctask.c index c6ade59b9ced..bfcdbcf9c27f 100644 --- a/module/zfs/zcp_synctask.c +++ b/module/zfs/zcp_synctask.c @@ -129,10 +129,10 @@ static zcp_synctask_info_t zcp_synctask_destroy_info = { .blocks_modified = 0 }; -/* ARGSUSED */ static int zcp_synctask_destroy(lua_State *state, boolean_t sync, nvlist_t *err_details) { + (void) err_details; int err; const char *dsname = lua_tostring(state, 1); @@ -251,10 +251,10 @@ static zcp_synctask_info_t zcp_synctask_snapshot_info = { .blocks_modified = 3 }; -/* ARGSUSED */ static int zcp_synctask_snapshot(lua_State *state, boolean_t sync, nvlist_t *err_details) { + (void) err_details; int err; dsl_dataset_snapshot_arg_t ddsa = { 0 }; const char *dsname = lua_tostring(state, 1); @@ -354,6 +354,7 @@ static int zcp_synctask_inherit_prop(lua_State *state, boolean_t sync, nvlist_t *err_details) { + (void) err_details; int err; zcp_inherit_prop_arg_t zipa = { 0 }; dsl_props_set_arg_t *dpsa = &zipa.zipa_dpsa; @@ -396,10 +397,10 @@ static zcp_synctask_info_t zcp_synctask_bookmark_info = { .blocks_modified = 1, }; -/* ARGSUSED */ static int zcp_synctask_bookmark(lua_State *state, boolean_t sync, nvlist_t *err_details) { + (void) err_details; int err; const char *source = lua_tostring(state, 1); const char *new = lua_tostring(state, 2); @@ -443,6 +444,7 @@ static zcp_synctask_info_t zcp_synctask_set_prop_info = { static int zcp_synctask_set_prop(lua_State *state, boolean_t sync, nvlist_t *err_details) { + (void) err_details; int err; zcp_set_prop_arg_t args = { 0 }; diff --git a/module/zfs/zfs_fm.c b/module/zfs/zfs_fm.c index 007f31b4e7b3..87f793cb4e92 100644 --- a/module/zfs/zfs_fm.c +++ b/module/zfs/zfs_fm.c @@ -59,7 +59,7 @@ * read I/Os, there are basically three 'types' of I/O, which form a roughly * layered diagram: * - * +---------------+ + * +---------------+ * | Aggregate I/O | No associated logical data or device * +---------------+ * | @@ -205,7 +205,6 @@ static void zfs_ereport_schedule_cleaner(void); /* * background task to clean stale recent event nodes. */ -/*ARGSUSED*/ static void zfs_ereport_cleaner(void *arg) { @@ -992,10 +991,10 @@ annotate_ecksum(nvlist_t *ereport, zio_bad_cksum_t *info, return (eip); } #else -/*ARGSUSED*/ void zfs_ereport_clear(spa_t *spa, vdev_t *vd) { + (void) spa, (void) vd; } #endif @@ -1072,6 +1071,8 @@ zfs_ereport_is_valid(const char *subclass, spa_t *spa, vdev_t *vd, zio_t *zio) (zio != NULL) && (!zio->io_timestamp)) { return (B_FALSE); } +#else + (void) subclass, (void) spa, (void) vd, (void) zio; #endif return (B_TRUE); } @@ -1112,6 +1113,9 @@ zfs_ereport_post(const char *subclass, spa_t *spa, vdev_t *vd, /* Cleanup is handled by the callback function */ rc = zfs_zevent_post(ereport, detector, zfs_zevent_post_cb); +#else + (void) subclass, (void) spa, (void) vd, (void) zb, (void) zio, + (void) state; #endif return (rc); } @@ -1141,6 +1145,8 @@ zfs_ereport_start_checksum(spa_t *spa, vdev_t *vd, const zbookmark_phys_t *zb, if (zfs_is_ratelimiting_event(FM_EREPORT_ZFS_CHECKSUM, vd)) return (SET_ERROR(EBUSY)); +#else + (void) zb, (void) offset; #endif report = kmem_zalloc(sizeof (*report), KM_SLEEP); @@ -1193,6 +1199,9 @@ zfs_ereport_finish_checksum(zio_cksum_report_t *report, const abd_t *good_data, report->zcr_ereport = report->zcr_detector = NULL; if (info != NULL) kmem_free(info, sizeof (*info)); +#else + (void) report, (void) good_data, (void) bad_data, + (void) drop_if_identical; #endif } @@ -1257,6 +1266,9 @@ zfs_ereport_post_checksum(spa_t *spa, vdev_t *vd, const zbookmark_phys_t *zb, rc = zfs_zevent_post(ereport, detector, zfs_zevent_post_cb); kmem_free(info, sizeof (*info)); } +#else + (void) spa, (void) vd, (void) zb, (void) zio, (void) offset, + (void) length, (void) good_data, (void) bad_data, (void) zbc; #endif return (rc); } @@ -1321,7 +1333,8 @@ zfs_event_create(spa_t *spa, vdev_t *vd, const char *type, const char *name, while ((elem = nvlist_next_nvpair(aux, elem)) != NULL) (void) nvlist_add_nvpair(resource, elem); } - +#else + (void) spa, (void) vd, (void) type, (void) name, (void) aux; #endif return (resource); } @@ -1336,6 +1349,8 @@ zfs_post_common(spa_t *spa, vdev_t *vd, const char *type, const char *name, resource = zfs_event_create(spa, vd, type, name, aux); if (resource) zfs_zevent_post(resource, NULL, zfs_zevent_post_cb); +#else + (void) spa, (void) vd, (void) type, (void) name, (void) aux; #endif } @@ -1399,6 +1414,8 @@ zfs_post_state_change(spa_t *spa, vdev_t *vd, uint64_t laststate) if (aux) fm_nvlist_destroy(aux, FM_NVA_FREE); +#else + (void) spa, (void) vd, (void) laststate; #endif } diff --git a/module/zfs/zil.c b/module/zfs/zil.c index 640e805d093a..b9f177daee53 100644 --- a/module/zfs/zil.c +++ b/module/zfs/zil.c @@ -432,11 +432,11 @@ zil_parse(zilog_t *zilog, zil_parse_blk_func_t *parse_blk_func, return (error); } -/* ARGSUSED */ static int zil_clear_log_block(zilog_t *zilog, const blkptr_t *bp, void *tx, uint64_t first_txg) { + (void) tx; ASSERT(!BP_IS_HOLE(bp)); /* @@ -455,11 +455,11 @@ zil_clear_log_block(zilog_t *zilog, const blkptr_t *bp, void *tx, return (0); } -/* ARGSUSED */ static int zil_noop_log_record(zilog_t *zilog, const lr_t *lrc, void *tx, uint64_t first_txg) { + (void) zilog, (void) lrc, (void) tx, (void) first_txg; return (0); } @@ -507,11 +507,12 @@ zil_claim_log_record(zilog_t *zilog, const lr_t *lrc, void *tx, return (zil_claim_log_block(zilog, &lr->lr_blkptr, tx, first_txg)); } -/* ARGSUSED */ static int zil_free_log_block(zilog_t *zilog, const blkptr_t *bp, void *tx, uint64_t claim_txg) { + (void) claim_txg; + zio_free(zilog->zl_spa, dmu_tx_get_txg(tx), bp); return (0); @@ -911,10 +912,10 @@ zil_claim(dsl_pool_t *dp, dsl_dataset_t *ds, void *txarg) * Checksum errors are ok as they indicate the end of the chain. * Any other error (no device or read failure) returns an error. */ -/* ARGSUSED */ int zil_check_log_chain(dsl_pool_t *dp, dsl_dataset_t *ds, void *tx) { + (void) dp; zilog_t *zilog; objset_t *os; blkptr_t *bp; @@ -3127,10 +3128,10 @@ zil_sync(zilog_t *zilog, dmu_tx_t *tx) mutex_exit(&zilog->zl_lock); } -/* ARGSUSED */ static int zil_lwb_cons(void *vbuf, void *unused, int kmflag) { + (void) unused, (void) kmflag; lwb_t *lwb = vbuf; list_create(&lwb->lwb_itxs, sizeof (itx_t), offsetof(itx_t, itx_node)); list_create(&lwb->lwb_waiters, sizeof (zil_commit_waiter_t), @@ -3141,10 +3142,10 @@ zil_lwb_cons(void *vbuf, void *unused, int kmflag) return (0); } -/* ARGSUSED */ static void zil_lwb_dest(void *vbuf, void *unused) { + (void) unused; lwb_t *lwb = vbuf; mutex_destroy(&lwb->lwb_vdev_lock); avl_destroy(&lwb->lwb_vdev_tree); @@ -3615,10 +3616,11 @@ zil_replay_log_record(zilog_t *zilog, const lr_t *lr, void *zra, return (0); } -/* ARGSUSED */ static int zil_incr_blks(zilog_t *zilog, const blkptr_t *bp, void *arg, uint64_t claim_txg) { + (void) bp, (void) arg, (void) claim_txg; + zilog->zl_replay_blks++; return (0); @@ -3677,13 +3679,12 @@ zil_replaying(zilog_t *zilog, dmu_tx_t *tx) return (B_FALSE); } -/* ARGSUSED */ int zil_reset(const char *osname, void *arg) { - int error; + (void) arg; - error = zil_suspend(osname, NULL); + int error = zil_suspend(osname, NULL); /* EACCES means crypto key not loaded */ if ((error == EACCES) || (error == EBUSY)) return (SET_ERROR(error)); diff --git a/module/zfs/zio.c b/module/zfs/zio.c index c016fa323b41..d222702392d5 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -369,6 +369,7 @@ zio_data_buf_free(void *buf, size_t size) static void zio_abd_free(void *abd, size_t size) { + (void) size; abd_free((abd_t *)abd); } @@ -1072,6 +1073,7 @@ zfs_blkptr_verify(spa_t *spa, const blkptr_t *bp, boolean_t config_held, boolean_t zfs_dva_valid(spa_t *spa, const dva_t *dva, const blkptr_t *bp) { + (void) bp; uint64_t vdevid = DVA_GET_VDEV(dva); if (vdevid >= spa->spa_root_vdev->vdev_children) @@ -2143,6 +2145,8 @@ zio_execute_stack_check(zio_t *zio) !zio_taskq_member(zio, ZIO_TASKQ_ISSUE) && !zio_taskq_member(zio, ZIO_TASKQ_ISSUE_HIGH)) return (B_TRUE); +#else + (void) zio; #endif /* HAVE_LARGE_STACKS */ return (B_FALSE); @@ -2555,11 +2559,12 @@ zio_rewrite_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data, return (zio); } -/* ARGSUSED */ static zio_t * zio_free_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data, uint64_t offset) { + (void) gn, (void) data, (void) offset; + zio_t *zio = zio_free_sync(pio, pio->io_spa, pio->io_txg, bp, ZIO_GANG_CHILD_FLAGS(pio)); if (zio == NULL) { @@ -2569,11 +2574,11 @@ zio_free_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data, return (zio); } -/* ARGSUSED */ static zio_t * zio_claim_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data, uint64_t offset) { + (void) gn, (void) data, (void) offset; return (zio_claim(pio, pio->io_spa, pio->io_txg, bp, NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio))); } @@ -3964,7 +3969,6 @@ zio_vsd_default_cksum_finish(zio_cksum_report_t *zcr, zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE); } -/*ARGSUSED*/ void zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr) { diff --git a/module/zfs/zio_checksum.c b/module/zfs/zio_checksum.c index f8fee78c6068..00837f07e8e7 100644 --- a/module/zfs/zio_checksum.c +++ b/module/zfs/zio_checksum.c @@ -91,29 +91,29 @@ * invocation and passed to the checksum function. */ -/*ARGSUSED*/ static void abd_checksum_off(abd_t *abd, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { + (void) abd, (void) size, (void) ctx_template; ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0); } -/*ARGSUSED*/ static void abd_fletcher_2_native(abd_t *abd, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { + (void) ctx_template; fletcher_init(zcp); (void) abd_iterate_func(abd, 0, size, fletcher_2_incremental_native, zcp); } -/*ARGSUSED*/ static void abd_fletcher_2_byteswap(abd_t *abd, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { + (void) ctx_template; fletcher_init(zcp); (void) abd_iterate_func(abd, 0, size, fletcher_2_incremental_byteswap, zcp); @@ -127,11 +127,11 @@ abd_fletcher_4_impl(abd_t *abd, uint64_t size, zio_abd_checksum_data_t *acdp) fletcher_4_abd_ops.acf_fini(acdp); } -/*ARGSUSED*/ void abd_fletcher_4_native(abd_t *abd, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { + (void) ctx_template; fletcher_4_ctx_t ctx; zio_abd_checksum_data_t acd = { @@ -144,11 +144,11 @@ abd_fletcher_4_native(abd_t *abd, uint64_t size, } -/*ARGSUSED*/ void abd_fletcher_4_byteswap(abd_t *abd, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { + (void) ctx_template; fletcher_4_ctx_t ctx; zio_abd_checksum_data_t acd = { diff --git a/module/zfs/zio_compress.c b/module/zfs/zio_compress.c index 1ff1e76d7f22..cded11f4cbd5 100644 --- a/module/zfs/zio_compress.c +++ b/module/zfs/zio_compress.c @@ -74,6 +74,7 @@ uint8_t zio_complevel_select(spa_t *spa, enum zio_compress compress, uint8_t child, uint8_t parent) { + (void) spa; uint8_t result; if (!ZIO_COMPRESS_HASLEVEL(compress)) @@ -110,10 +111,11 @@ zio_compress_select(spa_t *spa, enum zio_compress child, return (result); } -/*ARGSUSED*/ static int zio_compress_zeroed_cb(void *data, size_t len, void *private) { + (void) private; + uint64_t *end = (uint64_t *)((char *)data + len); for (uint64_t *word = (uint64_t *)data; word < end; word++) if (*word != 0) From 02309af096cbc11b520a8cd6ad7a6929419ca6a7 Mon Sep 17 00:00:00 2001 From: Tomohiro Kusumi Date: Wed, 9 Feb 2022 03:48:57 +0900 Subject: [PATCH 089/109] Remove unneeded "extern inline" function declarations All of these externs are already #included as static inline functions via corresponding headers. Reviewed-by: Igor Kozhukhov Reviewed-by: Brian Behlendorf Signed-off-by: Tomohiro Kusumi Closes #13073 --- module/zfs/dbuf.c | 5 ----- module/zfs/dsl_dataset.c | 2 -- module/zfs/dsl_dir.c | 2 -- module/zfs/zap.c | 2 -- module/zfs/zap_leaf.c | 2 -- module/zfs/zap_micro.c | 2 -- 6 files changed, 15 deletions(-) diff --git a/module/zfs/dbuf.c b/module/zfs/dbuf.c index a547fbb03e67..d61724be8a07 100644 --- a/module/zfs/dbuf.c +++ b/module/zfs/dbuf.c @@ -170,11 +170,6 @@ static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); static void dbuf_sync_leaf_verify_bonus_dnode(dbuf_dirty_record_t *dr); static int dbuf_read_verify_dnode_crypt(dmu_buf_impl_t *db, uint32_t flags); -extern inline void dmu_buf_init_user(dmu_buf_user_t *dbu, - dmu_buf_evict_func_t *evict_func_sync, - dmu_buf_evict_func_t *evict_func_async, - dmu_buf_t **clear_on_evict_dbufp); - /* * Global data structures and functions for the dbuf cache. */ diff --git a/module/zfs/dsl_dataset.c b/module/zfs/dsl_dataset.c index 4623bcec4ea8..e8692c95438a 100644 --- a/module/zfs/dsl_dataset.c +++ b/module/zfs/dsl_dataset.c @@ -90,8 +90,6 @@ int zfs_allow_redacted_dataset_mount = 0; #define DS_REF_MAX (1ULL << 62) -extern inline dsl_dataset_phys_t *dsl_dataset_phys(dsl_dataset_t *ds); - static void dsl_dataset_set_remap_deadlist_object(dsl_dataset_t *ds, uint64_t obj, dmu_tx_t *tx); static void dsl_dataset_unset_remap_deadlist_object(dsl_dataset_t *ds, diff --git a/module/zfs/dsl_dir.c b/module/zfs/dsl_dir.c index e4ff7d8e754f..43ef1897811c 100644 --- a/module/zfs/dsl_dir.c +++ b/module/zfs/dsl_dir.c @@ -121,8 +121,6 @@ * dsl_dir_init_fs_ss_count(). */ -extern inline dsl_dir_phys_t *dsl_dir_phys(dsl_dir_t *dd); - static uint64_t dsl_dir_space_towrite(dsl_dir_t *dd); typedef struct ddulrt_arg { diff --git a/module/zfs/zap.c b/module/zfs/zap.c index 6f03beef3bdb..4cd14287a5a2 100644 --- a/module/zfs/zap.c +++ b/module/zfs/zap.c @@ -80,8 +80,6 @@ int zap_iterate_prefetch = B_TRUE; int fzap_default_block_shift = 14; /* 16k blocksize */ -extern inline zap_phys_t *zap_f_phys(zap_t *zap); - static uint64_t zap_allocate_blocks(zap_t *zap, int nblocks); void diff --git a/module/zfs/zap_leaf.c b/module/zfs/zap_leaf.c index aa6c298c3b4b..aad923d512df 100644 --- a/module/zfs/zap_leaf.c +++ b/module/zfs/zap_leaf.c @@ -52,8 +52,6 @@ static uint16_t *zap_leaf_rehash_entry(zap_leaf_t *l, uint16_t entry); #define LEAF_HASH_ENTPTR(l, h) (&zap_leaf_phys(l)->l_hash[LEAF_HASH(l, h)]) -extern inline zap_leaf_phys_t *zap_leaf_phys(zap_leaf_t *l); - static void zap_memset(void *a, int c, size_t n) { diff --git a/module/zfs/zap_micro.c b/module/zfs/zap_micro.c index b4611685b204..1f32e4450522 100644 --- a/module/zfs/zap_micro.c +++ b/module/zfs/zap_micro.c @@ -41,8 +41,6 @@ #include #endif -extern inline mzap_phys_t *zap_m_phys(zap_t *zap); - static int mzap_upgrade(zap_t **zapp, void *tag, dmu_tx_t *tx, zap_flags_t flags); From 4b3fbf3c16955ab939305beeb3594647420dee04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Mon, 14 Feb 2022 13:20:33 +0100 Subject: [PATCH 090/109] zfs-receive.8: fix Op Fl x Ar encryption in running text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Brian Behlendorf Reviewed-by: George Amanakis Reviewed-by: Ryan Moeller Signed-off-by: Ahelenia Ziemiańska Closes #13101 --- man/man8/zfs-receive.8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/man8/zfs-receive.8 b/man/man8/zfs-receive.8 index 1f4f91bca2ee..171c43850b28 100644 --- a/man/man8/zfs-receive.8 +++ b/man/man8/zfs-receive.8 @@ -329,7 +329,7 @@ Once the receive has completed, you can use .Nm zfs Cm set to change this setting after the fact. Similarly, you can receive a dataset as an encrypted child by specifying -.Op Fl x Ar encryption +.Fl x Sy encryption to force the property to be inherited. Overriding encryption properties (except for .Sy keylocation ) From 336c6d5f54346ba71d955e54baf215abcfc588ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Mon, 14 Feb 2022 13:22:52 +0100 Subject: [PATCH 091/109] zfs-receive.8: properly unlight = in option setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Brian Behlendorf Reviewed-by: George Amanakis Reviewed-by: Ryan Moeller Signed-off-by: Ahelenia Ziemiańska Closes #13101 --- man/man8/zfs-receive.8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/man8/zfs-receive.8 b/man/man8/zfs-receive.8 index 171c43850b28..b81bfc507e1f 100644 --- a/man/man8/zfs-receive.8 +++ b/man/man8/zfs-receive.8 @@ -319,7 +319,7 @@ received as an encryption root, specify encryption properties in the same manner as is required for .Nm zfs Cm create . For instance: -.Dl # Nm zfs Cm send Pa tank/test@snap1 | Nm zfs Cm recv Fl o Sy encryption Ns = Ns Sy on Fl o Sy keyformat=passphrase Fl o Sy keylocation Ns = Ns Pa file:///path/to/keyfile +.Dl # Nm zfs Cm send Pa tank/test@snap1 | Nm zfs Cm recv Fl o Sy encryption Ns = Ns Sy on Fl o Sy keyformat Ns = Ns Sy passphrase Fl o Sy keylocation Ns = Ns Pa file:///path/to/keyfile .Pp Note that .Fl o Sy keylocation Ns = Ns Sy prompt From 6c6153e5b80d2a78b246ce7dc68853d447a231d8 Mon Sep 17 00:00:00 2001 From: George Amanakis Date: Wed, 16 Feb 2022 00:48:59 +0100 Subject: [PATCH 092/109] Avoid dirtying the final TXGs when exporting a pool There are two codepaths than can dirty final TXGs: 1) If calling spa_export_common()->spa_unload()-> spa_unload_log_sm_flush_all() after the spa_final_txg is set, then spa_sync()->spa_flush_metaslabs() may end up dirtying the final TXGs. Then we have the following panic: Call Trace: dump_stack_lvl+0x46/0x62 spl_panic+0xea/0x102 [spl] dbuf_dirty+0xcd6/0x11b0 [zfs] zap_lockdir_impl+0x321/0x590 [zfs] zap_lockdir+0xed/0x150 [zfs] zap_update+0x69/0x250 [zfs] feature_sync+0x5f/0x190 [zfs] space_map_alloc+0x83/0xc0 [zfs] spa_generate_syncing_log_sm+0x10b/0x2f0 [zfs] spa_flush_metaslabs+0xb2/0x350 [zfs] spa_sync_iterate_to_convergence+0x15a/0x320 [zfs] spa_sync+0x2e0/0x840 [zfs] txg_sync_thread+0x2b1/0x3f0 [zfs] thread_generic_wrapper+0x62/0xa0 [spl] kthread+0x127/0x150 ret_from_fork+0x22/0x30 2) Calling vdev_*_stop_all() for a second time in spa_unload() after spa_export_common() unnecessarily delays the final TXGs beyond what spa_final_txg is set at. Fix this by performing the check and call for spa_unload_log_sm_flush_all() before the spa_final_txg is set in spa_export_common(). Also check if the spa_final_txg has already been set in spa_unload() and skip those calls in this case. Reviewed-by: Brian Behlendorf Signed-off-by: George Amanakis External-issue: https://www.illumos.org/issues/9081 Closes #13048 Closes #13098 --- module/zfs/spa.c | 60 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/module/zfs/spa.c b/module/zfs/spa.c index 9bfaefb6c214..7c49ced9998a 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -1588,25 +1588,33 @@ spa_unload(spa_t *spa) spa_wake_waiters(spa); /* - * If the log space map feature is enabled and the pool is getting - * exported (but not destroyed), we want to spend some time flushing - * as many metaslabs as we can in an attempt to destroy log space - * maps and save import time. + * If we have set the spa_final_txg, we have already performed the + * tasks below in spa_export_common(). We should not redo it here since + * we delay the final TXGs beyond what spa_final_txg is set at. */ - if (spa_should_flush_logs_on_unload(spa)) - spa_unload_log_sm_flush_all(spa); + if (spa->spa_final_txg == UINT64_MAX) { + /* + * If the log space map feature is enabled and the pool is + * getting exported (but not destroyed), we want to spend some + * time flushing as many metaslabs as we can in an attempt to + * destroy log space maps and save import time. + */ + if (spa_should_flush_logs_on_unload(spa)) + spa_unload_log_sm_flush_all(spa); - /* - * Stop async tasks. - */ - spa_async_suspend(spa); + /* + * Stop async tasks. + */ + spa_async_suspend(spa); - if (spa->spa_root_vdev) { - vdev_t *root_vdev = spa->spa_root_vdev; - vdev_initialize_stop_all(root_vdev, VDEV_INITIALIZE_ACTIVE); - vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE); - vdev_autotrim_stop_all(spa); - vdev_rebuild_stop_all(spa); + if (spa->spa_root_vdev) { + vdev_t *root_vdev = spa->spa_root_vdev; + vdev_initialize_stop_all(root_vdev, + VDEV_INITIALIZE_ACTIVE); + vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE); + vdev_autotrim_stop_all(spa); + vdev_rebuild_stop_all(spa); + } } /* @@ -6416,9 +6424,27 @@ spa_export_common(const char *pool, int new_state, nvlist_t **oldconfig, if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); spa->spa_state = new_state; + vdev_config_dirty(spa->spa_root_vdev); + spa_config_exit(spa, SCL_ALL, FTAG); + } + + /* + * If the log space map feature is enabled and the pool is + * getting exported (but not destroyed), we want to spend some + * time flushing as many metaslabs as we can in an attempt to + * destroy log space maps and save import time. This has to be + * done before we set the spa_final_txg, otherwise + * spa_sync() -> spa_flush_metaslabs() may dirty the final TXGs. + * spa_should_flush_logs_on_unload() should be called after + * spa_state has been set to the new_state. + */ + if (spa_should_flush_logs_on_unload(spa)) + spa_unload_log_sm_flush_all(spa); + + if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { + spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); spa->spa_final_txg = spa_last_synced_txg(spa) + TXG_DEFER_SIZE + 1; - vdev_config_dirty(spa->spa_root_vdev); spa_config_exit(spa, SCL_ALL, FTAG); } } From bcddb18bae26692f83a4b56334108536895355f3 Mon Sep 17 00:00:00 2001 From: George Amanakis Date: Wed, 16 Feb 2022 20:52:02 +0100 Subject: [PATCH 093/109] Enable encrypted raw sending to pools with greater ashift Raw sending from pool1/encrypted with ashift=9 to pool2/encrypted with ashift=12 results to failure when mounting pool2/encrypted (Input/Output error). Notably, the opposite, raw sending from a greater ashift to a lower one does not fail. This happens because zio_compress_write() falsely checks only ZIO_FLAG_RAW_COMPRESS and not ZIO_FLAG_RAW_ENCRYPT which is also set in encrypted raw send streams. In this case it rounds up the psize and if not equal to the zio->io_size it modifies the block by zeroing out the extra bytes. Because this happens in a SA attr. registration object (type=46), the decryption fails upon mounting the filesystem, and zpool status falsely reports an error. Fix this by checking both ZIO_FLAG_RAW_COMPRESS and ZIO_FLAG_RAW_ENCRYPT before deciding whether to zero-pad a block. Reviewed-by: Brian Behlendorf Signed-off-by: George Amanakis Closes #13067 Closes #13074 --- module/zfs/zio.c | 8 +- tests/runfiles/common.run | 2 +- .../tests/functional/rsend/Makefile.am | 1 + .../functional/rsend/send_raw_ashift.ksh | 193 ++++++++++++++++++ 4 files changed, 202 insertions(+), 2 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/rsend/send_raw_ashift.ksh diff --git a/module/zfs/zio.c b/module/zfs/zio.c index d222702392d5..04b5d121e111 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -1773,7 +1773,13 @@ zio_write_compress(zio_t *zio) zio->io_abd, NULL, lsize, zp->zp_complevel); if (psize == 0 || psize >= lsize) compress = ZIO_COMPRESS_OFF; - } else if (zio->io_flags & ZIO_FLAG_RAW_COMPRESS) { + } else if (zio->io_flags & ZIO_FLAG_RAW_COMPRESS && + !(zio->io_flags & ZIO_FLAG_RAW_ENCRYPT)) { + /* + * If we are raw receiving an encrypted dataset we should not + * take this codepath because it will change the on-disk block + * and decryption will fail. + */ size_t rounded = MIN((size_t)roundup(psize, spa->spa_min_alloc), lsize); diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index bad2df2f97de..44115f6a97b4 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -833,7 +833,7 @@ tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos', 'send_realloc_encrypted_files', 'send_spill_block', 'send_holds', 'send_hole_birth', 'send_mixed_raw', 'send-wR_encrypted_zvol', 'send_partial_dataset', 'send_invalid', 'send_doall', - 'send_raw_spill_block'] + 'send_raw_spill_block', 'send_raw_ashift'] tags = ['functional', 'rsend'] [tests/functional/scrub_mirror] diff --git a/tests/zfs-tests/tests/functional/rsend/Makefile.am b/tests/zfs-tests/tests/functional/rsend/Makefile.am index ba1bcf24a501..b8eb54f64c3e 100644 --- a/tests/zfs-tests/tests/functional/rsend/Makefile.am +++ b/tests/zfs-tests/tests/functional/rsend/Makefile.am @@ -50,6 +50,7 @@ dist_pkgdata_SCRIPTS = \ send_realloc_encrypted_files.ksh \ send_spill_block.ksh \ send_raw_spill_block.ksh \ + send_raw_ashift.ksh \ send_holds.ksh \ send_hole_birth.ksh \ send_invalid.ksh \ diff --git a/tests/zfs-tests/tests/functional/rsend/send_raw_ashift.ksh b/tests/zfs-tests/tests/functional/rsend/send_raw_ashift.ksh new file mode 100755 index 000000000000..3cea334495d9 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send_raw_ashift.ksh @@ -0,0 +1,193 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2021, George Amanakis. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/include/properties.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify encrypted raw sending to pools with greater ashift succeeds. +# +# Strategy: +# 1) Create a set of files each containing some file data in an +# encrypted filesystem. +# 2) Snapshot and raw send these files to a pool with greater ashift +# 3) Verify that all the xattrs (and thus the spill block) were +# preserved when receiving the incremental stream. +# 4) Repeat the test for a non-encrypted filesystem using raw send +# + +verify_runnable "both" + +log_assert "Verify raw sending to pools with greater ashift succeeds" + +function cleanup +{ + rm -f $BACKDIR/fs@* + poolexists pool9 && destroy_pool pool9 + poolexists pool12 && destroy_pool pool12 + log_must rm -f $TESTDIR/vdev_a $TESTDIR/vdev_b +} + +function xattr_test +{ + log_must zfs set xattr=sa pool9/$1 + log_must zfs set dnodesize=legacy pool9/$1 + log_must zfs set recordsize=128k pool9/$1 + rand_set_prop pool9/$1 compression "${compress_prop_vals[@]}" + + # Create 40 files each with a spill block containing xattrs. Each file + # will be modified in a different way to validate the incremental receive. + for i in {1..40}; do + file="/pool9/$1/file$i" + + log_must mkfile 16384 $file + for j in {1..20}; do + log_must set_xattr "testattr$j" "$attrvalue" $file + done + done + + # Snapshot the pool and send it to the new dataset. + log_must zfs snapshot pool9/$1@snap1 + log_must eval "zfs send -w pool9/$1@snap1 >$BACKDIR/$1@snap1" + log_must eval "zfs recv pool12/$1 < $BACKDIR/$1@snap1" + + # + # Modify file[1-6]'s contents but not the spill blocks. + # + # file1 - Increase record size; single block + # file2 - Increase record size; multiple blocks + # file3 - Truncate file to zero size; single block + # file4 - Truncate file to smaller size; single block + # file5 - Truncate file to much larger size; add holes + # file6 - Truncate file to embedded size; embedded data + # + log_must mkfile 32768 /pool9/$1/file1 + log_must mkfile 1048576 /pool9/$1/file2 + log_must truncate -s 0 /pool9/$1/file3 + log_must truncate -s 8192 /pool9/$1/file4 + log_must truncate -s 1073741824 /pool9/$1/file5 + log_must truncate -s 50 /pool9/$1/file6 + + # + # Modify file[11-16]'s contents and their spill blocks. + # + # file11 - Increase record size; single block + # file12 - Increase record size; multiple blocks + # file13 - Truncate file to zero size; single block + # file14 - Truncate file to smaller size; single block + # file15 - Truncate file to much larger size; add holes + # file16 - Truncate file to embedded size; embedded data + # + log_must mkfile 32768 /pool9/$1/file11 + log_must mkfile 1048576 /pool9/$1/file12 + log_must truncate -s 0 /pool9/$1/file13 + log_must truncate -s 8192 /pool9/$1/file14 + log_must truncate -s 1073741824 /pool9/$1/file15 + log_must truncate -s 50 /pool9/$1/file16 + + for i in {11..20}; do + log_must rm_xattr testattr1 /pool9/$1/file$i + done + + # + # Modify file[21-26]'s contents and remove their spill blocks. + # + # file21 - Increase record size; single block + # file22 - Increase record size; multiple blocks + # file23 - Truncate file to zero size; single block + # file24 - Truncate file to smaller size; single block + # file25 - Truncate file to much larger size; add holes + # file26 - Truncate file to embedded size; embedded data + # + log_must mkfile 32768 /pool9/$1/file21 + log_must mkfile 1048576 /pool9/$1/file22 + log_must truncate -s 0 /pool9/$1/file23 + log_must truncate -s 8192 /pool9/$1/file24 + log_must truncate -s 1073741824 /pool9/$1/file25 + log_must truncate -s 50 /pool9/$1/file26 + + for i in {21..30}; do + for j in {1..20}; do + log_must rm_xattr testattr$j /pool9/$1/file$i + done + done + + # + # Modify file[31-40]'s spill blocks but not the file contents. + # + for i in {31..40}; do + file="/pool9/$1/file$i" + log_must rm_xattr testattr$(((RANDOM % 20) + 1)) $file + log_must set_xattr testattr$(((RANDOM % 20) + 1)) "$attrvalue" $file + done + + # Snapshot the pool and send the incremental snapshot. + log_must zfs snapshot pool9/$1@snap2 + log_must eval "zfs send -w -i pool9/$1@snap1 pool9/$1@snap2 >$BACKDIR/$1@snap2" + log_must eval "zfs recv pool12/$1 < $BACKDIR/$1@snap2" +} + +attrvalue="abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + +log_onexit cleanup + +# Create pools +truncate -s $MINVDEVSIZE $TESTDIR/vdev_a +truncate -s $MINVDEVSIZE $TESTDIR/vdev_b +log_must zpool create -f -o ashift=9 pool9 $TESTDIR/vdev_a +log_must zpool create -f -o ashift=12 pool12 $TESTDIR/vdev_b + +# Create encrypted fs +log_must eval "echo 'password' | zfs create -o encryption=on" \ + "-o keyformat=passphrase -o keylocation=prompt " \ + "pool9/encfs" + +# Run xattr tests for encrypted fs +xattr_test encfs + +# Calculate the expected recursive checksum for source encrypted fs +expected_cksum=$(recursive_cksum /pool9/encfs) + +# Mount target encrypted fs +log_must eval "echo 'password' | zfs load-key pool12/encfs" +log_must zfs mount pool12/encfs + +# Validate the received copy using the received recursive checksum +actual_cksum=$(recursive_cksum /pool12/encfs) +if [[ "$expected_cksum" != "$actual_cksum" ]]; then + log_fail "Checksums differ ($expected_cksum != $actual_cksum)" +fi + +# Perform the same test but without encryption (send -w) +log_must zfs create pool9/fs + +# Run xattr tests for non-encrypted fs +xattr_test fs + +# Calculate the expected recursive checksum for source non-encrypted fs +expected_cksum=$(recursive_cksum /pool9/fs) + +# Validate the received copy using the received recursive checksum +actual_cksum=$(recursive_cksum /pool12/fs) +if [[ "$expected_cksum" != "$actual_cksum" ]]; then + log_fail "Checksums differ ($expected_cksum != $actual_cksum)" +fi + +log_pass "Verify raw sending to pools with greater ashift succeeds" From b55ed8df92a8b79284d691c74e3ea4ddda52ba76 Mon Sep 17 00:00:00 2001 From: Damian Szuberski Date: Thu, 17 Feb 2022 00:17:16 +0100 Subject: [PATCH 094/109] Fix Linux kernel directories detection Most modern Linux distributions have separate locations for bare source and prebuilt ("build") files. Additionally, there are `source` and `build` symlinks in `/lib/modules/$(KERNEL_VERSION)` pointing to them. The order of directory search is now: - `configure` command line values if both `--with-linux` and `--with-linux-obj` were defined - If only `--with-linux` was defined, `--with-linux-obj` is assumed to have the same value as `--with-linux` - If neither `--with-linux` nor `--with-linux-obj` were defined autodetection is used: - `/lib/modules/$(uname -r)/{source,build}` respectively, if exist - The first directory in `/lib/modules` with the highest version number according to `sort -V` which contains `source` and `build` symlinks/directories - The first directory matching `/usr/src/kernels/*` and `/usr/src/linux-*` with the highest version number according to `sort -V`. Here the source and prebuilt directories are assumed to be the same. Reviewed-by: Brian Behlendorf Signed-off-by: szubersk Closes #9935 Closes #13096 --- config/kernel.m4 | 105 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 75 insertions(+), 30 deletions(-) diff --git a/config/kernel.m4 b/config/kernel.m4 index b918f045341b..3122e9dbaa94 100644 --- a/config/kernel.m4 +++ b/config/kernel.m4 @@ -280,6 +280,35 @@ AC_DEFUN([ZFS_AC_MODULE_SYMVERS], [ dnl # dnl # Detect the kernel to be built against dnl # +dnl # Most modern Linux distributions have separate locations for bare +dnl # source (source) and prebuilt (build) files. Additionally, there are +dnl # `source` and `build` symlinks in `/lib/modules/$(KERNEL_VERSION)` +dnl # pointing to them. The directory search order is now: +dnl # +dnl # - `configure` command line values if both `--with-linux` and +dnl # `--with-linux-obj` were defined +dnl # +dnl # - If only `--with-linux` was defined, `--with-linux-obj` is assumed +dnl # to have the same value as `--with-linux` +dnl # +dnl # - If neither `--with-linux` nor `--with-linux-obj` were defined +dnl # autodetection is used: +dnl # +dnl # - `/lib/modules/$(uname -r)/{source,build}` respectively, if exist. +dnl # +dnl # - If only `/lib/modules/$(uname -r)/build` exists, it is assumed +dnl # to be both source and build directory. +dnl # +dnl # - The first directory in `/lib/modules` with the highest version +dnl # number according to `sort -V` which contains both `source` and +dnl # `build` symlinks/directories. If module directory contains only +dnl # `build` component, it is assumed to be both source and build +dnl # directory. +dnl # +dnl # - Last resort: the first directory matching `/usr/src/kernels/*` +dnl # and `/usr/src/linux-*` with the highest version number according +dnl # to `sort -V` is assumed to be both source and build directory. +dnl # AC_DEFUN([ZFS_AC_KERNEL], [ AC_ARG_WITH([linux], AS_HELP_STRING([--with-linux=PATH], @@ -291,25 +320,52 @@ AC_DEFUN([ZFS_AC_KERNEL], [ [Path to kernel build objects]), [kernelbuild="$withval"]) - AC_MSG_CHECKING([kernel source directory]) - AS_IF([test -z "$kernelsrc"], [ - AS_IF([test -e "/lib/modules/$(uname -r)/source"], [ - headersdir="/lib/modules/$(uname -r)/source" - sourcelink=$(readlink -f "$headersdir") + AC_MSG_CHECKING([kernel source and build directories]) + AS_IF([test -n "$kernelsrc" && test -z "$kernelbuild"], [ + kernelbuild="$kernelsrc" + ], [test -z "$kernelsrc"], [ + AS_IF([test -e "/lib/modules/$(uname -r)/source" && \ + test -e "/lib/modules/$(uname -r)/build"], [ + src="/lib/modules/$(uname -r)/source" + build="/lib/modules/$(uname -r)/build" ], [test -e "/lib/modules/$(uname -r)/build"], [ - headersdir="/lib/modules/$(uname -r)/build" - sourcelink=$(readlink -f "$headersdir") + build="/lib/modules/$(uname -r)/build" + src="$build" ], [ - sourcelink=$(ls -1d /usr/src/kernels/* \ - /usr/src/linux-* \ - 2>/dev/null | grep -v obj | tail -1) + src= + + for d in $(ls -1d /lib/modules/* 2>/dev/null | sort -Vr); do + if test -e "$d/source" && test -e "$d/build"; then + src="$d/source" + build="$d/build" + break + fi + + if test -e "$d/build"; then + src="$d/build" + build="$d/build" + break + fi + done + + # the least reliable method + if test -z "$src"; then + src=$(ls -1d /usr/src/kernels/* /usr/src/linux-* \ + 2>/dev/null | grep -v obj | sort -Vr | head -1) + build="$src" + fi ]) - AS_IF([test -n "$sourcelink" && test -e ${sourcelink}], [ - kernelsrc=`readlink -f ${sourcelink}` + AS_IF([test -n "$src" && test -e "$src"], [ + kernelsrc=$(readlink -e "$src") ], [ kernelsrc="[Not found]" ]) + AS_IF([test -n "$build" && test -e "$build"], [ + kernelbuild=$(readlink -e "$build") + ], [ + kernelbuild="[Not found]" + ]) ], [ AS_IF([test "$kernelsrc" = "NONE"], [ kernsrcver=NONE @@ -317,30 +373,19 @@ AC_DEFUN([ZFS_AC_KERNEL], [ withlinux=yes ]) + AC_MSG_RESULT([done]) + AC_MSG_CHECKING([kernel source directory]) AC_MSG_RESULT([$kernelsrc]) - AS_IF([test ! -d "$kernelsrc"], [ + AC_MSG_CHECKING([kernel build directory]) + AC_MSG_RESULT([$kernelbuild]) + AS_IF([test ! -d "$kernelsrc" || test ! -d "$kernelbuild"], [ AC_MSG_ERROR([ *** Please make sure the kernel devel package for your distribution *** is installed and then try again. If that fails, you can specify the - *** location of the kernel source with the '--with-linux=PATH' option.]) + *** location of the kernel source and build with the '--with-linux=PATH' and + *** '--with-linux-obj=PATH' options respectively.]) ]) - AC_MSG_CHECKING([kernel build directory]) - AS_IF([test -z "$kernelbuild"], [ - AS_IF([test x$withlinux != xyes -a -e "/lib/modules/$(uname -r)/build"], [ - kernelbuild=`readlink -f /lib/modules/$(uname -r)/build` - ], [test -d ${kernelsrc}-obj/${target_cpu}/${target_cpu}], [ - kernelbuild=${kernelsrc}-obj/${target_cpu}/${target_cpu} - ], [test -d ${kernelsrc}-obj/${target_cpu}/default], [ - kernelbuild=${kernelsrc}-obj/${target_cpu}/default - ], [test -d `dirname ${kernelsrc}`/build-${target_cpu}], [ - kernelbuild=`dirname ${kernelsrc}`/build-${target_cpu} - ], [ - kernelbuild=${kernelsrc} - ]) - ]) - AC_MSG_RESULT([$kernelbuild]) - AC_MSG_CHECKING([kernel source version]) utsrelease1=$kernelbuild/include/linux/version.h utsrelease2=$kernelbuild/include/linux/utsrelease.h From 1d70698174cfb7124ec42e4462d1c8d35e0caf61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20F=C3=BCl=C3=B6p?= Date: Thu, 24 Feb 2022 18:23:41 +0100 Subject: [PATCH 095/109] Linux 5.11 compat: x86 SIMD: fix kernel_fpu_{begin,end}() detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linux 5.11 changed kernel_fpu_begin() to an inlined function and moved the functionality to kernel_fpu_begin_mask(). This breaks the existing detection mechanism since it checks if kernel_fpu_begin is an exported kernel symbol, which isn't the case for an inlined function. To avoid assumptions about internal implementation, replace ZFS_LINUX_TEST_RESULT_SYMBOL in favor of ZFS_LINUX_TEST_RESULT which already makes sure kernel_fpu_{begin,end}() is usable by us. Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf Signed-off-by: Attila Fülöp Closes #13147 --- config/kernel-fpu.m4 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/kernel-fpu.m4 b/config/kernel-fpu.m4 index 06280239ddf0..7f8b028d043b 100644 --- a/config/kernel-fpu.m4 +++ b/config/kernel-fpu.m4 @@ -8,6 +8,9 @@ dnl # dnl # XSTATE_XSAVE and XSTATE_XRESTORE aren't accessible any more dnl # HAVE_KERNEL_FPU_XSAVE_INTERNAL dnl # +dnl # 5.11: kernel_fpu_begin() is an inlined function now, so don't check +dnl # for it inside the kernel symbols. +dnl # dnl # 5.0: Wrappers have been introduced to save/restore the FPU state. dnl # This change was made to the 4.19.38 and 4.14.120 LTS kernels. dnl # HAVE_KERNEL_FPU_INTERNAL @@ -147,8 +150,7 @@ AC_DEFUN([ZFS_AC_KERNEL_FPU], [ dnl # Legacy kernel dnl # AC_MSG_CHECKING([whether kernel fpu is available]) - ZFS_LINUX_TEST_RESULT_SYMBOL([kernel_fpu_license], - [kernel_fpu_begin], [arch/x86/kernel/fpu/core.c], [ + ZFS_LINUX_TEST_RESULT([kernel_fpu_license], [ AC_MSG_RESULT(kernel_fpu_*) AC_DEFINE(HAVE_KERNEL_FPU, 1, [kernel has kernel_fpu_* functions]) From 5c80a25653ba38ff86a079ab9cee76477158dd5f Mon Sep 17 00:00:00 2001 From: Damian Szuberski Date: Thu, 24 Feb 2022 19:33:48 +0100 Subject: [PATCH 096/109] Fix directory detection in `dkms.mkconf` Fix `zfs-dkms` installation on Debian-derived distributions by aligning the directory detection logic to #13096. Reviewed-by: Brian Behlendorf Signed-off-by: szubersk Closes #11449 Closes #13141 --- scripts/dkms.mkconf | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/scripts/dkms.mkconf b/scripts/dkms.mkconf index 9d12a8c3b3fd..408322fa124a 100755 --- a/scripts/dkms.mkconf +++ b/scripts/dkms.mkconf @@ -29,22 +29,14 @@ PRE_BUILD="configure --prefix=/usr --with-config=kernel --with-linux=\$( - case \`lsb_release -is\` in - (Debian|Devuan) - if [[ -e \${kernel_source_dir/%build/source} ]] - then - echo \${kernel_source_dir/%build/source} - else - # A kpkg exception for Proxmox 2.0 - echo \${kernel_source_dir} - fi - ;; - (*) - echo \${kernel_source_dir} - ;; - esac + if [ -e "\${kernel_source_dir/%build/source}" ] + then + echo "\${kernel_source_dir/%build/source}" + else + echo "\${kernel_source_dir}" + fi ) - --with-linux-obj=\${kernel_source_dir} + --with-linux-obj="\${kernel_source_dir}" \$( [[ -n \"\${ICP_ROOT}\" ]] && \\ { From 7bd292e59b524faf66196ba6d4f6ca52d41df382 Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Tue, 2 Nov 2021 09:23:48 -0700 Subject: [PATCH 097/109] Fix cpu hotplug atomic sleep issue We move the spinlock unlock before the thread creation. This should be safe because the thread creation code doesn't actually manipulate any taskq data structures; that's done by the thread once it's created. We also remove the assertion that the maxthreads is the current threads plus one; that assertion could fail if multiple hotplug events come in quick succession, and the first new taskq thread hasn't had a chance to start processing yet. Reviewed-by: Brian Behlendorf Reviewed-by: Matthew Ahrens eviewed-by: Tony Nguyen Signed-off-by: Paul Dagnelie Closes #12714 --- module/os/linux/spl/spl-taskq.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/module/os/linux/spl/spl-taskq.c b/module/os/linux/spl/spl-taskq.c index 61631256c858..fb25a4154485 100644 --- a/module/os/linux/spl/spl-taskq.c +++ b/module/os/linux/spl/spl-taskq.c @@ -1298,8 +1298,10 @@ spl_taskq_expand(unsigned int cpu, struct hlist_node *node) ASSERT(tq); spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); - if (!(tq->tq_flags & TASKQ_ACTIVE)) - goto out; + if (!(tq->tq_flags & TASKQ_ACTIVE)) { + spin_unlock_irqrestore(&tq->tq_lock, flags); + return (err); + } ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT); int nthreads = MIN(tq->tq_cpu_pct, 100); @@ -1308,13 +1310,12 @@ spl_taskq_expand(unsigned int cpu, struct hlist_node *node) if (!((tq->tq_flags & TASKQ_DYNAMIC) && spl_taskq_thread_dynamic) && tq->tq_maxthreads > tq->tq_nthreads) { - ASSERT3U(tq->tq_maxthreads, ==, tq->tq_nthreads + 1); + spin_unlock_irqrestore(&tq->tq_lock, flags); taskq_thread_t *tqt = taskq_thread_create(tq); if (tqt == NULL) err = -1; + return (err); } - -out: spin_unlock_irqrestore(&tq->tq_lock, flags); return (err); } From f2eaa978402bd342ec84425f0722af709dd02e48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Sat, 26 Feb 2022 20:19:05 +0100 Subject: [PATCH 098/109] Fix FreeBSD reporting on reruns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turns out, when your test-suite fails on FreeBSD the rerun logic would fail as follows: Results Summary PASS 1358 FAIL 7 SKIP 47 Running Time: 04:00:02 Percent passed: 96.2% Log directory: /var/tmp/test_results/20220225T092538 mktemp: illegal option -- p usage: mktemp [-d] [-q] [-t prefix] [-u] template ... mktemp [-d] [-q] [-u] -t prefix mktemp: illegal option -- p usage: mktemp [-d] [-q] [-t prefix] [-u] template ... mktemp [-d] [-q] [-u] -t prefix /usr/local/share/zfs/zfs-tests.sh: cannot create : No such file or directory ... This change resolves a flaw from the original commit, 2320e6eb4 ("Add zfs-test facility to automatically rerun failing tests") Reviewed by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #13156 --- scripts/zfs-tests.sh | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/scripts/zfs-tests.sh b/scripts/zfs-tests.sh index f871a51d34c2..aa0829b28326 100755 --- a/scripts/zfs-tests.sh +++ b/scripts/zfs-tests.sh @@ -680,14 +680,16 @@ export __ZFS_POOL_EXCLUDE export TESTFAIL_CALLBACKS export PATH=$STF_PATH -if [ "$UNAME" = "FreeBSD" ] ; then - mkdir -p "$FILEDIR" || true - RESULTS_FILE=$(mktemp -u "${FILEDIR}/zts-results.XXXXXX") - REPORT_FILE=$(mktemp -u "${FILEDIR}/zts-report.XXXXXX") -else - RESULTS_FILE=$(mktemp -u -t zts-results.XXXXXX -p "$FILEDIR") - REPORT_FILE=$(mktemp -u -t zts-report.XXXXXX -p "$FILEDIR") -fi +mktemp_file() { + if [ "$UNAME" = "FreeBSD" ]; then + mktemp -u "${FILEDIR}/$1.XXXXXX" + else + mktemp -ut "$1.XXXXXX" -p "$FILEDIR" + fi +} +mkdir -p "$FILEDIR" || : +RESULTS_FILE=$(mktemp_file zts-results) +REPORT_FILE=$(mktemp_file zts-report) # # Run all the tests as specified. @@ -711,8 +713,8 @@ RESULT=$? if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then MAYBES="$($ZTS_REPORT --list-maybes)" - TEMP_RESULTS_FILE=$(mktemp -u -t zts-results-tmp.XXXXX -p "$FILEDIR") - TEST_LIST=$(mktemp -u -t test-list.XXXXX -p "$FILEDIR") + TEMP_RESULTS_FILE=$(mktemp_file zts-results-tmp) + TEST_LIST=$(mktemp_file test-list) grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE" for test_name in $MAYBES; do grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST" From ddcdccbcc448ad5dcdafc872a2fa777e0b953680 Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Sat, 26 Feb 2022 11:24:27 -0800 Subject: [PATCH 099/109] Fix erroneous zstreamdump warning Reviewed-by: Brian Behlendorf Reviewed-by: George Amanakis Signed-off-by: Paul Dagnelie Closes #13154 --- cmd/zstream/zstream_dump.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/zstream/zstream_dump.c b/cmd/zstream/zstream_dump.c index af69a1494f28..04a4986b45d8 100644 --- a/cmd/zstream/zstream_dump.c +++ b/cmd/zstream/zstream_dump.c @@ -297,6 +297,7 @@ zstream_do_dump(int argc, char *argv[]) fletcher_4_init(); while (read_hdr(drr, &zc)) { + uint64_t featureflags = 0; /* * If this is the first DMU record being processed, check for @@ -362,6 +363,9 @@ zstream_do_dump(int argc, char *argv[]) BSWAP_64(drrb->drr_fromguid); } + featureflags = + DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo); + (void) printf("BEGIN record\n"); (void) printf("\thdrtype = %lld\n", DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo)); @@ -461,7 +465,8 @@ zstream_do_dump(int argc, char *argv[]) BSWAP_64(drro->drr_maxblkid); } - if (drro->drr_bonuslen > drro->drr_raw_bonuslen) { + if (featureflags & DMU_BACKUP_FEATURE_RAW && + drro->drr_bonuslen > drro->drr_raw_bonuslen) { (void) fprintf(stderr, "Warning: Object %llu has bonuslen = " "%u > raw_bonuslen = %u\n\n", From 4cb88d7fdccfff1486840a379cdddcb4e0f027b0 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Tue, 1 Mar 2022 08:46:00 -0800 Subject: [PATCH 100/109] ZTS: Move largest_pool_001_pos.ksh to Linux runfile On FreeBSD pools are not allowed to be created using vdevs which are backed by ZFS volumes. This configuration is not recommended for any supported platform, nevertheless the largest_pool_001_pos.ksh test case makes use of it as a convenience. This causes the test case to fail reliably on FreeBSD. The layout is still tolerated on Linux so only perform this test on Linux. Reviewed-by: Igor Kozhukhov Reviewed by: George Melikov Reviewed-by: Ryan Moeller Signed-off-by: Brian Behlendorf Closes #13166 --- tests/runfiles/common.run | 6 ------ tests/runfiles/linux.run | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 44115f6a97b4..609de105a4e1 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -652,12 +652,6 @@ tags = ['functional', 'inuse'] tests = ['large_files_001_pos', 'large_files_002_pos'] tags = ['functional', 'large_files'] -[tests/functional/largest_pool] -tests = ['largest_pool_001_pos'] -pre = -post = -tags = ['functional', 'largest_pool'] - [tests/functional/limits] tests = ['filesystem_count', 'filesystem_limit', 'snapshot_count', 'snapshot_limit'] diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 5a9fbe994611..c01e1e3c4d53 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -113,6 +113,12 @@ tags = ['functional', 'features', 'large_dnode'] tests = ['libaio', 'io_uring'] tags = ['functional', 'io'] +[tests/functional/largest_pool:Linux] +tests = ['largest_pool_001_pos'] +pre = +post = +tags = ['functional', 'largest_pool'] + [tests/functional/mmap:Linux] tests = ['mmap_libaio_001_pos'] tags = ['functional', 'mmap'] From e2fddf07bdaeb1db448a25d3b28fcfa63fce02d8 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Tue, 1 Mar 2022 08:47:30 -0800 Subject: [PATCH 101/109] ZTS: Modify receive-o-x_props_override.ksh exception As previously noted in #12272 the receive-o-x_props_override.ksh test reliably fails on FreeBSD. Since we don't expect this test to pass move the exception from the "maybe" to "known" section. This way we don't retry the FAILED test when it is not expected to pass. Reviewed-by: George Melikov Reviewed-by: Ryan Moeller Signed-off-by: Brian Behlendorf Closes #13167 --- tests/test-runner/bin/zts-report.py.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index 6a8553d033a9..e8100d0aa953 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -181,6 +181,8 @@ known = { if sys.platform.startswith('freebsd'): known.update({ + 'cli_root/zfs_receive/receive-o-x_props_override': + ['FAIL', known_reason], 'cli_root/zpool_wait/zpool_wait_trim_basic': ['SKIP', trim_reason], 'cli_root/zpool_wait/zpool_wait_trim_cancel': ['SKIP', trim_reason], 'cli_root/zpool_wait/zpool_wait_trim_flag': ['SKIP', trim_reason], @@ -268,8 +270,6 @@ if sys.platform.startswith('freebsd'): maybe.update({ 'cli_root/zfs_copies/zfs_copies_002_pos': ['FAIL', known_reason], 'cli_root/zfs_inherit/zfs_inherit_001_neg': ['FAIL', known_reason], - 'cli_root/zfs_receive/receive-o-x_props_override': - ['FAIL', known_reason], 'cli_root/zfs_share/zfs_share_011_pos': ['FAIL', known_reason], 'cli_root/zfs_share/zfs_share_concurrent_shares': ['FAIL', known_reason], From 190516f0c514e96f6121c34bb2a8c2c8a18d765e Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Sun, 20 Feb 2022 19:21:31 -0800 Subject: [PATCH 102/109] ZTS: Retry in import_rewind_config_changed.ksh As explained by the disclaimer in the test case, "This test can fail since nothing guarantees that old MOS blocks aren't overwritten." This behavior is expected and correct, but results in a flaky test case which is problematic for the CI. The best we can do to resolve this is to retry the sub-test which failed when the MOS blocks have clearly been overwritten. When testing failures were rare enough that a single retry should normally be sufficient. However, we allow up to five for good measure. Reviewed by: George Melikov Signed-off-by: Brian Behlendorf Closes #13119 --- tests/test-runner/bin/zts-report.py.in | 2 - .../import_rewind_config_changed.ksh | 58 ++++++++++++++++--- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index e8100d0aa953..559e98dd07b2 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -224,8 +224,6 @@ maybe = { 'cli_root/zfs_unshare/setup': ['SKIP', share_reason], 'cli_root/zpool_add/zpool_add_004_pos': ['FAIL', known_reason], 'cli_root/zpool_destroy/zpool_destroy_001_pos': ['SKIP', '6145'], - 'cli_root/zpool_import/import_rewind_config_changed': - ['FAIL', rewind_reason], 'cli_root/zpool_import/zpool_import_missing_003_pos': ['SKIP', '6839'], 'cli_root/zpool_initialize/zpool_initialize_import_export': ['FAIL', '11948'], diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_config_changed.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_config_changed.ksh index 3ac8c104f1ca..d79c757d2406 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_config_changed.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_config_changed.ksh @@ -61,6 +61,7 @@ function test_common typeset detachvdev="${4:-}" typeset removevdev="${5:-}" typeset finalpool="${6:-}" + typeset retval=1 typeset poolcheck="$poolcreate" @@ -120,19 +121,30 @@ function test_common # while having a checkpoint, we take it after the # operation that changes the config. # + # However, it is possible the MOS data was overwritten + # in which case the pool will either be unimportable, or + # may have been rewound prior to the data being written. + # In which case an error is returned and test_common() + # is retried by the caller to minimize false positives. + # log_must zpool checkpoint $TESTPOOL1 log_must overwrite_data $TESTPOOL1 "" log_must zpool export $TESTPOOL1 - log_must zpool import -d $DEVICE_DIR -T $txg $TESTPOOL1 - log_must check_pool_config $TESTPOOL1 "$poolcheck" + zpool import -d $DEVICE_DIR -T $txg $TESTPOOL1 + if (( $? == 0 )); then + verify_data_md5sums $MD5FILE + if (( $? == 0 )); then + retval=0 + fi - log_must verify_data_md5sums $MD5FILE + log_must check_pool_config $TESTPOOL1 "$poolcheck" + log_must zpool destroy $TESTPOOL1 + fi # Cleanup - log_must zpool destroy $TESTPOOL1 if [[ -n $pathstochange ]]; then for dev in $pathstochange; do log_must mv "${dev}_new" $dev @@ -143,6 +155,7 @@ function test_common log_must zpool destroy $TESTPOOL2 log_note "" + return $retval } function test_add_vdevs @@ -152,7 +165,12 @@ function test_add_vdevs log_note "$0: pool '$poolcreate', add $addvdevs." - test_common "$poolcreate" "$addvdevs" + for retry in $(seq 1 5); do + test_common "$poolcreate" "$addvdevs" && return + log_note "Retry $retry / 5 for test_add_vdevs()" + done + + log_fail "Exhausted all 5 retries for test_add_vdevs()" } function test_attach_vdev @@ -163,7 +181,12 @@ function test_attach_vdev log_note "$0: pool '$poolcreate', attach $attachvdev to $attachto." - test_common "$poolcreate" "" "$attachto $attachvdev" + for retry in $(seq 1 5); do + test_common "$poolcreate" "" "$attachto $attachvdev" && return + log_note "Retry $retry / 5 for test_attach_vdev()" + done + + log_fail "Exhausted all 5 retries for test_attach_vdev()" } function test_detach_vdev @@ -173,7 +196,12 @@ function test_detach_vdev log_note "$0: pool '$poolcreate', detach $detachvdev." - test_common "$poolcreate" "" "" "$detachvdev" + for retry in $(seq 1 5); do + test_common "$poolcreate" "" "" "$detachvdev" && return + log_note "Retry $retry / 5 for test_detach_vdev()" + done + + log_fail "Exhausted all 5 retries for test_detach_vdev()" } function test_attach_detach_vdev @@ -186,7 +214,13 @@ function test_attach_detach_vdev log_note "$0: pool '$poolcreate', attach $attachvdev to $attachto," \ "then detach $detachvdev." - test_common "$poolcreate" "" "$attachto $attachvdev" "$detachvdev" + for retry in $(seq 1 5); do + test_common "$poolcreate" "" "$attachto $attachvdev" \ + "$detachvdev" && return + log_note "Retry $retry / 5 for test_attach_detach_vdev()" + done + + log_fail "Exhausted all 5 retries for test_attach_detach_vdev()" } function test_remove_vdev @@ -197,7 +231,13 @@ function test_remove_vdev log_note "$0: pool '$poolcreate', remove $removevdev." - test_common "$poolcreate" "" "" "" "$removevdev" "$finalpool" + for retry in $(seq 1 5); do + test_common "$poolcreate" "" "" "" "$removevdev" \ + "$finalpool" && return + log_note "Retry $retry / 5 for test_remove_vdev()" + done + + log_fail "Exhausted all 5 retries for test_remove_vdev()" } # Record txg history From 037434e4fc5e30f6bb38ce62c7aab8aaa30c6e0b Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 2 Mar 2022 11:03:53 -0800 Subject: [PATCH 103/109] ZTS: Fix import_devices_missing.ksh Related to commit 90b77a036. Retry the `zpool export` if the pool is "busy" indicating there is a process accessing the mount point. This can happen after an import, allowing it to be retried will avoid spurious test failures. Reviewed-by: Tony Hutter Signed-off-by: Brian Behlendorf Closes #13169 --- .../cli_root/zpool_import/import_devices_missing.ksh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_devices_missing.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_devices_missing.ksh index 53828c912ca3..af6ac8d78e4e 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_devices_missing.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_devices_missing.ksh @@ -68,7 +68,7 @@ function test_devices_missing log_must generate_data $TESTPOOL1 $MD5FILE2 "second" - log_must zpool export $TESTPOOL1 + log_must_busy zpool export $TESTPOOL1 log_must mv $missingvdevs $BACKUP_DEVICE_DIR @@ -85,7 +85,7 @@ function test_devices_missing "get suspended." verify_data_md5sums $MD5FILE >/dev/null 2>&1 - log_must zpool export $TESTPOOL1 + log_must_busy zpool export $TESTPOOL1 typeset newpaths=$(echo "$missingvdevs" | \ sed "s:$DEVICE_DIR:$BACKUP_DEVICE_DIR:g") From 0e2bb1a3ee395887a8e75f0273aca2b328a3f3cd Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Fri, 17 Sep 2021 13:17:18 -0400 Subject: [PATCH 104/109] Really zero the zero page While switching abd_zero_buf allocation KPI I've missed the fact that kmem_zalloc() zeroed the allocation, while kmem_cache_alloc() does not. Add explicit bzero() after it. I don't think it should have caused real problems, but leaking one memory page content all over the pool is not good. Reviewed-by: Brian Behlendorf Reviewed-by: Brian Atkinson Reviewed-by: Ryan Moeller Signed-off-by: Alexander Motin Closes #12569 --- module/os/freebsd/zfs/abd_os.c | 1 + 1 file changed, 1 insertion(+) diff --git a/module/os/freebsd/zfs/abd_os.c b/module/os/freebsd/zfs/abd_os.c index 95a83542fadc..41ceed1dc54c 100644 --- a/module/os/freebsd/zfs/abd_os.c +++ b/module/os/freebsd/zfs/abd_os.c @@ -250,6 +250,7 @@ abd_alloc_zero_scatter(void) n = abd_chunkcnt_for_bytes(SPA_MAXBLOCKSIZE); abd_zero_buf = kmem_cache_alloc(abd_chunk_cache, KM_PUSHPAGE); + bzero(abd_zero_buf, PAGE_SIZE); abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE); abd_zero_scatter->abd_flags |= ABD_FLAG_OWNER | ABD_FLAG_ZEROS; From b3427b18b1c694f9dbc4673a11a6cf57c7f5c5d8 Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Sat, 20 Nov 2021 11:21:25 -0500 Subject: [PATCH 105/109] zfs: Fix a deadlock between page busy and the teardown lock When rolling back a dataset, ZFS has to purge file data resident in the system page cache. To do this, it loops over all vnodes for the mountpoint and calls vn_pages_remove() to purge pages associated with the vnode's VM object. Each page is thus exclusively busied while the dataset's teardown write lock is held. When handling a page fault on a mapped ZFS file, FreeBSD's page fault handler busies newly allocated pages and then uses VOP_GETPAGES to fill them. The ZFS getpages VOP acquires the teardown read lock with vnode pages already busied. This represents a lock order reversal which can lead to deadlock. To break the deadlock, observe that zfs_rezget() need only purge those pages marked valid, and that pages busied by the page fault handler are, by definition, invalid. Furthermore, ZFS pages always transition from invalid to valid with the teardown lock held, and ZFS never creates partially valid pages. Thus, zfs_rezget() can use the new vn_pages_remove_valid() to skip over pages busied by the fault handler. PR: 258208 Tested by: pho Reviewed by: avg, sef, kib MFC after: 2 weeks Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D32931 Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Signed-off-by: Ryan Moeller Closes #12828 --- module/os/freebsd/zfs/zfs_znode.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/module/os/freebsd/zfs/zfs_znode.c b/module/os/freebsd/zfs/zfs_znode.c index 85f71f99ea16..317a35eefd0e 100644 --- a/module/os/freebsd/zfs/zfs_znode.c +++ b/module/os/freebsd/zfs/zfs_znode.c @@ -1083,9 +1083,18 @@ zfs_rezget(znode_t *zp) * the vnode in case of error, but currently we cannot do that * because of the LOR between the vnode lock and z_teardown_lock. * So, instead, we have to "doom" the znode in the illumos style. + * + * Ignore invalid pages during the scan. This is to avoid deadlocks + * between page busying and the teardown lock, as pages are busied prior + * to a VOP_GETPAGES operation, which acquires the teardown read lock. + * Such pages will be invalid and can safely be skipped here. */ vp = ZTOV(zp); +#if __FreeBSD_version >= 1400042 + vn_pages_remove_valid(vp, 0, 0); +#else vn_pages_remove(vp, 0, 0); +#endif ZFS_OBJ_HOLD_ENTER(zfsvfs, obj_num); From b3b6491ce99c91739b8a25d513404188f8769125 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Mon, 11 Oct 2021 10:49:13 -0700 Subject: [PATCH 106/109] ZTS: deadman_sync fix In the CI environment it's possible for events to be slightly delayed resulting in 4, instead of 5, events appearing in the log file. This isn't a problem and should be considered a success to avoid false positive test results. Reviewed-by: George Melikov Signed-off-by: Brian Behlendorf Closes #12625 --- tests/zfs-tests/tests/functional/deadman/deadman_sync.ksh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/zfs-tests/tests/functional/deadman/deadman_sync.ksh b/tests/zfs-tests/tests/functional/deadman/deadman_sync.ksh index b0b03f5d523e..fd6e8c858edd 100755 --- a/tests/zfs-tests/tests/functional/deadman/deadman_sync.ksh +++ b/tests/zfs-tests/tests/functional/deadman/deadman_sync.ksh @@ -80,10 +80,10 @@ else fi log_must zpool events -# Verify at least 5 deadman events were logged. The first after 5 seconds, +# Verify at least 4 deadman events were logged. The first after 5 seconds, # and another each second thereafter until the delay is clearer. events=$(zpool events | grep -c ereport.fs.zfs.deadman) -if [ "$events" -lt 5 ]; then +if [ "$events" -lt 4 ]; then log_fail "Expect >=5 deadman events, $events found" fi From 145af480d32a866c5732977688ce5cb132ac2faf Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Tue, 8 Mar 2022 09:16:35 -0800 Subject: [PATCH 107/109] Fix ENOSPC when unlinking multiple files from full pool When unlinking multiple files from a pool at 100% capacity, it was possible for ENOSPC to be returned after the first unlink. e.g. rm -f /mnt/fs/test1.0.0 /mnt/fs/test1.1.0 /mnt/fs/test1.2.0 rm: cannot remove '/mnt/fs/test1.1.0': No space left on device rm: cannot remove '/mnt/fs/test1.2.0': No space left on device After waiting for the pending deferred frees from the first unlink to be processed the remaining files can then be unlinked. This is caused by the quota limit in dsl_dir_tempreserve_impl() being temporarily decreased to the allocatable pool capacity less any deferred free space. This is resolved using the existing mechanism of returning ERESTART when over quota as long as we know enough space will shortly be available after processing the pending deferred frees. Reviewed-by: Alexander Motin Reviewed-by: Ryan Moeller Reviewed-by: Tony Hutter Signed-off-by: Brian Behlendorf Closes #13172 --- include/sys/dsl_pool.h | 1 + module/zfs/dsl_dir.c | 20 +++++-- module/zfs/dsl_pool.c | 6 ++ tests/runfiles/common.run | 2 +- .../tests/functional/no_space/Makefile.am | 3 +- .../tests/functional/no_space/enospc_rm.ksh | 60 +++++++++++++++++++ 6 files changed, 84 insertions(+), 8 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/no_space/enospc_rm.ksh diff --git a/include/sys/dsl_pool.h b/include/sys/dsl_pool.h index 8249bb8fc633..58fcae65db5a 100644 --- a/include/sys/dsl_pool.h +++ b/include/sys/dsl_pool.h @@ -158,6 +158,7 @@ int dsl_pool_sync_context(dsl_pool_t *dp); uint64_t dsl_pool_adjustedsize(dsl_pool_t *dp, zfs_space_check_t slop_policy); uint64_t dsl_pool_unreserved_space(dsl_pool_t *dp, zfs_space_check_t slop_policy); +uint64_t dsl_pool_deferred_space(dsl_pool_t *dp); void dsl_pool_dirty_space(dsl_pool_t *dp, int64_t space, dmu_tx_t *tx); void dsl_pool_undirty_space(dsl_pool_t *dp, int64_t space, uint64_t txg); void dsl_free(dsl_pool_t *dp, uint64_t txg, const blkptr_t *bpp); diff --git a/module/zfs/dsl_dir.c b/module/zfs/dsl_dir.c index 43ef1897811c..aca32ff9bbb9 100644 --- a/module/zfs/dsl_dir.c +++ b/module/zfs/dsl_dir.c @@ -1334,13 +1334,21 @@ dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree, /* * If they are requesting more space, and our current estimate * is over quota, they get to try again unless the actual - * on-disk is over quota and there are no pending changes (which - * may free up space for us). + * on-disk is over quota and there are no pending changes + * or deferred frees (which may free up space for us). */ if (used_on_disk + est_inflight >= quota) { - if (est_inflight > 0 || used_on_disk < quota || - (retval == ENOSPC && used_on_disk < quota)) - retval = ERESTART; + if (est_inflight > 0 || used_on_disk < quota) { + retval = SET_ERROR(ERESTART); + } else { + ASSERT3U(used_on_disk, >=, quota); + + if (retval == ENOSPC && (used_on_disk - quota) < + dsl_pool_deferred_space(dd->dd_pool)) { + retval = SET_ERROR(ERESTART); + } + } + dprintf_dd(dd, "failing: used=%lluK inflight = %lluK " "quota=%lluK tr=%lluK err=%d\n", (u_longlong_t)used_on_disk>>10, @@ -1348,7 +1356,7 @@ dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree, (u_longlong_t)quota>>10, (u_longlong_t)asize>>10, retval); mutex_exit(&dd->dd_lock); DMU_TX_STAT_BUMP(dmu_tx_quota); - return (SET_ERROR(retval)); + return (retval); } /* We need to up our estimated delta before dropping dd_lock */ diff --git a/module/zfs/dsl_pool.c b/module/zfs/dsl_pool.c index 9301641aad2d..af7fa5af4e1e 100644 --- a/module/zfs/dsl_pool.c +++ b/module/zfs/dsl_pool.c @@ -893,6 +893,12 @@ dsl_pool_unreserved_space(dsl_pool_t *dp, zfs_space_check_t slop_policy) return (quota); } +uint64_t +dsl_pool_deferred_space(dsl_pool_t *dp) +{ + return (metaslab_class_get_deferred(spa_normal_class(dp->dp_spa))); +} + boolean_t dsl_pool_need_dirty_delay(dsl_pool_t *dp) { diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 609de105a4e1..aefcd98436d7 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -686,7 +686,7 @@ tags = ['functional', 'nestedfs'] [tests/functional/no_space] tests = ['enospc_001_pos', 'enospc_002_pos', 'enospc_003_pos', - 'enospc_df'] + 'enospc_df', 'enospc_rm'] tags = ['functional', 'no_space'] [tests/functional/nopwrite] diff --git a/tests/zfs-tests/tests/functional/no_space/Makefile.am b/tests/zfs-tests/tests/functional/no_space/Makefile.am index c2e42bc2ada4..31584fb17583 100644 --- a/tests/zfs-tests/tests/functional/no_space/Makefile.am +++ b/tests/zfs-tests/tests/functional/no_space/Makefile.am @@ -5,7 +5,8 @@ dist_pkgdata_SCRIPTS = \ enospc_001_pos.ksh \ enospc_002_pos.ksh \ enospc_003_pos.ksh \ - enospc_df.ksh + enospc_df.ksh \ + enospc_rm.ksh dist_pkgdata_DATA = \ enospc.cfg diff --git a/tests/zfs-tests/tests/functional/no_space/enospc_rm.ksh b/tests/zfs-tests/tests/functional/no_space/enospc_rm.ksh new file mode 100755 index 000000000000..065abc75977e --- /dev/null +++ b/tests/zfs-tests/tests/functional/no_space/enospc_rm.ksh @@ -0,0 +1,60 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2014, 2016 by Delphix. All rights reserved. +# Copyright (c) 2022 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/no_space/enospc.cfg + +# +# DESCRIPTION: +# After filling a filesystem, verify the contents can be removed +# without encountering an ENOSPC error. +# + +verify_runnable "both" + +function cleanup +{ + destroy_pool $TESTPOOL + log_must rm -f $all_vdevs +} + +log_onexit cleanup + +log_assert "Files can be removed from full file system." + +all_vdevs=$(echo $TEST_BASE_DIR/file.{01..12}) + +log_must truncate -s $MINVDEVSIZE $all_vdevs + +log_must zpool create -f $TESTPOOL draid2:8d:2s $all_vdevs +log_must zfs create $TESTPOOL/$TESTFS +log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS +log_must zfs set compression=off $TESTPOOL/$TESTFS + +log_note "Writing files until ENOSPC." +log_mustnot_expect "No space left on device" fio --name=test \ + --fallocate=none --rw=write --bs=1M --size=1G --numjobs=4 \ + --sync=1 --directory=$TESTDIR/ --group_reporting + +log_must rm $TESTDIR/test.* +log_must test -z "$(ls -A $TESTDIR)" + +log_pass "All files removed without error" From ef83e07db53e5d1017d3afbf376f4dbb2f6feada Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Tue, 8 Mar 2022 18:03:54 -0800 Subject: [PATCH 108/109] Tag zfs-2.1.3 META file and changelog updated. Signed-off-by: Tony Hutter --- META | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/META b/META index 10130517955a..0437224b403f 100644 --- a/META +++ b/META @@ -1,10 +1,10 @@ Meta: 1 Name: zfs Branch: 1.0 -Version: 2.1.2 +Version: 2.1.3 Release: 1 Release-Tags: relext License: CDDL Author: OpenZFS -Linux-Maximum: 5.15 +Linux-Maximum: 5.16 Linux-Minimum: 3.10 From 0452ae895453a187a32c8512689bc20fadf91ed4 Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Thu, 10 Mar 2022 21:18:14 +0000 Subject: [PATCH 109/109] Update changelog for zfs-2.1.3 Signed-off-by: Ryan Moeller --- contrib/truenas/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/truenas/changelog b/contrib/truenas/changelog index 49842af77bb6..d5f417bc1db8 100644 --- a/contrib/truenas/changelog +++ b/contrib/truenas/changelog @@ -1,3 +1,9 @@ +openzfs (2.1.3-0) unstable; urgency=medium + + * Merged OpenZFS zfs-2.1.3 + + -- Ryan Moeller Thu, 10 Mar 2021 16:00:00 -0400 + openzfs (2.1.2-1) unstable; urgency=medium * Merged OpenZFS zfs-2.1.2-release