merged and fixed merge conflicts with zol upstream

cmd/arc_summary/Makefile.am

@@ -4,9 +4,7 @@ if USING_PYTHON_2
 dist_bin_SCRIPTS = arc_summary2
 install-exec-hook:
 	mv $(DESTDIR)$(bindir)/arc_summary2 $(DESTDIR)$(bindir)/arc_summary
-endif
-
-if USING_PYTHON_3
+else
 dist_bin_SCRIPTS = arc_summary3
 install-exec-hook:
 	mv $(DESTDIR)$(bindir)/arc_summary3 $(DESTDIR)$(bindir)/arc_summary

cmd/ztest/ztest.c

@@ -1692,7 +1692,7 @@ ztest_log_write(ztest_ds_t *zd, dmu_tx_t *tx, lr_write_t *lr)
 	if (zil_replaying(zd->zd_zilog, tx))
 		return;
 
-	if (lr->lr_length > ZIL_MAX_LOG_DATA)
+	if (lr->lr_length > zil_max_log_data(zd->zd_zilog))
 		write_state = WR_INDIRECT;
 
 	itx = zil_itx_create(TX_WRITE,

config/always-python.m4

@@ -1,47 +1,3 @@
-dnl #
-dnl # ZFS_AC_PYTHON_VERSION(version, [action-if-true], [action-if-false])
-dnl #
-dnl # Verify Python version
-dnl #
-AC_DEFUN([ZFS_AC_PYTHON_VERSION], [
-	ver_check=`$PYTHON -c "import sys; print (sys.version.split()[[0]] $1)"`
-	AS_IF([test "$ver_check" = "True"], [
-		m4_ifvaln([$2], [$2])
-	], [
-		m4_ifvaln([$3], [$3])
-	])
-])
-
-dnl #
-dnl # ZFS_AC_PYTHON_VERSION_IS_2
-dnl # ZFS_AC_PYTHON_VERSION_IS_3
-dnl #
-dnl # Tests if the $PYTHON_VERSION matches 2.x or 3.x.
-dnl #
-AC_DEFUN([ZFS_AC_PYTHON_VERSION_IS_2],
-	[test "${PYTHON_VERSION%%\.*}" = "2"])
-AC_DEFUN([ZFS_AC_PYTHON_VERSION_IS_3],
-	[test "${PYTHON_VERSION%%\.*}" = "3"])
-
-dnl #
-dnl # ZFS_AC_PYTHON_MODULE(module_name, [action-if-true], [action-if-false])
-dnl #
-dnl # Checks for Python module. Freely inspired by AX_PYTHON_MODULE
-dnl # https://www.gnu.org/software/autoconf-archive/ax_python_module.html
-dnl # Required by ZFS_AC_CONFIG_ALWAYS_PYZFS.
-dnl #
-AC_DEFUN([ZFS_AC_PYTHON_MODULE], [
-	PYTHON_NAME=`basename $PYTHON`
-	AC_MSG_CHECKING([for $PYTHON_NAME module: $1])
-	AS_IF([$PYTHON -c "import $1" 2>/dev/null], [
-		AC_MSG_RESULT(yes)
-		m4_ifvaln([$2], [$2])
-	], [
-		AC_MSG_RESULT(no)
-		m4_ifvaln([$3], [$3])
-	])
-])
-
 dnl #
 dnl # The majority of the python scripts are written to be compatible
 dnl # with Python 2.6 and Python 3.4.  Therefore, they may be installed
@@ -66,35 +22,38 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYTHON], [
 		[AC_MSG_ERROR([Unknown --with-python value '$with_python'])]
 	)
 
-	AS_IF([test $PYTHON != :], [
-		AS_IF([$PYTHON --version >/dev/null 2>&1],
-			[AM_PATH_PYTHON([2.6], [], [:])],
-			[AC_MSG_ERROR([Cannot find $PYTHON in your system path])]
-		)
-	])
-	AM_CONDITIONAL([USING_PYTHON], [test "$PYTHON" != :])
-	AM_CONDITIONAL([USING_PYTHON_2], [ZFS_AC_PYTHON_VERSION_IS_2])
-	AM_CONDITIONAL([USING_PYTHON_3], [ZFS_AC_PYTHON_VERSION_IS_3])
-
 	dnl #
 	dnl # Minimum supported Python versions for utilities:
-	dnl # Python 2.6.x, or Python 3.4.x
+	dnl # Python 2.6 or Python 3.4
 	dnl #
-	AS_IF([ZFS_AC_PYTHON_VERSION_IS_2], [
-		ZFS_AC_PYTHON_VERSION([>= '2.6'], [ true ],
-			[AC_MSG_ERROR("Python >= 2.6.x is not available")])
+	AM_PATH_PYTHON([], [], [:])
+	AS_IF([test -z "$PYTHON_VERSION"], [
+		PYTHON_VERSION=$(basename $PYTHON | tr -cd 0-9.)
 	])
+	PYTHON_MINOR=${PYTHON_VERSION#*\.}
 
-	AS_IF([ZFS_AC_PYTHON_VERSION_IS_3], [
-		ZFS_AC_PYTHON_VERSION([>= '3.4'], [ true ],
-			[AC_MSG_ERROR("Python >= 3.4.x is not available")])
-	])
+	AS_CASE([$PYTHON_VERSION],
+		[2.*], [
+			AS_IF([test $PYTHON_MINOR -lt 6],
+				[AC_MSG_ERROR("Python >= 2.6 is required")])
+		],
+		[3.*], [
+			AS_IF([test $PYTHON_MINOR -lt 4],
+				[AC_MSG_ERROR("Python >= 3.4 is required")])
+		],
+		[:|2|3], [],
+		[PYTHON_VERSION=3]
+	)
+
+	AM_CONDITIONAL([USING_PYTHON], [test "$PYTHON" != :])
+	AM_CONDITIONAL([USING_PYTHON_2], [test "x${PYTHON_VERSION%%\.*}" = x2])
+	AM_CONDITIONAL([USING_PYTHON_3], [test "x${PYTHON_VERSION%%\.*}" = x3])
 
 	dnl #
 	dnl # Request that packages be built for a specific Python version.
 	dnl #
-	AS_IF([test $with_python != check], [
-		PYTHON_PKG_VERSION=`echo ${PYTHON} | tr -d 'a-zA-Z.'`
+	AS_IF([test "x$with_python" != xcheck], [
+		PYTHON_PKG_VERSION=$(echo $PYTHON_VERSION | tr -d .)
 		DEFINE_PYTHON_PKG_VERSION='--define "__use_python_pkg_version '${PYTHON_PKG_VERSION}'"'
 		DEFINE_PYTHON_VERSION='--define "__use_python '${PYTHON}'"'
 	], [

config/always-pyzfs.m4

@@ -1,5 +1,24 @@
 dnl #
-dnl # Determines if pyzfs can be built, requires Python 2.7 or latter.
+dnl # ZFS_AC_PYTHON_MODULE(module_name, [action-if-true], [action-if-false])
+dnl #
+dnl # Checks for Python module. Freely inspired by AX_PYTHON_MODULE
+dnl # https://www.gnu.org/software/autoconf-archive/ax_python_module.html
+dnl # Required by ZFS_AC_CONFIG_ALWAYS_PYZFS.
+dnl #
+AC_DEFUN([ZFS_AC_PYTHON_MODULE], [
+	PYTHON_NAME=$(basename $PYTHON)
+	AC_MSG_CHECKING([for $PYTHON_NAME module: $1])
+	AS_IF([$PYTHON -c "import $1" 2>/dev/null], [
+		AC_MSG_RESULT(yes)
+		m4_ifvaln([$2], [$2])
+	], [
+		AC_MSG_RESULT(no)
+		m4_ifvaln([$3], [$3])
+	])
+])
+
+dnl #
+dnl # Determines if pyzfs can be built, requires Python 2.7 or later.
 dnl #
 AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
 	AC_ARG_ENABLE([pyzfs],
@@ -18,7 +37,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
 			DEFINE_PYZFS='--without pyzfs'
 		])
 	], [
-		AS_IF([test $PYTHON != :], [
+		AS_IF([test "$PYTHON" != :], [
 			DEFINE_PYZFS=''
 		], [
 			enable_pyzfs=no
@@ -31,20 +50,16 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
 	dnl # Require python-devel libraries
 	dnl #
 	AS_IF([test "x$enable_pyzfs" = xcheck  -o "x$enable_pyzfs" = xyes], [
-		AS_IF([ZFS_AC_PYTHON_VERSION_IS_2], [
-			PYTHON_REQUIRED_VERSION=">= '2.7.0'"
-		], [
-			AS_IF([ZFS_AC_PYTHON_VERSION_IS_3], [
-				PYTHON_REQUIRED_VERSION=">= '3.4.0'"
-			], [
-				AC_MSG_ERROR("Python $PYTHON_VERSION unknown")
-			])
-		])
+		AS_CASE([$PYTHON_VERSION],
+			[3.*], [PYTHON_REQUIRED_VERSION=">= '3.4.0'"],
+			[2.*], [PYTHON_REQUIRED_VERSION=">= '2.7.0'"],
+			[AC_MSG_ERROR("Python $PYTHON_VERSION unknown")]
+		)
 
 		AX_PYTHON_DEVEL([$PYTHON_REQUIRED_VERSION], [
 			AS_IF([test "x$enable_pyzfs" = xyes], [
 				AC_MSG_ERROR("Python $PYTHON_REQUIRED_VERSION development library is not installed")
-			], [test ! "x$enable_pyzfs" = xno], [
+			], [test "x$enable_pyzfs" != xno], [
 				enable_pyzfs=no
 			])
 		])
@@ -57,7 +72,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
 		ZFS_AC_PYTHON_MODULE([setuptools], [], [
 			AS_IF([test "x$enable_pyzfs" = xyes], [
 				AC_MSG_ERROR("Python $PYTHON_VERSION setuptools is not installed")
-			], [test ! "x$enable_pyzfs" = xno], [
+			], [test "x$enable_pyzfs" != xno], [
 				enable_pyzfs=no
 			])
 		])
@@ -70,7 +85,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
 		ZFS_AC_PYTHON_MODULE([cffi], [], [
 			AS_IF([test "x$enable_pyzfs" = xyes], [
 				AC_MSG_ERROR("Python $PYTHON_VERSION cffi is not installed")
-			], [test ! "x$enable_pyzfs" = xno], [
+			], [test "x$enable_pyzfs" != xno], [
 				enable_pyzfs=no
 			])
 		])
@@ -81,7 +96,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
 	dnl #
 	AS_IF([test "x$enable_pyzfs" = xcheck], [enable_pyzfs=yes])
 
-	AM_CONDITIONAL([PYZFS_ENABLED], [test x$enable_pyzfs = xyes])
+	AM_CONDITIONAL([PYZFS_ENABLED], [test "x$enable_pyzfs" = xyes])
 	AC_SUBST([PYZFS_ENABLED], [$enable_pyzfs])
 	AC_SUBST(pythonsitedir, [$PYTHON_SITE_PKG])
 

include/sys/abd.h

@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
- * Copyright (c) 2016 by Delphix. All rights reserved.
+ * Copyright (c) 2016, 2019 by Delphix. All rights reserved.
  */
 
 #ifndef _ABD_H
@@ -44,7 +44,8 @@ typedef enum abd_flags {
 	ABD_FLAG_OWNER	= 1 << 1,	/* does it own its data buffers? */
 	ABD_FLAG_META	= 1 << 2,	/* does this represent FS metadata? */
 	ABD_FLAG_MULTI_ZONE  = 1 << 3,	/* pages split over memory zones */
-	ABD_FLAG_MULTI_CHUNK = 1 << 4	/* pages split over multiple chunks */
+	ABD_FLAG_MULTI_CHUNK = 1 << 4,	/* pages split over multiple chunks */
+	ABD_FLAG_LINEAR_PAGE = 1 << 5,	/* linear but allocd from page */
 } abd_flags_t;
 
 typedef struct abd {
@@ -60,6 +61,7 @@ typedef struct abd {
 		} abd_scatter;
 		struct abd_linear {
 			void		*abd_buf;
+			struct scatterlist *abd_sgl; /* for LINEAR_PAGE */
 		} abd_linear;
 	} abd_u;
 } abd_t;
@@ -75,6 +77,13 @@ abd_is_linear(abd_t *abd)
 	return ((abd->abd_flags & ABD_FLAG_LINEAR) != 0 ? B_TRUE : B_FALSE);
 }
 
+static inline boolean_t
+abd_is_linear_page(abd_t *abd)
+{
+	return ((abd->abd_flags & ABD_FLAG_LINEAR_PAGE) != 0 ?
+	    B_TRUE : B_FALSE);
+}
+
 /*
  * Allocations and deallocations
  */

include/sys/vdev_removal.h

@@ -14,7 +14,7 @@
  */
 
 /*
- * Copyright (c) 2014, 2017 by Delphix. All rights reserved.
+ * Copyright (c) 2014, 2019 by Delphix. All rights reserved.
  */
 
 #ifndef _SYS_VDEV_REMOVAL_H
@@ -81,13 +81,13 @@ extern void spa_vdev_condense_suspend(spa_t *);
 extern int spa_vdev_remove(spa_t *, uint64_t, boolean_t);
 extern void free_from_removing_vdev(vdev_t *, uint64_t, uint64_t);
 extern int spa_removal_get_stats(spa_t *, pool_removal_stat_t *);
-extern void svr_sync(spa_t *spa, dmu_tx_t *tx);
+extern void svr_sync(spa_t *, dmu_tx_t *);
 extern void spa_vdev_remove_suspend(spa_t *);
 extern int spa_vdev_remove_cancel(spa_t *);
-extern void spa_vdev_removal_destroy(spa_vdev_removal_t *svr);
+extern void spa_vdev_removal_destroy(spa_vdev_removal_t *);
+extern uint64_t spa_remove_max_segment(spa_t *);
 
 extern int vdev_removal_max_span;
-extern int zfs_remove_max_segment;
 
 #ifdef	__cplusplus
 }

include/sys/zap.h

@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  * Copyright 2017 Nexenta Systems, Inc.
  */
 
@@ -350,6 +350,7 @@ typedef struct zap_cursor {
 	uint64_t zc_serialized;
 	uint64_t zc_hash;
 	uint32_t zc_cd;
+	boolean_t zc_prefetch;
 } zap_cursor_t;
 
 typedef struct {
@@ -375,7 +376,9 @@ typedef struct {
  * Initialize a zap cursor, pointing to the "first" attribute of the
  * zapobj.  You must _fini the cursor when you are done with it.
  */
-void zap_cursor_init(zap_cursor_t *zc, objset_t *ds, uint64_t zapobj);
+void zap_cursor_init(zap_cursor_t *zc, objset_t *os, uint64_t zapobj);
+void zap_cursor_init_noprefetch(zap_cursor_t *zc, objset_t *os,
+    uint64_t zapobj);
 void zap_cursor_fini(zap_cursor_t *zc);
 
 /*

include/sys/zil.h

@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2017 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -515,6 +515,9 @@ extern void	zil_set_sync(zilog_t *zilog, uint64_t syncval);
 
 extern void	zil_set_logbias(zilog_t *zilog, uint64_t slogval);
 
+extern uint64_t	zil_max_copied_data(zilog_t *zilog);
+extern uint64_t	zil_max_log_data(zilog_t *zilog);
+
 extern int zil_replay_disable;
 
 #ifdef	__cplusplus

include/sys/zil_impl.h

@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2017 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -209,33 +209,20 @@ struct zilog {
 	uint_t		zl_prev_rotor;	/* rotor for zl_prev[] */
 	txg_node_t	zl_dirty_link;	/* protected by dp_dirty_zilogs list */
 	uint64_t	zl_dirty_max_txg; /* highest txg used to dirty zilog */
+	/*
+	 * Max block size for this ZIL.  Note that this can not be changed
+	 * while the ZIL is in use because consumers (ZPL/zvol) need to take
+	 * this into account when deciding between WR_COPIED and WR_NEED_COPY
+	 * (see zil_max_copied_data()).
+	 */
+	uint64_t	zl_max_block_size;
 };
 
 typedef struct zil_bp_node {
 	dva_t		zn_dva;
 	avl_node_t	zn_node;
 } zil_bp_node_t;
 
-/*
- * Maximum amount of write data that can be put into single log block.
- */
-#define	ZIL_MAX_LOG_DATA (SPA_OLD_MAXBLOCKSIZE - sizeof (zil_chain_t) - \
-    sizeof (lr_write_t))
-
-/*
- * Maximum amount of log space we agree to waste to reduce number of
- * WR_NEED_COPY chunks to reduce zl_get_data() overhead (~12%).
- */
-#define	ZIL_MAX_WASTE_SPACE (ZIL_MAX_LOG_DATA / 8)
-
-/*
- * Maximum amount of write data for WR_COPIED.  Fall back to WR_NEED_COPY
- * as more space efficient if we can't fit at least two log records into
- * maximum sized log block.
- */
-#define	ZIL_MAX_COPIED_DATA ((SPA_OLD_MAXBLOCKSIZE - \
-    sizeof (zil_chain_t)) / 2 - sizeof (lr_write_t))
-
 #ifdef	__cplusplus
 }
 #endif

include/sys/zio_compress.h

@@ -172,8 +172,6 @@ extern size_t lz4_compress_zfs(void *src, void *dst, size_t s_len, size_t d_len,
     int level);
 extern int lz4_decompress_zfs(void *src, void *dst, size_t s_len, size_t d_len,
     int level);
-extern int lz4_decompress_abd(abd_t *src, void *dst, size_t s_len, size_t d_len,
-    int level);
 extern size_t zstd_compress(void *src, void *dst, size_t s_len, size_t d_len,
     int level);
 extern int zstd_decompress(void *src, void *dst, size_t s_len, size_t d_len,

include/zfs_namecheck.h

@@ -43,6 +43,8 @@ typedef enum {
 	NAME_ERR_RESERVED,		/* entire name is reserved */
 	NAME_ERR_DISKLIKE,		/* reserved disk name (c[0-9].*) */
 	NAME_ERR_TOOLONG,		/* name is too long */
+	NAME_ERR_SELF_REF,		/* reserved self path name ('.') */
+	NAME_ERR_PARENT_REF,		/* reserved parent path name ('..') */
 	NAME_ERR_NO_AT,			/* permission set is missing '@' */
 } namecheck_err_t;