Illumos 4171, 4172

4171 clean up spa_feature_*() interfaces
4172 implement extensible_dataset feature for use by other zpool features
Reviewed by: Max Grossman <max.grossman@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Approved by: Garrett D'Amore <garrett@damore.org>
Ported by: Tim Chase <tim@chase2k.com>

cmd/zdb/zdb.c

@@ -565,16 +565,20 @@ get_metaslab_refcount(vdev_t *vd)
 static int
 verify_spacemap_refcounts(spa_t *spa)
 {
-	int expected_refcount, actual_refcount;
+	uint64_t expected_refcount = 0;
+	uint64_t actual_refcount;
 
-	expected_refcount = spa_feature_get_refcount(spa,
-	    &spa_feature_table[SPA_FEATURE_SPACEMAP_HISTOGRAM]);
+	(void) feature_get_refcount(spa,
+	    &spa_feature_table[SPA_FEATURE_SPACEMAP_HISTOGRAM],
+	    &expected_refcount);
 	actual_refcount = get_dtl_refcount(spa->spa_root_vdev);
 	actual_refcount += get_metaslab_refcount(spa->spa_root_vdev);
 
 	if (expected_refcount != actual_refcount) {
-		(void) printf("space map refcount mismatch: expected %d != "
-		    "actual %d\n", expected_refcount, actual_refcount);
+		(void) printf("space map refcount mismatch: expected %lld != "
+		    "actual %lld\n",
+		    (longlong_t)expected_refcount,
+		    (longlong_t)actual_refcount);
 		return (2);
 	}
 	return (0);
@@ -676,8 +680,7 @@ dump_metaslab(metaslab_t *msp)
 	}
 
 	if (dump_opt['m'] > 1 && sm != NULL &&
-	    spa_feature_is_active(spa,
-	    &spa_feature_table[SPA_FEATURE_SPACEMAP_HISTOGRAM])) {
+	    spa_feature_is_active(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM)) {
 		/*
 		 * The space map histogram represents free space in chunks
 		 * of sm_shift (i.e. bucket 0 refers to 2^sm_shift).
@@ -2515,8 +2518,7 @@ dump_block_stats(spa_t *spa)
 		(void) bpobj_iterate_nofree(&spa->spa_dsl_pool->dp_free_bpobj,
 		    count_block_cb, &zcb, NULL);
 	}
-	if (spa_feature_is_active(spa,
-	    &spa_feature_table[SPA_FEATURE_ASYNC_DESTROY])) {
+	if (spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY)) {
 		VERIFY3U(0, ==, bptree_iterate(spa->spa_meta_objset,
 		    spa->spa_dsl_pool->dp_bptree_obj, B_FALSE, count_block_cb,
 		    &zcb, NULL));
@@ -2832,7 +2834,7 @@ dump_zpool(spa_t *spa)
 			}
 
 			if (spa_feature_is_active(spa,
-			    &spa_feature_table[SPA_FEATURE_ASYNC_DESTROY])) {
+			    SPA_FEATURE_ASYNC_DESTROY)) {
 				dump_bptree(spa->spa_meta_objset,
 				    spa->spa_dsl_pool->dp_bptree_obj,
 				    "Pool dataset frees");

cmd/zhack/zhack.c

@@ -84,10 +84,15 @@ usage(void)
 
 
 static void
-fatal(const char *fmt, ...)
+fatal(spa_t *spa, void *tag, const char *fmt, ...)
 {
 	va_list ap;
 
+	if (spa != NULL) {
+		spa_close(spa, tag);
+		(void) spa_export(g_pool, NULL, B_TRUE, B_FALSE);
+	}
+
 	va_start(ap, fmt);
 	(void) fprintf(stderr, "%s: ", cmdname);
 	(void) vfprintf(stderr, fmt, ap);
@@ -158,13 +163,13 @@ import_pool(const char *target, boolean_t readonly)
 			g_importargs.can_be_active = B_TRUE;
 			if (zpool_search_import(g_zfs, &g_importargs) != NULL ||
 			    spa_open(target, &spa, FTAG) == 0) {
-				fatal("cannot import '%s': pool is active; run "
+				fatal(spa, FTAG, "cannot import '%s': pool is active; run "
 				    "\"zpool export %s\" first\n",
 				    g_pool, g_pool);
 			}
 		}
 
-		fatal("cannot import '%s': no such pool available\n", g_pool);
+		fatal(NULL, FTAG, "cannot import '%s': no such pool available\n", g_pool);
 	}
 
 	elem = nvlist_next_nvpair(pools, NULL);
@@ -185,7 +190,7 @@ import_pool(const char *target, boolean_t readonly)
 		error = 0;
 
 	if (error)
-		fatal("can't import '%s': %s", name, strerror(error));
+		fatal(NULL, FTAG, "can't import '%s': %s", name, strerror(error));
 }
 
 static void
@@ -200,9 +205,9 @@ zhack_spa_open(const char *target, boolean_t readonly, void *tag, spa_t **spa)
 	zfeature_checks_disable = B_FALSE;
 
 	if (err != 0)
-		fatal("cannot open '%s': %s", target, strerror(err));
+		fatal(*spa, FTAG, "cannot open '%s': %s", target, strerror(err));
 	if (spa_version(*spa) < SPA_VERSION_FEATURES) {
-		fatal("'%s' has version %d, features not enabled", target,
+		fatal(*spa, FTAG, "'%s' has version %d, features not enabled", target,
 		    (int)spa_version(*spa));
 	}
 }
@@ -275,12 +280,13 @@ zhack_do_feature_stat(int argc, char **argv)
 }
 
 static void
-feature_enable_sync(void *arg, dmu_tx_t *tx)
+zhack_feature_enable_sync(void *arg, dmu_tx_t *tx)
 {
 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
 	zfeature_info_t *feature = arg;
 
-	spa_feature_enable(spa, feature, tx);
+	feature_enable_sync(spa, feature, tx);
+
 	spa_history_log_internal(spa, "zhack enable feature", tx,
 	    "name=%s can_readonly=%u",
 	    feature->fi_guid, feature->fi_can_readonly);
@@ -294,7 +300,7 @@ zhack_do_feature_enable(int argc, char **argv)
 	spa_t *spa;
 	objset_t *mos;
 	zfeature_info_t feature;
-	zfeature_info_t *nodeps[] = { NULL };
+	spa_feature_t nodeps[] = { SPA_FEATURE_NONE };
 
 	/*
 	 * Features are not added to the pool's label until their refcounts
@@ -336,18 +342,18 @@ zhack_do_feature_enable(int argc, char **argv)
 	feature.fi_guid = argv[1];
 
 	if (!zfeature_is_valid_guid(feature.fi_guid))
-		fatal("invalid feature guid: %s", feature.fi_guid);
+		fatal(NULL, FTAG, "invalid feature guid: %s", feature.fi_guid);
 
 	zhack_spa_open(target, B_FALSE, FTAG, &spa);
 	mos = spa->spa_meta_objset;
 
-	if (0 == zfeature_lookup_guid(feature.fi_guid, NULL))
-		fatal("'%s' is a real feature, will not enable");
+	if (zfeature_is_supported(feature.fi_guid))
+		fatal(spa, FTAG, "'%s' is a real feature, will not enable");
 	if (0 == zap_contains(mos, spa->spa_feat_desc_obj, feature.fi_guid))
-		fatal("feature already enabled: %s", feature.fi_guid);
+		fatal(spa, FTAG, "feature already enabled: %s", feature.fi_guid);
 
 	VERIFY0(dsl_sync_task(spa_name(spa), NULL,
-	    feature_enable_sync, &feature, 5));
+	    zhack_feature_enable_sync, &feature, 5));
 
 	spa_close(spa, FTAG);
 
@@ -359,8 +365,10 @@ feature_incr_sync(void *arg, dmu_tx_t *tx)
 {
 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
 	zfeature_info_t *feature = arg;
+	uint64_t refcount;
 
-	spa_feature_incr(spa, feature, tx);
+	VERIFY0(feature_get_refcount(spa, feature, &refcount));
+	feature_sync(spa, feature, refcount + 1, tx);
 	spa_history_log_internal(spa, "zhack feature incr", tx,
 	    "name=%s", feature->fi_guid);
 }
@@ -370,8 +378,10 @@ feature_decr_sync(void *arg, dmu_tx_t *tx)
 {
 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
 	zfeature_info_t *feature = arg;
+	uint64_t refcount;
 
-	spa_feature_decr(spa, feature, tx);
+	VERIFY0(feature_get_refcount(spa, feature, &refcount));
+	feature_sync(spa, feature, refcount - 1, tx);
 	spa_history_log_internal(spa, "zhack feature decr", tx,
 	    "name=%s", feature->fi_guid);
 }
@@ -385,7 +395,7 @@ zhack_do_feature_ref(int argc, char **argv)
 	spa_t *spa;
 	objset_t *mos;
 	zfeature_info_t feature;
-	zfeature_info_t *nodeps[] = { NULL };
+	spa_feature_t nodeps[] = { SPA_FEATURE_NONE };
 
 	/*
 	 * fi_desc does not matter here because it was written to disk
@@ -423,13 +433,15 @@ zhack_do_feature_ref(int argc, char **argv)
 	feature.fi_guid = argv[1];
 
 	if (!zfeature_is_valid_guid(feature.fi_guid))
-		fatal("invalid feature guid: %s", feature.fi_guid);
+		fatal(NULL, FTAG, "invalid feature guid: %s", feature.fi_guid);
 
 	zhack_spa_open(target, B_FALSE, FTAG, &spa);
 	mos = spa->spa_meta_objset;
 
-	if (0 == zfeature_lookup_guid(feature.fi_guid, NULL))
-		fatal("'%s' is a real feature, will not change refcount");
+	if (zfeature_is_supported(feature.fi_guid)) {
+		fatal(spa, FTAG,
+		    "'%s' is a real feature, will not change refcount");
+	}
 
 	if (0 == zap_contains(mos, spa->spa_feat_for_read_obj,
 	    feature.fi_guid)) {
@@ -438,11 +450,17 @@ zhack_do_feature_ref(int argc, char **argv)
 	    feature.fi_guid)) {
 		feature.fi_can_readonly = B_TRUE;
 	} else {
-		fatal("feature is not enabled: %s", feature.fi_guid);
+		fatal(spa, FTAG, "feature is not enabled: %s", feature.fi_guid);
 	}
 
-	if (decr && !spa_feature_is_active(spa, &feature))
-		fatal("feature refcount already 0: %s", feature.fi_guid);
+	if (decr) {
+		uint64_t count;
+		if (feature_get_refcount(spa, &feature, &count) == 0 &&
+		    count != 0) {
+			fatal(spa, FTAG, "feature refcount already 0: %s",
+			    feature.fi_guid);
+		}
+	}
 
 	VERIFY0(dsl_sync_task(spa_name(spa), NULL,
 	    decr ? feature_decr_sync : feature_incr_sync, &feature, 5));
@@ -531,7 +549,7 @@ main(int argc, char **argv)
 	}
 
 	if (!g_readonly && spa_export(g_pool, NULL, B_TRUE, B_TRUE) != 0) {
-		fatal("pool export failed; "
+		fatal(NULL, FTAG, "pool export failed; "
 		    "changes may not be committed to disk\n");
 	}
 

cmd/zpool/zpool_main.c

@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
- * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2013 by Delphix. All rights reserved.
  * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
  * Copyright (c) 2012 by Cyril Plisko. All rights reserved.
  */
@@ -1029,7 +1029,7 @@ zpool_do_create(int argc, char **argv)
 		 * Hand off to libzfs.
 		 */
 		if (enable_all_pool_feat) {
-			int i;
+			spa_feature_t i;
 			for (i = 0; i < SPA_FEATURES; i++) {
 				char propname[MAXPATHLEN];
 				zfeature_info_t *feat = &spa_feature_table[i];

include/sys/dmu_impl.h

@@ -273,6 +273,9 @@ typedef struct dmu_sendarg {
 	uint64_t dsa_last_data_offset;
 } dmu_sendarg_t;
 
+void dmu_object_zapify(objset_t *, uint64_t, dmu_object_type_t, dmu_tx_t *);
+void dmu_object_free_zapified(objset_t *, uint64_t, dmu_tx_t *);
+
 #ifdef	__cplusplus
 }
 #endif

include/sys/dnode.h

@@ -178,6 +178,7 @@ typedef struct dnode {
 	uint16_t dn_datablkszsec;	/* in 512b sectors */
 	uint32_t dn_datablksz;		/* in bytes */
 	uint64_t dn_maxblkid;
+	uint8_t dn_next_type[TXG_SIZE];
 	uint8_t dn_next_nblkptr[TXG_SIZE];
 	uint8_t dn_next_nlevels[TXG_SIZE];
 	uint8_t dn_next_indblkshift[TXG_SIZE];

include/sys/dsl_dataset.h

@@ -49,9 +49,9 @@ struct dsl_pool;
 #define	DS_FLAG_INCONSISTENT	(1ULL<<0)
 #define	DS_IS_INCONSISTENT(ds)	\
 	((ds)->ds_phys->ds_flags & DS_FLAG_INCONSISTENT)
+
 /*
- * Note: nopromote can not yet be set, but we want support for it in this
- * on-disk version, so that we don't need to upgrade for it later.
+ * Do not allow this dataset to be promoted.
  */
 #define	DS_FLAG_NOPROMOTE	(1ULL<<1)
 
@@ -70,6 +70,11 @@ struct dsl_pool;
 #define	DS_IS_DEFER_DESTROY(ds)	\
 	((ds)->ds_phys->ds_flags & DS_FLAG_DEFER_DESTROY)
 
+/*
+ * DS_FIELD_* are strings that are used in the "extensified" dataset zap object.
+ * They should be of the format <reverse-dns>:<field>.
+ */
+
 /*
  * DS_FLAG_CI_DATASET is set if the dataset contains a file system whose
  * name lookups should be performed case-insensitively.

include/sys/zap.h

@@ -140,6 +140,12 @@ uint64_t zap_create_flags(objset_t *os, int normflags, zap_flags_t flags,
 uint64_t zap_create_link(objset_t *os, dmu_object_type_t ot,
     uint64_t parent_obj, const char *name, dmu_tx_t *tx);
 
+/*
+ * Initialize an already-allocated object.
+ */
+void mzap_create_impl(objset_t *os, uint64_t obj, int normflags,
+    zap_flags_t flags, dmu_tx_t *tx);
+
 /*
  * Create a new zapobj with no attributes from the given (unallocated)
  * object number.

include/sys/zfeature.h

@@ -27,27 +27,40 @@
 #define	_SYS_ZFEATURE_H
 
 #include <sys/nvpair.h>
+#include <sys/txg.h>
 #include "zfeature_common.h"
 
 #ifdef	__cplusplus
 extern "C" {
 #endif
 
+#define	VALID_FEATURE_FID(fid)	((fid) >= 0 && (fid) < SPA_FEATURES)
+#define	VALID_FEATURE_OR_NONE(fid)	((fid) == SPA_FEATURE_NONE ||	\
+    VALID_FEATURE_FID(fid))
+
 struct spa;
 struct dmu_tx;
 struct objset;
 
-extern boolean_t feature_is_supported(struct objset *os, uint64_t obj,
-    uint64_t desc_obj, nvlist_t *unsup_feat, nvlist_t *enabled_feat);
-
 extern void spa_feature_create_zap_objects(struct spa *, struct dmu_tx *);
-extern void spa_feature_enable(struct spa *, zfeature_info_t *,
+extern void spa_feature_enable(struct spa *, spa_feature_t,
+    struct dmu_tx *);
+extern void spa_feature_incr(struct spa *, spa_feature_t, struct dmu_tx *);
+extern void spa_feature_decr(struct spa *, spa_feature_t, struct dmu_tx *);
+extern boolean_t spa_feature_is_enabled(struct spa *, spa_feature_t);
+extern boolean_t spa_feature_is_active(struct spa *, spa_feature_t);
+extern uint64_t spa_feature_refcount(spa_t *, spa_feature_t, uint64_t);
+extern boolean_t spa_features_check(spa_t *, boolean_t, nvlist_t *, nvlist_t *);
+
+/*
+ * These functions are only exported for zhack and zdb; normal callers should
+ * use the above interfaces.
+ */
+extern int feature_get_refcount(struct spa *, zfeature_info_t *, uint64_t *);
+extern void feature_enable_sync(struct spa *, zfeature_info_t *,
+    struct dmu_tx *);
+extern void feature_sync(struct spa *, zfeature_info_t *, uint64_t,
     struct dmu_tx *);
-extern void spa_feature_incr(struct spa *, zfeature_info_t *, struct dmu_tx *);
-extern void spa_feature_decr(struct spa *, zfeature_info_t *, struct dmu_tx *);
-extern boolean_t spa_feature_is_enabled(struct spa *, zfeature_info_t *);
-extern boolean_t spa_feature_is_active(struct spa *, zfeature_info_t *);
-extern int spa_feature_get_refcount(struct spa *, zfeature_info_t *);
 
 #ifdef	__cplusplus
 }