Count ABD memory usage

Adds two kstat entries to the `arcstats` file
for ABD memory usage - headers and data. Data
will always be a size multiple of the system
page size.

include/sys/arc.h

@@ -103,6 +103,8 @@ typedef enum arc_space_type {
 	ARC_SPACE_META,
 	ARC_SPACE_HDRS,
 	ARC_SPACE_L2HDRS,
+	ARC_SPACE_ABD_DATA,
+	ARC_SPACE_ABD_HDRS,
 	ARC_SPACE_OTHER,
 	ARC_SPACE_NUMTYPES
 } arc_space_type_t;

module/zfs/abd.c

@@ -25,6 +25,7 @@
 
 #include <sys/abd.h>
 #include <sys/zio.h>
+#include <sys/arc.h>
 #ifdef _KERNEL
 #include <linux/kernel.h>
 #include <linux/sched.h>
@@ -968,6 +969,7 @@ abd_get_offset(abd_t *sabd, size_t off)
 	ASSERT(off <= sabd->abd_size);
 
 	abd = kmem_cache_alloc(abd_struct_cache, KM_PUSHPAGE);
+	/* XXX: Not counting this as ARC space usage */
 
 	abd->abd_magic = ARC_BUF_DATA_MAGIC;
 	abd->abd_size = sabd->abd_size - off;
@@ -1011,6 +1013,7 @@ abd_get_from_buf(void *buf, size_t size)
 	abd_t *abd;
 
 	abd = kmem_cache_alloc(abd_struct_cache, KM_PUSHPAGE);
+	/* XXX: Not counting this as ARC space usage */
 
 	abd->abd_magic = ARC_BUF_DATA_MAGIC;
 	abd->abd_flags = ABD_F_LINEAR;
@@ -1118,6 +1121,8 @@ __abd_alloc_scatter(size_t size, int highmem)
 		sg_set_page(sg, page, (i == n-1 ? last_size : PAGESIZE), 0);
 	}
 
+	arc_space_consume(sizeof (abd_t), ARC_SPACE_ABD_HDRS);
+	arc_space_consume(abd->abd_nents * PAGESIZE, ARC_SPACE_ABD_DATA);
 	return (abd);
 }
 
@@ -1148,6 +1153,8 @@ abd_alloc_linear(size_t size)
 
 	abd->abd_buf = zio_buf_alloc(size);
 
+	arc_space_consume(sizeof (abd_t), ARC_SPACE_ABD_HDRS);
+
 	return (abd);
 }
 
@@ -1169,6 +1176,8 @@ abd_free_scatter(abd_t *abd, size_t size)
 	}
 
 	abd_sg_free_table(abd);
+	arc_space_return(sizeof (abd_t), ARC_SPACE_ABD_HDRS);
+	arc_space_return(n * PAGESIZE, ARC_SPACE_ABD_DATA);
 	kmem_cache_free(abd_struct_cache, abd);
 }
 
@@ -1178,8 +1187,10 @@ abd_free_linear(abd_t *abd, size_t size)
 	abd->abd_magic = 0;
 	zio_buf_free(abd->abd_buf, size);
 	kmem_cache_free(abd_struct_cache, abd);
+	arc_space_return(sizeof (abd_t), ARC_SPACE_ABD_HDRS);
 }
 
+
 /*
  * Free a ABD.
  * Only use this on ABD allocated with abd_alloc_{scatter,linear}.

module/zfs/arc.c

@@ -335,6 +335,8 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_meta_used;
 	kstat_named_t arcstat_meta_limit;
 	kstat_named_t arcstat_meta_max;
+	kstat_named_t arcstat_abd_data_size;
+	kstat_named_t arcstat_abd_hdr_size;
 } arc_stats_t;
 
 static arc_stats_t arc_stats = {
@@ -424,6 +426,8 @@ static arc_stats_t arc_stats = {
 	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
 	{ "arc_meta_limit",		KSTAT_DATA_UINT64 },
 	{ "arc_meta_max",		KSTAT_DATA_UINT64 },
+	{ "abd_data_size",		KSTAT_DATA_UINT64 },
+	{ "abd_hdr_size",		KSTAT_DATA_UINT64 },
 };
 
 #define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
@@ -1316,6 +1320,13 @@ arc_space_consume(uint64_t space, arc_space_type_t type)
 	case ARC_SPACE_L2HDRS:
 		ARCSTAT_INCR(arcstat_l2_hdr_size, space);
 		break;
+	case ARC_SPACE_ABD_DATA:
+		ARCSTAT_INCR(arcstat_abd_data_size, space);
+		/* Space is already considered data used, return now */
+		return;
+	case ARC_SPACE_ABD_HDRS:
+		ARCSTAT_INCR(arcstat_abd_hdr_size, space);
+		break;
 	}
 
 	if (type != ARC_SPACE_DATA) {
@@ -1350,6 +1361,13 @@ arc_space_return(uint64_t space, arc_space_type_t type)
 	case ARC_SPACE_L2HDRS:
 		ARCSTAT_INCR(arcstat_l2_hdr_size, -space);
 		break;
+	case ARC_SPACE_ABD_DATA:
+		ARCSTAT_INCR(arcstat_abd_data_size, -space);
+		/* Space is alredy considered used, return now */
+		return;
+	case ARC_SPACE_ABD_HDRS:
+		ARCSTAT_INCR(arcstat_abd_hdr_size, -space);
+		break;
 	}
 
 	if (type != ARC_SPACE_DATA) {