Enforce ARC meta-data limits

This change ensures the ARC meta-data limits are enforced.  Without
this enforcement meta-data can grow to consume all of the ARC cache
pushing out data and hurting performance.  The cache is aggressively
reclaimed but this is a soft and not a hard limit.  The cache may
exceed the set limit briefly before being brought under control.

By default 25% of the ARC capacity can be used for meta-data.  This
limit can be tuned by setting the 'zfs_arc_meta_limit' module option.
Once this limit is exceeded meta-data reclaim will occur in 3 percent
chunks, or may be tuned using 'arc_reduce_dnlc_percent'.

Closes #193

module/zfs/arc.c

@@ -180,6 +180,7 @@ unsigned long zfs_arc_meta_limit = 0;
 int zfs_arc_grow_retry = 0;
 int zfs_arc_shrink_shift = 0;
 int zfs_arc_p_min_shift = 0;
+int zfs_arc_reduce_dnlc_percent = 0;
 
 /*
  * Note that buffers can be in one of 6 states:
@@ -2084,14 +2085,16 @@ arc_kmem_reap_now(arc_reclaim_strategy_t strat)
 	kmem_cache_t		*prev_data_cache = NULL;
 	extern kmem_cache_t	*zio_buf_cache[];
 	extern kmem_cache_t	*zio_data_buf_cache[];
-
 #ifdef _KERNEL
-	if (arc_meta_used >= arc_meta_limit) {
+	int			retry = 0;
+
+	while ((arc_meta_used >= arc_meta_limit) && (retry < 10)) {
 		/*
 		 * We are exceeding our meta-data cache limit.
 		 * Purge some DNLC entries to release holds on meta-data.
 		 */
 		dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
+		retry++;
 	}
 #if defined(__i386)
 	/*
@@ -2157,6 +2160,10 @@ arc_reclaim_thread(void)
 			arc_no_grow = FALSE;
 		}
 
+		/* Keep meta data usage within limits */
+		if (arc_meta_used >= arc_meta_limit)
+			arc_kmem_reap_now(ARC_RECLAIM_CONS);
+
 		arc_adjust();
 
 		if (arc_eviction_list != NULL)
@@ -3583,6 +3590,9 @@ arc_init(void)
 	if (zfs_arc_p_min_shift > 0)
 		arc_p_min_shift = zfs_arc_p_min_shift;
 
+	if (zfs_arc_reduce_dnlc_percent > 0)
+		arc_reduce_dnlc_percent = zfs_arc_reduce_dnlc_percent;
+
 	/* if kmem_flags are set, lets try to use less memory */
 	if (kmem_debugging())
 		arc_c = arc_c / 2;
@@ -4765,4 +4775,7 @@ MODULE_PARM_DESC(zfs_arc_max, "Maximum arc size");
 
 module_param(zfs_arc_meta_limit, ulong, 0644);
 MODULE_PARM_DESC(zfs_arc_meta_limit, "Meta limit for arc size");
+
+module_param(arc_reduce_dnlc_percent, uint, 0644);
+MODULE_PARM_DESC(arc_reduce_dnlc_percent, "Meta reclaim percentage");
 #endif