lookasidelist cache for abd_chunk_cache

include/os/windows/spl/sys/lookasidelist.h

@@ -0,0 +1,21 @@
+#ifndef _SPL_LOOKASIDELIST_H
+#define	_SPL_LOOKASIDELIST_H
+
+extern void* osif_malloc(uint64_t);
+extern void osif_free(void*, uint64_t);
+
+#define	ZFS_LookAsideList_DRV_TAG '!SFZ'
+
+typedef struct lookasidelist_cache {
+    ULONG cache_active_allocations;
+    size_t cache_chunksize; // cache object size
+    LOOKASIDE_LIST_EX lookasideField;
+} lookasidelist_cache_t;
+
+lookasidelist_cache_t *lookasidelist_cache_create(size_t size);
+void lookasidelist_cache_destroy(lookasidelist_cache_t *pLookasidelist_cache);
+void* lookasidelist_cache_alloc(lookasidelist_cache_t *pLookasidelist_cache);
+void lookasidelist_cache_free(lookasidelist_cache_t *pLookasidelist_cache,
+    void *buf);
+
+#endif

module/os/windows/spl/CMakeLists.txt

@@ -37,6 +37,7 @@ wdk_add_library(splkern
   spl-vnode.c
   spl-windows.c
   spl-xdr.c
+  spl-lookasidelist.c
   ${TMH_FILE_LIST}
 )
 

module/os/windows/spl/spl-lookasidelist.c

@@ -0,0 +1,99 @@
+#include <sys/zfs_context.h>
+#include <sys/lookasidelist.h>
+
+void *
+allocate_func(
+    __in POOL_TYPE  PoolType,
+    __in SIZE_T  NumberOfBytes,
+    __in ULONG  Tag,
+    __inout PLOOKASIDE_LIST_EX  Lookaside)
+{
+	lookasidelist_cache_t *pLookasidelist_cache;
+	pLookasidelist_cache = CONTAINING_RECORD(Lookaside,
+	    lookasidelist_cache_t, lookasideField);
+	void *buf = osif_malloc(NumberOfBytes);
+	ASSERT(buf != NULL);
+
+	if (buf != NULL) {
+		atomic_inc_32(&pLookasidelist_cache->cache_active_allocations);
+		// =Lookaside->L.TotalAllocates - Lookaside->L.TotalFrees
+	}
+
+	return (buf);
+}
+
+void free_func(
+    __in PVOID  Buffer,
+    __inout PLOOKASIDE_LIST_EX  Lookaside
+) {
+	lookasidelist_cache_t *pLookasidelist_cache;
+	pLookasidelist_cache = CONTAINING_RECORD(Lookaside,
+	    lookasidelist_cache_t, lookasideField);
+	osif_free(Buffer, pLookasidelist_cache->cache_chunksize);
+	atomic_dec_32(&pLookasidelist_cache->cache_active_allocations);
+}
+
+lookasidelist_cache_t *
+lookasidelist_cache_create(size_t size)
+{
+	lookasidelist_cache_t *pLookasidelist_cache;
+	pLookasidelist_cache = ExAllocatePoolWithTag(NonPagedPool,
+	    sizeof (lookasidelist_cache_t), ZFS_LookAsideList_DRV_TAG);
+
+	if (pLookasidelist_cache != NULL) {
+		NTSTATUS retval = ExInitializeLookasideListEx(
+		    &pLookasidelist_cache->lookasideField,
+		    allocate_func,
+		    free_func,
+		    NonPagedPoolNx,
+		    0,
+		    size,
+		    ZFS_LookAsideList_DRV_TAG,
+		    0);
+
+		ASSERT(retval == STATUS_SUCCESS);
+
+		if (retval == STATUS_SUCCESS) {
+			pLookasidelist_cache->cache_chunksize = size;
+			pLookasidelist_cache->cache_active_allocations = 0;
+		}
+		else
+		{
+			ExFreePoolWithTag(pLookasidelist_cache,
+			    ZFS_LookAsideList_DRV_TAG);
+			pLookasidelist_cache = NULL;
+		}
+	}
+
+	return (pLookasidelist_cache);
+}
+
+void
+lookasidelist_cache_destroy(lookasidelist_cache_t *pLookasidelist_cache)
+{
+	if (pLookasidelist_cache != NULL) {
+		ExFlushLookasideListEx(&pLookasidelist_cache->lookasideField);
+		ExDeleteLookasideListEx(&pLookasidelist_cache->lookasideField);
+		ExFreePoolWithTag(pLookasidelist_cache,
+		    ZFS_LookAsideList_DRV_TAG);
+	}
+}
+
+void *
+lookasidelist_cache_alloc(lookasidelist_cache_t *pLookasidelist_cache)
+{
+	void *buf = ExAllocateFromLookasideListEx(
+	    &pLookasidelist_cache->lookasideField);
+	ASSERT(buf != NULL);
+	return (buf);
+}
+
+void
+lookasidelist_cache_free(lookasidelist_cache_t *pLookasidelist_cache, void *buf)
+{
+	ASSERT(buf != NULL);
+	if (buf != NULL) {
+		ExFreeToLookasideListEx(&pLookasidelist_cache->lookasideField,
+		    buf);
+	}
+}

module/os/windows/zfs/abd_os.c

@@ -32,6 +32,7 @@
 #include <sys/zio.h>
 #include <sys/zfs_context.h>
 #include <sys/zfs_znode.h>
+#include <sys/lookasidelist.h>
 
 
 typedef struct abd_stats {
@@ -89,7 +90,7 @@ struct {
  */
 size_t zfs_abd_chunk_size = 4096;
 
-kmem_cache_t *abd_chunk_cache;
+lookasidelist_cache_t *abd_chunk_cache;
 static kstat_t *abd_ksp;
 
 
@@ -105,7 +106,7 @@ static char *abd_zero_buf = NULL;
 static void
 abd_free_chunk(void *c)
 {
-	kmem_cache_free(abd_chunk_cache, c);
+	lookasidelist_cache_free(abd_chunk_cache, c);
 }
 
 static size_t
@@ -182,7 +183,7 @@ abd_alloc_chunks(abd_t *abd, size_t size)
 {
 	size_t n = abd_chunkcnt_for_bytes(size);
 	for (int i = 0; i < n; i++) {
-		void *c = kmem_cache_alloc(abd_chunk_cache, KM_SLEEP);
+		void* c = lookasidelist_cache_alloc(abd_chunk_cache);
 		ABD_SCATTER(abd).abd_chunks[i] = c;
 	}
 	ABD_SCATTER(abd).abd_chunk_size = zfs_abd_chunk_size;
@@ -236,7 +237,7 @@ static void
 abd_alloc_zero_scatter(void)
 {
 	size_t n = abd_chunkcnt_for_bytes(SPA_MAXBLOCKSIZE);
-	abd_zero_buf = kmem_cache_alloc(abd_chunk_cache, KM_SLEEP);
+	abd_zero_buf = lookasidelist_cache_alloc(abd_chunk_cache);
 	bzero(abd_zero_buf, zfs_abd_chunk_size);
 	abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE);
 
@@ -264,15 +265,13 @@ abd_free_zero_scatter(void)
 
 	abd_free_struct(abd_zero_scatter);
 	abd_zero_scatter = NULL;
-	kmem_cache_free(abd_chunk_cache, abd_zero_buf);
+	lookasidelist_cache_free(abd_chunk_cache, abd_zero_buf);
 }
 
 void
 abd_init(void)
 {
-	abd_chunk_cache = kmem_cache_create("abd_chunk", zfs_abd_chunk_size,
-	    MIN(PAGE_SIZE, 4096),
-	    NULL, NULL, NULL, NULL, abd_arena, KMC_NOTOUCH);
+	abd_chunk_cache = lookasidelist_cache_create(zfs_abd_chunk_size);
 
 	abd_ksp = kstat_create("zfs", 0, "abdstats", "misc", KSTAT_TYPE_NAMED,
 	    sizeof (abd_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
@@ -294,7 +293,7 @@ abd_fini(void)
 		abd_ksp = NULL;
 	}
 
-	kmem_cache_destroy(abd_chunk_cache);
+	lookasidelist_cache_destroy(abd_chunk_cache);
 	abd_chunk_cache = NULL;
 }
 
@@ -487,5 +486,5 @@ abd_iter_unmap(struct abd_iter *aiter)
 void
 abd_cache_reap_now(void)
 {
-	kmem_cache_reap_now(abd_chunk_cache);
+	// do nothing
 }

module/os/windows/zfs/arc_os.c

@@ -253,8 +253,7 @@ arc_reclaim_thread(void *unused)
 		 * it is worth reaping the abd_chunk_cache
 		 */
 		if (d_adj >= 64LL*1024LL*1024LL) {
-			extern kmem_cache_t *abd_chunk_cache;
-			kmem_cache_reap_now(abd_chunk_cache);
+			abd_cache_reap_now();
 		}
 
 		free_memory = post_adjust_free_memory;