From d0489f5c0f167020a07a8a637d9870a26376085e Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Wed, 16 Dec 2015 17:11:02 +0100 Subject: [PATCH 1/7] kasan: Change the behavior of kmalloc_large_oob_right test depending on which allocator (SLAB or SLUB) is being used Signed-off-by: Alexander Potapenko --- lib/test_kasan.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/test_kasan.c b/lib/test_kasan.c index c32f3b0048dc82..eb5a9d0d73b2fc 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -68,7 +68,21 @@ static noinline void __init kmalloc_node_oob_right(void) static noinline void __init kmalloc_large_oob_right(void) { char *ptr; - size_t size = KMALLOC_MAX_CACHE_SIZE + 10; + size_t size; + if (KMALLOC_MAX_CACHE_SIZE == KMALLOC_MAX_SIZE) { + /* + * We're using the SLAB allocator. Allocate a chunk that fits + * into a slab. + */ + size = KMALLOC_MAX_CACHE_SIZE - 256; + } else { + /* + * KMALLOC_MAX_SIZE > KMALLOC_MAX_CACHE_SIZE. + * We're using the SLUB allocator. Allocate a chunk that does + * not fit into a slab to trigger the page allocator. + */ + size = KMALLOC_MAX_CACHE_SIZE + 10; + } pr_info("kmalloc large allocation: out-of-bounds to right\n"); ptr = kmalloc(size, GFP_KERNEL); From c0fa4d6bcad93fdf6a2a1acce82de931d82f363c Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Wed, 16 Dec 2015 17:18:29 +0100 Subject: [PATCH 2/7] mm, kasan: SLAB support. This patch adds KASAN hooks to SLAB allocator. This patch is based on the "mm: kasan: unified support for SLUB and SLAB allocators" patch originally prepared by Dmitry Chernenkov. Signed-off-by: Dmitry Chernenkov Signed-off-by: Alexander Potapenko --- Documentation/kasan.txt | 5 +-- include/linux/kasan.h | 14 +++++++ include/linux/slab.h | 6 +++ include/linux/slab_def.h | 13 ++++++ include/linux/slub_def.h | 11 +++++ lib/Kconfig.kasan | 4 +- mm/Makefile | 1 + mm/kasan/kasan.c | 91 ++++++++++++++++++++++++++++++++++++++++ mm/kasan/kasan.h | 34 +++++++++++++++ mm/kasan/report.c | 59 +++++++++++++++++++++----- mm/slab.c | 45 +++++++++++++++++--- mm/slab_common.c | 2 +- 12 files changed, 264 insertions(+), 21 deletions(-) diff --git a/Documentation/kasan.txt b/Documentation/kasan.txt index aa1e0c91e36888..7dd95b35cd7cd2 100644 --- a/Documentation/kasan.txt +++ b/Documentation/kasan.txt @@ -12,8 +12,7 @@ KASAN uses compile-time instrumentation for checking every memory access, therefore you will need a GCC version 4.9.2 or later. GCC 5.0 or later is required for detection of out-of-bounds accesses to stack or global variables. -Currently KASAN is supported only for x86_64 architecture and requires the -kernel to be built with the SLUB allocator. +Currently KASAN is supported only for x86_64 architecture. 1. Usage ======== @@ -27,7 +26,7 @@ inline are compiler instrumentation types. The former produces smaller binary the latter is 1.1 - 2 times faster. Inline instrumentation requires a GCC version 5.0 or later. -Currently KASAN works only with the SLUB memory allocator. +KASAN works with both SLUB and SLAB memory allocators. For better bug detection and nicer reporting, enable CONFIG_STACKTRACE. To disable instrumentation for specific files or directories, add a line diff --git a/include/linux/kasan.h b/include/linux/kasan.h index 4b9f85c963d073..cba31cada2329c 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -7,6 +7,8 @@ struct kmem_cache; struct page; struct vm_struct; +typedef size_t cache_size_t; + #ifdef CONFIG_KASAN #define KASAN_SHADOW_SCALE_SHIFT 3 @@ -46,6 +48,9 @@ void kasan_unpoison_shadow(const void *address, size_t size); void kasan_alloc_pages(struct page *page, unsigned int order); void kasan_free_pages(struct page *page, unsigned int order); +void kasan_cache_create(struct kmem_cache *cache, cache_size_t *size, + unsigned long *flags); + void kasan_poison_slab(struct page *page); void kasan_unpoison_object_data(struct kmem_cache *cache, void *object); void kasan_poison_object_data(struct kmem_cache *cache, void *object); @@ -59,6 +64,11 @@ void kasan_krealloc(const void *object, size_t new_size); void kasan_slab_alloc(struct kmem_cache *s, void *object); void kasan_slab_free(struct kmem_cache *s, void *object); +struct kasan_cache { + int alloc_meta_offset; + int free_meta_offset; +}; + int kasan_module_alloc(void *addr, size_t size); void kasan_free_shadow(const struct vm_struct *vm); @@ -72,6 +82,10 @@ static inline void kasan_disable_current(void) {} static inline void kasan_alloc_pages(struct page *page, unsigned int order) {} static inline void kasan_free_pages(struct page *page, unsigned int order) {} +static inline void kasan_cache_create(struct kmem_cache *cache, + cache_size_t *size, + unsigned long *flags) {} + static inline void kasan_poison_slab(struct page *page) {} static inline void kasan_unpoison_object_data(struct kmem_cache *cache, void *object) {} diff --git a/include/linux/slab.h b/include/linux/slab.h index 2037a861e36799..973855db33570d 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -87,6 +87,12 @@ # define SLAB_FAILSLAB 0x00000000UL #endif +#ifdef CONFIG_KASAN +#define SLAB_KASAN 0x08000000UL +#else +#define SLAB_KASAN 0x00000000UL +#endif + /* The following flags affect the page allocator grouping pages by mobility */ #define SLAB_RECLAIM_ACCOUNT 0x00020000UL /* Objects are reclaimable */ #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 33d049066c3d43..b971d6bee68a2e 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -72,8 +72,21 @@ struct kmem_cache { #ifdef CONFIG_MEMCG_KMEM struct memcg_cache_params memcg_params; #endif +#ifdef CONFIG_KASAN + struct kasan_cache kasan_info; +#endif struct kmem_cache_node *node[MAX_NUMNODES]; }; +static inline void *nearest_obj(struct kmem_cache *cache, struct page *page, + void *x) { + void *object = x - (x - page->s_mem) % cache->size; + void *last_object = page->s_mem + (cache->num - 1) * cache->size; + if (unlikely(object > last_object)) + return last_object; + else + return object; +} + #endif /* _LINUX_SLAB_DEF_H */ diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 33885118523c7c..c553dad412d2aa 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -129,4 +129,15 @@ static inline void *virt_to_obj(struct kmem_cache *s, void object_err(struct kmem_cache *s, struct page *page, u8 *object, char *reason); +static inline void *nearest_obj(struct kmem_cache *cache, struct page *page, + void *x) { + void *object = x - (x - page_address(page)) % cache->size; + void *last_object = page_address(page) + + (page->objects - 1) * cache->size; + if (unlikely(object > last_object)) + return last_object; + else + return object; +} + #endif /* _LINUX_SLUB_DEF_H */ diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index 0fee5acd5aa09e..0e4d2b3b0aee18 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -5,7 +5,7 @@ if HAVE_ARCH_KASAN config KASAN bool "KASan: runtime memory debugger" - depends on SLUB_DEBUG + depends on SLUB_DEBUG || (SLAB && !DEBUG_SLAB) select CONSTRUCTORS help Enables kernel address sanitizer - runtime memory debugger, @@ -16,6 +16,8 @@ config KASAN This feature consumes about 1/8 of available memory and brings about ~x3 performance slowdown. For better error detection enable CONFIG_STACKTRACE. + Currently CONFIG_KASAN doesn't work with CONFIG_DEBUG_SLAB + (the resulting kernel does not boot). choice prompt "Instrumentation type" diff --git a/mm/Makefile b/mm/Makefile index 2ed43191fc3bf7..d675b372edcd80 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -3,6 +3,7 @@ # KASAN_SANITIZE_slab_common.o := n +KASAN_SANITIZE_slab.o := n KASAN_SANITIZE_slub.o := n mmu-y := nommu.o diff --git a/mm/kasan/kasan.c b/mm/kasan/kasan.c index bc0a8d8b8f42fa..f924f8f1ebc564 100644 --- a/mm/kasan/kasan.c +++ b/mm/kasan/kasan.c @@ -314,6 +314,59 @@ void kasan_free_pages(struct page *page, unsigned int order) KASAN_FREE_PAGE); } +#ifdef CONFIG_SLAB +/* + * Adaptive redzone policy taken from the userspace AddressSanitizer runtime. + * For larger allocations larger redzones are used. + */ +static size_t optimal_redzone(size_t object_size) +{ + int rz = + object_size <= 64 - 16 ? 16 : + object_size <= 128 - 32 ? 32 : + object_size <= 512 - 64 ? 64 : + object_size <= 4096 - 128 ? 128 : + object_size <= (1 << 14) - 256 ? 256 : + object_size <= (1 << 15) - 512 ? 512 : + object_size <= (1 << 16) - 1024 ? 1024 : 2048; + return rz; +} + +void kasan_cache_create(struct kmem_cache *cache, cache_size_t *size, + unsigned long *flags) +{ + int redzone_adjust; + /* Make sure the adjusted size is still less than + * KMALLOC_MAX_CACHE_SIZE. + * TODO: this check is only useful for SLAB, but not SLUB. We'll need + * to skip it for SLUB when it starts using kasan_cache_create(). + */ + if (*size > KMALLOC_MAX_CACHE_SIZE - + sizeof(struct kasan_alloc_meta) - + sizeof(struct kasan_free_meta)) + return; + *flags |= SLAB_KASAN; + /* Add alloc meta. */ + cache->kasan_info.alloc_meta_offset = *size; + *size += sizeof(struct kasan_alloc_meta); + + /* Add free meta. */ + if (cache->flags & SLAB_DESTROY_BY_RCU || cache->ctor || + cache->object_size < sizeof(struct kasan_free_meta)) { + cache->kasan_info.free_meta_offset = *size; + *size += sizeof(struct kasan_free_meta); + } + redzone_adjust = optimal_redzone(cache->object_size) - + (*size - cache->object_size); + if (redzone_adjust > 0) + *size += redzone_adjust; + *size = min(KMALLOC_MAX_CACHE_SIZE, + max(*size, + cache->object_size + + optimal_redzone(cache->object_size))); +} +#endif + void kasan_poison_slab(struct page *page) { kasan_poison_shadow(page_address(page), @@ -331,8 +384,36 @@ void kasan_poison_object_data(struct kmem_cache *cache, void *object) kasan_poison_shadow(object, round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE), KASAN_KMALLOC_REDZONE); +#ifdef CONFIG_SLAB + if (cache->flags & SLAB_KASAN) { + struct kasan_alloc_meta *alloc_info = + get_alloc_info(cache, object); + alloc_info->state = KASAN_STATE_INIT; + } +#endif } +static inline void set_track(struct kasan_track *track) +{ + track->cpu = raw_smp_processor_id(); + track->pid = current->pid; + track->when = jiffies; +} + +#ifdef CONFIG_SLAB +struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache, + const void *object) +{ + return (void *)object + cache->kasan_info.alloc_meta_offset; +} + +struct kasan_free_meta *get_free_info(struct kmem_cache *cache, + const void *object) +{ + return (void *)object + cache->kasan_info.free_meta_offset; +} +#endif + void kasan_slab_alloc(struct kmem_cache *cache, void *object) { kasan_kmalloc(cache, object, cache->object_size); @@ -366,6 +447,16 @@ void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size) kasan_unpoison_shadow(object, size); kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start, KASAN_KMALLOC_REDZONE); +#ifdef CONFIG_SLAB + if (cache->flags & SLAB_KASAN) { + struct kasan_alloc_meta *alloc_info = + get_alloc_info(cache, object); + + alloc_info->state = KASAN_STATE_ALLOC; + alloc_info->alloc_size = size; + set_track(&alloc_info->track); + } +#endif } EXPORT_SYMBOL(kasan_kmalloc); diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index 4f6c62e5c21edd..7b9e4ab9b66b43 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -54,6 +54,40 @@ struct kasan_global { #endif }; +/** + * Structures to keep alloc and free tracks * + */ + +enum kasan_state { + KASAN_STATE_INIT, + KASAN_STATE_ALLOC, + KASAN_STATE_FREE +}; + +struct kasan_track { + u64 cpu : 6; /* for NR_CPUS = 64 */ + u64 pid : 16; /* 65536 processes */ + u64 when : 42; /* ~140 years */ +}; + +struct kasan_alloc_meta { + u32 state : 2; /* enum kasan_state */ + u32 alloc_size : 30; + struct kasan_track track; +}; + +struct kasan_free_meta { + /* Allocator freelist pointer, unused by KASAN. */ + void **freelist; + struct kasan_track track; +}; + +struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache, + const void *object); +struct kasan_free_meta *get_free_info(struct kmem_cache *cache, + const void *object); + + static inline const void *kasan_shadow_to_mem(const void *shadow_addr) { return (void *)(((unsigned long)shadow_addr - KASAN_SHADOW_OFFSET) diff --git a/mm/kasan/report.c b/mm/kasan/report.c index 12f222d0224b93..2bf7218fef12e3 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -115,6 +115,42 @@ static inline bool init_task_stack_addr(const void *addr) sizeof(init_thread_union.stack)); } +static void print_track(struct kasan_track *track) +{ + pr_err("PID = %lu, CPU = %lu, timestamp = %lu\n", track->pid, + track->cpu, track->when); +} + +static void print_object(struct kmem_cache *cache, void *object) +{ + struct kasan_alloc_meta *alloc_info = get_alloc_info(cache, object); + struct kasan_free_meta *free_info; + + pr_err("Object at %p, in cache %s\n", object, cache->name); + if (!(cache->flags & SLAB_KASAN)) + return; + switch (alloc_info->state) { + case KASAN_STATE_INIT: + pr_err("Object not allocated yet\n"); + break; + case KASAN_STATE_ALLOC: + pr_err("Object allocated with size %u bytes.\n", + alloc_info->alloc_size); + pr_err("Allocation:\n"); + print_track(&alloc_info->track); + break; + case KASAN_STATE_FREE: + pr_err("Object freed, allocated with size %u bytes\n", + alloc_info->alloc_size); + free_info = get_free_info(cache, object); + pr_err("Allocation:\n"); + print_track(&alloc_info->track); + pr_err("Deallocation:\n"); + print_track(&free_info->track); + break; + } +} + static void print_address_description(struct kasan_access_info *info) { const void *addr = info->access_addr; @@ -126,17 +162,14 @@ static void print_address_description(struct kasan_access_info *info) if (PageSlab(page)) { void *object; struct kmem_cache *cache = page->slab_cache; - void *last_object; - - object = virt_to_obj(cache, page_address(page), addr); - last_object = page_address(page) + - page->objects * cache->size; - - if (unlikely(object > last_object)) - object = last_object; /* we hit into padding */ - + object = nearest_obj(cache, page, + (void *)info->access_addr); +#ifdef CONFIG_SLAB + print_object(cache, object); +#else object_err(cache, page, object, - "kasan: bad access detected"); + "kasan: bad access detected"); +#endif return; } dump_page(page, "kasan: bad access detected"); @@ -146,8 +179,9 @@ static void print_address_description(struct kasan_access_info *info) if (!init_task_stack_addr(addr)) pr_err("Address belongs to variable %pS\n", addr); } - +#ifdef CONFIG_SLUB dump_stack(); +#endif } static bool row_is_guilty(const void *row, const void *guilty) @@ -233,6 +267,9 @@ static void kasan_report_error(struct kasan_access_info *info) dump_stack(); } else { print_error_description(info); +#ifdef CONFIG_SLAB + dump_stack(); +#endif print_address_description(info); print_shadow_for_address(info->first_bad_addr); } diff --git a/mm/slab.c b/mm/slab.c index 4765c97ce6900d..7741c48805930a 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -2196,6 +2196,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) #endif #endif + kasan_cache_create(cachep, &size, &flags); /* * Determine if the slab management is 'on' or 'off' slab. * (bootstrapping cannot cope with offslab caches so don't do @@ -2503,8 +2504,13 @@ static void cache_init_objs(struct kmem_cache *cachep, * cache which they are a constructor for. Otherwise, deadlock. * They must also be threaded. */ - if (cachep->ctor && !(cachep->flags & SLAB_POISON)) + if (cachep->ctor && !(cachep->flags & SLAB_POISON)) { + kasan_unpoison_object_data(cachep, + objp + obj_offset(cachep)); cachep->ctor(objp + obj_offset(cachep)); + kasan_poison_object_data( + cachep, objp + obj_offset(cachep)); + } if (cachep->flags & SLAB_RED_ZONE) { if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) @@ -2519,8 +2525,11 @@ static void cache_init_objs(struct kmem_cache *cachep, kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, 0); #else - if (cachep->ctor) + if (cachep->ctor) { + kasan_unpoison_object_data(cachep, objp); cachep->ctor(objp); + kasan_poison_object_data(cachep, objp); + } #endif set_obj_status(page, i, OBJECT_FREE); set_free_obj(page, i, i); @@ -2650,6 +2659,7 @@ static int cache_grow(struct kmem_cache *cachep, slab_map_pages(cachep, page, freelist); + kasan_poison_slab(page); cache_init_objs(cachep, page); if (gfpflags_allow_blocking(local_flags)) @@ -3366,7 +3376,10 @@ static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) static inline void __cache_free(struct kmem_cache *cachep, void *objp, unsigned long caller) { - struct array_cache *ac = cpu_cache_get(cachep); + struct array_cache *ac; + + kasan_slab_free(cachep, objp); + ac = cpu_cache_get(cachep); check_irq_off(); kmemleak_free_recursive(objp, cachep->flags); @@ -3405,6 +3418,8 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp, void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) { void *ret = slab_alloc(cachep, flags, _RET_IP_); + if (ret) + kasan_slab_alloc(cachep, ret); trace_kmem_cache_alloc(_RET_IP_, ret, cachep->object_size, cachep->size, flags); @@ -3434,6 +3449,8 @@ kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) ret = slab_alloc(cachep, flags, _RET_IP_); + if (ret) + kasan_kmalloc(cachep, ret, size); trace_kmalloc(_RET_IP_, ret, size, cachep->size, flags); return ret; @@ -3457,6 +3474,8 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) { void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); + if (ret) + kasan_slab_alloc(cachep, ret); trace_kmem_cache_alloc_node(_RET_IP_, ret, cachep->object_size, cachep->size, flags, nodeid); @@ -3475,6 +3494,8 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); + if (ret) + kasan_kmalloc(cachep, ret, size); trace_kmalloc_node(_RET_IP_, ret, size, cachep->size, flags, nodeid); @@ -3487,11 +3508,16 @@ static __always_inline void * __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) { struct kmem_cache *cachep; + void *ret; cachep = kmalloc_slab(size, flags); if (unlikely(ZERO_OR_NULL_PTR(cachep))) return cachep; - return kmem_cache_alloc_node_trace(cachep, flags, node, size); + ret = kmem_cache_alloc_node_trace(cachep, flags, node, size); + if (ret) + kasan_kmalloc(cachep, ret, size); + + return ret; } void *__kmalloc_node(size_t size, gfp_t flags, int node) @@ -3525,6 +3551,8 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, return cachep; ret = slab_alloc(cachep, flags, caller); + if (ret) + kasan_kmalloc(cachep, ret, size); trace_kmalloc(caller, ret, size, cachep->size, flags); @@ -4242,10 +4270,17 @@ module_init(slab_proc_init); */ size_t ksize(const void *objp) { + size_t size; + BUG_ON(!objp); if (unlikely(objp == ZERO_SIZE_PTR)) return 0; - return virt_to_cache(objp)->object_size; + size = virt_to_cache(objp)->object_size; + /* We assume that ksize callers could use whole allocated area, + so we need to unpoison this area. */ + kasan_krealloc(objp, size); + + return size; } EXPORT_SYMBOL(ksize); diff --git a/mm/slab_common.c b/mm/slab_common.c index 3c6a86b4ec25f8..ae7cf03671a270 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -35,7 +35,7 @@ struct kmem_cache *kmem_cache; */ #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE | \ - SLAB_FAILSLAB) + SLAB_FAILSLAB | SLAB_KASAN) #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | SLAB_NOTRACK) From 8c113b90f36b33b99a92c7726122e3040d5211c0 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Thu, 22 Oct 2015 20:15:45 +0200 Subject: [PATCH 3/7] mm, kasan: Added GFP flags to KASAN API Add GFP flags to KASAN hooks for future patches to use. This patch is based on the "mm: kasan: unified support for SLUB and SLAB allocators" patch originally prepared by Dmitry Chernenkov. Signed-off-by: Dmitry Chernenkov Signed-off-by: Alexander Potapenko --- include/linux/kasan.h | 19 +++++++++++-------- mm/kasan/kasan.c | 15 ++++++++------- mm/mempool.c | 16 ++++++++-------- mm/slab.c | 14 +++++++------- mm/slab_common.c | 4 ++-- mm/slub.c | 17 +++++++++-------- 6 files changed, 45 insertions(+), 40 deletions(-) diff --git a/include/linux/kasan.h b/include/linux/kasan.h index cba31cada2329c..b75a605b4ef7ae 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -55,13 +55,14 @@ void kasan_poison_slab(struct page *page); void kasan_unpoison_object_data(struct kmem_cache *cache, void *object); void kasan_poison_object_data(struct kmem_cache *cache, void *object); -void kasan_kmalloc_large(const void *ptr, size_t size); +void kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags); void kasan_kfree_large(const void *ptr); void kasan_kfree(void *ptr); -void kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size); -void kasan_krealloc(const void *object, size_t new_size); +void kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size, + gfp_t flags); +void kasan_krealloc(const void *object, size_t new_size, gfp_t flags); -void kasan_slab_alloc(struct kmem_cache *s, void *object); +void kasan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags); void kasan_slab_free(struct kmem_cache *s, void *object); struct kasan_cache { @@ -92,14 +93,16 @@ static inline void kasan_unpoison_object_data(struct kmem_cache *cache, static inline void kasan_poison_object_data(struct kmem_cache *cache, void *object) {} -static inline void kasan_kmalloc_large(void *ptr, size_t size) {} +static inline void kasan_kmalloc_large(void *ptr, size_t size, gfp_t flags) {} static inline void kasan_kfree_large(const void *ptr) {} static inline void kasan_kfree(void *ptr) {} static inline void kasan_kmalloc(struct kmem_cache *s, const void *object, - size_t size) {} -static inline void kasan_krealloc(const void *object, size_t new_size) {} + size_t size, gfp_t flags) {} +static inline void kasan_krealloc(const void *object, size_t new_size, + gfp_t flags) {} -static inline void kasan_slab_alloc(struct kmem_cache *s, void *object) {} +static inline void kasan_slab_alloc(struct kmem_cache *s, void *object, + gfp_t flags) {} static inline void kasan_slab_free(struct kmem_cache *s, void *object) {} static inline int kasan_module_alloc(void *addr, size_t size) { return 0; } diff --git a/mm/kasan/kasan.c b/mm/kasan/kasan.c index f924f8f1ebc564..b030e19d9e1ebf 100644 --- a/mm/kasan/kasan.c +++ b/mm/kasan/kasan.c @@ -414,9 +414,9 @@ struct kasan_free_meta *get_free_info(struct kmem_cache *cache, } #endif -void kasan_slab_alloc(struct kmem_cache *cache, void *object) +void kasan_slab_alloc(struct kmem_cache *cache, void *object, gfp_t flags) { - kasan_kmalloc(cache, object, cache->object_size); + kasan_kmalloc(cache, object, cache->object_size, flags); } void kasan_slab_free(struct kmem_cache *cache, void *object) @@ -431,7 +431,8 @@ void kasan_slab_free(struct kmem_cache *cache, void *object) kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE); } -void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size) +void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size, + gfp_t flags) { unsigned long redzone_start; unsigned long redzone_end; @@ -460,7 +461,7 @@ void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size) } EXPORT_SYMBOL(kasan_kmalloc); -void kasan_kmalloc_large(const void *ptr, size_t size) +void kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags) { struct page *page; unsigned long redzone_start; @@ -479,7 +480,7 @@ void kasan_kmalloc_large(const void *ptr, size_t size) KASAN_PAGE_REDZONE); } -void kasan_krealloc(const void *object, size_t size) +void kasan_krealloc(const void *object, size_t size, gfp_t flags) { struct page *page; @@ -489,9 +490,9 @@ void kasan_krealloc(const void *object, size_t size) page = virt_to_head_page(object); if (unlikely(!PageSlab(page))) - kasan_kmalloc_large(object, size); + kasan_kmalloc_large(object, size, flags); else - kasan_kmalloc(page->slab_cache, object, size); + kasan_kmalloc(page->slab_cache, object, size, flags); } void kasan_kfree(void *ptr) diff --git a/mm/mempool.c b/mm/mempool.c index 004d42b1dfaf92..b47c8a75d7c77a 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -112,12 +112,12 @@ static void kasan_poison_element(mempool_t *pool, void *element) kasan_free_pages(element, (unsigned long)pool->pool_data); } -static void kasan_unpoison_element(mempool_t *pool, void *element) +static void kasan_unpoison_element(mempool_t *pool, void *element, gfp_t flags) { if (pool->alloc == mempool_alloc_slab) - kasan_slab_alloc(pool->pool_data, element); + kasan_slab_alloc(pool->pool_data, element, flags); if (pool->alloc == mempool_kmalloc) - kasan_krealloc(element, (size_t)pool->pool_data); + kasan_krealloc(element, (size_t)pool->pool_data, flags); if (pool->alloc == mempool_alloc_pages) kasan_alloc_pages(element, (unsigned long)pool->pool_data); } @@ -130,13 +130,13 @@ static void add_element(mempool_t *pool, void *element) pool->elements[pool->curr_nr++] = element; } -static void *remove_element(mempool_t *pool) +static void *remove_element(mempool_t *pool, gfp_t flags) { void *element = pool->elements[--pool->curr_nr]; BUG_ON(pool->curr_nr < 0); check_element(pool, element); - kasan_unpoison_element(pool, element); + kasan_unpoison_element(pool, element, flags); return element; } @@ -154,7 +154,7 @@ void mempool_destroy(mempool_t *pool) return; while (pool->curr_nr) { - void *element = remove_element(pool); + void *element = remove_element(pool, GFP_KERNEL); pool->free(element, pool->pool_data); } kfree(pool->elements); @@ -250,7 +250,7 @@ int mempool_resize(mempool_t *pool, int new_min_nr) spin_lock_irqsave(&pool->lock, flags); if (new_min_nr <= pool->min_nr) { while (new_min_nr < pool->curr_nr) { - element = remove_element(pool); + element = remove_element(pool, GFP_KERNEL); spin_unlock_irqrestore(&pool->lock, flags); pool->free(element, pool->pool_data); spin_lock_irqsave(&pool->lock, flags); @@ -336,7 +336,7 @@ void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask) spin_lock_irqsave(&pool->lock, flags); if (likely(pool->curr_nr)) { - element = remove_element(pool); + element = remove_element(pool, gfp_temp); spin_unlock_irqrestore(&pool->lock, flags); /* paired with rmb in mempool_free(), read comment there */ smp_wmb(); diff --git a/mm/slab.c b/mm/slab.c index 7741c48805930a..743e920da5f61f 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3419,7 +3419,7 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) { void *ret = slab_alloc(cachep, flags, _RET_IP_); if (ret) - kasan_slab_alloc(cachep, ret); + kasan_slab_alloc(cachep, ret, flags); trace_kmem_cache_alloc(_RET_IP_, ret, cachep->object_size, cachep->size, flags); @@ -3450,7 +3450,7 @@ kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) ret = slab_alloc(cachep, flags, _RET_IP_); if (ret) - kasan_kmalloc(cachep, ret, size); + kasan_kmalloc(cachep, ret, size, flags); trace_kmalloc(_RET_IP_, ret, size, cachep->size, flags); return ret; @@ -3475,7 +3475,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); if (ret) - kasan_slab_alloc(cachep, ret); + kasan_slab_alloc(cachep, ret, flags); trace_kmem_cache_alloc_node(_RET_IP_, ret, cachep->object_size, cachep->size, flags, nodeid); @@ -3495,7 +3495,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); if (ret) - kasan_kmalloc(cachep, ret, size); + kasan_kmalloc(cachep, ret, size, flags); trace_kmalloc_node(_RET_IP_, ret, size, cachep->size, flags, nodeid); @@ -3515,7 +3515,7 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) return cachep; ret = kmem_cache_alloc_node_trace(cachep, flags, node, size); if (ret) - kasan_kmalloc(cachep, ret, size); + kasan_kmalloc(cachep, ret, size, flags); return ret; } @@ -3552,7 +3552,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, ret = slab_alloc(cachep, flags, caller); if (ret) - kasan_kmalloc(cachep, ret, size); + kasan_kmalloc(cachep, ret, size, flags); trace_kmalloc(caller, ret, size, cachep->size, flags); @@ -4279,7 +4279,7 @@ size_t ksize(const void *objp) size = virt_to_cache(objp)->object_size; /* We assume that ksize callers could use whole allocated area, so we need to unpoison this area. */ - kasan_krealloc(objp, size); + kasan_krealloc(objp, size, GFP_NOWAIT); return size; } diff --git a/mm/slab_common.c b/mm/slab_common.c index ae7cf03671a270..3f645da7a22fcf 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1007,7 +1007,7 @@ void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) page = alloc_kmem_pages(flags, order); ret = page ? page_address(page) : NULL; kmemleak_alloc(ret, size, 1, flags); - kasan_kmalloc_large(ret, size); + kasan_kmalloc_large(ret, size, flags); return ret; } EXPORT_SYMBOL(kmalloc_order); @@ -1188,7 +1188,7 @@ static __always_inline void *__do_krealloc(const void *p, size_t new_size, ks = ksize(p); if (ks >= new_size) { - kasan_krealloc((void *)p, new_size); + kasan_krealloc((void *)p, new_size, flags); return (void *)p; } diff --git a/mm/slub.c b/mm/slub.c index 46997517406ede..8b00acc157eaa8 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1270,7 +1270,7 @@ static inline void dec_slabs_node(struct kmem_cache *s, int node, static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) { kmemleak_alloc(ptr, size, 1, flags); - kasan_kmalloc_large(ptr, size); + kasan_kmalloc_large(ptr, size, flags); } static inline void kfree_hook(const void *x) @@ -1304,7 +1304,7 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, kmemcheck_slab_alloc(s, flags, object, slab_ksize(s)); kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags); - kasan_slab_alloc(s, object); + kasan_slab_alloc(s, object, flags); } memcg_kmem_put_cache(s); } @@ -2588,7 +2588,7 @@ void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) { void *ret = slab_alloc(s, gfpflags, _RET_IP_); trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags); - kasan_kmalloc(s, ret, size); + kasan_kmalloc(s, ret, size, gfpflags); return ret; } EXPORT_SYMBOL(kmem_cache_alloc_trace); @@ -2616,7 +2616,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *s, trace_kmalloc_node(_RET_IP_, ret, size, s->size, gfpflags, node); - kasan_kmalloc(s, ret, size); + kasan_kmalloc(s, ret, size, gfpflags); return ret; } EXPORT_SYMBOL(kmem_cache_alloc_node_trace); @@ -3160,7 +3160,8 @@ static void early_kmem_cache_node_alloc(int node) init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); init_tracking(kmem_cache_node, n); #endif - kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node)); + kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node), + GFP_KERNEL); init_kmem_cache_node(n); inc_slabs_node(kmem_cache_node, node, page->objects); @@ -3533,7 +3534,7 @@ void *__kmalloc(size_t size, gfp_t flags) trace_kmalloc(_RET_IP_, ret, size, s->size, flags); - kasan_kmalloc(s, ret, size); + kasan_kmalloc(s, ret, size, flags); return ret; } @@ -3578,7 +3579,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node) trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node); - kasan_kmalloc(s, ret, size); + kasan_kmalloc(s, ret, size, flags); return ret; } @@ -3607,7 +3608,7 @@ size_t ksize(const void *object) size_t size = __ksize(object); /* We assume that ksize callers could use whole allocated area, so we need unpoison this area. */ - kasan_krealloc(object, size); + kasan_krealloc(object, size, GFP_NOWAIT); return size; } EXPORT_SYMBOL(ksize); From 7043b13800e8a787c3c770a3bec82c10bef64b53 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Fri, 20 Nov 2015 19:07:34 +0100 Subject: [PATCH 4/7] mm, kasan: Initial stackdepot implementation. Stack depot will allow KASAN store allocation/deallocation stack traces for memory chunks. The stack traces are stored in a hash table and referenced by handles which reside in the kasan_alloc_meta/kasan_free_meta structures in the allocated memory chunks. Right now stackdepot support is only enabled in SLAB allocator. Once KASAN features in SLAB are on par with those in SLUB we can switch SLUB to stackdepot as well, thus removing the dependency on SLUB_DEBUG. This patch is based on the "mm: kasan: stack depots" patch originally prepared by Dmitry Chernenkov. Signed-off-by: Dmitry Chernenkov Signed-off-by: Alexander Potapenko --- arch/x86/kernel/Makefile | 1 + mm/kasan/Makefile | 3 + mm/kasan/kasan.c | 54 ++++++++- mm/kasan/kasan.h | 11 ++ mm/kasan/report.c | 8 ++ mm/kasan/stackdepot.c | 254 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 mm/kasan/stackdepot.c diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index b1b78ffe01d060..500584d729d9c8 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -19,6 +19,7 @@ endif KASAN_SANITIZE_head$(BITS).o := n KASAN_SANITIZE_dumpstack.o := n KASAN_SANITIZE_dumpstack_$(BITS).o := n +KASAN_SANITIZE_stacktrace.o := n CFLAGS_irq.o := -I$(src)/../include/asm/trace diff --git a/mm/kasan/Makefile b/mm/kasan/Makefile index 64710148941ed3..f952515f1ed22a 100644 --- a/mm/kasan/Makefile +++ b/mm/kasan/Makefile @@ -6,3 +6,6 @@ CFLAGS_REMOVE_kasan.o = -pg CFLAGS_kasan.o := $(call cc-option, -fno-conserve-stack -fno-stack-protector) obj-y := kasan.o report.o kasan_init.o +ifdef CONFIG_SLAB + obj-y += stackdepot.o +endif diff --git a/mm/kasan/kasan.c b/mm/kasan/kasan.c index b030e19d9e1ebf..e51ca077da2ef5 100644 --- a/mm/kasan/kasan.c +++ b/mm/kasan/kasan.c @@ -393,23 +393,73 @@ void kasan_poison_object_data(struct kmem_cache *cache, void *object) #endif } -static inline void set_track(struct kasan_track *track) +static inline int +in_irq_stack(unsigned long *stack, unsigned long *irq_stack, + unsigned long *irq_stack_end) +{ + return (stack >= irq_stack && stack < irq_stack_end); +} + +static inline void filter_irq_stacks(struct stack_trace *trace) +{ + const unsigned cpu = get_cpu(); + unsigned long *irq_stack = (unsigned long *)per_cpu(irq_stack_ptr, cpu); + unsigned long stack_end, stack; + int index = 0; + if (!trace->nr_entries) + return; + stack = trace->entries[0]; + if (!in_irq_stack(stack, irq_stack, &stack_end)) + return; + while (in_irq_stack(stack, irq_stack, &stack_end)) { + index++; + if (index >= trace->nr_entries) + break; + stack = trace->entries[index]; + } + if (index < trace->nr_entries) + trace->nr_entries = index; +} + +static inline kasan_stack_handle save_stack(gfp_t flags) +{ + unsigned long entries[KASAN_STACK_DEPTH]; + struct stack_trace trace = { + .nr_entries = 0, + .entries = entries, + .max_entries = KASAN_STACK_DEPTH, + .skip = 0 + }; + + save_stack_trace(&trace); + filter_irq_stacks(&trace); + if (trace.nr_entries != 0 && + trace.entries[trace.nr_entries-1] == ULONG_MAX) + trace.nr_entries--; + + return kasan_save_stack(&trace, flags); +} + +static inline void set_track(struct kasan_track *track, gfp_t flags) { track->cpu = raw_smp_processor_id(); track->pid = current->pid; track->when = jiffies; + track->stack = save_stack(flags); } #ifdef CONFIG_SLAB struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache, const void *object) { + BUILD_BUG_ON(sizeof(struct kasan_alloc_meta) > 32); return (void *)object + cache->kasan_info.alloc_meta_offset; } struct kasan_free_meta *get_free_info(struct kmem_cache *cache, const void *object) { + BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32); return (void *)object + cache->kasan_info.free_meta_offset; } #endif @@ -455,7 +505,7 @@ void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size, alloc_info->state = KASAN_STATE_ALLOC; alloc_info->alloc_size = size; - set_track(&alloc_info->track); + set_track(&alloc_info->track, flags); } #endif } diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index 7b9e4ab9b66b43..eb9de3696df81c 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -64,10 +64,15 @@ enum kasan_state { KASAN_STATE_FREE }; +#define KASAN_STACK_DEPTH 64 +#define KASAN_STACK_BITS (32) /* up to 16GB of stack storage */ +typedef u32 kasan_stack_handle; + struct kasan_track { u64 cpu : 6; /* for NR_CPUS = 64 */ u64 pid : 16; /* 65536 processes */ u64 when : 42; /* ~140 years */ + kasan_stack_handle stack : KASAN_STACK_BITS; }; struct kasan_alloc_meta { @@ -102,4 +107,10 @@ static inline bool kasan_report_enabled(void) void kasan_report(unsigned long addr, size_t size, bool is_write, unsigned long ip); +struct stack_trace; + +kasan_stack_handle kasan_save_stack(struct stack_trace *trace, gfp_t flags); + +void kasan_fetch_stack(kasan_stack_handle handle, struct stack_trace *trace); + #endif diff --git a/mm/kasan/report.c b/mm/kasan/report.c index 2bf7218fef12e3..6c4afcdbd41272 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -119,6 +119,14 @@ static void print_track(struct kasan_track *track) { pr_err("PID = %lu, CPU = %lu, timestamp = %lu\n", track->pid, track->cpu, track->when); + if (track->stack) { + struct stack_trace trace; + + kasan_fetch_stack(track->stack, &trace); + print_stack_trace(&trace, 0); + } else { + pr_err("(stack is not available)\n"); + } } static void print_object(struct kmem_cache *cache, void *object) diff --git a/mm/kasan/stackdepot.c b/mm/kasan/stackdepot.c new file mode 100644 index 00000000000000..1d073b9e354e63 --- /dev/null +++ b/mm/kasan/stackdepot.c @@ -0,0 +1,254 @@ +/* + * Stack depot + * KASAN needs to safe alloc and free stacks per object, but storing 2 stack + * traces per object is too much overhead (e.g. SLUB_DEBUG needs 256 bytes per + * object). + * + * Instead, stack depot maintains a hashtable of unique stacktraces. Since alloc + * and free stacks repeat a lot, we save about 100x space. + * Stacks are never removed from depot, so we store them contiguously one after + * another in a contiguos memory allocation. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kasan.h" + +#define STACK_ALLOC_ORDER 4 /* 'Slab' size order for stack depot, 16 pages */ +#define STACK_ALLOC_SIZE (1L << (PAGE_SHIFT + STACK_ALLOC_ORDER)) +// TODO: take a closer look +#define STACK_ALLOC_GFP_MASK (__GFP_RECLAIM | __GFP_IO | __GFP_FS | \ + __GFP_NOWARN | __GFP_NORETRY | \ + __GFP_NOMEMALLOC | __GFP_DIRECT_RECLAIM ) +#define STACK_ALLOC_ALIGN 4 +#define STACK_ALLOC_OFFSET_BITS (STACK_ALLOC_ORDER + PAGE_SHIFT - \ + STACK_ALLOC_ALIGN) +#define STACK_ALLOC_INDEX_BITS (KASAN_STACK_BITS - STACK_ALLOC_OFFSET_BITS) +#define STACK_ALLOC_SLABS_CAP 2048 +#define STACK_ALLOC_MAX_SLABS \ + (((1L << (STACK_ALLOC_INDEX_BITS)) < STACK_ALLOC_SLABS_CAP) ? \ + (1L << (STACK_ALLOC_INDEX_BITS)) : STACK_ALLOC_SLABS_CAP) + +/* The compact structure to store the reference to stacks. */ +union handle_parts { + kasan_stack_handle handle; + struct { + u32 slabindex : STACK_ALLOC_INDEX_BITS; + u32 offset : STACK_ALLOC_OFFSET_BITS; + }; +}; + +struct kasan_stack { + struct kasan_stack *next; /* Link in the hashtable */ + u32 hash; /* Hash in the hastable */ + u32 size; /* Number of frames in the stack */ + union handle_parts handle; + unsigned long entries[1]; /* Variable-sized array of entries. */ +}; + +static void *stack_slabs[STACK_ALLOC_MAX_SLABS] = { + [0 ... STACK_ALLOC_MAX_SLABS - 1] = NULL +}; + +static int depot_index; +static int next_slab_inited; +static size_t depot_offset; +static DEFINE_SPINLOCK(depot_lock); + +static int init_stack_slab_false = 0, init_stack_slab_true = 0, + init_stack_slab_next_changed = 0, init_stack_slab_inited_current = 0, + init_stack_slab_inited_next = 0; + +static bool init_stack_slab(void **prealloc) +{ + if (!*prealloc) { + init_stack_slab_false++; + return false; + } + if (smp_load_acquire(&next_slab_inited)) { + init_stack_slab_true++; + init_stack_slab_next_changed++; + return true; + } + if (stack_slabs[depot_index] == NULL) { + stack_slabs[depot_index] = *prealloc; + init_stack_slab_inited_current++; + } else { + stack_slabs[depot_index + 1] = *prealloc; + smp_store_release(&next_slab_inited, 1); + init_stack_slab_inited_next++; + } + *prealloc = NULL; + init_stack_slab_true++; + return true; +} + +/* Allocation of a new stack in raw storage */ +static struct kasan_stack *kasan_alloc_stack(unsigned long *entries, int size, + u32 hash, void **prealloc, gfp_t alloc_flags) +{ + int required_size = offsetof(struct kasan_stack, entries) + + sizeof(unsigned long) * size; + struct kasan_stack *stack; + + required_size = ALIGN(required_size, 1 << STACK_ALLOC_ALIGN); + + if (unlikely(depot_offset + required_size > STACK_ALLOC_SIZE)) { + if (unlikely(depot_index + 1 >= STACK_ALLOC_MAX_SLABS)) { + WARN_ONCE(1, "Stack depot reached limit capacity"); + return NULL; + } + depot_index++; + depot_offset = 0; + if (depot_index + 1 < STACK_ALLOC_MAX_SLABS) + smp_store_release(&next_slab_inited, 0); + } + init_stack_slab(prealloc); + if (stack_slabs[depot_index] == NULL) { + //pr_warn("Failed to allocate stack in kasan depot #1\n"); + //dump_stack(); + return NULL; + } + + stack = stack_slabs[depot_index] + depot_offset; + + stack->hash = hash; + stack->size = size; + stack->handle.slabindex = depot_index; + stack->handle.offset = depot_offset >> STACK_ALLOC_ALIGN; + __memcpy(stack->entries, entries, size * sizeof(unsigned long)); + depot_offset += required_size; + + return stack; +} + +#define STACK_HASH_ORDER 20 +#define STACK_HASH_SIZE (1L << STACK_HASH_ORDER) +#define STACK_HASH_MASK (STACK_HASH_SIZE - 1) +#define STACK_HASH_SEED 0x9747b28c + +static struct kasan_stack *stack_table[STACK_HASH_SIZE] = { + [0 ... STACK_HASH_SIZE - 1] = NULL +}; + +/* Calculate hash for a stack */ +static inline u32 hash_stack(unsigned long *entries, unsigned int size) +{ + return jhash2((u32 *)entries, + size * sizeof(unsigned long) / sizeof(u32), + STACK_HASH_SEED); +} + +/* Find a stack that is equal to the one stored in entries in the hash */ +static inline struct kasan_stack *find_stack(struct kasan_stack *bucket, + unsigned long *entries, int size, + u32 hash) +{ + struct kasan_stack *found; + + for (found = bucket; found; found = found->next) { + if (found->hash == hash && + found->size == size && + !memcmp(entries, found->entries, + size * sizeof(unsigned long))) { + return found; + } + } + return NULL; +} + +void kasan_fetch_stack(kasan_stack_handle handle, struct stack_trace *trace) +{ + union handle_parts parts = { .handle = handle }; + void *slab = stack_slabs[parts.slabindex]; + size_t offset = parts.offset << STACK_ALLOC_ALIGN; + struct kasan_stack *stack = slab + offset; + + trace->nr_entries = trace->max_entries = stack->size; + trace->entries = stack->entries; + trace->skip = 0; +} + +/* + * kasan_save_stack - save stack in a stack depot. + * @trace - the stacktrace to save. + * @alloc_flags - flags for allocating additional memory if required. + * + * Returns the handle of the stack struct stored in depot. + */ +kasan_stack_handle kasan_save_stack(struct stack_trace *trace, + gfp_t alloc_flags) +{ + u32 hash; + kasan_stack_handle retval = 0; + struct kasan_stack *found = NULL, **bucket; + unsigned long flags; + struct page *page = NULL; + void *prealloc = NULL; + + if (unlikely(trace->nr_entries == 0)) + goto fin; + + hash = hash_stack(trace->entries, trace->nr_entries); + // Bad luck, we won't store this stack. + if (hash == 0) + goto fin; + + bucket = &stack_table[hash & STACK_HASH_MASK]; + + /* Fast path: look the stack trace up without locking. */ + found = find_stack(smp_load_acquire(bucket), trace->entries, + trace->nr_entries, hash); + if (found) + goto fin; + + /* Check if the current or the next stack slab need to be + * initialized. If so, allocate the memory - we won't be able to do + * that under the lock. + */ + if (unlikely(!smp_load_acquire(&next_slab_inited))) { + if (!preempt_count() && !in_irq()) { + page = alloc_pages(alloc_flags & STACK_ALLOC_GFP_MASK, + STACK_ALLOC_ORDER); + if (page) + prealloc = page_address(page); + } + } + + spin_lock_irqsave(&depot_lock, flags); + + found = find_stack(*bucket, trace->entries, trace->nr_entries, hash); + if (!found) { + struct kasan_stack *new = + kasan_alloc_stack(trace->entries, trace->nr_entries, + hash, &prealloc, alloc_flags); + if (new) { + new->next = *bucket; + smp_store_release(bucket, new); + found = new; + } + } else if (prealloc) { + /* We didn't need to store this stack trace, but let's keep + * the preallocated memory for the future. + */ + BUG_ON(!init_stack_slab(&prealloc)); + } + + spin_unlock_irqrestore(&depot_lock, flags); +fin: + if (prealloc) + /* Nobody used this memory, ok to free it. */ + free_pages((unsigned long)prealloc, STACK_ALLOC_ORDER); + if (found) + retval = found->handle.handle; + return retval; +} From c379512f84aad8a155f8cd1db9624876a8c3fe04 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Thu, 26 Nov 2015 15:32:10 +0100 Subject: [PATCH 5/7] kasan: Test fix: Warn if the UAF could not be detected in kmalloc_uaf2 Signed-off-by: Alexander Potapenko --- lib/test_kasan.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/test_kasan.c b/lib/test_kasan.c index eb5a9d0d73b2fc..d25b96d784af37 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -285,6 +285,8 @@ static noinline void __init kmalloc_uaf2(void) } ptr1[40] = 'x'; + if (ptr1 == ptr2) + pr_err("Could not detect use-after-free: ptr1 == ptr2\n"); kfree(ptr2); } From 6d1ad51f3c46d3ae2d30af700b9894895ded148b Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Mon, 7 Dec 2015 14:11:14 +0100 Subject: [PATCH 6/7] kasan: Reworked kmalloc_large_oob_right, added kmalloc_pagealloc_oob_right Signed-off-by: Alexander Potapenko --- lib/test_kasan.c | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/lib/test_kasan.c b/lib/test_kasan.c index d25b96d784af37..040dd26c3cd657 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -65,25 +65,34 @@ static noinline void __init kmalloc_node_oob_right(void) kfree(ptr); } -static noinline void __init kmalloc_large_oob_right(void) +#ifdef CONFIG_SLUB +static noinline void __init kmalloc_pagealloc_oob_right(void) { char *ptr; - size_t size; - if (KMALLOC_MAX_CACHE_SIZE == KMALLOC_MAX_SIZE) { - /* - * We're using the SLAB allocator. Allocate a chunk that fits - * into a slab. - */ - size = KMALLOC_MAX_CACHE_SIZE - 256; - } else { - /* - * KMALLOC_MAX_SIZE > KMALLOC_MAX_CACHE_SIZE. - * We're using the SLUB allocator. Allocate a chunk that does - * not fit into a slab to trigger the page allocator. - */ - size = KMALLOC_MAX_CACHE_SIZE + 10; + /* Allocate a chunk that does not fit into a slab to trigger the page + * allocator fallback. + */ + size_t size = KMALLOC_MAX_CACHE_SIZE + 10; + + pr_info("kmalloc page allocator allocation: out-of-bounds to right\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; } + ptr[size] = 0; + kfree(ptr); +} +#endif + +static noinline void __init kmalloc_large_oob_right(void) +{ + char *ptr; + size_t size = KMALLOC_MAX_CACHE_SIZE - 256; + /* Allocate a chunk that is large enough, but still fits into a slab + * and does not trigger the page allocator fallback in SLUB. + */ pr_info("kmalloc large allocation: out-of-bounds to right\n"); ptr = kmalloc(size, GFP_KERNEL); if (!ptr) { @@ -340,6 +349,9 @@ static int __init kmalloc_tests_init(void) kmalloc_oob_right(); kmalloc_oob_left(); kmalloc_node_oob_right(); +#ifdef CONFIG_SLUB + kmalloc_pagealloc_oob_right(); +#endif kmalloc_large_oob_right(); kmalloc_oob_krealloc_more(); kmalloc_oob_krealloc_less(); From edc5284b08147dd182a1a793dd341dde5af21433 Mon Sep 17 00:00:00 2001 From: Dmitry Chernenkov Date: Thu, 2 Apr 2015 14:28:24 +0300 Subject: [PATCH 7/7] mm: kasan: Initial memory quarantine implementation. Quarantine isolates freed objects in a separate queue. The objects are returned to the allocator later, which helps to detect use-after-free errors. Freed objects are first added to per-cpu quarantine queues. When a cache is destroyed or memory shrinking is requested, those objects are moved into the global quarantine queue. Whenever a kmalloc call allows memory reclaiming, the oldest objects are popped out of the global queue until the total size of objects in quarantine is less than 3/4 of the maximum quarantine size (which is a fraction of installed physical memory). This patch is based on the "mm: kasan: quarantine" patch originally prepared by Dmitry Chernenkov. Signed-off-by: Dmitry Chernenkov Signed-off-by: Alexander Potapenko --- include/linux/kasan.h | 34 +++-- lib/test_kasan.c | 29 +++++ mm/kasan/Makefile | 2 +- mm/kasan/kasan.c | 73 +++++++++-- mm/kasan/kasan.h | 10 +- mm/kasan/quarantine.c | 280 ++++++++++++++++++++++++++++++++++++++++++ mm/kasan/report.c | 3 +- mm/mempool.c | 7 +- mm/page_alloc.c | 2 +- mm/slab.c | 12 +- mm/slab.h | 4 + mm/slab_common.c | 2 + mm/slub.c | 4 +- 13 files changed, 434 insertions(+), 28 deletions(-) create mode 100644 mm/kasan/quarantine.c diff --git a/include/linux/kasan.h b/include/linux/kasan.h index b75a605b4ef7ae..4db6ff3424d03b 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -46,24 +46,29 @@ static inline void kasan_disable_current(void) void kasan_unpoison_shadow(const void *address, size_t size); void kasan_alloc_pages(struct page *page, unsigned int order); -void kasan_free_pages(struct page *page, unsigned int order); +void kasan_poison_free_pages(struct page *page, unsigned int order); void kasan_cache_create(struct kmem_cache *cache, cache_size_t *size, unsigned long *flags); +void kasan_cache_shrink(struct kmem_cache *cache); +void kasan_cache_destroy(struct kmem_cache *cache); void kasan_poison_slab(struct page *page); void kasan_unpoison_object_data(struct kmem_cache *cache, void *object); void kasan_poison_object_data(struct kmem_cache *cache, void *object); void kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags); -void kasan_kfree_large(const void *ptr); -void kasan_kfree(void *ptr); +void kasan_poison_kfree_large(const void *ptr); +void kasan_poison_kfree(void *ptr); void kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size, gfp_t flags); void kasan_krealloc(const void *object, size_t new_size, gfp_t flags); void kasan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags); -void kasan_slab_free(struct kmem_cache *s, void *object); +/* kasan_slab_free() returns true if the object has been put into quarantine. + */ +bool kasan_slab_free(struct kmem_cache *s, void *object); +void kasan_poison_slab_free(struct kmem_cache *s, void *object); struct kasan_cache { int alloc_meta_offset; @@ -81,11 +86,14 @@ static inline void kasan_enable_current(void) {} static inline void kasan_disable_current(void) {} static inline void kasan_alloc_pages(struct page *page, unsigned int order) {} -static inline void kasan_free_pages(struct page *page, unsigned int order) {} +static inline void kasan_poison_free_pages(struct page *page, + unsigned int order) {} static inline void kasan_cache_create(struct kmem_cache *cache, - cache_size_t *size, - unsigned long *flags) {} + cache_size_t *size, + unsigned long *flags) {} +static inline void kasan_cache_shrink(struct kmem_cache *cache) {} +static inline void kasan_cache_destroy(struct kmem_cache *cache) {} static inline void kasan_poison_slab(struct page *page) {} static inline void kasan_unpoison_object_data(struct kmem_cache *cache, @@ -94,8 +102,8 @@ static inline void kasan_poison_object_data(struct kmem_cache *cache, void *object) {} static inline void kasan_kmalloc_large(void *ptr, size_t size, gfp_t flags) {} -static inline void kasan_kfree_large(const void *ptr) {} -static inline void kasan_kfree(void *ptr) {} +static inline void kasan_poison_kfree_large(const void *ptr) {} +static inline void kasan_poison_kfree(void *ptr) {} static inline void kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size, gfp_t flags) {} static inline void kasan_krealloc(const void *object, size_t new_size, @@ -103,7 +111,13 @@ static inline void kasan_krealloc(const void *object, size_t new_size, static inline void kasan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags) {} -static inline void kasan_slab_free(struct kmem_cache *s, void *object) {} +/* kasan_slab_free() returns true if the object has been put into quarantine. + */ +static inline bool kasan_slab_free(struct kmem_cache *s, void *object) +{ + return false; +} +static inline void kasan_poison_slab_free(struct kmem_cache *s, void *object) {} static inline int kasan_module_alloc(void *addr, size_t size) { return 0; } static inline void kasan_free_shadow(const struct vm_struct *vm) {} diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 040dd26c3cd657..48c2ae38d2261c 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -344,6 +344,32 @@ static noinline void __init kasan_stack_oob(void) *(volatile char *)p; } +#ifdef CONFIG_SLAB +static noinline void __init kasan_quarantine_cache(void) +{ + struct kmem_cache *cache = kmem_cache_create( + "test", 137, 8, GFP_KERNEL, NULL); + int i; + + for (i = 0; i < 100; i++) { + void *p = kmem_cache_alloc(cache, GFP_KERNEL); + + kmem_cache_free(cache, p); + p = kmalloc(sizeof(u64), GFP_KERNEL); + kfree(p); + } + kmem_cache_shrink(cache); + for (i = 0; i < 100; i++) { + u64 *p = kmem_cache_alloc(cache, GFP_KERNEL); + + kmem_cache_free(cache, p); + p = kmalloc(sizeof(u64), GFP_KERNEL); + kfree(p); + } + kmem_cache_destroy(cache); +} +#endif + static int __init kmalloc_tests_init(void) { kmalloc_oob_right(); @@ -367,6 +393,9 @@ static int __init kmalloc_tests_init(void) kmem_cache_oob(); kasan_stack_oob(); kasan_global_oob(); +#ifdef CONFIG_SLAB + kasan_quarantine_cache(); +#endif return -EAGAIN; } diff --git a/mm/kasan/Makefile b/mm/kasan/Makefile index f952515f1ed22a..8e59350bf1cbcc 100644 --- a/mm/kasan/Makefile +++ b/mm/kasan/Makefile @@ -7,5 +7,5 @@ CFLAGS_kasan.o := $(call cc-option, -fno-conserve-stack -fno-stack-protector) obj-y := kasan.o report.o kasan_init.o ifdef CONFIG_SLAB - obj-y += stackdepot.o + obj-y += stackdepot.o quarantine.o endif diff --git a/mm/kasan/kasan.c b/mm/kasan/kasan.c index e51ca077da2ef5..b7e371af3ac200 100644 --- a/mm/kasan/kasan.c +++ b/mm/kasan/kasan.c @@ -306,7 +306,7 @@ void kasan_alloc_pages(struct page *page, unsigned int order) kasan_unpoison_shadow(page_address(page), PAGE_SIZE << order); } -void kasan_free_pages(struct page *page, unsigned int order) +void kasan_poison_free_pages(struct page *page, unsigned int order) { if (likely(!PageHighMem(page))) kasan_poison_shadow(page_address(page), @@ -367,6 +367,20 @@ void kasan_cache_create(struct kmem_cache *cache, cache_size_t *size, } #endif +void kasan_cache_shrink(struct kmem_cache *cache) +{ +#ifdef CONFIG_SLAB + quarantine_remove_cache(cache); +#endif +} + +void kasan_cache_destroy(struct kmem_cache *cache) +{ +#ifdef CONFIG_SLAB + quarantine_remove_cache(cache); +#endif +} + void kasan_poison_slab(struct page *page) { kasan_poison_shadow(page_address(page), @@ -469,24 +483,64 @@ void kasan_slab_alloc(struct kmem_cache *cache, void *object, gfp_t flags) kasan_kmalloc(cache, object, cache->object_size, flags); } -void kasan_slab_free(struct kmem_cache *cache, void *object) -{ +void kasan_poison_slab_free(struct kmem_cache *cache, void *object) { unsigned long size = cache->object_size; - unsigned long rounded_up_size = round_up(size, KASAN_SHADOW_SCALE_SIZE); + unsigned long rounded_up_size = round_up(size, KASAN_SHADOW_SCALE_SIZE); /* RCU slabs could be legally used after free within the RCU period */ if (unlikely(cache->flags & SLAB_DESTROY_BY_RCU)) return; - + kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE); } +bool kasan_slab_free(struct kmem_cache *cache, void *object) +{ +#ifdef CONFIG_SLAB + /* RCU slabs could be legally used after free within the RCU period */ + if (unlikely(cache->flags & SLAB_DESTROY_BY_RCU)) + return false; + + if (likely(cache->flags & SLAB_KASAN)) { + struct kasan_alloc_meta *alloc_info = + get_alloc_info(cache, object); + struct kasan_free_meta *free_info = + get_free_info(cache, object); + + switch (alloc_info->state) { + case KASAN_STATE_ALLOC: + alloc_info->state = KASAN_STATE_QUARANTINE; + quarantine_put(free_info, cache); + set_track(&free_info->track, GFP_NOWAIT); + kasan_poison_slab_free(cache, object); + return true; + case KASAN_STATE_QUARANTINE: + case KASAN_STATE_FREE: + pr_err("Double free"); + dump_stack(); + break; + default: + break; + } + } + return false; +#else + kasan_poison_slab_free(cache, object); + return false; +#endif +} + void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size, gfp_t flags) { unsigned long redzone_start; unsigned long redzone_end; +#ifdef CONFIG_SLAB + if (flags & __GFP_RECLAIM) + quarantine_reduce(); +#endif + if (unlikely(object == NULL)) return; @@ -517,6 +571,11 @@ void kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags) unsigned long redzone_start; unsigned long redzone_end; +#ifdef CONFIG_SLAB + if (flags & __GFP_RECLAIM) + quarantine_reduce(); +#endif + if (unlikely(ptr == NULL)) return; @@ -545,7 +604,7 @@ void kasan_krealloc(const void *object, size_t size, gfp_t flags) kasan_kmalloc(page->slab_cache, object, size, flags); } -void kasan_kfree(void *ptr) +void kasan_poison_kfree(void *ptr) { struct page *page; @@ -558,7 +617,7 @@ void kasan_kfree(void *ptr) kasan_slab_free(page->slab_cache, ptr); } -void kasan_kfree_large(const void *ptr) +void kasan_poison_kfree_large(const void *ptr) { struct page *page = virt_to_page(ptr); diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index eb9de3696df81c..4c466a22f3256d 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -61,6 +61,7 @@ struct kasan_global { enum kasan_state { KASAN_STATE_INIT, KASAN_STATE_ALLOC, + KASAN_STATE_QUARANTINE, KASAN_STATE_FREE }; @@ -82,8 +83,9 @@ struct kasan_alloc_meta { }; struct kasan_free_meta { - /* Allocator freelist pointer, unused by KASAN. */ - void **freelist; + /* This field is used while the object is in quarantine. + * Otherwise it might be used by the freelist */ + void **quarantine_link; struct kasan_track track; }; @@ -113,4 +115,8 @@ kasan_stack_handle kasan_save_stack(struct stack_trace *trace, gfp_t flags); void kasan_fetch_stack(kasan_stack_handle handle, struct stack_trace *trace); +void quarantine_put(struct kasan_free_meta *info, struct kmem_cache *cache); +void quarantine_reduce(void); +void quarantine_remove_cache(struct kmem_cache *cache); + #endif diff --git a/mm/kasan/quarantine.c b/mm/kasan/quarantine.c new file mode 100644 index 00000000000000..295cc28d5cac01 --- /dev/null +++ b/mm/kasan/quarantine.c @@ -0,0 +1,280 @@ +/* Kasan quarantine */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../slab.h" +#include "kasan.h" + +/* Data structure and operations for quarantine queues */ + +/* Each queue is a signled-linked list, which also stores the total size of + * objects inside of it */ +struct qlist { + void **head; + void **tail; + size_t bytes; +}; + +#define QLIST_INIT { NULL, NULL, 0 } + +static inline bool empty_qlist(struct qlist *q) +{ + return !q->head; +} + +static inline void init_qlist(struct qlist *q) +{ + q->head = q->tail = NULL; + q->bytes = 0; +} + +static inline void qlist_put(struct qlist *q, void **qlink, size_t size) +{ + if (unlikely(empty_qlist(q))) + q->head = qlink; + else + *q->tail = qlink; + q->tail = qlink; + *qlink = NULL; + q->bytes += size; +} + +static inline void** qlist_remove(struct qlist *q, void ***prev, + size_t size) { + void **qlink = *prev; + + *prev = *qlink; + if (q->tail == qlink) { + if (q->head == qlink) + q->tail = NULL; + else + q->tail = (void**) prev; + } + q->bytes -= size; + + return qlink; +} + +static inline void qlist_move_all(struct qlist *from, struct qlist *to) +{ + if (unlikely(empty_qlist(from))) + return; + + if (empty_qlist(to)) { + *to = *from; + init_qlist(from); + return; + } + + *to->tail = from->head; + to->tail = from->tail; + to->bytes += from->bytes; + + init_qlist(from); +} + +static inline void qlist_move(struct qlist *from, void **last, struct qlist *to, + size_t size) +{ + if (unlikely(last == from->tail)) { + qlist_move_all(from, to); + return; + } + if (empty_qlist(to)) + to->head = from->head; + else + *to->tail = from->head; + to->tail = last; + from->head = *last; + *last = NULL; + from->bytes -= size; + to->bytes += size; +} + + +/* The object quarantine consists of per-cpu queues and a global queue, + * guarded by quarantine_lock */ + +static DEFINE_PER_CPU(struct qlist, cpu_quarantine); + +static struct qlist global_quarantine; +static DEFINE_SPINLOCK(quarantine_lock); + +static volatile unsigned long quarantine_size; /* The maximum size of global + queue */ + +/* The fraction of physical memory the quarantine is allowed to occupy. */ +#define QUARANTINE_FRACTION 32 /* Quarantine doesn't support memory shrinker + with SLAB allocator, so we keep the ratio + low to avoid OOM. */ + +#define QUARANTINE_LOW_SIZE (quarantine_size * 3 / 4) +#define QUARANTINE_PERCPU_SIZE (1 << 20) + +static inline struct kmem_cache *qlink_to_cache(void **qlink) +{ + return virt_to_head_page(qlink)->slab_cache; +} + +static inline void *qlink_to_object(void **qlink, struct kmem_cache *cache) +{ + struct kasan_free_meta *free_info = + container_of((void ***)qlink, struct kasan_free_meta, + quarantine_link); + + return ((void *)free_info) - cache->kasan_info.free_meta_offset; +} + +static inline void qlink_free(void **qlink, struct kmem_cache *cache) +{ + void *object = qlink_to_object(qlink, cache); + struct kasan_alloc_meta *alloc_info = get_alloc_info(cache, object); + unsigned long flags; + + local_irq_save(flags); + alloc_info->state = KASAN_STATE_FREE; + nokasan_free(cache, object, _THIS_IP_); + local_irq_restore(flags); +} + +static inline void qlist_free_all(struct qlist *q, struct kmem_cache *cache) +{ + void **qlink; + + if (unlikely(empty_qlist(q))) + return; + + qlink = q->head; + while (qlink) { + struct kmem_cache *obj_cache = + cache ? cache : qlink_to_cache(qlink); + void **next = *qlink; + + qlink_free(qlink, obj_cache); + qlink = next; + } + init_qlist(q); +} + +void quarantine_put(struct kasan_free_meta *info, struct kmem_cache *cache) +{ + unsigned long flags; + struct qlist *q; + struct qlist temp = QLIST_INIT; + + local_irq_save(flags); + + q = this_cpu_ptr(&cpu_quarantine); + qlist_put(q, (void **) &info->quarantine_link, cache->size); + if (unlikely(q->bytes > QUARANTINE_PERCPU_SIZE)) + qlist_move_all(q, &temp); + + local_irq_restore(flags); + + if (unlikely(!empty_qlist(&temp))) { + spin_lock_irqsave(&quarantine_lock, flags); + qlist_move_all(&temp, &global_quarantine); + spin_unlock_irqrestore(&quarantine_lock, flags); + } +} + +void quarantine_reduce(void) +{ + size_t new_quarantine_size; + unsigned long flags; + struct qlist to_free = QLIST_INIT; + size_t size_to_free = 0; + void **last; + + if (likely(ACCESS_ONCE(global_quarantine.bytes) <= + smp_load_acquire(&quarantine_size))) + return; + + spin_lock_irqsave(&quarantine_lock, flags); + + /* Update quarantine size in case of hotplug. Allocate fraction of installed + * memory to quarantine minus per-cpu queue limits. */ + new_quarantine_size = (ACCESS_ONCE(totalram_pages) << PAGE_SHIFT) / + QUARANTINE_FRACTION; + new_quarantine_size -= QUARANTINE_PERCPU_SIZE * num_online_cpus(); + smp_store_release(&quarantine_size, new_quarantine_size); + + last = global_quarantine.head; + while (last) { + struct kmem_cache *cache = qlink_to_cache(last); + + size_to_free += cache->size; + if (!*last || size_to_free > + global_quarantine.bytes - QUARANTINE_LOW_SIZE) + break; + last = (void **) *last; + } + qlist_move(&global_quarantine, last, &to_free, size_to_free); + + spin_unlock_irqrestore(&quarantine_lock, flags); + + qlist_free_all(&to_free, NULL); +} + +static inline void qlist_move_cache(struct qlist *from, + struct qlist *to, + struct kmem_cache *cache) +{ + void ***prev; + + if (unlikely(empty_qlist(from))) + return; + + prev = &from->head; + while (*prev) { + void **qlink = *prev; + struct kmem_cache *obj_cache = qlink_to_cache(qlink); + + if (obj_cache == cache) { + if (unlikely(from->tail == qlink)) + from->tail = (void **) prev; + *prev = (void **) *qlink; + from->bytes -= cache->size; + qlist_put(to, qlink, cache->size); + } else + prev = (void ***) *prev; + } +} + +static void per_cpu_remove_cache(void *arg) +{ + struct kmem_cache *cache = arg; + struct qlist to_free = QLIST_INIT; + struct qlist *q; + unsigned long flags; + + local_irq_save(flags); + q = this_cpu_ptr(&cpu_quarantine); + qlist_move_cache(q, &to_free, cache); + local_irq_restore(flags); + + qlist_free_all(&to_free, cache); +} + +void quarantine_remove_cache(struct kmem_cache *cache) +{ + unsigned long flags; + struct qlist to_free = QLIST_INIT; + + on_each_cpu(per_cpu_remove_cache, cache, 0); + + spin_lock_irqsave(&quarantine_lock, flags); + qlist_move_cache(&global_quarantine, &to_free, cache); + spin_unlock_irqrestore(&quarantine_lock, flags); + + qlist_free_all(&to_free, cache); +} diff --git a/mm/kasan/report.c b/mm/kasan/report.c index 6c4afcdbd41272..a4dca254300e3b 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -148,7 +148,8 @@ static void print_object(struct kmem_cache *cache, void *object) print_track(&alloc_info->track); break; case KASAN_STATE_FREE: - pr_err("Object freed, allocated with size %u bytes\n", + case KASAN_STATE_QUARANTINE: + pr_err("Object freed, allocated with size %lu bytes\n", alloc_info->alloc_size); free_info = get_free_info(cache, object); pr_err("Allocation:\n"); diff --git a/mm/mempool.c b/mm/mempool.c index b47c8a75d7c77a..4beeeef3640e1a 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -105,11 +105,12 @@ static inline void poison_element(mempool_t *pool, void *element) static void kasan_poison_element(mempool_t *pool, void *element) { if (pool->alloc == mempool_alloc_slab) - kasan_slab_free(pool->pool_data, element); + kasan_poison_slab_free(pool->pool_data, element); if (pool->alloc == mempool_kmalloc) - kasan_kfree(element); + kasan_poison_kfree(element); if (pool->alloc == mempool_alloc_pages) - kasan_free_pages(element, (unsigned long)pool->pool_data); + kasan_poison_free_pages(element, + (unsigned long)pool->pool_data); } static void kasan_unpoison_element(mempool_t *pool, void *element, gfp_t flags) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 9d666df5ef9502..613fafa5bd25ee 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -960,7 +960,7 @@ static bool free_pages_prepare(struct page *page, unsigned int order) trace_mm_page_free(page, order); kmemcheck_free_shadow(page, order); - kasan_free_pages(page, order); + kasan_poison_free_pages(page, order); if (PageAnon(page)) page->mapping = NULL; diff --git a/mm/slab.c b/mm/slab.c index 743e920da5f61f..92697f11e32aa3 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3376,9 +3376,19 @@ static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) static inline void __cache_free(struct kmem_cache *cachep, void *objp, unsigned long caller) { +#ifdef CONFIG_KASAN + if (!kasan_slab_free(cachep, objp)) + /* The object has been put into the quarantine, don't touch it + * for now. + */ + nokasan_free(cachep, objp, caller); +} + +void nokasan_free(struct kmem_cache *cachep, void *objp, unsigned long caller) +{ +#endif struct array_cache *ac; - kasan_slab_free(cachep, objp); ac = cpu_cache_get(cachep); check_irq_off(); diff --git a/mm/slab.h b/mm/slab.h index 7b608719799763..2739457a395e59 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -371,4 +371,8 @@ void *slab_next(struct seq_file *m, void *p, loff_t *pos); void slab_stop(struct seq_file *m, void *p); int memcg_slab_show(struct seq_file *m, void *p); +#ifdef CONFIG_KASAN +void nokasan_free(struct kmem_cache *cache, void *x, unsigned long addr); +#endif + #endif /* MM_SLAB_H */ diff --git a/mm/slab_common.c b/mm/slab_common.c index 3f645da7a22fcf..d255f7770b68a6 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -709,6 +709,7 @@ void kmem_cache_destroy(struct kmem_cache *s) get_online_cpus(); get_online_mems(); + kasan_cache_destroy(s); mutex_lock(&slab_mutex); s->refcount--; @@ -747,6 +748,7 @@ int kmem_cache_shrink(struct kmem_cache *cachep) get_online_cpus(); get_online_mems(); + kasan_cache_shrink(cachep); ret = __kmem_cache_shrink(cachep, false); put_online_mems(); put_online_cpus(); diff --git a/mm/slub.c b/mm/slub.c index 8b00acc157eaa8..5462eddd486e1a 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1276,7 +1276,7 @@ static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) static inline void kfree_hook(const void *x) { kmemleak_free(x); - kasan_kfree_large(x); + kasan_poison_kfree_large(x); } static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, @@ -1331,7 +1331,7 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x) if (!(s->flags & SLAB_DEBUG_OBJECTS)) debug_check_no_obj_freed(x, s->object_size); - kasan_slab_free(s, x); + kasan_poison_slab_free(s, x); } static inline void slab_free_freelist_hook(struct kmem_cache *s,