From f1b6eb6e6be149b40ebb013f5bfe2ac86b6f1c1b Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 4 Sep 2013 16:35:34 +0000 Subject: mm/sl[aou]b: Move kmallocXXX functions to common code The kmalloc* functions of all slab allcoators are similar now so lets move them into slab.h. This requires some function naming changes in slob. As a results of this patch there is a common set of functions for all allocators. Also means that kmalloc_large() is now available in general to perform large order allocations that go directly via the page allocator. kmalloc_large() can be substituted if kmalloc() throws warnings because of too large allocations. kmalloc_large() has exactly the same semantics as kmalloc but can only used for allocations > PAGE_SIZE. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slab.h | 156 +++++++++++++++++++++++++++++++++++++---------- include/linux/slab_def.h | 106 -------------------------------- include/linux/slob_def.h | 31 ---------- include/linux/slub_def.h | 97 ----------------------------- 4 files changed, 124 insertions(+), 266 deletions(-) delete mode 100644 include/linux/slob_def.h (limited to 'include') diff --git a/include/linux/slab.h b/include/linux/slab.h index 6c5cc0ea8713..386af639dcaa 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -4,6 +4,8 @@ * (C) SGI 2006, Christoph Lameter * Cleaned up and restructured to ease the addition of alternative * implementations of SLAB allocators. + * (C) Linux Foundation 2008-2013 + * Unified interface for all slab allocators */ #ifndef _LINUX_SLAB_H @@ -94,6 +96,7 @@ #define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \ (unsigned long)ZERO_SIZE_PTR) +#include struct mem_cgroup; /* @@ -289,6 +292,57 @@ static __always_inline int kmalloc_index(size_t size) } #endif /* !CONFIG_SLOB */ +void *__kmalloc(size_t size, gfp_t flags); +void *kmem_cache_alloc(struct kmem_cache *, gfp_t flags); + +#ifdef CONFIG_NUMA +void *__kmalloc_node(size_t size, gfp_t flags, int node); +void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); +#else +static __always_inline void *__kmalloc_node(size_t size, gfp_t flags, int node) +{ + return __kmalloc(size, flags); +} + +static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) +{ + return kmem_cache_alloc(s, flags); +} +#endif + +#ifdef CONFIG_TRACING +extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t); + +#ifdef CONFIG_NUMA +extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, + gfp_t gfpflags, + int node, size_t size); +#else +static __always_inline void * +kmem_cache_alloc_node_trace(struct kmem_cache *s, + gfp_t gfpflags, + int node, size_t size) +{ + return kmem_cache_alloc_trace(s, gfpflags, size); +} +#endif /* CONFIG_NUMA */ + +#else /* CONFIG_TRACING */ +static __always_inline void *kmem_cache_alloc_trace(struct kmem_cache *s, + gfp_t flags, size_t size) +{ + return kmem_cache_alloc(s, flags); +} + +static __always_inline void * +kmem_cache_alloc_node_trace(struct kmem_cache *s, + gfp_t gfpflags, + int node, size_t size) +{ + return kmem_cache_alloc_node(s, gfpflags, node); +} +#endif /* CONFIG_TRACING */ + #ifdef CONFIG_SLAB #include #endif @@ -297,9 +351,60 @@ static __always_inline int kmalloc_index(size_t size) #include #endif -#ifdef CONFIG_SLOB -#include +static __always_inline void * +kmalloc_order(size_t size, gfp_t flags, unsigned int order) +{ + void *ret; + + flags |= (__GFP_COMP | __GFP_KMEMCG); + ret = (void *) __get_free_pages(flags, order); + kmemleak_alloc(ret, size, 1, flags); + return ret; +} + +#ifdef CONFIG_TRACING +extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order); +#else +static __always_inline void * +kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) +{ + return kmalloc_order(size, flags, order); +} +#endif + +static __always_inline void *kmalloc_large(size_t size, gfp_t flags) +{ + unsigned int order = get_order(size); + return kmalloc_order_trace(size, flags, order); +} + +/** + * kmalloc - allocate memory + * @size: how many bytes of memory are required. + * @flags: the type of memory to allocate (see kcalloc). + * + * kmalloc is the normal method of allocating memory + * for objects smaller than page size in the kernel. + */ +static __always_inline void *kmalloc(size_t size, gfp_t flags) +{ + if (__builtin_constant_p(size)) { + if (size > KMALLOC_MAX_CACHE_SIZE) + return kmalloc_large(size, flags); +#ifndef CONFIG_SLOB + if (!(flags & GFP_DMA)) { + int index = kmalloc_index(size); + + if (!index) + return ZERO_SIZE_PTR; + + return kmem_cache_alloc_trace(kmalloc_caches[index], + flags, size); + } #endif + } + return __kmalloc(size, flags); +} /* * Determine size used for the nth kmalloc cache. @@ -321,6 +426,23 @@ static __always_inline int kmalloc_size(int n) return 0; } +static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) +{ +#ifndef CONFIG_SLOB + if (__builtin_constant_p(size) && + size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLAB_CACHE_DMA)) { + int i = kmalloc_index(size); + + if (!i) + return ZERO_SIZE_PTR; + + return kmem_cache_alloc_node_trace(kmalloc_caches[i], + flags, node, size); + } +#endif + return __kmalloc_node(size, flags, node); +} + /* * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment. * Intended for arches that get misalignment faults even for 64 bit integer @@ -451,36 +573,6 @@ static inline void *kcalloc(size_t n, size_t size, gfp_t flags) return kmalloc_array(n, size, flags | __GFP_ZERO); } -#if !defined(CONFIG_NUMA) && !defined(CONFIG_SLOB) -/** - * kmalloc_node - allocate memory from a specific node - * @size: how many bytes of memory are required. - * @flags: the type of memory to allocate (see kmalloc). - * @node: node to allocate from. - * - * kmalloc() for non-local nodes, used to allocate from a specific node - * if available. Equivalent to kmalloc() in the non-NUMA single-node - * case. - */ -static inline void *kmalloc_node(size_t size, gfp_t flags, int node) -{ - return kmalloc(size, flags); -} - -static inline void *__kmalloc_node(size_t size, gfp_t flags, int node) -{ - return __kmalloc(size, flags); -} - -void *kmem_cache_alloc(struct kmem_cache *, gfp_t); - -static inline void *kmem_cache_alloc_node(struct kmem_cache *cachep, - gfp_t flags, int node) -{ - return kmem_cache_alloc(cachep, flags); -} -#endif /* !CONFIG_NUMA && !CONFIG_SLOB */ - /* * kmalloc_track_caller is a special version of kmalloc that records the * calling function of the routine calling it for slab leak tracking instead diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index cd401580bdd3..e9346b4f1ef4 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -3,20 +3,6 @@ /* * Definitions unique to the original Linux SLAB allocator. - * - * What we provide here is a way to optimize the frequent kmalloc - * calls in the kernel by selecting the appropriate general cache - * if kmalloc was called with a size that can be established at - * compile time. - */ - -#include -#include - -/* - * struct kmem_cache - * - * manages a cache. */ struct kmem_cache { @@ -102,96 +88,4 @@ struct kmem_cache { */ }; -void *kmem_cache_alloc(struct kmem_cache *, gfp_t); -void *__kmalloc(size_t size, gfp_t flags); - -#ifdef CONFIG_TRACING -extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t); -#else -static __always_inline void * -kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) -{ - return kmem_cache_alloc(cachep, flags); -} -#endif - -static __always_inline void *kmalloc(size_t size, gfp_t flags) -{ - struct kmem_cache *cachep; - void *ret; - - if (__builtin_constant_p(size)) { - int i; - - if (!size) - return ZERO_SIZE_PTR; - - if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE)) - return NULL; - - i = kmalloc_index(size); - -#ifdef CONFIG_ZONE_DMA - if (flags & GFP_DMA) - cachep = kmalloc_dma_caches[i]; - else -#endif - cachep = kmalloc_caches[i]; - - ret = kmem_cache_alloc_trace(cachep, flags, size); - - return ret; - } - return __kmalloc(size, flags); -} - -#ifdef CONFIG_NUMA -extern void *__kmalloc_node(size_t size, gfp_t flags, int node); -extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); - -#ifdef CONFIG_TRACING -extern void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, - gfp_t flags, - int nodeid, - size_t size); -#else -static __always_inline void * -kmem_cache_alloc_node_trace(struct kmem_cache *cachep, - gfp_t flags, - int nodeid, - size_t size) -{ - return kmem_cache_alloc_node(cachep, flags, nodeid); -} -#endif - -static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) -{ - struct kmem_cache *cachep; - - if (__builtin_constant_p(size)) { - int i; - - if (!size) - return ZERO_SIZE_PTR; - - if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE)) - return NULL; - - i = kmalloc_index(size); - -#ifdef CONFIG_ZONE_DMA - if (flags & GFP_DMA) - cachep = kmalloc_dma_caches[i]; - else -#endif - cachep = kmalloc_caches[i]; - - return kmem_cache_alloc_node_trace(cachep, flags, node, size); - } - return __kmalloc_node(size, flags, node); -} - -#endif /* CONFIG_NUMA */ - #endif /* _LINUX_SLAB_DEF_H */ diff --git a/include/linux/slob_def.h b/include/linux/slob_def.h deleted file mode 100644 index 095a5a4a8516..000000000000 --- a/include/linux/slob_def.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef __LINUX_SLOB_DEF_H -#define __LINUX_SLOB_DEF_H - -#include - -void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); - -static __always_inline void *kmem_cache_alloc(struct kmem_cache *cachep, - gfp_t flags) -{ - return kmem_cache_alloc_node(cachep, flags, NUMA_NO_NODE); -} - -void *__kmalloc_node(size_t size, gfp_t flags, int node); - -static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) -{ - return __kmalloc_node(size, flags, node); -} - -static __always_inline void *kmalloc(size_t size, gfp_t flags) -{ - return __kmalloc_node(size, flags, NUMA_NO_NODE); -} - -static __always_inline void *__kmalloc(size_t size, gfp_t flags) -{ - return kmalloc(size, flags); -} - -#endif /* __LINUX_SLOB_DEF_H */ diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 027276fa8713..901fb6eb7467 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -6,14 +6,8 @@ * * (C) 2007 SGI, Christoph Lameter */ -#include -#include -#include -#include #include -#include - enum stat_item { ALLOC_FASTPATH, /* Allocation from cpu slab */ ALLOC_SLOWPATH, /* Allocation by getting a new cpu slab */ @@ -104,20 +98,6 @@ struct kmem_cache { struct kmem_cache_node *node[MAX_NUMNODES]; }; -void *kmem_cache_alloc(struct kmem_cache *, gfp_t); -void *__kmalloc(size_t size, gfp_t flags); - -static __always_inline void * -kmalloc_order(size_t size, gfp_t flags, unsigned int order) -{ - void *ret; - - flags |= (__GFP_COMP | __GFP_KMEMCG); - ret = (void *) __get_free_pages(flags, order); - kmemleak_alloc(ret, size, 1, flags); - return ret; -} - /** * Calling this on allocated memory will check that the memory * is expected to be in use, and print warnings if not. @@ -131,81 +111,4 @@ static inline bool verify_mem_not_deleted(const void *x) } #endif -#ifdef CONFIG_TRACING -extern void * -kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size); -extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order); -#else -static __always_inline void * -kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) -{ - return kmem_cache_alloc(s, gfpflags); -} - -static __always_inline void * -kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) -{ - return kmalloc_order(size, flags, order); -} -#endif - -static __always_inline void *kmalloc_large(size_t size, gfp_t flags) -{ - unsigned int order = get_order(size); - return kmalloc_order_trace(size, flags, order); -} - -static __always_inline void *kmalloc(size_t size, gfp_t flags) -{ - if (__builtin_constant_p(size)) { - if (size > KMALLOC_MAX_CACHE_SIZE) - return kmalloc_large(size, flags); - - if (!(flags & GFP_DMA)) { - int index = kmalloc_index(size); - - if (!index) - return ZERO_SIZE_PTR; - - return kmem_cache_alloc_trace(kmalloc_caches[index], - flags, size); - } - } - return __kmalloc(size, flags); -} - -#ifdef CONFIG_NUMA -void *__kmalloc_node(size_t size, gfp_t flags, int node); -void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); - -#ifdef CONFIG_TRACING -extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, - gfp_t gfpflags, - int node, size_t size); -#else -static __always_inline void * -kmem_cache_alloc_node_trace(struct kmem_cache *s, - gfp_t gfpflags, - int node, size_t size) -{ - return kmem_cache_alloc_node(s, gfpflags, node); -} -#endif - -static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) -{ - if (__builtin_constant_p(size) && - size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) { - int index = kmalloc_index(size); - - if (!index) - return ZERO_SIZE_PTR; - - return kmem_cache_alloc_node_trace(kmalloc_caches[index], - flags, node, size); - } - return __kmalloc_node(size, flags, node); -} -#endif - #endif /* _LINUX_SLUB_DEF_H */ -- cgit v1.2.3 From 76b6f3d255a327383c89cb8c8384872dd4a0a054 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 4 Sep 2013 16:35:35 +0000 Subject: slub: remove verify_mem_not_deleted() I do not see any user for this code in the tree. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slub_def.h | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'include') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 901fb6eb7467..cc0b67eada42 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -98,17 +98,4 @@ struct kmem_cache { struct kmem_cache_node *node[MAX_NUMNODES]; }; -/** - * Calling this on allocated memory will check that the memory - * is expected to be in use, and print warnings if not. - */ -#ifdef CONFIG_SLUB_DEBUG -extern bool verify_mem_not_deleted(const void *x); -#else -static inline bool verify_mem_not_deleted(const void *x) -{ - return true; -} -#endif - #endif /* _LINUX_SLUB_DEF_H */ -- cgit v1.2.3 From 23774a2f6fee0848503bfb8004eeeb5adef94f5c Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 4 Sep 2013 19:58:08 +0000 Subject: slab: Use correct GFP_DMA constant On Thu, 5 Sep 2013, kbuild test robot wrote: > >> include/linux/slab.h:433:53: sparse: restricted gfp_t degrades to integer > 429 static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) > 430 { > 431 #ifndef CONFIG_SLOB > 432 if (__builtin_constant_p(size) && > > 433 size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLAB_CACHE_DMA)) { > 434 int i = kmalloc_index(size); > 435 flags is of type gfp_t and not a slab internal flag. Therefore use GFP_DMA. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slab.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/slab.h b/include/linux/slab.h index 386af639dcaa..74f105847d13 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -430,7 +430,7 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) { #ifndef CONFIG_SLOB if (__builtin_constant_p(size) && - size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLAB_CACHE_DMA)) { + size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) { int i = kmalloc_index(size); if (!i) -- cgit v1.2.3