/* SPDX-License-Identifier: GPL-2.0 */
/*
* Written by Mark Hemment, 1996 ([email protected]).
*
* (C) SGI 2006, Christoph Lameter
* Cleaned up and restructured to ease the addition of alternative
* implementations of SLAB allocators.
* (C) Linux Foundation 2008-2013
* Unified interface for all slab allocators
*/
#ifndef _LINUX_SLAB_H
#define _LINUX_SLAB_H
#include <linux/cache.h>
#include <linux/gfp.h>
#include <linux/overflow.h>
#include <linux/types.h>
#include <linux/workqueue.h>
#include <linux/percpu-refcount.h>
#include <linux/cleanup.h>
#include <linux/hash.h>
#include <linux/autoslab.h>
#include <linux/vmalloc.h>
#include <linux/err.h>
enum _slab_flag_bits {
_SLAB_NO_SANITIZE,
#ifdef CONFIG_PAX_USERCOPY
_SLAB_USERCOPY,
#endif
#ifdef CONFIG_PAX_EXACT_USERCOPY
_SLAB_EXACT_USERCOPY,
#endif
#ifdef CONFIG_PAX_KERNSEAL
_SLAB_SEALED,
_SLAB_HIDDEN,
#endif
#ifdef CONFIG_GRKERNSEC_SLAB_OBJREUSE_HARDEN
_SLAB_RAND_PAD,
#endif
#ifdef AUTOSLAB_PLUGIN
_SLAB_MAY_LEAK,
#endif
_SLAB_CONSISTENCY_CHECKS,
_SLAB_RED_ZONE,
_SLAB_POISON,
_SLAB_KMALLOC,
_SLAB_HWCACHE_ALIGN,
_SLAB_CACHE_DMA,
_SLAB_CACHE_DMA32,
_SLAB_STORE_USER,
_SLAB_PANIC,
_SLAB_TYPESAFE_BY_RCU,
_SLAB_MEM_SPREAD,
_SLAB_TRACE,
#ifdef CONFIG_DEBUG_OBJECTS
_SLAB_DEBUG_OBJECTS,
#endif
_SLAB_NOLEAKTRACE,
_SLAB_NO_MERGE,
#ifdef CONFIG_FAILSLAB
_SLAB_FAILSLAB,
#endif
#ifdef CONFIG_MEMCG_KMEM
_SLAB_ACCOUNT,
#endif
#ifdef CONFIG_KASAN_GENERIC
_SLAB_KASAN,
#endif
_SLAB_NO_USER_FLAGS,
#ifdef CONFIG_KFENCE
_SLAB_SKIP_KFENCE,
#endif
#ifndef CONFIG_SLUB_TINY
_SLAB_RECLAIM_ACCOUNT,
#endif
_SLAB_OBJECT_POISON,
_SLAB_CMPXCHG_DOUBLE,
_SLAB_FLAGS_LAST_BIT
};
static_assert(_SLAB_FLAGS_LAST_BIT <= 8 * sizeof(slab_flags_t));
#define __SLAB_FLAG_BIT(nr) ((slab_flags_t __force)(1U << (nr)))
#define __SLAB_FLAG_UNUSED ((slab_flags_t __force)(0U))
/*
* Flags to pass to kmem_cache_create().
* The ones marked DEBUG are only valid if CONFIG_DEBUG_SLAB is set.
*/
/* PaX: Do not sanitize objs on free */
#define SLAB_NO_SANITIZE __SLAB_FLAG_BIT(_SLAB_NO_SANITIZE)
#ifdef CONFIG_PAX_USERCOPY
/* PaX: Allow copying objs to/from userland */
#define SLAB_USERCOPY __SLAB_FLAG_BIT(_SLAB_USERCOPY)
#else
#define SLAB_USERCOPY __SLAB_FLAG_UNUSED
#endif
#ifdef CONFIG_PAX_EXACT_USERCOPY
/* PaX: Precise USERCOPY checks for kmalloc */
#define SLAB_EXACT_USERCOPY __SLAB_FLAG_BIT(_SLAB_EXACT_USERCOPY)
#else
/* PaX: Precise USERCOPY checks for kmalloc */
#define SLAB_EXACT_USERCOPY __SLAB_FLAG_UNUSED
#endif
#ifdef CONFIG_GRKERNSEC_SLAB_OBJREUSE_HARDEN
/* PaX: random padding between objects */
#define SLAB_RAND_PAD __SLAB_FLAG_BIT(_SLAB_RAND_PAD)
#else
#define SLAB_RAND_PAD __SLAB_FLAG_UNUSED
#endif
#ifdef CONFIG_PAX_KERNSEAL
#define SLAB_SEALED __SLAB_FLAG_BIT(_SLAB_SEALED)
#define SLAB_HIDDEN __SLAB_FLAG_BIT(_SLAB_HIDDEN)
#else
#define SLAB_SEALED __SLAB_FLAG_UNUSED
#define SLAB_HIDDEN __SLAB_FLAG_UNUSED
#endif
#ifdef AUTOSLAB_PLUGIN
#define SLAB_MAY_LEAK __SLAB_FLAG_BIT(_SLAB_MAY_LEAK)
#else
#define SLAB_MAY_LEAK __SLAB_FLAG_UNUSED
#endif
/* DEBUG: Perform (expensive) checks on alloc/free */
#define SLAB_CONSISTENCY_CHECKS __SLAB_FLAG_BIT(_SLAB_CONSISTENCY_CHECKS)
/* DEBUG: Red zone objs in a cache */
#define SLAB_RED_ZONE __SLAB_FLAG_BIT(_SLAB_RED_ZONE)
/* DEBUG: Poison objects */
#define SLAB_POISON __SLAB_FLAG_BIT(_SLAB_POISON)
/* Indicate a kmalloc slab */
#define SLAB_KMALLOC __SLAB_FLAG_BIT(_SLAB_KMALLOC)
/* Align objs on cache lines */
#define SLAB_HWCACHE_ALIGN __SLAB_FLAG_BIT(_SLAB_HWCACHE_ALIGN)
/* Use GFP_DMA memory */
#define SLAB_CACHE_DMA __SLAB_FLAG_BIT(_SLAB_CACHE_DMA)
/* Use GFP_DMA32 memory */
#define SLAB_CACHE_DMA32 __SLAB_FLAG_BIT(_SLAB_CACHE_DMA32)
/* DEBUG: Store the last owner for bug hunting */
#define SLAB_STORE_USER __SLAB_FLAG_BIT(_SLAB_STORE_USER)
/* Panic if kmem_cache_create() fails */
#define SLAB_PANIC __SLAB_FLAG_BIT(_SLAB_PANIC)
/*
* SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS!
*
* This delays freeing the SLAB page by a grace period, it does _NOT_
* delay object freeing. This means that if you do kmem_cache_free()
* that memory location is free to be reused at any time. Thus it may
* be possible to see another object there in the same RCU grace period.
*
* This feature only ensures the memory location backing the object
* stays valid, the trick to using this is relying on an independent
* object validation pass. Something like:
*
* begin:
* rcu_read_lock();
* obj = lockless_lookup(key);
* if (obj) {
* if (!try_get_ref(obj)) // might fail for free objects
* rcu_read_unlock();
* goto begin;
*
* if (obj->key != key) { // not the object we expected
* put_ref(obj);
* rcu_read_unlock();
* goto begin;
* }
* }
* rcu_read_unlock();
*
* This is useful if we need to approach a kernel structure obliquely,
* from its address obtained without the usual locking. We can lock
* the structure to stabilize it and check it's still at the given address,
* only if we can be sure that the memory has not been meanwhile reused
* for some other kind of object (which our subsystem's lock might corrupt).
*
* rcu_read_lock before reading the address, then rcu_read_unlock after
* taking the spinlock within the structure expected at that address.
*
* Note that it is not possible to acquire a lock within a structure
* allocated with SLAB_TYPESAFE_BY_RCU without first acquiring a reference
* as described above. The reason is that SLAB_TYPESAFE_BY_RCU pages
* are not zeroed before being given to the slab, which means that any
* locks must be initialized after each and every kmem_struct_alloc().
* Alternatively, make the ctor passed to kmem_cache_create() initialize
* the locks at page-allocation time, as is done in __i915_request_ctor(),
* sighand_ctor(), and anon_vma_ctor(). Such a ctor permits readers
* to safely acquire those ctor-initialized locks under rcu_read_lock()
* protection.
*
* Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU.
*/
/* Defer freeing slabs to RCU */
#define SLAB_TYPESAFE_BY_RCU __SLAB_FLAG_BIT(_SLAB_TYPESAFE_BY_RCU)
/* Spread some memory over cpuset */
#define SLAB_MEM_SPREAD __SLAB_FLAG_BIT(_SLAB_MEM_SPREAD)
/* Trace allocations and frees */
#define SLAB_TRACE __SLAB_FLAG_BIT(_SLAB_TRACE)
/* Flag to prevent checks on free */
#ifdef CONFIG_DEBUG_OBJECTS
# define SLAB_DEBUG_OBJECTS __SLAB_FLAG_BIT(_SLAB_DEBUG_OBJECTS)
#else
# define SLAB_DEBUG_OBJECTS __SLAB_FLAG_UNUSED
#endif
/* Avoid kmemleak tracing */
#define SLAB_NOLEAKTRACE __SLAB_FLAG_BIT(_SLAB_NOLEAKTRACE)
/*
* Prevent merging with compatible kmem caches. This flag should be used
* cautiously. Valid use cases:
*
* - caches created for self-tests (e.g. kunit)
* - general caches created and used by a subsystem, only when a
* (subsystem-specific) debug option is enabled
* - performance critical caches, should be very rare and consulted with slab
* maintainers, and not used together with CONFIG_SLUB_TINY
*/
#define SLAB_NO_MERGE __SLAB_FLAG_BIT(_SLAB_NO_MERGE)
/* Fault injection mark */
#ifdef CONFIG_FAILSLAB
# define SLAB_FAILSLAB __SLAB_FLAG_BIT(_SLAB_FAILSLAB)
#else
# define SLAB_FAILSLAB __SLAB_FLAG_UNUSED
#endif
/* Account to memcg */
#ifdef CONFIG_MEMCG_KMEM
# define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT)
#else
# define SLAB_ACCOUNT __SLAB_FLAG_UNUSED
#endif
#ifdef CONFIG_KASAN_GENERIC
#define SLAB_KASAN __SLAB_FLAG_BIT(_SLAB_KASAN)
#else
#define SLAB_KASAN __SLAB_FLAG_UNUSED
#endif
/*
* Beware that SLAB and SLUB use the two bits 0x4000000000
* and 0x8000000000 for internal purposes!
*/
/*
* Ignore user specified debugging flags.
* Intended for caches created for self-tests so they have only flags
* specified in the code and other flags are ignored.
*/
#define SLAB_NO_USER_FLAGS __SLAB_FLAG_BIT(_SLAB_NO_USER_FLAGS)
#ifdef CONFIG_KFENCE
#define SLAB_SKIP_KFENCE __SLAB_FLAG_BIT(_SLAB_SKIP_KFENCE)
#else
#define SLAB_SKIP_KFENCE __SLAB_FLAG_UNUSED
#endif
/* The following flags affect the page allocator grouping pages by mobility */
/* Objects are reclaimable */
#ifndef CONFIG_SLUB_TINY
#define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_BIT(_SLAB_RECLAIM_ACCOUNT)
#else
#define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_UNUSED
#endif
#define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */
/*
* freeptr_t represents a SLUB freelist pointer, which might be encoded
* and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled.
*/
typedef struct { unsigned long v; } freeptr_t;
/*
* ZERO_SIZE_PTR will be returned for zero sized kmalloc requests.
*
* Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault.
*
* ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can.
* Both make kfree a no-op.
*/
#define ZERO_SIZE_PTR ((void *)(-MAX_ERRNO-1L))
static_assert(MAX_ERRNO & ~PAGE_MASK);
#define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) - 1 >= (unsigned long)ZERO_SIZE_PTR - 1)
#include <linux/kasan.h>
struct list_lru;
struct mem_cgroup;
/*
* struct kmem_cache related prototypes
*/
bool slab_is_available(void);
/**
* struct kmem_cache_args - Less common arguments for kmem_cache_create()
*
* Any uninitialized fields of the structure are interpreted as unused. The
* exception is @freeptr_offset where %0 is a valid value, so
* @use_freeptr_offset must be also set to %true in order to interpret the field
* as used. For @useroffset %0 is also valid, but only with non-%0
* @usersize.
*
* When %NULL args is passed to kmem_cache_create(), it is equivalent to all
* fields unused.
*/
struct kmem_cache_args {
/**
* @typename: The name of the underlying object type.
*
* %NULL means unknown type or not a struct type.
*/
const char *typename;
/**
* @typesize: The size of the underlying object type.
*
* This may be different from the slab object size in case of varsize*
* AUTOSLABs of arrays.
*
* %0 means unknown type or not a struct type.
*/
unsigned int typesize;
/**
* @align: The required alignment for the objects.
*
* %0 means no specific alignment is requested.
*/
unsigned int align;
/**
* @useroffset: USERCOPY region offset.
*
* %0 is a valid offset, when @usersize is non-%0
*/
unsigned int useroffset;
/**
* @usersize: USERCOPY region size.
*
* %0 means no USERCOPY region is specified.
*/
unsigned int usersize;
/**
* @freeptr_offset: Custom offset for the free pointer
* in &SLAB_TYPESAFE_BY_RCU caches
*
* By default &SLAB_TYPESAFE_BY_RCU caches place the free pointer
* outside of the object. This might cause the object to grow in size.
* Cache creators that have a reason to avoid this can specify a custom
* free pointer offset in their struct where the free pointer will be
* placed.
*
* Note that placing the free pointer inside the object requires the
* caller to ensure that no fields are invalidated that are required to
* guard against object recycling (See &SLAB_TYPESAFE_BY_RCU for
* details).
*
* Using %0 as a value for @freeptr_offset is valid. If @freeptr_offset
* is specified, %use_freeptr_offset must be set %true.
*
* Note that @ctor currently isn't supported with custom free pointers
* as a @ctor requires an external free pointer.
*/
unsigned int freeptr_offset;
/**
* @use_freeptr_offset: Whether a @freeptr_offset is used.
*/
bool use_freeptr_offset;
/**
* @ctor: A constructor for the objects.
*
* The constructor is invoked for each object in a newly allocated slab
* page. It is the cache user's responsibility to free object in the
* same state as after calling the constructor, or deal appropriately
* with any differences between a freshly constructed and a reallocated
* object.
*
* %NULL means no constructor.
*/
void (*ctor)(void *);
/**
* @print: A callback for interpreting ```%pS``` for objects
*
* If the type of the objects stored in the slab is known then providing
* this callback will allow emitting extra information whenever such an
* object pointer is provided to ```%pS```.
*
* %NULL means no ```%pS``` support for the slab objects.
*/
ssize_t (*print)(char *buffer, size_t buflen, void *obj, const char *name);
};
struct kmem_cache *__kmem_cache_create_args(const char *name,
unsigned int object_size,
struct kmem_cache_args *args,
slab_flags_t flags);
static inline struct kmem_cache *
__kmem_cache_create(const char *name, unsigned int size, unsigned int align,
slab_flags_t flags, void (*ctor)(void *))
{
struct kmem_cache_args kmem_args = {
.align = align,
.ctor = ctor,
};
return __kmem_cache_create_args(name, size, &kmem_args, flags);
}
struct kmem_cache *kmem_cache_create_usercopy_typename_typesize_print(
const char *name, const char *typename,
unsigned int typesize, unsigned int size, unsigned int align,
slab_flags_t flags,
unsigned int useroffset, unsigned int usersize,
void (*ctor)(void *),
ssize_t (*print)(char *buffer, size_t buflen, void *obj, const char *name));
#define kmem_cache_create_usercopy_typename_print(name, typename, size, align, flags, useroffset, usersize, ctor, print) \
kmem_cache_create_usercopy_typename_typesize_print(name, typename, size, size, align, flags, useroffset, usersize, ctor, print) \
#define kmem_cache_create_usercopy_typename(name, typename, size, align, flags, useroffset, usersize, ctor) \
kmem_cache_create_usercopy_typename_print(name, typename, size, align, flags, useroffset, usersize, ctor, NULL) \
/**
* kmem_cache_create_usercopy - Create a kmem cache with a region suitable
* for copying to userspace.
* @name: A string which is used in /proc/slabinfo to identify this cache.
* @size: The size of objects to be created in this cache.
* @align: The required alignment for the objects.
* @flags: SLAB flags
* @useroffset: USERCOPY region offset
* @usersize: USERCOPY region size
* @ctor: A constructor for the objects, or %NULL.
*
* This is a legacy wrapper, new code should use either KMEM_CACHE_USERCOPY()
* if whitelisting a single field is sufficient, or kmem_cache_create() with
* the necessary parameters passed via the args parameter (see
* &struct kmem_cache_args)
*
* Return: a pointer to the cache on success, NULL on failure.
*/
#define kmem_cache_create_usercopy(name, size, align, flags, useroffset, usersize, ctor) \
kmem_cache_create_usercopy_typename(name, "\0" __stringify(size), size, align, flags, useroffset, usersize, ctor)
/* If NULL is passed for @args, use this variant with default arguments. */
static inline struct kmem_cache *
__kmem_cache_default_args(const char *name, unsigned int size,
struct kmem_cache_args *args,
slab_flags_t flags)
{
struct kmem_cache_args kmem_default_args = {};
/* Make sure we don't get passed garbage. */
if (WARN_ON_ONCE(args))
return ERR_PTR(-EINVAL);
return __kmem_cache_create_args(name, size, &kmem_default_args, flags);
}
/**
* kmem_cache_create - Create a kmem cache.
* @__name: A string which is used in /proc/slabinfo to identify this cache.
* @__object_size: The size of objects to be created in this cache.
* @__args: Optional arguments, see &struct kmem_cache_args. Passing %NULL
* means defaults will be used for all the arguments.
*
* This is currently implemented as a macro using ``_Generic()`` to call
* either the new variant of the function, or a legacy one.
*
* The new variant has 4 parameters:
* ``kmem_cache_create(name, object_size, args, flags)``
*
* See __kmem_cache_create_args() which implements this.
*
* The legacy variant has 5 parameters:
* ``kmem_cache_create(name, object_size, align, flags, ctor)``
*
* The align and ctor parameters map to the respective fields of
* &struct kmem_cache_args
*
* Context: Cannot be called within a interrupt, but can be interrupted.
*
* Return: a pointer to the cache on success, NULL on failure.
*/
#define kmem_cache_create(__name, __object_size, __args, ...) \
_Generic((__args), \
struct kmem_cache_args *: __kmem_cache_create_args, \
void *: __kmem_cache_default_args, \
default: __kmem_cache_create)(__name, __object_size, __args, __VA_ARGS__)
int kmem_cache_destroy(struct kmem_cache *s);
int kmem_cache_shrink(struct kmem_cache *s);
#if defined(AUTOSLAB_PLUGIN) && !defined(CONFIG_KASAN)
void autoslab_shrink_caches(void);
#else
/* PaX: avoid boot-time cache shrinking due to its performance impact under KASAN */
static inline void autoslab_shrink_caches(void) { }
#endif
#ifdef AUTOSLAB_PLUGIN
void __autoslab_shrink_caches(union autoslab_caches *autoslabs_start,
union autoslab_caches *autoslabs_stop,
union autoslab_caches *autoslabs_init_start,
union autoslab_caches *autoslabs_init_stop,
const char *name);
#endif
/*
* Please use this macro to create slab caches. Simply specify the
* name of the structure and maybe some flags that are listed above.
*
* The alignment of the struct determines object alignment. If you
* f.e. add ____cacheline_aligned_in_smp to the struct declaration
* then the objects will be properly aligned in SMP configurations.
*/
#define KMEM_CACHE(__struct, __flags) \
__kmem_cache_create_args(#__struct, sizeof(struct __struct), \
&(struct kmem_cache_args) { \
.typename = #__struct, \
.align = __alignof__(struct __struct), \
}, (__flags))
/*
* To whitelist a single field for copying to/from usercopy, use this
* macro instead for KMEM_CACHE() above.
*/
#define KMEM_CACHE_USERCOPY(__struct, __flags, __field) \
__kmem_cache_create_args(#__struct, sizeof(struct __struct), \
&(struct kmem_cache_args) { \
.typename = #__struct, \
.align = __alignof__(struct __struct), \
.useroffset = offsetof(struct __struct, __field), \
.usersize = sizeof_field(struct __struct, __field), \
}, (__flags))
#define KMEM_CACHE_USERCOPY_PRINT(__struct, __flags, __field, __print) \
__kmem_cache_create_args(#__struct, sizeof(struct __struct), \
&(struct kmem_cache_args) { \
.typename = #__struct, \
.align = __alignof__(struct __struct), \
.useroffset = offsetof(struct __struct, __field), \
.usersize = sizeof_field(struct __struct, __field), \
.print = __print, \
}, (__flags))
/*
* Common kmalloc functions provided by all allocators
*/
void * __must_check krealloc(const void *objp, size_t new_size, gfp_t flags) __realloc_size(2);
void kfree(const void *objp);
void kfree_sensitive(const void *objp);
size_t __ksize(const void *objp);
DEFINE_FREE(kfree, void *, if (!IS_ERR_OR_NULL(_T)) kfree(_T))
/**
* ksize - Report actual allocation size of associated object
*
* @objp: Pointer returned from a prior kmalloc()-family allocation.
*
* This should not be used for writing beyond the originally requested
* allocation size. Either use krealloc() or round up the allocation size
* with kmalloc_size_roundup() prior to allocation. If this is used to
* access beyond the originally requested allocation size, UBSAN_BOUNDS
* and/or FORTIFY_SOURCE may trip, since they only know about the
* originally allocated size via the __alloc_size attribute.
*/
size_t ksize(const void *objp);
bool is_usercopy_object(const void *ptr);
#ifdef CONFIG_PRINTK
bool kmem_dump_obj(void *object);
#else
static inline bool kmem_dump_obj(void *object) { return false; }
#endif
/*
* Some archs want to perform DMA into kmalloc caches and need a guaranteed
* alignment larger than the alignment of a 64-bit integer.
* Setting ARCH_DMA_MINALIGN in arch headers allows that.
*/
#ifdef ARCH_HAS_DMA_MINALIGN
#if ARCH_DMA_MINALIGN > 8 && !defined(ARCH_KMALLOC_MINALIGN)
#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN
#endif
#endif
#ifndef ARCH_KMALLOC_MINALIGN
#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
#elif ARCH_KMALLOC_MINALIGN > 8
#define KMALLOC_MIN_SIZE ARCH_KMALLOC_MINALIGN
#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
#endif
/*
* Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment.
* Intended for arches that get misalignment faults even for 64 bit integer
* aligned buffers.
*/
#ifndef ARCH_SLAB_MINALIGN
#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
#endif
/*
* Arches can define this function if they want to decide the minimum slab
* alignment at runtime. The value returned by the function must be a power
* of two and >= ARCH_SLAB_MINALIGN.
*/
#ifndef arch_slab_minalign
static inline unsigned int arch_slab_minalign(void)
{
return ARCH_SLAB_MINALIGN;
}
#endif
/*
* kmem_cache_alloc and friends return pointers aligned to ARCH_SLAB_MINALIGN.
* kmalloc and friends return pointers aligned to both ARCH_KMALLOC_MINALIGN
* and ARCH_SLAB_MINALIGN, but here we only assume the former alignment.
*/
#define __assume_kmalloc_alignment __assume_aligned(ARCH_KMALLOC_MINALIGN)
#define __assume_slab_alignment __assume_aligned(ARCH_SLAB_MINALIGN)
#define __assume_page_alignment __assume_aligned(PAGE_SIZE)
/*
* Kmalloc array related definitions
*/
#ifdef CONFIG_SLAB
/*
* SLAB and SLUB directly allocates requests fitting in to an order-1 page
* (PAGE_SIZE*2). Larger requests are passed to the page allocator.
*/
#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)
#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT)
#ifndef KMALLOC_SHIFT_LOW
#define KMALLOC_SHIFT_LOW 5
#endif
#endif
#ifdef CONFIG_SLUB
#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)
#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT)
#ifndef KMALLOC_SHIFT_LOW
#define KMALLOC_SHIFT_LOW 3
#endif
#endif
/* Maximum allocatable size */
#define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX)
/* Maximum size for which we actually use a slab cache */
#define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH)
/* Maximum order allocatable via the slab allocator */
#define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT)
/*
* Kmalloc subsystem.
*/
#ifndef KMALLOC_MIN_SIZE
#define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW)
#endif
/*
* This restriction comes from byte sized index implementation.
* Page size is normally 2^12 bytes and, in this case, if we want to use
* byte sized index which can represent 2^8 entries, the size of the object
* should be equal or greater to 2^12 / 2^8 = 2^4 = 16.
* If minimum size of kmalloc is less than 16, we use it as minimum object
* size and give up to use byte sized index.
*/
#define SLAB_OBJ_MIN_SIZE (KMALLOC_MIN_SIZE < 16 ? \
(KMALLOC_MIN_SIZE) : 16)
#ifdef CONFIG_RANDOM_KMALLOC_CACHES
#define RANDOM_KMALLOC_CACHES_NR 15 // # of cache copies
#else
#define RANDOM_KMALLOC_CACHES_NR 0
#endif
/*
* Whenever changing this, take care of that kmalloc_type() and
* create_kmalloc_caches() still work as intended.
*
* KMALLOC_NORMAL can contain only unaccounted objects whereas KMALLOC_CGROUP
* is for accounted but unreclaimable and non-dma objects. All the other
* kmem caches can have both accounted and unaccounted objects.
*/
enum kmalloc_cache_type {
KMALLOC_NORMAL = 0,
KMALLOC_RANDOM_START = KMALLOC_NORMAL,
KMALLOC_RANDOM_END = KMALLOC_RANDOM_START + RANDOM_KMALLOC_CACHES_NR,
#ifdef CONFIG_ZONE_DMA
KMALLOC_DMA,
#endif
#ifdef CONFIG_ZONE_DMA32
KMALLOC_DMA32,
#endif
#ifndef CONFIG_SLUB_TINY
KMALLOC_RECLAIM,
#endif
#ifdef CONFIG_MEMCG_KMEM
KMALLOC_CGROUP,
#endif
#ifdef CONFIG_PAX_USERCOPY
KMALLOC_USERCOPY,
#ifdef CONFIG_PAX_EXACT_USERCOPY
KMALLOC_EXACT_USERCOPY,
#endif
#endif
#ifdef CONFIG_PAX_KERNSEAL
KMALLOC_SEALED,
#endif
#if !defined(CONFIG_SLUB_TINY) && defined(CONFIG_PAX_KERNSEAL)
KMALLOC_SEALED_RECLAIM,
#endif
#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_PAX_KERNSEAL)
KMALLOC_SEALED_CGROUP,
#endif
NR_KMALLOC_TYPES
};
extern struct kmem_cache *
kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1];
/*
* Define gfp bits that should not be set for KMALLOC_NORMAL.
*/
#define KMALLOC_NOT_NORMAL_BITS \
(__GFP_RECLAIMABLE | \
(IS_ENABLED(CONFIG_ZONE_DMA) ? __GFP_DMA : 0) | \
(IS_ENABLED(CONFIG_MEMCG_KMEM) ? __GFP_ACCOUNT : 0))
extern unsigned long random_kmalloc_seed;
#ifdef CONFIG_PAX_EXACT_USERCOPY
void pax_store_exact_size(struct kmem_cache *cachep, void *obj, size_t size);
#else
static inline void pax_store_exact_size(struct kmem_cache *cachep, void *obj, size_t size) {}
#endif
static __always_inline __size_overflow(1) unsigned int __kmalloc_index(size_t size, bool size_is_constant);
static __always_inline __pure enum kmalloc_cache_type kmalloc_type(size_t size, gfp_t flags, unsigned long caller)
{
#ifndef CONFIG_ZONE_DMA
flags &= ~__GFP_DMA;
#endif
#ifndef CONFIG_ZONE_DMA32
flags &= ~__GFP_DMA32;
#endif
#ifndef CONFIG_PAX_USERCOPY
flags &= ~__GFP_USERCOPY;
#elif defined(AUTOSLAB_PLUGIN)
flags &= ~(GFP_USERCOPY | GFP_USER);
#endif
#ifndef CONFIG_PAX_KERNSEAL
flags &= ~__GFP_SEALED;
#endif
switch (flags & (__GFP_DMA | __GFP_DMA32 | __GFP_RECLAIMABLE | GFP_USERCOPY)) {
/*
* At least one of the flags has to be set. Their priorities in
* decreasing order are:
* 1) __GFP_DMA
* 2) __GFP_RECLAIMABLE
* 3) __GFP_ACCOUNT
*/
#ifdef CONFIG_ZONE_DMA
case __GFP_DMA | __GFP_RECLAIMABLE | GFP_USERCOPY:
case __GFP_DMA | __GFP_RECLAIMABLE:
case __GFP_DMA | GFP_USERCOPY:
case __GFP_DMA:
return KMALLOC_DMA;
#endif
#ifdef CONFIG_ZONE_DMA32
case __GFP_DMA32 | __GFP_RECLAIMABLE | GFP_USERCOPY:
case __GFP_DMA32 | __GFP_RECLAIMABLE:
case __GFP_DMA32 | GFP_USERCOPY:
case __GFP_DMA32:
return KMALLOC_DMA32;
#endif
#ifdef CONFIG_PAX_USERCOPY
case __GFP_RECLAIMABLE | GFP_USERCOPY:
case GFP_USERCOPY:
return KMALLOC_USERCOPY;
#endif
#ifndef CONFIG_SLUB_TINY
case __GFP_RECLAIMABLE:
#ifdef CONFIG_PAX_KERNSEAL
return flags & __GFP_SEALED ? KMALLOC_SEALED_RECLAIM : KMALLOC_RECLAIM;
#else
return KMALLOC_RECLAIM;
#endif
#endif
default:
#ifdef CONFIG_PAX_USERCOPY
if ((flags & GFP_USER) == GFP_USER)
return KMALLOC_USERCOPY;
#endif
#if defined(CONFIG_MEMCG_KMEM) && !defined(AUTOSLAB_PLUGIN)
if (flags & __GFP_ACCOUNT)
#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_PAX_KERNSEAL)
return flags & __GFP_SEALED ? KMALLOC_SEALED_CGROUP : KMALLOC_CGROUP;
#else
return KMALLOC_CGROUP;
#endif
#endif
#ifdef CONFIG_PAX_KERNSEAL
if (flags & __GFP_SEALED)
return KMALLOC_SEALED;
#endif
#ifdef CONFIG_PAX_EXACT_USERCOPY
if (size <= kmalloc_caches[KMALLOC_EXACT_USERCOPY][__kmalloc_index(size, true)]->usersize)
return KMALLOC_EXACT_USERCOPY;
#endif
#ifdef CONFIG_RANDOM_KMALLOC_CACHES
/* RANDOM_KMALLOC_CACHES_NR (=15) copies + the KMALLOC_NORMAL */
return KMALLOC_RANDOM_START + hash_64(caller ^ random_kmalloc_seed,
ilog2(RANDOM_KMALLOC_CACHES_NR + 1));
#else
return KMALLOC_NORMAL;
#endif
}
}
/*
* Figure out which kmalloc slab an allocation of a certain size
* belongs to.
* 0 = zero alloc
* 1 = 65 .. 96 bytes
* 2 = 129 .. 192 bytes
* n = 2^(n-1)+1 .. 2^n
*
* Note: __kmalloc_index() is compile-time optimized, and not runtime optimized;
* typical usage is via kmalloc_index() and therefore evaluated at compile-time.
* Callers where !size_is_constant should only be test modules, where runtime
* overheads of __kmalloc_index() can be tolerated. Also see kmalloc_slab().
*/
static __always_inline __size_overflow(1) unsigned int __kmalloc_index(size_t size,
bool size_is_constant)
{
if (!size)
return 0;
if (size <= KMALLOC_MIN_SIZE)
return KMALLOC_SHIFT_LOW;
if (!IS_ENABLED(AUTOSLAB_PLUGIN) && KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
return 1;
if (!IS_ENABLED(AUTOSLAB_PLUGIN) && KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
return 2;
if (size <= 8) return 3;
if (size <= 16) return 4;
if (size <= 32) return 5;
if (size <= 64) return 6;
if (size <= 128) return 7;
if (size <= 256) return 8;
if (size <= 512) return 9;
if (size <= 1024) return 10;
if (size <= 2 * 1024) return 11;
if (size <= 4 * 1024) return 12;
if (size <= 8 * 1024) return 13;
if (size <= 16 * 1024) return 14;
if (size <= 32 * 1024) return 15;
if (size <= 64 * 1024) return 16;
if (size <= 128 * 1024) return 17;
if (size <= 256 * 1024) return 18;
if (size <= 512 * 1024) return 19;
if (size <= 1024 * 1024) return 20;
if (size <= 2 * 1024 * 1024) return 21;
if (0 && !IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant)
BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()");
else
BUG();
/* Will never be reached. Needed because the compiler may complain */
return -1;
}
static_assert(PAGE_SHIFT <= 20);
#define kmalloc_index(s) __kmalloc_index(s, true)
void *__kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __malloc __alloc_size(1) __alloc_gfp_flags(2);
/**
* kmem_cache_alloc - Allocate an object
* @cachep: The cache to allocate from.
* @flags: See kmalloc().
*
* Allocate an object from this cache.
* See kmem_cache_zalloc() for a shortcut of adding __GFP_ZERO to flags.
*
* Return: pointer to the new object or %NULL in case of error
*/
void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) __assume_slab_alignment __malloc __alloc_gfp_flags(2);
void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, gfp_t gfpflags) __assume_slab_alignment __malloc __alloc_gfp_flags(3);
#ifdef AUTOSLAB_PLUGIN
void *kmem_cache_alloc_type(struct kmem_cache **, gfp_t flags) __assume_slab_alignment __malloc __alloc_gfp_flags(2);
void *kmem_cache_alloc_index(struct kmem_cache **, size_t size, gfp_t flags, unsigned int minshift, unsigned int maxshift) __assume_slab_alignment __malloc __alloc_size(2) __alloc_gfp_flags(3);
void *kmem_cache_alloc_index_type(struct kmem_cache **, size_t size, gfp_t flags, unsigned int minshift, unsigned int maxshift) __assume_slab_alignment __malloc __alloc_size(2) __alloc_gfp_flags(3);
#endif
void kmem_cache_free(struct kmem_cache *s, void *objp);
/*
* Bulk allocation and freeing operations. These are accelerated in an
* allocator specific way to avoid taking locks repeatedly or building
* metadata structures unnecessarily.
*
* Note that interrupts must be enabled when calling these functions.
*/
void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, void **p);
static __always_inline void kfree_bulk(size_t size, void **p)
{
kmem_cache_free_bulk(NULL, size, p);
}
void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment __malloc __alloc_size(1) __alloc_gfp_flags(2) __alloc_node(3);
void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node) __assume_slab_alignment __malloc __alloc_gfp_flags(2) __alloc_node(3);
#ifdef AUTOSLAB_PLUGIN
void *kmem_cache_alloc_node_type(struct kmem_cache **, gfp_t flags, int node) __assume_slab_alignment __malloc __alloc_gfp_flags(2) __alloc_node(3);
void *kmem_cache_alloc_node_index(struct kmem_cache **, size_t size, gfp_t flags, int node, unsigned int minshift, unsigned int maxshift) __assume_slab_alignment __malloc __alloc_size(2) __alloc_gfp_flags(3) __alloc_node(4);
void *kmem_cache_alloc_node_index_type(struct kmem_cache **, size_t size, gfp_t flags, int node, unsigned int minshift, unsigned int maxshift) __assume_slab_alignment __malloc __alloc_size(2) __alloc_gfp_flags(3) __alloc_node(4);
#endif
void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size) __assume_kmalloc_alignment __malloc __alloc_gfp_flags(2) __alloc_size(3);
void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, int node, size_t size) __assume_kmalloc_alignment __malloc __alloc_gfp_flags(2) __alloc_node(3) __alloc_size(4);
void *kmalloc_large(size_t size, gfp_t flags) __assume_page_alignment __alloc_size(1) __alloc_gfp_flags(2);
void *kmalloc_large_node(size_t size, gfp_t flags, int node) __assume_page_alignment __alloc_size(1) __alloc_gfp_flags(2) __alloc_node(3);
/**
* kmalloc - allocate kernel memory
* @size: how many bytes of memory are required.
* @flags: describe the allocation context
*
* kmalloc is the normal method of allocating memory
* for objects smaller than page size in the kernel.
*
* The allocated object address is aligned to at least ARCH_KMALLOC_MINALIGN
* bytes. For @size of power of two bytes, the alignment is also guaranteed
* to be at least to the size. For other sizes, the alignment is guaranteed to
* be at least the largest power-of-two divisor of @size.
*
* The @flags argument may be one of the GFP flags defined at
* include/linux/gfp_types.h and described at
* :ref:`Documentation/core-api/mm-api.rst <mm-api-gfp-flags>`
*
* The recommended usage of the @flags is described at
* :ref:`Documentation/core-api/memory-allocation.rst <memory_allocation>`
*
* Below is a brief outline of the most useful GFP flags
*
* %GFP_KERNEL
* Allocate normal kernel ram. May sleep.
*
* %GFP_NOWAIT
* Allocation will not sleep.
*
* %GFP_ATOMIC
* Allocation will not sleep. May use emergency pools.
*
* Also it is possible to set different flags by OR'ing
* in one or more of the following additional @flags:
*
* %__GFP_ZERO
* Zero the allocated memory before returning. Also see kzalloc().
*
* %__GFP_HIGH
* This allocation has high priority and may use emergency pools.
*
* %__GFP_NOFAIL
* Indicate that this allocation is in no way allowed to fail
* (think twice before using).
*
* %__GFP_NORETRY
* If memory is not immediately available,
* then give up at once.
*
* %__GFP_NOWARN
* If allocation fails, don't issue any warnings.
*
* %__GFP_RETRY_MAYFAIL
* Try really hard to succeed the allocation but fail
* eventually.
*/
#ifdef AUTOSLAB_PLUGIN
/* !!! keep in sync with the copies in include/linux/string.h and include/linux/sprintf.h !!! */
void *kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __malloc __alloc_size(1) __alloc_gfp_flags(2);
#else
static __always_inline __malloc __alloc_size(1) __alloc_gfp_flags(2)
void *kmalloc(size_t size, gfp_t flags)
{
if (__builtin_constant_p(size) && size) {
unsigned int index;
if (size > INT_MAX)
return NULL;
if (size > KMALLOC_MAX_CACHE_SIZE)
return kmalloc_large(size, flags);
index = kmalloc_index(size);
#ifdef CONFIG_PAX_EXACT_USERCOPY
if ((size < 256 && index == kmalloc_index(size + 1)) ||
(size < 65536 && index == kmalloc_index(size + 2)) ||
index == kmalloc_index(size + 4)) {
struct kmem_cache *cachep = kmalloc_caches[KMALLOC_EXACT_USERCOPY][index];
if (cachep) {
void *obj = kmalloc_trace(cachep, flags, size);
pax_store_exact_size(cachep, obj, size);
return obj;
}
}
#endif
return kmalloc_trace(
kmalloc_caches[kmalloc_type(size, flags, _RET_IP_)][index],
flags, size);
}
return __kmalloc(size, flags);
}
#endif /* AUTOSLAB_PLUGIN */
#ifdef AUTOSLAB_PLUGIN
#ifdef CONFIG_NUMA
void *kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment __malloc __alloc_size(1) __alloc_gfp_flags(2) __alloc_node(3);
#else
#define kmalloc_node(size, flags, node) ({ (void)(node);kmalloc((size), (flags)); })
#endif
#else
static __always_inline __malloc __alloc_size(1) __alloc_gfp_flags(2) __alloc_node(3)
void *kmalloc_node(size_t size, gfp_t flags, int node)
{
if (__builtin_constant_p(size) && size) {
unsigned int index;
if (size > KMALLOC_MAX_CACHE_SIZE)
return kmalloc_large_node(size, flags, node);
index = kmalloc_index(size);
#ifdef CONFIG_PAX_EXACT_USERCOPY
if ((size < 256 && i == kmalloc_index(size + 1)) ||
(size < 65536 && i == kmalloc_index(size + 2)) ||
i == kmalloc_index(size + 4)) {
struct kmem_cache *cachep = kmalloc_caches[KMALLOC_EXACT_USERCOPY][i];
if (cachep) {
void *obj = kmalloc_node_trace(cachep, flags, node, size);
pax_store_exact_size(cachep, obj, size);
return obj;
}
}
#endif
return kmalloc_node_trace(
kmalloc_caches[kmalloc_type(size, flags, _RET_IP_)][index],
flags, node, size);
}
return __kmalloc_node(size, flags, node);
}
#endif /* AUTOSLAB_PLUGIN */
/**
* kmalloc_array - allocate memory for an array.
* @n: number of elements.
* @size: element size.
* @flags: the type of memory to allocate (see kmalloc).
*/
#ifdef AUTOSLAB_PLUGIN
#define kmalloc_array(n, size, autoslab_flags) \
({ \
size_t __n = (n); \
size_t __size = (size); \
size_t bytes; \
\
(unlikely(check_mul_overflow((size_t)(__n), (size_t)(__size), &bytes))) ?\
NULL : \
kmalloc_typename(bytes, (autoslab_flags), "\0" __stringify(size));\
})
#else
static __always_inline __alloc __alloc_size(1, 2) __alloc_gfp_flags(3)
void *kmalloc_array(size_t n, size_t size, gfp_t flags)
{
size_t bytes;
if (unlikely(check_mul_overflow(n, size, &bytes)))
return NULL;
if (bytes > INT_MAX)
return NULL;
return kmalloc(bytes, flags);
}
#endif
/**
* krealloc_array - reallocate memory for an array.
* @p: pointer to the memory chunk to reallocate
* @new_n: new number of elements to alloc
* @new_size: new size of a single member of the array
* @flags: the type of memory to allocate (see kmalloc)
*
* If __GFP_ZERO logic is requested, callers must ensure that, starting with the
* initial memory allocation, every subsequent call to this API for the same
* memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
* __GFP_ZERO is not fully honored by this API.
*
* See krealloc() for further details.
*
* In any case, the contents of the object pointed to are preserved up to the
* lesser of the new and old sizes.
*/
static __always_inline __realloc_size(2, 3) __must_check
void *krealloc_array(void *p, size_t new_n, size_t new_size, gfp_t flags)
{
size_t bytes;
if (unlikely(check_mul_overflow(new_n, new_size, &bytes)))
return NULL;
return krealloc(p, bytes, flags);
}
/**
* kcalloc - allocate memory for an array. The memory is set to zero.
* @n: number of elements.
* @size: element size.
* @flags: the type of memory to allocate (see kmalloc).
*/
#ifdef AUTOSLAB_PLUGIN
#define kcalloc(n, size, flags) kmalloc_array((n), (size), (flags) | __GFP_ZERO)
#else
static __always_inline __malloc __alloc_size(1, 2) __alloc_gfp_flags(3)
void *kcalloc(size_t n, size_t size, gfp_t flags)
{
return kmalloc_array(n, size, flags | __GFP_ZERO);
}
#endif
void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node, unsigned long caller) __malloc __alloc_size(1) __alloc_gfp_flags(2) __alloc_node(3);
#define kmalloc_node_track_caller(size, flags, node) \
__kmalloc_node_track_caller(size, flags, node, \
_RET_IP_)
/*
* kmalloc_track_caller is a special version of kmalloc that records the
* calling function of the routine calling it for slab leak tracking instead
* of just the calling function (confusing, eh?).
* It's useful when the call to kmalloc comes from a widely-used standard
* allocator where we care about the real place the memory allocation
* request comes from.
*/
#define kmalloc_track_caller(size, flags) \
__kmalloc_node_track_caller(size, flags, \
NUMA_NO_NODE, _RET_IP_)
#ifdef AUTOSLAB_PLUGIN
#define kmalloc_array_node(n, size, autoslab_flags, node) ({ \
size_t bytes; \
\
(unlikely(check_mul_overflow((size_t)(n), (size_t)(size), &bytes))) ?\
NULL : \
kmalloc_node(bytes, (autoslab_flags), (node)); \
})
#else
static __always_inline __alloc __alloc_size(1, 2) __alloc_gfp_flags(3) __alloc_node(4)
void *kmalloc_array_node(size_t n, size_t size, gfp_t flags, int node)
{
size_t bytes;
if (unlikely(check_mul_overflow(n, size, &bytes)))
return NULL;
if (__builtin_constant_p(n) && __builtin_constant_p(size))
return kmalloc_node(bytes, flags, node);
return __kmalloc_node(bytes, flags, node);
}
#endif
#ifdef AUTOSLAB_PLUGIN
#define kcalloc_node(n, size, flags, node) kmalloc_array_node((n), (size), (flags) | __GFP_ZERO, (node))
#else
static __always_inline __malloc __alloc_size(1, 2) __alloc_gfp_flags(3) __alloc_node(4)
void *kcalloc_node(size_t n, size_t size, gfp_t flags, int node)
{
return kmalloc_array_node(n, size, flags | __GFP_ZERO, node);
}
#endif
/*
* Shortcuts
*/
static inline void *kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags)
{
return kmem_cache_alloc(k, flags | __GFP_ZERO);
}
static inline void *kmem_cache_zalloc_node(struct kmem_cache *k, gfp_t flags, int node)
{
return kmem_cache_alloc_node(k, flags | __GFP_ZERO, node);
}
/**
* kzalloc - allocate memory. The memory is set to zero.
* @size: how many bytes of memory are required.
* @flags: the type of memory to allocate (see kmalloc).
*/
#ifdef AUTOSLAB_PLUGIN
#define kzalloc(size, flags) kmalloc((size), (flags) | __GFP_ZERO)
#else
static __always_inline __malloc __alloc_size(1) __alloc_gfp_flags(2)
void *kzalloc(size_t size, gfp_t flags)
{
return kmalloc(size, flags | __GFP_ZERO);
}
#endif
/**
* kzalloc_node - allocate zeroed memory from a particular memory node.
* @size: how many bytes of memory are required.
* @flags: the type of memory to allocate (see kmalloc).
* @node: memory node from which to allocate
*/
#ifdef AUTOSLAB_PLUGIN
#define kzalloc_node(size, flags, node) kmalloc_node((size), (flags) | __GFP_ZERO, (node))
#else
static __always_inline __malloc __alloc_size(1) __alloc_gfp_flags(2) __alloc_node(3)
void *kzalloc_node(size_t size, gfp_t flags, int node)
{
return kmalloc_node(size, flags | __GFP_ZERO, node);
}
#endif
static __always_inline gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size)
{
/*
* We want to attempt a large physically contiguous block first because
* it is less likely to fragment multiple larger blocks and therefore
* contribute to a long term fragmentation less than vmalloc fallback.
* However make sure that larger requests are not too disruptive - no
* OOM killer and no allocation failure warnings as we have a fallback.
*/
if (size > PAGE_SIZE) {
flags |= __GFP_NOWARN;
if (!(flags & __GFP_RETRY_MAYFAIL))
flags |= __GFP_NORETRY;
/* nofail semantic is implemented by the vmalloc fallback */
flags &= ~__GFP_NOFAIL;
}
return flags;
}
static __always_inline __malloc __alloc_size(1) __alloc_gfp_flags(2) __alloc_node(3)
void *__kvmalloc_node(size_t size, gfp_t gfp_flags, int node)
{
#ifdef CONFIG_GRKERNSEC
unsigned long vm_flags = 0;
unsigned long align = 1;
#endif
void *ret;
if (__builtin_constant_p(size) && size > INT_MAX)
return NULL;
ret = kmalloc_node(size, kmalloc_gfp_adjust(gfp_flags, size), node);
/*
* It doesn't really make sense to fallback to vmalloc for sub page
* requests
*/
if (ret || size <= PAGE_SIZE)
return ret;
/* non-sleeping allocations are not supported by vmalloc */
if (!gfpflags_allow_blocking(gfp_flags))
return NULL;
/* Don't even allow crazy sizes */
if (unlikely(size > INT_MAX)) {
WARN_ON_ONCE(!(gfp_flags & __GFP_NOWARN));
return NULL;
}
#ifdef CONFIG_GRKERNSEC
/*
* Align non-zeroed requests towards the page end to prevent info leaks
* and make buffer overflows hit the guard page. For zeroed requests
* there's no reason to do so, as there's no opportunity for info leaks.
*
* Keep alignment requirements to what kmalloc() and vmalloc() would
* guarantee.
*/
if (!(gfp_flags & __GFP_ZERO)) {
align = ARCH_KMALLOC_MINALIGN;
vm_flags |= VM_ALIGN_END;
}
ret = __vmalloc_node_flags(size, align, gfp_flags, vm_flags | VM_ALLOW_HUGE_VMAP,
node, __builtin_return_address(0));
/* Wipe possible padding bytes for right aligned allocations. */
if (ret && (vm_flags & VM_ALIGN_END) && (size & (align - 1))) {
unsigned int padding = align - (size & (align - 1));
memset(ret + size - padding, PAX_MEMORY_SANITIZE_VALUE, padding);
}
#else
/*
* kvmalloc() can always use VM_ALLOW_HUGE_VMAP,
* since the callers already cannot assume anything
* about the resulting pointer, and cannot play
* protection games.
*/
ret = __vmalloc_node_flags(size, 1, gfp_flags, VM_ALLOW_HUGE_VMAP,
node, __builtin_return_address(0));
#endif
return ret;
}
#ifdef AUTOSLAB_PLUGIN
static __always_inline __malloc __alloc_size(1) __alloc_gfp_flags(2) __alloc_node(3)
void *kvmalloc_node(size_t size, gfp_t flags, int node)
{
return __kvmalloc_node(size, flags, node);
}
#else
void *kvmalloc_node(size_t size, gfp_t flags, int node) __malloc __alloc_size(1) __alloc_gfp_flags(2) __alloc_node(3);
#endif
/* !!! keep in sync with the copy in include/linux/string.h !!! */
static inline __malloc __alloc_size(1) __alloc_gfp_flags(2)
void *kvmalloc(size_t size, gfp_t flags)
{
return kvmalloc_node(size, flags, NUMA_NO_NODE);
}
static inline __malloc __alloc_size(1) __alloc_gfp_flags(2) __alloc_node(3)
void *kvzalloc_node(size_t size, gfp_t flags, int node)
{
return kvmalloc_node(size, flags | __GFP_ZERO, node);
}
static inline __malloc __alloc_size(1) __alloc_gfp_flags(2)
void *kvzalloc(size_t size, gfp_t flags)
{
return kvmalloc(size, flags | __GFP_ZERO);
}
static inline __alloc __alloc_size(1,2) __alloc_gfp_flags(3)
void *kvmalloc_array_node(size_t n, size_t size, gfp_t flags, int node)
{
size_t bytes;
if (unlikely(check_mul_overflow(n, size, &bytes)))
return NULL;
return kvmalloc_node(bytes, flags, node);
}
static inline __alloc __alloc_size(1,2) __alloc_gfp_flags(3)
void *kvmalloc_array(size_t n, size_t size, gfp_t flags)
{
return kvmalloc_array_node(n, size, flags, NUMA_NO_NODE);
}
static inline __malloc __alloc_size(1,2) __alloc_gfp_flags(3)
void *kvcalloc_node(size_t n, size_t size, gfp_t flags, int node)
{
return kvmalloc_array_node(n, size, flags | __GFP_ZERO, node);
}
static inline __malloc __alloc_size(1,2) __alloc_gfp_flags(3)
void *kvcalloc(size_t n, size_t size, gfp_t flags)
{
return kvmalloc_array(n, size, flags | __GFP_ZERO);
}
void *kvrealloc(const void *p, size_t size, gfp_t flags) __realloc_size(2);
extern void kvfree(const void *addr);
extern void kvfree_sensitive(const void *addr, size_t len);
unsigned int kmem_cache_size(const struct kmem_cache *s);
unsigned int kmem_cache_pad_space(const struct kmem_cache *s);
const char *kmem_cache_name(const struct kmem_cache *s);
const char *kmem_cache_name_safe(const struct kmem_cache *s);
const void *kmem_cache_offset(struct kmem_cache **s, struct folio *folio, const void *ptr);
/**
* kmalloc_size_roundup - Report allocation bucket size for the given size
*
* @size: Number of bytes to round up from.
*
* This returns the number of bytes that would be available in a kmalloc()
* allocation of @size bytes. For example, a 126 byte request would be
* rounded up to the next sized kmalloc bucket, 128 bytes. (This is strictly
* for the general-purpose kmalloc()-based allocations, and is not for the
* pre-sized kmem_cache_alloc()-based allocations.)
*
* Use this to kmalloc() the full bucket size ahead of time instead of using
* ksize() to query the size after an allocation.
*/
size_t kmalloc_size_roundup(size_t size);
void __init kmem_cache_init_late(void);
#ifdef CONFIG_KALLSYMS
ssize_t kmem_cache_print(struct kmem_cache *s, char *buffer, size_t buflen, void *obj, const char *name);
void *__kmem_cache_lookup(struct kmem_cache **s, void *ptr);
const char *kmem_cache_lookup(void *ptr,
unsigned long *size,
unsigned long *offset,
char *buffer,
char **modname);
#endif
#if defined(CONFIG_SMP) && defined(CONFIG_SLAB)
int slab_prepare_cpu(unsigned int cpu);
int slab_dead_cpu(unsigned int cpu);
#else
#define slab_prepare_cpu NULL
#define slab_dead_cpu NULL
#endif
#endif /* _LINUX_SLAB_H */