/* Python's malloc wrappers (see pymem.h) */

#include "Python.h"

#include "pycore_code.h" // stats

#include "pycore_object.h" // _PyDebugAllocatorStats() definition

#include "pycore_obmalloc.h"

#include "pycore_pyerrors.h" // _Py_FatalErrorFormat()

#include "pycore_pymem.h"

#include "pycore_pystate.h" // _PyInterpreterState_GET

#include "pycore_obmalloc_init.h"

#include <stdlib.h> // malloc()

#include <stdbool.h>

#ifdef WITH_MIMALLOC

# include "pycore_mimalloc.h"

# include "mimalloc/static.c"

# include "mimalloc/internal.h" // for stats

#endif

#if defined(Py_GIL_DISABLED) && !defined(WITH_MIMALLOC)

# error "Py_GIL_DISABLED requires WITH_MIMALLOC"

#endif

#undef uint

#define uint pymem_uint

/* Defined in tracemalloc.c */

extern void _PyMem_DumpTraceback(int fd, const void *ptr);

static void _PyObject_DebugDumpAddress(const void *p);

static void _PyMem_DebugCheckAddress(const char *func, char api_id, const void *p);

static void set_up_debug_hooks_domain_unlocked(PyMemAllocatorDomain domain);

static void set_up_debug_hooks_unlocked(void);

static void get_allocator_unlocked(PyMemAllocatorDomain, PyMemAllocatorEx *);

static void set_allocator_unlocked(PyMemAllocatorDomain, PyMemAllocatorEx *);

/***************************************/

/* low-level allocator implementations */

/***************************************/

/* the default raw allocator (wraps malloc) */

void *

_PyMem_RawMalloc(void *Py_UNUSED(ctx), size_t size)

{

/* PyMem_RawMalloc(0) means malloc(1). Some systems would return NULL

for malloc(0), which would be treated as an error. Some platforms would

return a pointer with no memory behind it, which would break pymalloc.

To solve these problems, allocate an extra byte. */

if (size == 0)

size = 1;

return malloc(size);

}

void *

_PyMem_RawCalloc(void *Py_UNUSED(ctx), size_t nelem, size_t elsize)

{

/* PyMem_RawCalloc(0, 0) means calloc(1, 1). Some systems would return NULL

for calloc(0, 0), which would be treated as an error. Some platforms

would return a pointer with no memory behind it, which would break

pymalloc. To solve these problems, allocate an extra byte. */

if (nelem == 0 || elsize == 0) {

nelem = 1;

elsize = 1;

}

return calloc(nelem, elsize);

}

void *

_PyMem_RawRealloc(void *Py_UNUSED(ctx), void *ptr, size_t size)

{

if (size == 0)

size = 1;

return realloc(ptr, size);

}

void

_PyMem_RawFree(void *Py_UNUSED(ctx), void *ptr)

{

free(ptr);

}

#ifdef WITH_MIMALLOC

void *

_PyMem_MiMalloc(void *ctx, size_t size)

{

#ifdef Py_GIL_DISABLED

_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();

mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM];

return mi_heap_malloc(heap, size);

#else

return mi_malloc(size);

#endif

}

void *

_PyMem_MiCalloc(void *ctx, size_t nelem, size_t elsize)

{

#ifdef Py_GIL_DISABLED

_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();

mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM];

return mi_heap_calloc(heap, nelem, elsize);

#else

return mi_calloc(nelem, elsize);

#endif

}

void *

_PyMem_MiRealloc(void *ctx, void *ptr, size_t size)

{

#ifdef Py_GIL_DISABLED

_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();

mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM];

return mi_heap_realloc(heap, ptr, size);

#else

return mi_realloc(ptr, size);

#endif

}

void

_PyMem_MiFree(void *ctx, void *ptr)

{

mi_free(ptr);

}

void *

_PyObject_MiMalloc(void *ctx, size_t nbytes)

{

#ifdef Py_GIL_DISABLED

_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();

mi_heap_t *heap = tstate->mimalloc.current_object_heap;

return mi_heap_malloc(heap, nbytes);

#else

return mi_malloc(nbytes);

#endif

}

void *

_PyObject_MiCalloc(void *ctx, size_t nelem, size_t elsize)

{

#ifdef Py_GIL_DISABLED

_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();

mi_heap_t *heap = tstate->mimalloc.current_object_heap;

return mi_heap_calloc(heap, nelem, elsize);

#else

return mi_calloc(nelem, elsize);

#endif

}

void *

_PyObject_MiRealloc(void *ctx, void *ptr, size_t nbytes)

{

#ifdef Py_GIL_DISABLED

_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();

mi_heap_t *heap = tstate->mimalloc.current_object_heap;

return mi_heap_realloc(heap, ptr, nbytes);

#else

return mi_realloc(ptr, nbytes);

#endif

}

void

_PyObject_MiFree(void *ctx, void *ptr)

{

mi_free(ptr);

}

#endif // WITH_MIMALLOC

#define MALLOC_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree}

#ifdef WITH_MIMALLOC

# define MIMALLOC_ALLOC {NULL, _PyMem_MiMalloc, _PyMem_MiCalloc, _PyMem_MiRealloc, _PyMem_MiFree}

# define MIMALLOC_OBJALLOC {NULL, _PyObject_MiMalloc, _PyObject_MiCalloc, _PyObject_MiRealloc, _PyObject_MiFree}

#endif

/* the pymalloc allocator */

// The actual implementation is further down.

#if defined(WITH_PYMALLOC)

void* _PyObject_Malloc(void *ctx, size_t size);

void* _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize);

void _PyObject_Free(void *ctx, void *p);

void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);

# define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free}

#endif // WITH_PYMALLOC

#if defined(Py_GIL_DISABLED)

// Py_GIL_DISABLED requires using mimalloc for "mem" and "obj" domains.

# define PYRAW_ALLOC MALLOC_ALLOC

# define PYMEM_ALLOC MIMALLOC_ALLOC

# define PYOBJ_ALLOC MIMALLOC_OBJALLOC

#elif defined(WITH_PYMALLOC)

# define PYRAW_ALLOC MALLOC_ALLOC

# define PYMEM_ALLOC PYMALLOC_ALLOC

# define PYOBJ_ALLOC PYMALLOC_ALLOC

#else

# define PYRAW_ALLOC MALLOC_ALLOC

# define PYMEM_ALLOC MALLOC_ALLOC

# define PYOBJ_ALLOC MALLOC_ALLOC

#endif

/* the default debug allocators */

// The actual implementation is further down.

void* _PyMem_DebugRawMalloc(void *ctx, size_t size);

void* _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize);

void* _PyMem_DebugRawRealloc(void *ctx, void *ptr, size_t size);

void _PyMem_DebugRawFree(void *ctx, void *ptr);

void* _PyMem_DebugMalloc(void *ctx, size_t size);

void* _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize);

void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size);

void _PyMem_DebugFree(void *ctx, void *p);

#define PYDBGRAW_ALLOC \

{&_PyRuntime.allocators.debug.raw, _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree}

#define PYDBGMEM_ALLOC \

{&_PyRuntime.allocators.debug.mem, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree}

#define PYDBGOBJ_ALLOC \

{&_PyRuntime.allocators.debug.obj, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree}

/* the low-level virtual memory allocator */

#ifdef WITH_PYMALLOC

# ifdef MS_WINDOWS

# include <windows.h>

# elif defined(HAVE_MMAP)

# include <sys/mman.h>

# ifdef MAP_ANONYMOUS

# define ARENAS_USE_MMAP

# endif

# endif

#endif

void *

_PyMem_ArenaAlloc(void *Py_UNUSED(ctx), size_t size)

{

#ifdef MS_WINDOWS

return VirtualAlloc(NULL, size,

MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);

#elif defined(ARENAS_USE_MMAP)

void *ptr;

ptr = mmap(NULL, size, PROT_READ|PROT_WRITE,

MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);

if (ptr == MAP_FAILED)

return NULL;

assert(ptr != NULL);

return ptr;

#else

return malloc(size);

#endif

}

void

_PyMem_ArenaFree(void *Py_UNUSED(ctx), void *ptr,

#if defined(ARENAS_USE_MMAP)

size_t size

#else

size_t Py_UNUSED(size)

#endif

)

{

#ifdef MS_WINDOWS

VirtualFree(ptr, 0, MEM_RELEASE);

#elif defined(ARENAS_USE_MMAP)

munmap(ptr, size);

#else

free(ptr);

#endif

}

/*******************************************/

/* end low-level allocator implementations */

/*******************************************/

#if defined(__has_feature) /* Clang */

# if __has_feature(address_sanitizer) /* is ASAN enabled? */

# define _Py_NO_SANITIZE_ADDRESS \

__attribute__((no_sanitize("address")))

# endif

# if __has_feature(thread_sanitizer) /* is TSAN enabled? */

# define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))

# endif

# if __has_feature(memory_sanitizer) /* is MSAN enabled? */

# define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory))

# endif

#elif defined(__GNUC__)

# if defined(__SANITIZE_ADDRESS__) /* GCC 4.8+, is ASAN enabled? */

# define _Py_NO_SANITIZE_ADDRESS \

__attribute__((no_sanitize_address))

# endif

// TSAN is supported since GCC 5.1, but __SANITIZE_THREAD__ macro

// is provided only since GCC 7.

# if __GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1)

# define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))

# endif

#endif

#ifndef _Py_NO_SANITIZE_ADDRESS

# define _Py_NO_SANITIZE_ADDRESS

#endif

#ifndef _Py_NO_SANITIZE_THREAD

# define _Py_NO_SANITIZE_THREAD

#endif

#ifndef _Py_NO_SANITIZE_MEMORY

# define _Py_NO_SANITIZE_MEMORY

#endif

#define ALLOCATORS_MUTEX (_PyRuntime.allocators.mutex)

#define _PyMem_Raw (_PyRuntime.allocators.standard.raw)

#define _PyMem (_PyRuntime.allocators.standard.mem)

#define _PyObject (_PyRuntime.allocators.standard.obj)

#define _PyMem_Debug (_PyRuntime.allocators.debug)

#define _PyObject_Arena (_PyRuntime.allocators.obj_arena)

/***************************/

/* managing the allocators */

/***************************/

static int

set_default_allocator_unlocked(PyMemAllocatorDomain domain, int debug,

PyMemAllocatorEx *old_alloc)

{

if (old_alloc != NULL) {

get_allocator_unlocked(domain, old_alloc);

}

PyMemAllocatorEx new_alloc;

switch(domain)

{

case PYMEM_DOMAIN_RAW:

new_alloc = (PyMemAllocatorEx)PYRAW_ALLOC;

break;

case PYMEM_DOMAIN_MEM:

new_alloc = (PyMemAllocatorEx)PYMEM_ALLOC;

break;

case PYMEM_DOMAIN_OBJ:

new_alloc = (PyMemAllocatorEx)PYOBJ_ALLOC;

break;

default:

/* unknown domain */

return -1;

}

set_allocator_unlocked(domain, &new_alloc);

if (debug) {

set_up_debug_hooks_domain_unlocked(domain);

}

return 0;

}

#ifdef Py_DEBUG

static const int pydebug = 1;

#else

static const int pydebug = 0;

#endif

int

_PyMem_SetDefaultAllocator(PyMemAllocatorDomain domain,

PyMemAllocatorEx *old_alloc)

{

PyMutex_Lock(&ALLOCATORS_MUTEX);

int res = set_default_allocator_unlocked(domain, pydebug, old_alloc);

PyMutex_Unlock(&ALLOCATORS_MUTEX);

return res;

}

int

_PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator)

{

if (name == NULL || *name == '\0') {

/* PYTHONMALLOC is empty or is not set or ignored (-E/-I command line

nameions): use default memory allocators */

*allocator = PYMEM_ALLOCATOR_DEFAULT;

}

else if (strcmp(name, "default") == 0) {

*allocator = PYMEM_ALLOCATOR_DEFAULT;

}

else if (strcmp(name, "debug") == 0) {

*allocator = PYMEM_ALLOCATOR_DEBUG;

}

#if defined(WITH_PYMALLOC) && !defined(Py_GIL_DISABLED)

else if (strcmp(name, "pymalloc") == 0) {

*allocator = PYMEM_ALLOCATOR_PYMALLOC;

}

else if (strcmp(name, "pymalloc_debug") == 0) {

*allocator = PYMEM_ALLOCATOR_PYMALLOC_DEBUG;

}

#endif

#ifdef WITH_MIMALLOC

else if (strcmp(name, "mimalloc") == 0) {

*allocator = PYMEM_ALLOCATOR_MIMALLOC;

}

else if (strcmp(name, "mimalloc_debug") == 0) {

*allocator = PYMEM_ALLOCATOR_MIMALLOC_DEBUG;

}

#endif

#ifndef Py_GIL_DISABLED

else if (strcmp(name, "malloc") == 0) {

*allocator = PYMEM_ALLOCATOR_MALLOC;

}

else if (strcmp(name, "malloc_debug") == 0) {

*allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG;

}

#endif

else {

/* unknown allocator */

return -1;

}

return 0;

}

static int

set_up_allocators_unlocked(PyMemAllocatorName allocator)

{

switch (allocator) {

case PYMEM_ALLOCATOR_NOT_SET:

/* do nothing */

break;

case PYMEM_ALLOCATOR_DEFAULT:

(void)set_default_allocator_unlocked(PYMEM_DOMAIN_RAW, pydebug, NULL);

(void)set_default_allocator_unlocked(PYMEM_DOMAIN_MEM, pydebug, NULL);

(void)set_default_allocator_unlocked(PYMEM_DOMAIN_OBJ, pydebug, NULL);

_PyRuntime.allocators.is_debug_enabled = pydebug;

break;

case PYMEM_ALLOCATOR_DEBUG:

(void)set_default_allocator_unlocked(PYMEM_DOMAIN_RAW, 1, NULL);

(void)set_default_allocator_unlocked(PYMEM_DOMAIN_MEM, 1, NULL);

(void)set_default_allocator_unlocked(PYMEM_DOMAIN_OBJ, 1, NULL);

_PyRuntime.allocators.is_debug_enabled = 1;

break;

#ifdef WITH_PYMALLOC

case PYMEM_ALLOCATOR_PYMALLOC:

case PYMEM_ALLOCATOR_PYMALLOC_DEBUG:

{

PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;

set_allocator_unlocked(PYMEM_DOMAIN_RAW, &malloc_alloc);

PyMemAllocatorEx pymalloc = PYMALLOC_ALLOC;

set_allocator_unlocked(PYMEM_DOMAIN_MEM, &pymalloc);

set_allocator_unlocked(PYMEM_DOMAIN_OBJ, &pymalloc);

int is_debug = (allocator == PYMEM_ALLOCATOR_PYMALLOC_DEBUG);

_PyRuntime.allocators.is_debug_enabled = is_debug;

if (is_debug) {

set_up_debug_hooks_unlocked();

}

break;

}

#endif

#ifdef WITH_MIMALLOC

case PYMEM_ALLOCATOR_MIMALLOC:

case PYMEM_ALLOCATOR_MIMALLOC_DEBUG:

{

PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;

set_allocator_unlocked(PYMEM_DOMAIN_RAW, &malloc_alloc);

PyMemAllocatorEx pymalloc = MIMALLOC_ALLOC;

set_allocator_unlocked(PYMEM_DOMAIN_MEM, &pymalloc);

PyMemAllocatorEx objmalloc = MIMALLOC_OBJALLOC;

set_allocator_unlocked(PYMEM_DOMAIN_OBJ, &objmalloc);

int is_debug = (allocator == PYMEM_ALLOCATOR_MIMALLOC_DEBUG);

_PyRuntime.allocators.is_debug_enabled = is_debug;

if (is_debug) {

set_up_debug_hooks_unlocked();

}

break;

}

#endif

case PYMEM_ALLOCATOR_MALLOC:

case PYMEM_ALLOCATOR_MALLOC_DEBUG:

{

PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;

set_allocator_unlocked(PYMEM_DOMAIN_RAW, &malloc_alloc);

set_allocator_unlocked(PYMEM_DOMAIN_MEM, &malloc_alloc);

set_allocator_unlocked(PYMEM_DOMAIN_OBJ, &malloc_alloc);

int is_debug = (allocator == PYMEM_ALLOCATOR_MALLOC_DEBUG);

_PyRuntime.allocators.is_debug_enabled = is_debug;

if (is_debug) {

set_up_debug_hooks_unlocked();

}

break;

}

default:

/* unknown allocator */

return -1;

}

return 0;

}

int

_PyMem_SetupAllocators(PyMemAllocatorName allocator)

{

PyMutex_Lock(&ALLOCATORS_MUTEX);

int res = set_up_allocators_unlocked(allocator);

PyMutex_Unlock(&ALLOCATORS_MUTEX);

return res;

}

static int

pymemallocator_eq(PyMemAllocatorEx *a, PyMemAllocatorEx *b)

{

return (memcmp(a, b, sizeof(PyMemAllocatorEx)) == 0);

}

static const char*

get_current_allocator_name_unlocked(void)

{

PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;

#ifdef WITH_PYMALLOC

PyMemAllocatorEx pymalloc = PYMALLOC_ALLOC;

#endif

#ifdef WITH_MIMALLOC

PyMemAllocatorEx mimalloc = MIMALLOC_ALLOC;

PyMemAllocatorEx mimalloc_obj = MIMALLOC_OBJALLOC;

#endif

if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) &&

pymemallocator_eq(&_PyMem, &malloc_alloc) &&

pymemallocator_eq(&_PyObject, &malloc_alloc))

{

return "malloc";

}

#ifdef WITH_PYMALLOC

if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) &&

pymemallocator_eq(&_PyMem, &pymalloc) &&

pymemallocator_eq(&_PyObject, &pymalloc))

{

return "pymalloc";

}

#endif

#ifdef WITH_MIMALLOC

if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) &&

pymemallocator_eq(&_PyMem, &mimalloc) &&

pymemallocator_eq(&_PyObject, &mimalloc_obj))

{

return "mimalloc";

}

#endif

PyMemAllocatorEx dbg_raw = PYDBGRAW_ALLOC;

PyMemAllocatorEx dbg_mem = PYDBGMEM_ALLOC;

PyMemAllocatorEx dbg_obj = PYDBGOBJ_ALLOC;

if (pymemallocator_eq(&_PyMem_Raw, &dbg_raw) &&

pymemallocator_eq(&_PyMem, &dbg_mem) &&

pymemallocator_eq(&_PyObject, &dbg_obj))

{

/* Debug hooks installed */

if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) &&

pymemallocator_eq(&_PyMem_Debug.mem.alloc, &malloc_alloc) &&

pymemallocator_eq(&_PyMem_Debug.obj.alloc, &malloc_alloc))

{

return "malloc_debug";

}

#ifdef WITH_PYMALLOC

if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) &&

pymemallocator_eq(&_PyMem_Debug.mem.alloc, &pymalloc) &&

pymemallocator_eq(&_PyMem_Debug.obj.alloc, &pymalloc))

{

return "pymalloc_debug";

}

#endif

#ifdef WITH_MIMALLOC

if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) &&

pymemallocator_eq(&_PyMem_Debug.mem.alloc, &mimalloc) &&

pymemallocator_eq(&_PyMem_Debug.obj.alloc, &mimalloc_obj))

{

return "mimalloc_debug";

}

#endif

}

return NULL;

}

const char*

_PyMem_GetCurrentAllocatorName(void)

{

PyMutex_Lock(&ALLOCATORS_MUTEX);

const char *name = get_current_allocator_name_unlocked();

PyMutex_Unlock(&ALLOCATORS_MUTEX);

return name;

}

int

_PyMem_DebugEnabled(void)

{

return _PyRuntime.allocators.is_debug_enabled;

}

#ifdef WITH_PYMALLOC

static int

_PyMem_PymallocEnabled(void)

{

if (_PyMem_DebugEnabled()) {

return (_PyMem_Debug.obj.alloc.malloc == _PyObject_Malloc);

}

else {

return (_PyObject.malloc == _PyObject_Malloc);

}

}

#ifdef WITH_MIMALLOC

static int

_PyMem_MimallocEnabled(void)

{

#ifdef Py_GIL_DISABLED

return 1;

#else

if (_PyMem_DebugEnabled()) {

return (_PyMem_Debug.obj.alloc.malloc == _PyObject_MiMalloc);

}

else {

return (_PyObject.malloc == _PyObject_MiMalloc);

}

#endif

}

#endif // WITH_MIMALLOC

#endif // WITH_PYMALLOC

static void

set_up_debug_hooks_domain_unlocked(PyMemAllocatorDomain domain)

{

PyMemAllocatorEx alloc;

if (domain == PYMEM_DOMAIN_RAW) {

if (_PyMem_Raw.malloc == _PyMem_DebugRawMalloc) {

return;

}

get_allocator_unlocked(domain, &_PyMem_Debug.raw.alloc);

alloc.ctx = &_PyMem_Debug.raw;

alloc.malloc = _PyMem_DebugRawMalloc;

alloc.calloc = _PyMem_DebugRawCalloc;

alloc.realloc = _PyMem_DebugRawRealloc;

alloc.free = _PyMem_DebugRawFree;

set_allocator_unlocked(domain, &alloc);

}

else if (domain == PYMEM_DOMAIN_MEM) {

if (_PyMem.malloc == _PyMem_DebugMalloc) {

return;

}

get_allocator_unlocked(domain, &_PyMem_Debug.mem.alloc);

alloc.ctx = &_PyMem_Debug.mem;

alloc.malloc = _PyMem_DebugMalloc;

alloc.calloc = _PyMem_DebugCalloc;

alloc.realloc = _PyMem_DebugRealloc;

alloc.free = _PyMem_DebugFree;

set_allocator_unlocked(domain, &alloc);

}

else if (domain == PYMEM_DOMAIN_OBJ) {

if (_PyObject.malloc == _PyMem_DebugMalloc) {

return;

}

get_allocator_unlocked(domain, &_PyMem_Debug.obj.alloc);

alloc.ctx = &_PyMem_Debug.obj;

alloc.malloc = _PyMem_DebugMalloc;

alloc.calloc = _PyMem_DebugCalloc;

alloc.realloc = _PyMem_DebugRealloc;

alloc.free = _PyMem_DebugFree;

set_allocator_unlocked(domain, &alloc);

}

}

static void

set_up_debug_hooks_unlocked(void)

{

set_up_debug_hooks_domain_unlocked(PYMEM_DOMAIN_RAW);

set_up_debug_hooks_domain_unlocked(PYMEM_DOMAIN_MEM);

set_up_debug_hooks_domain_unlocked(PYMEM_DOMAIN_OBJ);

_PyRuntime.allocators.is_debug_enabled = 1;

}

void

PyMem_SetupDebugHooks(void)

{

PyMutex_Lock(&ALLOCATORS_MUTEX);

set_up_debug_hooks_unlocked();

PyMutex_Unlock(&ALLOCATORS_MUTEX);

}

static void

get_allocator_unlocked(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)

{

switch(domain)

{

case PYMEM_DOMAIN_RAW: *allocator = _PyMem_Raw; break;

case PYMEM_DOMAIN_MEM: *allocator = _PyMem; break;

case PYMEM_DOMAIN_OBJ: *allocator = _PyObject; break;

default:

/* unknown domain: set all attributes to NULL */

allocator->ctx = NULL;

allocator->malloc = NULL;

allocator->calloc = NULL;

allocator->realloc = NULL;

allocator->free = NULL;

}

}

static void

set_allocator_unlocked(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)

{

switch(domain)

{

case PYMEM_DOMAIN_RAW: _PyMem_Raw = *allocator; break;

case PYMEM_DOMAIN_MEM: _PyMem = *allocator; break;

case PYMEM_DOMAIN_OBJ: _PyObject = *allocator; break;

/* ignore unknown domain */

}

}

void

PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)

{

PyMutex_Lock(&ALLOCATORS_MUTEX);

get_allocator_unlocked(domain, allocator);

PyMutex_Unlock(&ALLOCATORS_MUTEX);

}

void

PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)

{

PyMutex_Lock(&ALLOCATORS_MUTEX);

set_allocator_unlocked(domain, allocator);

PyMutex_Unlock(&ALLOCATORS_MUTEX);

}

void

PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)

{

PyMutex_Lock(&ALLOCATORS_MUTEX);

*allocator = _PyObject_Arena;

PyMutex_Unlock(&ALLOCATORS_MUTEX);

}

void

PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)

{

PyMutex_Lock(&ALLOCATORS_MUTEX);

_PyObject_Arena = *allocator;

PyMutex_Unlock(&ALLOCATORS_MUTEX);

}

/* Note that there is a possible, but very unlikely, race in any place

* below where we call one of the allocator functions. We access two

* fields in each case: "malloc", etc. and "ctx".

*

* It is unlikely that the allocator will be changed while one of those

* calls is happening, much less in that very narrow window.

* Furthermore, the likelihood of a race is drastically reduced by the

* fact that the allocator may not be changed after runtime init

* (except with a wrapper).

*

* With the above in mind, we currently don't worry about locking

* around these uses of the runtime-global allocators state. */

/*************************/

/* the "arena" allocator */

/*************************/

void *

_PyObject_VirtualAlloc(size_t size)

{

return _PyObject_Arena.alloc(_PyObject_Arena.ctx, size);

}

void

_PyObject_VirtualFree(void *obj, size_t size)

{

_PyObject_Arena.free(_PyObject_Arena.ctx, obj, size);

}

/***********************/

/* the "raw" allocator */

/***********************/

void *

PyMem_RawMalloc(size_t size)

{

/*

* Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.

* Most python internals blindly use a signed Py_ssize_t to track

* things without checking for overflows or negatives.

* As size_t is unsigned, checking for size < 0 is not required.

*/

if (size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size);

}

void *

PyMem_RawCalloc(size_t nelem, size_t elsize)

{

/* see PyMem_RawMalloc() */

if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)

return NULL;

return _PyMem_Raw.calloc(_PyMem_Raw.ctx, nelem, elsize);

}

void*

PyMem_RawRealloc(void *ptr, size_t new_size)

{

/* see PyMem_RawMalloc() */

if (new_size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size);

}

void PyMem_RawFree(void *ptr)

{

_PyMem_Raw.free(_PyMem_Raw.ctx, ptr);

}

/***********************/

/* the "mem" allocator */

/***********************/

void *

PyMem_Malloc(size_t size)

{

/* see PyMem_RawMalloc() */

if (size > (size_t)PY_SSIZE_T_MAX)

return NULL;

OBJECT_STAT_INC_COND(allocations512, size < 512);

OBJECT_STAT_INC_COND(allocations4k, size >= 512 && size < 4094);

OBJECT_STAT_INC_COND(allocations_big, size >= 4094);

OBJECT_STAT_INC(allocations);

return _PyMem.malloc(_PyMem.ctx, size);

}

void *

PyMem_Calloc(size_t nelem, size_t elsize)

{

/* see PyMem_RawMalloc() */

if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)

return NULL;

OBJECT_STAT_INC_COND(allocations512, elsize < 512);

OBJECT_STAT_INC_COND(allocations4k, elsize >= 512 && elsize < 4094);

OBJECT_STAT_INC_COND(allocations_big, elsize >= 4094);

OBJECT_STAT_INC(allocations);

return _PyMem.calloc(_PyMem.ctx, nelem, elsize);

}

void *

PyMem_Realloc(void *ptr, size_t new_size)

{

/* see PyMem_RawMalloc() */

if (new_size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyMem.realloc(_PyMem.ctx, ptr, new_size);

}

void

PyMem_Free(void *ptr)

{

OBJECT_STAT_INC(frees);

_PyMem.free(_PyMem.ctx, ptr);

}

/***************************/

/* pymem utility functions */

/***************************/

wchar_t*

_PyMem_RawWcsdup(const wchar_t *str)

{

assert(str != NULL);

size_t len = wcslen(str);

if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) {

return NULL;

}

size_t size = (len + 1) * sizeof(wchar_t);

wchar_t *str2 = PyMem_RawMalloc(size);

if (str2 == NULL) {

return NULL;

}

memcpy(str2, str, size);

return str2;

}

char *

_PyMem_RawStrdup(const char *str)

{

assert(str != NULL);

size_t size = strlen(str) + 1;

char *copy = PyMem_RawMalloc(size);

if (copy == NULL) {

return NULL;

}

memcpy(copy, str, size);

return copy;

}

char *

_PyMem_Strdup(const char *str)

{

assert(str != NULL);

size_t size = strlen(str) + 1;

char *copy = PyMem_Malloc(size);

if (copy == NULL) {

return NULL;

}

memcpy(copy, str, size);

return copy;

}

/***********************************************/

/* Delayed freeing support for Py_GIL_DISABLED */

/***********************************************/

// So that sizeof(struct _mem_work_chunk) is 4096 bytes on 64-bit platforms.

#define WORK_ITEMS_PER_CHUNK 254

// A pointer to be freed once the QSBR read sequence reaches qsbr_goal.

struct _mem_work_item {

void *ptr;

uint64_t qsbr_goal;

};

// A fixed-size buffer of pointers to be freed

struct _mem_work_chunk {

// Linked list node of chunks in queue

struct llist_node node;

Py_ssize_t rd_idx; // index of next item to read

Py_ssize_t wr_idx; // index of next item to write

struct _mem_work_item array[WORK_ITEMS_PER_CHUNK];

};

void

_PyMem_FreeDelayed(void *ptr)

{

#ifndef Py_GIL_DISABLED

PyMem_Free(ptr);

#else

if (_PyRuntime.stoptheworld.world_stopped) {

// Free immediately if the world is stopped, including during

// interpreter shutdown.

PyMem_Free(ptr);

return;

}

_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();

struct llist_node *head = &tstate->mem_free_queue;

struct _mem_work_chunk *buf = NULL;

if (!llist_empty(head)) {

// Try to re-use the last buffer

buf = llist_data(head->prev, struct _mem_work_chunk, node);

if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) {

// already full

buf = NULL;

}

}

if (buf == NULL) {