#include "Python.h"

#include "pycore_code.h" // stats

#include "pycore_pystate.h" // _PyInterpreterState_GET

#include "pycore_obmalloc.h"

#include "pycore_pymem.h"

#include <stdlib.h> // malloc()

#include <stdbool.h>

#undef uint

#define uint pymem_uint

/* Defined in tracemalloc.c */

extern void _PyMem_DumpTraceback(int fd, const void *ptr);

/* Python's malloc wrappers (see pymem.h) */

static void _PyObject_DebugDumpAddress(const void *p);

static void _PyMem_DebugCheckAddress(const char *func, char api_id, const void *p);

static void _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain);

/***************************************/

/* low-level allocator implementations */

/***************************************/

/* the default raw allocator (wraps malloc) */

void *

_PyMem_RawMalloc(void *Py_UNUSED(ctx), size_t size)

{

/* PyMem_RawMalloc(0) means malloc(1). Some systems would return NULL

for malloc(0), which would be treated as an error. Some platforms would

return a pointer with no memory behind it, which would break pymalloc.

To solve these problems, allocate an extra byte. */

if (size == 0)

size = 1;

return malloc(size);

}

void *

_PyMem_RawCalloc(void *Py_UNUSED(ctx), size_t nelem, size_t elsize)

{

/* PyMem_RawCalloc(0, 0) means calloc(1, 1). Some systems would return NULL

for calloc(0, 0), which would be treated as an error. Some platforms

would return a pointer with no memory behind it, which would break

pymalloc. To solve these problems, allocate an extra byte. */

if (nelem == 0 || elsize == 0) {

nelem = 1;

elsize = 1;

}

return calloc(nelem, elsize);

}

void *

_PyMem_RawRealloc(void *Py_UNUSED(ctx), void *ptr, size_t size)

{

if (size == 0)

size = 1;

return realloc(ptr, size);

}

void

_PyMem_RawFree(void *Py_UNUSED(ctx), void *ptr)

{

free(ptr);

}

#define MALLOC_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree}

#define PYRAW_ALLOC MALLOC_ALLOC

/* the default object allocator */

// The actual implementation is further down.

#ifdef WITH_PYMALLOC

void* _PyObject_Malloc(void *ctx, size_t size);

void* _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize);

void _PyObject_Free(void *ctx, void *p);

void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);

# define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free}

# define PYOBJ_ALLOC PYMALLOC_ALLOC

#else

# define PYOBJ_ALLOC MALLOC_ALLOC

#endif // WITH_PYMALLOC

#define PYMEM_ALLOC PYOBJ_ALLOC

/* the default debug allocators */

// The actual implementation is further down.

void* _PyMem_DebugRawMalloc(void *ctx, size_t size);

void* _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize);

void* _PyMem_DebugRawRealloc(void *ctx, void *ptr, size_t size);

void _PyMem_DebugRawFree(void *ctx, void *ptr);

void* _PyMem_DebugMalloc(void *ctx, size_t size);

void* _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize);

void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size);

void _PyMem_DebugFree(void *ctx, void *p);

#define PYDBGRAW_ALLOC \

{&_PyRuntime.allocators.debug.raw, _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree}

#define PYDBGMEM_ALLOC \

{&_PyRuntime.allocators.debug.mem, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree}

#define PYDBGOBJ_ALLOC \

{&_PyRuntime.allocators.debug.obj, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree}

/* the low-level virtual memory allocator */

#ifdef WITH_PYMALLOC

# ifdef MS_WINDOWS

# include <windows.h>

# elif defined(HAVE_MMAP)

# include <sys/mman.h>

# ifdef MAP_ANONYMOUS

# define ARENAS_USE_MMAP

# endif

# endif

#endif

void *

_PyMem_ArenaAlloc(void *Py_UNUSED(ctx), size_t size)

{

#ifdef MS_WINDOWS

return VirtualAlloc(NULL, size,

MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);

#elif defined(ARENAS_USE_MMAP)

void *ptr;

ptr = mmap(NULL, size, PROT_READ|PROT_WRITE,

MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);

if (ptr == MAP_FAILED)

return NULL;

assert(ptr != NULL);

return ptr;

#else

return malloc(size);

#endif

}

void

_PyMem_ArenaFree(void *Py_UNUSED(ctx), void *ptr,

#if defined(ARENAS_USE_MMAP)

size_t size

#else

size_t Py_UNUSED(size)

#endif

)

{

#ifdef MS_WINDOWS

VirtualFree(ptr, 0, MEM_RELEASE);

#elif defined(ARENAS_USE_MMAP)

munmap(ptr, size);

#else

free(ptr);

#endif

}

/*******************************************/

/* end low-level allocator implementations */

/*******************************************/

#if defined(__has_feature) /* Clang */

# if __has_feature(address_sanitizer) /* is ASAN enabled? */

# define _Py_NO_SANITIZE_ADDRESS \

__attribute__((no_sanitize("address")))

# endif

# if __has_feature(thread_sanitizer) /* is TSAN enabled? */

# define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))

# endif

# if __has_feature(memory_sanitizer) /* is MSAN enabled? */

# define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory))

# endif

#elif defined(__GNUC__)

# if defined(__SANITIZE_ADDRESS__) /* GCC 4.8+, is ASAN enabled? */

# define _Py_NO_SANITIZE_ADDRESS \

__attribute__((no_sanitize_address))

# endif

// TSAN is supported since GCC 5.1, but __SANITIZE_THREAD__ macro

// is provided only since GCC 7.

# if __GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1)

# define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))

# endif

#endif

#ifndef _Py_NO_SANITIZE_ADDRESS

# define _Py_NO_SANITIZE_ADDRESS

#endif

#ifndef _Py_NO_SANITIZE_THREAD

# define _Py_NO_SANITIZE_THREAD

#endif

#ifndef _Py_NO_SANITIZE_MEMORY

# define _Py_NO_SANITIZE_MEMORY

#endif

/* bpo-35053: Declare tracemalloc configuration here rather than

Modules/_tracemalloc.c because _tracemalloc can be compiled as dynamic

library, whereas _Py_NewReference() requires it. */

struct _PyTraceMalloc_Config _Py_tracemalloc_config = _PyTraceMalloc_Config_INIT;

#define _PyMem_Raw (_PyRuntime.allocators.standard.raw)

#define _PyMem (_PyRuntime.allocators.standard.mem)

#define _PyObject (_PyRuntime.allocators.standard.obj)

#define _PyMem_Debug (_PyRuntime.allocators.debug)

#define _PyObject_Arena (_PyRuntime.allocators.obj_arena)

static int

pymem_set_default_allocator(PyMemAllocatorDomain domain, int debug,

PyMemAllocatorEx *old_alloc)

{

if (old_alloc != NULL) {

PyMem_GetAllocator(domain, old_alloc);

}

PyMemAllocatorEx new_alloc;

switch(domain)

{

case PYMEM_DOMAIN_RAW:

new_alloc = (PyMemAllocatorEx)PYRAW_ALLOC;

break;

case PYMEM_DOMAIN_MEM:

new_alloc = (PyMemAllocatorEx)PYMEM_ALLOC;

break;

case PYMEM_DOMAIN_OBJ:

new_alloc = (PyMemAllocatorEx)PYOBJ_ALLOC;

break;

default:

/* unknown domain */

return -1;

}

PyMem_SetAllocator(domain, &new_alloc);

if (debug) {

_PyMem_SetupDebugHooksDomain(domain);

}

return 0;

}

int

_PyMem_SetDefaultAllocator(PyMemAllocatorDomain domain,

PyMemAllocatorEx *old_alloc)

{

#ifdef Py_DEBUG

const int debug = 1;

#else

const int debug = 0;

#endif

return pymem_set_default_allocator(domain, debug, old_alloc);

}

int

_PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator)

{

if (name == NULL || *name == '\0') {

/* PYTHONMALLOC is empty or is not set or ignored (-E/-I command line

nameions): use default memory allocators */

*allocator = PYMEM_ALLOCATOR_DEFAULT;

}

else if (strcmp(name, "default") == 0) {

*allocator = PYMEM_ALLOCATOR_DEFAULT;

}

else if (strcmp(name, "debug") == 0) {

*allocator = PYMEM_ALLOCATOR_DEBUG;

}

#ifdef WITH_PYMALLOC

else if (strcmp(name, "pymalloc") == 0) {

*allocator = PYMEM_ALLOCATOR_PYMALLOC;

}

else if (strcmp(name, "pymalloc_debug") == 0) {

*allocator = PYMEM_ALLOCATOR_PYMALLOC_DEBUG;

}

#endif

else if (strcmp(name, "malloc") == 0) {

*allocator = PYMEM_ALLOCATOR_MALLOC;

}

else if (strcmp(name, "malloc_debug") == 0) {

*allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG;

}

else {

/* unknown allocator */

return -1;

}

return 0;

}

int

_PyMem_SetupAllocators(PyMemAllocatorName allocator)

{

switch (allocator) {

case PYMEM_ALLOCATOR_NOT_SET:

/* do nothing */

break;

case PYMEM_ALLOCATOR_DEFAULT:

(void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, NULL);

(void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_MEM, NULL);

(void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_OBJ, NULL);

break;

case PYMEM_ALLOCATOR_DEBUG:

(void)pymem_set_default_allocator(PYMEM_DOMAIN_RAW, 1, NULL);

(void)pymem_set_default_allocator(PYMEM_DOMAIN_MEM, 1, NULL);

(void)pymem_set_default_allocator(PYMEM_DOMAIN_OBJ, 1, NULL);

break;

#ifdef WITH_PYMALLOC

case PYMEM_ALLOCATOR_PYMALLOC:

case PYMEM_ALLOCATOR_PYMALLOC_DEBUG:

{

PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &malloc_alloc);

PyMemAllocatorEx pymalloc = PYMALLOC_ALLOC;

PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &pymalloc);

PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &pymalloc);

if (allocator == PYMEM_ALLOCATOR_PYMALLOC_DEBUG) {

PyMem_SetupDebugHooks();

}

break;

}

#endif

case PYMEM_ALLOCATOR_MALLOC:

case PYMEM_ALLOCATOR_MALLOC_DEBUG:

{

PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &malloc_alloc);

PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &malloc_alloc);

PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &malloc_alloc);

if (allocator == PYMEM_ALLOCATOR_MALLOC_DEBUG) {

PyMem_SetupDebugHooks();

}

break;

}

default:

/* unknown allocator */

return -1;

}

return 0;

}

static int

pymemallocator_eq(PyMemAllocatorEx *a, PyMemAllocatorEx *b)

{

return (memcmp(a, b, sizeof(PyMemAllocatorEx)) == 0);

}

const char*

_PyMem_GetCurrentAllocatorName(void)

{

PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;

#ifdef WITH_PYMALLOC

PyMemAllocatorEx pymalloc = PYMALLOC_ALLOC;

#endif

if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) &&

pymemallocator_eq(&_PyMem, &malloc_alloc) &&

pymemallocator_eq(&_PyObject, &malloc_alloc))

{

return "malloc";

}

#ifdef WITH_PYMALLOC

if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) &&

pymemallocator_eq(&_PyMem, &pymalloc) &&

pymemallocator_eq(&_PyObject, &pymalloc))

{

return "pymalloc";

}

#endif

PyMemAllocatorEx dbg_raw = PYDBGRAW_ALLOC;

PyMemAllocatorEx dbg_mem = PYDBGMEM_ALLOC;

PyMemAllocatorEx dbg_obj = PYDBGOBJ_ALLOC;

if (pymemallocator_eq(&_PyMem_Raw, &dbg_raw) &&

pymemallocator_eq(&_PyMem, &dbg_mem) &&

pymemallocator_eq(&_PyObject, &dbg_obj))

{

/* Debug hooks installed */

if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) &&

pymemallocator_eq(&_PyMem_Debug.mem.alloc, &malloc_alloc) &&

pymemallocator_eq(&_PyMem_Debug.obj.alloc, &malloc_alloc))

{

return "malloc_debug";

}

#ifdef WITH_PYMALLOC

if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) &&

pymemallocator_eq(&_PyMem_Debug.mem.alloc, &pymalloc) &&

pymemallocator_eq(&_PyMem_Debug.obj.alloc, &pymalloc))

{

return "pymalloc_debug";

}

#endif

}

return NULL;

}

#ifdef WITH_PYMALLOC

static int

_PyMem_DebugEnabled(void)

{

return (_PyObject.malloc == _PyMem_DebugMalloc);

}

static int

_PyMem_PymallocEnabled(void)

{

if (_PyMem_DebugEnabled()) {

return (_PyMem_Debug.obj.alloc.malloc == _PyObject_Malloc);

}

else {

return (_PyObject.malloc == _PyObject_Malloc);

}

}

#endif

static void

_PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain)

{

PyMemAllocatorEx alloc;

if (domain == PYMEM_DOMAIN_RAW) {

if (_PyMem_Raw.malloc == _PyMem_DebugRawMalloc) {

return;

}

PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &_PyMem_Debug.raw.alloc);

alloc.ctx = &_PyMem_Debug.raw;

alloc.malloc = _PyMem_DebugRawMalloc;

alloc.calloc = _PyMem_DebugRawCalloc;

alloc.realloc = _PyMem_DebugRawRealloc;

alloc.free = _PyMem_DebugRawFree;

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc);

}

else if (domain == PYMEM_DOMAIN_MEM) {

if (_PyMem.malloc == _PyMem_DebugMalloc) {

return;

}

PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc);

alloc.ctx = &_PyMem_Debug.mem;

alloc.malloc = _PyMem_DebugMalloc;

alloc.calloc = _PyMem_DebugCalloc;

alloc.realloc = _PyMem_DebugRealloc;

alloc.free = _PyMem_DebugFree;

PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc);

}

else if (domain == PYMEM_DOMAIN_OBJ) {

if (_PyObject.malloc == _PyMem_DebugMalloc) {

return;

}

PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc);

alloc.ctx = &_PyMem_Debug.obj;

alloc.malloc = _PyMem_DebugMalloc;

alloc.calloc = _PyMem_DebugCalloc;

alloc.realloc = _PyMem_DebugRealloc;

alloc.free = _PyMem_DebugFree;

PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc);

}

}

void

PyMem_SetupDebugHooks(void)

{

_PyMem_SetupDebugHooksDomain(PYMEM_DOMAIN_RAW);

_PyMem_SetupDebugHooksDomain(PYMEM_DOMAIN_MEM);

_PyMem_SetupDebugHooksDomain(PYMEM_DOMAIN_OBJ);

}

void

PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)

{

switch(domain)

{

case PYMEM_DOMAIN_RAW: *allocator = _PyMem_Raw; break;

case PYMEM_DOMAIN_MEM: *allocator = _PyMem; break;

case PYMEM_DOMAIN_OBJ: *allocator = _PyObject; break;

default:

/* unknown domain: set all attributes to NULL */

allocator->ctx = NULL;

allocator->malloc = NULL;

allocator->calloc = NULL;

allocator->realloc = NULL;

allocator->free = NULL;

}

}

void

PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)

{

switch(domain)

{

case PYMEM_DOMAIN_RAW: _PyMem_Raw = *allocator; break;

case PYMEM_DOMAIN_MEM: _PyMem = *allocator; break;

case PYMEM_DOMAIN_OBJ: _PyObject = *allocator; break;

/* ignore unknown domain */

}

}

void

PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)

{

*allocator = _PyObject_Arena;

}

void *

_PyObject_VirtualAlloc(size_t size)

{

return _PyObject_Arena.alloc(_PyObject_Arena.ctx, size);

}

void

_PyObject_VirtualFree(void *obj, size_t size)

{

_PyObject_Arena.free(_PyObject_Arena.ctx, obj, size);

}

void

PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)

{

_PyObject_Arena = *allocator;

}

void *

PyMem_RawMalloc(size_t size)

{

/*

* Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.

* Most python internals blindly use a signed Py_ssize_t to track

* things without checking for overflows or negatives.

* As size_t is unsigned, checking for size < 0 is not required.

*/

if (size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size);

}

void *

PyMem_RawCalloc(size_t nelem, size_t elsize)

{

/* see PyMem_RawMalloc() */

if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)

return NULL;

return _PyMem_Raw.calloc(_PyMem_Raw.ctx, nelem, elsize);

}

void*

PyMem_RawRealloc(void *ptr, size_t new_size)

{

/* see PyMem_RawMalloc() */

if (new_size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size);

}

void PyMem_RawFree(void *ptr)

{

_PyMem_Raw.free(_PyMem_Raw.ctx, ptr);

}

void *

PyMem_Malloc(size_t size)

{

/* see PyMem_RawMalloc() */

if (size > (size_t)PY_SSIZE_T_MAX)

return NULL;

OBJECT_STAT_INC_COND(allocations512, size < 512);

OBJECT_STAT_INC_COND(allocations4k, size >= 512 && size < 4094);

OBJECT_STAT_INC_COND(allocations_big, size >= 4094);

OBJECT_STAT_INC(allocations);

return _PyMem.malloc(_PyMem.ctx, size);

}

void *

PyMem_Calloc(size_t nelem, size_t elsize)

{

/* see PyMem_RawMalloc() */

if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)

return NULL;

OBJECT_STAT_INC_COND(allocations512, elsize < 512);

OBJECT_STAT_INC_COND(allocations4k, elsize >= 512 && elsize < 4094);

OBJECT_STAT_INC_COND(allocations_big, elsize >= 4094);

OBJECT_STAT_INC(allocations);

return _PyMem.calloc(_PyMem.ctx, nelem, elsize);

}

void *

PyMem_Realloc(void *ptr, size_t new_size)

{

/* see PyMem_RawMalloc() */

if (new_size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyMem.realloc(_PyMem.ctx, ptr, new_size);

}

void

PyMem_Free(void *ptr)

{

OBJECT_STAT_INC(frees);

_PyMem.free(_PyMem.ctx, ptr);

}

wchar_t*

_PyMem_RawWcsdup(const wchar_t *str)

{

assert(str != NULL);

size_t len = wcslen(str);

if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) {

return NULL;

}

size_t size = (len + 1) * sizeof(wchar_t);

wchar_t *str2 = PyMem_RawMalloc(size);

if (str2 == NULL) {

return NULL;

}

memcpy(str2, str, size);

return str2;

}

char *

_PyMem_RawStrdup(const char *str)

{

assert(str != NULL);

size_t size = strlen(str) + 1;

char *copy = PyMem_RawMalloc(size);

if (copy == NULL) {

return NULL;

}

memcpy(copy, str, size);

return copy;

}

char *

_PyMem_Strdup(const char *str)

{

assert(str != NULL);

size_t size = strlen(str) + 1;

char *copy = PyMem_Malloc(size);

if (copy == NULL) {

return NULL;

}

memcpy(copy, str, size);

return copy;

}

void *

PyObject_Malloc(size_t size)

{

/* see PyMem_RawMalloc() */

if (size > (size_t)PY_SSIZE_T_MAX)

return NULL;

OBJECT_STAT_INC_COND(allocations512, size < 512);

OBJECT_STAT_INC_COND(allocations4k, size >= 512 && size < 4094);

OBJECT_STAT_INC_COND(allocations_big, size >= 4094);

OBJECT_STAT_INC(allocations);

return _PyObject.malloc(_PyObject.ctx, size);

}

void *

PyObject_Calloc(size_t nelem, size_t elsize)

{

/* see PyMem_RawMalloc() */

if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)

return NULL;

OBJECT_STAT_INC_COND(allocations512, elsize < 512);

OBJECT_STAT_INC_COND(allocations4k, elsize >= 512 && elsize < 4094);

OBJECT_STAT_INC_COND(allocations_big, elsize >= 4094);

OBJECT_STAT_INC(allocations);

return _PyObject.calloc(_PyObject.ctx, nelem, elsize);

}

void *

PyObject_Realloc(void *ptr, size_t new_size)

{

/* see PyMem_RawMalloc() */

if (new_size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyObject.realloc(_PyObject.ctx, ptr, new_size);

}

void

PyObject_Free(void *ptr)

{

OBJECT_STAT_INC(frees);

_PyObject.free(_PyObject.ctx, ptr);

}

/* If we're using GCC, use __builtin_expect() to reduce overhead of

the valgrind checks */

#if defined(__GNUC__) && (__GNUC__ > 2) && defined(__OPTIMIZE__)

# define UNLIKELY(value) __builtin_expect((value), 0)

# define LIKELY(value) __builtin_expect((value), 1)

#else

# define UNLIKELY(value) (value)

# define LIKELY(value) (value)

#endif

#ifdef WITH_PYMALLOC

#ifdef WITH_VALGRIND

#include <valgrind/valgrind.h>

/* -1 indicates that we haven't checked that we're running on valgrind yet. */

static int running_on_valgrind = -1;

#endif

#define allarenas (_PyRuntime.obmalloc.mgmt.arenas)

#define maxarenas (_PyRuntime.obmalloc.mgmt.maxarenas)

#define unused_arena_objects (_PyRuntime.obmalloc.mgmt.unused_arena_objects)

#define usable_arenas (_PyRuntime.obmalloc.mgmt.usable_arenas)

#define nfp2lasta (_PyRuntime.obmalloc.mgmt.nfp2lasta)

#define narenas_currently_allocated (_PyRuntime.obmalloc.mgmt.narenas_currently_allocated)

#define ntimes_arena_allocated (_PyRuntime.obmalloc.mgmt.ntimes_arena_allocated)

#define narenas_highwater (_PyRuntime.obmalloc.mgmt.narenas_highwater)

#define raw_allocated_blocks (_PyRuntime.obmalloc.mgmt.raw_allocated_blocks)

Py_ssize_t

_Py_GetAllocatedBlocks(void)

{

Py_ssize_t n = raw_allocated_blocks;

/* add up allocated blocks for used pools */

for (uint i = 0; i < maxarenas; ++i) {

/* Skip arenas which are not allocated. */

if (allarenas[i].address == 0) {

continue;

}

uintptr_t base = (uintptr_t)_Py_ALIGN_UP(allarenas[i].address, POOL_SIZE);

/* visit every pool in the arena */

assert(base <= (uintptr_t) allarenas[i].pool_address);

for (; base < (uintptr_t) allarenas[i].pool_address; base += POOL_SIZE) {

poolp p = (poolp)base;

n += p->ref.count;

}

}

return n;

}

#if WITH_PYMALLOC_RADIX_TREE

/*==========================================================================*/

/* radix tree for tracking arena usage. */

#define arena_map_root (_PyRuntime.obmalloc.usage.arena_map_root)

#ifdef USE_INTERIOR_NODES

#define arena_map_mid_count (_PyRuntime.obmalloc.usage.arena_map_mid_count)

#define arena_map_bot_count (_PyRuntime.obmalloc.usage.arena_map_bot_count)

#endif

/* Return a pointer to a bottom tree node, return NULL if it doesn't exist or

* it cannot be created */

static Py_ALWAYS_INLINE arena_map_bot_t *

arena_map_get(pymem_block *p, int create)

{

#ifdef USE_INTERIOR_NODES

/* sanity check that IGNORE_BITS is correct */

assert(HIGH_BITS(p) == HIGH_BITS(&arena_map_root));

int i1 = MAP_TOP_INDEX(p);

if (arena_map_root.ptrs[i1] == NULL) {

if (!create) {

return NULL;

}

arena_map_mid_t *n = PyMem_RawCalloc(1, sizeof(arena_map_mid_t));

if (n == NULL) {

return NULL;

}

arena_map_root.ptrs[i1] = n;

arena_map_mid_count++;

}

int i2 = MAP_MID_INDEX(p);

if (arena_map_root.ptrs[i1]->ptrs[i2] == NULL) {

if (!create) {

return NULL;

}

arena_map_bot_t *n = PyMem_RawCalloc(1, sizeof(arena_map_bot_t));

if (n == NULL) {

return NULL;

}

arena_map_root.ptrs[i1]->ptrs[i2] = n;

arena_map_bot_count++;

}

return arena_map_root.ptrs[i1]->ptrs[i2];

#else

return &arena_map_root;

#endif

}

/* The radix tree only tracks arenas. So, for 16 MiB arenas, we throw

* away 24 bits of the address. That reduces the space requirement of

* the tree compared to similar radix tree page-map schemes. In

* exchange for slashing the space requirement, it needs more

* computation to check an address.

*

* Tracking coverage is done by "ideal" arena address. It is easier to

* explain in decimal so let's say that the arena size is 100 bytes.

* Then, ideal addresses are 100, 200, 300, etc. For checking if a

* pointer address is inside an actual arena, we have to check two ideal

* arena addresses. E.g. if pointer is 357, we need to check 200 and

* 300. In the rare case that an arena is aligned in the ideal way

* (e.g. base address of arena is 200) then we only have to check one

* ideal address.

*

* The tree nodes for 200 and 300 both store the address of arena.

* There are two cases: the arena starts at a lower ideal arena and

* extends to this one, or the arena starts in this arena and extends to

* the next ideal arena. The tail_lo and tail_hi members correspond to

* these two cases.

*/

/* mark or unmark addresses covered by arena */

static int

arena_map_mark_used(uintptr_t arena_base, int is_used)

{

/* sanity check that IGNORE_BITS is correct */

assert(HIGH_BITS(arena_base) == HIGH_BITS(&arena_map_root));

arena_map_bot_t *n_hi = arena_map_get((pymem_block *)arena_base, is_used);

if (n_hi == NULL) {

assert(is_used); /* otherwise node should already exist */

return 0; /* failed to allocate space for node */

}

int i3 = MAP_BOT_INDEX((pymem_block *)arena_base);

int32_t tail = (int32_t)(arena_base & ARENA_SIZE_MASK);

if (tail == 0) {

/* is ideal arena address */

n_hi->arenas[i3].tail_hi = is_used ? -1 : 0;

}

else {

/* arena_base address is not ideal (aligned to arena size) and

* so it potentially covers two MAP_BOT nodes. Get the MAP_BOT node

* for the next arena. Note that it might be in different MAP_TOP

* and MAP_MID nodes as well so we need to call arena_map_get()

* again (do the full tree traversal).

*/

n_hi->arenas[i3].tail_hi = is_used ? tail : 0;

uintptr_t arena_base_next = arena_base + ARENA_SIZE;

/* If arena_base is a legit arena address, so is arena_base_next - 1

* (last address in arena). If arena_base_next overflows then it

* must overflow to 0. However, that would mean arena_base was

* "ideal" and we should not be in this case. */

assert(arena_base < arena_base_next);

arena_map_bot_t *n_lo = arena_map_get((pymem_block *)arena_base_next, is_used);

if (n_lo == NULL) {

assert(is_used); /* otherwise should already exist */

n_hi->arenas[i3].tail_hi = 0;

return 0; /* failed to allocate space for node */

}

int i3_next = MAP_BOT_INDEX(arena_base_next);

n_lo->arenas[i3_next].tail_lo = is_used ? tail : 0;

}

return 1;

}

/* Return true if 'p' is a pointer inside an obmalloc arena.

* _PyObject_Free() calls this so it needs to be very fast. */

static int

arena_map_is_used(pymem_block *p)

{

arena_map_bot_t *n = arena_map_get(p, 0);

if (n == NULL) {

return 0;

}

int i3 = MAP_BOT_INDEX(p);

/* ARENA_BITS must be < 32 so that the tail is a non-negative int32_t. */

int32_t hi = n->arenas[i3].tail_hi;

int32_t lo = n->arenas[i3].tail_lo;

int32_t tail = (int32_t)(AS_UINT(p) & ARENA_SIZE_MASK);

return (tail < lo) || (tail >= hi && hi != 0);

}

/* end of radix tree logic */

/*==========================================================================*/

#endif /* WITH_PYMALLOC_RADIX_TREE */

/* Allocate a new arena. If we run out of memory, return NULL. Else

* allocate a new arena, and return the address of an arena_object

* describing the new arena. It's expected that the caller will set

* `usable_arenas` to the return value.

*/

static struct arena_object*

new_arena(void)

{

struct arena_object* arenaobj;

uint excess; /* number of bytes above pool alignment */

void *address;

static int debug_stats = -1;

if (debug_stats == -1) {

const char *opt = Py_GETENV("PYTHONMALLOCSTATS");

debug_stats = (opt != NULL && *opt != '\0');

}

if (debug_stats) {

_PyObject_DebugMallocStats(stderr);

}

if (unused_arena_objects == NULL) {

uint i;

uint numarenas;

size_t nbytes;

/* Double the number of arena objects on each allocation.

* Note that it's possible for `numarenas` to overflow.

*/

numarenas = maxarenas ? maxarenas << 1 : INITIAL_ARENA_OBJECTS;

if (numarenas <= maxarenas)

return NULL; /* overflow */

#if SIZEOF_SIZE_T <= SIZEOF_INT

if (numarenas > SIZE_MAX / sizeof(*allarenas))

return NULL; /* overflow */

#endif

nbytes = numarenas * sizeof(*allarenas);

arenaobj = (struct arena_object *)PyMem_RawRealloc(allarenas, nbytes);

if (arenaobj == NULL)

return NULL;

allarenas = arenaobj;

/* We might need to fix pointers that were copied. However,

* new_arena only gets called when all the pages in the

* previous arenas are full. Thus, there are *no* pointers

* into the old array. Thus, we don't have to worry about

* invalid pointers. Just to be sure, some asserts:

*/

assert(usable_arenas == NULL);

assert(unused_arena_objects == NULL);

/* Put the new arenas on the unused_arena_objects list. */

for (i = maxarenas; i < numarenas; ++i) {

allarenas[i].address = 0; /* mark as unassociated */

allarenas[i].nextarena = i < numarenas - 1 ?

&allarenas[i+1] : NULL;

}

/* Update globals. */

unused_arena_objects = &allarenas[maxarenas];

maxarenas = numarenas;

}

/* Take the next available arena object off the head of the list. */

assert(unused_arena_objects != NULL);

arenaobj = unused_arena_objects;

unused_arena_objects = arenaobj->nextarena;

assert(arenaobj->address == 0);

address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE);

#if WITH_PYMALLOC_RADIX_TREE

if (address != NULL) {

if (!arena_map_mark_used((uintptr_t)address, 1)) {

/* marking arena in radix tree failed, abort */

_PyObject_Arena.free(_PyObject_Arena.ctx, address, ARENA_SIZE);

address = NULL;

}

}

#endif

if (address == NULL) {

/* The allocation failed: return NULL after putting the

* arenaobj back.

*/

arenaobj->nextarena = unused_arena_objects;

unused_arena_objects = arenaobj;

return NULL;

}

arenaobj->address = (uintptr_t)address;

++narenas_currently_allocated;

++ntimes_arena_allocated;

if (narenas_currently_allocated > narenas_highwater)

narenas_highwater = narenas_currently_allocated;

arenaobj->freepools = NULL;

/* pool_address <- first pool-aligned address in the arena

nfreepools <- number of whole pools that fit after alignment */