#include "Python.h"

#include "pycore_pymem.h" // _PyTraceMalloc_Config

#include "pycore_code.h" // stats

#include <stdbool.h>

#include <stdlib.h> // malloc()

/* Defined in tracemalloc.c */

extern void _PyMem_DumpTraceback(int fd, const void *ptr);

/* Python's malloc wrappers (see pymem.h) */

#undef uint

#define uint unsigned int /* assuming >= 16 bits */

/* Forward declaration */

static void* _PyMem_DebugRawMalloc(void *ctx, size_t size);

static void* _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize);

static void* _PyMem_DebugRawRealloc(void *ctx, void *ptr, size_t size);

static void _PyMem_DebugRawFree(void *ctx, void *ptr);

static void* _PyMem_DebugMalloc(void *ctx, size_t size);

static void* _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize);

static void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size);

static void _PyMem_DebugFree(void *ctx, void *p);

static void _PyObject_DebugDumpAddress(const void *p);

static void _PyMem_DebugCheckAddress(const char *func, char api_id, const void *p);

static void _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain);

#if defined(__has_feature) /* Clang */

# if __has_feature(address_sanitizer) /* is ASAN enabled? */

# define _Py_NO_SANITIZE_ADDRESS \

__attribute__((no_sanitize("address")))

# endif

# if __has_feature(thread_sanitizer) /* is TSAN enabled? */

# define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))

# endif

# if __has_feature(memory_sanitizer) /* is MSAN enabled? */

# define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory))

# endif

#elif defined(__GNUC__)

# if defined(__SANITIZE_ADDRESS__) /* GCC 4.8+, is ASAN enabled? */

# define _Py_NO_SANITIZE_ADDRESS \

__attribute__((no_sanitize_address))

# endif

// TSAN is supported since GCC 5.1, but __SANITIZE_THREAD__ macro

// is provided only since GCC 7.

# if __GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1)

# define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))

# endif

#endif

#ifndef _Py_NO_SANITIZE_ADDRESS

# define _Py_NO_SANITIZE_ADDRESS

#endif

#ifndef _Py_NO_SANITIZE_THREAD

# define _Py_NO_SANITIZE_THREAD

#endif

#ifndef _Py_NO_SANITIZE_MEMORY

# define _Py_NO_SANITIZE_MEMORY

#endif

#ifdef WITH_PYMALLOC

#ifdef MS_WINDOWS

# include <windows.h>

#elif defined(HAVE_MMAP)

# include <sys/mman.h>

# ifdef MAP_ANONYMOUS

# define ARENAS_USE_MMAP

# endif

#endif

/* Forward declaration */

static void* _PyObject_Malloc(void *ctx, size_t size);

static void* _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize);

static void _PyObject_Free(void *ctx, void *p);

static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);

#endif

/* bpo-35053: Declare tracemalloc configuration here rather than

Modules/_tracemalloc.c because _tracemalloc can be compiled as dynamic

library, whereas _Py_NewReference() requires it. */

struct _PyTraceMalloc_Config _Py_tracemalloc_config = _PyTraceMalloc_Config_INIT;

static void *

_PyMem_RawMalloc(void *ctx, size_t size)

{

/* PyMem_RawMalloc(0) means malloc(1). Some systems would return NULL

for malloc(0), which would be treated as an error. Some platforms would

return a pointer with no memory behind it, which would break pymalloc.

To solve these problems, allocate an extra byte. */

if (size == 0)

size = 1;

return malloc(size);

}

static void *

_PyMem_RawCalloc(void *ctx, size_t nelem, size_t elsize)

{

/* PyMem_RawCalloc(0, 0) means calloc(1, 1). Some systems would return NULL

for calloc(0, 0), which would be treated as an error. Some platforms

would return a pointer with no memory behind it, which would break

pymalloc. To solve these problems, allocate an extra byte. */

if (nelem == 0 || elsize == 0) {

nelem = 1;

elsize = 1;

}

return calloc(nelem, elsize);

}

static void *

_PyMem_RawRealloc(void *ctx, void *ptr, size_t size)

{

if (size == 0)

size = 1;

return realloc(ptr, size);

}

static void

_PyMem_RawFree(void *ctx, void *ptr)

{

free(ptr);

}

#ifdef MS_WINDOWS

static void *

_PyObject_ArenaVirtualAlloc(void *ctx, size_t size)

{

return VirtualAlloc(NULL, size,

MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);

}

static void

_PyObject_ArenaVirtualFree(void *ctx, void *ptr, size_t size)

{

VirtualFree(ptr, 0, MEM_RELEASE);

}

#elif defined(ARENAS_USE_MMAP)

static void *

_PyObject_ArenaMmap(void *ctx, size_t size)

{

void *ptr;

ptr = mmap(NULL, size, PROT_READ|PROT_WRITE,

MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);

if (ptr == MAP_FAILED)

return NULL;

assert(ptr != NULL);

return ptr;

}

static void

_PyObject_ArenaMunmap(void *ctx, void *ptr, size_t size)

{

munmap(ptr, size);

}

#else

static void *

_PyObject_ArenaMalloc(void *ctx, size_t size)

{

return malloc(size);

}

static void

_PyObject_ArenaFree(void *ctx, void *ptr, size_t size)

{

free(ptr);

}

#endif

#define MALLOC_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree}

#ifdef WITH_PYMALLOC

# define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free}

#endif

#define PYRAW_ALLOC MALLOC_ALLOC

#ifdef WITH_PYMALLOC

# define PYOBJ_ALLOC PYMALLOC_ALLOC

#else

# define PYOBJ_ALLOC MALLOC_ALLOC

#endif

#define PYMEM_ALLOC PYOBJ_ALLOC

typedef struct {

/* We tag each block with an API ID in order to tag API violations */

char api_id;

PyMemAllocatorEx alloc;

} debug_alloc_api_t;

static struct {

debug_alloc_api_t raw;

debug_alloc_api_t mem;

debug_alloc_api_t obj;

} _PyMem_Debug = {

{'r', PYRAW_ALLOC},

{'m', PYMEM_ALLOC},

{'o', PYOBJ_ALLOC}

};

#define PYDBGRAW_ALLOC \

{&_PyMem_Debug.raw, _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree}

#define PYDBGMEM_ALLOC \

{&_PyMem_Debug.mem, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree}

#define PYDBGOBJ_ALLOC \

{&_PyMem_Debug.obj, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree}

#ifdef Py_DEBUG

static PyMemAllocatorEx _PyMem_Raw = PYDBGRAW_ALLOC;

static PyMemAllocatorEx _PyMem = PYDBGMEM_ALLOC;

static PyMemAllocatorEx _PyObject = PYDBGOBJ_ALLOC;

#else

static PyMemAllocatorEx _PyMem_Raw = PYRAW_ALLOC;

static PyMemAllocatorEx _PyMem = PYMEM_ALLOC;

static PyMemAllocatorEx _PyObject = PYOBJ_ALLOC;

#endif

static int

pymem_set_default_allocator(PyMemAllocatorDomain domain, int debug,

PyMemAllocatorEx *old_alloc)

{

if (old_alloc != NULL) {

PyMem_GetAllocator(domain, old_alloc);

}

PyMemAllocatorEx new_alloc;

switch(domain)

{

case PYMEM_DOMAIN_RAW:

new_alloc = (PyMemAllocatorEx)PYRAW_ALLOC;

break;

case PYMEM_DOMAIN_MEM:

new_alloc = (PyMemAllocatorEx)PYMEM_ALLOC;

break;

case PYMEM_DOMAIN_OBJ:

new_alloc = (PyMemAllocatorEx)PYOBJ_ALLOC;

break;

default:

/* unknown domain */

return -1;

}

PyMem_SetAllocator(domain, &new_alloc);

if (debug) {

_PyMem_SetupDebugHooksDomain(domain);

}

return 0;

}

int

_PyMem_SetDefaultAllocator(PyMemAllocatorDomain domain,

PyMemAllocatorEx *old_alloc)

{

#ifdef Py_DEBUG

const int debug = 1;

#else

const int debug = 0;

#endif

return pymem_set_default_allocator(domain, debug, old_alloc);

}

int

_PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator)

{

if (name == NULL || *name == '\0') {

/* PYTHONMALLOC is empty or is not set or ignored (-E/-I command line

nameions): use default memory allocators */

*allocator = PYMEM_ALLOCATOR_DEFAULT;

}

else if (strcmp(name, "default") == 0) {

*allocator = PYMEM_ALLOCATOR_DEFAULT;

}

else if (strcmp(name, "debug") == 0) {

*allocator = PYMEM_ALLOCATOR_DEBUG;

}

#ifdef WITH_PYMALLOC

else if (strcmp(name, "pymalloc") == 0) {

*allocator = PYMEM_ALLOCATOR_PYMALLOC;

}

else if (strcmp(name, "pymalloc_debug") == 0) {

*allocator = PYMEM_ALLOCATOR_PYMALLOC_DEBUG;

}

#endif

else if (strcmp(name, "malloc") == 0) {

*allocator = PYMEM_ALLOCATOR_MALLOC;

}

else if (strcmp(name, "malloc_debug") == 0) {

*allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG;

}

else {

/* unknown allocator */

return -1;

}

return 0;

}

int

_PyMem_SetupAllocators(PyMemAllocatorName allocator)

{

switch (allocator) {

case PYMEM_ALLOCATOR_NOT_SET:

/* do nothing */

break;

case PYMEM_ALLOCATOR_DEFAULT:

(void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, NULL);

(void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_MEM, NULL);

(void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_OBJ, NULL);

break;

case PYMEM_ALLOCATOR_DEBUG:

(void)pymem_set_default_allocator(PYMEM_DOMAIN_RAW, 1, NULL);

(void)pymem_set_default_allocator(PYMEM_DOMAIN_MEM, 1, NULL);

(void)pymem_set_default_allocator(PYMEM_DOMAIN_OBJ, 1, NULL);

break;

#ifdef WITH_PYMALLOC

case PYMEM_ALLOCATOR_PYMALLOC:

case PYMEM_ALLOCATOR_PYMALLOC_DEBUG:

{

PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &malloc_alloc);

PyMemAllocatorEx pymalloc = PYMALLOC_ALLOC;

PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &pymalloc);

PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &pymalloc);

if (allocator == PYMEM_ALLOCATOR_PYMALLOC_DEBUG) {

PyMem_SetupDebugHooks();

}

break;

}

#endif

case PYMEM_ALLOCATOR_MALLOC:

case PYMEM_ALLOCATOR_MALLOC_DEBUG:

{

PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &malloc_alloc);

PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &malloc_alloc);

PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &malloc_alloc);

if (allocator == PYMEM_ALLOCATOR_MALLOC_DEBUG) {

PyMem_SetupDebugHooks();

}

break;

}

default:

/* unknown allocator */

return -1;

}

return 0;

}

static int

pymemallocator_eq(PyMemAllocatorEx *a, PyMemAllocatorEx *b)

{

return (memcmp(a, b, sizeof(PyMemAllocatorEx)) == 0);

}

const char*

_PyMem_GetCurrentAllocatorName(void)

{

PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;

#ifdef WITH_PYMALLOC

PyMemAllocatorEx pymalloc = PYMALLOC_ALLOC;

#endif

if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) &&

pymemallocator_eq(&_PyMem, &malloc_alloc) &&

pymemallocator_eq(&_PyObject, &malloc_alloc))

{

return "malloc";

}

#ifdef WITH_PYMALLOC

if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) &&

pymemallocator_eq(&_PyMem, &pymalloc) &&

pymemallocator_eq(&_PyObject, &pymalloc))

{

return "pymalloc";

}

#endif

PyMemAllocatorEx dbg_raw = PYDBGRAW_ALLOC;

PyMemAllocatorEx dbg_mem = PYDBGMEM_ALLOC;

PyMemAllocatorEx dbg_obj = PYDBGOBJ_ALLOC;

if (pymemallocator_eq(&_PyMem_Raw, &dbg_raw) &&

pymemallocator_eq(&_PyMem, &dbg_mem) &&

pymemallocator_eq(&_PyObject, &dbg_obj))

{

/* Debug hooks installed */

if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) &&

pymemallocator_eq(&_PyMem_Debug.mem.alloc, &malloc_alloc) &&

pymemallocator_eq(&_PyMem_Debug.obj.alloc, &malloc_alloc))

{

return "malloc_debug";

}

#ifdef WITH_PYMALLOC

if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) &&

pymemallocator_eq(&_PyMem_Debug.mem.alloc, &pymalloc) &&

pymemallocator_eq(&_PyMem_Debug.obj.alloc, &pymalloc))

{

return "pymalloc_debug";

}

#endif

}

return NULL;

}

#undef MALLOC_ALLOC

#undef PYMALLOC_ALLOC

#undef PYRAW_ALLOC

#undef PYMEM_ALLOC

#undef PYOBJ_ALLOC

#undef PYDBGRAW_ALLOC

#undef PYDBGMEM_ALLOC

#undef PYDBGOBJ_ALLOC

static PyObjectArenaAllocator _PyObject_Arena = {NULL,

#ifdef MS_WINDOWS

_PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree

#elif defined(ARENAS_USE_MMAP)

_PyObject_ArenaMmap, _PyObject_ArenaMunmap

#else

_PyObject_ArenaMalloc, _PyObject_ArenaFree

#endif

};

#ifdef WITH_PYMALLOC

static int

_PyMem_DebugEnabled(void)

{

return (_PyObject.malloc == _PyMem_DebugMalloc);

}

static int

_PyMem_PymallocEnabled(void)

{

if (_PyMem_DebugEnabled()) {

return (_PyMem_Debug.obj.alloc.malloc == _PyObject_Malloc);

}

else {

return (_PyObject.malloc == _PyObject_Malloc);

}

}

#endif

static void

_PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain)

{

PyMemAllocatorEx alloc;

if (domain == PYMEM_DOMAIN_RAW) {

if (_PyMem_Raw.malloc == _PyMem_DebugRawMalloc) {

return;

}

PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &_PyMem_Debug.raw.alloc);

alloc.ctx = &_PyMem_Debug.raw;

alloc.malloc = _PyMem_DebugRawMalloc;

alloc.calloc = _PyMem_DebugRawCalloc;

alloc.realloc = _PyMem_DebugRawRealloc;

alloc.free = _PyMem_DebugRawFree;

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc);

}

else if (domain == PYMEM_DOMAIN_MEM) {

if (_PyMem.malloc == _PyMem_DebugMalloc) {

return;

}

PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc);

alloc.ctx = &_PyMem_Debug.mem;

alloc.malloc = _PyMem_DebugMalloc;

alloc.calloc = _PyMem_DebugCalloc;

alloc.realloc = _PyMem_DebugRealloc;

alloc.free = _PyMem_DebugFree;

PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc);

}

else if (domain == PYMEM_DOMAIN_OBJ) {

if (_PyObject.malloc == _PyMem_DebugMalloc) {

return;

}

PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc);

alloc.ctx = &_PyMem_Debug.obj;

alloc.malloc = _PyMem_DebugMalloc;

alloc.calloc = _PyMem_DebugCalloc;

alloc.realloc = _PyMem_DebugRealloc;

alloc.free = _PyMem_DebugFree;

PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc);

}

}

void

PyMem_SetupDebugHooks(void)

{

_PyMem_SetupDebugHooksDomain(PYMEM_DOMAIN_RAW);

_PyMem_SetupDebugHooksDomain(PYMEM_DOMAIN_MEM);

_PyMem_SetupDebugHooksDomain(PYMEM_DOMAIN_OBJ);

}

void

PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)

{

switch(domain)

{

case PYMEM_DOMAIN_RAW: *allocator = _PyMem_Raw; break;

case PYMEM_DOMAIN_MEM: *allocator = _PyMem; break;

case PYMEM_DOMAIN_OBJ: *allocator = _PyObject; break;

default:

/* unknown domain: set all attributes to NULL */

allocator->ctx = NULL;

allocator->malloc = NULL;

allocator->calloc = NULL;

allocator->realloc = NULL;

allocator->free = NULL;

}

}

void

PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)

{

switch(domain)

{

case PYMEM_DOMAIN_RAW: _PyMem_Raw = *allocator; break;

case PYMEM_DOMAIN_MEM: _PyMem = *allocator; break;

case PYMEM_DOMAIN_OBJ: _PyObject = *allocator; break;

/* ignore unknown domain */

}

}

void

PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)

{

*allocator = _PyObject_Arena;

}

void *

_PyObject_VirtualAlloc(size_t size)

{

return _PyObject_Arena.alloc(_PyObject_Arena.ctx, size);

}

void

_PyObject_VirtualFree(void *obj, size_t size)

{

_PyObject_Arena.free(_PyObject_Arena.ctx, obj, size);

}

void

PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)

{

_PyObject_Arena = *allocator;

}

void *

PyMem_RawMalloc(size_t size)

{

/*

* Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.

* Most python internals blindly use a signed Py_ssize_t to track

* things without checking for overflows or negatives.

* As size_t is unsigned, checking for size < 0 is not required.

*/

if (size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size);

}

void *

PyMem_RawCalloc(size_t nelem, size_t elsize)

{

/* see PyMem_RawMalloc() */

if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)

return NULL;

return _PyMem_Raw.calloc(_PyMem_Raw.ctx, nelem, elsize);

}

void*

PyMem_RawRealloc(void *ptr, size_t new_size)

{

/* see PyMem_RawMalloc() */

if (new_size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size);

}

void PyMem_RawFree(void *ptr)

{

_PyMem_Raw.free(_PyMem_Raw.ctx, ptr);

}

void *

PyMem_Malloc(size_t size)

{

/* see PyMem_RawMalloc() */

if (size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyMem.malloc(_PyMem.ctx, size);

}

void *

PyMem_Calloc(size_t nelem, size_t elsize)

{

/* see PyMem_RawMalloc() */

if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)

return NULL;

return _PyMem.calloc(_PyMem.ctx, nelem, elsize);

}

void *

PyMem_Realloc(void *ptr, size_t new_size)

{

/* see PyMem_RawMalloc() */

if (new_size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyMem.realloc(_PyMem.ctx, ptr, new_size);

}

void

PyMem_Free(void *ptr)

{

_PyMem.free(_PyMem.ctx, ptr);

}

wchar_t*

_PyMem_RawWcsdup(const wchar_t *str)

{

assert(str != NULL);

size_t len = wcslen(str);

if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) {

return NULL;

}

size_t size = (len + 1) * sizeof(wchar_t);

wchar_t *str2 = PyMem_RawMalloc(size);

if (str2 == NULL) {

return NULL;

}

memcpy(str2, str, size);

return str2;

}

char *

_PyMem_RawStrdup(const char *str)

{

assert(str != NULL);

size_t size = strlen(str) + 1;

char *copy = PyMem_RawMalloc(size);

if (copy == NULL) {

return NULL;

}

memcpy(copy, str, size);

return copy;

}

char *

_PyMem_Strdup(const char *str)

{

assert(str != NULL);

size_t size = strlen(str) + 1;

char *copy = PyMem_Malloc(size);

if (copy == NULL) {

return NULL;

}

memcpy(copy, str, size);

return copy;

}

void *

PyObject_Malloc(size_t size)

{

/* see PyMem_RawMalloc() */

if (size > (size_t)PY_SSIZE_T_MAX)

return NULL;

OBJECT_STAT_INC(allocations);

return _PyObject.malloc(_PyObject.ctx, size);

}

void *

PyObject_Calloc(size_t nelem, size_t elsize)

{

/* see PyMem_RawMalloc() */

if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)

return NULL;

OBJECT_STAT_INC(allocations);

return _PyObject.calloc(_PyObject.ctx, nelem, elsize);

}

void *

PyObject_Realloc(void *ptr, size_t new_size)

{

/* see PyMem_RawMalloc() */

if (new_size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyObject.realloc(_PyObject.ctx, ptr, new_size);

}

void

PyObject_Free(void *ptr)

{

OBJECT_STAT_INC(frees);

_PyObject.free(_PyObject.ctx, ptr);

}

/* If we're using GCC, use __builtin_expect() to reduce overhead of

the valgrind checks */

#if defined(__GNUC__) && (__GNUC__ > 2) && defined(__OPTIMIZE__)

# define UNLIKELY(value) __builtin_expect((value), 0)

# define LIKELY(value) __builtin_expect((value), 1)

#else

# define UNLIKELY(value) (value)

# define LIKELY(value) (value)

#endif

#ifdef WITH_PYMALLOC

#ifdef WITH_VALGRIND

#include <valgrind/valgrind.h>

/* -1 indicates that we haven't checked that we're running on valgrind yet. */

static int running_on_valgrind = -1;

#endif

/* An object allocator for Python.

Here is an introduction to the layers of the Python memory architecture,

showing where the object allocator is actually used (layer +2), It is

called for every object allocation and deallocation (PyObject_New/Del),

unless the object-specific allocators implement a proprietary allocation

scheme (ex.: ints use a simple free list). This is also the place where

the cyclic garbage collector operates selectively on container objects.

Object-specific allocators

_____ ______ ______ ________

[ int ] [ dict ] [ list ] ... [ string ] Python core |

+3 | <----- Object-specific memory -----> | <-- Non-object memory --> |

_______________________________ | |

[ Python's object allocator ] | |

+2 | ####### Object memory ####### | <------ Internal buffers ------> |

______________________________________________________________ |

[ Python's raw memory allocator (PyMem_ API) ] |

+1 | <----- Python memory (under PyMem manager's control) ------> | |

__________________________________________________________________

[ Underlying general-purpose allocator (ex: C library malloc) ]

0 | <------ Virtual memory allocated for the python process -------> |

=========================================================================

_______________________________________________________________________

[ OS-specific Virtual Memory Manager (VMM) ]

-1 | <--- Kernel dynamic storage allocation & management (page-based) ---> |

__________________________________ __________________________________

[ ] [ ]

-2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> |

*/

/*==========================================================================*/

/* A fast, special-purpose memory allocator for small blocks, to be used

on top of a general-purpose malloc -- heavily based on previous art. */

/* Vladimir Marangozov -- August 2000 */

/*

* "Memory management is where the rubber meets the road -- if we do the wrong

* thing at any level, the results will not be good. And if we don't make the

* levels work well together, we are in serious trouble." (1)

*

* (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles,

* "Dynamic Storage Allocation: A Survey and Critical Review",

* in Proc. 1995 Int'l. Workshop on Memory Management, September 1995.

*/

/* #undef WITH_MEMORY_LIMITS */ /* disable mem limit checks */

/*==========================================================================*/

/*

* Allocation strategy abstract:

*

* For small requests, the allocator sub-allocates <Big> blocks of memory.

* Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the

* system's allocator.

*

* Small requests are grouped in size classes spaced 8 bytes apart, due

* to the required valid alignment of the returned address. Requests of

* a particular size are serviced from memory pools of 4K (one VMM page).

* Pools are fragmented on demand and contain free lists of blocks of one

* particular size class. In other words, there is a fixed-size allocator

* for each size class. Free pools are shared by the different allocators

* thus minimizing the space reserved for a particular size class.

*

* This allocation strategy is a variant of what is known as "simple

* segregated storage based on array of free lists". The main drawback of

* simple segregated storage is that we might end up with lot of reserved

* memory for the different free lists, which degenerate in time. To avoid

* this, we partition each free list in pools and we share dynamically the

* reserved space between all free lists. This technique is quite efficient

* for memory intensive programs which allocate mainly small-sized blocks.

*

* For small requests we have the following table:

*

* Request in bytes Size of allocated block Size class idx

* ----------------------------------------------------------------

* 1-8 8 0

* 9-16 16 1

* 17-24 24 2

* 25-32 32 3

* 33-40 40 4

* 41-48 48 5

* 49-56 56 6

* 57-64 64 7

* 65-72 72 8

* ... ... ...

* 497-504 504 62

* 505-512 512 63

*

* 0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying

* allocator.

*/

/*==========================================================================*/

/*

* -- Main tunable settings section --

*/

/*

* Alignment of addresses returned to the user. 8-bytes alignment works

* on most current architectures (with 32-bit or 64-bit address buses).

* The alignment value is also used for grouping small requests in size

* classes spaced ALIGNMENT bytes apart.

*

* You shouldn't change this unless you know what you are doing.

*/

#if SIZEOF_VOID_P > 4

#define ALIGNMENT 16 /* must be 2^N */

#define ALIGNMENT_SHIFT 4

#else

#define ALIGNMENT 8 /* must be 2^N */

#define ALIGNMENT_SHIFT 3

#endif

/* Return the number of bytes in size class I, as a uint. */

#define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT)

/*

* Max size threshold below which malloc requests are considered to be

* small enough in order to use preallocated memory pools. You can tune

* this value according to your application behaviour and memory needs.

*

* Note: a size threshold of 512 guarantees that newly created dictionaries

* will be allocated from preallocated memory pools on 64-bit.

*

* The following invariants must hold:

* 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512

* 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT

*

* Although not required, for better performance and space efficiency,

* it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2.

*/

#define SMALL_REQUEST_THRESHOLD 512

#define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT)

/*

* The system's VMM page size can be obtained on most unices with a

* getpagesize() call or deduced from various header files. To make

* things simpler, we assume that it is 4K, which is OK for most systems.

* It is probably better if this is the native page size, but it doesn't

* have to be. In theory, if SYSTEM_PAGE_SIZE is larger than the native page

* size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation

* violation fault. 4K is apparently OK for all the platforms that python

* currently targets.

*/

#define SYSTEM_PAGE_SIZE (4 * 1024)

/*

* Maximum amount of memory managed by the allocator for small requests.

*/

#ifdef WITH_MEMORY_LIMITS

#ifndef SMALL_MEMORY_LIMIT

#define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MB -- more? */

#endif

#endif

#if !defined(WITH_PYMALLOC_RADIX_TREE)

/* Use radix-tree to track arena memory regions, for address_in_range().

* Enable by default since it allows larger pool sizes. Can be disabled

* using -DWITH_PYMALLOC_RADIX_TREE=0 */

#define WITH_PYMALLOC_RADIX_TREE 1

#endif

#if SIZEOF_VOID_P > 4

/* on 64-bit platforms use larger pools and arenas if we can */

#define USE_LARGE_ARENAS

#if WITH_PYMALLOC_RADIX_TREE

/* large pools only supported if radix-tree is enabled */

#define USE_LARGE_POOLS

#endif

#endif

/*

* The allocator sub-allocates <Big> blocks of memory (called arenas) aligned

* on a page boundary. This is a reserved virtual address space for the

* current process (obtained through a malloc()/mmap() call). In no way this

* means that the memory arenas will be used entirely. A malloc(<Big>) is

* usually an address range reservation for <Big> bytes, unless all pages within

* this space are referenced subsequently. So malloc'ing big blocks and not

* using them does not mean "wasting memory". It's an addressable range

* wastage...

*

* Arenas are allocated with mmap() on systems supporting anonymous memory

* mappings to reduce heap fragmentation.

*/

#ifdef USE_LARGE_ARENAS

#define ARENA_BITS 20 /* 1 MiB */

#else

#define ARENA_BITS 18 /* 256 KiB */

#endif

#define ARENA_SIZE (1 << ARENA_BITS)

#define ARENA_SIZE_MASK (ARENA_SIZE - 1)

#ifdef WITH_MEMORY_LIMITS

#define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE)

#endif

/*

* Size of the pools used for small blocks. Must be a power of 2.

*/

#ifdef USE_LARGE_POOLS

#define POOL_BITS 14 /* 16 KiB */

#else

#define POOL_BITS 12 /* 4 KiB */

#endif

#define POOL_SIZE (1 << POOL_BITS)

#define POOL_SIZE_MASK (POOL_SIZE - 1)

#if !WITH_PYMALLOC_RADIX_TREE

#if POOL_SIZE != SYSTEM_PAGE_SIZE

# error "pool size must be equal to system page size"

#endif

#endif

#define MAX_POOLS_IN_ARENA (ARENA_SIZE / POOL_SIZE)

#if MAX_POOLS_IN_ARENA * POOL_SIZE != ARENA_SIZE

# error "arena size not an exact multiple of pool size"

#endif

/*

* -- End of tunable settings section --

*/

/*==========================================================================*/

/* When you say memory, my mind reasons in terms of (pointers to) blocks */

typedef uint8_t block;

/* Pool for small blocks. */

struct pool_header {

union { block *_padding;

uint count; } ref; /* number of allocated blocks */

block *freeblock; /* pool's free list head */

struct pool_header *nextpool; /* next pool of this size class */

struct pool_header *prevpool; /* previous pool "" */

uint arenaindex; /* index into arenas of base adr */

uint szidx; /* block size class index */

uint nextoffset; /* bytes to virgin block */

uint maxnextoffset; /* largest valid nextoffset */

};

typedef struct pool_header *poolp;

/* Record keeping for arenas. */