diff options
| author | shadchin <[email protected]> | 2022-02-10 16:44:30 +0300 |
|---|---|---|
| committer | Daniil Cherednik <[email protected]> | 2022-02-10 16:44:30 +0300 |
| commit | 2598ef1d0aee359b4b6d5fdd1758916d5907d04f (patch) | |
| tree | 012bb94d777798f1f56ac1cec429509766d05181 /contrib/tools/python3/src/Objects/obmalloc.c | |
| parent | 6751af0b0c1b952fede40b19b71da8025b5d8bcf (diff) | |
Restoring authorship annotation for <[email protected]>. Commit 1 of 2.
Diffstat (limited to 'contrib/tools/python3/src/Objects/obmalloc.c')
| -rw-r--r-- | contrib/tools/python3/src/Objects/obmalloc.c | 1050 |
1 files changed, 525 insertions, 525 deletions
diff --git a/contrib/tools/python3/src/Objects/obmalloc.c b/contrib/tools/python3/src/Objects/obmalloc.c index 9f8e0d114ff..1bff84b2325 100644 --- a/contrib/tools/python3/src/Objects/obmalloc.c +++ b/contrib/tools/python3/src/Objects/obmalloc.c @@ -1,5 +1,5 @@ #include "Python.h" -#include "pycore_pymem.h" // _PyTraceMalloc_Config +#include "pycore_pymem.h" // _PyTraceMalloc_Config #include <stdbool.h> @@ -25,14 +25,14 @@ static void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size); static void _PyMem_DebugFree(void *ctx, void *p); static void _PyObject_DebugDumpAddress(const void *p); -static void _PyMem_DebugCheckAddress(const char *func, char api_id, const void *p); +static void _PyMem_DebugCheckAddress(const char *func, char api_id, const void *p); static void _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain); #if defined(__has_feature) /* Clang */ # if __has_feature(address_sanitizer) /* is ASAN enabled? */ -# define _Py_NO_SANITIZE_ADDRESS \ - __attribute__((no_sanitize("address"))) +# define _Py_NO_SANITIZE_ADDRESS \ + __attribute__((no_sanitize("address"))) # endif # if __has_feature(thread_sanitizer) /* is TSAN enabled? */ # define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread)) @@ -42,18 +42,18 @@ static void _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain); # endif #elif defined(__GNUC__) # if defined(__SANITIZE_ADDRESS__) /* GCC 4.8+, is ASAN enabled? */ -# define _Py_NO_SANITIZE_ADDRESS \ - __attribute__((no_sanitize_address)) +# define _Py_NO_SANITIZE_ADDRESS \ + __attribute__((no_sanitize_address)) # endif - // TSAN is supported since GCC 5.1, but __SANITIZE_THREAD__ macro + // TSAN is supported since GCC 5.1, but __SANITIZE_THREAD__ macro // is provided only since GCC 7. -# if __GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1) +# if __GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1) # define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread)) # endif #endif -#ifndef _Py_NO_SANITIZE_ADDRESS -# define _Py_NO_SANITIZE_ADDRESS +#ifndef _Py_NO_SANITIZE_ADDRESS +# define _Py_NO_SANITIZE_ADDRESS #endif #ifndef _Py_NO_SANITIZE_THREAD # define _Py_NO_SANITIZE_THREAD @@ -81,12 +81,12 @@ static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size); #endif -/* bpo-35053: Declare tracemalloc configuration here rather than - Modules/_tracemalloc.c because _tracemalloc can be compiled as dynamic - library, whereas _Py_NewReference() requires it. */ -struct _PyTraceMalloc_Config _Py_tracemalloc_config = _PyTraceMalloc_Config_INIT; - - +/* bpo-35053: Declare tracemalloc configuration here rather than + Modules/_tracemalloc.c because _tracemalloc can be compiled as dynamic + library, whereas _Py_NewReference() requires it. */ +struct _PyTraceMalloc_Config _Py_tracemalloc_config = _PyTraceMalloc_Config_INIT; + + static void * _PyMem_RawMalloc(void *ctx, size_t size) { @@ -268,65 +268,65 @@ _PyMem_SetDefaultAllocator(PyMemAllocatorDomain domain, int -_PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator) +_PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator) { - if (name == NULL || *name == '\0') { + if (name == NULL || *name == '\0') { /* PYTHONMALLOC is empty or is not set or ignored (-E/-I command line - nameions): use default memory allocators */ - *allocator = PYMEM_ALLOCATOR_DEFAULT; - } - else if (strcmp(name, "default") == 0) { - *allocator = PYMEM_ALLOCATOR_DEFAULT; - } - else if (strcmp(name, "debug") == 0) { - *allocator = PYMEM_ALLOCATOR_DEBUG; - } -#ifdef WITH_PYMALLOC - else if (strcmp(name, "pymalloc") == 0) { - *allocator = PYMEM_ALLOCATOR_PYMALLOC; - } - else if (strcmp(name, "pymalloc_debug") == 0) { - *allocator = PYMEM_ALLOCATOR_PYMALLOC_DEBUG; - } -#endif - else if (strcmp(name, "malloc") == 0) { - *allocator = PYMEM_ALLOCATOR_MALLOC; - } - else if (strcmp(name, "malloc_debug") == 0) { - *allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG; - } - else { - /* unknown allocator */ - return -1; - } - return 0; -} - - -int -_PyMem_SetupAllocators(PyMemAllocatorName allocator) -{ - switch (allocator) { - case PYMEM_ALLOCATOR_NOT_SET: - /* do nothing */ - break; - - case PYMEM_ALLOCATOR_DEFAULT: + nameions): use default memory allocators */ + *allocator = PYMEM_ALLOCATOR_DEFAULT; + } + else if (strcmp(name, "default") == 0) { + *allocator = PYMEM_ALLOCATOR_DEFAULT; + } + else if (strcmp(name, "debug") == 0) { + *allocator = PYMEM_ALLOCATOR_DEBUG; + } +#ifdef WITH_PYMALLOC + else if (strcmp(name, "pymalloc") == 0) { + *allocator = PYMEM_ALLOCATOR_PYMALLOC; + } + else if (strcmp(name, "pymalloc_debug") == 0) { + *allocator = PYMEM_ALLOCATOR_PYMALLOC_DEBUG; + } +#endif + else if (strcmp(name, "malloc") == 0) { + *allocator = PYMEM_ALLOCATOR_MALLOC; + } + else if (strcmp(name, "malloc_debug") == 0) { + *allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG; + } + else { + /* unknown allocator */ + return -1; + } + return 0; +} + + +int +_PyMem_SetupAllocators(PyMemAllocatorName allocator) +{ + switch (allocator) { + case PYMEM_ALLOCATOR_NOT_SET: + /* do nothing */ + break; + + case PYMEM_ALLOCATOR_DEFAULT: (void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, NULL); (void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_MEM, NULL); (void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_OBJ, NULL); - break; - - case PYMEM_ALLOCATOR_DEBUG: + break; + + case PYMEM_ALLOCATOR_DEBUG: (void)pymem_set_default_allocator(PYMEM_DOMAIN_RAW, 1, NULL); (void)pymem_set_default_allocator(PYMEM_DOMAIN_MEM, 1, NULL); (void)pymem_set_default_allocator(PYMEM_DOMAIN_OBJ, 1, NULL); - break; - + break; + #ifdef WITH_PYMALLOC - case PYMEM_ALLOCATOR_PYMALLOC: - case PYMEM_ALLOCATOR_PYMALLOC_DEBUG: - { + case PYMEM_ALLOCATOR_PYMALLOC: + case PYMEM_ALLOCATOR_PYMALLOC_DEBUG: + { PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC; PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &malloc_alloc); @@ -334,28 +334,28 @@ _PyMem_SetupAllocators(PyMemAllocatorName allocator) PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &pymalloc); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &pymalloc); - if (allocator == PYMEM_ALLOCATOR_PYMALLOC_DEBUG) { + if (allocator == PYMEM_ALLOCATOR_PYMALLOC_DEBUG) { PyMem_SetupDebugHooks(); } - break; + break; } #endif - - case PYMEM_ALLOCATOR_MALLOC: - case PYMEM_ALLOCATOR_MALLOC_DEBUG: - { + + case PYMEM_ALLOCATOR_MALLOC: + case PYMEM_ALLOCATOR_MALLOC_DEBUG: + { PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC; PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &malloc_alloc); PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &malloc_alloc); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &malloc_alloc); - if (allocator == PYMEM_ALLOCATOR_MALLOC_DEBUG) { + if (allocator == PYMEM_ALLOCATOR_MALLOC_DEBUG) { PyMem_SetupDebugHooks(); } - break; + break; } - - default: + + default: /* unknown allocator */ return -1; } @@ -371,7 +371,7 @@ pymemallocator_eq(PyMemAllocatorEx *a, PyMemAllocatorEx *b) const char* -_PyMem_GetCurrentAllocatorName(void) +_PyMem_GetCurrentAllocatorName(void) { PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC; #ifdef WITH_PYMALLOC @@ -714,17 +714,17 @@ PyObject_Free(void *ptr) the valgrind checks */ #if defined(__GNUC__) && (__GNUC__ > 2) && defined(__OPTIMIZE__) # define UNLIKELY(value) __builtin_expect((value), 0) -# define LIKELY(value) __builtin_expect((value), 1) +# define LIKELY(value) __builtin_expect((value), 1) #else # define UNLIKELY(value) (value) -# define LIKELY(value) (value) +# define LIKELY(value) (value) #endif -#ifdef WITH_PYMALLOC - -#ifdef WITH_VALGRIND -#include <valgrind/valgrind.h> - +#ifdef WITH_PYMALLOC + +#ifdef WITH_VALGRIND +#include <valgrind/valgrind.h> + /* -1 indicates that we haven't checked that we're running on valgrind yet. */ static int running_on_valgrind = -1; #endif @@ -836,20 +836,20 @@ static int running_on_valgrind = -1; /* * Alignment of addresses returned to the user. 8-bytes alignment works - * on most current architectures (with 32-bit or 64-bit address buses). + * on most current architectures (with 32-bit or 64-bit address buses). * The alignment value is also used for grouping small requests in size * classes spaced ALIGNMENT bytes apart. * * You shouldn't change this unless you know what you are doing. */ - -#if SIZEOF_VOID_P > 4 -#define ALIGNMENT 16 /* must be 2^N */ -#define ALIGNMENT_SHIFT 4 -#else + +#if SIZEOF_VOID_P > 4 +#define ALIGNMENT 16 /* must be 2^N */ +#define ALIGNMENT_SHIFT 4 +#else #define ALIGNMENT 8 /* must be 2^N */ #define ALIGNMENT_SHIFT 3 -#endif +#endif /* Return the number of bytes in size class I, as a uint. */ #define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT) @@ -920,11 +920,11 @@ static int running_on_valgrind = -1; #define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */ #define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK -#define MAX_POOLS_IN_ARENA (ARENA_SIZE / POOL_SIZE) -#if MAX_POOLS_IN_ARENA * POOL_SIZE != ARENA_SIZE -# error "arena size not an exact multiple of pool size" -#endif - +#define MAX_POOLS_IN_ARENA (ARENA_SIZE / POOL_SIZE) +#if MAX_POOLS_IN_ARENA * POOL_SIZE != ARENA_SIZE +# error "arena size not an exact multiple of pool size" +#endif + /* * -- End of tunable settings section -- */ @@ -1162,18 +1162,18 @@ usable_arenas Note that an arena_object associated with an arena all of whose pools are currently in use isn't on either list. - -Changed in Python 3.8: keeping usable_arenas sorted by number of free pools -used to be done by one-at-a-time linear search when an arena's number of -free pools changed. That could, overall, consume time quadratic in the -number of arenas. That didn't really matter when there were only a few -hundred arenas (typical!), but could be a timing disaster when there were -hundreds of thousands. See bpo-37029. - -Now we have a vector of "search fingers" to eliminate the need to search: -nfp2lasta[nfp] returns the last ("rightmost") arena in usable_arenas -with nfp free pools. This is NULL if and only if there is no arena with -nfp free pools in usable_arenas. + +Changed in Python 3.8: keeping usable_arenas sorted by number of free pools +used to be done by one-at-a-time linear search when an arena's number of +free pools changed. That could, overall, consume time quadratic in the +number of arenas. That didn't really matter when there were only a few +hundred arenas (typical!), but could be a timing disaster when there were +hundreds of thousands. See bpo-37029. + +Now we have a vector of "search fingers" to eliminate the need to search: +nfp2lasta[nfp] returns the last ("rightmost") arena in usable_arenas +with nfp free pools. This is NULL if and only if there is no arena with +nfp free pools in usable_arenas. */ /* Array of objects used to track chunks of memory (arenas). */ @@ -1191,9 +1191,9 @@ static struct arena_object* unused_arena_objects = NULL; */ static struct arena_object* usable_arenas = NULL; -/* nfp2lasta[nfp] is the last arena in usable_arenas with nfp free pools */ -static struct arena_object* nfp2lasta[MAX_POOLS_IN_ARENA + 1] = { NULL }; - +/* nfp2lasta[nfp] is the last arena in usable_arenas with nfp free pools */ +static struct arena_object* nfp2lasta[MAX_POOLS_IN_ARENA + 1] = { NULL }; + /* How many arena_objects do we initially allocate? * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the * `arenas` vector. @@ -1208,29 +1208,29 @@ static size_t ntimes_arena_allocated = 0; /* High water mark (max value ever seen) for narenas_currently_allocated. */ static size_t narenas_highwater = 0; -static Py_ssize_t raw_allocated_blocks; +static Py_ssize_t raw_allocated_blocks; Py_ssize_t _Py_GetAllocatedBlocks(void) { - Py_ssize_t n = raw_allocated_blocks; - /* add up allocated blocks for used pools */ - for (uint i = 0; i < maxarenas; ++i) { - /* Skip arenas which are not allocated. */ - if (arenas[i].address == 0) { - continue; - } - - uintptr_t base = (uintptr_t)_Py_ALIGN_UP(arenas[i].address, POOL_SIZE); - - /* visit every pool in the arena */ - assert(base <= (uintptr_t) arenas[i].pool_address); - for (; base < (uintptr_t) arenas[i].pool_address; base += POOL_SIZE) { - poolp p = (poolp)base; - n += p->ref.count; - } - } - return n; + Py_ssize_t n = raw_allocated_blocks; + /* add up allocated blocks for used pools */ + for (uint i = 0; i < maxarenas; ++i) { + /* Skip arenas which are not allocated. */ + if (arenas[i].address == 0) { + continue; + } + + uintptr_t base = (uintptr_t)_Py_ALIGN_UP(arenas[i].address, POOL_SIZE); + + /* visit every pool in the arena */ + assert(base <= (uintptr_t) arenas[i].pool_address); + for (; base < (uintptr_t) arenas[i].pool_address; base += POOL_SIZE) { + poolp p = (poolp)base; + n += p->ref.count; + } + } + return n; } @@ -1320,7 +1320,7 @@ new_arena(void) /* pool_address <- first pool-aligned address in the arena nfreepools <- number of whole pools that fit after alignment */ arenaobj->pool_address = (block*)arenaobj->address; - arenaobj->nfreepools = MAX_POOLS_IN_ARENA; + arenaobj->nfreepools = MAX_POOLS_IN_ARENA; excess = (uint)(arenaobj->address & POOL_SIZE_MASK); if (excess != 0) { --arenaobj->nfreepools; @@ -1407,7 +1407,7 @@ obmalloc controls. Since this test is needed at every entry point, it's extremely desirable that it be this fast. */ -static bool _Py_NO_SANITIZE_ADDRESS +static bool _Py_NO_SANITIZE_ADDRESS _Py_NO_SANITIZE_THREAD _Py_NO_SANITIZE_MEMORY address_in_range(void *p, poolp pool) @@ -1426,76 +1426,76 @@ address_in_range(void *p, poolp pool) /*==========================================================================*/ -// Called when freelist is exhausted. Extend the freelist if there is -// space for a block. Otherwise, remove this pool from usedpools. -static void -pymalloc_pool_extend(poolp pool, uint size) -{ - if (UNLIKELY(pool->nextoffset <= pool->maxnextoffset)) { - /* There is room for another block. */ - pool->freeblock = (block*)pool + pool->nextoffset; - pool->nextoffset += INDEX2SIZE(size); - *(block **)(pool->freeblock) = NULL; - return; - } - - /* Pool is full, unlink from used pools. */ - poolp next; - next = pool->nextpool; - pool = pool->prevpool; - next->prevpool = pool; - pool->nextpool = next; -} - -/* called when pymalloc_alloc can not allocate a block from usedpool. - * This function takes new pool and allocate a block from it. - */ -static void* -allocate_from_new_pool(uint size) +// Called when freelist is exhausted. Extend the freelist if there is +// space for a block. Otherwise, remove this pool from usedpools. +static void +pymalloc_pool_extend(poolp pool, uint size) +{ + if (UNLIKELY(pool->nextoffset <= pool->maxnextoffset)) { + /* There is room for another block. */ + pool->freeblock = (block*)pool + pool->nextoffset; + pool->nextoffset += INDEX2SIZE(size); + *(block **)(pool->freeblock) = NULL; + return; + } + + /* Pool is full, unlink from used pools. */ + poolp next; + next = pool->nextpool; + pool = pool->prevpool; + next->prevpool = pool; + pool->nextpool = next; +} + +/* called when pymalloc_alloc can not allocate a block from usedpool. + * This function takes new pool and allocate a block from it. + */ +static void* +allocate_from_new_pool(uint size) { /* There isn't a pool of the right size class immediately * available: use a free pool. */ - if (UNLIKELY(usable_arenas == NULL)) { + if (UNLIKELY(usable_arenas == NULL)) { /* No arena has a free pool: allocate a new arena. */ #ifdef WITH_MEMORY_LIMITS if (narenas_currently_allocated >= MAX_ARENAS) { - return NULL; + return NULL; } #endif usable_arenas = new_arena(); if (usable_arenas == NULL) { - return NULL; + return NULL; } - usable_arenas->nextarena = usable_arenas->prevarena = NULL; - assert(nfp2lasta[usable_arenas->nfreepools] == NULL); - nfp2lasta[usable_arenas->nfreepools] = usable_arenas; + usable_arenas->nextarena = usable_arenas->prevarena = NULL; + assert(nfp2lasta[usable_arenas->nfreepools] == NULL); + nfp2lasta[usable_arenas->nfreepools] = usable_arenas; } assert(usable_arenas->address != 0); - /* This arena already had the smallest nfreepools value, so decreasing - * nfreepools doesn't change that, and we don't need to rearrange the - * usable_arenas list. However, if the arena becomes wholly allocated, - * we need to remove its arena_object from usable_arenas. - */ - assert(usable_arenas->nfreepools > 0); - if (nfp2lasta[usable_arenas->nfreepools] == usable_arenas) { - /* It's the last of this size, so there won't be any. */ - nfp2lasta[usable_arenas->nfreepools] = NULL; - } - /* If any free pools will remain, it will be the new smallest. */ - if (usable_arenas->nfreepools > 1) { - assert(nfp2lasta[usable_arenas->nfreepools - 1] == NULL); - nfp2lasta[usable_arenas->nfreepools - 1] = usable_arenas; - } - + /* This arena already had the smallest nfreepools value, so decreasing + * nfreepools doesn't change that, and we don't need to rearrange the + * usable_arenas list. However, if the arena becomes wholly allocated, + * we need to remove its arena_object from usable_arenas. + */ + assert(usable_arenas->nfreepools > 0); + if (nfp2lasta[usable_arenas->nfreepools] == usable_arenas) { + /* It's the last of this size, so there won't be any. */ + nfp2lasta[usable_arenas->nfreepools] = NULL; + } + /* If any free pools will remain, it will be the new smallest. */ + if (usable_arenas->nfreepools > 1) { + assert(nfp2lasta[usable_arenas->nfreepools - 1] == NULL); + nfp2lasta[usable_arenas->nfreepools - 1] = usable_arenas; + } + /* Try to get a cached free pool. */ - poolp pool = usable_arenas->freepools; - if (LIKELY(pool != NULL)) { + poolp pool = usable_arenas->freepools; + if (LIKELY(pool != NULL)) { /* Unlink from cached pools. */ usable_arenas->freepools = pool->nextpool; - usable_arenas->nfreepools--; - if (UNLIKELY(usable_arenas->nfreepools == 0)) { + usable_arenas->nfreepools--; + if (UNLIKELY(usable_arenas->nfreepools == 0)) { /* Wholly allocated: remove. */ assert(usable_arenas->freepools == NULL); assert(usable_arenas->nextarena == NULL || @@ -1518,133 +1518,133 @@ allocate_from_new_pool(uint size) (block*)usable_arenas->address + ARENA_SIZE - POOL_SIZE); } - } - else { - /* Carve off a new pool. */ - assert(usable_arenas->nfreepools > 0); - assert(usable_arenas->freepools == NULL); - pool = (poolp)usable_arenas->pool_address; - assert((block*)pool <= (block*)usable_arenas->address + - ARENA_SIZE - POOL_SIZE); - pool->arenaindex = (uint)(usable_arenas - arenas); - assert(&arenas[pool->arenaindex] == usable_arenas); - pool->szidx = DUMMY_SIZE_IDX; - usable_arenas->pool_address += POOL_SIZE; - --usable_arenas->nfreepools; - - if (usable_arenas->nfreepools == 0) { - assert(usable_arenas->nextarena == NULL || - usable_arenas->nextarena->prevarena == - usable_arenas); - /* Unlink the arena: it is completely allocated. */ - usable_arenas = usable_arenas->nextarena; - if (usable_arenas != NULL) { - usable_arenas->prevarena = NULL; - assert(usable_arenas->address != 0); - } + } + else { + /* Carve off a new pool. */ + assert(usable_arenas->nfreepools > 0); + assert(usable_arenas->freepools == NULL); + pool = (poolp)usable_arenas->pool_address; + assert((block*)pool <= (block*)usable_arenas->address + + ARENA_SIZE - POOL_SIZE); + pool->arenaindex = (uint)(usable_arenas - arenas); + assert(&arenas[pool->arenaindex] == usable_arenas); + pool->szidx = DUMMY_SIZE_IDX; + usable_arenas->pool_address += POOL_SIZE; + --usable_arenas->nfreepools; + + if (usable_arenas->nfreepools == 0) { + assert(usable_arenas->nextarena == NULL || + usable_arenas->nextarena->prevarena == + usable_arenas); + /* Unlink the arena: it is completely allocated. */ + usable_arenas = usable_arenas->nextarena; + if (usable_arenas != NULL) { + usable_arenas->prevarena = NULL; + assert(usable_arenas->address != 0); + } } - } - - /* Frontlink to used pools. */ - block *bp; - poolp next = usedpools[size + size]; /* == prev */ - pool->nextpool = next; - pool->prevpool = next; - next->nextpool = pool; - next->prevpool = pool; - pool->ref.count = 1; - if (pool->szidx == size) { - /* Luckily, this pool last contained blocks - * of the same size class, so its header - * and free list are already initialized. - */ - bp = pool->freeblock; - assert(bp != NULL); - pool->freeblock = *(block **)bp; - return bp; - } - /* - * Initialize the pool header, set up the free list to - * contain just the second block, and return the first - * block. - */ - pool->szidx = size; - size = INDEX2SIZE(size); - bp = (block *)pool + POOL_OVERHEAD; - pool->nextoffset = POOL_OVERHEAD + (size << 1); - pool->maxnextoffset = POOL_SIZE - size; - pool->freeblock = bp + size; - *(block **)(pool->freeblock) = NULL; - return bp; -} - -/* pymalloc allocator - - Return a pointer to newly allocated memory if pymalloc allocated memory. - - Return NULL if pymalloc failed to allocate the memory block: on bigger - requests, on error in the code below (as a last chance to serve the request) - or when the max memory limit has been reached. -*/ -static inline void* -pymalloc_alloc(void *ctx, size_t nbytes) -{ -#ifdef WITH_VALGRIND - if (UNLIKELY(running_on_valgrind == -1)) { - running_on_valgrind = RUNNING_ON_VALGRIND; - } - if (UNLIKELY(running_on_valgrind)) { - return NULL; - } -#endif - - if (UNLIKELY(nbytes == 0)) { - return NULL; - } - if (UNLIKELY(nbytes > SMALL_REQUEST_THRESHOLD)) { - return NULL; - } - - uint size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT; - poolp pool = usedpools[size + size]; - block *bp; - - if (LIKELY(pool != pool->nextpool)) { - /* - * There is a used pool for this size class. - * Pick up the head block of its free list. + } + + /* Frontlink to used pools. */ + block *bp; + poolp next = usedpools[size + size]; /* == prev */ + pool->nextpool = next; + pool->prevpool = next; + next->nextpool = pool; + next->prevpool = pool; + pool->ref.count = 1; + if (pool->szidx == size) { + /* Luckily, this pool last contained blocks + * of the same size class, so its header + * and free list are already initialized. */ - ++pool->ref.count; - bp = pool->freeblock; - assert(bp != NULL); - - if (UNLIKELY((pool->freeblock = *(block **)bp) == NULL)) { - // Reached the end of the free list, try to extend it. - pymalloc_pool_extend(pool, size); + bp = pool->freeblock; + assert(bp != NULL); + pool->freeblock = *(block **)bp; + return bp; + } + /* + * Initialize the pool header, set up the free list to + * contain just the second block, and return the first + * block. + */ + pool->szidx = size; + size = INDEX2SIZE(size); + bp = (block *)pool + POOL_OVERHEAD; + pool->nextoffset = POOL_OVERHEAD + (size << 1); + pool->maxnextoffset = POOL_SIZE - size; + pool->freeblock = bp + size; + *(block **)(pool->freeblock) = NULL; + return bp; +} + +/* pymalloc allocator + + Return a pointer to newly allocated memory if pymalloc allocated memory. + + Return NULL if pymalloc failed to allocate the memory block: on bigger + requests, on error in the code below (as a last chance to serve the request) + or when the max memory limit has been reached. +*/ +static inline void* +pymalloc_alloc(void *ctx, size_t nbytes) +{ +#ifdef WITH_VALGRIND + if (UNLIKELY(running_on_valgrind == -1)) { + running_on_valgrind = RUNNING_ON_VALGRIND; + } + if (UNLIKELY(running_on_valgrind)) { + return NULL; + } +#endif + + if (UNLIKELY(nbytes == 0)) { + return NULL; + } + if (UNLIKELY(nbytes > SMALL_REQUEST_THRESHOLD)) { + return NULL; + } + + uint size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT; + poolp pool = usedpools[size + size]; + block *bp; + + if (LIKELY(pool != pool->nextpool)) { + /* + * There is a used pool for this size class. + * Pick up the head block of its free list. + */ + ++pool->ref.count; + bp = pool->freeblock; + assert(bp != NULL); + + if (UNLIKELY((pool->freeblock = *(block **)bp) == NULL)) { + // Reached the end of the free list, try to extend it. + pymalloc_pool_extend(pool, size); } } - else { - /* There isn't a pool of the right size class immediately - * available: use a free pool. - */ - bp = allocate_from_new_pool(size); - } + else { + /* There isn't a pool of the right size class immediately + * available: use a free pool. + */ + bp = allocate_from_new_pool(size); + } - return (void *)bp; + return (void *)bp; } static void * _PyObject_Malloc(void *ctx, size_t nbytes) { - void* ptr = pymalloc_alloc(ctx, nbytes); - if (LIKELY(ptr != NULL)) { + void* ptr = pymalloc_alloc(ctx, nbytes); + if (LIKELY(ptr != NULL)) { return ptr; } ptr = PyMem_RawMalloc(nbytes); if (ptr != NULL) { - raw_allocated_blocks++; + raw_allocated_blocks++; } return ptr; } @@ -1656,77 +1656,77 @@ _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize) assert(elsize == 0 || nelem <= (size_t)PY_SSIZE_T_MAX / elsize); size_t nbytes = nelem * elsize; - void* ptr = pymalloc_alloc(ctx, nbytes); - if (LIKELY(ptr != NULL)) { + void* ptr = pymalloc_alloc(ctx, nbytes); + if (LIKELY(ptr != NULL)) { memset(ptr, 0, nbytes); return ptr; } ptr = PyMem_RawCalloc(nelem, elsize); if (ptr != NULL) { - raw_allocated_blocks++; + raw_allocated_blocks++; } return ptr; } -static void -insert_to_usedpool(poolp pool) +static void +insert_to_usedpool(poolp pool) { - assert(pool->ref.count > 0); /* else the pool is empty */ + assert(pool->ref.count > 0); /* else the pool is empty */ - uint size = pool->szidx; - poolp next = usedpools[size + size]; - poolp prev = next->prevpool; + uint size = pool->szidx; + poolp next = usedpools[size + size]; + poolp prev = next->prevpool; - /* insert pool before next: prev <-> pool <-> next */ - pool->nextpool = next; - pool->prevpool = prev; - next->prevpool = pool; - prev->nextpool = pool; -} + /* insert pool before next: prev <-> pool <-> next */ + pool->nextpool = next; + pool->prevpool = prev; + next->prevpool = pool; + prev->nextpool = pool; +} -static void -insert_to_freepool(poolp pool) -{ - poolp next = pool->nextpool; - poolp prev = pool->prevpool; +static void +insert_to_freepool(poolp pool) +{ + poolp next = pool->nextpool; + poolp prev = pool->prevpool; next->prevpool = prev; prev->nextpool = next; /* Link the pool to freepools. This is a singly-linked * list, and pool->prevpool isn't used there. */ - struct arena_object *ao = &arenas[pool->arenaindex]; + struct arena_object *ao = &arenas[pool->arenaindex]; pool->nextpool = ao->freepools; ao->freepools = pool; - uint nf = ao->nfreepools; - /* If this is the rightmost arena with this number of free pools, - * nfp2lasta[nf] needs to change. Caution: if nf is 0, there - * are no arenas in usable_arenas with that value. - */ - struct arena_object* lastnf = nfp2lasta[nf]; - assert((nf == 0 && lastnf == NULL) || - (nf > 0 && - lastnf != NULL && - lastnf->nfreepools == nf && - (lastnf->nextarena == NULL || - nf < lastnf->nextarena->nfreepools))); - if (lastnf == ao) { /* it is the rightmost */ - struct arena_object* p = ao->prevarena; - nfp2lasta[nf] = (p != NULL && p->nfreepools == nf) ? p : NULL; - } - ao->nfreepools = ++nf; + uint nf = ao->nfreepools; + /* If this is the rightmost arena with this number of free pools, + * nfp2lasta[nf] needs to change. Caution: if nf is 0, there + * are no arenas in usable_arenas with that value. + */ + struct arena_object* lastnf = nfp2lasta[nf]; + assert((nf == 0 && lastnf == NULL) || + (nf > 0 && + lastnf != NULL && + lastnf->nfreepools == nf && + (lastnf->nextarena == NULL || + nf < lastnf->nextarena->nfreepools))); + if (lastnf == ao) { /* it is the rightmost */ + struct arena_object* p = ao->prevarena; + nfp2lasta[nf] = (p != NULL && p->nfreepools == nf) ? p : NULL; + } + ao->nfreepools = ++nf; /* All the rest is arena management. We just freed * a pool, and there are 4 cases for arena mgmt: * 1. If all the pools are free, return the arena to - * the system free(). Except if this is the last - * arena in the list, keep it to avoid thrashing: - * keeping one wholly free arena in the list avoids - * pathological cases where a simple loop would - * otherwise provoke needing to allocate and free an - * arena on every iteration. See bpo-37257. + * the system free(). Except if this is the last + * arena in the list, keep it to avoid thrashing: + * keeping one wholly free arena in the list avoids + * pathological cases where a simple loop would + * otherwise provoke needing to allocate and free an + * arena on every iteration. See bpo-37257. * 2. If this is the only free pool in the arena, * add the arena back to the `usable_arenas` list. * 3. If the "next" arena has a smaller count of free @@ -1735,7 +1735,7 @@ insert_to_freepool(poolp pool) * nfreepools. * 4. Else there's nothing more to do. */ - if (nf == ao->ntotalpools && ao->nextarena != NULL) { + if (nf == ao->ntotalpools && ao->nextarena != NULL) { /* Case 1. First unlink ao from usable_arenas. */ assert(ao->prevarena == NULL || @@ -1774,7 +1774,7 @@ insert_to_freepool(poolp pool) ao->address = 0; /* mark unassociated */ --narenas_currently_allocated; - return; + return; } if (nf == 1) { @@ -1789,11 +1789,11 @@ insert_to_freepool(poolp pool) usable_arenas->prevarena = ao; usable_arenas = ao; assert(usable_arenas->address != 0); - if (nfp2lasta[1] == NULL) { - nfp2lasta[1] = ao; - } + if (nfp2lasta[1] == NULL) { + nfp2lasta[1] = ao; + } - return; + return; } /* If this arena is now out of order, we need to keep @@ -1803,23 +1803,23 @@ insert_to_freepool(poolp pool) * a few un-scientific tests, it seems like this * approach allowed a lot more memory to be freed. */ - /* If this is the only arena with nf, record that. */ - if (nfp2lasta[nf] == NULL) { - nfp2lasta[nf] = ao; - } /* else the rightmost with nf doesn't change */ - /* If this was the rightmost of the old size, it remains in place. */ - if (ao == lastnf) { + /* If this is the only arena with nf, record that. */ + if (nfp2lasta[nf] == NULL) { + nfp2lasta[nf] = ao; + } /* else the rightmost with nf doesn't change */ + /* If this was the rightmost of the old size, it remains in place. */ + if (ao == lastnf) { /* Case 4. Nothing to do. */ - return; - } - /* If ao were the only arena in the list, the last block would have - * gotten us out. - */ - assert(ao->nextarena != NULL); - - /* Case 3: We have to move the arena towards the end of the list, - * because it has more free pools than the arena to its right. It needs - * to move to follow lastnf. + return; + } + /* If ao were the only arena in the list, the last block would have + * gotten us out. + */ + assert(ao->nextarena != NULL); + + /* Case 3: We have to move the arena towards the end of the list, + * because it has more free pools than the arena to its right. It needs + * to move to follow lastnf. * First unlink ao from usable_arenas. */ if (ao->prevarena != NULL) { @@ -1833,78 +1833,78 @@ insert_to_freepool(poolp pool) usable_arenas = ao->nextarena; } ao->nextarena->prevarena = ao->prevarena; - /* And insert after lastnf. */ - ao->prevarena = lastnf; - ao->nextarena = lastnf->nextarena; + /* And insert after lastnf. */ + ao->prevarena = lastnf; + ao->nextarena = lastnf->nextarena; if (ao->nextarena != NULL) { ao->nextarena->prevarena = ao; } - lastnf->nextarena = ao; + lastnf->nextarena = ao; /* Verify that the swaps worked. */ assert(ao->nextarena == NULL || nf <= ao->nextarena->nfreepools); assert(ao->prevarena == NULL || nf > ao->prevarena->nfreepools); assert(ao->nextarena == NULL || ao->nextarena->prevarena == ao); assert((usable_arenas == ao && ao->prevarena == NULL) || ao->prevarena->nextarena == ao); -} - -/* Free a memory block allocated by pymalloc_alloc(). - Return 1 if it was freed. - Return 0 if the block was not allocated by pymalloc_alloc(). */ -static inline int -pymalloc_free(void *ctx, void *p) -{ - assert(p != NULL); - -#ifdef WITH_VALGRIND - if (UNLIKELY(running_on_valgrind > 0)) { - return 0; - } -#endif - - poolp pool = POOL_ADDR(p); - if (UNLIKELY(!address_in_range(p, pool))) { - return 0; - } - /* We allocated this address. */ - - /* Link p to the start of the pool's freeblock list. Since - * the pool had at least the p block outstanding, the pool - * wasn't empty (so it's already in a usedpools[] list, or - * was full and is in no list -- it's not in the freeblocks - * list in any case). - */ - assert(pool->ref.count > 0); /* else it was empty */ - block *lastfree = pool->freeblock; - *(block **)p = lastfree; - pool->freeblock = (block *)p; - pool->ref.count--; - - if (UNLIKELY(lastfree == NULL)) { - /* Pool was full, so doesn't currently live in any list: - * link it to the front of the appropriate usedpools[] list. - * This mimics LRU pool usage for new allocations and - * targets optimal filling when several pools contain - * blocks of the same size class. - */ - insert_to_usedpool(pool); - return 1; - } - - /* freeblock wasn't NULL, so the pool wasn't full, - * and the pool is in a usedpools[] list. - */ - if (LIKELY(pool->ref.count != 0)) { - /* pool isn't empty: leave it in usedpools */ - return 1; - } - - /* Pool is now empty: unlink from usedpools, and - * link to the front of freepools. This ensures that - * previously freed pools will be allocated later - * (being not referenced, they are perhaps paged out). - */ - insert_to_freepool(pool); +} + +/* Free a memory block allocated by pymalloc_alloc(). + Return 1 if it was freed. + Return 0 if the block was not allocated by pymalloc_alloc(). */ +static inline int +pymalloc_free(void *ctx, void *p) +{ + assert(p != NULL); + +#ifdef WITH_VALGRIND + if (UNLIKELY(running_on_valgrind > 0)) { + return 0; + } +#endif + + poolp pool = POOL_ADDR(p); + if (UNLIKELY(!address_in_range(p, pool))) { + return 0; + } + /* We allocated this address. */ + + /* Link p to the start of the pool's freeblock list. Since + * the pool had at least the p block outstanding, the pool + * wasn't empty (so it's already in a usedpools[] list, or + * was full and is in no list -- it's not in the freeblocks + * list in any case). + */ + assert(pool->ref.count > 0); /* else it was empty */ + block *lastfree = pool->freeblock; + *(block **)p = lastfree; + pool->freeblock = (block *)p; + pool->ref.count--; + + if (UNLIKELY(lastfree == NULL)) { + /* Pool was full, so doesn't currently live in any list: + * link it to the front of the appropriate usedpools[] list. + * This mimics LRU pool usage for new allocations and + * targets optimal filling when several pools contain + * blocks of the same size class. + */ + insert_to_usedpool(pool); + return 1; + } + + /* freeblock wasn't NULL, so the pool wasn't full, + * and the pool is in a usedpools[] list. + */ + if (LIKELY(pool->ref.count != 0)) { + /* pool isn't empty: leave it in usedpools */ + return 1; + } + + /* Pool is now empty: unlink from usedpools, and + * link to the front of freepools. This ensures that + * previously freed pools will be allocated later + * (being not referenced, they are perhaps paged out). + */ + insert_to_freepool(pool); return 1; } @@ -1917,10 +1917,10 @@ _PyObject_Free(void *ctx, void *p) return; } - if (UNLIKELY(!pymalloc_free(ctx, p))) { + if (UNLIKELY(!pymalloc_free(ctx, p))) { /* pymalloc didn't allocate this address */ PyMem_RawFree(p); - raw_allocated_blocks--; + raw_allocated_blocks--; } } @@ -2031,10 +2031,10 @@ _Py_GetAllocatedBlocks(void) * it wraps a real allocator, adding extra debugging info to the memory blocks. */ -/* Uncomment this define to add the "serialno" field */ -/* #define PYMEM_DEBUG_SERIALNO */ +/* Uncomment this define to add the "serialno" field */ +/* #define PYMEM_DEBUG_SERIALNO */ -#ifdef PYMEM_DEBUG_SERIALNO +#ifdef PYMEM_DEBUG_SERIALNO static size_t serialno = 0; /* incremented on each debug {m,re}alloc */ /* serialno is always incremented via calling this routine. The point is @@ -2045,16 +2045,16 @@ bumpserialno(void) { ++serialno; } -#endif +#endif #define SST SIZEOF_SIZE_T -#ifdef PYMEM_DEBUG_SERIALNO -# define PYMEM_DEBUG_EXTRA_BYTES 4 * SST -#else -# define PYMEM_DEBUG_EXTRA_BYTES 3 * SST -#endif - +#ifdef PYMEM_DEBUG_SERIALNO +# define PYMEM_DEBUG_EXTRA_BYTES 4 * SST +#else +# define PYMEM_DEBUG_EXTRA_BYTES 3 * SST +#endif + /* Read sizeof(size_t) bytes at p as a big-endian size_t. */ static size_t read_size_t(const void *p) @@ -2083,7 +2083,7 @@ write_size_t(void *p, size_t n) } } -/* Let S = sizeof(size_t). The debug malloc asks for 4 * S extra bytes and +/* Let S = sizeof(size_t). The debug malloc asks for 4 * S extra bytes and fills them with useful stuff, here calling the underlying malloc's result p: p[0: S] @@ -2092,14 +2092,14 @@ p[0: S] p[S] API ID. See PEP 445. This is a character, but seems undocumented. p[S+1: 2*S] - Copies of PYMEM_FORBIDDENBYTE. Used to catch under- writes and reads. + Copies of PYMEM_FORBIDDENBYTE. Used to catch under- writes and reads. p[2*S: 2*S+n] - The requested memory, filled with copies of PYMEM_CLEANBYTE. + The requested memory, filled with copies of PYMEM_CLEANBYTE. Used to catch reference to uninitialized memory. &p[2*S] is returned. Note that this is 8-byte aligned if pymalloc handled the request itself. p[2*S+n: 2*S+n+S] - Copies of PYMEM_FORBIDDENBYTE. Used to catch over- writes and reads. + Copies of PYMEM_FORBIDDENBYTE. Used to catch over- writes and reads. p[2*S+n+S: 2*S+n+2*S] A serial number, incremented by 1 on each call to _PyMem_DebugMalloc and _PyMem_DebugRealloc. @@ -2107,9 +2107,9 @@ p[2*S+n+S: 2*S+n+2*S] If "bad memory" is detected later, the serial number gives an excellent way to set a breakpoint on the next run, to capture the instant at which this block was passed out. - -If PYMEM_DEBUG_SERIALNO is not defined (default), the debug malloc only asks -for 3 * S extra bytes, and omits the last serialno field. + +If PYMEM_DEBUG_SERIALNO is not defined (default), the debug malloc only asks +for 3 * S extra bytes, and omits the last serialno field. */ static void * @@ -2119,25 +2119,25 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) uint8_t *p; /* base address of malloc'ed pad block */ uint8_t *data; /* p + 2*SST == pointer to data bytes */ uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */ - size_t total; /* nbytes + PYMEM_DEBUG_EXTRA_BYTES */ + size_t total; /* nbytes + PYMEM_DEBUG_EXTRA_BYTES */ - if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) { + if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) { /* integer overflow: can't represent total as a Py_ssize_t */ return NULL; } - total = nbytes + PYMEM_DEBUG_EXTRA_BYTES; + total = nbytes + PYMEM_DEBUG_EXTRA_BYTES; /* Layout: [SSSS IFFF CCCC...CCCC FFFF NNNN] - ^--- p ^--- data ^--- tail + ^--- p ^--- data ^--- tail S: nbytes stored as size_t I: API identifier (1 byte) F: Forbidden bytes (size_t - 1 bytes before, size_t bytes after) C: Clean bytes used later to store actual data - N: Serial number stored as size_t - - If PYMEM_DEBUG_SERIALNO is not defined (default), the last NNNN field - is omitted. */ + N: Serial number stored as size_t + If PYMEM_DEBUG_SERIALNO is not defined (default), the last NNNN field + is omitted. */ + if (use_calloc) { p = (uint8_t *)api->alloc.calloc(api->alloc.ctx, 1, total); } @@ -2149,25 +2149,25 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) } data = p + 2*SST; -#ifdef PYMEM_DEBUG_SERIALNO +#ifdef PYMEM_DEBUG_SERIALNO bumpserialno(); -#endif +#endif /* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */ write_size_t(p, nbytes); p[SST] = (uint8_t)api->api_id; - memset(p + SST + 1, PYMEM_FORBIDDENBYTE, SST-1); + memset(p + SST + 1, PYMEM_FORBIDDENBYTE, SST-1); if (nbytes > 0 && !use_calloc) { - memset(data, PYMEM_CLEANBYTE, nbytes); + memset(data, PYMEM_CLEANBYTE, nbytes); } /* at tail, write pad (SST bytes) and serialno (SST bytes) */ tail = data + nbytes; - memset(tail, PYMEM_FORBIDDENBYTE, SST); -#ifdef PYMEM_DEBUG_SERIALNO + memset(tail, PYMEM_FORBIDDENBYTE, SST); +#ifdef PYMEM_DEBUG_SERIALNO write_size_t(tail + SST, serialno); -#endif +#endif return data; } @@ -2190,7 +2190,7 @@ _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize) /* The debug free first checks the 2*SST bytes on each end for sanity (in particular, that the FORBIDDENBYTEs with the api ID are still intact). - Then fills the original bytes with PYMEM_DEADBYTE. + Then fills the original bytes with PYMEM_DEADBYTE. Then calls the underlying free. */ static void @@ -2205,10 +2205,10 @@ _PyMem_DebugRawFree(void *ctx, void *p) uint8_t *q = (uint8_t *)p - 2*SST; /* address returned from malloc */ size_t nbytes; - _PyMem_DebugCheckAddress(__func__, api->api_id, p); + _PyMem_DebugCheckAddress(__func__, api->api_id, p); nbytes = read_size_t(q); - nbytes += PYMEM_DEBUG_EXTRA_BYTES; - memset(q, PYMEM_DEADBYTE, nbytes); + nbytes += PYMEM_DEBUG_EXTRA_BYTES; + memset(q, PYMEM_DEADBYTE, nbytes); api->alloc.free(api->alloc.ctx, q); } @@ -2230,62 +2230,62 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) #define ERASED_SIZE 64 uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ - _PyMem_DebugCheckAddress(__func__, api->api_id, p); + _PyMem_DebugCheckAddress(__func__, api->api_id, p); data = (uint8_t *)p; head = data - 2*SST; original_nbytes = read_size_t(head); - if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) { + if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) { /* integer overflow: can't represent total as a Py_ssize_t */ return NULL; } - total = nbytes + PYMEM_DEBUG_EXTRA_BYTES; + total = nbytes + PYMEM_DEBUG_EXTRA_BYTES; tail = data + original_nbytes; -#ifdef PYMEM_DEBUG_SERIALNO - size_t block_serialno = read_size_t(tail + SST); -#endif +#ifdef PYMEM_DEBUG_SERIALNO + size_t block_serialno = read_size_t(tail + SST); +#endif /* Mark the header, the trailer, ERASED_SIZE bytes at the begin and ERASED_SIZE bytes at the end as dead and save the copy of erased bytes. */ if (original_nbytes <= sizeof(save)) { memcpy(save, data, original_nbytes); - memset(data - 2 * SST, PYMEM_DEADBYTE, - original_nbytes + PYMEM_DEBUG_EXTRA_BYTES); + memset(data - 2 * SST, PYMEM_DEADBYTE, + original_nbytes + PYMEM_DEBUG_EXTRA_BYTES); } else { memcpy(save, data, ERASED_SIZE); - memset(head, PYMEM_DEADBYTE, ERASED_SIZE + 2 * SST); + memset(head, PYMEM_DEADBYTE, ERASED_SIZE + 2 * SST); memcpy(&save[ERASED_SIZE], tail - ERASED_SIZE, ERASED_SIZE); - memset(tail - ERASED_SIZE, PYMEM_DEADBYTE, - ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST); + memset(tail - ERASED_SIZE, PYMEM_DEADBYTE, + ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST); } /* Resize and add decorations. */ r = (uint8_t *)api->alloc.realloc(api->alloc.ctx, head, total); if (r == NULL) { - /* if realloc() failed: rewrite header and footer which have - just been erased */ + /* if realloc() failed: rewrite header and footer which have + just been erased */ nbytes = original_nbytes; } else { head = r; -#ifdef PYMEM_DEBUG_SERIALNO +#ifdef PYMEM_DEBUG_SERIALNO bumpserialno(); block_serialno = serialno; -#endif +#endif } - data = head + 2*SST; + data = head + 2*SST; write_size_t(head, nbytes); head[SST] = (uint8_t)api->api_id; - memset(head + SST + 1, PYMEM_FORBIDDENBYTE, SST-1); + memset(head + SST + 1, PYMEM_FORBIDDENBYTE, SST-1); tail = data + nbytes; - memset(tail, PYMEM_FORBIDDENBYTE, SST); -#ifdef PYMEM_DEBUG_SERIALNO + memset(tail, PYMEM_FORBIDDENBYTE, SST); +#ifdef PYMEM_DEBUG_SERIALNO write_size_t(tail + SST, block_serialno); -#endif +#endif /* Restore saved bytes. */ if (original_nbytes <= sizeof(save)) { @@ -2305,35 +2305,35 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) } if (nbytes > original_nbytes) { - /* growing: mark new extra memory clean */ - memset(data + original_nbytes, PYMEM_CLEANBYTE, - nbytes - original_nbytes); + /* growing: mark new extra memory clean */ + memset(data + original_nbytes, PYMEM_CLEANBYTE, + nbytes - original_nbytes); } return data; } -static inline void -_PyMem_DebugCheckGIL(const char *func) +static inline void +_PyMem_DebugCheckGIL(const char *func) { - if (!PyGILState_Check()) { - _Py_FatalErrorFunc(func, - "Python memory allocator called " - "without holding the GIL"); - } + if (!PyGILState_Check()) { + _Py_FatalErrorFunc(func, + "Python memory allocator called " + "without holding the GIL"); + } } static void * _PyMem_DebugMalloc(void *ctx, size_t nbytes) { - _PyMem_DebugCheckGIL(__func__); + _PyMem_DebugCheckGIL(__func__); return _PyMem_DebugRawMalloc(ctx, nbytes); } static void * _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize) { - _PyMem_DebugCheckGIL(__func__); + _PyMem_DebugCheckGIL(__func__); return _PyMem_DebugRawCalloc(ctx, nelem, elsize); } @@ -2341,7 +2341,7 @@ _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize) static void _PyMem_DebugFree(void *ctx, void *ptr) { - _PyMem_DebugCheckGIL(__func__); + _PyMem_DebugCheckGIL(__func__); _PyMem_DebugRawFree(ctx, ptr); } @@ -2349,7 +2349,7 @@ _PyMem_DebugFree(void *ctx, void *ptr) static void * _PyMem_DebugRealloc(void *ctx, void *ptr, size_t nbytes) { - _PyMem_DebugCheckGIL(__func__); + _PyMem_DebugCheckGIL(__func__); return _PyMem_DebugRawRealloc(ctx, ptr, nbytes); } @@ -2359,10 +2359,10 @@ _PyMem_DebugRealloc(void *ctx, void *ptr, size_t nbytes) * The API id, is also checked. */ static void -_PyMem_DebugCheckAddress(const char *func, char api, const void *p) +_PyMem_DebugCheckAddress(const char *func, char api, const void *p) { - assert(p != NULL); - + assert(p != NULL); + const uint8_t *q = (const uint8_t *)p; size_t nbytes; const uint8_t *tail; @@ -2372,11 +2372,11 @@ _PyMem_DebugCheckAddress(const char *func, char api, const void *p) /* Check the API id */ id = (char)q[-SST]; if (id != api) { - _PyObject_DebugDumpAddress(p); - _Py_FatalErrorFormat(func, - "bad ID: Allocated using API '%c', " - "verified using API '%c'", - id, api); + _PyObject_DebugDumpAddress(p); + _Py_FatalErrorFormat(func, + "bad ID: Allocated using API '%c', " + "verified using API '%c'", + id, api); } /* Check the stuff at the start of p first: if there's underwrite @@ -2384,18 +2384,18 @@ _PyMem_DebugCheckAddress(const char *func, char api, const void *p) * the tail could lead to a segfault then. */ for (i = SST-1; i >= 1; --i) { - if (*(q-i) != PYMEM_FORBIDDENBYTE) { - _PyObject_DebugDumpAddress(p); - _Py_FatalErrorFunc(func, "bad leading pad byte"); + if (*(q-i) != PYMEM_FORBIDDENBYTE) { + _PyObject_DebugDumpAddress(p); + _Py_FatalErrorFunc(func, "bad leading pad byte"); } } nbytes = read_size_t(q - 2*SST); tail = q + nbytes; for (i = 0; i < SST; ++i) { - if (tail[i] != PYMEM_FORBIDDENBYTE) { - _PyObject_DebugDumpAddress(p); - _Py_FatalErrorFunc(func, "bad trailing pad byte"); + if (tail[i] != PYMEM_FORBIDDENBYTE) { + _PyObject_DebugDumpAddress(p); + _Py_FatalErrorFunc(func, "bad trailing pad byte"); } } } @@ -2406,7 +2406,7 @@ _PyObject_DebugDumpAddress(const void *p) { const uint8_t *q = (const uint8_t *)p; const uint8_t *tail; - size_t nbytes; + size_t nbytes; int i; int ok; char id; @@ -2427,7 +2427,7 @@ _PyObject_DebugDumpAddress(const void *p) fprintf(stderr, " The %d pad bytes at p-%d are ", SST-1, SST-1); ok = 1; for (i = 1; i <= SST-1; ++i) { - if (*(q-i) != PYMEM_FORBIDDENBYTE) { + if (*(q-i) != PYMEM_FORBIDDENBYTE) { ok = 0; break; } @@ -2436,11 +2436,11 @@ _PyObject_DebugDumpAddress(const void *p) fputs("FORBIDDENBYTE, as expected.\n", stderr); else { fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n", - PYMEM_FORBIDDENBYTE); + PYMEM_FORBIDDENBYTE); for (i = SST-1; i >= 1; --i) { const uint8_t byte = *(q-i); fprintf(stderr, " at p-%d: 0x%02x", i, byte); - if (byte != PYMEM_FORBIDDENBYTE) + if (byte != PYMEM_FORBIDDENBYTE) fputs(" *** OUCH", stderr); fputc('\n', stderr); } @@ -2452,10 +2452,10 @@ _PyObject_DebugDumpAddress(const void *p) } tail = q + nbytes; - fprintf(stderr, " The %d pad bytes at tail=%p are ", SST, (void *)tail); + fprintf(stderr, " The %d pad bytes at tail=%p are ", SST, (void *)tail); ok = 1; for (i = 0; i < SST; ++i) { - if (tail[i] != PYMEM_FORBIDDENBYTE) { + if (tail[i] != PYMEM_FORBIDDENBYTE) { ok = 0; break; } @@ -2464,22 +2464,22 @@ _PyObject_DebugDumpAddress(const void *p) fputs("FORBIDDENBYTE, as expected.\n", stderr); else { fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n", - PYMEM_FORBIDDENBYTE); + PYMEM_FORBIDDENBYTE); for (i = 0; i < SST; ++i) { const uint8_t byte = tail[i]; fprintf(stderr, " at tail+%d: 0x%02x", i, byte); - if (byte != PYMEM_FORBIDDENBYTE) + if (byte != PYMEM_FORBIDDENBYTE) fputs(" *** OUCH", stderr); fputc('\n', stderr); } } -#ifdef PYMEM_DEBUG_SERIALNO - size_t serial = read_size_t(tail + SST); +#ifdef PYMEM_DEBUG_SERIALNO + size_t serial = read_size_t(tail + SST); fprintf(stderr, " The block was made by call #%" PY_FORMAT_SIZE_T "u to debug malloc/realloc.\n", serial); -#endif +#endif if (nbytes > 0) { i = 0; @@ -2705,11 +2705,11 @@ _PyObject_DebugMallocStats(FILE *out) quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); } fputc('\n', out); -#ifdef PYMEM_DEBUG_SERIALNO - if (_PyMem_DebugEnabled()) { +#ifdef PYMEM_DEBUG_SERIALNO + if (_PyMem_DebugEnabled()) { (void)printone(out, "# times object malloc called", serialno); - } -#endif + } +#endif (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); (void)printone(out, "# arenas highwater mark", narenas_highwater); |
