summaryrefslogtreecommitdiffstats
path: root/contrib/tools/python3/src/Objects/obmalloc.c
diff options
context:
space:
mode:
authorshadchin <[email protected]>2022-02-10 16:44:30 +0300
committerDaniil Cherednik <[email protected]>2022-02-10 16:44:30 +0300
commit2598ef1d0aee359b4b6d5fdd1758916d5907d04f (patch)
tree012bb94d777798f1f56ac1cec429509766d05181 /contrib/tools/python3/src/Objects/obmalloc.c
parent6751af0b0c1b952fede40b19b71da8025b5d8bcf (diff)
Restoring authorship annotation for <[email protected]>. Commit 1 of 2.
Diffstat (limited to 'contrib/tools/python3/src/Objects/obmalloc.c')
-rw-r--r--contrib/tools/python3/src/Objects/obmalloc.c1050
1 files changed, 525 insertions, 525 deletions
diff --git a/contrib/tools/python3/src/Objects/obmalloc.c b/contrib/tools/python3/src/Objects/obmalloc.c
index 9f8e0d114ff..1bff84b2325 100644
--- a/contrib/tools/python3/src/Objects/obmalloc.c
+++ b/contrib/tools/python3/src/Objects/obmalloc.c
@@ -1,5 +1,5 @@
#include "Python.h"
-#include "pycore_pymem.h" // _PyTraceMalloc_Config
+#include "pycore_pymem.h" // _PyTraceMalloc_Config
#include <stdbool.h>
@@ -25,14 +25,14 @@ static void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size);
static void _PyMem_DebugFree(void *ctx, void *p);
static void _PyObject_DebugDumpAddress(const void *p);
-static void _PyMem_DebugCheckAddress(const char *func, char api_id, const void *p);
+static void _PyMem_DebugCheckAddress(const char *func, char api_id, const void *p);
static void _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain);
#if defined(__has_feature) /* Clang */
# if __has_feature(address_sanitizer) /* is ASAN enabled? */
-# define _Py_NO_SANITIZE_ADDRESS \
- __attribute__((no_sanitize("address")))
+# define _Py_NO_SANITIZE_ADDRESS \
+ __attribute__((no_sanitize("address")))
# endif
# if __has_feature(thread_sanitizer) /* is TSAN enabled? */
# define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))
@@ -42,18 +42,18 @@ static void _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain);
# endif
#elif defined(__GNUC__)
# if defined(__SANITIZE_ADDRESS__) /* GCC 4.8+, is ASAN enabled? */
-# define _Py_NO_SANITIZE_ADDRESS \
- __attribute__((no_sanitize_address))
+# define _Py_NO_SANITIZE_ADDRESS \
+ __attribute__((no_sanitize_address))
# endif
- // TSAN is supported since GCC 5.1, but __SANITIZE_THREAD__ macro
+ // TSAN is supported since GCC 5.1, but __SANITIZE_THREAD__ macro
// is provided only since GCC 7.
-# if __GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1)
+# if __GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1)
# define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))
# endif
#endif
-#ifndef _Py_NO_SANITIZE_ADDRESS
-# define _Py_NO_SANITIZE_ADDRESS
+#ifndef _Py_NO_SANITIZE_ADDRESS
+# define _Py_NO_SANITIZE_ADDRESS
#endif
#ifndef _Py_NO_SANITIZE_THREAD
# define _Py_NO_SANITIZE_THREAD
@@ -81,12 +81,12 @@ static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);
#endif
-/* bpo-35053: Declare tracemalloc configuration here rather than
- Modules/_tracemalloc.c because _tracemalloc can be compiled as dynamic
- library, whereas _Py_NewReference() requires it. */
-struct _PyTraceMalloc_Config _Py_tracemalloc_config = _PyTraceMalloc_Config_INIT;
-
-
+/* bpo-35053: Declare tracemalloc configuration here rather than
+ Modules/_tracemalloc.c because _tracemalloc can be compiled as dynamic
+ library, whereas _Py_NewReference() requires it. */
+struct _PyTraceMalloc_Config _Py_tracemalloc_config = _PyTraceMalloc_Config_INIT;
+
+
static void *
_PyMem_RawMalloc(void *ctx, size_t size)
{
@@ -268,65 +268,65 @@ _PyMem_SetDefaultAllocator(PyMemAllocatorDomain domain,
int
-_PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator)
+_PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator)
{
- if (name == NULL || *name == '\0') {
+ if (name == NULL || *name == '\0') {
/* PYTHONMALLOC is empty or is not set or ignored (-E/-I command line
- nameions): use default memory allocators */
- *allocator = PYMEM_ALLOCATOR_DEFAULT;
- }
- else if (strcmp(name, "default") == 0) {
- *allocator = PYMEM_ALLOCATOR_DEFAULT;
- }
- else if (strcmp(name, "debug") == 0) {
- *allocator = PYMEM_ALLOCATOR_DEBUG;
- }
-#ifdef WITH_PYMALLOC
- else if (strcmp(name, "pymalloc") == 0) {
- *allocator = PYMEM_ALLOCATOR_PYMALLOC;
- }
- else if (strcmp(name, "pymalloc_debug") == 0) {
- *allocator = PYMEM_ALLOCATOR_PYMALLOC_DEBUG;
- }
-#endif
- else if (strcmp(name, "malloc") == 0) {
- *allocator = PYMEM_ALLOCATOR_MALLOC;
- }
- else if (strcmp(name, "malloc_debug") == 0) {
- *allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG;
- }
- else {
- /* unknown allocator */
- return -1;
- }
- return 0;
-}
-
-
-int
-_PyMem_SetupAllocators(PyMemAllocatorName allocator)
-{
- switch (allocator) {
- case PYMEM_ALLOCATOR_NOT_SET:
- /* do nothing */
- break;
-
- case PYMEM_ALLOCATOR_DEFAULT:
+ nameions): use default memory allocators */
+ *allocator = PYMEM_ALLOCATOR_DEFAULT;
+ }
+ else if (strcmp(name, "default") == 0) {
+ *allocator = PYMEM_ALLOCATOR_DEFAULT;
+ }
+ else if (strcmp(name, "debug") == 0) {
+ *allocator = PYMEM_ALLOCATOR_DEBUG;
+ }
+#ifdef WITH_PYMALLOC
+ else if (strcmp(name, "pymalloc") == 0) {
+ *allocator = PYMEM_ALLOCATOR_PYMALLOC;
+ }
+ else if (strcmp(name, "pymalloc_debug") == 0) {
+ *allocator = PYMEM_ALLOCATOR_PYMALLOC_DEBUG;
+ }
+#endif
+ else if (strcmp(name, "malloc") == 0) {
+ *allocator = PYMEM_ALLOCATOR_MALLOC;
+ }
+ else if (strcmp(name, "malloc_debug") == 0) {
+ *allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG;
+ }
+ else {
+ /* unknown allocator */
+ return -1;
+ }
+ return 0;
+}
+
+
+int
+_PyMem_SetupAllocators(PyMemAllocatorName allocator)
+{
+ switch (allocator) {
+ case PYMEM_ALLOCATOR_NOT_SET:
+ /* do nothing */
+ break;
+
+ case PYMEM_ALLOCATOR_DEFAULT:
(void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, NULL);
(void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_MEM, NULL);
(void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_OBJ, NULL);
- break;
-
- case PYMEM_ALLOCATOR_DEBUG:
+ break;
+
+ case PYMEM_ALLOCATOR_DEBUG:
(void)pymem_set_default_allocator(PYMEM_DOMAIN_RAW, 1, NULL);
(void)pymem_set_default_allocator(PYMEM_DOMAIN_MEM, 1, NULL);
(void)pymem_set_default_allocator(PYMEM_DOMAIN_OBJ, 1, NULL);
- break;
-
+ break;
+
#ifdef WITH_PYMALLOC
- case PYMEM_ALLOCATOR_PYMALLOC:
- case PYMEM_ALLOCATOR_PYMALLOC_DEBUG:
- {
+ case PYMEM_ALLOCATOR_PYMALLOC:
+ case PYMEM_ALLOCATOR_PYMALLOC_DEBUG:
+ {
PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;
PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &malloc_alloc);
@@ -334,28 +334,28 @@ _PyMem_SetupAllocators(PyMemAllocatorName allocator)
PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &pymalloc);
PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &pymalloc);
- if (allocator == PYMEM_ALLOCATOR_PYMALLOC_DEBUG) {
+ if (allocator == PYMEM_ALLOCATOR_PYMALLOC_DEBUG) {
PyMem_SetupDebugHooks();
}
- break;
+ break;
}
#endif
-
- case PYMEM_ALLOCATOR_MALLOC:
- case PYMEM_ALLOCATOR_MALLOC_DEBUG:
- {
+
+ case PYMEM_ALLOCATOR_MALLOC:
+ case PYMEM_ALLOCATOR_MALLOC_DEBUG:
+ {
PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;
PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &malloc_alloc);
PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &malloc_alloc);
PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &malloc_alloc);
- if (allocator == PYMEM_ALLOCATOR_MALLOC_DEBUG) {
+ if (allocator == PYMEM_ALLOCATOR_MALLOC_DEBUG) {
PyMem_SetupDebugHooks();
}
- break;
+ break;
}
-
- default:
+
+ default:
/* unknown allocator */
return -1;
}
@@ -371,7 +371,7 @@ pymemallocator_eq(PyMemAllocatorEx *a, PyMemAllocatorEx *b)
const char*
-_PyMem_GetCurrentAllocatorName(void)
+_PyMem_GetCurrentAllocatorName(void)
{
PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;
#ifdef WITH_PYMALLOC
@@ -714,17 +714,17 @@ PyObject_Free(void *ptr)
the valgrind checks */
#if defined(__GNUC__) && (__GNUC__ > 2) && defined(__OPTIMIZE__)
# define UNLIKELY(value) __builtin_expect((value), 0)
-# define LIKELY(value) __builtin_expect((value), 1)
+# define LIKELY(value) __builtin_expect((value), 1)
#else
# define UNLIKELY(value) (value)
-# define LIKELY(value) (value)
+# define LIKELY(value) (value)
#endif
-#ifdef WITH_PYMALLOC
-
-#ifdef WITH_VALGRIND
-#include <valgrind/valgrind.h>
-
+#ifdef WITH_PYMALLOC
+
+#ifdef WITH_VALGRIND
+#include <valgrind/valgrind.h>
+
/* -1 indicates that we haven't checked that we're running on valgrind yet. */
static int running_on_valgrind = -1;
#endif
@@ -836,20 +836,20 @@ static int running_on_valgrind = -1;
/*
* Alignment of addresses returned to the user. 8-bytes alignment works
- * on most current architectures (with 32-bit or 64-bit address buses).
+ * on most current architectures (with 32-bit or 64-bit address buses).
* The alignment value is also used for grouping small requests in size
* classes spaced ALIGNMENT bytes apart.
*
* You shouldn't change this unless you know what you are doing.
*/
-
-#if SIZEOF_VOID_P > 4
-#define ALIGNMENT 16 /* must be 2^N */
-#define ALIGNMENT_SHIFT 4
-#else
+
+#if SIZEOF_VOID_P > 4
+#define ALIGNMENT 16 /* must be 2^N */
+#define ALIGNMENT_SHIFT 4
+#else
#define ALIGNMENT 8 /* must be 2^N */
#define ALIGNMENT_SHIFT 3
-#endif
+#endif
/* Return the number of bytes in size class I, as a uint. */
#define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT)
@@ -920,11 +920,11 @@ static int running_on_valgrind = -1;
#define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */
#define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK
-#define MAX_POOLS_IN_ARENA (ARENA_SIZE / POOL_SIZE)
-#if MAX_POOLS_IN_ARENA * POOL_SIZE != ARENA_SIZE
-# error "arena size not an exact multiple of pool size"
-#endif
-
+#define MAX_POOLS_IN_ARENA (ARENA_SIZE / POOL_SIZE)
+#if MAX_POOLS_IN_ARENA * POOL_SIZE != ARENA_SIZE
+# error "arena size not an exact multiple of pool size"
+#endif
+
/*
* -- End of tunable settings section --
*/
@@ -1162,18 +1162,18 @@ usable_arenas
Note that an arena_object associated with an arena all of whose pools are
currently in use isn't on either list.
-
-Changed in Python 3.8: keeping usable_arenas sorted by number of free pools
-used to be done by one-at-a-time linear search when an arena's number of
-free pools changed. That could, overall, consume time quadratic in the
-number of arenas. That didn't really matter when there were only a few
-hundred arenas (typical!), but could be a timing disaster when there were
-hundreds of thousands. See bpo-37029.
-
-Now we have a vector of "search fingers" to eliminate the need to search:
-nfp2lasta[nfp] returns the last ("rightmost") arena in usable_arenas
-with nfp free pools. This is NULL if and only if there is no arena with
-nfp free pools in usable_arenas.
+
+Changed in Python 3.8: keeping usable_arenas sorted by number of free pools
+used to be done by one-at-a-time linear search when an arena's number of
+free pools changed. That could, overall, consume time quadratic in the
+number of arenas. That didn't really matter when there were only a few
+hundred arenas (typical!), but could be a timing disaster when there were
+hundreds of thousands. See bpo-37029.
+
+Now we have a vector of "search fingers" to eliminate the need to search:
+nfp2lasta[nfp] returns the last ("rightmost") arena in usable_arenas
+with nfp free pools. This is NULL if and only if there is no arena with
+nfp free pools in usable_arenas.
*/
/* Array of objects used to track chunks of memory (arenas). */
@@ -1191,9 +1191,9 @@ static struct arena_object* unused_arena_objects = NULL;
*/
static struct arena_object* usable_arenas = NULL;
-/* nfp2lasta[nfp] is the last arena in usable_arenas with nfp free pools */
-static struct arena_object* nfp2lasta[MAX_POOLS_IN_ARENA + 1] = { NULL };
-
+/* nfp2lasta[nfp] is the last arena in usable_arenas with nfp free pools */
+static struct arena_object* nfp2lasta[MAX_POOLS_IN_ARENA + 1] = { NULL };
+
/* How many arena_objects do we initially allocate?
* 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the
* `arenas` vector.
@@ -1208,29 +1208,29 @@ static size_t ntimes_arena_allocated = 0;
/* High water mark (max value ever seen) for narenas_currently_allocated. */
static size_t narenas_highwater = 0;
-static Py_ssize_t raw_allocated_blocks;
+static Py_ssize_t raw_allocated_blocks;
Py_ssize_t
_Py_GetAllocatedBlocks(void)
{
- Py_ssize_t n = raw_allocated_blocks;
- /* add up allocated blocks for used pools */
- for (uint i = 0; i < maxarenas; ++i) {
- /* Skip arenas which are not allocated. */
- if (arenas[i].address == 0) {
- continue;
- }
-
- uintptr_t base = (uintptr_t)_Py_ALIGN_UP(arenas[i].address, POOL_SIZE);
-
- /* visit every pool in the arena */
- assert(base <= (uintptr_t) arenas[i].pool_address);
- for (; base < (uintptr_t) arenas[i].pool_address; base += POOL_SIZE) {
- poolp p = (poolp)base;
- n += p->ref.count;
- }
- }
- return n;
+ Py_ssize_t n = raw_allocated_blocks;
+ /* add up allocated blocks for used pools */
+ for (uint i = 0; i < maxarenas; ++i) {
+ /* Skip arenas which are not allocated. */
+ if (arenas[i].address == 0) {
+ continue;
+ }
+
+ uintptr_t base = (uintptr_t)_Py_ALIGN_UP(arenas[i].address, POOL_SIZE);
+
+ /* visit every pool in the arena */
+ assert(base <= (uintptr_t) arenas[i].pool_address);
+ for (; base < (uintptr_t) arenas[i].pool_address; base += POOL_SIZE) {
+ poolp p = (poolp)base;
+ n += p->ref.count;
+ }
+ }
+ return n;
}
@@ -1320,7 +1320,7 @@ new_arena(void)
/* pool_address <- first pool-aligned address in the arena
nfreepools <- number of whole pools that fit after alignment */
arenaobj->pool_address = (block*)arenaobj->address;
- arenaobj->nfreepools = MAX_POOLS_IN_ARENA;
+ arenaobj->nfreepools = MAX_POOLS_IN_ARENA;
excess = (uint)(arenaobj->address & POOL_SIZE_MASK);
if (excess != 0) {
--arenaobj->nfreepools;
@@ -1407,7 +1407,7 @@ obmalloc controls. Since this test is needed at every entry point, it's
extremely desirable that it be this fast.
*/
-static bool _Py_NO_SANITIZE_ADDRESS
+static bool _Py_NO_SANITIZE_ADDRESS
_Py_NO_SANITIZE_THREAD
_Py_NO_SANITIZE_MEMORY
address_in_range(void *p, poolp pool)
@@ -1426,76 +1426,76 @@ address_in_range(void *p, poolp pool)
/*==========================================================================*/
-// Called when freelist is exhausted. Extend the freelist if there is
-// space for a block. Otherwise, remove this pool from usedpools.
-static void
-pymalloc_pool_extend(poolp pool, uint size)
-{
- if (UNLIKELY(pool->nextoffset <= pool->maxnextoffset)) {
- /* There is room for another block. */
- pool->freeblock = (block*)pool + pool->nextoffset;
- pool->nextoffset += INDEX2SIZE(size);
- *(block **)(pool->freeblock) = NULL;
- return;
- }
-
- /* Pool is full, unlink from used pools. */
- poolp next;
- next = pool->nextpool;
- pool = pool->prevpool;
- next->prevpool = pool;
- pool->nextpool = next;
-}
-
-/* called when pymalloc_alloc can not allocate a block from usedpool.
- * This function takes new pool and allocate a block from it.
- */
-static void*
-allocate_from_new_pool(uint size)
+// Called when freelist is exhausted. Extend the freelist if there is
+// space for a block. Otherwise, remove this pool from usedpools.
+static void
+pymalloc_pool_extend(poolp pool, uint size)
+{
+ if (UNLIKELY(pool->nextoffset <= pool->maxnextoffset)) {
+ /* There is room for another block. */
+ pool->freeblock = (block*)pool + pool->nextoffset;
+ pool->nextoffset += INDEX2SIZE(size);
+ *(block **)(pool->freeblock) = NULL;
+ return;
+ }
+
+ /* Pool is full, unlink from used pools. */
+ poolp next;
+ next = pool->nextpool;
+ pool = pool->prevpool;
+ next->prevpool = pool;
+ pool->nextpool = next;
+}
+
+/* called when pymalloc_alloc can not allocate a block from usedpool.
+ * This function takes new pool and allocate a block from it.
+ */
+static void*
+allocate_from_new_pool(uint size)
{
/* There isn't a pool of the right size class immediately
* available: use a free pool.
*/
- if (UNLIKELY(usable_arenas == NULL)) {
+ if (UNLIKELY(usable_arenas == NULL)) {
/* No arena has a free pool: allocate a new arena. */
#ifdef WITH_MEMORY_LIMITS
if (narenas_currently_allocated >= MAX_ARENAS) {
- return NULL;
+ return NULL;
}
#endif
usable_arenas = new_arena();
if (usable_arenas == NULL) {
- return NULL;
+ return NULL;
}
- usable_arenas->nextarena = usable_arenas->prevarena = NULL;
- assert(nfp2lasta[usable_arenas->nfreepools] == NULL);
- nfp2lasta[usable_arenas->nfreepools] = usable_arenas;
+ usable_arenas->nextarena = usable_arenas->prevarena = NULL;
+ assert(nfp2lasta[usable_arenas->nfreepools] == NULL);
+ nfp2lasta[usable_arenas->nfreepools] = usable_arenas;
}
assert(usable_arenas->address != 0);
- /* This arena already had the smallest nfreepools value, so decreasing
- * nfreepools doesn't change that, and we don't need to rearrange the
- * usable_arenas list. However, if the arena becomes wholly allocated,
- * we need to remove its arena_object from usable_arenas.
- */
- assert(usable_arenas->nfreepools > 0);
- if (nfp2lasta[usable_arenas->nfreepools] == usable_arenas) {
- /* It's the last of this size, so there won't be any. */
- nfp2lasta[usable_arenas->nfreepools] = NULL;
- }
- /* If any free pools will remain, it will be the new smallest. */
- if (usable_arenas->nfreepools > 1) {
- assert(nfp2lasta[usable_arenas->nfreepools - 1] == NULL);
- nfp2lasta[usable_arenas->nfreepools - 1] = usable_arenas;
- }
-
+ /* This arena already had the smallest nfreepools value, so decreasing
+ * nfreepools doesn't change that, and we don't need to rearrange the
+ * usable_arenas list. However, if the arena becomes wholly allocated,
+ * we need to remove its arena_object from usable_arenas.
+ */
+ assert(usable_arenas->nfreepools > 0);
+ if (nfp2lasta[usable_arenas->nfreepools] == usable_arenas) {
+ /* It's the last of this size, so there won't be any. */
+ nfp2lasta[usable_arenas->nfreepools] = NULL;
+ }
+ /* If any free pools will remain, it will be the new smallest. */
+ if (usable_arenas->nfreepools > 1) {
+ assert(nfp2lasta[usable_arenas->nfreepools - 1] == NULL);
+ nfp2lasta[usable_arenas->nfreepools - 1] = usable_arenas;
+ }
+
/* Try to get a cached free pool. */
- poolp pool = usable_arenas->freepools;
- if (LIKELY(pool != NULL)) {
+ poolp pool = usable_arenas->freepools;
+ if (LIKELY(pool != NULL)) {
/* Unlink from cached pools. */
usable_arenas->freepools = pool->nextpool;
- usable_arenas->nfreepools--;
- if (UNLIKELY(usable_arenas->nfreepools == 0)) {
+ usable_arenas->nfreepools--;
+ if (UNLIKELY(usable_arenas->nfreepools == 0)) {
/* Wholly allocated: remove. */
assert(usable_arenas->freepools == NULL);
assert(usable_arenas->nextarena == NULL ||
@@ -1518,133 +1518,133 @@ allocate_from_new_pool(uint size)
(block*)usable_arenas->address +
ARENA_SIZE - POOL_SIZE);
}
- }
- else {
- /* Carve off a new pool. */
- assert(usable_arenas->nfreepools > 0);
- assert(usable_arenas->freepools == NULL);
- pool = (poolp)usable_arenas->pool_address;
- assert((block*)pool <= (block*)usable_arenas->address +
- ARENA_SIZE - POOL_SIZE);
- pool->arenaindex = (uint)(usable_arenas - arenas);
- assert(&arenas[pool->arenaindex] == usable_arenas);
- pool->szidx = DUMMY_SIZE_IDX;
- usable_arenas->pool_address += POOL_SIZE;
- --usable_arenas->nfreepools;
-
- if (usable_arenas->nfreepools == 0) {
- assert(usable_arenas->nextarena == NULL ||
- usable_arenas->nextarena->prevarena ==
- usable_arenas);
- /* Unlink the arena: it is completely allocated. */
- usable_arenas = usable_arenas->nextarena;
- if (usable_arenas != NULL) {
- usable_arenas->prevarena = NULL;
- assert(usable_arenas->address != 0);
- }
+ }
+ else {
+ /* Carve off a new pool. */
+ assert(usable_arenas->nfreepools > 0);
+ assert(usable_arenas->freepools == NULL);
+ pool = (poolp)usable_arenas->pool_address;
+ assert((block*)pool <= (block*)usable_arenas->address +
+ ARENA_SIZE - POOL_SIZE);
+ pool->arenaindex = (uint)(usable_arenas - arenas);
+ assert(&arenas[pool->arenaindex] == usable_arenas);
+ pool->szidx = DUMMY_SIZE_IDX;
+ usable_arenas->pool_address += POOL_SIZE;
+ --usable_arenas->nfreepools;
+
+ if (usable_arenas->nfreepools == 0) {
+ assert(usable_arenas->nextarena == NULL ||
+ usable_arenas->nextarena->prevarena ==
+ usable_arenas);
+ /* Unlink the arena: it is completely allocated. */
+ usable_arenas = usable_arenas->nextarena;
+ if (usable_arenas != NULL) {
+ usable_arenas->prevarena = NULL;
+ assert(usable_arenas->address != 0);
+ }
}
- }
-
- /* Frontlink to used pools. */
- block *bp;
- poolp next = usedpools[size + size]; /* == prev */
- pool->nextpool = next;
- pool->prevpool = next;
- next->nextpool = pool;
- next->prevpool = pool;
- pool->ref.count = 1;
- if (pool->szidx == size) {
- /* Luckily, this pool last contained blocks
- * of the same size class, so its header
- * and free list are already initialized.
- */
- bp = pool->freeblock;
- assert(bp != NULL);
- pool->freeblock = *(block **)bp;
- return bp;
- }
- /*
- * Initialize the pool header, set up the free list to
- * contain just the second block, and return the first
- * block.
- */
- pool->szidx = size;
- size = INDEX2SIZE(size);
- bp = (block *)pool + POOL_OVERHEAD;
- pool->nextoffset = POOL_OVERHEAD + (size << 1);
- pool->maxnextoffset = POOL_SIZE - size;
- pool->freeblock = bp + size;
- *(block **)(pool->freeblock) = NULL;
- return bp;
-}
-
-/* pymalloc allocator
-
- Return a pointer to newly allocated memory if pymalloc allocated memory.
-
- Return NULL if pymalloc failed to allocate the memory block: on bigger
- requests, on error in the code below (as a last chance to serve the request)
- or when the max memory limit has been reached.
-*/
-static inline void*
-pymalloc_alloc(void *ctx, size_t nbytes)
-{
-#ifdef WITH_VALGRIND
- if (UNLIKELY(running_on_valgrind == -1)) {
- running_on_valgrind = RUNNING_ON_VALGRIND;
- }
- if (UNLIKELY(running_on_valgrind)) {
- return NULL;
- }
-#endif
-
- if (UNLIKELY(nbytes == 0)) {
- return NULL;
- }
- if (UNLIKELY(nbytes > SMALL_REQUEST_THRESHOLD)) {
- return NULL;
- }
-
- uint size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT;
- poolp pool = usedpools[size + size];
- block *bp;
-
- if (LIKELY(pool != pool->nextpool)) {
- /*
- * There is a used pool for this size class.
- * Pick up the head block of its free list.
+ }
+
+ /* Frontlink to used pools. */
+ block *bp;
+ poolp next = usedpools[size + size]; /* == prev */
+ pool->nextpool = next;
+ pool->prevpool = next;
+ next->nextpool = pool;
+ next->prevpool = pool;
+ pool->ref.count = 1;
+ if (pool->szidx == size) {
+ /* Luckily, this pool last contained blocks
+ * of the same size class, so its header
+ * and free list are already initialized.
*/
- ++pool->ref.count;
- bp = pool->freeblock;
- assert(bp != NULL);
-
- if (UNLIKELY((pool->freeblock = *(block **)bp) == NULL)) {
- // Reached the end of the free list, try to extend it.
- pymalloc_pool_extend(pool, size);
+ bp = pool->freeblock;
+ assert(bp != NULL);
+ pool->freeblock = *(block **)bp;
+ return bp;
+ }
+ /*
+ * Initialize the pool header, set up the free list to
+ * contain just the second block, and return the first
+ * block.
+ */
+ pool->szidx = size;
+ size = INDEX2SIZE(size);
+ bp = (block *)pool + POOL_OVERHEAD;
+ pool->nextoffset = POOL_OVERHEAD + (size << 1);
+ pool->maxnextoffset = POOL_SIZE - size;
+ pool->freeblock = bp + size;
+ *(block **)(pool->freeblock) = NULL;
+ return bp;
+}
+
+/* pymalloc allocator
+
+ Return a pointer to newly allocated memory if pymalloc allocated memory.
+
+ Return NULL if pymalloc failed to allocate the memory block: on bigger
+ requests, on error in the code below (as a last chance to serve the request)
+ or when the max memory limit has been reached.
+*/
+static inline void*
+pymalloc_alloc(void *ctx, size_t nbytes)
+{
+#ifdef WITH_VALGRIND
+ if (UNLIKELY(running_on_valgrind == -1)) {
+ running_on_valgrind = RUNNING_ON_VALGRIND;
+ }
+ if (UNLIKELY(running_on_valgrind)) {
+ return NULL;
+ }
+#endif
+
+ if (UNLIKELY(nbytes == 0)) {
+ return NULL;
+ }
+ if (UNLIKELY(nbytes > SMALL_REQUEST_THRESHOLD)) {
+ return NULL;
+ }
+
+ uint size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT;
+ poolp pool = usedpools[size + size];
+ block *bp;
+
+ if (LIKELY(pool != pool->nextpool)) {
+ /*
+ * There is a used pool for this size class.
+ * Pick up the head block of its free list.
+ */
+ ++pool->ref.count;
+ bp = pool->freeblock;
+ assert(bp != NULL);
+
+ if (UNLIKELY((pool->freeblock = *(block **)bp) == NULL)) {
+ // Reached the end of the free list, try to extend it.
+ pymalloc_pool_extend(pool, size);
}
}
- else {
- /* There isn't a pool of the right size class immediately
- * available: use a free pool.
- */
- bp = allocate_from_new_pool(size);
- }
+ else {
+ /* There isn't a pool of the right size class immediately
+ * available: use a free pool.
+ */
+ bp = allocate_from_new_pool(size);
+ }
- return (void *)bp;
+ return (void *)bp;
}
static void *
_PyObject_Malloc(void *ctx, size_t nbytes)
{
- void* ptr = pymalloc_alloc(ctx, nbytes);
- if (LIKELY(ptr != NULL)) {
+ void* ptr = pymalloc_alloc(ctx, nbytes);
+ if (LIKELY(ptr != NULL)) {
return ptr;
}
ptr = PyMem_RawMalloc(nbytes);
if (ptr != NULL) {
- raw_allocated_blocks++;
+ raw_allocated_blocks++;
}
return ptr;
}
@@ -1656,77 +1656,77 @@ _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize)
assert(elsize == 0 || nelem <= (size_t)PY_SSIZE_T_MAX / elsize);
size_t nbytes = nelem * elsize;
- void* ptr = pymalloc_alloc(ctx, nbytes);
- if (LIKELY(ptr != NULL)) {
+ void* ptr = pymalloc_alloc(ctx, nbytes);
+ if (LIKELY(ptr != NULL)) {
memset(ptr, 0, nbytes);
return ptr;
}
ptr = PyMem_RawCalloc(nelem, elsize);
if (ptr != NULL) {
- raw_allocated_blocks++;
+ raw_allocated_blocks++;
}
return ptr;
}
-static void
-insert_to_usedpool(poolp pool)
+static void
+insert_to_usedpool(poolp pool)
{
- assert(pool->ref.count > 0); /* else the pool is empty */
+ assert(pool->ref.count > 0); /* else the pool is empty */
- uint size = pool->szidx;
- poolp next = usedpools[size + size];
- poolp prev = next->prevpool;
+ uint size = pool->szidx;
+ poolp next = usedpools[size + size];
+ poolp prev = next->prevpool;
- /* insert pool before next: prev <-> pool <-> next */
- pool->nextpool = next;
- pool->prevpool = prev;
- next->prevpool = pool;
- prev->nextpool = pool;
-}
+ /* insert pool before next: prev <-> pool <-> next */
+ pool->nextpool = next;
+ pool->prevpool = prev;
+ next->prevpool = pool;
+ prev->nextpool = pool;
+}
-static void
-insert_to_freepool(poolp pool)
-{
- poolp next = pool->nextpool;
- poolp prev = pool->prevpool;
+static void
+insert_to_freepool(poolp pool)
+{
+ poolp next = pool->nextpool;
+ poolp prev = pool->prevpool;
next->prevpool = prev;
prev->nextpool = next;
/* Link the pool to freepools. This is a singly-linked
* list, and pool->prevpool isn't used there.
*/
- struct arena_object *ao = &arenas[pool->arenaindex];
+ struct arena_object *ao = &arenas[pool->arenaindex];
pool->nextpool = ao->freepools;
ao->freepools = pool;
- uint nf = ao->nfreepools;
- /* If this is the rightmost arena with this number of free pools,
- * nfp2lasta[nf] needs to change. Caution: if nf is 0, there
- * are no arenas in usable_arenas with that value.
- */
- struct arena_object* lastnf = nfp2lasta[nf];
- assert((nf == 0 && lastnf == NULL) ||
- (nf > 0 &&
- lastnf != NULL &&
- lastnf->nfreepools == nf &&
- (lastnf->nextarena == NULL ||
- nf < lastnf->nextarena->nfreepools)));
- if (lastnf == ao) { /* it is the rightmost */
- struct arena_object* p = ao->prevarena;
- nfp2lasta[nf] = (p != NULL && p->nfreepools == nf) ? p : NULL;
- }
- ao->nfreepools = ++nf;
+ uint nf = ao->nfreepools;
+ /* If this is the rightmost arena with this number of free pools,
+ * nfp2lasta[nf] needs to change. Caution: if nf is 0, there
+ * are no arenas in usable_arenas with that value.
+ */
+ struct arena_object* lastnf = nfp2lasta[nf];
+ assert((nf == 0 && lastnf == NULL) ||
+ (nf > 0 &&
+ lastnf != NULL &&
+ lastnf->nfreepools == nf &&
+ (lastnf->nextarena == NULL ||
+ nf < lastnf->nextarena->nfreepools)));
+ if (lastnf == ao) { /* it is the rightmost */
+ struct arena_object* p = ao->prevarena;
+ nfp2lasta[nf] = (p != NULL && p->nfreepools == nf) ? p : NULL;
+ }
+ ao->nfreepools = ++nf;
/* All the rest is arena management. We just freed
* a pool, and there are 4 cases for arena mgmt:
* 1. If all the pools are free, return the arena to
- * the system free(). Except if this is the last
- * arena in the list, keep it to avoid thrashing:
- * keeping one wholly free arena in the list avoids
- * pathological cases where a simple loop would
- * otherwise provoke needing to allocate and free an
- * arena on every iteration. See bpo-37257.
+ * the system free(). Except if this is the last
+ * arena in the list, keep it to avoid thrashing:
+ * keeping one wholly free arena in the list avoids
+ * pathological cases where a simple loop would
+ * otherwise provoke needing to allocate and free an
+ * arena on every iteration. See bpo-37257.
* 2. If this is the only free pool in the arena,
* add the arena back to the `usable_arenas` list.
* 3. If the "next" arena has a smaller count of free
@@ -1735,7 +1735,7 @@ insert_to_freepool(poolp pool)
* nfreepools.
* 4. Else there's nothing more to do.
*/
- if (nf == ao->ntotalpools && ao->nextarena != NULL) {
+ if (nf == ao->ntotalpools && ao->nextarena != NULL) {
/* Case 1. First unlink ao from usable_arenas.
*/
assert(ao->prevarena == NULL ||
@@ -1774,7 +1774,7 @@ insert_to_freepool(poolp pool)
ao->address = 0; /* mark unassociated */
--narenas_currently_allocated;
- return;
+ return;
}
if (nf == 1) {
@@ -1789,11 +1789,11 @@ insert_to_freepool(poolp pool)
usable_arenas->prevarena = ao;
usable_arenas = ao;
assert(usable_arenas->address != 0);
- if (nfp2lasta[1] == NULL) {
- nfp2lasta[1] = ao;
- }
+ if (nfp2lasta[1] == NULL) {
+ nfp2lasta[1] = ao;
+ }
- return;
+ return;
}
/* If this arena is now out of order, we need to keep
@@ -1803,23 +1803,23 @@ insert_to_freepool(poolp pool)
* a few un-scientific tests, it seems like this
* approach allowed a lot more memory to be freed.
*/
- /* If this is the only arena with nf, record that. */
- if (nfp2lasta[nf] == NULL) {
- nfp2lasta[nf] = ao;
- } /* else the rightmost with nf doesn't change */
- /* If this was the rightmost of the old size, it remains in place. */
- if (ao == lastnf) {
+ /* If this is the only arena with nf, record that. */
+ if (nfp2lasta[nf] == NULL) {
+ nfp2lasta[nf] = ao;
+ } /* else the rightmost with nf doesn't change */
+ /* If this was the rightmost of the old size, it remains in place. */
+ if (ao == lastnf) {
/* Case 4. Nothing to do. */
- return;
- }
- /* If ao were the only arena in the list, the last block would have
- * gotten us out.
- */
- assert(ao->nextarena != NULL);
-
- /* Case 3: We have to move the arena towards the end of the list,
- * because it has more free pools than the arena to its right. It needs
- * to move to follow lastnf.
+ return;
+ }
+ /* If ao were the only arena in the list, the last block would have
+ * gotten us out.
+ */
+ assert(ao->nextarena != NULL);
+
+ /* Case 3: We have to move the arena towards the end of the list,
+ * because it has more free pools than the arena to its right. It needs
+ * to move to follow lastnf.
* First unlink ao from usable_arenas.
*/
if (ao->prevarena != NULL) {
@@ -1833,78 +1833,78 @@ insert_to_freepool(poolp pool)
usable_arenas = ao->nextarena;
}
ao->nextarena->prevarena = ao->prevarena;
- /* And insert after lastnf. */
- ao->prevarena = lastnf;
- ao->nextarena = lastnf->nextarena;
+ /* And insert after lastnf. */
+ ao->prevarena = lastnf;
+ ao->nextarena = lastnf->nextarena;
if (ao->nextarena != NULL) {
ao->nextarena->prevarena = ao;
}
- lastnf->nextarena = ao;
+ lastnf->nextarena = ao;
/* Verify that the swaps worked. */
assert(ao->nextarena == NULL || nf <= ao->nextarena->nfreepools);
assert(ao->prevarena == NULL || nf > ao->prevarena->nfreepools);
assert(ao->nextarena == NULL || ao->nextarena->prevarena == ao);
assert((usable_arenas == ao && ao->prevarena == NULL)
|| ao->prevarena->nextarena == ao);
-}
-
-/* Free a memory block allocated by pymalloc_alloc().
- Return 1 if it was freed.
- Return 0 if the block was not allocated by pymalloc_alloc(). */
-static inline int
-pymalloc_free(void *ctx, void *p)
-{
- assert(p != NULL);
-
-#ifdef WITH_VALGRIND
- if (UNLIKELY(running_on_valgrind > 0)) {
- return 0;
- }
-#endif
-
- poolp pool = POOL_ADDR(p);
- if (UNLIKELY(!address_in_range(p, pool))) {
- return 0;
- }
- /* We allocated this address. */
-
- /* Link p to the start of the pool's freeblock list. Since
- * the pool had at least the p block outstanding, the pool
- * wasn't empty (so it's already in a usedpools[] list, or
- * was full and is in no list -- it's not in the freeblocks
- * list in any case).
- */
- assert(pool->ref.count > 0); /* else it was empty */
- block *lastfree = pool->freeblock;
- *(block **)p = lastfree;
- pool->freeblock = (block *)p;
- pool->ref.count--;
-
- if (UNLIKELY(lastfree == NULL)) {
- /* Pool was full, so doesn't currently live in any list:
- * link it to the front of the appropriate usedpools[] list.
- * This mimics LRU pool usage for new allocations and
- * targets optimal filling when several pools contain
- * blocks of the same size class.
- */
- insert_to_usedpool(pool);
- return 1;
- }
-
- /* freeblock wasn't NULL, so the pool wasn't full,
- * and the pool is in a usedpools[] list.
- */
- if (LIKELY(pool->ref.count != 0)) {
- /* pool isn't empty: leave it in usedpools */
- return 1;
- }
-
- /* Pool is now empty: unlink from usedpools, and
- * link to the front of freepools. This ensures that
- * previously freed pools will be allocated later
- * (being not referenced, they are perhaps paged out).
- */
- insert_to_freepool(pool);
+}
+
+/* Free a memory block allocated by pymalloc_alloc().
+ Return 1 if it was freed.
+ Return 0 if the block was not allocated by pymalloc_alloc(). */
+static inline int
+pymalloc_free(void *ctx, void *p)
+{
+ assert(p != NULL);
+
+#ifdef WITH_VALGRIND
+ if (UNLIKELY(running_on_valgrind > 0)) {
+ return 0;
+ }
+#endif
+
+ poolp pool = POOL_ADDR(p);
+ if (UNLIKELY(!address_in_range(p, pool))) {
+ return 0;
+ }
+ /* We allocated this address. */
+
+ /* Link p to the start of the pool's freeblock list. Since
+ * the pool had at least the p block outstanding, the pool
+ * wasn't empty (so it's already in a usedpools[] list, or
+ * was full and is in no list -- it's not in the freeblocks
+ * list in any case).
+ */
+ assert(pool->ref.count > 0); /* else it was empty */
+ block *lastfree = pool->freeblock;
+ *(block **)p = lastfree;
+ pool->freeblock = (block *)p;
+ pool->ref.count--;
+
+ if (UNLIKELY(lastfree == NULL)) {
+ /* Pool was full, so doesn't currently live in any list:
+ * link it to the front of the appropriate usedpools[] list.
+ * This mimics LRU pool usage for new allocations and
+ * targets optimal filling when several pools contain
+ * blocks of the same size class.
+ */
+ insert_to_usedpool(pool);
+ return 1;
+ }
+
+ /* freeblock wasn't NULL, so the pool wasn't full,
+ * and the pool is in a usedpools[] list.
+ */
+ if (LIKELY(pool->ref.count != 0)) {
+ /* pool isn't empty: leave it in usedpools */
+ return 1;
+ }
+
+ /* Pool is now empty: unlink from usedpools, and
+ * link to the front of freepools. This ensures that
+ * previously freed pools will be allocated later
+ * (being not referenced, they are perhaps paged out).
+ */
+ insert_to_freepool(pool);
return 1;
}
@@ -1917,10 +1917,10 @@ _PyObject_Free(void *ctx, void *p)
return;
}
- if (UNLIKELY(!pymalloc_free(ctx, p))) {
+ if (UNLIKELY(!pymalloc_free(ctx, p))) {
/* pymalloc didn't allocate this address */
PyMem_RawFree(p);
- raw_allocated_blocks--;
+ raw_allocated_blocks--;
}
}
@@ -2031,10 +2031,10 @@ _Py_GetAllocatedBlocks(void)
* it wraps a real allocator, adding extra debugging info to the memory blocks.
*/
-/* Uncomment this define to add the "serialno" field */
-/* #define PYMEM_DEBUG_SERIALNO */
+/* Uncomment this define to add the "serialno" field */
+/* #define PYMEM_DEBUG_SERIALNO */
-#ifdef PYMEM_DEBUG_SERIALNO
+#ifdef PYMEM_DEBUG_SERIALNO
static size_t serialno = 0; /* incremented on each debug {m,re}alloc */
/* serialno is always incremented via calling this routine. The point is
@@ -2045,16 +2045,16 @@ bumpserialno(void)
{
++serialno;
}
-#endif
+#endif
#define SST SIZEOF_SIZE_T
-#ifdef PYMEM_DEBUG_SERIALNO
-# define PYMEM_DEBUG_EXTRA_BYTES 4 * SST
-#else
-# define PYMEM_DEBUG_EXTRA_BYTES 3 * SST
-#endif
-
+#ifdef PYMEM_DEBUG_SERIALNO
+# define PYMEM_DEBUG_EXTRA_BYTES 4 * SST
+#else
+# define PYMEM_DEBUG_EXTRA_BYTES 3 * SST
+#endif
+
/* Read sizeof(size_t) bytes at p as a big-endian size_t. */
static size_t
read_size_t(const void *p)
@@ -2083,7 +2083,7 @@ write_size_t(void *p, size_t n)
}
}
-/* Let S = sizeof(size_t). The debug malloc asks for 4 * S extra bytes and
+/* Let S = sizeof(size_t). The debug malloc asks for 4 * S extra bytes and
fills them with useful stuff, here calling the underlying malloc's result p:
p[0: S]
@@ -2092,14 +2092,14 @@ p[0: S]
p[S]
API ID. See PEP 445. This is a character, but seems undocumented.
p[S+1: 2*S]
- Copies of PYMEM_FORBIDDENBYTE. Used to catch under- writes and reads.
+ Copies of PYMEM_FORBIDDENBYTE. Used to catch under- writes and reads.
p[2*S: 2*S+n]
- The requested memory, filled with copies of PYMEM_CLEANBYTE.
+ The requested memory, filled with copies of PYMEM_CLEANBYTE.
Used to catch reference to uninitialized memory.
&p[2*S] is returned. Note that this is 8-byte aligned if pymalloc
handled the request itself.
p[2*S+n: 2*S+n+S]
- Copies of PYMEM_FORBIDDENBYTE. Used to catch over- writes and reads.
+ Copies of PYMEM_FORBIDDENBYTE. Used to catch over- writes and reads.
p[2*S+n+S: 2*S+n+2*S]
A serial number, incremented by 1 on each call to _PyMem_DebugMalloc
and _PyMem_DebugRealloc.
@@ -2107,9 +2107,9 @@ p[2*S+n+S: 2*S+n+2*S]
If "bad memory" is detected later, the serial number gives an
excellent way to set a breakpoint on the next run, to capture the
instant at which this block was passed out.
-
-If PYMEM_DEBUG_SERIALNO is not defined (default), the debug malloc only asks
-for 3 * S extra bytes, and omits the last serialno field.
+
+If PYMEM_DEBUG_SERIALNO is not defined (default), the debug malloc only asks
+for 3 * S extra bytes, and omits the last serialno field.
*/
static void *
@@ -2119,25 +2119,25 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
uint8_t *p; /* base address of malloc'ed pad block */
uint8_t *data; /* p + 2*SST == pointer to data bytes */
uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */
- size_t total; /* nbytes + PYMEM_DEBUG_EXTRA_BYTES */
+ size_t total; /* nbytes + PYMEM_DEBUG_EXTRA_BYTES */
- if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) {
+ if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) {
/* integer overflow: can't represent total as a Py_ssize_t */
return NULL;
}
- total = nbytes + PYMEM_DEBUG_EXTRA_BYTES;
+ total = nbytes + PYMEM_DEBUG_EXTRA_BYTES;
/* Layout: [SSSS IFFF CCCC...CCCC FFFF NNNN]
- ^--- p ^--- data ^--- tail
+ ^--- p ^--- data ^--- tail
S: nbytes stored as size_t
I: API identifier (1 byte)
F: Forbidden bytes (size_t - 1 bytes before, size_t bytes after)
C: Clean bytes used later to store actual data
- N: Serial number stored as size_t
-
- If PYMEM_DEBUG_SERIALNO is not defined (default), the last NNNN field
- is omitted. */
+ N: Serial number stored as size_t
+ If PYMEM_DEBUG_SERIALNO is not defined (default), the last NNNN field
+ is omitted. */
+
if (use_calloc) {
p = (uint8_t *)api->alloc.calloc(api->alloc.ctx, 1, total);
}
@@ -2149,25 +2149,25 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
}
data = p + 2*SST;
-#ifdef PYMEM_DEBUG_SERIALNO
+#ifdef PYMEM_DEBUG_SERIALNO
bumpserialno();
-#endif
+#endif
/* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
write_size_t(p, nbytes);
p[SST] = (uint8_t)api->api_id;
- memset(p + SST + 1, PYMEM_FORBIDDENBYTE, SST-1);
+ memset(p + SST + 1, PYMEM_FORBIDDENBYTE, SST-1);
if (nbytes > 0 && !use_calloc) {
- memset(data, PYMEM_CLEANBYTE, nbytes);
+ memset(data, PYMEM_CLEANBYTE, nbytes);
}
/* at tail, write pad (SST bytes) and serialno (SST bytes) */
tail = data + nbytes;
- memset(tail, PYMEM_FORBIDDENBYTE, SST);
-#ifdef PYMEM_DEBUG_SERIALNO
+ memset(tail, PYMEM_FORBIDDENBYTE, SST);
+#ifdef PYMEM_DEBUG_SERIALNO
write_size_t(tail + SST, serialno);
-#endif
+#endif
return data;
}
@@ -2190,7 +2190,7 @@ _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize)
/* The debug free first checks the 2*SST bytes on each end for sanity (in
particular, that the FORBIDDENBYTEs with the api ID are still intact).
- Then fills the original bytes with PYMEM_DEADBYTE.
+ Then fills the original bytes with PYMEM_DEADBYTE.
Then calls the underlying free.
*/
static void
@@ -2205,10 +2205,10 @@ _PyMem_DebugRawFree(void *ctx, void *p)
uint8_t *q = (uint8_t *)p - 2*SST; /* address returned from malloc */
size_t nbytes;
- _PyMem_DebugCheckAddress(__func__, api->api_id, p);
+ _PyMem_DebugCheckAddress(__func__, api->api_id, p);
nbytes = read_size_t(q);
- nbytes += PYMEM_DEBUG_EXTRA_BYTES;
- memset(q, PYMEM_DEADBYTE, nbytes);
+ nbytes += PYMEM_DEBUG_EXTRA_BYTES;
+ memset(q, PYMEM_DEADBYTE, nbytes);
api->alloc.free(api->alloc.ctx, q);
}
@@ -2230,62 +2230,62 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
#define ERASED_SIZE 64
uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */
- _PyMem_DebugCheckAddress(__func__, api->api_id, p);
+ _PyMem_DebugCheckAddress(__func__, api->api_id, p);
data = (uint8_t *)p;
head = data - 2*SST;
original_nbytes = read_size_t(head);
- if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) {
+ if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) {
/* integer overflow: can't represent total as a Py_ssize_t */
return NULL;
}
- total = nbytes + PYMEM_DEBUG_EXTRA_BYTES;
+ total = nbytes + PYMEM_DEBUG_EXTRA_BYTES;
tail = data + original_nbytes;
-#ifdef PYMEM_DEBUG_SERIALNO
- size_t block_serialno = read_size_t(tail + SST);
-#endif
+#ifdef PYMEM_DEBUG_SERIALNO
+ size_t block_serialno = read_size_t(tail + SST);
+#endif
/* Mark the header, the trailer, ERASED_SIZE bytes at the begin and
ERASED_SIZE bytes at the end as dead and save the copy of erased bytes.
*/
if (original_nbytes <= sizeof(save)) {
memcpy(save, data, original_nbytes);
- memset(data - 2 * SST, PYMEM_DEADBYTE,
- original_nbytes + PYMEM_DEBUG_EXTRA_BYTES);
+ memset(data - 2 * SST, PYMEM_DEADBYTE,
+ original_nbytes + PYMEM_DEBUG_EXTRA_BYTES);
}
else {
memcpy(save, data, ERASED_SIZE);
- memset(head, PYMEM_DEADBYTE, ERASED_SIZE + 2 * SST);
+ memset(head, PYMEM_DEADBYTE, ERASED_SIZE + 2 * SST);
memcpy(&save[ERASED_SIZE], tail - ERASED_SIZE, ERASED_SIZE);
- memset(tail - ERASED_SIZE, PYMEM_DEADBYTE,
- ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST);
+ memset(tail - ERASED_SIZE, PYMEM_DEADBYTE,
+ ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST);
}
/* Resize and add decorations. */
r = (uint8_t *)api->alloc.realloc(api->alloc.ctx, head, total);
if (r == NULL) {
- /* if realloc() failed: rewrite header and footer which have
- just been erased */
+ /* if realloc() failed: rewrite header and footer which have
+ just been erased */
nbytes = original_nbytes;
}
else {
head = r;
-#ifdef PYMEM_DEBUG_SERIALNO
+#ifdef PYMEM_DEBUG_SERIALNO
bumpserialno();
block_serialno = serialno;
-#endif
+#endif
}
- data = head + 2*SST;
+ data = head + 2*SST;
write_size_t(head, nbytes);
head[SST] = (uint8_t)api->api_id;
- memset(head + SST + 1, PYMEM_FORBIDDENBYTE, SST-1);
+ memset(head + SST + 1, PYMEM_FORBIDDENBYTE, SST-1);
tail = data + nbytes;
- memset(tail, PYMEM_FORBIDDENBYTE, SST);
-#ifdef PYMEM_DEBUG_SERIALNO
+ memset(tail, PYMEM_FORBIDDENBYTE, SST);
+#ifdef PYMEM_DEBUG_SERIALNO
write_size_t(tail + SST, block_serialno);
-#endif
+#endif
/* Restore saved bytes. */
if (original_nbytes <= sizeof(save)) {
@@ -2305,35 +2305,35 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
}
if (nbytes > original_nbytes) {
- /* growing: mark new extra memory clean */
- memset(data + original_nbytes, PYMEM_CLEANBYTE,
- nbytes - original_nbytes);
+ /* growing: mark new extra memory clean */
+ memset(data + original_nbytes, PYMEM_CLEANBYTE,
+ nbytes - original_nbytes);
}
return data;
}
-static inline void
-_PyMem_DebugCheckGIL(const char *func)
+static inline void
+_PyMem_DebugCheckGIL(const char *func)
{
- if (!PyGILState_Check()) {
- _Py_FatalErrorFunc(func,
- "Python memory allocator called "
- "without holding the GIL");
- }
+ if (!PyGILState_Check()) {
+ _Py_FatalErrorFunc(func,
+ "Python memory allocator called "
+ "without holding the GIL");
+ }
}
static void *
_PyMem_DebugMalloc(void *ctx, size_t nbytes)
{
- _PyMem_DebugCheckGIL(__func__);
+ _PyMem_DebugCheckGIL(__func__);
return _PyMem_DebugRawMalloc(ctx, nbytes);
}
static void *
_PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize)
{
- _PyMem_DebugCheckGIL(__func__);
+ _PyMem_DebugCheckGIL(__func__);
return _PyMem_DebugRawCalloc(ctx, nelem, elsize);
}
@@ -2341,7 +2341,7 @@ _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize)
static void
_PyMem_DebugFree(void *ctx, void *ptr)
{
- _PyMem_DebugCheckGIL(__func__);
+ _PyMem_DebugCheckGIL(__func__);
_PyMem_DebugRawFree(ctx, ptr);
}
@@ -2349,7 +2349,7 @@ _PyMem_DebugFree(void *ctx, void *ptr)
static void *
_PyMem_DebugRealloc(void *ctx, void *ptr, size_t nbytes)
{
- _PyMem_DebugCheckGIL(__func__);
+ _PyMem_DebugCheckGIL(__func__);
return _PyMem_DebugRawRealloc(ctx, ptr, nbytes);
}
@@ -2359,10 +2359,10 @@ _PyMem_DebugRealloc(void *ctx, void *ptr, size_t nbytes)
* The API id, is also checked.
*/
static void
-_PyMem_DebugCheckAddress(const char *func, char api, const void *p)
+_PyMem_DebugCheckAddress(const char *func, char api, const void *p)
{
- assert(p != NULL);
-
+ assert(p != NULL);
+
const uint8_t *q = (const uint8_t *)p;
size_t nbytes;
const uint8_t *tail;
@@ -2372,11 +2372,11 @@ _PyMem_DebugCheckAddress(const char *func, char api, const void *p)
/* Check the API id */
id = (char)q[-SST];
if (id != api) {
- _PyObject_DebugDumpAddress(p);
- _Py_FatalErrorFormat(func,
- "bad ID: Allocated using API '%c', "
- "verified using API '%c'",
- id, api);
+ _PyObject_DebugDumpAddress(p);
+ _Py_FatalErrorFormat(func,
+ "bad ID: Allocated using API '%c', "
+ "verified using API '%c'",
+ id, api);
}
/* Check the stuff at the start of p first: if there's underwrite
@@ -2384,18 +2384,18 @@ _PyMem_DebugCheckAddress(const char *func, char api, const void *p)
* the tail could lead to a segfault then.
*/
for (i = SST-1; i >= 1; --i) {
- if (*(q-i) != PYMEM_FORBIDDENBYTE) {
- _PyObject_DebugDumpAddress(p);
- _Py_FatalErrorFunc(func, "bad leading pad byte");
+ if (*(q-i) != PYMEM_FORBIDDENBYTE) {
+ _PyObject_DebugDumpAddress(p);
+ _Py_FatalErrorFunc(func, "bad leading pad byte");
}
}
nbytes = read_size_t(q - 2*SST);
tail = q + nbytes;
for (i = 0; i < SST; ++i) {
- if (tail[i] != PYMEM_FORBIDDENBYTE) {
- _PyObject_DebugDumpAddress(p);
- _Py_FatalErrorFunc(func, "bad trailing pad byte");
+ if (tail[i] != PYMEM_FORBIDDENBYTE) {
+ _PyObject_DebugDumpAddress(p);
+ _Py_FatalErrorFunc(func, "bad trailing pad byte");
}
}
}
@@ -2406,7 +2406,7 @@ _PyObject_DebugDumpAddress(const void *p)
{
const uint8_t *q = (const uint8_t *)p;
const uint8_t *tail;
- size_t nbytes;
+ size_t nbytes;
int i;
int ok;
char id;
@@ -2427,7 +2427,7 @@ _PyObject_DebugDumpAddress(const void *p)
fprintf(stderr, " The %d pad bytes at p-%d are ", SST-1, SST-1);
ok = 1;
for (i = 1; i <= SST-1; ++i) {
- if (*(q-i) != PYMEM_FORBIDDENBYTE) {
+ if (*(q-i) != PYMEM_FORBIDDENBYTE) {
ok = 0;
break;
}
@@ -2436,11 +2436,11 @@ _PyObject_DebugDumpAddress(const void *p)
fputs("FORBIDDENBYTE, as expected.\n", stderr);
else {
fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
- PYMEM_FORBIDDENBYTE);
+ PYMEM_FORBIDDENBYTE);
for (i = SST-1; i >= 1; --i) {
const uint8_t byte = *(q-i);
fprintf(stderr, " at p-%d: 0x%02x", i, byte);
- if (byte != PYMEM_FORBIDDENBYTE)
+ if (byte != PYMEM_FORBIDDENBYTE)
fputs(" *** OUCH", stderr);
fputc('\n', stderr);
}
@@ -2452,10 +2452,10 @@ _PyObject_DebugDumpAddress(const void *p)
}
tail = q + nbytes;
- fprintf(stderr, " The %d pad bytes at tail=%p are ", SST, (void *)tail);
+ fprintf(stderr, " The %d pad bytes at tail=%p are ", SST, (void *)tail);
ok = 1;
for (i = 0; i < SST; ++i) {
- if (tail[i] != PYMEM_FORBIDDENBYTE) {
+ if (tail[i] != PYMEM_FORBIDDENBYTE) {
ok = 0;
break;
}
@@ -2464,22 +2464,22 @@ _PyObject_DebugDumpAddress(const void *p)
fputs("FORBIDDENBYTE, as expected.\n", stderr);
else {
fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
- PYMEM_FORBIDDENBYTE);
+ PYMEM_FORBIDDENBYTE);
for (i = 0; i < SST; ++i) {
const uint8_t byte = tail[i];
fprintf(stderr, " at tail+%d: 0x%02x",
i, byte);
- if (byte != PYMEM_FORBIDDENBYTE)
+ if (byte != PYMEM_FORBIDDENBYTE)
fputs(" *** OUCH", stderr);
fputc('\n', stderr);
}
}
-#ifdef PYMEM_DEBUG_SERIALNO
- size_t serial = read_size_t(tail + SST);
+#ifdef PYMEM_DEBUG_SERIALNO
+ size_t serial = read_size_t(tail + SST);
fprintf(stderr, " The block was made by call #%" PY_FORMAT_SIZE_T
"u to debug malloc/realloc.\n", serial);
-#endif
+#endif
if (nbytes > 0) {
i = 0;
@@ -2705,11 +2705,11 @@ _PyObject_DebugMallocStats(FILE *out)
quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size);
}
fputc('\n', out);
-#ifdef PYMEM_DEBUG_SERIALNO
- if (_PyMem_DebugEnabled()) {
+#ifdef PYMEM_DEBUG_SERIALNO
+ if (_PyMem_DebugEnabled()) {
(void)printone(out, "# times object malloc called", serialno);
- }
-#endif
+ }
+#endif
(void)printone(out, "# arenas allocated total", ntimes_arena_allocated);
(void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas);
(void)printone(out, "# arenas highwater mark", narenas_highwater);