gh-101659: Isolate "obmalloc" State to Each Interpreter (gh-101660) · python/cpython@df3173d · GitHub
Skip to content

Commit df3173d

Browse files
ericsnowcurrentlyericsnowcurrently
authored
gh-101659: Isolate "obmalloc" State to Each Interpreter (gh-101660)
This is strictly about moving the "obmalloc" runtime state from `_PyRuntimeState` to `PyInterpreterState`. Doing so improves isolation between interpreters, specifically most of the memory (incl. objects) allocated for each interpreter's use. This is important for a per-interpreter GIL, but such isolation is valuable even without it. FWIW, a per-interpreter obmalloc is the proverbial canary-in-the-coalmine when it comes to the isolation of objects between interpreters. Any object that leaks (unintentionally) to another interpreter is highly likely to cause a crash (on debug builds at least). That's a useful thing to know, relative to interpreter isolation.
1 parent 01be52e commit df3173d

20 files changed

Lines changed: 322 additions & 73 deletions

‎Include/cpython/initconfig.h‎

Lines changed: 4 additions & 0 deletions

‎Include/cpython/pystate.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ is available in a given context. For example, forking the process
1111
might not be allowed in the current interpreter (i.e. os.fork() would fail).
1212
*/
1313

14+
/* Set if the interpreter share obmalloc runtime state
15+
with the main interpreter. */
16+
#define Py_RTFLAGS_USE_MAIN_OBMALLOC (1UL << 5)
17+
1418
/* Set if import should check a module for subinterpreter support. */
1519
#define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8)
1620

‎Include/internal/pycore_interp.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,12 @@ extern "C" {
2323
#include "pycore_function.h" // FUNC_MAX_WATCHERS
2424
#include "pycore_genobject.h" // struct _Py_async_gen_state
2525
#include "pycore_gc.h" // struct _gc_runtime_state
26+
#include "pycore_global_objects.h" // struct _Py_interp_static_objects
2627
#include "pycore_import.h" // struct _import_state
2728
#include "pycore_instruments.h" // PY_MONITORING_EVENTS
2829
#include "pycore_list.h" // struct _Py_list_state
29-
#include "pycore_global_objects.h" // struct _Py_interp_static_objects
3030
#include "pycore_object_state.h" // struct _py_object_state
31+
#include "pycore_obmalloc.h" // struct obmalloc_state
3132
#include "pycore_tuple.h" // struct _Py_tuple_state
3233
#include "pycore_typeobject.h" // struct type_cache
3334
#include "pycore_unicodeobject.h" // struct _Py_unicode_state
@@ -82,6 +83,8 @@ struct _is {
8283
int _initialized;
8384
int finalizing;
8485

86+
struct _obmalloc_state obmalloc;
87+
8588
struct _ceval_state ceval;
8689
struct _gc_runtime_state gc;
8790

‎Include/internal/pycore_obmalloc.h‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -657,8 +657,12 @@ struct _obmalloc_usage {
657657
#endif /* WITH_PYMALLOC_RADIX_TREE */
658658

659659

660-
struct _obmalloc_state {
660+
struct _obmalloc_global_state {
661661
int dump_debug_stats;
662+
Py_ssize_t interpreter_leaks;
663+
};
664+
665+
struct _obmalloc_state {
662666
struct _obmalloc_pools pools;
663667
struct _obmalloc_mgmt mgmt;
664668
struct _obmalloc_usage usage;
@@ -675,7 +679,11 @@ void _PyObject_VirtualFree(void *, size_t size);
675679

676680

677681
/* This function returns the number of allocated memory blocks, regardless of size */
678-
PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
682+
extern Py_ssize_t _Py_GetGlobalAllocatedBlocks(void);
683+
#define _Py_GetAllocatedBlocks() \
684+
_Py_GetGlobalAllocatedBlocks()
685+
extern Py_ssize_t _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *);
686+
extern void _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *);
679687

680688

681689
#ifdef WITH_PYMALLOC

‎Include/internal/pycore_obmalloc_init.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,13 @@ extern "C" {
5454
# error "NB_SMALL_SIZE_CLASSES should be less than 64"
5555
#endif
5656

57-
#define _obmalloc_state_INIT(obmalloc) \
57+
#define _obmalloc_global_state_INIT \
5858
{ \
5959
.dump_debug_stats = -1, \
60+
}
61+
62+
#define _obmalloc_state_INIT(obmalloc) \
63+
{ \
6064
.pools = { \
6165
.used = _obmalloc_pools_INIT(obmalloc.pools), \
6266
}, \

‎Include/internal/pycore_pylifecycle.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ extern void _PyAtExit_Fini(PyInterpreterState *interp);
6464
extern void _PyThread_FiniType(PyInterpreterState *interp);
6565
extern void _Py_Deepfreeze_Fini(void);
6666
extern void _PyArg_Fini(void);
67+
extern void _Py_FinalizeAllocatedBlocks(_PyRuntimeState *);
6768

6869
extern PyStatus _PyGILState_Init(PyInterpreterState *interp);
6970
extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate);

‎Include/internal/pycore_pystate.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@ _Py_IsMainInterpreter(PyInterpreterState *interp)
3333
return (interp == _PyInterpreterState_Main());
3434
}
3535

36+
static inline int
37+
_Py_IsMainInterpreterFinalizing(PyInterpreterState *interp)
38+
{
39+
return (_PyRuntimeState_GetFinalizing(interp->runtime) != NULL &&
40+
interp == &interp->runtime->_main_interpreter);
41+
}
42+
3643

3744
static inline const PyConfig *
3845
_Py_GetMainConfig(void)

‎Include/internal/pycore_runtime.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ extern "C" {
2121
#include "pycore_pymem.h" // struct _pymem_allocators
2222
#include "pycore_pyhash.h" // struct pyhash_runtime_state
2323
#include "pycore_pythread.h" // struct _pythread_runtime_state
24-
#include "pycore_obmalloc.h" // struct obmalloc_state
2524
#include "pycore_signal.h" // struct _signals_runtime_state
2625
#include "pycore_time.h" // struct _time_runtime_state
2726
#include "pycore_tracemalloc.h" // struct _tracemalloc_runtime_state
@@ -88,7 +87,7 @@ typedef struct pyruntimestate {
8887
_Py_atomic_address _finalizing;
8988

9089
struct _pymem_allocators allocators;
91-
struct _obmalloc_state obmalloc;
90+
struct _obmalloc_global_state obmalloc;
9291
struct pyhash_runtime_state pyhash_state;
9392
struct _time_runtime_state time;
9493
struct _pythread_runtime_state threads;

‎Include/internal/pycore_runtime_init.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ extern PyTypeObject _PyExc_MemoryError;
2929
_pymem_allocators_debug_INIT, \
3030
_pymem_allocators_obj_arena_INIT, \
3131
}, \
32-
.obmalloc = _obmalloc_state_INIT(runtime.obmalloc), \
32+
.obmalloc = _obmalloc_global_state_INIT, \
3333
.pyhash_state = pyhash_state_INIT, \
3434
.signals = _signals_RUNTIME_INIT, \
3535
.interpreters = { \
@@ -93,6 +93,7 @@ extern PyTypeObject _PyExc_MemoryError;
9393
{ \
9494
.id_refcount = -1, \
9595
.imports = IMPORTS_INIT, \
96+
.obmalloc = _obmalloc_state_INIT(INTERP.obmalloc), \
9697
.ceval = { \
9798
.recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \
9899
}, \

‎Lib/test/test_capi/test_misc.py‎

Lines changed: 27 additions & 6 deletions

0 commit comments

Comments
 (0)