#include "Python.h"
#include "pycore_abstract.h"
#include "pycore_audit.h"
#include "pycore_ceval.h"
#include "pycore_code.h"
#include "pycore_critical_section.h"
#include "pycore_dtoa.h"
#include "pycore_emscripten_trampoline.h"
#include "pycore_frame.h"
#include "pycore_freelist.h"
#include "pycore_initconfig.h"
#include "pycore_object.h"
#include "pycore_parking_lot.h"
#include "pycore_pyerrors.h"
#include "pycore_pylifecycle.h"
#include "pycore_pymem.h"
#include "pycore_pystate.h"
#include "pycore_runtime_init.h"
#include "pycore_obmalloc.h"
#include "pycore_uniqueid.h"
#ifdef HAVE_DLOPEN
# ifdef HAVE_DLFCN_H
# include <dlfcn.h>
# endif
# if !HAVE_DECL_RTLD_LAZY
#define RTLD_LAZY …
# endif
#endif
#ifdef HAVE_THREAD_LOCAL
_Py_thread_local PyThreadState *_Py_tss_tstate = …;
#endif
static inline PyThreadState *
current_fast_get(void)
{ … }
static inline void
current_fast_set(_PyRuntimeState *Py_UNUSED(runtime), PyThreadState *tstate)
{ … }
static inline void
current_fast_clear(_PyRuntimeState *Py_UNUSED(runtime))
{ … }
#define tstate_verify_not_active(tstate) …
PyThreadState *
_PyThreadState_GetCurrent(void)
{ … }
static inline int
tstate_tss_initialized(Py_tss_t *key)
{ … }
static inline int
tstate_tss_init(Py_tss_t *key)
{ … }
static inline void
tstate_tss_fini(Py_tss_t *key)
{ … }
static inline PyThreadState *
tstate_tss_get(Py_tss_t *key)
{ … }
static inline int
tstate_tss_set(Py_tss_t *key, PyThreadState *tstate)
{ … }
static inline int
tstate_tss_clear(Py_tss_t *key)
{ … }
#ifdef HAVE_FORK
static PyStatus
tstate_tss_reinit(Py_tss_t *key)
{ … }
#endif
#define gilstate_tss_initialized(runtime) …
#define gilstate_tss_init(runtime) …
#define gilstate_tss_fini(runtime) …
#define gilstate_tss_get(runtime) …
#define _gilstate_tss_set(runtime, tstate) …
#define _gilstate_tss_clear(runtime) …
#define gilstate_tss_reinit(runtime) …
static inline void
gilstate_tss_set(_PyRuntimeState *runtime, PyThreadState *tstate)
{ … }
static inline void
gilstate_tss_clear(_PyRuntimeState *runtime)
{ … }
#ifndef NDEBUG
static inline int tstate_is_alive(PyThreadState *tstate);
static inline int
tstate_is_bound(PyThreadState *tstate)
{
return tstate->_status.bound && !tstate->_status.unbound;
}
#endif
static void bind_gilstate_tstate(PyThreadState *);
static void unbind_gilstate_tstate(PyThreadState *);
static void tstate_mimalloc_bind(PyThreadState *);
static void
bind_tstate(PyThreadState *tstate)
{ … }
static void
unbind_tstate(PyThreadState *tstate)
{ … }
static void
bind_gilstate_tstate(PyThreadState *tstate)
{ … }
static void
unbind_gilstate_tstate(PyThreadState *tstate)
{ … }
static int
holds_gil(PyThreadState *tstate)
{ … }
_Py_COMP_DIAG_PUSH
_Py_COMP_DIAG_IGNORE_DEPR_DECLS
static const _PyRuntimeState initial = …;
_Py_COMP_DIAG_POP
#define LOCKS_INIT(runtime) …
static void
init_runtime(_PyRuntimeState *runtime,
void *open_code_hook, void *open_code_userdata,
_Py_AuditHookEntry *audit_hook_head,
Py_ssize_t unicode_next_index)
{ … }
PyStatus
_PyRuntimeState_Init(_PyRuntimeState *runtime)
{ … }
void
_PyRuntimeState_Fini(_PyRuntimeState *runtime)
{ … }
#ifdef HAVE_FORK
PyStatus
_PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime)
{ … }
#endif
PyStatus
_PyInterpreterState_Enable(_PyRuntimeState *runtime)
{ … }
static PyInterpreterState *
alloc_interpreter(void)
{ … }
static void
free_interpreter(PyInterpreterState *interp)
{ … }
#ifndef NDEBUG
static inline int check_interpreter_whence(long);
#endif
static PyStatus
init_interpreter(PyInterpreterState *interp,
_PyRuntimeState *runtime, int64_t id,
PyInterpreterState *next,
long whence)
{ … }
PyStatus
_PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp)
{ … }
PyInterpreterState *
PyInterpreterState_New(void)
{ … }
static void
interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
{ … }
void
PyInterpreterState_Clear(PyInterpreterState *interp)
{ … }
void
_PyInterpreterState_Clear(PyThreadState *tstate)
{ … }
static inline void tstate_deactivate(PyThreadState *tstate);
static void tstate_set_detached(PyThreadState *tstate, int detached_state);
static void zapthreads(PyInterpreterState *interp);
void
PyInterpreterState_Delete(PyInterpreterState *interp)
{ … }
#ifdef HAVE_FORK
PyStatus
_PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime)
{ … }
#endif
static inline void
set_main_thread(PyInterpreterState *interp, PyThreadState *tstate)
{ … }
static inline PyThreadState *
get_main_thread(PyInterpreterState *interp)
{ … }
void
_PyErr_SetInterpreterAlreadyRunning(void)
{ … }
int
_PyInterpreterState_SetRunningMain(PyInterpreterState *interp)
{ … }
void
_PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp)
{ … }
int
_PyInterpreterState_IsRunningMain(PyInterpreterState *interp)
{ … }
int
_PyThreadState_IsRunningMain(PyThreadState *tstate)
{ … }
void
_PyInterpreterState_ReinitRunningMain(PyThreadState *tstate)
{ … }
int
_PyInterpreterState_IsReady(PyInterpreterState *interp)
{ … }
#ifndef NDEBUG
static inline int
check_interpreter_whence(long whence)
{
if(whence < 0) {
return -1;
}
if (whence > _PyInterpreterState_WHENCE_MAX) {
return -1;
}
return 0;
}
#endif
long
_PyInterpreterState_GetWhence(PyInterpreterState *interp)
{ … }
void
_PyInterpreterState_SetWhence(PyInterpreterState *interp, long whence)
{ … }
PyObject *
PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp)
{ … }
PyObject *
PyInterpreterState_GetDict(PyInterpreterState *interp)
{ … }
int64_t
_PyInterpreterState_ObjectToID(PyObject *idobj)
{ … }
int64_t
PyInterpreterState_GetID(PyInterpreterState *interp)
{ … }
PyObject *
_PyInterpreterState_GetIDObject(PyInterpreterState *interp)
{ … }
void
_PyInterpreterState_IDIncref(PyInterpreterState *interp)
{ … }
void
_PyInterpreterState_IDDecref(PyInterpreterState *interp)
{ … }
int
_PyInterpreterState_RequiresIDRef(PyInterpreterState *interp)
{ … }
void
_PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required)
{ … }
PyInterpreterState*
PyInterpreterState_Get(void)
{ … }
static PyInterpreterState *
interp_look_up_id(_PyRuntimeState *runtime, int64_t requested_id)
{ … }
PyInterpreterState *
_PyInterpreterState_LookUpID(int64_t requested_id)
{ … }
PyInterpreterState *
_PyInterpreterState_LookUpIDObject(PyObject *requested_id)
{ … }
#ifndef NDEBUG
static inline int
tstate_is_alive(PyThreadState *tstate)
{
return (tstate->_status.initialized &&
!tstate->_status.finalized &&
!tstate->_status.cleared &&
!tstate->_status.finalizing);
}
#endif
#define DATA_STACK_CHUNK_SIZE …
static _PyStackChunk*
allocate_chunk(int size_in_bytes, _PyStackChunk* previous)
{ … }
static void
reset_threadstate(_PyThreadStateImpl *tstate)
{ … }
static _PyThreadStateImpl *
alloc_threadstate(PyInterpreterState *interp)
{ … }
static void
free_threadstate(_PyThreadStateImpl *tstate)
{ … }
static void
init_threadstate(_PyThreadStateImpl *_tstate,
PyInterpreterState *interp, uint64_t id, int whence)
{ … }
static void
add_threadstate(PyInterpreterState *interp, PyThreadState *tstate,
PyThreadState *next)
{ … }
static PyThreadState *
new_threadstate(PyInterpreterState *interp, int whence)
{ … }
PyThreadState *
PyThreadState_New(PyInterpreterState *interp)
{ … }
PyThreadState *
_PyThreadState_NewBound(PyInterpreterState *interp, int whence)
{ … }
PyThreadState *
_PyThreadState_New(PyInterpreterState *interp, int whence)
{ … }
PyAPI_FUNC(PyThreadState*)
_PyThreadState_Prealloc(PyInterpreterState *interp)
{ … }
PyAPI_FUNC(void)
_PyThreadState_Init(PyThreadState *tstate)
{ … }
static void
clear_datastack(PyThreadState *tstate)
{ … }
void
PyThreadState_Clear(PyThreadState *tstate)
{ … }
static void
decrement_stoptheworld_countdown(struct _stoptheworld_state *stw);
static void
tstate_delete_common(PyThreadState *tstate, int release_gil)
{ … }
static void
zapthreads(PyInterpreterState *interp)
{ … }
void
PyThreadState_Delete(PyThreadState *tstate)
{ … }
void
_PyThreadState_DeleteCurrent(PyThreadState *tstate)
{ … }
void
PyThreadState_DeleteCurrent(void)
{ … }
PyThreadState *
_PyThreadState_RemoveExcept(PyThreadState *tstate)
{ … }
void
_PyThreadState_DeleteList(PyThreadState *list)
{ … }
PyObject *
_PyThreadState_GetDict(PyThreadState *tstate)
{ … }
PyObject *
PyThreadState_GetDict(void)
{ … }
PyInterpreterState *
PyThreadState_GetInterpreter(PyThreadState *tstate)
{ … }
PyFrameObject*
PyThreadState_GetFrame(PyThreadState *tstate)
{ … }
uint64_t
PyThreadState_GetID(PyThreadState *tstate)
{ … }
static inline void
tstate_activate(PyThreadState *tstate)
{ … }
static inline void
tstate_deactivate(PyThreadState *tstate)
{ … }
static int
tstate_try_attach(PyThreadState *tstate)
{ … }
static void
tstate_set_detached(PyThreadState *tstate, int detached_state)
{ … }
static void
tstate_wait_attach(PyThreadState *tstate)
{ … }
void
_PyThreadState_Attach(PyThreadState *tstate)
{ … }
static void
detach_thread(PyThreadState *tstate, int detached_state)
{ … }
void
_PyThreadState_Detach(PyThreadState *tstate)
{ … }
void
_PyThreadState_Suspend(PyThreadState *tstate)
{ … }
static void
decrement_stoptheworld_countdown(struct _stoptheworld_state *stw)
{ … }
#ifdef Py_GIL_DISABLED
static PyInterpreterState *
interp_for_stop_the_world(struct _stoptheworld_state *stw)
{
return (stw->is_global
? PyInterpreterState_Head()
: _Py_CONTAINER_OF(stw, PyInterpreterState, stoptheworld));
}
#define _Py_FOR_EACH_STW_INTERP …
static bool
park_detached_threads(struct _stoptheworld_state *stw)
{
int num_parked = 0;
_Py_FOR_EACH_STW_INTERP(stw, i) {
_Py_FOR_EACH_TSTATE_UNLOCKED(i, t) {
int state = _Py_atomic_load_int_relaxed(&t->state);
if (state == _Py_THREAD_DETACHED) {
if (_Py_atomic_compare_exchange_int(
&t->state, &state, _Py_THREAD_SUSPENDED)) {
num_parked++;
}
}
else if (state == _Py_THREAD_ATTACHED && t != stw->requester) {
_Py_set_eval_breaker_bit(t, _PY_EVAL_PLEASE_STOP_BIT);
}
}
}
stw->thread_countdown -= num_parked;
assert(stw->thread_countdown >= 0);
return num_parked > 0 && stw->thread_countdown == 0;
}
static void
stop_the_world(struct _stoptheworld_state *stw)
{
_PyRuntimeState *runtime = &_PyRuntime;
PyMutex_Lock(&stw->mutex);
if (stw->is_global) {
_PyRWMutex_Lock(&runtime->stoptheworld_mutex);
}
else {
_PyRWMutex_RLock(&runtime->stoptheworld_mutex);
}
HEAD_LOCK(runtime);
stw->requested = 1;
stw->thread_countdown = 0;
stw->stop_event = (PyEvent){0};
stw->requester = _PyThreadState_GET();
_Py_FOR_EACH_STW_INTERP(stw, i) {
_Py_FOR_EACH_TSTATE_UNLOCKED(i, t) {
if (t != stw->requester) {
stw->thread_countdown++;
}
}
}
if (stw->thread_countdown == 0) {
HEAD_UNLOCK(runtime);
stw->world_stopped = 1;
return;
}
for (;;) {
bool stopped_all_threads = park_detached_threads(stw);
HEAD_UNLOCK(runtime);
if (stopped_all_threads) {
break;
}
PyTime_t wait_ns = 1000*1000;
int detach = 0;
if (PyEvent_WaitTimed(&stw->stop_event, wait_ns, detach)) {
assert(stw->thread_countdown == 0);
break;
}
HEAD_LOCK(runtime);
}
stw->world_stopped = 1;
}
static void
start_the_world(struct _stoptheworld_state *stw)
{
_PyRuntimeState *runtime = &_PyRuntime;
assert(PyMutex_IsLocked(&stw->mutex));
HEAD_LOCK(runtime);
stw->requested = 0;
stw->world_stopped = 0;
_Py_FOR_EACH_STW_INTERP(stw, i) {
_Py_FOR_EACH_TSTATE_UNLOCKED(i, t) {
if (t != stw->requester) {
assert(_Py_atomic_load_int_relaxed(&t->state) ==
_Py_THREAD_SUSPENDED);
_Py_atomic_store_int(&t->state, _Py_THREAD_DETACHED);
_PyParkingLot_UnparkAll(&t->state);
}
}
}
stw->requester = NULL;
HEAD_UNLOCK(runtime);
if (stw->is_global) {
_PyRWMutex_Unlock(&runtime->stoptheworld_mutex);
}
else {
_PyRWMutex_RUnlock(&runtime->stoptheworld_mutex);
}
PyMutex_Unlock(&stw->mutex);
}
#endif
void
_PyEval_StopTheWorldAll(_PyRuntimeState *runtime)
{ … }
void
_PyEval_StartTheWorldAll(_PyRuntimeState *runtime)
{ … }
void
_PyEval_StopTheWorld(PyInterpreterState *interp)
{ … }
void
_PyEval_StartTheWorld(PyInterpreterState *interp)
{ … }
int
PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc)
{ … }
PyThreadState *
PyThreadState_GetUnchecked(void)
{ … }
PyThreadState *
PyThreadState_Get(void)
{ … }
PyThreadState *
_PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts)
{ … }
PyThreadState *
PyThreadState_Swap(PyThreadState *newts)
{ … }
void
_PyThreadState_Bind(PyThreadState *tstate)
{ … }
#if defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API)
uintptr_t
_Py_GetThreadLocal_Addr(void)
{
#ifdef HAVE_THREAD_LOCAL
return (uintptr_t)&_Py_tss_tstate;
#else
# error "no supported thread-local variable storage classifier"
#endif
}
#endif
PyInterpreterState *
PyInterpreterState_Head(void)
{ … }
PyInterpreterState *
PyInterpreterState_Main(void)
{ … }
PyInterpreterState *
PyInterpreterState_Next(PyInterpreterState *interp) { … }
PyThreadState *
PyInterpreterState_ThreadHead(PyInterpreterState *interp) { … }
PyThreadState *
PyThreadState_Next(PyThreadState *tstate) { … }
PyObject *
_PyThread_CurrentFrames(void)
{ … }
PyObject *
_PyThread_CurrentExceptions(void)
{ … }
PyStatus
_PyGILState_Init(PyInterpreterState *interp)
{ … }
void
_PyGILState_Fini(PyInterpreterState *interp)
{ … }
void
_PyGILState_SetTstate(PyThreadState *tstate)
{ … }
PyInterpreterState *
_PyGILState_GetInterpreterStateUnsafe(void)
{ … }
PyThreadState *
PyGILState_GetThisThreadState(void)
{ … }
int
PyGILState_Check(void)
{ … }
PyGILState_STATE
PyGILState_Ensure(void)
{ … }
void
PyGILState_Release(PyGILState_STATE oldstate)
{ … }
_PyFrameEvalFunction
_PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp)
{ … }
void
_PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp,
_PyFrameEvalFunction eval_frame)
{ … }
const PyConfig*
_PyInterpreterState_GetConfig(PyInterpreterState *interp)
{ … }
int
_PyInterpreterState_GetConfigCopy(PyConfig *config)
{ … }
const PyConfig*
_Py_GetConfig(void)
{ … }
int
_PyInterpreterState_HasFeature(PyInterpreterState *interp, unsigned long feature)
{ … }
#define MINIMUM_OVERHEAD …
static PyObject **
push_chunk(PyThreadState *tstate, int size)
{ … }
_PyInterpreterFrame *
_PyThreadState_PushFrame(PyThreadState *tstate, size_t size)
{ … }
void
_PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame * frame)
{ … }
#ifndef NDEBUG
int
_PyThreadState_CheckConsistency(PyThreadState *tstate)
{
assert(!_PyMem_IsPtrFreed(tstate));
assert(!_PyMem_IsPtrFreed(tstate->interp));
return 1;
}
#endif
int
_PyThreadState_MustExit(PyThreadState *tstate)
{ … }
static void
tstate_mimalloc_bind(PyThreadState *tstate)
{ … }
void
_PyThreadState_ClearMimallocHeaps(PyThreadState *tstate)
{ … }