#include "Python.h" #include "pycore_ceval.h" // _PyEval_SignalReceived() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // _Py_RunGC() #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystats.h" // _Py_PrintSpecializationStats() #include "pycore_pythread.h" // PyThread_hang_thread() /* Notes about the implementation: - The GIL is just a boolean variable (locked) whose access is protected by a mutex (gil_mutex), and whose changes are signalled by a condition variable (gil_cond). gil_mutex is taken for short periods of time, and therefore mostly uncontended. - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be able to release the GIL on demand by another thread. A volatile boolean variable (gil_drop_request) is used for that purpose, which is checked at every turn of the eval loop. That variable is set after a wait of `interval` microseconds on `gil_cond` has timed out. [Actually, another volatile boolean variable (eval_breaker) is used which ORs several conditions into one. Volatile booleans are sufficient as inter-thread signalling means since Python is run on cache-coherent architectures only.] - A thread wanting to take the GIL will first let pass a given amount of time (`interval` microseconds) before setting gil_drop_request. This encourages a defined switching period, but doesn't enforce it since opcodes can take an arbitrary time to execute. The `interval` value is available for the user to read and modify using the Python API `sys.{get,set}switchinterval()`. - When a thread releases the GIL and gil_drop_request is set, that thread ensures that another GIL-awaiting thread gets scheduled. It does so by waiting on a condition variable (switch_cond) until the value of last_holder is changed to something else than its own thread state pointer, indicating that another thread was able to take the GIL. This is meant to prohibit the latency-adverse behaviour on multi-core machines where one thread would speculatively release the GIL, but still run and end up being the first to re-acquire it, making the "timeslices" much longer than expected. (Note: this mechanism is enabled with FORCE_SWITCHING above) */ // Atomically copy the bits indicated by mask between two values. static inline void copy_eval_breaker_bits(uintptr_t *from, uintptr_t *to, uintptr_t mask) { … } // When attaching a thread, set the global instrumentation version and // _PY_CALLS_TO_DO_BIT from the current state of the interpreter. static inline void update_eval_breaker_for_thread(PyInterpreterState *interp, PyThreadState *tstate) { … } /* * Implementation of the Global Interpreter Lock (GIL). */ #include <stdlib.h> #include <errno.h> #include "condvar.h" #define MUTEX_INIT(mut) … #define MUTEX_FINI(mut) … #define MUTEX_LOCK(mut) … #define MUTEX_UNLOCK(mut) … #define COND_INIT(cond) … #define COND_FINI(cond) … #define COND_SIGNAL(cond) … #define COND_WAIT(cond, mut) … #define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) … \ #define DEFAULT_INTERVAL … static void _gil_initialize(struct _gil_runtime_state *gil) { … } static int gil_created(struct _gil_runtime_state *gil) { … } static void create_gil(struct _gil_runtime_state *gil) { … } static void destroy_gil(struct _gil_runtime_state *gil) { … } #ifdef HAVE_FORK static void recreate_gil(struct _gil_runtime_state *gil) { … } #endif static inline void drop_gil_impl(PyThreadState *tstate, struct _gil_runtime_state *gil) { … } static void drop_gil(PyInterpreterState *interp, PyThreadState *tstate, int final_release) { … } /* Take the GIL. The function saves errno at entry and restores its value at exit. It may hang rather than return if the interpreter has been finalized. tstate must be non-NULL. */ static void take_gil(PyThreadState *tstate) { … } void _PyEval_SetSwitchInterval(unsigned long microseconds) { … } unsigned long _PyEval_GetSwitchInterval(void) { … } int _PyEval_ThreadsInitialized(void) { … } // Function removed in the Python 3.13 API but kept in the stable ABI. PyAPI_FUNC(int) PyEval_ThreadsInitialized(void) { … } #ifndef NDEBUG static inline int current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate) { int holds_gil = tstate->_status.holds_gil; // holds_gil is the source of truth; check that last_holder and gil->locked // are consistent with it. int locked = _Py_atomic_load_int_relaxed(&gil->locked); int is_last_holder = ((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) == tstate; assert(!holds_gil || locked); assert(!holds_gil || is_last_holder); return holds_gil; } #endif static void init_shared_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) { … } static void init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) { … } void _PyEval_InitGIL(PyThreadState *tstate, int own_gil) { … } void _PyEval_FiniGIL(PyInterpreterState *interp) { … } void PyEval_InitThreads(void) { … } void _PyEval_Fini(void) { … } // Function removed in the Python 3.13 API but kept in the stable ABI. PyAPI_FUNC(void) PyEval_AcquireLock(void) { … } // Function removed in the Python 3.13 API but kept in the stable ABI. PyAPI_FUNC(void) PyEval_ReleaseLock(void) { … } void _PyEval_AcquireLock(PyThreadState *tstate) { … } void _PyEval_ReleaseLock(PyInterpreterState *interp, PyThreadState *tstate, int final_release) { … } void PyEval_AcquireThread(PyThreadState *tstate) { … } void PyEval_ReleaseThread(PyThreadState *tstate) { … } #ifdef HAVE_FORK /* This function is called from PyOS_AfterFork_Child to re-initialize the GIL and pending calls lock. */ PyStatus _PyEval_ReInitThreads(PyThreadState *tstate) { … } #endif PyThreadState * PyEval_SaveThread(void) { … } void PyEval_RestoreThread(PyThreadState *tstate) { … } void _PyEval_SignalReceived(void) { … } #ifndef Py_GIL_DISABLED static void signal_active_thread(PyInterpreterState *interp, uintptr_t bit) { … } #endif /* Mechanism whereby asynchronously executing callbacks (e.g. UNIX signal handlers or Mac I/O completion routines) can schedule calls to a function to be called synchronously. The synchronous function is called with one void* argument. It should return 0 for success or -1 for failure -- failure should be accompanied by an exception. If registry succeeds, the registry function returns 0; if it fails (e.g. due to too many pending calls) it returns -1 (without setting an exception condition). Note that because registry may occur from within signal handlers, or other asynchronous events, calling malloc() is unsafe! Any thread can schedule pending calls, but only the main thread will execute them. There is no facility to schedule calls to a particular thread, but that should be easy to change, should that ever be required. In that case, the static variables here should go into the python threadstate. */ /* Push one item onto the queue while holding the lock. */ static int _push_pending_call(struct _pending_calls *pending, _Py_pending_call_func func, void *arg, int flags) { … } static int _next_pending_call(struct _pending_calls *pending, int (**func)(void *), void **arg, int *flags) { … } /* Pop one item off the queue while holding the lock. */ static void _pop_pending_call(struct _pending_calls *pending, int (**func)(void *), void **arg, int *flags) { … } /* This implementation is thread-safe. It allows scheduling to be made from any thread, and even from an executing callback. */ _Py_add_pending_call_result _PyEval_AddPendingCall(PyInterpreterState *interp, _Py_pending_call_func func, void *arg, int flags) { … } int Py_AddPendingCall(_Py_pending_call_func func, void *arg) { … } static int handle_signals(PyThreadState *tstate) { … } static int _make_pending_calls(struct _pending_calls *pending, int32_t *p_npending) { … } static void signal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) { … } static void unsignal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) { … } static void clear_pending_handling_thread(struct _pending_calls *pending) { … } static int make_pending_calls(PyThreadState *tstate) { … } void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit) { … } void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit) { … } void _Py_FinishPendingCalls(PyThreadState *tstate) { … } int _PyEval_MakePendingCalls(PyThreadState *tstate) { … } /* Py_MakePendingCalls() is a simple wrapper for the sake of backward-compatibility. */ int Py_MakePendingCalls(void) { … } void _PyEval_InitState(PyInterpreterState *interp) { … } #ifdef Py_GIL_DISABLED int _PyEval_EnableGILTransient(PyThreadState *tstate) { const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); if (config->enable_gil != _PyConfig_GIL_DEFAULT) { return 0; } struct _gil_runtime_state *gil = tstate->interp->ceval.gil; int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); if (enabled == INT_MAX) { // The GIL is already enabled permanently. return 0; } if (enabled == INT_MAX - 1) { Py_FatalError("Too many transient requests to enable the GIL"); } if (enabled > 0) { // If enabled is nonzero, we know we hold the GIL. This means that no // other threads are attached, and nobody else can be concurrently // mutating it. _Py_atomic_store_int_relaxed(&gil->enabled, enabled + 1); return 0; } // Enabling the GIL changes what it means to be an "attached" thread. To // safely make this transition, we: // 1. Detach the current thread. // 2. Stop the world to detach (and suspend) all other threads. // 3. Enable the GIL, if nobody else did between our check above and when // our stop-the-world begins. // 4. Start the world. // 5. Attach the current thread. Other threads may attach and hold the GIL // before this thread, which is harmless. _PyThreadState_Detach(tstate); // This could be an interpreter-local stop-the-world in situations where we // know that this interpreter's GIL is not shared, and that it won't become // shared before the stop-the-world begins. For now, we always stop all // interpreters for simplicity. _PyEval_StopTheWorldAll(&_PyRuntime); enabled = _Py_atomic_load_int_relaxed(&gil->enabled); int this_thread_enabled = enabled == 0; _Py_atomic_store_int_relaxed(&gil->enabled, enabled + 1); _PyEval_StartTheWorldAll(&_PyRuntime); _PyThreadState_Attach(tstate); return this_thread_enabled; } int _PyEval_EnableGILPermanent(PyThreadState *tstate) { const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); if (config->enable_gil != _PyConfig_GIL_DEFAULT) { return 0; } struct _gil_runtime_state *gil = tstate->interp->ceval.gil; assert(current_thread_holds_gil(gil, tstate)); int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); if (enabled == INT_MAX) { return 0; } _Py_atomic_store_int_relaxed(&gil->enabled, INT_MAX); return 1; } int _PyEval_DisableGIL(PyThreadState *tstate) { const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); if (config->enable_gil != _PyConfig_GIL_DEFAULT) { return 0; } struct _gil_runtime_state *gil = tstate->interp->ceval.gil; assert(current_thread_holds_gil(gil, tstate)); int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); if (enabled == INT_MAX) { return 0; } assert(enabled >= 1); enabled--; // Disabling the GIL is much simpler than enabling it, since we know we are // the only attached thread. Other threads may start free-threading as soon // as this store is complete, if it sets gil->enabled to 0. _Py_atomic_store_int_relaxed(&gil->enabled, enabled); if (enabled == 0) { // We're attached, so we know the GIL will remain disabled until at // least the next time we detach, which must be after this function // returns. // // Drop the GIL, which will wake up any threads waiting in take_gil() // and let them resume execution without the GIL. drop_gil_impl(tstate, gil); // If another thread asked us to drop the GIL, they should be // free-threading by now. Remove any such request so we have a clean // slate if/when the GIL is enabled again. _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); return 1; } return 0; } #endif /* Do periodic things, like check for signals and async I/0. * We need to do reasonably frequently, but not too frequently. * All loops should include a check of the eval breaker. * We also check on return from any builtin function. * * ## More Details ### * * The eval loop (this function) normally executes the instructions * of a code object sequentially. However, the runtime supports a * number of out-of-band execution scenarios that may pause that * sequential execution long enough to do that out-of-band work * in the current thread using the current PyThreadState. * * The scenarios include: * * - cyclic garbage collection * - GIL drop requests * - "async" exceptions * - "pending calls" (some only in the main thread) * - signal handling (only in the main thread) * * When the need for one of the above is detected, the eval loop * pauses long enough to handle the detected case. Then, if doing * so didn't trigger an exception, the eval loop resumes executing * the sequential instructions. * * To make this work, the eval loop periodically checks if any * of the above needs to happen. The individual checks can be * expensive if computed each time, so a while back we switched * to using pre-computed, per-interpreter variables for the checks, * and later consolidated that to a single "eval breaker" variable * (now a PyInterpreterState field). * * For the longest time, the eval breaker check would happen * frequently, every 5 or so times through the loop, regardless * of what instruction ran last or what would run next. Then, in * early 2021 (gh-18334, commit 4958f5d), we switched to checking * the eval breaker less frequently, by hard-coding the check to * specific places in the eval loop (e.g. certain instructions). * The intent then was to check after returning from calls * and on the back edges of loops. * * In addition to being more efficient, that approach keeps * the eval loop from running arbitrary code between instructions * that don't handle that well. (See gh-74174.) * * Currently, the eval breaker check happens on back edges in * the control flow graph, which pretty much applies to all loops, * and most calls. * (See bytecodes.c for exact information.) * * One consequence of this approach is that it might not be obvious * how to force any specific thread to pick up the eval breaker, * or for any specific thread to not pick it up. Mostly this * involves judicious uses of locks and careful ordering of code, * while avoiding code that might trigger the eval breaker * until so desired. */ int _Py_HandlePending(PyThreadState *tstate) { … }
Codebase Browser
cpython