cpython/Python/ceval.c

/* Execute compiled code */

#define _PY_INTERPRETER

#include "Python.h"
#include "pycore_abstract.h"      // _PyIndex_Check()
#include "pycore_backoff.h"
#include "pycore_call.h"          // _PyObject_CallNoArgs()
#include "pycore_cell.h"          // PyCell_GetRef()
#include "pycore_ceval.h"
#include "pycore_code.h"
#include "pycore_emscripten_signal.h"  // _Py_CHECK_EMSCRIPTEN_SIGNALS
#include "pycore_function.h"
#include "pycore_instruments.h"
#include "pycore_intrinsics.h"
#include "pycore_jit.h"
#include "pycore_long.h"          // _PyLong_GetZero()
#include "pycore_moduleobject.h"  // PyModuleObject
#include "pycore_object.h"        // _PyObject_GC_TRACK()
#include "pycore_opcode_metadata.h" // EXTRA_CASES
#include "pycore_optimizer.h"     // _PyUOpExecutor_Type
#include "pycore_opcode_utils.h"  // MAKE_FUNCTION_*
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_*
#include "pycore_pyerrors.h"      // _PyErr_GetRaisedException()
#include "pycore_pystate.h"       // _PyInterpreterState_GET()
#include "pycore_range.h"         // _PyRangeIterObject
#include "pycore_setobject.h"     // _PySet_Update()
#include "pycore_sliceobject.h"   // _PyBuildSlice_ConsumeRefs
#include "pycore_sysmodule.h"     // _PySys_Audit()
#include "pycore_tuple.h"         // _PyTuple_ITEMS()
#include "pycore_typeobject.h"    // _PySuper_Lookup()
#include "pycore_uop_ids.h"       // Uops
#include "pycore_pyerrors.h"

#include "pycore_dict.h"
#include "dictobject.h"
#include "pycore_frame.h"
#include "frameobject.h"          // _PyInterpreterFrame_GetLine
#include "opcode.h"
#include "pydtrace.h"
#include "setobject.h"
#include "pycore_stackref.h"

#include <stdbool.h>              // bool

#ifdef Py_DEBUG
   /* For debugging the interpreter: */
#define LLTRACE …
#endif

#if !defined(Py_BUILD_CORE)
#  error "ceval.c must be build with Py_BUILD_CORE define for best performance"
#endif

#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS)
// GH-89279: The MSVC compiler does not inline these static inline functions
// in PGO build in _PyEval_EvalFrameDefault(), because this function is over
// the limit of PGO, and that limit cannot be configured.
// Define them as macros to make sure that they are always inlined by the
// preprocessor.

#undef Py_IS_TYPE
#define Py_IS_TYPE(ob, type) …

#undef Py_XDECREF
#define Py_XDECREF(arg) …

#ifndef Py_GIL_DISABLED

#undef Py_DECREF
#define Py_DECREF(arg) …

#undef _Py_DECREF_SPECIALIZED
#define _Py_DECREF_SPECIALIZED(arg, dealloc) …

#else // Py_GIL_DISABLED

#undef Py_DECREF
#define Py_DECREF …

#undef _Py_DECREF_SPECIALIZED
#define _Py_DECREF_SPECIALIZED …

#endif
#endif


#ifdef LLTRACE
static void
dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
{
    _PyStackRef *stack_base = _PyFrame_Stackbase(frame);
    PyObject *exc = PyErr_GetRaisedException();
    printf("    stack=[");
    for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) {
        if (ptr != stack_base) {
            printf(", ");
        }
        PyObject *obj = PyStackRef_AsPyObjectBorrow(*ptr);
        if (obj == NULL) {
            printf("<nil>");
            continue;
        }
        if (
            obj == Py_None
            || PyBool_Check(obj)
            || PyLong_CheckExact(obj)
            || PyFloat_CheckExact(obj)
            || PyUnicode_CheckExact(obj)
        ) {
            if (PyObject_Print(obj, stdout, 0) == 0) {
                continue;
            }
            PyErr_Clear();
        }
        // Don't call __repr__(), it might recurse into the interpreter.
        printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)(ptr->bits));
    }
    printf("]\n");
    fflush(stdout);
    PyErr_SetRaisedException(exc);
}

static void
lltrace_instruction(_PyInterpreterFrame *frame,
                    _PyStackRef *stack_pointer,
                    _Py_CODEUNIT *next_instr,
                    int opcode,
                    int oparg)
{
    if (frame->owner == FRAME_OWNED_BY_CSTACK) {
        return;
    }
    dump_stack(frame, stack_pointer);
    const char *opname = _PyOpcode_OpName[opcode];
    assert(opname != NULL);
    int offset = (int)(next_instr - _PyCode_CODE(_PyFrame_GetCode(frame)));
    if (OPCODE_HAS_ARG((int)_PyOpcode_Deopt[opcode])) {
        printf("%d: %s %d\n", offset * 2, opname, oparg);
    }
    else {
        printf("%d: %s\n", offset * 2, opname);
    }
    fflush(stdout);
}
static void
lltrace_resume_frame(_PyInterpreterFrame *frame)
{
    PyObject *fobj = frame->f_funcobj;
    if (!PyCode_Check(frame->f_executable) ||
        fobj == NULL ||
        !PyFunction_Check(fobj)
    ) {
        printf("\nResuming frame.\n");
        return;
    }
    PyFunctionObject *f = (PyFunctionObject *)fobj;
    PyObject *exc = PyErr_GetRaisedException();
    PyObject *name = f->func_qualname;
    if (name == NULL) {
        name = f->func_name;
    }
    printf("\nResuming frame");
    if (name) {
        printf(" for ");
        if (PyObject_Print(name, stdout, 0) < 0) {
            PyErr_Clear();
        }
    }
    if (f->func_module) {
        printf(" in module ");
        if (PyObject_Print(f->func_module, stdout, 0) < 0) {
            PyErr_Clear();
        }
    }
    printf("\n");
    fflush(stdout);
    PyErr_SetRaisedException(exc);
}

static int
maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, _PyInterpreterFrame *skip_frame, PyObject *globals)
{
    if (globals == NULL) {
        return 0;
    }
    if (frame == skip_frame) {
        return 0;
    }
    int r = PyDict_Contains(globals, &_Py_ID(__lltrace__));
    if (r < 0) {
        return -1;
    }
    int lltrace = r * 5;  // Levels 1-4 only trace uops
    if (!lltrace) {
        // Can also be controlled by environment variable
        char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
        if (python_lltrace != NULL && *python_lltrace >= '0') {
            lltrace = *python_lltrace - '0';  // TODO: Parse an int and all that
        }
    }
    if (lltrace >= 5) {
        lltrace_resume_frame(frame);
    }
    return lltrace;
}

#endif

static void monitor_reraise(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr);
static int monitor_stop_iteration(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr,
                 PyObject *value);
static void monitor_unwind(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr);
static int monitor_handled(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr, PyObject *exc);
static void monitor_throw(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr);

static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg);
static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);
static  _PyInterpreterFrame *
_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func,
    PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous);

#ifdef HAVE_ERRNO_H
#include <errno.h>
#endif

int
Py_GetRecursionLimit(void)
{ … }

void
Py_SetRecursionLimit(int new_limit)
{ … }

/* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall()
   if the recursion_depth reaches recursion_limit. */
int
_Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
{ … }


const binaryfunc _PyEval_BinaryOps[] = …;

const conversion_func _PyEval_ConversionFuncs[4] = …;

const _Py_SpecialMethod _Py_SpecialMethods[] = …;


// PEP 634: Structural Pattern Matching


// Return a tuple of values corresponding to keys, with error checks for
// duplicate/missing keys.
PyObject *
_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys)
{ … }

// Extract a named attribute from the subject, with additional bookkeeping to
// raise TypeErrors for repeated lookups. On failure, return NULL (with no
// error set). Use _PyErr_Occurred(tstate) to disambiguate.
static PyObject *
match_class_attr(PyThreadState *tstate, PyObject *subject, PyObject *type,
                 PyObject *name, PyObject *seen)
{ … }

// On success (match), return a tuple of extracted attributes. On failure (no
// match), return NULL. Use _PyErr_Occurred(tstate) to disambiguate.
PyObject*
_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type,
                   Py_ssize_t nargs, PyObject *kwargs)
{ … }


static int do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause);

PyObject *
PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals)
{ … }


/* Interpreter main loop */

PyObject *
PyEval_EvalFrame(PyFrameObject *f)
{ … }

PyObject *
PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
{ … }

#include "ceval_macros.h"

int _Py_CheckRecursiveCallPy(
    PyThreadState *tstate)
{ … }

static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = …;

#ifdef Py_DEBUG
extern void _PyUOpPrint(const _PyUOpInstruction *uop);
#endif


/* Disable unused label warnings.  They are handy for debugging, even
   if computed gotos aren't used. */

/* TBD - what about other compilers? */
#if defined(__GNUC__)
#  pragma GCC diagnostic push
#  pragma GCC diagnostic ignored "-Wunused-label"
#elif defined(_MSC_VER) /* MS_WINDOWS */
#  pragma warning(push)
#  pragma warning(disable:4102)
#endif


PyObject **
_PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch)
{ … }

void
_PyObjectArray_Free(PyObject **array, PyObject **scratch)
{ … }


/* _PyEval_EvalFrameDefault() is a *big* function,
 * so consume 3 units of C stack */
#define PY_EVAL_C_STACK_UNITS …

PyObject* _Py_HOT_FUNCTION
_PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
{ … }

#if defined(__GNUC__)
#  pragma GCC diagnostic pop
#elif defined(_MSC_VER) /* MS_WINDOWS */
#  pragma warning(pop)
#endif

static void
format_missing(PyThreadState *tstate, const char *kind,
               PyCodeObject *co, PyObject *names, PyObject *qualname)
{ … }

static void
missing_arguments(PyThreadState *tstate, PyCodeObject *co,
                  Py_ssize_t missing, Py_ssize_t defcount,
                  _PyStackRef *localsplus, PyObject *qualname)
{ … }

static void
too_many_positional(PyThreadState *tstate, PyCodeObject *co,
                    Py_ssize_t given, PyObject *defaults,
                    _PyStackRef *localsplus, PyObject *qualname)
{ … }

static int
positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co,
                                  Py_ssize_t kwcount, PyObject* kwnames,
                                  PyObject *qualname)
{ … }


static inline unsigned char *
scan_back_to_entry_start(unsigned char *p) { … }

static inline unsigned char *
skip_to_next_entry(unsigned char *p, unsigned char *end) { … }


#define MAX_LINEAR_SEARCH …

static int
get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti)
{ … }

static int
initialize_locals(PyThreadState *tstate, PyFunctionObject *func,
    _PyStackRef *localsplus, _PyStackRef const *args,
    Py_ssize_t argcount, PyObject *kwnames)
{ … }

static void
clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame)
{ … }

static void
clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame)
{ … }

void
_PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame)
{ … }

/* Consumes references to func, locals and all the args */
_PyInterpreterFrame *
_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
                        PyObject *locals, _PyStackRef const* args,
                        size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous)
{ … }

static _PyInterpreterFrame *
_PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func,
                        PyObject *locals, PyObject *const* args,
                        size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous)
{ … }

/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict.
   Steals references to func, callargs and kwargs.
*/
static _PyInterpreterFrame *
_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func,
    PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous)
{ … }

PyObject *
_PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func,
               PyObject *locals,
               PyObject* const* args, size_t argcount,
               PyObject *kwnames)
{ … }

/* Legacy API */
PyObject *
PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals,
                  PyObject *const *args, int argcount,
                  PyObject *const *kws, int kwcount,
                  PyObject *const *defs, int defcount,
                  PyObject *kwdefs, PyObject *closure)
{ … }


/* Logic for the raise statement (too complicated for inlining).
   This *consumes* a reference count to each of its arguments. */
static int
do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause)
{ … }

/* Logic for matching an exception in an except* clause (too
   complicated for inlining).
*/

int
_PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type,
                            PyObject **match, PyObject **rest)
{ … }

/* Iterate v argcnt times and store the results on the stack (via decreasing
   sp).  Return 1 for success, 0 if error.

   If argcntafter == -1, do a simple unpack. If it is >= 0, do an unpack
   with a variable target.
*/

int
_PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v_stackref,
                       int argcnt, int argcntafter, _PyStackRef *sp)
{ … }

static int
do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame,
               _Py_CODEUNIT *instr, int event)
{ … }

static inline bool
no_tools_for_global_event(PyThreadState *tstate, int event)
{ … }

static inline bool
no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event)
{ … }

void
_PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame,
              _Py_CODEUNIT *instr)
{ … }

static void
monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame,
              _Py_CODEUNIT *instr)
{ … }

static int
monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame,
                       _Py_CODEUNIT *instr, PyObject *value)
{ … }

static void
monitor_unwind(PyThreadState *tstate,
               _PyInterpreterFrame *frame,
               _Py_CODEUNIT *instr)
{ … }


static int
monitor_handled(PyThreadState *tstate,
                _PyInterpreterFrame *frame,
                _Py_CODEUNIT *instr, PyObject *exc)
{ … }

static void
monitor_throw(PyThreadState *tstate,
              _PyInterpreterFrame *frame,
              _Py_CODEUNIT *instr)
{ … }

void
PyThreadState_EnterTracing(PyThreadState *tstate)
{ … }

void
PyThreadState_LeaveTracing(PyThreadState *tstate)
{ … }


PyObject*
_PyEval_CallTracing(PyObject *func, PyObject *args)
{ … }

void
PyEval_SetProfile(Py_tracefunc func, PyObject *arg)
{ … }

void
PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *arg)
{ … }

void
PyEval_SetTrace(Py_tracefunc func, PyObject *arg)
{ … }

void
PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *arg)
{ … }

int
_PyEval_SetCoroutineOriginTrackingDepth(int depth)
{ … }


int
_PyEval_GetCoroutineOriginTrackingDepth(void)
{ … }

int
_PyEval_SetAsyncGenFirstiter(PyObject *firstiter)
{ … }

PyObject *
_PyEval_GetAsyncGenFirstiter(void)
{ … }

int
_PyEval_SetAsyncGenFinalizer(PyObject *finalizer)
{ … }

PyObject *
_PyEval_GetAsyncGenFinalizer(void)
{ … }

_PyInterpreterFrame *
_PyEval_GetFrame(void)
{ … }

PyFrameObject *
PyEval_GetFrame(void)
{ … }

PyObject *
_PyEval_GetBuiltins(PyThreadState *tstate)
{ … }

PyObject *
PyEval_GetBuiltins(void)
{ … }

/* Convenience function to get a builtin from its name */
PyObject *
_PyEval_GetBuiltin(PyObject *name)
{ … }

PyObject *
_PyEval_GetBuiltinId(_Py_Identifier *name)
{ … }

PyObject *
PyEval_GetLocals(void)
{ … }

PyObject *
_PyEval_GetFrameLocals(void)
{ … }

PyObject *
PyEval_GetGlobals(void)
{ … }

PyObject*
PyEval_GetFrameLocals(void)
{ … }

PyObject* PyEval_GetFrameGlobals(void)
{ … }

PyObject* PyEval_GetFrameBuiltins(void)
{ … }

int
PyEval_MergeCompilerFlags(PyCompilerFlags *cf)
{ … }


const char *
PyEval_GetFuncName(PyObject *func)
{ … }

const char *
PyEval_GetFuncDesc(PyObject *func)
{ … }

/* Extract a slice index from a PyLong or an object with the
   nb_index slot defined, and store in *pi.
   Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX,
   and silently boost values less than PY_SSIZE_T_MIN to PY_SSIZE_T_MIN.
   Return 0 on error, 1 on success.
*/
int
_PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi)
{ … }

int
_PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi)
{ … }

PyObject *
_PyEval_ImportName(PyThreadState *tstate, _PyInterpreterFrame *frame,
            PyObject *name, PyObject *fromlist, PyObject *level)
{ … }

PyObject *
_PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name)
{ … }

#define CANNOT_CATCH_MSG …

#define CANNOT_EXCEPT_STAR_EG …

int
_PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right)
{ … }

int
_PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right)
{ … }

static int
check_args_iterable(PyThreadState *tstate, PyObject *func, PyObject *args)
{ … }

void
_PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs)
{ … }

void
_PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc,
                          const char *format_str, PyObject *obj)
{ … }

void
_PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg)
{ … }

void
_PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg)
{ … }


Py_ssize_t
PyUnstable_Eval_RequestCodeExtraIndex(freefunc free)
{ … }

/* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions
   for the limited API. */

int Py_EnterRecursiveCall(const char *where)
{ … }

void Py_LeaveRecursiveCall(void)
{ … }

PyObject *
_PyEval_GetANext(PyObject *aiter)
{ … }

PyObject *
_PyEval_LoadGlobal(PyObject *globals, PyObject *builtins, PyObject *name)
{ … }

PyObject *
_PyEval_GetAwaitable(PyObject *iterable, int oparg)
{ … }

PyObject *
_PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name)
{ … }