#ifndef Py_INTERNAL_CODE_H #define Py_INTERNAL_CODE_H #ifdef __cplusplus extern "C" { #endif #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif #include "pycore_stackref.h" // _PyStackRef #include "pycore_lock.h" // PyMutex #include "pycore_backoff.h" // _Py_BackoffCounter #include "pycore_tstate.h" // _PyThreadStateImpl /* Each instruction in a code object is a fixed-width value, * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG * opcode allows for larger values but the current limit is 3 uses * of EXTENDED_ARG (see Python/compile.c), for a maximum * 32-bit value. This aligns with the note in Python/compile.c * (compiler_addop_i_line) indicating that the max oparg value is * 2**32 - 1, rather than INT_MAX. */ _Py_CODEUNIT; #define _PyCode_CODE(CO) … #define _PyCode_NBYTES(CO) … /* These macros only remain defined for compatibility. */ #define _Py_OPCODE(word) … #define _Py_OPARG(word) … static inline _Py_CODEUNIT _py_make_codeunit(uint8_t opcode, uint8_t oparg) { … } static inline void _py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode) { … } #define _Py_MAKE_CODEUNIT(opcode, oparg) … #define _Py_SET_OPCODE(word, opcode) … // We hide some of the newer PyCodeObject fields behind macros. // This helps with backporting certain changes to 3.12. #define _PyCode_HAS_EXECUTORS(CODE) … #define _PyCode_HAS_INSTRUMENTATION(CODE) … struct _py_code_state { … }; extern PyStatus _PyCode_Init(PyInterpreterState *interp); extern void _PyCode_Fini(PyInterpreterState *interp); #define CODE_MAX_WATCHERS … /* PEP 659 * Specialization and quickening structs and helper functions */ // Inline caches. If you change the number of cache entries for an instruction, // you must *also* update the number of cache entries in Lib/opcode.py and bump // the magic number in Lib/importlib/_bootstrap_external.py! #define CACHE_ENTRIES(cache) … _PyLoadGlobalCache; #define INLINE_CACHE_ENTRIES_LOAD_GLOBAL … _PyBinaryOpCache; #define INLINE_CACHE_ENTRIES_BINARY_OP … _PyUnpackSequenceCache; #define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE … _PyCompareOpCache; #define INLINE_CACHE_ENTRIES_COMPARE_OP … _PyBinarySubscrCache; #define INLINE_CACHE_ENTRIES_BINARY_SUBSCR … _PySuperAttrCache; #define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR … _PyAttrCache; _PyLoadMethodCache; // MUST be the max(_PyAttrCache, _PyLoadMethodCache) #define INLINE_CACHE_ENTRIES_LOAD_ATTR … #define INLINE_CACHE_ENTRIES_STORE_ATTR … _PyCallCache; #define INLINE_CACHE_ENTRIES_CALL … #define INLINE_CACHE_ENTRIES_CALL_KW … _PyStoreSubscrCache; #define INLINE_CACHE_ENTRIES_STORE_SUBSCR … _PyForIterCache; #define INLINE_CACHE_ENTRIES_FOR_ITER … _PySendCache; #define INLINE_CACHE_ENTRIES_SEND … _PyToBoolCache; #define INLINE_CACHE_ENTRIES_TO_BOOL … _PyContainsOpCache; #define INLINE_CACHE_ENTRIES_CONTAINS_OP … // Borrowed references to common callables: struct callable_cache { … }; /* "Locals plus" for a code object is the set of locals + cell vars + * free vars. This relates to variable names as well as offsets into * the "fast locals" storage array of execution frames. The compiler * builds the list of names, their offsets, and the corresponding * kind of local. * * Those kinds represent the source of the initial value and the * variable's scope (as related to closures). A "local" is an * argument or other variable defined in the current scope. A "free" * variable is one that is defined in an outer scope and comes from * the function's closure. A "cell" variable is a local that escapes * into an inner function as part of a closure, and thus must be * wrapped in a cell. Any "local" can also be a "cell", but the * "free" kind is mutually exclusive with both. */ // Note that these all fit within a byte, as do combinations. // Later, we will use the smaller numbers to differentiate the different // kinds of locals (e.g. pos-only arg, varkwargs, local-only). #define CO_FAST_HIDDEN … #define CO_FAST_LOCAL … #define CO_FAST_CELL … #define CO_FAST_FREE … _PyLocals_Kind; static inline _PyLocals_Kind _PyLocals_GetKind(PyObject *kinds, int i) { … } static inline void _PyLocals_SetKind(PyObject *kinds, int i, _PyLocals_Kind kind) { … } struct _PyCodeConstructor { … }; // Using an "arguments struct" like this is helpful for maintainability // in a case such as this with many parameters. It does bear a risk: // if the struct changes and callers are not updated properly then the // compiler will not catch problems (like a missing argument). This can // cause hard-to-debug problems. The risk is mitigated by the use of // check_code() in codeobject.c. However, we may decide to switch // back to a regular function signature. Regardless, this approach // wouldn't be appropriate if this weren't a strictly internal API. // (See the comments in https://github.com/python/cpython/pull/26258.) extern int _PyCode_Validate(struct _PyCodeConstructor *); extern PyCodeObject* _PyCode_New(struct _PyCodeConstructor *); /* Private API */ /* Getters for internal PyCodeObject data. */ extern PyObject* _PyCode_GetVarnames(PyCodeObject *); extern PyObject* _PyCode_GetCellvars(PyCodeObject *); extern PyObject* _PyCode_GetFreevars(PyCodeObject *); extern PyObject* _PyCode_GetCode(PyCodeObject *); /** API for initializing the line number tables. */ extern int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds); /** Out of process API for initializing the location table. */ extern void _PyLineTable_InitAddressRange( const char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range); /** API for traversing the line number table. */ extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range); extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); /** API for executors */ extern void _PyCode_Clear_Executors(PyCodeObject *code); #ifdef Py_GIL_DISABLED // gh-115999 tracks progress on addressing this. #define ENABLE_SPECIALIZATION … // Use this to enable specialization families once they are thread-safe. All // uses will be replaced with ENABLE_SPECIALIZATION once all families are // thread-safe. #define ENABLE_SPECIALIZATION_FT … #else #define ENABLE_SPECIALIZATION … #define ENABLE_SPECIALIZATION_FT … #endif /* Specialization functions */ extern void _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls, _Py_CODEUNIT *instr, int load_method); extern void _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); extern void _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name); extern void _Py_Specialize_BinarySubscr(_PyStackRef sub, _PyStackRef container, _Py_CODEUNIT *instr); extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); extern void _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg, _PyStackRef *locals); extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); #ifdef Py_STATS #include "pycore_bitutils.h" // _Py_bit_length #define STAT_INC … #define STAT_DEC … #define OPCODE_EXE_INC … #define CALL_STAT_INC … #define OBJECT_STAT_INC … #define OBJECT_STAT_INC_COND … #define EVAL_CALL_STAT_INC … #define EVAL_CALL_STAT_INC_IF_FUNCTION … #define GC_STAT_ADD … #define OPT_STAT_INC … #define UOP_STAT_INC … #define UOP_PAIR_INC … #define OPT_UNSUPPORTED_OPCODE … #define OPT_ERROR_IN_OPCODE … #define OPT_HIST … #define RARE_EVENT_STAT_INC … #define OPCODE_DEFERRED_INC … // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #else #define STAT_INC(opname, name) … #define STAT_DEC(opname, name) … #define OPCODE_EXE_INC(opname) … #define CALL_STAT_INC(name) … #define OBJECT_STAT_INC(name) … #define OBJECT_STAT_INC_COND(name, cond) … #define EVAL_CALL_STAT_INC(name) … #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) … #define GC_STAT_ADD(gen, name, n) … #define OPT_STAT_INC(name) … #define UOP_STAT_INC(opname, name) … #define UOP_PAIR_INC(uopcode, lastuop) … #define OPT_UNSUPPORTED_OPCODE(opname) … #define OPT_ERROR_IN_OPCODE(opname) … #define OPT_HIST(length, name) … #define RARE_EVENT_STAT_INC(name) … #define OPCODE_DEFERRED_INC(opname) … #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. // Great care should be taken to ensure that these functions remain correct and // performant! They should compile to just "move" instructions on all supported // compilers and platforms. // We use memcpy to let the C compiler handle unaligned accesses and endianness // issues for us. It also seems to produce better code than manual copying for // most compilers (see https://blog.regehr.org/archives/959 for more info). static inline void write_u32(uint16_t *p, uint32_t val) { … } static inline void write_u64(uint16_t *p, uint64_t val) { … } static inline void write_obj(uint16_t *p, PyObject *val) { … } static inline uint16_t read_u16(uint16_t *p) { … } static inline uint32_t read_u32(uint16_t *p) { … } static inline uint64_t read_u64(uint16_t *p) { … } static inline PyObject * read_obj(uint16_t *p) { … } /* See Objects/exception_handling_notes.txt for details. */ static inline unsigned char * parse_varint(unsigned char *p, int *result) { … } static inline int write_varint(uint8_t *ptr, unsigned int val) { … } static inline int write_signed_varint(uint8_t *ptr, int val) { … } static inline int write_location_entry_start(uint8_t *ptr, int code, int length) { … } /** Counters * The first 16-bit value in each inline cache is a counter. * * When counting executions until the next specialization attempt, * exponential backoff is used to reduce the number of specialization failures. * See pycore_backoff.h for more details. * On a specialization failure, the backoff counter is restarted. */ #include "pycore_backoff.h" // A value of 1 means that we attempt to specialize the *second* time each // instruction is executed. Executing twice is a much better indicator of // "hotness" than executing once, but additional warmup delays only prevent // specialization. Most types stabilize by the second execution, too: #define ADAPTIVE_WARMUP_VALUE … #define ADAPTIVE_WARMUP_BACKOFF … // A value of 52 means that we attempt to re-specialize after 53 misses (a prime // number, useful for avoiding artifacts if every nth value is a different type // or something). Setting the backoff to 0 means that the counter is reset to // the same state as a warming-up instruction (value == 1, backoff == 1) after // deoptimization. This isn't strictly necessary, but it is bit easier to reason // about when thinking about the opcode transitions as a state machine: #define ADAPTIVE_COOLDOWN_VALUE … #define ADAPTIVE_COOLDOWN_BACKOFF … // Can't assert this in pycore_backoff.h because of header order dependencies #if SIDE_EXIT_INITIAL_VALUE <= ADAPTIVE_COOLDOWN_VALUE # error "Cold exit value should be larger than adaptive cooldown value" #endif static inline _Py_BackoffCounter adaptive_counter_bits(uint16_t value, uint16_t backoff) { … } static inline _Py_BackoffCounter adaptive_counter_warmup(void) { … } static inline _Py_BackoffCounter adaptive_counter_cooldown(void) { … } static inline _Py_BackoffCounter adaptive_counter_backoff(_Py_BackoffCounter counter) { … } /* Comparison bit masks. */ /* Note this evaluates its arguments twice each */ #define COMPARISON_BIT(x, y) … /* * The following bits are chosen so that the value of * COMPARSION_BIT(left, right) * masked by the values below will be non-zero if the * comparison is true, and zero if it is false */ /* This is for values that are unordered, ie. NaN, not types that are unordered, e.g. sets */ #define COMPARISON_UNORDERED … #define COMPARISON_LESS_THAN … #define COMPARISON_GREATER_THAN … #define COMPARISON_EQUALS … #define COMPARISON_NOT_EQUALS … extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); struct _PyCode8 _PyCode_DEF(8); PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup; #ifdef Py_GIL_DISABLED static inline _PyCodeArray * _PyCode_GetTLBCArray(PyCodeObject *co) { return _Py_STATIC_CAST(_PyCodeArray *, _Py_atomic_load_ptr_acquire(&co->co_tlbc)); } // Return a pointer to the thread-local bytecode for the current thread, if it // exists. static inline _Py_CODEUNIT * _PyCode_GetTLBCFast(PyThreadState *tstate, PyCodeObject *co) { _PyCodeArray *code = _PyCode_GetTLBCArray(co); int32_t idx = ((_PyThreadStateImpl*) tstate)->tlbc_index; if (idx < code->size && code->entries[idx] != NULL) { return (_Py_CODEUNIT *) code->entries[idx]; } return NULL; } // Return a pointer to the thread-local bytecode for the current thread, // creating it if necessary. extern _Py_CODEUNIT *_PyCode_GetTLBC(PyCodeObject *co); // Reserve an index for the current thread into thread-local bytecode // arrays // // Returns the reserved index or -1 on error. extern int32_t _Py_ReserveTLBCIndex(PyInterpreterState *interp); // Release the current thread's index into thread-local bytecode arrays extern void _Py_ClearTLBCIndex(_PyThreadStateImpl *tstate); // Free all TLBC copies not associated with live threads. // // Returns 0 on success or -1 on error. extern int _Py_ClearUnusedTLBC(PyInterpreterState *interp); #endif #ifdef __cplusplus } #endif #endif /* !Py_INTERNAL_CODE_H */
Codebase Browser
cpython