#include "Python.h" #include "opcode.h" #include "pycore_code.h" #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // DICT_KEYS_UNICODE #include "pycore_function.h" // _PyFunction_GetVersionForCurrentState() #include "pycore_long.h" // _PyLong_IsNonNegativeCompact() #include "pycore_moduleobject.h" #include "pycore_object.h" #include "pycore_opcode_metadata.h" // _PyOpcode_Caches #include "pycore_uop_metadata.h" // _PyOpcode_uop_name #include "pycore_uop_ids.h" // MAX_UOP_ID #include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() #include "pycore_runtime.h" // _Py_ID() #include <stdlib.h> // rand() extern const char *_PyUOpName(int index); /* For guidance on adding or extending families of instructions see * ./adaptive.md */ #ifdef Py_STATS GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 }; static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats }; PyStats *_Py_stats = NULL; #if PYSTATS_MAX_UOP_ID < MAX_UOP_ID #error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_ID" #endif #define ADD_STAT_TO_DICT … static PyObject* stats_to_dict(SpecializationStats *stats) { PyObject *res = PyDict_New(); if (res == NULL) { return NULL; } ADD_STAT_TO_DICT(res, success); ADD_STAT_TO_DICT(res, failure); ADD_STAT_TO_DICT(res, hit); ADD_STAT_TO_DICT(res, deferred); ADD_STAT_TO_DICT(res, miss); ADD_STAT_TO_DICT(res, deopt); PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS); if (failure_kinds == NULL) { Py_DECREF(res); return NULL; } for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) { PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]); if (stat == NULL) { Py_DECREF(res); Py_DECREF(failure_kinds); return NULL; } PyTuple_SET_ITEM(failure_kinds, i, stat); } if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) { Py_DECREF(res); Py_DECREF(failure_kinds); return NULL; } Py_DECREF(failure_kinds); return res; } #undef ADD_STAT_TO_DICT static int add_stat_dict( PyObject *res, int opcode, const char *name) { SpecializationStats *stats = &_Py_stats_struct.opcode_stats[opcode].specialization; PyObject *d = stats_to_dict(stats); if (d == NULL) { return -1; } int err = PyDict_SetItemString(res, name, d); Py_DECREF(d); return err; } PyObject* _Py_GetSpecializationStats(void) { PyObject *stats = PyDict_New(); if (stats == NULL) { return NULL; } int err = 0; err += add_stat_dict(stats, CONTAINS_OP, "contains_op"); err += add_stat_dict(stats, LOAD_SUPER_ATTR, "load_super_attr"); err += add_stat_dict(stats, LOAD_ATTR, "load_attr"); err += add_stat_dict(stats, LOAD_GLOBAL, "load_global"); err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr"); err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr"); err += add_stat_dict(stats, STORE_ATTR, "store_attr"); err += add_stat_dict(stats, CALL, "call"); err += add_stat_dict(stats, CALL_KW, "call_kw"); err += add_stat_dict(stats, BINARY_OP, "binary_op"); err += add_stat_dict(stats, COMPARE_OP, "compare_op"); err += add_stat_dict(stats, UNPACK_SEQUENCE, "unpack_sequence"); err += add_stat_dict(stats, FOR_ITER, "for_iter"); err += add_stat_dict(stats, TO_BOOL, "to_bool"); err += add_stat_dict(stats, SEND, "send"); if (err < 0) { Py_DECREF(stats); return NULL; } return stats; } #define PRINT_STAT … static void print_spec_stats(FILE *out, OpcodeStats *stats) { /* Mark some opcodes as specializable for stats, * even though we don't specialize them yet. */ fprintf(out, "opcode[BINARY_SLICE].specializable : 1\n"); fprintf(out, "opcode[STORE_SLICE].specializable : 1\n"); for (int i = 0; i < 256; i++) { if (_PyOpcode_Caches[i]) { /* Ignore jumps as they cannot be specialized */ switch (i) { case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: case POP_JUMP_IF_NONE: case POP_JUMP_IF_NOT_NONE: case JUMP_BACKWARD: break; default: fprintf(out, "opcode[%s].specializable : 1\n", _PyOpcode_OpName[i]); } } PRINT_STAT(i, specialization.success); PRINT_STAT(i, specialization.failure); PRINT_STAT(i, specialization.hit); PRINT_STAT(i, specialization.deferred); PRINT_STAT(i, specialization.miss); PRINT_STAT(i, specialization.deopt); PRINT_STAT(i, execution_count); for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) { uint64_t val = stats[i].specialization.failure_kinds[j]; if (val) { fprintf(out, " opcode[%s].specialization.failure_kinds[%d] : %" PRIu64 "\n", _PyOpcode_OpName[i], j, val); } } for (int j = 0; j < 256; j++) { if (stats[i].pair_count[j]) { fprintf(out, "opcode[%s].pair_count[%s] : %" PRIu64 "\n", _PyOpcode_OpName[i], _PyOpcode_OpName[j], stats[i].pair_count[j]); } } } } #undef PRINT_STAT static void print_call_stats(FILE *out, CallStats *stats) { fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls); fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls); fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed); fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created); for (int i = 0; i < EVAL_CALL_KINDS; i++) { fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]); } } static void print_object_stats(FILE *out, ObjectStats *stats) { fprintf(out, "Object allocations from freelist: %" PRIu64 "\n", stats->from_freelist); fprintf(out, "Object frees to freelist: %" PRIu64 "\n", stats->to_freelist); fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations); fprintf(out, "Object allocations to 512 bytes: %" PRIu64 "\n", stats->allocations512); fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k); fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big); fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values); fprintf(out, "Object interpreter mortal increfs: %" PRIu64 "\n", stats->interpreter_increfs); fprintf(out, "Object interpreter mortal decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); fprintf(out, "Object mortal increfs: %" PRIu64 "\n", stats->increfs); fprintf(out, "Object mortal decrefs: %" PRIu64 "\n", stats->decrefs); fprintf(out, "Object interpreter immortal increfs: %" PRIu64 "\n", stats->interpreter_immortal_increfs); fprintf(out, "Object interpreter immortal decrefs: %" PRIu64 "\n", stats->interpreter_immortal_decrefs); fprintf(out, "Object immortal increfs: %" PRIu64 "\n", stats->immortal_increfs); fprintf(out, "Object immortal decrefs: %" PRIu64 "\n", stats->immortal_decrefs); fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request); fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits); fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses); } static void print_gc_stats(FILE *out, GCStats *stats) { for (int i = 0; i < NUM_GENERATIONS; i++) { fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); fprintf(out, "GC[%d] objects reachable from roots: %" PRIu64 "\n", i, stats[i].objects_transitively_reachable); fprintf(out, "GC[%d] objects not reachable from roots: %" PRIu64 "\n", i, stats[i].objects_not_transitively_reachable); } } #ifdef _Py_TIER2 static void print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE]) { for (int i = 0; i < _Py_UOP_HIST_SIZE; i++) { fprintf(out, "%s[%" PRIu64"]: %" PRIu64 "\n", name, (uint64_t)1 << i, hist[i]); } } static void print_optimization_stats(FILE *out, OptimizationStats *stats) { fprintf(out, "Optimization attempts: %" PRIu64 "\n", stats->attempts); fprintf(out, "Optimization traces created: %" PRIu64 "\n", stats->traces_created); fprintf(out, "Optimization traces executed: %" PRIu64 "\n", stats->traces_executed); fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->uops_executed); fprintf(out, "Optimization trace stack overflow: %" PRIu64 "\n", stats->trace_stack_overflow); fprintf(out, "Optimization trace stack underflow: %" PRIu64 "\n", stats->trace_stack_underflow); fprintf(out, "Optimization trace too long: %" PRIu64 "\n", stats->trace_too_long); fprintf(out, "Optimization trace too short: %" PRIu64 "\n", stats->trace_too_short); fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated); print_histogram(out, "Trace length", stats->trace_length_hist); print_histogram(out, "Trace run length", stats->trace_run_length_hist); print_histogram(out, "Optimized trace length", stats->optimized_trace_length_hist); fprintf(out, "Optimization optimizer attempts: %" PRIu64 "\n", stats->optimizer_attempts); fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes); fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n", stats->optimizer_failure_reason_no_memory); fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); for (int i = 0; i <= MAX_UOP_ID; i++) { if (stats->opcode[i].execution_count) { fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); } if (stats->opcode[i].miss) { fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss); } } for (int i = 0; i < 256; i++) { if (stats->unsupported_opcode[i]) { fprintf( out, "unsupported_opcode[%s].count : %" PRIu64 "\n", _PyOpcode_OpName[i], stats->unsupported_opcode[i] ); } } for (int i = 1; i <= MAX_UOP_ID; i++){ for (int j = 1; j <= MAX_UOP_ID; j++) { if (stats->opcode[i].pair_count[j]) { fprintf(out, "uop[%s].pair_count[%s] : %" PRIu64 "\n", _PyOpcode_uop_name[i], _PyOpcode_uop_name[j], stats->opcode[i].pair_count[j]); } } } for (int i = 0; i < MAX_UOP_ID; i++) { if (stats->error_in_opcode[i]) { fprintf( out, "error_in_opcode[%s].count : %" PRIu64 "\n", _PyUOpName(i), stats->error_in_opcode[i] ); } } } #endif static void print_rare_event_stats(FILE *out, RareEventStats *stats) { fprintf(out, "Rare event (set_class): %" PRIu64 "\n", stats->set_class); fprintf(out, "Rare event (set_bases): %" PRIu64 "\n", stats->set_bases); fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func); fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict); fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification); fprintf(out, "Rare event (watched_dict_modification): %" PRIu64 "\n", stats->watched_dict_modification); fprintf(out, "Rare event (watched_globals_modification): %" PRIu64 "\n", stats->watched_globals_modification); } static void print_stats(FILE *out, PyStats *stats) { print_spec_stats(out, stats->opcode_stats); print_call_stats(out, &stats->call_stats); print_object_stats(out, &stats->object_stats); print_gc_stats(out, stats->gc_stats); #ifdef _Py_TIER2 print_optimization_stats(out, &stats->optimization_stats); #endif print_rare_event_stats(out, &stats->rare_event_stats); } void _Py_StatsOn(void) { _Py_stats = &_Py_stats_struct; } void _Py_StatsOff(void) { _Py_stats = NULL; } void _Py_StatsClear(void) { memset(&_py_gc_stats, 0, sizeof(_py_gc_stats)); memset(&_Py_stats_struct, 0, sizeof(_Py_stats_struct)); _Py_stats_struct.gc_stats = _py_gc_stats; } static int mem_is_zero(unsigned char *ptr, size_t size) { for (size_t i=0; i < size; i++) { if (*ptr != 0) { return 0; } ptr++; } return 1; } int _Py_PrintSpecializationStats(int to_file) { PyStats *stats = &_Py_stats_struct; #define MEM_IS_ZERO … int is_zero = ( MEM_IS_ZERO(stats->gc_stats) // is a pointer && MEM_IS_ZERO(&stats->opcode_stats) && MEM_IS_ZERO(&stats->call_stats) && MEM_IS_ZERO(&stats->object_stats) ); #undef MEM_IS_ZERO if (is_zero) { // gh-108753: -X pystats command line was used, but then _stats_off() // and _stats_clear() have been called: in this case, avoid printing // useless "all zeros" statistics. return 0; } FILE *out = stderr; if (to_file) { /* Write to a file instead of stderr. */ # ifdef MS_WINDOWS const char *dirname = "c:\\temp\\py_stats\\"; # else const char *dirname = "/tmp/py_stats/"; # endif /* Use random 160 bit number as file name, * to avoid both accidental collisions and * symlink attacks. */ unsigned char rand[20]; char hex_name[41]; _PyOS_URandomNonblock(rand, 20); for (int i = 0; i < 20; i++) { hex_name[2*i] = Py_hexdigits[rand[i]&15]; hex_name[2*i+1] = Py_hexdigits[(rand[i]>>4)&15]; } hex_name[40] = '\0'; char buf[64]; assert(strlen(dirname) + 40 + strlen(".txt") < 64); sprintf(buf, "%s%s.txt", dirname, hex_name); FILE *fout = fopen(buf, "w"); if (fout) { out = fout; } } else { fprintf(out, "Specialization stats:\n"); } print_stats(out, stats); if (out != stderr) { fclose(out); } return 1; } #define SPECIALIZATION_FAIL … #endif // Py_STATS #ifndef SPECIALIZATION_FAIL #define SPECIALIZATION_FAIL(opcode, kind) … #endif // Initialize warmup counters and optimize instructions. This cannot fail. void _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts, int enable_counters) { … } #define SIMPLE_FUNCTION … /* Common */ #define SPEC_FAIL_OTHER … #define SPEC_FAIL_NO_DICT … #define SPEC_FAIL_OVERRIDDEN … #define SPEC_FAIL_OUT_OF_VERSIONS … #define SPEC_FAIL_OUT_OF_RANGE … #define SPEC_FAIL_EXPECTED_ERROR … #define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS … #define SPEC_FAIL_CODE_COMPLEX_PARAMETERS … #define SPEC_FAIL_CODE_NOT_OPTIMIZED … #define SPEC_FAIL_LOAD_GLOBAL_NON_DICT … #define SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT … /* Super */ #define SPEC_FAIL_SUPER_BAD_CLASS … #define SPEC_FAIL_SUPER_SHADOWED … /* Attributes */ #define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR … #define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR … #define SPEC_FAIL_ATTR_NOT_DESCRIPTOR … #define SPEC_FAIL_ATTR_METHOD … #define SPEC_FAIL_ATTR_MUTABLE_CLASS … #define SPEC_FAIL_ATTR_PROPERTY … #define SPEC_FAIL_ATTR_NON_OBJECT_SLOT … #define SPEC_FAIL_ATTR_READ_ONLY … #define SPEC_FAIL_ATTR_AUDITED_SLOT … #define SPEC_FAIL_ATTR_NOT_MANAGED_DICT … #define SPEC_FAIL_ATTR_NON_STRING … #define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND … #define SPEC_FAIL_ATTR_SHADOWED … #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD … #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ … #define SPEC_FAIL_ATTR_OBJECT_SLOT … #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE … #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE … #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION … #define SPEC_FAIL_ATTR_NOT_IN_KEYS … #define SPEC_FAIL_ATTR_NOT_IN_DICT … #define SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE … #define SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR … #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ … #define SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN … #define SPEC_FAIL_ATTR_SPLIT_DICT … /* Binary subscr and store subscr */ #define SPEC_FAIL_SUBSCR_ARRAY_INT … #define SPEC_FAIL_SUBSCR_ARRAY_SLICE … #define SPEC_FAIL_SUBSCR_LIST_SLICE … #define SPEC_FAIL_SUBSCR_TUPLE_SLICE … #define SPEC_FAIL_SUBSCR_STRING_SLICE … #define SPEC_FAIL_SUBSCR_BUFFER_INT … #define SPEC_FAIL_SUBSCR_BUFFER_SLICE … #define SPEC_FAIL_SUBSCR_SEQUENCE_INT … /* Store subscr */ #define SPEC_FAIL_SUBSCR_BYTEARRAY_INT … #define SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE … #define SPEC_FAIL_SUBSCR_PY_SIMPLE … #define SPEC_FAIL_SUBSCR_PY_OTHER … #define SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE … #define SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE … /* Binary op */ #define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES … #define SPEC_FAIL_BINARY_OP_ADD_OTHER … #define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES … #define SPEC_FAIL_BINARY_OP_AND_INT … #define SPEC_FAIL_BINARY_OP_AND_OTHER … #define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE … #define SPEC_FAIL_BINARY_OP_LSHIFT … #define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY … #define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES … #define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER … #define SPEC_FAIL_BINARY_OP_OR … #define SPEC_FAIL_BINARY_OP_POWER … #define SPEC_FAIL_BINARY_OP_REMAINDER … #define SPEC_FAIL_BINARY_OP_RSHIFT … #define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES … #define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER … #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES … #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT … #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER … #define SPEC_FAIL_BINARY_OP_XOR … /* Calls */ #define SPEC_FAIL_CALL_INSTANCE_METHOD … #define SPEC_FAIL_CALL_CMETHOD … #define SPEC_FAIL_CALL_CFUNC_VARARGS … #define SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS … #define SPEC_FAIL_CALL_CFUNC_NOARGS … #define SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS … #define SPEC_FAIL_CALL_METH_DESCR_VARARGS … #define SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS … #define SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS … #define SPEC_FAIL_CALL_BAD_CALL_FLAGS … #define SPEC_FAIL_CALL_INIT_NOT_PYTHON … #define SPEC_FAIL_CALL_PEP_523 … #define SPEC_FAIL_CALL_BOUND_METHOD … #define SPEC_FAIL_CALL_CLASS_MUTABLE … #define SPEC_FAIL_CALL_METHOD_WRAPPER … #define SPEC_FAIL_CALL_OPERATOR_WRAPPER … #define SPEC_FAIL_CALL_INIT_NOT_SIMPLE … #define SPEC_FAIL_CALL_METACLASS … #define SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES … /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES … #define SPEC_FAIL_COMPARE_OP_STRING … #define SPEC_FAIL_COMPARE_OP_BIG_INT … #define SPEC_FAIL_COMPARE_OP_BYTES … #define SPEC_FAIL_COMPARE_OP_TUPLE … #define SPEC_FAIL_COMPARE_OP_LIST … #define SPEC_FAIL_COMPARE_OP_SET … #define SPEC_FAIL_COMPARE_OP_BOOL … #define SPEC_FAIL_COMPARE_OP_BASEOBJECT … #define SPEC_FAIL_COMPARE_OP_FLOAT_LONG … #define SPEC_FAIL_COMPARE_OP_LONG_FLOAT … /* FOR_ITER and SEND */ #define SPEC_FAIL_ITER_GENERATOR … #define SPEC_FAIL_ITER_COROUTINE … #define SPEC_FAIL_ITER_ASYNC_GENERATOR … #define SPEC_FAIL_ITER_LIST … #define SPEC_FAIL_ITER_TUPLE … #define SPEC_FAIL_ITER_SET … #define SPEC_FAIL_ITER_STRING … #define SPEC_FAIL_ITER_BYTES … #define SPEC_FAIL_ITER_RANGE … #define SPEC_FAIL_ITER_ITERTOOLS … #define SPEC_FAIL_ITER_DICT_KEYS … #define SPEC_FAIL_ITER_DICT_ITEMS … #define SPEC_FAIL_ITER_DICT_VALUES … #define SPEC_FAIL_ITER_ENUMERATE … #define SPEC_FAIL_ITER_MAP … #define SPEC_FAIL_ITER_ZIP … #define SPEC_FAIL_ITER_SEQ_ITER … #define SPEC_FAIL_ITER_REVERSED_LIST … #define SPEC_FAIL_ITER_CALLABLE … #define SPEC_FAIL_ITER_ASCII_STRING … #define SPEC_FAIL_ITER_ASYNC_GENERATOR_SEND … // UNPACK_SEQUENCE #define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR … #define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE … // TO_BOOL #define SPEC_FAIL_TO_BOOL_BYTEARRAY … #define SPEC_FAIL_TO_BOOL_BYTES … #define SPEC_FAIL_TO_BOOL_DICT … #define SPEC_FAIL_TO_BOOL_FLOAT … #define SPEC_FAIL_TO_BOOL_MAPPING … #define SPEC_FAIL_TO_BOOL_MEMORY_VIEW … #define SPEC_FAIL_TO_BOOL_NUMBER … #define SPEC_FAIL_TO_BOOL_SEQUENCE … #define SPEC_FAIL_TO_BOOL_SET … #define SPEC_FAIL_TO_BOOL_TUPLE … // CONTAINS_OP #define SPEC_FAIL_CONTAINS_OP_STR … #define SPEC_FAIL_CONTAINS_OP_TUPLE … #define SPEC_FAIL_CONTAINS_OP_LIST … #define SPEC_FAIL_CONTAINS_OP_USER_CLASS … static inline int set_opcode(_Py_CODEUNIT *instr, uint8_t opcode) { … } static inline void set_counter(_Py_BackoffCounter *counter, _Py_BackoffCounter value) { … } static inline _Py_BackoffCounter load_counter(_Py_BackoffCounter *counter) { … } static inline void specialize(_Py_CODEUNIT *instr, uint8_t specialized_opcode) { … } static inline void unspecialize(_Py_CODEUNIT *instr) { … } static int function_kind(PyCodeObject *code); static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); static uint32_t type_get_version(PyTypeObject *t, int opcode); static int specialize_module_load_attr( PyObject *owner, _Py_CODEUNIT *instr, PyObject *name ) { … } /* Attribute specialization */ void _Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _Py_CODEUNIT *instr, int load_method) { … } DescriptorClassification; static DescriptorClassification classify_descriptor(PyObject *descriptor, bool has_getattr) { … } static DescriptorClassification analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store) { … } static int specialize_dict_access( PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, DescriptorClassification kind, PyObject *name, int base_op, int values_op, int hint_op) { … } static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, PyObject* descr, DescriptorClassification kind, bool is_method); static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name); /* Returns true if instances of obj's class are * likely to have `name` in their __dict__. * For objects with inline values, we check in the shared keys. * For other objects, we check their actual dictionary. */ static bool instance_has_key(PyObject *obj, PyObject* name) { … } static int specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name) { … } void _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) { … } void _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) { … } #ifdef Py_STATS static int load_attr_fail_kind(DescriptorClassification kind) { switch (kind) { case OVERRIDING: return SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR; case METHOD: return SPEC_FAIL_ATTR_METHOD; case PROPERTY: return SPEC_FAIL_ATTR_PROPERTY; case OBJECT_SLOT: return SPEC_FAIL_ATTR_OBJECT_SLOT; case OTHER_SLOT: return SPEC_FAIL_ATTR_NON_OBJECT_SLOT; case DUNDER_CLASS: return SPEC_FAIL_OTHER; case MUTABLE: return SPEC_FAIL_ATTR_MUTABLE_CLASS; case GETSET_OVERRIDDEN: case GETATTRIBUTE_IS_PYTHON_FUNCTION: return SPEC_FAIL_OVERRIDDEN; case BUILTIN_CLASSMETHOD: return SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD; case PYTHON_CLASSMETHOD: return SPEC_FAIL_ATTR_CLASS_METHOD_OBJ; case NON_OVERRIDING: return SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR; case NON_DESCRIPTOR: return SPEC_FAIL_ATTR_NOT_DESCRIPTOR; case ABSENT: return SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE; } Py_UNREACHABLE(); } #endif // Py_STATS static int specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { … } // Please collect stats carefully before and after modifying. A subtle change // can cause a significant drop in cache hits. A possible test is // python.exe -m test_typing test_re test_dis test_zlib. static int specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, PyObject *descr, DescriptorClassification kind, bool is_method) { … } static void specialize_load_global_lock_held( PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name) { … } void _Py_Specialize_LoadGlobal( PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name) { … } #ifdef Py_STATS static int binary_subscr_fail_kind(PyTypeObject *container_type, PyObject *sub) { if (strcmp(container_type->tp_name, "array.array") == 0) { if (PyLong_CheckExact(sub)) { return SPEC_FAIL_SUBSCR_ARRAY_INT; } if (PySlice_Check(sub)) { return SPEC_FAIL_SUBSCR_ARRAY_SLICE; } return SPEC_FAIL_OTHER; } else if (container_type->tp_as_buffer) { if (PyLong_CheckExact(sub)) { return SPEC_FAIL_SUBSCR_BUFFER_INT; } if (PySlice_Check(sub)) { return SPEC_FAIL_SUBSCR_BUFFER_SLICE; } return SPEC_FAIL_OTHER; } else if (container_type->tp_as_sequence) { if (PyLong_CheckExact(sub) && container_type->tp_as_sequence->sq_item) { return SPEC_FAIL_SUBSCR_SEQUENCE_INT; } } return SPEC_FAIL_OTHER; } #endif // Py_STATS static int function_kind(PyCodeObject *code) { … } /* Returning false indicates a failure. */ static bool function_check_args(PyObject *o, int expected_argcount, int opcode) { … } /* Returning 0 indicates a failure. */ static uint32_t function_get_version(PyObject *o, int opcode) { … } /* Returning 0 indicates a failure. */ static uint32_t type_get_version(PyTypeObject *t, int opcode) { … } void _Py_Specialize_BinarySubscr( _PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) { … } #ifdef Py_STATS static int store_subscr_fail_kind(PyObject *container, PyObject *sub) { PyTypeObject *container_type = Py_TYPE(container); PyMappingMethods *as_mapping = container_type->tp_as_mapping; if (as_mapping && (as_mapping->mp_ass_subscript == PyDict_Type.tp_as_mapping->mp_ass_subscript)) { return SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE; } if (PyObject_CheckBuffer(container)) { if (PyLong_CheckExact(sub) && (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub))) { return SPEC_FAIL_OUT_OF_RANGE; } else if (strcmp(container_type->tp_name, "array.array") == 0) { if (PyLong_CheckExact(sub)) { return SPEC_FAIL_SUBSCR_ARRAY_INT; } else if (PySlice_Check(sub)) { return SPEC_FAIL_SUBSCR_ARRAY_SLICE; } else { return SPEC_FAIL_OTHER; } } else if (PyByteArray_CheckExact(container)) { if (PyLong_CheckExact(sub)) { return SPEC_FAIL_SUBSCR_BYTEARRAY_INT; } else if (PySlice_Check(sub)) { return SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE; } else { return SPEC_FAIL_OTHER; } } else { if (PyLong_CheckExact(sub)) { return SPEC_FAIL_SUBSCR_BUFFER_INT; } else if (PySlice_Check(sub)) { return SPEC_FAIL_SUBSCR_BUFFER_SLICE; } else { return SPEC_FAIL_OTHER; } } return SPEC_FAIL_OTHER; } PyObject *descriptor = _PyType_Lookup(container_type, &_Py_ID(__setitem__)); if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) { PyFunctionObject *func = (PyFunctionObject *)descriptor; PyCodeObject *code = (PyCodeObject *)func->func_code; int kind = function_kind(code); if (kind == SIMPLE_FUNCTION) { return SPEC_FAIL_SUBSCR_PY_SIMPLE; } else { return SPEC_FAIL_SUBSCR_PY_OTHER; } } return SPEC_FAIL_OTHER; } #endif void _Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) { … } /* Returns a borrowed reference. * The reference is only valid if guarded by a type version check. */ static PyFunctionObject * get_init_for_simple_managed_python_class(PyTypeObject *tp) { … } static int specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) { … } static int specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, int nargs) { … } static int specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, bool bound_method) { … } static int specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, bool bound_method) { … } static int specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) { … } void _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) { … } void _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) { … } #ifdef Py_STATS static int binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) { switch (oparg) { case NB_ADD: case NB_INPLACE_ADD: if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { return SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES; } return SPEC_FAIL_BINARY_OP_ADD_OTHER; case NB_AND: case NB_INPLACE_AND: if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { return SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES; } if (PyLong_CheckExact(lhs)) { return SPEC_FAIL_BINARY_OP_AND_INT; } return SPEC_FAIL_BINARY_OP_AND_OTHER; case NB_FLOOR_DIVIDE: case NB_INPLACE_FLOOR_DIVIDE: return SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE; case NB_LSHIFT: case NB_INPLACE_LSHIFT: return SPEC_FAIL_BINARY_OP_LSHIFT; case NB_MATRIX_MULTIPLY: case NB_INPLACE_MATRIX_MULTIPLY: return SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY; case NB_MULTIPLY: case NB_INPLACE_MULTIPLY: if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { return SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES; } return SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER; case NB_OR: case NB_INPLACE_OR: return SPEC_FAIL_BINARY_OP_OR; case NB_POWER: case NB_INPLACE_POWER: return SPEC_FAIL_BINARY_OP_POWER; case NB_REMAINDER: case NB_INPLACE_REMAINDER: return SPEC_FAIL_BINARY_OP_REMAINDER; case NB_RSHIFT: case NB_INPLACE_RSHIFT: return SPEC_FAIL_BINARY_OP_RSHIFT; case NB_SUBTRACT: case NB_INPLACE_SUBTRACT: if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { return SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES; } return SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER; case NB_TRUE_DIVIDE: case NB_INPLACE_TRUE_DIVIDE: if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES; } if (PyFloat_CheckExact(lhs)) { return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT; } return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER; case NB_XOR: case NB_INPLACE_XOR: return SPEC_FAIL_BINARY_OP_XOR; } Py_UNREACHABLE(); } #endif void _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr, int oparg, _PyStackRef *locals) { … } #ifdef Py_STATS static int compare_op_fail_kind(PyObject *lhs, PyObject *rhs) { if (Py_TYPE(lhs) != Py_TYPE(rhs)) { if (PyFloat_CheckExact(lhs) && PyLong_CheckExact(rhs)) { return SPEC_FAIL_COMPARE_OP_FLOAT_LONG; } if (PyLong_CheckExact(lhs) && PyFloat_CheckExact(rhs)) { return SPEC_FAIL_COMPARE_OP_LONG_FLOAT; } return SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES; } if (PyBytes_CheckExact(lhs)) { return SPEC_FAIL_COMPARE_OP_BYTES; } if (PyTuple_CheckExact(lhs)) { return SPEC_FAIL_COMPARE_OP_TUPLE; } if (PyList_CheckExact(lhs)) { return SPEC_FAIL_COMPARE_OP_LIST; } if (PySet_CheckExact(lhs) || PyFrozenSet_CheckExact(lhs)) { return SPEC_FAIL_COMPARE_OP_SET; } if (PyBool_Check(lhs)) { return SPEC_FAIL_COMPARE_OP_BOOL; } if (Py_TYPE(lhs)->tp_richcompare == PyBaseObject_Type.tp_richcompare) { return SPEC_FAIL_COMPARE_OP_BASEOBJECT; } return SPEC_FAIL_OTHER; } #endif // Py_STATS void _Py_Specialize_CompareOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr, int oparg) { … } #ifdef Py_STATS static int unpack_sequence_fail_kind(PyObject *seq) { if (PySequence_Check(seq)) { return SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE; } if (PyIter_Check(seq)) { return SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR; } return SPEC_FAIL_OTHER; } #endif // Py_STATS void _Py_Specialize_UnpackSequence(_PyStackRef seq_st, _Py_CODEUNIT *instr, int oparg) { … } #ifdef Py_STATS int _PySpecialization_ClassifyIterator(PyObject *iter) { if (PyGen_CheckExact(iter)) { return SPEC_FAIL_ITER_GENERATOR; } if (PyCoro_CheckExact(iter)) { return SPEC_FAIL_ITER_COROUTINE; } if (PyAsyncGen_CheckExact(iter)) { return SPEC_FAIL_ITER_ASYNC_GENERATOR; } if (PyAsyncGenASend_CheckExact(iter)) { return SPEC_FAIL_ITER_ASYNC_GENERATOR_SEND; } PyTypeObject *t = Py_TYPE(iter); if (t == &PyListIter_Type) { return SPEC_FAIL_ITER_LIST; } if (t == &PyTupleIter_Type) { return SPEC_FAIL_ITER_TUPLE; } if (t == &PyDictIterKey_Type) { return SPEC_FAIL_ITER_DICT_KEYS; } if (t == &PyDictIterValue_Type) { return SPEC_FAIL_ITER_DICT_VALUES; } if (t == &PyDictIterItem_Type) { return SPEC_FAIL_ITER_DICT_ITEMS; } if (t == &PySetIter_Type) { return SPEC_FAIL_ITER_SET; } if (t == &PyUnicodeIter_Type) { return SPEC_FAIL_ITER_STRING; } if (t == &PyBytesIter_Type) { return SPEC_FAIL_ITER_BYTES; } if (t == &PyRangeIter_Type) { return SPEC_FAIL_ITER_RANGE; } if (t == &PyEnum_Type) { return SPEC_FAIL_ITER_ENUMERATE; } if (t == &PyMap_Type) { return SPEC_FAIL_ITER_MAP; } if (t == &PyZip_Type) { return SPEC_FAIL_ITER_ZIP; } if (t == &PySeqIter_Type) { return SPEC_FAIL_ITER_SEQ_ITER; } if (t == &PyListRevIter_Type) { return SPEC_FAIL_ITER_REVERSED_LIST; } if (t == &_PyUnicodeASCIIIter_Type) { return SPEC_FAIL_ITER_ASCII_STRING; } const char *name = t->tp_name; if (strncmp(name, "itertools", 9) == 0) { return SPEC_FAIL_ITER_ITERTOOLS; } if (strncmp(name, "callable_iterator", 17) == 0) { return SPEC_FAIL_ITER_CALLABLE; } return SPEC_FAIL_OTHER; } #endif // Py_STATS void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) { … } void _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) { … } #ifdef Py_STATS static int to_bool_fail_kind(PyObject *value) { if (PyByteArray_CheckExact(value)) { return SPEC_FAIL_TO_BOOL_BYTEARRAY; } if (PyBytes_CheckExact(value)) { return SPEC_FAIL_TO_BOOL_BYTES; } if (PyDict_CheckExact(value)) { return SPEC_FAIL_TO_BOOL_DICT; } if (PyFloat_CheckExact(value)) { return SPEC_FAIL_TO_BOOL_FLOAT; } if (PyMemoryView_Check(value)) { return SPEC_FAIL_TO_BOOL_MEMORY_VIEW; } if (PyAnySet_CheckExact(value)) { return SPEC_FAIL_TO_BOOL_SET; } if (PyTuple_CheckExact(value)) { return SPEC_FAIL_TO_BOOL_TUPLE; } return SPEC_FAIL_OTHER; } #endif // Py_STATS static int check_type_always_true(PyTypeObject *ty) { … } void _Py_Specialize_ToBool(_PyStackRef value_o, _Py_CODEUNIT *instr) { … } #ifdef Py_STATS static int containsop_fail_kind(PyObject *value) { if (PyUnicode_CheckExact(value)) { return SPEC_FAIL_CONTAINS_OP_STR; } if (PyList_CheckExact(value)) { return SPEC_FAIL_CONTAINS_OP_LIST; } if (PyTuple_CheckExact(value)) { return SPEC_FAIL_CONTAINS_OP_TUPLE; } if (PyType_Check(value)) { return SPEC_FAIL_CONTAINS_OP_USER_CLASS; } return SPEC_FAIL_OTHER; } #endif void _Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr) { … } /* Code init cleanup. * CALL_ALLOC_AND_ENTER_INIT will set up * the frame to execute the EXIT_INIT_CHECK * instruction. * Ends with a RESUME so that it is not traced. * This is used as a plain code object, not a function, * so must not access globals or builtins. */ #define NO_LOC_4 … static const PyBytesObject no_location = …; const struct _PyCode8 _Py_InitCleanup = …;
Codebase Browser
cpython