#include "Python.h" #include "pycore_ast.h" // stmt_ty #include "pycore_parser.h" // _PyParser_ASTFromString() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // PySTEntryObject // Set this to 1 to dump all symtables to stdout for debugging #define _PY_DUMP_SYMTABLE … /* error strings used for warnings */ #define GLOBAL_PARAM … #define NONLOCAL_PARAM … #define GLOBAL_AFTER_ASSIGN … #define NONLOCAL_AFTER_ASSIGN … #define GLOBAL_AFTER_USE … #define NONLOCAL_AFTER_USE … #define GLOBAL_ANNOT … #define NONLOCAL_ANNOT … #define IMPORT_STAR_WARNING … #define NAMED_EXPR_COMP_IN_CLASS … #define NAMED_EXPR_COMP_IN_TYPEVAR_BOUND … #define NAMED_EXPR_COMP_IN_TYPEALIAS … #define NAMED_EXPR_COMP_IN_TYPEPARAM … #define NAMED_EXPR_COMP_CONFLICT … #define NAMED_EXPR_COMP_INNER_LOOP_CONFLICT … #define NAMED_EXPR_COMP_ITER_EXPR … #define ANNOTATION_NOT_ALLOWED … #define EXPR_NOT_ALLOWED_IN_TYPE_VARIABLE … #define EXPR_NOT_ALLOWED_IN_TYPE_ALIAS … #define EXPR_NOT_ALLOWED_IN_TYPE_PARAMETERS … #define DUPLICATE_TYPE_PARAM … #define ASYNC_WITH_OUTSIDE_ASYNC_FUNC … #define ASYNC_FOR_OUTSIDE_ASYNC_FUNC … #define LOCATION(x) … #define SET_ERROR_LOCATION(FNAME, L) … #define IS_ASYNC_DEF(st) … static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, void *key, _Py_SourceLocation loc) { … } static PyObject * ste_repr(PySTEntryObject *ste) { … } static void ste_dealloc(PySTEntryObject *ste) { … } #define OFF(x) … static PyMemberDef ste_memberlist[] = …; PyTypeObject PySTEntry_Type = …; static int symtable_analyze(struct symtable *st); static int symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, void *ast, _Py_SourceLocation loc); static int symtable_exit_block(struct symtable *st); static int symtable_visit_stmt(struct symtable *st, stmt_ty s); static int symtable_visit_expr(struct symtable *st, expr_ty s); static int symtable_visit_type_param(struct symtable *st, type_param_ty s); static int symtable_visit_genexp(struct symtable *st, expr_ty s); static int symtable_visit_listcomp(struct symtable *st, expr_ty s); static int symtable_visit_setcomp(struct symtable *st, expr_ty s); static int symtable_visit_dictcomp(struct symtable *st, expr_ty s); static int symtable_visit_arguments(struct symtable *st, arguments_ty); static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty); static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty); static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args); static int symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key); static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty, struct _symtable_entry *parent_ste); static int symtable_visit_withitem(struct symtable *st, withitem_ty item); static int symtable_visit_match_case(struct symtable *st, match_case_ty m); static int symtable_visit_pattern(struct symtable *st, pattern_ty s); static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty); static int symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_SourceLocation loc); static int symtable_raise_if_comprehension_block(struct symtable *st, expr_ty); static int symtable_add_def(struct symtable *st, PyObject *name, int flag, _Py_SourceLocation loc); /* For debugging purposes only */ #if _PY_DUMP_SYMTABLE static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) { const char *blocktype = ""; switch (ste->ste_type) { case FunctionBlock: blocktype = "FunctionBlock"; break; case ClassBlock: blocktype = "ClassBlock"; break; case ModuleBlock: blocktype = "ModuleBlock"; break; case AnnotationBlock: blocktype = "AnnotationBlock"; break; case TypeVariableBlock: blocktype = "TypeVariableBlock"; break; case TypeAliasBlock: blocktype = "TypeAliasBlock"; break; case TypeParametersBlock: blocktype = "TypeParametersBlock"; break; } const char *comptype = ""; switch (ste->ste_comprehension) { case ListComprehension: comptype = " ListComprehension"; break; case DictComprehension: comptype = " DictComprehension"; break; case SetComprehension: comptype = " SetComprehension"; break; case GeneratorExpression: comptype = " GeneratorExpression"; break; case NoComprehension: break; } PyObject* msg = PyUnicode_FromFormat( ( "%U=== Symtable for %U ===\n" "%U%s%s\n" "%U%s%s%s%s%s%s%s%s%s%s%s\n" "%Ulineno: %d col_offset: %d\n" "%U--- Symbols ---\n" ), prefix, ste->ste_name, prefix, blocktype, comptype, prefix, ste->ste_nested ? " nested" : "", ste->ste_generator ? " generator" : "", ste->ste_coroutine ? " coroutine" : "", ste->ste_varargs ? " varargs" : "", ste->ste_varkeywords ? " varkeywords" : "", ste->ste_returns_value ? " returns_value" : "", ste->ste_needs_class_closure ? " needs_class_closure" : "", ste->ste_needs_classdict ? " needs_classdict" : "", ste->ste_comp_inlined ? " comp_inlined" : "", ste->ste_comp_iter_target ? " comp_iter_target" : "", ste->ste_can_see_class_scope ? " can_see_class_scope" : "", prefix, ste->ste_loc.lineno, ste->ste_loc.col_offset, prefix ); assert(msg != NULL); printf("%s", PyUnicode_AsUTF8(msg)); Py_DECREF(msg); PyObject *name, *value; Py_ssize_t pos = 0; while (PyDict_Next(ste->ste_symbols, &pos, &name, &value)) { int scope = _PyST_GetScope(ste, name); long flags = _PyST_GetSymbol(ste, name); printf("%s %s: ", PyUnicode_AsUTF8(prefix), PyUnicode_AsUTF8(name)); if (flags & DEF_GLOBAL) printf(" DEF_GLOBAL"); if (flags & DEF_LOCAL) printf(" DEF_LOCAL"); if (flags & DEF_PARAM) printf(" DEF_PARAM"); if (flags & DEF_NONLOCAL) printf(" DEF_NONLOCAL"); if (flags & USE) printf(" USE"); if (flags & DEF_FREE_CLASS) printf(" DEF_FREE_CLASS"); if (flags & DEF_IMPORT) printf(" DEF_IMPORT"); if (flags & DEF_ANNOT) printf(" DEF_ANNOT"); if (flags & DEF_COMP_ITER) printf(" DEF_COMP_ITER"); if (flags & DEF_TYPE_PARAM) printf(" DEF_TYPE_PARAM"); if (flags & DEF_COMP_CELL) printf(" DEF_COMP_CELL"); switch (scope) { case LOCAL: printf(" LOCAL"); break; case GLOBAL_EXPLICIT: printf(" GLOBAL_EXPLICIT"); break; case GLOBAL_IMPLICIT: printf(" GLOBAL_IMPLICIT"); break; case FREE: printf(" FREE"); break; case CELL: printf(" CELL"); break; } printf("\n"); } printf("%s--- Children ---\n", PyUnicode_AsUTF8(prefix)); PyObject *new_prefix = PyUnicode_FromFormat(" %U", prefix); assert(new_prefix != NULL); for (Py_ssize_t i = 0; i < PyList_GET_SIZE(ste->ste_children); i++) { PyObject *child = PyList_GetItem(ste->ste_children, i); assert(child != NULL && PySTEntry_Check(child)); _dump_symtable((PySTEntryObject *)child, new_prefix); } Py_DECREF(new_prefix); } static void dump_symtable(PySTEntryObject* ste) { PyObject *empty = Py_GetConstant(Py_CONSTANT_EMPTY_STR); assert(empty != NULL); _dump_symtable(ste, empty); Py_DECREF(empty); } #endif #define DUPLICATE_ARGUMENT … static struct symtable * symtable_new(void) { … } struct symtable * _PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) { … } void _PySymtable_Free(struct symtable *st) { … } PySTEntryObject * _PySymtable_Lookup(struct symtable *st, void *key) { … } int _PySymtable_LookupOptional(struct symtable *st, void *key, PySTEntryObject **out) { … } long _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name) { … } int _PyST_GetScope(PySTEntryObject *ste, PyObject *name) { … } int _PyST_IsFunctionLike(PySTEntryObject *ste) { … } static int error_at_directive(PySTEntryObject *ste, PyObject *name) { … } /* Analyze raw symbol information to determine scope of each name. The next several functions are helpers for symtable_analyze(), which determines whether a name is local, global, or free. In addition, it determines which local variables are cell variables; they provide bindings that are used for free variables in enclosed blocks. There are also two kinds of global variables, implicit and explicit. An explicit global is declared with the global statement. An implicit global is a free variable for which the compiler has found no binding in an enclosing function scope. The implicit global is either a global or a builtin. Python's module and class blocks use the xxx_NAME opcodes to handle these names to implement slightly odd semantics. In such a block, the name is treated as global until it is assigned to; then it is treated as a local. The symbol table requires two passes to determine the scope of each name. The first pass collects raw facts from the AST via the symtable_visit_* functions: the name is a parameter here, the name is used but not defined here, etc. The second pass analyzes these facts during a pass over the PySTEntryObjects created during pass 1. When a function is entered during the second pass, the parent passes the set of all name bindings visible to its children. These bindings are used to determine if non-local variables are free or implicit globals. Names which are explicitly declared nonlocal must exist in this set of visible names - if they do not, a syntax error is raised. After doing the local analysis, it analyzes each of its child blocks using an updated set of name bindings. The children update the free variable set. If a local variable is added to the free variable set by the child, the variable is marked as a cell. The function object being defined must provide runtime storage for the variable that may outlive the function's frame. Cell variables are removed from the free set before the analyze function returns to its parent. During analysis, the names are: symbols: dict mapping from symbol names to flag values (including offset scope values) scopes: dict mapping from symbol names to scope values (no offset) local: set of all symbol names local to the current scope bound: set of all symbol names local to a containing function scope free: set of all symbol names referenced but not bound in child scopes global: set of all symbol names explicitly declared as global */ #define SET_SCOPE … /* Decide on scope of name, given flags. The namespace dictionaries may be modified to record information about the new name. For example, a new global will add an entry to global. A name that was global can be changed to local. */ static int analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, PyObject *bound, PyObject *local, PyObject *free, PyObject *global, PyObject *type_params, PySTEntryObject *class_entry) { … } static int is_free_in_any_child(PySTEntryObject *entry, PyObject *key) { … } static int inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, PyObject *scopes, PyObject *comp_free, PyObject *inlined_cells) { … } #undef SET_SCOPE /* If a name is defined in free and also in locals, then this block provides the binding for the free variable. The name should be marked CELL in this block and removed from the free list. Note that the current block's free variables are included in free. That's safe because no name can be free and local in the same scope. */ static int analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells) { … } static int drop_class_free(PySTEntryObject *ste, PyObject *free) { … } /* Enter the final scope information into the ste_symbols dict. * * All arguments are dicts. Modifies symbols, others are read-only. */ static int update_symbols(PyObject *symbols, PyObject *scopes, PyObject *bound, PyObject *free, PyObject *inlined_cells, int classflag) { … } /* Make final symbol table decisions for block of ste. Arguments: ste -- current symtable entry (input/output) bound -- set of variables bound in enclosing scopes (input). bound is NULL for module blocks. free -- set of free variables in enclosed scopes (output) globals -- set of declared global variables in enclosing scopes (input) The implementation uses two mutually recursive functions, analyze_block() and analyze_child_block(). analyze_block() is responsible for analyzing the individual names defined in a block. analyze_child_block() prepares temporary namespace dictionaries used to evaluated nested blocks. The two functions exist because a child block should see the name bindings of its enclosing blocks, but those bindings should not propagate back to a parent block. */ static int analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, PyObject *global, PyObject *type_params, PySTEntryObject *class_entry, PyObject **child_free); static int analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, PyObject *global, PyObject *type_params, PySTEntryObject *class_entry) { … } static int analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, PyObject *global, PyObject *type_params, PySTEntryObject *class_entry, PyObject** child_free) { … } static int symtable_analyze(struct symtable *st) { … } /* symtable_enter_block() gets a reference via ste_new. This reference is released when the block is exited, via the DECREF in symtable_exit_block(). */ static int symtable_exit_block(struct symtable *st) { … } static int symtable_enter_existing_block(struct symtable *st, PySTEntryObject* ste) { … } static int symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, void *ast, _Py_SourceLocation loc) { … } static long symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name) { … } static long symtable_lookup(struct symtable *st, PyObject *name) { … } static int symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _symtable_entry *ste, _Py_SourceLocation loc) { … } static int check_name(struct symtable *st, PyObject *name, _Py_SourceLocation loc, expr_context_ty ctx) { … } static int check_keywords(struct symtable *st, asdl_keyword_seq *keywords) { … } static int check_kwd_patterns(struct symtable *st, pattern_ty p) { … } static int symtable_add_def_ctx(struct symtable *st, PyObject *name, int flag, _Py_SourceLocation loc, expr_context_ty ctx) { … } static int symtable_add_def(struct symtable *st, PyObject *name, int flag, _Py_SourceLocation loc) { … } static int symtable_enter_type_param_block(struct symtable *st, identifier name, void *ast, int has_defaults, int has_kwdefaults, enum _stmt_kind kind, _Py_SourceLocation loc) { … } /* VISIT, VISIT_SEQ and VIST_SEQ_TAIL take an ASDL type as their second argument. They use the ASDL name to synthesize the name of the C type and the visit function. VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is useful if the first node in the sequence requires special treatment. ENTER_RECURSIVE macro increments the current recursion depth counter. It should be used at the beginning of the recursive function. LEAVE_RECURSIVE macro decrements the current recursion depth counter. It should be used at the end of the recursive function. */ #define VISIT(ST, TYPE, V) … #define VISIT_SEQ(ST, TYPE, SEQ) … #define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) … #define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) … #define ENTER_RECURSIVE(ST) … #define LEAVE_RECURSIVE(ST) … static int symtable_record_directive(struct symtable *st, identifier name, _Py_SourceLocation loc) { … } static int has_kwonlydefaults(asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults) { … } static int check_import_from(struct symtable *st, stmt_ty s) { … } static bool allows_top_level_await(struct symtable *st) { … } static void maybe_set_ste_coroutine_for_module(struct symtable *st, stmt_ty s) { … } static int symtable_visit_stmt(struct symtable *st, stmt_ty s) { … } static int symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) { … } static int symtable_handle_namedexpr(struct symtable *st, expr_ty e) { … } static int symtable_visit_expr(struct symtable *st, expr_ty e) { … } static int symtable_visit_type_param_bound_or_default( struct symtable *st, expr_ty e, identifier name, void *key, const char *ste_scope_info) { … } static int symtable_visit_type_param(struct symtable *st, type_param_ty tp) { … } static int symtable_visit_pattern(struct symtable *st, pattern_ty p) { … } static int symtable_implicit_arg(struct symtable *st, int pos) { … } static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args) { … } static int symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key) { … } static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args) { … } static int symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns, struct _symtable_entry *function_ste) { … } static int symtable_visit_arguments(struct symtable *st, arguments_ty a) { … } static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh) { … } static int symtable_visit_withitem(struct symtable *st, withitem_ty item) { … } static int symtable_visit_match_case(struct symtable *st, match_case_ty m) { … } static int symtable_visit_alias(struct symtable *st, alias_ty a) { … } static int symtable_visit_comprehension(struct symtable *st, comprehension_ty lc) { … } static int symtable_visit_keyword(struct symtable *st, keyword_ty k) { … } static int symtable_handle_comprehension(struct symtable *st, expr_ty e, identifier scope_name, asdl_comprehension_seq *generators, expr_ty elt, expr_ty value) { … } static int symtable_visit_genexp(struct symtable *st, expr_ty e) { … } static int symtable_visit_listcomp(struct symtable *st, expr_ty e) { … } static int symtable_visit_setcomp(struct symtable *st, expr_ty e) { … } static int symtable_visit_dictcomp(struct symtable *st, expr_ty e) { … } static int symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e) { … } static int symtable_raise_if_comprehension_block(struct symtable *st, expr_ty e) { … } static int symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_SourceLocation loc) { … } struct symtable * _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, PyCompilerFlags *flags) { … } PyObject * _Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name) { … } PyObject * _Py_Mangle(PyObject *privateobj, PyObject *ident) { … }
Codebase Browser
cpython