| """Top-level logic for the semantic analyzer. |
| |
| The semantic analyzer binds names, resolves imports, detects various |
| special constructs that don't have dedicated AST nodes after parse |
| (such as 'cast' which looks like a call), populates symbol tables, and |
| performs various simple consistency checks. |
| |
| Semantic analysis of each SCC (strongly connected component; import |
| cycle) is performed in one unit. Each module is analyzed as multiple |
| separate *targets*; the module top level is one target and each function |
| is a target. Nested functions are not separate targets, however. This is |
| mostly identical to targets used by mypy daemon (but classes aren't |
| targets in semantic analysis). |
| |
| We first analyze each module top level in an SCC. If we encounter some |
| names that we can't bind because the target of the name may not have |
| been processed yet, we *defer* the current target for further |
| processing. Deferred targets will be analyzed additional times until |
| everything can be bound, or we reach a maximum number of iterations. |
| |
| We keep track of a set of incomplete namespaces, i.e. namespaces that we |
| haven't finished populating yet. References to these namespaces cause a |
| deferral if they can't be satisfied. Initially every module in the SCC |
| will be incomplete. |
| """ |
| |
| from __future__ import annotations |
| |
| from contextlib import nullcontext |
| from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union |
| from typing_extensions import Final, TypeAlias as _TypeAlias |
| |
| import mypy.build |
| import mypy.state |
| from mypy.checker import FineGrainedDeferredNode |
| from mypy.errors import Errors |
| from mypy.nodes import Decorator, FuncDef, MypyFile, OverloadedFuncDef, TypeInfo, Var |
| from mypy.options import Options |
| from mypy.plugin import ClassDefContext |
| from mypy.plugins import dataclasses as dataclasses_plugin |
| from mypy.semanal import ( |
| SemanticAnalyzer, |
| apply_semantic_analyzer_patches, |
| is_dataclass_transform_decorator, |
| remove_imported_names_from_symtable, |
| ) |
| from mypy.semanal_classprop import ( |
| add_type_promotion, |
| calculate_class_abstract_status, |
| calculate_class_vars, |
| check_protocol_status, |
| ) |
| from mypy.semanal_infer import infer_decorator_signature_if_simple |
| from mypy.semanal_typeargs import TypeArgumentAnalyzer |
| from mypy.server.aststrip import SavedAttributes |
| from mypy.util import is_typeshed_file |
| |
| if TYPE_CHECKING: |
| from mypy.build import Graph, State |
| |
| |
| Patches: _TypeAlias = List[Tuple[int, Callable[[], None]]] |
| |
| |
| # If we perform this many iterations, raise an exception since we are likely stuck. |
| MAX_ITERATIONS: Final = 20 |
| |
| |
| # Number of passes over core modules before going on to the rest of the builtin SCC. |
| CORE_WARMUP: Final = 2 |
| core_modules: Final = [ |
| "typing", |
| "_collections_abc", |
| "builtins", |
| "abc", |
| "collections", |
| "collections.abc", |
| ] |
| |
| |
| def semantic_analysis_for_scc(graph: Graph, scc: list[str], errors: Errors) -> None: |
| """Perform semantic analysis for all modules in a SCC (import cycle). |
| |
| Assume that reachability analysis has already been performed. |
| |
| The scc will be processed roughly in the order the modules are included |
| in the list. |
| """ |
| patches: Patches = [] |
| # Note that functions can't define new module-level attributes |
| # using 'global x', since module top levels are fully processed |
| # before functions. This limitation is unlikely to go away soon. |
| process_top_levels(graph, scc, patches) |
| process_functions(graph, scc, patches) |
| # We use patch callbacks to fix up things when we expect relatively few |
| # callbacks to be required. |
| apply_semantic_analyzer_patches(patches) |
| # Run class decorator hooks (they requite complete MROs and no placeholders). |
| apply_class_plugin_hooks(graph, scc, errors) |
| # This pass might need fallbacks calculated above and the results of hooks. |
| check_type_arguments(graph, scc, errors) |
| calculate_class_properties(graph, scc, errors) |
| check_blockers(graph, scc) |
| # Clean-up builtins, so that TypeVar etc. are not accessible without importing. |
| if "builtins" in scc: |
| cleanup_builtin_scc(graph["builtins"]) |
| |
| |
| def cleanup_builtin_scc(state: State) -> None: |
| """Remove imported names from builtins namespace. |
| |
| This way names imported from typing in builtins.pyi aren't available |
| by default (without importing them). We can only do this after processing |
| the whole SCC is finished, when the imported names aren't needed for |
| processing builtins.pyi itself. |
| """ |
| assert state.tree is not None |
| remove_imported_names_from_symtable(state.tree.names, "builtins") |
| |
| |
| def semantic_analysis_for_targets( |
| state: State, nodes: list[FineGrainedDeferredNode], graph: Graph, saved_attrs: SavedAttributes |
| ) -> None: |
| """Semantically analyze only selected nodes in a given module. |
| |
| This essentially mirrors the logic of semantic_analysis_for_scc() |
| except that we process only some targets. This is used in fine grained |
| incremental mode, when propagating an update. |
| |
| The saved_attrs are implicitly declared instance attributes (attributes |
| defined on self) removed by AST stripper that may need to be reintroduced |
| here. They must be added before any methods are analyzed. |
| """ |
| patches: Patches = [] |
| if any(isinstance(n.node, MypyFile) for n in nodes): |
| # Process module top level first (if needed). |
| process_top_levels(graph, [state.id], patches) |
| restore_saved_attrs(saved_attrs) |
| analyzer = state.manager.semantic_analyzer |
| for n in nodes: |
| if isinstance(n.node, MypyFile): |
| # Already done above. |
| continue |
| process_top_level_function( |
| analyzer, state, state.id, n.node.fullname, n.node, n.active_typeinfo, patches |
| ) |
| apply_semantic_analyzer_patches(patches) |
| apply_class_plugin_hooks(graph, [state.id], state.manager.errors) |
| check_type_arguments_in_targets(nodes, state, state.manager.errors) |
| calculate_class_properties(graph, [state.id], state.manager.errors) |
| |
| |
| def restore_saved_attrs(saved_attrs: SavedAttributes) -> None: |
| """Restore instance variables removed during AST strip that haven't been added yet.""" |
| for (cdef, name), sym in saved_attrs.items(): |
| info = cdef.info |
| existing = info.get(name) |
| defined_in_this_class = name in info.names |
| assert isinstance(sym.node, Var) |
| # This needs to mimic the logic in SemanticAnalyzer.analyze_member_lvalue() |
| # regarding the existing variable in class body or in a superclass: |
| # If the attribute of self is not defined in superclasses, create a new Var. |
| if ( |
| existing is None |
| or |
| # (An abstract Var is considered as not defined.) |
| (isinstance(existing.node, Var) and existing.node.is_abstract_var) |
| or |
| # Also an explicit declaration on self creates a new Var unless |
| # there is already one defined in the class body. |
| sym.node.explicit_self_type |
| and not defined_in_this_class |
| ): |
| info.names[name] = sym |
| |
| |
| def process_top_levels(graph: Graph, scc: list[str], patches: Patches) -> None: |
| # Process top levels until everything has been bound. |
| |
| # Reverse order of the scc so the first modules in the original list will be |
| # be processed first. This helps with performance. |
| scc = list(reversed(scc)) |
| |
| # Initialize ASTs and symbol tables. |
| for id in scc: |
| state = graph[id] |
| assert state.tree is not None |
| state.manager.semantic_analyzer.prepare_file(state.tree) |
| |
| # Initially all namespaces in the SCC are incomplete (well they are empty). |
| state.manager.incomplete_namespaces.update(scc) |
| |
| worklist = scc[:] |
| # HACK: process core stuff first. This is mostly needed to support defining |
| # named tuples in builtin SCC. |
| if all(m in worklist for m in core_modules): |
| worklist += list(reversed(core_modules)) * CORE_WARMUP |
| final_iteration = False |
| iteration = 0 |
| analyzer = state.manager.semantic_analyzer |
| analyzer.deferral_debug_context.clear() |
| |
| while worklist: |
| iteration += 1 |
| if iteration > MAX_ITERATIONS: |
| # Just pick some module inside the current SCC for error context. |
| assert state.tree is not None |
| with analyzer.file_context(state.tree, state.options): |
| analyzer.report_hang() |
| break |
| if final_iteration: |
| # Give up. It's impossible to bind all names. |
| state.manager.incomplete_namespaces.clear() |
| all_deferred: list[str] = [] |
| any_progress = False |
| while worklist: |
| next_id = worklist.pop() |
| state = graph[next_id] |
| assert state.tree is not None |
| deferred, incomplete, progress = semantic_analyze_target( |
| next_id, state, state.tree, None, final_iteration, patches |
| ) |
| all_deferred += deferred |
| any_progress = any_progress or progress |
| if not incomplete: |
| state.manager.incomplete_namespaces.discard(next_id) |
| if final_iteration: |
| assert not all_deferred, "Must not defer during final iteration" |
| # Reverse to process the targets in the same order on every iteration. This avoids |
| # processing the same target twice in a row, which is inefficient. |
| worklist = list(reversed(all_deferred)) |
| final_iteration = not any_progress |
| |
| |
| def process_functions(graph: Graph, scc: list[str], patches: Patches) -> None: |
| # Process functions. |
| for module in scc: |
| tree = graph[module].tree |
| assert tree is not None |
| analyzer = graph[module].manager.semantic_analyzer |
| # In principle, functions can be processed in arbitrary order, |
| # but _methods_ must be processed in the order they are defined, |
| # because some features (most notably partial types) depend on |
| # order of definitions on self. |
| # |
| # There can be multiple generated methods per line. Use target |
| # name as the second sort key to get a repeatable sort order on |
| # Python 3.5, which doesn't preserve dictionary order. |
| targets = sorted(get_all_leaf_targets(tree), key=lambda x: (x[1].line, x[0])) |
| for target, node, active_type in targets: |
| assert isinstance(node, (FuncDef, OverloadedFuncDef, Decorator)) |
| process_top_level_function( |
| analyzer, graph[module], module, target, node, active_type, patches |
| ) |
| |
| |
| def process_top_level_function( |
| analyzer: SemanticAnalyzer, |
| state: State, |
| module: str, |
| target: str, |
| node: FuncDef | OverloadedFuncDef | Decorator, |
| active_type: TypeInfo | None, |
| patches: Patches, |
| ) -> None: |
| """Analyze single top-level function or method. |
| |
| Process the body of the function (including nested functions) again and again, |
| until all names have been resolved (or iteration limit reached). |
| """ |
| # We need one more iteration after incomplete is False (e.g. to report errors, if any). |
| final_iteration = False |
| incomplete = True |
| # Start in the incomplete state (no missing names will be reported on first pass). |
| # Note that we use module name, since functions don't create qualified names. |
| deferred = [module] |
| analyzer.deferral_debug_context.clear() |
| analyzer.incomplete_namespaces.add(module) |
| iteration = 0 |
| while deferred: |
| iteration += 1 |
| if iteration == MAX_ITERATIONS: |
| # Just pick some module inside the current SCC for error context. |
| assert state.tree is not None |
| with analyzer.file_context(state.tree, state.options): |
| analyzer.report_hang() |
| break |
| if not (deferred or incomplete) or final_iteration: |
| # OK, this is one last pass, now missing names will be reported. |
| analyzer.incomplete_namespaces.discard(module) |
| deferred, incomplete, progress = semantic_analyze_target( |
| target, state, node, active_type, final_iteration, patches |
| ) |
| if final_iteration: |
| assert not deferred, "Must not defer during final iteration" |
| if not progress: |
| final_iteration = True |
| |
| analyzer.incomplete_namespaces.discard(module) |
| # After semantic analysis is done, discard local namespaces |
| # to avoid memory hoarding. |
| analyzer.saved_locals.clear() |
| |
| |
| TargetInfo: _TypeAlias = Tuple[ |
| str, Union[MypyFile, FuncDef, OverloadedFuncDef, Decorator], Optional[TypeInfo] |
| ] |
| |
| |
| def get_all_leaf_targets(file: MypyFile) -> list[TargetInfo]: |
| """Return all leaf targets in a symbol table (module-level and methods).""" |
| result: list[TargetInfo] = [] |
| for fullname, node, active_type in file.local_definitions(): |
| if isinstance(node.node, (FuncDef, OverloadedFuncDef, Decorator)): |
| result.append((fullname, node.node, active_type)) |
| return result |
| |
| |
| def semantic_analyze_target( |
| target: str, |
| state: State, |
| node: MypyFile | FuncDef | OverloadedFuncDef | Decorator, |
| active_type: TypeInfo | None, |
| final_iteration: bool, |
| patches: Patches, |
| ) -> tuple[list[str], bool, bool]: |
| """Semantically analyze a single target. |
| |
| Return tuple with these items: |
| - list of deferred targets |
| - was some definition incomplete (need to run another pass) |
| - were any new names defined (or placeholders replaced) |
| """ |
| state.manager.processed_targets.append(target) |
| tree = state.tree |
| assert tree is not None |
| analyzer = state.manager.semantic_analyzer |
| # TODO: Move initialization to somewhere else |
| analyzer.global_decls = [set()] |
| analyzer.nonlocal_decls = [set()] |
| analyzer.globals = tree.names |
| analyzer.progress = False |
| with state.wrap_context(check_blockers=False): |
| refresh_node = node |
| if isinstance(refresh_node, Decorator): |
| # Decorator expressions will be processed as part of the module top level. |
| refresh_node = refresh_node.func |
| analyzer.refresh_partial( |
| refresh_node, |
| patches, |
| final_iteration, |
| file_node=tree, |
| options=state.options, |
| active_type=active_type, |
| ) |
| if isinstance(node, Decorator): |
| infer_decorator_signature_if_simple(node, analyzer) |
| for dep in analyzer.imports: |
| state.add_dependency(dep) |
| priority = mypy.build.PRI_LOW |
| if priority <= state.priorities.get(dep, priority): |
| state.priorities[dep] = priority |
| |
| # Clear out some stale data to avoid memory leaks and astmerge |
| # validity check confusion |
| analyzer.statement = None |
| del analyzer.cur_mod_node |
| |
| if analyzer.deferred: |
| return [target], analyzer.incomplete, analyzer.progress |
| else: |
| return [], analyzer.incomplete, analyzer.progress |
| |
| |
| def check_type_arguments(graph: Graph, scc: list[str], errors: Errors) -> None: |
| for module in scc: |
| state = graph[module] |
| assert state.tree |
| analyzer = TypeArgumentAnalyzer( |
| errors, |
| state.options, |
| is_typeshed_file(state.options.abs_custom_typeshed_dir, state.path or ""), |
| ) |
| with state.wrap_context(): |
| with mypy.state.state.strict_optional_set(state.options.strict_optional): |
| state.tree.accept(analyzer) |
| |
| |
| def check_type_arguments_in_targets( |
| targets: list[FineGrainedDeferredNode], state: State, errors: Errors |
| ) -> None: |
| """Check type arguments against type variable bounds and restrictions. |
| |
| This mirrors the logic in check_type_arguments() except that we process only |
| some targets. This is used in fine grained incremental mode. |
| """ |
| analyzer = TypeArgumentAnalyzer( |
| errors, |
| state.options, |
| is_typeshed_file(state.options.abs_custom_typeshed_dir, state.path or ""), |
| ) |
| with state.wrap_context(): |
| with mypy.state.state.strict_optional_set(state.options.strict_optional): |
| for target in targets: |
| func: FuncDef | OverloadedFuncDef | None = None |
| if isinstance(target.node, (FuncDef, OverloadedFuncDef)): |
| func = target.node |
| saved = (state.id, target.active_typeinfo, func) # module, class, function |
| with errors.scope.saved_scope(saved) if errors.scope else nullcontext(): |
| analyzer.recurse_into_functions = func is not None |
| target.node.accept(analyzer) |
| |
| |
| def apply_class_plugin_hooks(graph: Graph, scc: list[str], errors: Errors) -> None: |
| """Apply class plugin hooks within a SCC. |
| |
| We run these after to the main semantic analysis so that the hooks |
| don't need to deal with incomplete definitions such as placeholder |
| types. |
| |
| Note that some hooks incorrectly run during the main semantic |
| analysis pass, for historical reasons. |
| """ |
| num_passes = 0 |
| incomplete = True |
| # If we encounter a base class that has not been processed, we'll run another |
| # pass. This should eventually reach a fixed point. |
| while incomplete: |
| assert num_passes < 10, "Internal error: too many class plugin hook passes" |
| num_passes += 1 |
| incomplete = False |
| for module in scc: |
| state = graph[module] |
| tree = state.tree |
| assert tree |
| for _, node, _ in tree.local_definitions(): |
| if isinstance(node.node, TypeInfo): |
| if not apply_hooks_to_class( |
| state.manager.semantic_analyzer, |
| module, |
| node.node, |
| state.options, |
| tree, |
| errors, |
| ): |
| incomplete = True |
| |
| |
| def apply_hooks_to_class( |
| self: SemanticAnalyzer, |
| module: str, |
| info: TypeInfo, |
| options: Options, |
| file_node: MypyFile, |
| errors: Errors, |
| ) -> bool: |
| # TODO: Move more class-related hooks here? |
| defn = info.defn |
| ok = True |
| for decorator in defn.decorators: |
| with self.file_context(file_node, options, info): |
| hook = None |
| |
| decorator_name = self.get_fullname_for_hook(decorator) |
| if decorator_name: |
| hook = self.plugin.get_class_decorator_hook_2(decorator_name) |
| # Special case: if the decorator is itself decorated with |
| # typing.dataclass_transform, apply the hook for the dataclasses plugin |
| # TODO: remove special casing here |
| if hook is None and is_dataclass_transform_decorator(decorator): |
| hook = dataclasses_plugin.dataclass_class_maker_callback |
| |
| if hook: |
| ok = ok and hook(ClassDefContext(defn, decorator, self)) |
| return ok |
| |
| |
| def calculate_class_properties(graph: Graph, scc: list[str], errors: Errors) -> None: |
| builtins = graph["builtins"].tree |
| assert builtins |
| for module in scc: |
| state = graph[module] |
| tree = state.tree |
| assert tree |
| for _, node, _ in tree.local_definitions(): |
| if isinstance(node.node, TypeInfo): |
| with state.manager.semantic_analyzer.file_context(tree, state.options, node.node): |
| calculate_class_abstract_status(node.node, tree.is_stub, errors) |
| check_protocol_status(node.node, errors) |
| calculate_class_vars(node.node) |
| add_type_promotion( |
| node.node, tree.names, graph[module].options, builtins.names |
| ) |
| |
| |
| def check_blockers(graph: Graph, scc: list[str]) -> None: |
| for module in scc: |
| graph[module].check_blockers() |