| # Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html |
| # For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE |
| # Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt |
| |
| from __future__ import annotations |
| |
| import abc |
| import enum |
| import importlib |
| import importlib.machinery |
| import importlib.util |
| import os |
| import pathlib |
| import sys |
| import types |
| import warnings |
| import zipimport |
| from collections.abc import Iterator, Sequence |
| from functools import lru_cache |
| from pathlib import Path |
| from typing import Any, Literal, NamedTuple, Protocol |
| |
| from astroid.const import PY310_PLUS |
| from astroid.modutils import EXT_LIB_DIRS |
| |
| from . import util |
| |
| |
| # The MetaPathFinder protocol comes from typeshed, which says: |
| # Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder` |
| class _MetaPathFinder(Protocol): |
| def find_spec( |
| self, |
| fullname: str, |
| path: Sequence[str] | None, |
| target: types.ModuleType | None = ..., |
| ) -> importlib.machinery.ModuleSpec | None: ... # pragma: no cover |
| |
| |
| class ModuleType(enum.Enum): |
| """Python module types used for ModuleSpec.""" |
| |
| C_BUILTIN = enum.auto() |
| C_EXTENSION = enum.auto() |
| PKG_DIRECTORY = enum.auto() |
| PY_CODERESOURCE = enum.auto() |
| PY_COMPILED = enum.auto() |
| PY_FROZEN = enum.auto() |
| PY_RESOURCE = enum.auto() |
| PY_SOURCE = enum.auto() |
| PY_ZIPMODULE = enum.auto() |
| PY_NAMESPACE = enum.auto() |
| |
| |
| _MetaPathFinderModuleTypes: dict[str, ModuleType] = { |
| # Finders created by setuptools editable installs |
| "_EditableFinder": ModuleType.PY_SOURCE, |
| "_EditableNamespaceFinder": ModuleType.PY_NAMESPACE, |
| # Finders create by six |
| "_SixMetaPathImporter": ModuleType.PY_SOURCE, |
| } |
| |
| _EditableFinderClasses: set[str] = { |
| "_EditableFinder", |
| "_EditableNamespaceFinder", |
| } |
| |
| |
| class ModuleSpec(NamedTuple): |
| """Defines a class similar to PEP 420's ModuleSpec. |
| |
| A module spec defines a name of a module, its type, location |
| and where submodules can be found, if the module is a package. |
| """ |
| |
| name: str |
| type: ModuleType | None |
| location: str | None = None |
| origin: str | None = None |
| submodule_search_locations: Sequence[str] | None = None |
| |
| |
| class Finder: |
| """A finder is a class which knows how to find a particular module.""" |
| |
| def __init__(self, path: Sequence[str] | None = None) -> None: |
| self._path = path or sys.path |
| |
| @abc.abstractmethod |
| def find_module( |
| self, |
| modname: str, |
| module_parts: Sequence[str], |
| processed: list[str], |
| submodule_path: Sequence[str] | None, |
| ) -> ModuleSpec | None: |
| """Find the given module. |
| |
| Each finder is responsible for each protocol of finding, as long as |
| they all return a ModuleSpec. |
| |
| :param modname: The module which needs to be searched. |
| :param module_parts: It should be a list of strings, |
| where each part contributes to the module's |
| namespace. |
| :param processed: What parts from the module parts were processed |
| so far. |
| :param submodule_path: A list of paths where the module |
| can be looked into. |
| :returns: A ModuleSpec, describing how and where the module was found, |
| None, otherwise. |
| """ |
| |
| def contribute_to_path( |
| self, spec: ModuleSpec, processed: list[str] |
| ) -> Sequence[str] | None: |
| """Get a list of extra paths where this finder can search.""" |
| |
| |
| class ImportlibFinder(Finder): |
| """A finder based on the importlib module.""" |
| |
| _SUFFIXES: Sequence[tuple[str, ModuleType]] = ( |
| [(s, ModuleType.C_EXTENSION) for s in importlib.machinery.EXTENSION_SUFFIXES] |
| + [(s, ModuleType.PY_SOURCE) for s in importlib.machinery.SOURCE_SUFFIXES] |
| + [(s, ModuleType.PY_COMPILED) for s in importlib.machinery.BYTECODE_SUFFIXES] |
| ) |
| |
| def find_module( |
| self, |
| modname: str, |
| module_parts: Sequence[str], |
| processed: list[str], |
| submodule_path: Sequence[str] | None, |
| ) -> ModuleSpec | None: |
| if submodule_path is not None: |
| submodule_path = list(submodule_path) |
| elif modname in sys.builtin_module_names: |
| return ModuleSpec( |
| name=modname, |
| location=None, |
| type=ModuleType.C_BUILTIN, |
| ) |
| else: |
| try: |
| with warnings.catch_warnings(): |
| warnings.filterwarnings("ignore", category=UserWarning) |
| spec = importlib.util.find_spec(modname) |
| if ( |
| spec |
| and spec.loader # type: ignore[comparison-overlap] # noqa: E501 |
| is importlib.machinery.FrozenImporter |
| ): |
| # No need for BuiltinImporter; builtins handled above |
| return ModuleSpec( |
| name=modname, |
| location=getattr(spec.loader_state, "filename", None), |
| type=ModuleType.PY_FROZEN, |
| ) |
| except ValueError: |
| pass |
| submodule_path = sys.path |
| |
| for entry in submodule_path: |
| package_directory = os.path.join(entry, modname) |
| for suffix in (".py", ".pyi", importlib.machinery.BYTECODE_SUFFIXES[0]): |
| package_file_name = "__init__" + suffix |
| file_path = os.path.join(package_directory, package_file_name) |
| if os.path.isfile(file_path): |
| return ModuleSpec( |
| name=modname, |
| location=package_directory, |
| type=ModuleType.PKG_DIRECTORY, |
| ) |
| for suffix, type_ in ImportlibFinder._SUFFIXES: |
| file_name = modname + suffix |
| file_path = os.path.join(entry, file_name) |
| if os.path.isfile(file_path): |
| return ModuleSpec(name=modname, location=file_path, type=type_) |
| return None |
| |
| def contribute_to_path( |
| self, spec: ModuleSpec, processed: list[str] |
| ) -> Sequence[str] | None: |
| if spec.location is None: |
| # Builtin. |
| return None |
| |
| if _is_setuptools_namespace(Path(spec.location)): |
| # extend_path is called, search sys.path for module/packages |
| # of this name see pkgutil.extend_path documentation |
| path = [ |
| os.path.join(p, *processed) |
| for p in sys.path |
| if os.path.isdir(os.path.join(p, *processed)) |
| ] |
| elif spec.name == "distutils" and not any( |
| spec.location.lower().startswith(ext_lib_dir.lower()) |
| for ext_lib_dir in EXT_LIB_DIRS |
| ): |
| # virtualenv below 20.0 patches distutils in an unexpected way |
| # so we just find the location of distutils that will be |
| # imported to avoid spurious import-error messages |
| # https://github.com/pylint-dev/pylint/issues/5645 |
| # A regression test to create this scenario exists in release-tests.yml |
| # and can be triggered manually from GitHub Actions |
| distutils_spec = importlib.util.find_spec("distutils") |
| if distutils_spec and distutils_spec.origin: |
| origin_path = Path( |
| distutils_spec.origin |
| ) # e.g. .../distutils/__init__.py |
| path = [str(origin_path.parent)] # e.g. .../distutils |
| else: |
| path = [spec.location] |
| else: |
| path = [spec.location] |
| return path |
| |
| |
| class ExplicitNamespacePackageFinder(ImportlibFinder): |
| """A finder for the explicit namespace packages.""" |
| |
| def find_module( |
| self, |
| modname: str, |
| module_parts: Sequence[str], |
| processed: list[str], |
| submodule_path: Sequence[str] | None, |
| ) -> ModuleSpec | None: |
| if processed: |
| modname = ".".join([*processed, modname]) |
| if util.is_namespace(modname) and modname in sys.modules: |
| submodule_path = sys.modules[modname].__path__ |
| return ModuleSpec( |
| name=modname, |
| location="", |
| origin="namespace", |
| type=ModuleType.PY_NAMESPACE, |
| submodule_search_locations=submodule_path, |
| ) |
| return None |
| |
| def contribute_to_path( |
| self, spec: ModuleSpec, processed: list[str] |
| ) -> Sequence[str] | None: |
| return spec.submodule_search_locations |
| |
| |
| class ZipFinder(Finder): |
| """Finder that knows how to find a module inside zip files.""" |
| |
| def __init__(self, path: Sequence[str]) -> None: |
| super().__init__(path) |
| for entry_path in path: |
| if entry_path not in sys.path_importer_cache: |
| try: |
| sys.path_importer_cache[entry_path] = zipimport.zipimporter( # type: ignore[assignment] |
| entry_path |
| ) |
| except zipimport.ZipImportError: |
| continue |
| |
| def find_module( |
| self, |
| modname: str, |
| module_parts: Sequence[str], |
| processed: list[str], |
| submodule_path: Sequence[str] | None, |
| ) -> ModuleSpec | None: |
| try: |
| file_type, filename, path = _search_zip(module_parts) |
| except ImportError: |
| return None |
| |
| return ModuleSpec( |
| name=modname, |
| location=filename, |
| origin="egg", |
| type=file_type, |
| submodule_search_locations=path, |
| ) |
| |
| |
| class PathSpecFinder(Finder): |
| """Finder based on importlib.machinery.PathFinder.""" |
| |
| def find_module( |
| self, |
| modname: str, |
| module_parts: Sequence[str], |
| processed: list[str], |
| submodule_path: Sequence[str] | None, |
| ) -> ModuleSpec | None: |
| spec = importlib.machinery.PathFinder.find_spec(modname, path=submodule_path) |
| if spec is not None: |
| is_namespace_pkg = spec.origin is None |
| location = spec.origin if not is_namespace_pkg else None |
| module_type = ModuleType.PY_NAMESPACE if is_namespace_pkg else None |
| return ModuleSpec( |
| name=spec.name, |
| location=location, |
| origin=spec.origin, |
| type=module_type, |
| submodule_search_locations=list(spec.submodule_search_locations or []), |
| ) |
| return spec |
| |
| def contribute_to_path( |
| self, spec: ModuleSpec, processed: list[str] |
| ) -> Sequence[str] | None: |
| if spec.type == ModuleType.PY_NAMESPACE: |
| return spec.submodule_search_locations |
| return None |
| |
| |
| _SPEC_FINDERS = ( |
| ImportlibFinder, |
| ZipFinder, |
| PathSpecFinder, |
| ExplicitNamespacePackageFinder, |
| ) |
| |
| |
| def _is_setuptools_namespace(location: pathlib.Path) -> bool: |
| try: |
| with open(location / "__init__.py", "rb") as stream: |
| data = stream.read(4096) |
| except OSError: |
| return False |
| extend_path = b"pkgutil" in data and b"extend_path" in data |
| declare_namespace = ( |
| b"pkg_resources" in data and b"declare_namespace(__name__)" in data |
| ) |
| return extend_path or declare_namespace |
| |
| |
| def _get_zipimporters() -> Iterator[tuple[str, zipimport.zipimporter]]: |
| for filepath, importer in sys.path_importer_cache.items(): |
| if isinstance(importer, zipimport.zipimporter): |
| yield filepath, importer |
| |
| |
| def _search_zip( |
| modpath: Sequence[str], |
| ) -> tuple[Literal[ModuleType.PY_ZIPMODULE], str, str]: |
| for filepath, importer in _get_zipimporters(): |
| if PY310_PLUS: |
| found: Any = importer.find_spec(modpath[0]) |
| else: |
| found = importer.find_module(modpath[0]) |
| if found: |
| if PY310_PLUS: |
| if not importer.find_spec(os.path.sep.join(modpath)): |
| raise ImportError( |
| "No module named %s in %s/%s" |
| % (".".join(modpath[1:]), filepath, modpath) |
| ) |
| elif not importer.find_module(os.path.sep.join(modpath)): |
| raise ImportError( |
| "No module named %s in %s/%s" |
| % (".".join(modpath[1:]), filepath, modpath) |
| ) |
| return ( |
| ModuleType.PY_ZIPMODULE, |
| os.path.abspath(filepath) + os.path.sep + os.path.sep.join(modpath), |
| filepath, |
| ) |
| raise ImportError(f"No module named {'.'.join(modpath)}") |
| |
| |
| def _find_spec_with_path( |
| search_path: Sequence[str], |
| modname: str, |
| module_parts: list[str], |
| processed: list[str], |
| submodule_path: Sequence[str] | None, |
| ) -> tuple[Finder | _MetaPathFinder, ModuleSpec]: |
| for finder in _SPEC_FINDERS: |
| finder_instance = finder(search_path) |
| spec = finder_instance.find_module( |
| modname, module_parts, processed, submodule_path |
| ) |
| if spec is None: |
| continue |
| return finder_instance, spec |
| |
| # Support for custom finders |
| for meta_finder in sys.meta_path: |
| # See if we support the customer import hook of the meta_finder |
| meta_finder_name = meta_finder.__class__.__name__ |
| if meta_finder_name not in _MetaPathFinderModuleTypes: |
| # Setuptools>62 creates its EditableFinders dynamically and have |
| # "type" as their __class__.__name__. We check __name__ as well |
| # to see if we can support the finder. |
| try: |
| meta_finder_name = meta_finder.__name__ # type: ignore[attr-defined] |
| except AttributeError: |
| continue |
| if meta_finder_name not in _MetaPathFinderModuleTypes: |
| continue |
| |
| module_type = _MetaPathFinderModuleTypes[meta_finder_name] |
| |
| # Meta path finders are supposed to have a find_spec method since |
| # Python 3.4. However, some third-party finders do not implement it. |
| # PEP302 does not refer to find_spec as well. |
| # See: https://github.com/pylint-dev/astroid/pull/1752/ |
| if not hasattr(meta_finder, "find_spec"): |
| continue |
| |
| spec = meta_finder.find_spec(modname, submodule_path) |
| if spec: |
| return ( |
| meta_finder, |
| ModuleSpec( |
| spec.name, |
| module_type, |
| spec.origin, |
| spec.origin, |
| spec.submodule_search_locations, |
| ), |
| ) |
| |
| raise ImportError(f"No module named {'.'.join(module_parts)}") |
| |
| |
| def find_spec(modpath: list[str], path: Sequence[str] | None = None) -> ModuleSpec: |
| """Find a spec for the given module. |
| |
| :type modpath: list or tuple |
| :param modpath: |
| split module's name (i.e name of a module or package split |
| on '.'), with leading empty strings for explicit relative import |
| |
| :type path: list or None |
| :param path: |
| optional list of path where the module or package should be |
| searched (use sys.path if nothing or None is given) |
| |
| :rtype: ModuleSpec |
| :return: A module spec, which describes how the module was |
| found and where. |
| """ |
| return _find_spec(tuple(modpath), tuple(path) if path else None) |
| |
| |
| @lru_cache(maxsize=1024) |
| def _find_spec(modpath: tuple, path: tuple) -> ModuleSpec: |
| _path = path or sys.path |
| |
| # Need a copy for not mutating the argument. |
| modpath = list(modpath) |
| |
| submodule_path = None |
| module_parts = modpath[:] |
| processed: list[str] = [] |
| |
| while modpath: |
| modname = modpath.pop(0) |
| finder, spec = _find_spec_with_path( |
| _path, modname, module_parts, processed, submodule_path or path |
| ) |
| processed.append(modname) |
| if modpath: |
| if isinstance(finder, Finder): |
| submodule_path = finder.contribute_to_path(spec, processed) |
| # If modname is a package from an editable install, update submodule_path |
| # so that the next module in the path will be found inside of it using importlib. |
| # Existence of __name__ is guaranteed by _find_spec_with_path. |
| elif finder.__name__ in _EditableFinderClasses: # type: ignore[attr-defined] |
| submodule_path = spec.submodule_search_locations |
| |
| if spec.type == ModuleType.PKG_DIRECTORY: |
| spec = spec._replace(submodule_search_locations=submodule_path) |
| |
| return spec |