| """Routines for finding the sources that mypy will check""" | |
| from __future__ import annotations | |
| import functools | |
| import os | |
| from collections.abc import Mapping, Sequence | |
| from typing import Final | |
| from mypy.fscache import FileSystemCache | |
| from mypy.modulefinder import ( | |
| PYTHON_EXTENSIONS, | |
| BuildSource, | |
| matches_exclude, | |
| matches_gitignore, | |
| mypy_path, | |
| ) | |
| from mypy.options import Options | |
| PY_EXTENSIONS: Final = tuple(PYTHON_EXTENSIONS) | |
| class InvalidSourceList(Exception): | |
| """Exception indicating a problem in the list of sources given to mypy.""" | |
| def create_source_list( | |
| paths: Sequence[str], | |
| options: Options, | |
| fscache: FileSystemCache | None = None, | |
| allow_empty_dir: bool = False, | |
| known_modules: Mapping[str, str] | None = None, | |
| ) -> list[BuildSource]: | |
| """From a list of source files/directories, makes a list of BuildSources. | |
| known_modules maps files to their modules, known already, as from a cache: such a | |
| file's module and base directory come from it, without crawling up its directories. | |
| Raises InvalidSourceList on errors. | |
| """ | |
| fscache = fscache or FileSystemCache() | |
| finder = SourceFinder(fscache, options) | |
| sources = [] | |
| for path in paths: | |
| path = os.path.normpath(path) | |
| if path.endswith(PY_EXTENSIONS): | |
| module = known_modules.get(path) if known_modules else None | |
| base_dir = base_dir_of(path, module) if module is not None else None | |
| if module is not None and base_dir is not None: | |
| sources.append(BuildSource(path, module, None, base_dir)) | |
| continue | |
| # Can raise InvalidSourceList if a directory doesn't have a valid module name. | |
| name, base_dir = finder.crawl_up(path) | |
| sources.append(BuildSource(path, name, None, base_dir)) | |
| elif fscache.isdir(path): | |
| sub_sources = finder.find_sources_in_dir(path) | |
| if not sub_sources and not allow_empty_dir: | |
| raise InvalidSourceList(f"There are no .py[i] files in directory '{path}'") | |
| sources.extend(sub_sources) | |
| else: | |
| mod = os.path.basename(path) if options.scripts_are_modules else None | |
| sources.append(BuildSource(path, mod, None)) | |
| return sources | |
| def base_dir_of(path: str, module: str) -> str | None: | |
| """The base directory crawl_up would find for a file holding module, without looking. | |
| Each component of a module but its last is a directory above the file, so the base | |
| directory is as many directories up. None if the path's directories do not match the | |
| module's components. | |
| """ | |
| dir = os.path.dirname(os.path.abspath(path)) | |
| components = module.split(".") | |
| stem = strip_py(os.path.basename(path)) | |
| if stem == "__init__": | |
| packages = components | |
| elif components[-1] == stem: | |
| packages = components[:-1] | |
| else: | |
| return None | |
| for package in reversed(packages): | |
| dir, name = os.path.split(dir) | |
| if name.removesuffix("-stubs") != package: | |
| return None | |
| return dir | |
| def keyfunc(name: str) -> tuple[bool, int, str]: | |
| """Determines sort order for directory listing. | |
| The desirable properties are: | |
| 1) foo < foo.pyi < foo.py | |
| 2) __init__.py[i] < foo | |
| """ | |
| base, suffix = os.path.splitext(name) | |
| for i, ext in enumerate(PY_EXTENSIONS): | |
| if suffix == ext: | |
| return (base != "__init__", i, base) | |
| return (base != "__init__", -1, name) | |
| def normalise_package_base(root: str) -> str: | |
| if not root: | |
| root = os.curdir | |
| root = os.path.abspath(root) | |
| if root.endswith(os.sep): | |
| root = root[:-1] | |
| return root | |
| def get_explicit_package_bases(options: Options) -> list[str] | None: | |
| """Returns explicit package bases to use if the option is enabled, or None if disabled. | |
| We currently use MYPYPATH and the current directory as the package bases. In the future, | |
| when --namespace-packages is the default could also use the values passed with the | |
| --package-root flag, see #9632. | |
| Values returned are normalised so we can use simple string comparisons in | |
| SourceFinder.is_explicit_package_base | |
| """ | |
| if not options.explicit_package_bases: | |
| return None | |
| roots = mypy_path() + options.mypy_path + [os.getcwd()] | |
| return [normalise_package_base(root) for root in roots] | |
| class SourceFinder: | |
| def __init__(self, fscache: FileSystemCache, options: Options) -> None: | |
| self.fscache = fscache | |
| self.explicit_package_bases = get_explicit_package_bases(options) | |
| self.namespace_packages = options.namespace_packages | |
| self.exclude = options.exclude | |
| self.exclude_gitignore = options.exclude_gitignore | |
| self.verbosity = options.verbosity | |
| def is_explicit_package_base(self, path: str) -> bool: | |
| assert self.explicit_package_bases | |
| return normalise_package_base(path) in self.explicit_package_bases | |
| def find_sources_in_dir(self, path: str) -> list[BuildSource]: | |
| sources = [] | |
| seen: set[str] = set() | |
| names = sorted(self.fscache.listdir(path), key=keyfunc) | |
| for name in names: | |
| # Skip certain names altogether | |
| if name in ("__pycache__", "site-packages", "node_modules") or name.startswith("."): | |
| continue | |
| subpath = os.path.join(path, name) | |
| if matches_exclude(subpath, self.exclude, self.fscache, self.verbosity >= 2): | |
| continue | |
| if self.exclude_gitignore and matches_gitignore( | |
| subpath, self.fscache, self.verbosity >= 2 | |
| ): | |
| continue | |
| if self.fscache.isdir(subpath): | |
| sub_sources = self.find_sources_in_dir(subpath) | |
| if sub_sources: | |
| seen.add(name) | |
| sources.extend(sub_sources) | |
| else: | |
| stem, suffix = os.path.splitext(name) | |
| if stem not in seen and suffix in PY_EXTENSIONS: | |
| seen.add(stem) | |
| module, base_dir = self.crawl_up(subpath) | |
| sources.append(BuildSource(subpath, module, None, base_dir)) | |
| return sources | |
| def crawl_up(self, path: str) -> tuple[str, str]: | |
| """Given a .py[i] filename, return module and base directory. | |
| For example, given "xxx/yyy/foo/bar.py", we might return something like: | |
| ("foo.bar", "xxx/yyy") | |
| If namespace packages is off, we crawl upwards until we find a directory without | |
| an __init__.py | |
| If namespace packages is on, we crawl upwards until the nearest explicit base directory. | |
| Failing that, we return one past the highest directory containing an __init__.py | |
| We won't crawl past directories with invalid package names. | |
| The base directory returned is an absolute path. | |
| """ | |
| path = os.path.abspath(path) | |
| parent, filename = os.path.split(path) | |
| module_name = strip_py(filename) or filename | |
| parent_module, base_dir = self.crawl_up_dir(parent) | |
| if module_name == "__init__": | |
| return parent_module, base_dir | |
| # Note that module_name might not actually be a valid identifier, but that's okay | |
| # Ignoring this possibility sidesteps some search path confusion | |
| module = module_join(parent_module, module_name) | |
| return module, base_dir | |
| def crawl_up_dir(self, dir: str) -> tuple[str, str]: | |
| return self._crawl_up_helper(dir) or ("", dir) | |
| @functools.lru_cache # noqa: B019 | |
| def _crawl_up_helper(self, dir: str) -> tuple[str, str] | None: | |
| """Given a directory, maybe returns module and base directory. | |
| We return a non-None value if we were able to find something clearly intended as a base | |
| directory (as adjudicated by being an explicit base directory or by containing a package | |
| with __init__.py). | |
| This distinction is necessary for namespace packages, so that we know when to treat | |
| ourselves as a subpackage. | |
| """ | |
| # stop crawling if we're an explicit base directory | |
| if self.explicit_package_bases is not None and self.is_explicit_package_base(dir): | |
| return "", dir | |
| parent, name = os.path.split(dir) | |
| name = name.removesuffix("-stubs") # PEP-561 stub-only directory | |
| # recurse if there's an __init__.py | |
| init_file = self.get_init_file(dir) | |
| if init_file is not None: | |
| if not name.isidentifier(): | |
| # in most cases the directory name is invalid, we'll just stop crawling upwards | |
| # but if there's an __init__.py in the directory, something is messed up | |
| raise InvalidSourceList( | |
| f"{name} contains {os.path.basename(init_file)} " | |
| "but is not a valid Python package name" | |
| ) | |
| # we're definitely a package, so we always return a non-None value | |
| mod_prefix, base_dir = self.crawl_up_dir(parent) | |
| return module_join(mod_prefix, name), base_dir | |
| # stop crawling if we're out of path components or our name is an invalid identifier | |
| if not name or not parent or not name.isidentifier(): | |
| return None | |
| # stop crawling if namespace packages is off (since we don't have an __init__.py) | |
| if not self.namespace_packages: | |
| return None | |
| # at this point: namespace packages is on, we don't have an __init__.py and we're not an | |
| # explicit base directory | |
| result = self._crawl_up_helper(parent) | |
| if result is None: | |
| # we're not an explicit base directory and we don't have an __init__.py | |
| # and none of our parents are either, so return | |
| return None | |
| # one of our parents was an explicit base directory or had an __init__.py, so we're | |
| # definitely a subpackage! chain our name to the module. | |
| mod_prefix, base_dir = result | |
| return module_join(mod_prefix, name), base_dir | |
| def get_init_file(self, dir: str) -> str | None: | |
| """Check whether a directory contains a file named __init__.py[i]. | |
| If so, return the file's name (with dir prefixed). If not, return None. | |
| This prefers .pyi over .py (because of the ordering of PY_EXTENSIONS). | |
| """ | |
| for ext in PY_EXTENSIONS: | |
| f = os.path.join(dir, "__init__" + ext) | |
| if self.fscache.isfile(f): | |
| return f | |
| if ext == ".py" and self.fscache.init_under_package_root(f): | |
| return f | |
| return None | |
| def module_join(parent: str, child: str) -> str: | |
| """Join module ids, accounting for a possibly empty parent.""" | |
| if parent: | |
| return parent + "." + child | |
| return child | |
| def strip_py(arg: str) -> str | None: | |
| """Strip a trailing .py or .pyi suffix. | |
| Return None if no such suffix is found. | |
| """ | |
| for ext in PY_EXTENSIONS: | |
| if arg.endswith(ext): | |
| return arg[: -len(ext)] | |
| return None |