| from __future__ import annotations |
| |
| import argparse |
| import configparser |
| import glob as fileglob |
| import os |
| import re |
| import sys |
| from io import StringIO |
| |
| from mypy.errorcodes import error_codes |
| |
| if sys.version_info >= (3, 11): |
| import tomllib |
| else: |
| import tomli as tomllib |
| |
| from typing import ( |
| Any, |
| Callable, |
| Dict, |
| Final, |
| Iterable, |
| List, |
| Mapping, |
| MutableMapping, |
| Sequence, |
| TextIO, |
| Tuple, |
| Union, |
| ) |
| from typing_extensions import TypeAlias as _TypeAlias |
| |
| from mypy import defaults |
| from mypy.options import PER_MODULE_OPTIONS, Options |
| |
| _CONFIG_VALUE_TYPES: _TypeAlias = Union[ |
| str, bool, int, float, Dict[str, str], List[str], Tuple[int, int] |
| ] |
| _INI_PARSER_CALLABLE: _TypeAlias = Callable[[Any], _CONFIG_VALUE_TYPES] |
| |
| |
| def parse_version(v: str | float) -> tuple[int, int]: |
| m = re.match(r"\A(\d)\.(\d+)\Z", str(v)) |
| if not m: |
| raise argparse.ArgumentTypeError(f"Invalid python version '{v}' (expected format: 'x.y')") |
| major, minor = int(m.group(1)), int(m.group(2)) |
| if major == 2 and minor == 7: |
| pass # Error raised elsewhere |
| elif major == 3: |
| if minor < defaults.PYTHON3_VERSION_MIN[1]: |
| msg = "Python 3.{} is not supported (must be {}.{} or higher)".format( |
| minor, *defaults.PYTHON3_VERSION_MIN |
| ) |
| |
| if isinstance(v, float): |
| msg += ". You may need to put quotes around your Python version" |
| |
| raise argparse.ArgumentTypeError(msg) |
| else: |
| raise argparse.ArgumentTypeError( |
| f"Python major version '{major}' out of range (must be 3)" |
| ) |
| return major, minor |
| |
| |
| def try_split(v: str | Sequence[str], split_regex: str = "[,]") -> list[str]: |
| """Split and trim a str or list of str into a list of str""" |
| if isinstance(v, str): |
| return [p.strip() for p in re.split(split_regex, v)] |
| |
| return [p.strip() for p in v] |
| |
| |
| def validate_codes(codes: list[str]) -> list[str]: |
| invalid_codes = set(codes) - set(error_codes.keys()) |
| if invalid_codes: |
| raise argparse.ArgumentTypeError( |
| f"Invalid error code(s): {', '.join(sorted(invalid_codes))}" |
| ) |
| return codes |
| |
| |
| def expand_path(path: str) -> str: |
| """Expand the user home directory and any environment variables contained within |
| the provided path. |
| """ |
| |
| return os.path.expandvars(os.path.expanduser(path)) |
| |
| |
| def str_or_array_as_list(v: str | Sequence[str]) -> list[str]: |
| if isinstance(v, str): |
| return [v.strip()] if v.strip() else [] |
| return [p.strip() for p in v if p.strip()] |
| |
| |
| def split_and_match_files_list(paths: Sequence[str]) -> list[str]: |
| """Take a list of files/directories (with support for globbing through the glob library). |
| |
| Where a path/glob matches no file, we still include the raw path in the resulting list. |
| |
| Returns a list of file paths |
| """ |
| expanded_paths = [] |
| |
| for path in paths: |
| path = expand_path(path.strip()) |
| globbed_files = fileglob.glob(path, recursive=True) |
| if globbed_files: |
| expanded_paths.extend(globbed_files) |
| else: |
| expanded_paths.append(path) |
| |
| return expanded_paths |
| |
| |
| def split_and_match_files(paths: str) -> list[str]: |
| """Take a string representing a list of files/directories (with support for globbing |
| through the glob library). |
| |
| Where a path/glob matches no file, we still include the raw path in the resulting list. |
| |
| Returns a list of file paths |
| """ |
| |
| return split_and_match_files_list(paths.split(",")) |
| |
| |
| def check_follow_imports(choice: str) -> str: |
| choices = ["normal", "silent", "skip", "error"] |
| if choice not in choices: |
| raise argparse.ArgumentTypeError( |
| "invalid choice '{}' (choose from {})".format( |
| choice, ", ".join(f"'{x}'" for x in choices) |
| ) |
| ) |
| return choice |
| |
| |
| def split_commas(value: str) -> list[str]: |
| # Uses a bit smarter technique to allow last trailing comma |
| # and to remove last `""` item from the split. |
| items = value.split(",") |
| if items and items[-1] == "": |
| items.pop(-1) |
| return items |
| |
| |
| # For most options, the type of the default value set in options.py is |
| # sufficient, and we don't have to do anything here. This table |
| # exists to specify types for values initialized to None or container |
| # types. |
| ini_config_types: Final[dict[str, _INI_PARSER_CALLABLE]] = { |
| "python_version": parse_version, |
| "custom_typing_module": str, |
| "custom_typeshed_dir": expand_path, |
| "mypy_path": lambda s: [expand_path(p.strip()) for p in re.split("[,:]", s)], |
| "files": split_and_match_files, |
| "quickstart_file": expand_path, |
| "junit_xml": expand_path, |
| "follow_imports": check_follow_imports, |
| "no_site_packages": bool, |
| "plugins": lambda s: [p.strip() for p in split_commas(s)], |
| "always_true": lambda s: [p.strip() for p in split_commas(s)], |
| "always_false": lambda s: [p.strip() for p in split_commas(s)], |
| "enable_incomplete_feature": lambda s: [p.strip() for p in split_commas(s)], |
| "disable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]), |
| "enable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]), |
| "package_root": lambda s: [p.strip() for p in split_commas(s)], |
| "cache_dir": expand_path, |
| "python_executable": expand_path, |
| "strict": bool, |
| "exclude": lambda s: [s.strip()], |
| "packages": try_split, |
| "modules": try_split, |
| } |
| |
| # Reuse the ini_config_types and overwrite the diff |
| toml_config_types: Final[dict[str, _INI_PARSER_CALLABLE]] = ini_config_types.copy() |
| toml_config_types.update( |
| { |
| "python_version": parse_version, |
| "mypy_path": lambda s: [expand_path(p) for p in try_split(s, "[,:]")], |
| "files": lambda s: split_and_match_files_list(try_split(s)), |
| "follow_imports": lambda s: check_follow_imports(str(s)), |
| "plugins": try_split, |
| "always_true": try_split, |
| "always_false": try_split, |
| "enable_incomplete_feature": try_split, |
| "disable_error_code": lambda s: validate_codes(try_split(s)), |
| "enable_error_code": lambda s: validate_codes(try_split(s)), |
| "package_root": try_split, |
| "exclude": str_or_array_as_list, |
| "packages": try_split, |
| "modules": try_split, |
| } |
| ) |
| |
| |
| def parse_config_file( |
| options: Options, |
| set_strict_flags: Callable[[], None], |
| filename: str | None, |
| stdout: TextIO | None = None, |
| stderr: TextIO | None = None, |
| ) -> None: |
| """Parse a config file into an Options object. |
| |
| Errors are written to stderr but are not fatal. |
| |
| If filename is None, fall back to default config files. |
| """ |
| stdout = stdout or sys.stdout |
| stderr = stderr or sys.stderr |
| |
| if filename is not None: |
| config_files: tuple[str, ...] = (filename,) |
| else: |
| config_files_iter: Iterable[str] = map(os.path.expanduser, defaults.CONFIG_FILES) |
| config_files = tuple(config_files_iter) |
| |
| config_parser = configparser.RawConfigParser() |
| |
| for config_file in config_files: |
| if not os.path.exists(config_file): |
| continue |
| try: |
| if is_toml(config_file): |
| with open(config_file, "rb") as f: |
| toml_data = tomllib.load(f) |
| # Filter down to just mypy relevant toml keys |
| toml_data = toml_data.get("tool", {}) |
| if "mypy" not in toml_data: |
| continue |
| toml_data = {"mypy": toml_data["mypy"]} |
| parser: MutableMapping[str, Any] = destructure_overrides(toml_data) |
| config_types = toml_config_types |
| else: |
| config_parser.read(config_file) |
| parser = config_parser |
| config_types = ini_config_types |
| except (tomllib.TOMLDecodeError, configparser.Error, ConfigTOMLValueError) as err: |
| print(f"{config_file}: {err}", file=stderr) |
| else: |
| if config_file in defaults.SHARED_CONFIG_FILES and "mypy" not in parser: |
| continue |
| file_read = config_file |
| options.config_file = file_read |
| break |
| else: |
| return |
| |
| os.environ["MYPY_CONFIG_FILE_DIR"] = os.path.dirname(os.path.abspath(config_file)) |
| |
| if "mypy" not in parser: |
| if filename or file_read not in defaults.SHARED_CONFIG_FILES: |
| print(f"{file_read}: No [mypy] section in config file", file=stderr) |
| else: |
| section = parser["mypy"] |
| prefix = f"{file_read}: [mypy]: " |
| updates, report_dirs = parse_section( |
| prefix, options, set_strict_flags, section, config_types, stderr |
| ) |
| for k, v in updates.items(): |
| setattr(options, k, v) |
| options.report_dirs.update(report_dirs) |
| |
| for name, section in parser.items(): |
| if name.startswith("mypy-"): |
| prefix = get_prefix(file_read, name) |
| updates, report_dirs = parse_section( |
| prefix, options, set_strict_flags, section, config_types, stderr |
| ) |
| if report_dirs: |
| print( |
| "%sPer-module sections should not specify reports (%s)" |
| % (prefix, ", ".join(s + "_report" for s in sorted(report_dirs))), |
| file=stderr, |
| ) |
| if set(updates) - PER_MODULE_OPTIONS: |
| print( |
| "%sPer-module sections should only specify per-module flags (%s)" |
| % (prefix, ", ".join(sorted(set(updates) - PER_MODULE_OPTIONS))), |
| file=stderr, |
| ) |
| updates = {k: v for k, v in updates.items() if k in PER_MODULE_OPTIONS} |
| |
| globs = name[5:] |
| for glob in globs.split(","): |
| # For backwards compatibility, replace (back)slashes with dots. |
| glob = glob.replace(os.sep, ".") |
| if os.altsep: |
| glob = glob.replace(os.altsep, ".") |
| |
| if any(c in glob for c in "?[]!") or any( |
| "*" in x and x != "*" for x in glob.split(".") |
| ): |
| print( |
| "%sPatterns must be fully-qualified module names, optionally " |
| "with '*' in some components (e.g spam.*.eggs.*)" % prefix, |
| file=stderr, |
| ) |
| else: |
| options.per_module_options[glob] = updates |
| |
| |
| def get_prefix(file_read: str, name: str) -> str: |
| if is_toml(file_read): |
| module_name_str = 'module = "%s"' % "-".join(name.split("-")[1:]) |
| else: |
| module_name_str = name |
| |
| return f"{file_read}: [{module_name_str}]: " |
| |
| |
| def is_toml(filename: str) -> bool: |
| return filename.lower().endswith(".toml") |
| |
| |
| def destructure_overrides(toml_data: dict[str, Any]) -> dict[str, Any]: |
| """Take the new [[tool.mypy.overrides]] section array in the pyproject.toml file, |
| and convert it back to a flatter structure that the existing config_parser can handle. |
| |
| E.g. the following pyproject.toml file: |
| |
| [[tool.mypy.overrides]] |
| module = [ |
| "a.b", |
| "b.*" |
| ] |
| disallow_untyped_defs = true |
| |
| [[tool.mypy.overrides]] |
| module = 'c' |
| disallow_untyped_defs = false |
| |
| Would map to the following config dict that it would have gotten from parsing an equivalent |
| ini file: |
| |
| { |
| "mypy-a.b": { |
| disallow_untyped_defs = true, |
| }, |
| "mypy-b.*": { |
| disallow_untyped_defs = true, |
| }, |
| "mypy-c": { |
| disallow_untyped_defs: false, |
| }, |
| } |
| """ |
| if "overrides" not in toml_data["mypy"]: |
| return toml_data |
| |
| if not isinstance(toml_data["mypy"]["overrides"], list): |
| raise ConfigTOMLValueError( |
| "tool.mypy.overrides sections must be an array. Please make " |
| "sure you are using double brackets like so: [[tool.mypy.overrides]]" |
| ) |
| |
| result = toml_data.copy() |
| for override in result["mypy"]["overrides"]: |
| if "module" not in override: |
| raise ConfigTOMLValueError( |
| "toml config file contains a [[tool.mypy.overrides]] " |
| "section, but no module to override was specified." |
| ) |
| |
| if isinstance(override["module"], str): |
| modules = [override["module"]] |
| elif isinstance(override["module"], list): |
| modules = override["module"] |
| else: |
| raise ConfigTOMLValueError( |
| "toml config file contains a [[tool.mypy.overrides]] " |
| "section with a module value that is not a string or a list of " |
| "strings" |
| ) |
| |
| for module in modules: |
| module_overrides = override.copy() |
| del module_overrides["module"] |
| old_config_name = f"mypy-{module}" |
| if old_config_name not in result: |
| result[old_config_name] = module_overrides |
| else: |
| for new_key, new_value in module_overrides.items(): |
| if ( |
| new_key in result[old_config_name] |
| and result[old_config_name][new_key] != new_value |
| ): |
| raise ConfigTOMLValueError( |
| "toml config file contains " |
| "[[tool.mypy.overrides]] sections with conflicting " |
| "values. Module '%s' has two different values for '%s'" |
| % (module, new_key) |
| ) |
| result[old_config_name][new_key] = new_value |
| |
| del result["mypy"]["overrides"] |
| return result |
| |
| |
| def parse_section( |
| prefix: str, |
| template: Options, |
| set_strict_flags: Callable[[], None], |
| section: Mapping[str, Any], |
| config_types: dict[str, Any], |
| stderr: TextIO = sys.stderr, |
| ) -> tuple[dict[str, object], dict[str, str]]: |
| """Parse one section of a config file. |
| |
| Returns a dict of option values encountered, and a dict of report directories. |
| """ |
| results: dict[str, object] = {} |
| report_dirs: dict[str, str] = {} |
| for key in section: |
| invert = False |
| options_key = key |
| if key in config_types: |
| ct = config_types[key] |
| else: |
| dv = None |
| # We have to keep new_semantic_analyzer in Options |
| # for plugin compatibility but it is not a valid option anymore. |
| assert hasattr(template, "new_semantic_analyzer") |
| if key != "new_semantic_analyzer": |
| dv = getattr(template, key, None) |
| if dv is None: |
| if key.endswith("_report"): |
| report_type = key[:-7].replace("_", "-") |
| if report_type in defaults.REPORTER_NAMES: |
| report_dirs[report_type] = str(section[key]) |
| else: |
| print(f"{prefix}Unrecognized report type: {key}", file=stderr) |
| continue |
| if key.startswith("x_"): |
| pass # Don't complain about `x_blah` flags |
| elif key.startswith("no_") and hasattr(template, key[3:]): |
| options_key = key[3:] |
| invert = True |
| elif key.startswith("allow") and hasattr(template, "dis" + key): |
| options_key = "dis" + key |
| invert = True |
| elif key.startswith("disallow") and hasattr(template, key[3:]): |
| options_key = key[3:] |
| invert = True |
| elif key.startswith("show_") and hasattr(template, "hide_" + key[5:]): |
| options_key = "hide_" + key[5:] |
| invert = True |
| elif key == "strict": |
| pass # Special handling below |
| else: |
| print(f"{prefix}Unrecognized option: {key} = {section[key]}", file=stderr) |
| if invert: |
| dv = getattr(template, options_key, None) |
| else: |
| continue |
| ct = type(dv) |
| v: Any = None |
| try: |
| if ct is bool: |
| if isinstance(section, dict): |
| v = convert_to_boolean(section.get(key)) |
| else: |
| v = section.getboolean(key) # type: ignore[attr-defined] # Until better stub |
| if invert: |
| v = not v |
| elif callable(ct): |
| if invert: |
| print(f"{prefix}Can not invert non-boolean key {options_key}", file=stderr) |
| continue |
| try: |
| v = ct(section.get(key)) |
| except argparse.ArgumentTypeError as err: |
| print(f"{prefix}{key}: {err}", file=stderr) |
| continue |
| else: |
| print(f"{prefix}Don't know what type {key} should have", file=stderr) |
| continue |
| except ValueError as err: |
| print(f"{prefix}{key}: {err}", file=stderr) |
| continue |
| if key == "strict": |
| if v: |
| set_strict_flags() |
| continue |
| results[options_key] = v |
| |
| # These two flags act as per-module overrides, so store the empty defaults. |
| if "disable_error_code" not in results: |
| results["disable_error_code"] = [] |
| if "enable_error_code" not in results: |
| results["enable_error_code"] = [] |
| |
| return results, report_dirs |
| |
| |
| def convert_to_boolean(value: Any | None) -> bool: |
| """Return a boolean value translating from other types if necessary.""" |
| if isinstance(value, bool): |
| return value |
| if not isinstance(value, str): |
| value = str(value) |
| if value.lower() not in configparser.RawConfigParser.BOOLEAN_STATES: |
| raise ValueError(f"Not a boolean: {value}") |
| return configparser.RawConfigParser.BOOLEAN_STATES[value.lower()] |
| |
| |
| def split_directive(s: str) -> tuple[list[str], list[str]]: |
| """Split s on commas, except during quoted sections. |
| |
| Returns the parts and a list of error messages.""" |
| parts = [] |
| cur: list[str] = [] |
| errors = [] |
| i = 0 |
| while i < len(s): |
| if s[i] == ",": |
| parts.append("".join(cur).strip()) |
| cur = [] |
| elif s[i] == '"': |
| i += 1 |
| while i < len(s) and s[i] != '"': |
| cur.append(s[i]) |
| i += 1 |
| if i == len(s): |
| errors.append("Unterminated quote in configuration comment") |
| cur.clear() |
| else: |
| cur.append(s[i]) |
| i += 1 |
| if cur: |
| parts.append("".join(cur).strip()) |
| |
| return parts, errors |
| |
| |
| def mypy_comments_to_config_map(line: str, template: Options) -> tuple[dict[str, str], list[str]]: |
| """Rewrite the mypy comment syntax into ini file syntax.""" |
| options = {} |
| entries, errors = split_directive(line) |
| for entry in entries: |
| if "=" not in entry: |
| name = entry |
| value = None |
| else: |
| name, value = (x.strip() for x in entry.split("=", 1)) |
| |
| name = name.replace("-", "_") |
| if value is None: |
| value = "True" |
| options[name] = value |
| |
| return options, errors |
| |
| |
| def parse_mypy_comments( |
| args: list[tuple[int, str]], template: Options |
| ) -> tuple[dict[str, object], list[tuple[int, str]]]: |
| """Parse a collection of inline mypy: configuration comments. |
| |
| Returns a dictionary of options to be applied and a list of error messages |
| generated. |
| """ |
| |
| errors: list[tuple[int, str]] = [] |
| sections = {} |
| |
| for lineno, line in args: |
| # In order to easily match the behavior for bools, we abuse configparser. |
| # Oddly, the only way to get the SectionProxy object with the getboolean |
| # method is to create a config parser. |
| parser = configparser.RawConfigParser() |
| options, parse_errors = mypy_comments_to_config_map(line, template) |
| parser["dummy"] = options |
| errors.extend((lineno, x) for x in parse_errors) |
| |
| stderr = StringIO() |
| strict_found = False |
| |
| def set_strict_flags() -> None: |
| nonlocal strict_found |
| strict_found = True |
| |
| new_sections, reports = parse_section( |
| "", template, set_strict_flags, parser["dummy"], ini_config_types, stderr=stderr |
| ) |
| errors.extend((lineno, x) for x in stderr.getvalue().strip().split("\n") if x) |
| if reports: |
| errors.append((lineno, "Reports not supported in inline configuration")) |
| if strict_found: |
| errors.append( |
| ( |
| lineno, |
| 'Setting "strict" not supported in inline configuration: specify it in ' |
| "a configuration file instead, or set individual inline flags " |
| '(see "mypy -h" for the list of flags enabled in strict mode)', |
| ) |
| ) |
| |
| sections.update(new_sections) |
| |
| return sections, errors |
| |
| |
| def get_config_module_names(filename: str | None, modules: list[str]) -> str: |
| if not filename or not modules: |
| return "" |
| |
| if not is_toml(filename): |
| return ", ".join(f"[mypy-{module}]" for module in modules) |
| |
| return "module = ['%s']" % ("', '".join(sorted(modules))) |
| |
| |
| class ConfigTOMLValueError(ValueError): |
| pass |