[mypyc] Reject unknown bases for native classes (#22204)
Fixes https://github.com/mypyc/mypyc/issues/1200.
Reject native classes with an `Any` base, since their instance layout is
not known during compilation. Suggest using
`@mypyc_attr(native_class=False)`
instead.
Preserve the original base expressions for non-native classes with
unknown bases so Python determines their inheritance and instance
layout at runtime.
diff --git a/mypyc/doc/native_classes.rst b/mypyc/doc/native_classes.rst
index f7d9180..a97e4d3 100644
--- a/mypyc/doc/native_classes.rst
+++ b/mypyc/doc/native_classes.rst
@@ -66,6 +66,12 @@
* ``LookupError``
* ``UserWarning``
+A native class can't inherit from a base class with type ``Any``, such
+as a class imported from a module that isn't type checked, even
+indirectly through other base classes, since mypyc needs to know the
+layout of all base classes. Use a
+:ref:`non-native class <non-native-classes>` instead.
+
By default, a non-native class can't inherit a native class, and you
can't inherit from a native class outside the compilation unit that
defines the class. You can enable these through
@@ -231,6 +237,8 @@
If a class definition uses an unsupported class decorator, *mypyc
compiles the class into a regular Python class* (non-native class).
+.. _non-native-classes:
+
Defining non-native classes
---------------------------
diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py
index ae63ff9..a0aa7c4 100644
--- a/mypyc/irbuild/classdef.py
+++ b/mypyc/irbuild/classdef.py
@@ -599,14 +599,16 @@
"""Should the base classes of a non-extension class be evaluated as written?
Resolved base types erase Generic[...] bases and type arguments, but they are
- needed at runtime, e.g. to make the class generic.
+ needed at runtime, e.g. to make the class generic. Any bases are missing from
+ resolved base types, so Python must determine the bases and instance layout.
"""
info = cdef.info
if info.is_named_tuple or info.typeddict_type is not None:
# These have synthesized base classes that need special casing.
return False
return bool(
- cdef.type_args
+ info.fallback_to_any
+ or cdef.type_args
or cdef.removed_base_type_exprs
or any(isinstance(expr, IndexExpr) for expr in cdef.base_type_exprs)
)
diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py
index 301e8cf..e358938 100644
--- a/mypyc/irbuild/prepare.py
+++ b/mypyc/irbuild/prepare.py
@@ -34,6 +34,7 @@
OverloadedFuncDef,
RefExpr,
SymbolNode,
+ TypeAlias,
TypeInfo,
Var,
)
@@ -541,6 +542,48 @@
)
+def report_needs_non_native_class(errors: Errors, msg: str, path: str, line: int) -> None:
+ errors.error(msg, path, line)
+ errors.note(
+ "Potential workaround: @mypy_extensions.mypyc_attr(native_class=False)", path, line
+ )
+ errors.note(
+ "https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes",
+ path,
+ line,
+ )
+
+
+def check_unknown_base_classes(path: str, cdef: ClassDef, errors: Errors, mapper: Mapper) -> None:
+ """Reject native classes whose layout depends on an Any base."""
+ if not cdef.info.fallback_to_any:
+ return
+
+ base_description = "a base class with an unknown layout"
+ for base_expr in cdef.base_type_exprs:
+ if isinstance(base_expr, IndexExpr):
+ base_expr = base_expr.base
+ if not isinstance(base_expr, (NameExpr, MemberExpr)):
+ continue
+ base_node = base_expr.node
+ if isinstance(base_node, TypeAlias):
+ target = get_proper_type(base_node.target)
+ if isinstance(target, Instance):
+ base_node = target.type
+ if not isinstance(base_node, TypeInfo) or base_node.fallback_to_any:
+ base_description = f'"{base_expr.name}" (unknown base class layout)'
+ break
+ msg = f"Native classes cannot inherit from {base_description}"
+ if any(
+ base.fallback_to_any and base in mapper.type_to_ir and mapper.type_to_ir[base].is_ext_class
+ for base in cdef.info.mro[1:]
+ ):
+ # A native base class already reported the error with the notes
+ errors.error(msg, path, cdef.line)
+ else:
+ report_needs_non_native_class(errors, msg, path, cdef.line)
+
+
def prepare_class_def(
path: str,
module_name: str,
@@ -558,6 +601,8 @@
ir = mapper.type_to_ir[cdef.info]
info = cdef.info
+ check_unknown_base_classes(path, cdef, errors, mapper)
+
attrs, attrs_lines = get_mypyc_attrs(cdef, path, errors)
if attrs.get("allow_interpreted_subclasses") is True:
ir.allow_interpreted_subclasses = True
@@ -582,18 +627,8 @@
# isn't in builtins, bad things will happen and we won't
# catch it here! But this should catch a lot of the most
# common pitfalls.
- errors.error(
- "Inheriting from most builtin types is unimplemented", path, cdef.line
- )
- errors.note(
- "Potential workaround: @mypy_extensions.mypyc_attr(native_class=False)",
- path,
- cdef.line,
- )
- errors.note(
- "https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes",
- path,
- cdef.line,
+ report_needs_non_native_class(
+ errors, "Inheriting from most builtin types is unimplemented", path, cdef.line
)
free_list_len = attrs.get("free_list_len")
diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test
index 43e9c2b..7e75ddd 100644
--- a/mypyc/test-data/irbuild-classes.test
+++ b/mypyc/test-data/irbuild-classes.test
@@ -1772,6 +1772,90 @@
class BadUse(): # E: native_class must be used with True or False only
pass
+[case testNativeClassUnknownBase]
+from mypy_extensions import mypyc_attr
+from typing import Any
+from missing_module import Base # type: ignore
+
+class C(Base): # E: Native classes cannot inherit from "Base" (unknown base class layout) \
+ # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \
+ # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes
+ pass
+
+class D(C): # E: Native classes cannot inherit from "C" (unknown base class layout)
+ pass
+
+@mypyc_attr(native_class=False)
+class NonNative(Base):
+ pass
+
+class NativeChild(NonNative): # E: Native classes cannot inherit from "NonNative" (unknown base class layout) \
+ # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \
+ # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes
+ pass
+
+ExplicitAny: Any = str
+class S(ExplicitAny): # E: Native classes cannot inherit from "ExplicitAny" (unknown base class layout) \
+ # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \
+ # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes
+ pass
+
+class Known:
+ pass
+
+class KnownFirst(Known, Base): # E: Native classes cannot inherit from "Base" (unknown base class layout) \
+ # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \
+ # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes
+ pass
+
+class UnknownFirst(Base, Known): # E: Native classes cannot inherit from "Base" (unknown base class layout) \
+ # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \
+ # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes
+ pass
+
+class TwoUnknown(Base, ExplicitAny): # E: Native classes cannot inherit from "Base" (unknown base class layout) \
+ # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \
+ # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes
+ pass
+
+Alias = Known
+class AliasFirst(Alias, Base): # E: Native classes cannot inherit from "Base" (unknown base class layout) \
+ # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \
+ # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes
+ pass
+
+class Indexed(Base[int]): # type: ignore # E: Native classes cannot inherit from "Base" (unknown base class layout) \
+ # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \
+ # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes
+ pass
+
+def make_base() -> Any: ...
+class Dynamic(make_base()): # type: ignore # E: Native classes cannot inherit from a base class with an unknown layout \
+ # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \
+ # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes
+ pass
+
+[case testNativeClassIndirectUnknownBase]
+# Any base class anywhere in the MRO makes the layout unknown
+from stubbed import Indirect, IndirectAlias
+
+class C(Indirect): # E: Native classes cannot inherit from "Indirect" (unknown base class layout) \
+ # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \
+ # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes
+ pass
+
+class D(IndirectAlias): # E: Native classes cannot inherit from "IndirectAlias" (unknown base class layout) \
+ # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \
+ # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes
+ pass
+
+[file stubbed.pyi]
+from typing import Any
+
+class AnyBase(Any): ...
+class Indirect(AnyBase): ...
+IndirectAlias = Indirect
+
[case testMypycAttrNativeClassMetaError]
from mypy_extensions import mypyc_attr
diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test
index 19a1804..2eb4dc2 100644
--- a/mypyc/test-data/run-classes.test
+++ b/mypyc/test-data/run-classes.test
@@ -1543,6 +1543,161 @@
assert False
assert x
+[case testSubclassUnknownPythonBase]
+# The base class is unknown during compilation, so it has type Any
+from mypy_extensions import mypyc_attr
+from typing import Any, Generic, TypeVar
+from unchecked import Base, MetaBase, SecondBase, StrBase # type: ignore
+
+T = TypeVar("T")
+
+@mypyc_attr(native_class=False)
+class C(Base):
+ def __init__(self, x: int) -> None:
+ super().__init__()
+ self.x = x
+
+ def marker(self) -> str:
+ return "override " + super().marker()
+
+@mypyc_attr(native_class=False)
+class InheritedInit(Base):
+ pass
+
+@mypyc_attr(native_class=False)
+class D(InheritedInit):
+ pass
+
+@mypyc_attr(native_class=False)
+class S(StrBase):
+ pass
+
+@mypyc_attr(native_class=False)
+class WithMeta(MetaBase):
+ pass
+
+@mypyc_attr(native_class=False)
+class Known:
+ def marker(self) -> str:
+ return "known"
+
+@mypyc_attr(native_class=False)
+class KnownFirst(Known, Base):
+ pass
+
+@mypyc_attr(native_class=False)
+class UnknownFirst(Base, Known):
+ pass
+
+@mypyc_attr(native_class=False)
+class TwoUnknown(Base, SecondBase):
+ pass
+
+@mypyc_attr(native_class=False)
+class GenericLast(Base, Generic[T]):
+ pass
+
+@mypyc_attr(native_class=False)
+class GenericFirst(Generic[T], Base):
+ pass
+
+@mypyc_attr(native_class=False)
+class KnownGeneric(Generic[T]):
+ pass
+
+@mypyc_attr(native_class=False)
+class KnownGenericBase(Base, KnownGeneric[int]):
+ pass
+
+def test_unknown_base() -> None:
+ c = C(1)
+ assert c.x == 1
+ assert c.y == 2
+ assert c.label == "base attribute"
+ assert c.marker() == "override base"
+ c.z = 3
+ assert c.z == 3
+ assert c.__dict__ == {"x": 1, "y": 2, "z": 3}
+ assert isinstance(c, Base)
+
+def test_inherited_init() -> None:
+ assert InheritedInit().y == 2
+ assert InheritedInit(7).y == 7
+ assert InheritedInit(x=11).y == 11
+ d = D(13)
+ assert d.y == 13
+ assert d.label == "base attribute"
+ assert d.marker() == "base"
+ assert D(x=17).y == 17
+ try:
+ InheritedInit(-1)
+ except ValueError as e:
+ assert str(e) == "negative"
+ else:
+ assert False
+
+def test_unknown_extension_base() -> None:
+ assert S.__basicsize__ >= StrBase.__basicsize__
+ c = S("abc")
+ assert isinstance(c, str)
+ assert c == "abc"
+ assert len(c) == 3
+ assert c.upper() == "ABC"
+
+def test_unknown_base_metaclass() -> None:
+ assert isinstance(WithMeta, type(MetaBase))
+ assert isinstance(WithMeta(), MetaBase)
+
+def test_multiple_inheritance() -> None:
+ assert KnownFirst.__bases__ == (Known, Base)
+ assert UnknownFirst.__bases__ == (Base, Known)
+ assert TwoUnknown.__bases__ == (Base, SecondBase)
+ assert KnownFirst(3).y == 3
+ assert UnknownFirst(x=5).y == 5
+ assert KnownFirst().marker() == "known"
+ assert UnknownFirst().marker() == "base"
+ assert TwoUnknown(7).y == 7
+ assert TwoUnknown().other() == "second"
+
+def test_generic_bases() -> None:
+ generic: Any = globals()["Generic"]
+ type_var: Any = globals()["T"]
+ assert GenericLast.__bases__ == (Base, generic)
+ assert GenericFirst.__bases__ == (generic, Base)
+ assert GenericLast.__orig_bases__ == (Base, generic[type_var])
+ assert GenericFirst.__orig_bases__ == (generic[type_var], Base)
+ assert GenericLast.__parameters__ == (type_var,)
+ assert GenericFirst.__parameters__ == (type_var,)
+ assert GenericLast[int](3).y == 3
+ assert GenericFirst[str](5).y == 5
+ assert KnownGenericBase.__bases__ == (Base, KnownGeneric)
+ assert KnownGenericBase.__orig_bases__ == (Base, KnownGeneric[int])
+ assert KnownGenericBase(7).y == 7
+
+[file unchecked.py]
+StrBase = str
+
+class SecondBase:
+ def other(self):
+ return "second"
+
+class Meta(type):
+ pass
+
+class MetaBase(metaclass=Meta):
+ pass
+
+class Base:
+ label = "base attribute"
+
+ def __init__(self, x=2):
+ if x < 0:
+ raise ValueError("negative")
+ self.y = x
+
+ def marker(self):
+ return "base"
+
[case testSubclassPy]
from b import B, V
class A(B):
@@ -2066,26 +2221,30 @@
[out]
ok
-[case testNoMetaclass]
-from foo import Base
+[case testNativeClassHiddenMetaclass]
+# The stub hides the metaclass of the base class, so it's only detected at runtime
+from hidden import Base
-class Nothing(Base): # type: ignore
+class C(Base):
pass
-[file foo.py]
-from typing import Any
+[file hidden.pyi]
+class Base: ...
+
+[file hidden.py]
class Meta(type):
pass
-class _Base(metaclass=Meta):
+class Base(metaclass=Meta):
pass
-Base = _Base # type: Any
[file driver.py]
try:
import native
except TypeError as e:
- assert(str(e) == "mypyc classes can't have a metaclass")
+ assert str(e) == "mypyc classes can't have a metaclass"
+else:
+ assert False
[case testMetaclass]
from meta import Meta
