[mypyc] Add internal flag to control python target (#22073) The purpose of this is to allow compiling extensions for any supported python version from a host running a potentially different version (or perhaps type of a different build variant; mainly free threading) This is not exposed externally because you are required to compile the C from outside of mypyc itself; therefore I suspect this is currently only useful to those using a monorepo build system (e.g. bazel or buck2) like we do internally. This requires passing through the target python which is represented as a class with version and free threaded fields as well as properties representing semantics. --------- Co-authored-by: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
diff --git a/mypyc/build.py b/mypyc/build.py index 281daaf..e61c09a 100644 --- a/mypyc/build.py +++ b/mypyc/build.py
@@ -40,12 +40,12 @@ from mypy.util import write_junit_xml from mypyc.annotate import generate_annotated_html from mypyc.codegen import emitmodule -from mypyc.common import IS_FREE_THREADED, RUNTIME_C_FILES, shared_lib_name +from mypyc.common import RUNTIME_C_FILES, shared_lib_name from mypyc.errors import Errors from mypyc.ir.deps import SourceDep from mypyc.ir.pprint import format_modules from mypyc.namegen import exported_name -from mypyc.options import CompilerOptions +from mypyc.options import CompilerOptions, TargetPython class ModDesc(NamedTuple): @@ -261,8 +261,8 @@ return mypyc_sources, all_sources, options # Override whatever python_version is inferred from the .ini file, - # and set the python_version to be the currently used version. - options.python_version = sys.version_info[:2] + # and set the python_version to be the target version. + options.python_version = compiler_options.target_python.version if options.python_version[0] == 2: fail("Python 2 not supported") @@ -299,7 +299,7 @@ cname = "%s.c" % full_module_name.replace(".", os.sep) cpath = os.path.join(dir_name, cname) - if IS_FREE_THREADED: + if TargetPython.host().free_threaded: # We use multi-phase init in free-threaded builds to enable free threading. shim_name = "module_shim_no_gil_multiphase.tmpl" else:
diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index e78b3f3..c747aa6 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py
@@ -2,7 +2,6 @@ from __future__ import annotations -import sys from collections.abc import Callable from typing import Final @@ -12,7 +11,6 @@ ATTR_PREFIX, BITMAP_BITS, FAST_ISINSTANCE_MAX_SUBCLASSES, - HAVE_IMMORTAL, MODULE_PREFIX, NATIVE_PREFIX, PREFIX, @@ -73,6 +71,7 @@ vec_item_type_tags, ) from mypyc.namegen import NameGenerator, exported_name +from mypyc.options import TargetPython from mypyc.primitives.registry import builtin_names from mypyc.sametype import is_same_type @@ -141,6 +140,7 @@ strict_traceback_checks: bool, group_name: str | None = None, group_map: dict[str, str | None] | None = None, + target_python: TargetPython | None = None, ) -> None: """Setup shared emitter state. @@ -148,6 +148,7 @@ names: The name generator to use group_map: Map from module names to group name group_name: Current group name + target_python: The Python build to generate code for (default: running Python) """ self.temp_counter = 0 self.names = names @@ -168,6 +169,7 @@ self.literals = Literals() # See mypyc/options.py for context. self.strict_traceback_checks = strict_traceback_checks + self.target_python = target_python or TargetPython.host() class ErrorHandler: @@ -211,17 +213,20 @@ self, context: EmitterContext, value_names: dict[Value, str] | None = None, - capi_version: tuple[int, int] | None = None, filepath: str | None = None, ) -> None: self.context = context - self.capi_version = capi_version or sys.version_info[:2] + self.target_python = context.target_python self.names = context.names self.value_names = value_names or {} self.fragments: list[str] = [] self._indent = 0 self.filepath = filepath + @property + def capi_version(self) -> tuple[int, int]: + return self.target_python.version + # Low-level operations def indent(self) -> None: @@ -598,10 +603,10 @@ self.emit_line(f"{prefix}_INCREF({dest});") elif not rtype.is_unboxed: # Always inline, since this is a simple but very hot op - if rtype.may_be_immortal or not HAVE_IMMORTAL: - self.emit_line("CPy_INCREF(%s);" % dest) - else: + if rtype.should_use_nonimmortal_variant(self.target_python): self.emit_line("CPy_INCREF_NO_IMM(%s);" % dest) + else: + self.emit_line("CPy_INCREF(%s);" % dest) # Otherwise assume it's an unboxed, pointerless value and do nothing. def emit_dec_ref( @@ -632,10 +637,10 @@ self.emit_line(f"CPy_{x}DecRef({dest});") else: # Inlined - if rtype.may_be_immortal or not HAVE_IMMORTAL: - self.emit_line(f"CPy_{x}DECREF({dest});") - else: + if rtype.should_use_nonimmortal_variant(self.target_python): self.emit_line(f"CPy_{x}DECREF_NO_IMM({dest});") + else: + self.emit_line(f"CPy_{x}DECREF({dest});") elif rtype.is_refcounted: assert False, f"dec_ref not implemented for {rtype}" # Otherwise assume it's an unboxed, pointerless value and do nothing.
diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 4281876..1756905 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py
@@ -31,7 +31,6 @@ BITMAP_TYPE, CPYFUNCTION_NAME, GENERATOR_HELPER_NAME, - IS_FREE_THREADED, MYPYC_DEFAULTS_SETUP, NATIVE_PREFIX, PREFIX, @@ -1319,7 +1318,7 @@ emitter.emit_line("{") attr_expr = f"self->{attr_field}" - if IS_FREE_THREADED and is_simple_refcounted_pointer(rtype): + if emitter.target_python.free_threaded and is_simple_refcounted_pointer(rtype): # In free-threaded builds, load the attribute and take a new reference with # an optimistic validated incref to avoid racing with a concurrent setter. # CPy_GetAttrRef returns NULL if the attribute is undefined (NULL field), @@ -1383,7 +1382,7 @@ emitter.emit_line("return -1;") emitter.emit_line("}") - if IS_FREE_THREADED and is_simple_refcounted_pointer(rtype): + if emitter.target_python.free_threaded and is_simple_refcounted_pointer(rtype): # In free-threaded builds, publish the new value via CPy_SetAttrRef, which # takes the owner's critical section so a concurrent reader (see # CPy_GetAttrRef) can always secure a reference to the value it observes, @@ -1562,7 +1561,7 @@ wrapper_name = emit_instance(cl.methods["__call__"], cl.coroutine_name) struct_name = cl.struct_name(emitter.names) attr = emitter.attr(CPYFUNCTION_NAME) - if IS_FREE_THREADED: + if emitter.target_python.free_threaded: emitter.emit_line( f"CPy_InitAttrRefIfNull(type, (PyObject **)&(({struct_name} *)type)->{attr}, " f"{wrapper_name});"
diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index a1864b3..215b889 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py
@@ -13,8 +13,6 @@ c_array_initializer, ) from mypyc.common import ( - HAVE_IMMORTAL, - IS_FREE_THREADED, NATIVE_PREFIX, REG_PREFIX, RUNNING_FIELD, @@ -466,7 +464,7 @@ ClassIR.attrs_are_thread_confined. """ use_get_attr_ref = ( - IS_FREE_THREADED + self.emitter.target_python.free_threaded and is_simple_refcounted_pointer(attr_rtype) and not op.is_borrowed and not cl.attrs_are_thread_confined() @@ -629,7 +627,7 @@ self.emit_line(f"{dest} = 1;") self.emitter.emit_error_check(tmp, ret_type, f"{dest} = 0;") elif ( - IS_FREE_THREADED + self.emitter.target_python.free_threaded and is_simple_refcounted_pointer(attr_rtype) and not cl.attrs_are_thread_confined() ): @@ -763,12 +761,12 @@ if ( isinstance(op.src, Box) and (is_none_rprimitive(op.src.src.type) or is_bool_or_bit_rprimitive(op.src.src.type)) - and HAVE_IMMORTAL + and self.emitter.target_python.have_immortal ): # On Python 3.12+, None/True/False are immortal, and we can skip inc ref return - if isinstance(op.src, LoadLiteral) and HAVE_IMMORTAL: + if isinstance(op.src, LoadLiteral) and self.emitter.target_python.have_immortal: value = op.src.value # We can skip inc ref for immortal literals on Python 3.12+ if type(value) is int and -5 <= value <= 256:
diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 6ebc74a..518d1a0 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py
@@ -7,7 +7,6 @@ import json import os -import sys from collections.abc import Iterable from typing import TypeVar @@ -46,9 +45,7 @@ ) from mypyc.codegen.literals import Literals from mypyc.common import ( - EXT_SUFFIX, GENERATOR_HELPER_NAME, - IS_FREE_THREADED, MODULE_PREFIX, PREFIX, RUNTIME_C_FILES, @@ -688,7 +685,11 @@ self.modules = modules self.source_paths = source_paths self.context = EmitterContext( - names, compiler_options.strict_traceback_checks, group_name, group_map + names, + compiler_options.strict_traceback_checks, + group_name, + group_map, + target_python=compiler_options.target_python, ) self.names = names # Initializations of globals to simple values that we can't @@ -700,7 +701,7 @@ self.multi_file = compiler_options.multi_file # Multi-phase init is needed to enable free-threading. In the future we'll # probably want to enable it always, but we'll wait until it's stable. - self.multi_phase_init = IS_FREE_THREADED + self.multi_phase_init = compiler_options.target_python.free_threaded @property def group_suffix(self) -> str: @@ -1223,13 +1224,13 @@ emitter.emit_line(f"static PyModuleDef_Slot {name}[] = {{") emitter.emit_line(f"{{Py_mod_exec, {exec_name}}},") - if sys.version_info >= (3, 12): + if emitter.capi_version >= (3, 12): # Multiple interpreter support requires not using any C global state, # which we don't support yet. emitter.emit_line( "{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}," ) - if sys.version_info >= (3, 13): + if emitter.capi_version >= (3, 13): # Declare support for free-threading to enable experimentation, # even if we don't properly support it. emitter.emit_line("{Py_mod_gil, Py_MOD_GIL_NOT_USED},") @@ -1502,6 +1503,7 @@ emitter.emit_line(f'modname = PyUnicode_FromString("{module_name}");') emitter.emit_line("if (modname == NULL) CPyError_OutOfMemory();") emitter.emit_line("int rv = 0;") + ext_suffix = emitter.target_python.ext_suffix if self.group_name: shared_lib_mod_name = shared_lib_name(self.group_name) emitter.emit_line("PyObject *mod_dict = PyImport_GetModuleDict();") @@ -1516,10 +1518,10 @@ emitter.emit_line("if (shared_lib_file == NULL) goto fail;") else: emitter.emit_line( - f'PyObject *shared_lib_file = PyUnicode_FromString("{module_name + EXT_SUFFIX}");' + f'PyObject *shared_lib_file = PyUnicode_FromString("{module_name + ext_suffix}");' ) emitter.emit_line("if (shared_lib_file == NULL) CPyError_OutOfMemory();") - emitter.emit_line(f'PyObject *ext_suffix = PyUnicode_FromString("{EXT_SUFFIX}");') + emitter.emit_line(f'PyObject *ext_suffix = PyUnicode_FromString("{ext_suffix}");') emitter.emit_line("if (ext_suffix == NULL) CPyError_OutOfMemory();") is_pkg = int(self.source_paths[module_name].endswith("__init__.py")) emitter.emit_line(f"Py_ssize_t is_pkg = {is_pkg};")
diff --git a/mypyc/common.py b/mypyc/common.py index 430a65b..e3967af 100644 --- a/mypyc/common.py +++ b/mypyc/common.py
@@ -146,16 +146,6 @@ "function_wrapper.c", ] -# Python 3.12 introduced immortal objects, specified via a special reference count -# value. The reference counts of immortal objects are normally not modified, but it's -# not strictly wrong to modify them. See PEP 683 for more information, but note that -# some details in the PEP are out of date. -HAVE_IMMORTAL: Final = sys.version_info >= (3, 12) - -# Are we running on a free-threaded build (GIL disabled)? This implies that -# we are on Python 3.13 or later. -IS_FREE_THREADED: Final = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) - # The file extension suffix for C extension modules on the current platform # (e.g. ".cpython-312-x86_64-linux-gnu.so" or ".pyd"). _EXT_SUFFIXES: Final = importlib.machinery.EXTENSION_SUFFIXES
diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index febd076..bde6730 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py
@@ -718,6 +718,7 @@ experimental: bool, dependencies: list[Dependency] | None, type_params: list[RTypeVar] | None, + gil_only: bool = False, ) -> None: # Each primitive much have a distinct name, but otherwise they are arbitrary. self.name: Final = name @@ -743,6 +744,9 @@ # Experimental primitives are not used unless mypyc experimental features are # explicitly enabled self.experimental = experimental + # GIL-only primitives are not used when targeting a free-threaded build + # (e.g. because they return a borrowed reference that could be unsafe) + self.gil_only = gil_only # Dependencies for the primitive, such as a capsule that needs to imported # and configured to call the primitive. self.dependencies = dependencies
diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index afe3d3e..84b260d 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py
@@ -40,13 +40,14 @@ from abc import abstractmethod from typing import TYPE_CHECKING, ClassVar, Final, Generic, TypeGuard, TypeVar, Union, final -from mypyc.common import HAVE_IMMORTAL, IS_32_BIT_PLATFORM, PLATFORM_SIZE, JsonDict, short_name +from mypyc.common import IS_32_BIT_PLATFORM, PLATFORM_SIZE, JsonDict, short_name from mypyc.ir.deps import LIBRT_RANDOM, LIBRT_STRINGS, LIBRT_THREADING, LIBRT_VECS, Dependency from mypyc.namegen import NameGenerator if TYPE_CHECKING: from mypyc.ir.class_ir import ClassIR from mypyc.ir.ops import DeserMaps + from mypyc.options import TargetPython T = TypeVar("T") @@ -98,11 +99,24 @@ def short_name(self) -> str: return short_name(self.name) - @property @abstractmethod - def may_be_immortal(self) -> bool: + def may_be_immortal(self, target: TargetPython) -> bool: + """Can a value of this type be an immortal object on the target Python? + + This is always False if the target has no immortal objects (before 3.12), + so False doesn't imply that ref count ops can skip the immortality check. + Use should_use_nonimmortal_variant() for that. + """ raise NotImplementedError + def should_use_nonimmortal_variant(self, target: TargetPython) -> bool: + """Can ref count ops skip the immortality check (e.g. CPy_INCREF_NO_IMM)? + + This is only possible if the target has immortal objects, since otherwise + the regular ops don't check for immortality. + """ + return target.have_immortal and not self.may_be_immortal(target) + def __str__(self) -> str: return short_name(self.name) @@ -199,8 +213,7 @@ def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_rvoid(self) - @property - def may_be_immortal(self) -> bool: + def may_be_immortal(self, target: TargetPython) -> bool: return False def serialize(self) -> str: @@ -259,7 +272,9 @@ self._ctype = ctype self.size = size self.error_overlap = error_overlap - self._may_be_immortal = may_be_immortal and HAVE_IMMORTAL + # Whether values may be immortal on targets that have immortal objects + # (see may_be_immortal(), which also takes the target into account) + self._may_be_immortal = may_be_immortal self.dependencies = dependencies if ctype == "CPyTagged": self.c_undefined = "CPY_INT_TAG" @@ -285,9 +300,8 @@ def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_rprimitive(self) - @property - def may_be_immortal(self) -> bool: - return self._may_be_immortal + def may_be_immortal(self, target: TargetPython) -> bool: + return self._may_be_immortal and target.have_immortal def serialize(self) -> str: return self.name @@ -811,8 +825,7 @@ def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_rtuple(self) - @property - def may_be_immortal(self) -> bool: + def may_be_immortal(self, target: TargetPython) -> bool: return False def __str__(self) -> str: @@ -945,8 +958,7 @@ def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_rstruct(self) - @property - def may_be_immortal(self) -> bool: + def may_be_immortal(self, target: TargetPython) -> bool: return False def __str__(self) -> str: @@ -1010,8 +1022,7 @@ def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_rinstance(self) - @property - def may_be_immortal(self) -> bool: + def may_be_immortal(self, target: TargetPython) -> bool: return False def struct_name(self, names: NameGenerator) -> str: @@ -1058,8 +1069,7 @@ def __init__(self, id: int) -> None: self.id = id - @property - def may_be_immortal(self) -> bool: + def may_be_immortal(self, target: TargetPython) -> bool: # RTypeVar must always be substituted before use, so this should never matter. return False @@ -1114,8 +1124,7 @@ self.types = [c_pyssize_t_rprimitive, pointer_rprimitive] self.buf_type = VecTBufObject - @property - def may_be_immortal(self) -> bool: + def may_be_immortal(self, target: TargetPython) -> bool: return False def unwrap_item_type(self) -> RPrimitive | RInstance: @@ -1218,9 +1227,8 @@ def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_runion(self) - @property - def may_be_immortal(self) -> bool: - return any(item.may_be_immortal for item in self.items) + def may_be_immortal(self, target: TargetPython) -> bool: + return any(item.may_be_immortal(target) for item in self.items) def __repr__(self) -> str: return "<RUnion %s>" % ", ".join(str(item) for item in self.items) @@ -1293,8 +1301,7 @@ def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_rarray(self) - @property - def may_be_immortal(self) -> bool: + def may_be_immortal(self, target: TargetPython) -> bool: return False def __str__(self) -> str:
diff --git a/mypyc/irbuild/ast_helpers.py b/mypyc/irbuild/ast_helpers.py index 57b5e15..9e1acd9 100644 --- a/mypyc/irbuild/ast_helpers.py +++ b/mypyc/irbuild/ast_helpers.py
@@ -20,7 +20,6 @@ UnaryExpr, Var, ) -from mypyc.common import IS_FREE_THREADED from mypyc.ir.ops import BasicBlock from mypyc.ir.rtypes import RVec, is_fixed_width_rtype, is_tagged from mypyc.irbuild.builder import IRBuilder @@ -126,7 +125,7 @@ # - Native Final attributes are read-only at runtime, so they can never be reassigned. # - Vec-typed attributes require manual synchronization, so we borrow them liberally. return ( - not IS_FREE_THREADED + not self.options.target_python.free_threaded or isinstance(self.node_type(expr), RVec) or self.is_final_native_attr_ref(expr) )
diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index bc47174..a622315 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py
@@ -69,9 +69,7 @@ from mypy.visitor import ExpressionVisitor, StatementVisitor from mypyc.common import ( BITMAP_BITS, - EXT_SUFFIX, GENERATOR_ATTRIBUTE_PREFIX, - IS_FREE_THREADED, KEEP_ALIVE_SHORT_LIVED, KEEP_ALIVE_WHOLE_EXPRESSION, SELF_NAME, @@ -589,7 +587,7 @@ shared_lib_file = self.py_get_attr(shared_lib_obj, "__file__", line) else: shared_lib_file = self.none_object(line) - ext_suffix = self.load_str(EXT_SUFFIX, line) + ext_suffix = self.load_str(self.options.target_python.ext_suffix, line) is_pkg = self.is_package_module(module) value = self.call_c( native_import_op, @@ -994,7 +992,7 @@ index: Value if is_list_rprimitive(rvalue.type): index = Integer(i, c_pyssize_t_rprimitive) - if not IS_FREE_THREADED: + if not self.options.target_python.free_threaded: item_value = self.primitive_op(list_get_item_unsafe_op, [rvalue, index], line) else: item_value = self.primitive_op(list_get_item_int64_op, [rvalue, index], line)
diff --git a/mypyc/irbuild/callable_class.py b/mypyc/irbuild/callable_class.py index 3adda15..c0022ef 100644 --- a/mypyc/irbuild/callable_class.py +++ b/mypyc/irbuild/callable_class.py
@@ -6,13 +6,7 @@ from __future__ import annotations -from mypyc.common import ( - CPYFUNCTION_NAME, - ENV_ATTR_NAME, - IS_FREE_THREADED, - PROPSET_PREFIX, - SELF_NAME, -) +from mypyc.common import CPYFUNCTION_NAME, ENV_ATTR_NAME, PROPSET_PREFIX, SELF_NAME from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature, RuntimeArg from mypyc.ir.ops import ( @@ -144,7 +138,7 @@ self_reg, CPYFUNCTION_NAME, line, - borrow=not IS_FREE_THREADED, + borrow=not builder.options.target_python.free_threaded, allow_error_value=True, ) )
diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 991a654..7f6ef5a 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py
@@ -60,12 +60,7 @@ UnionType, get_proper_type, ) -from mypyc.common import ( - IS_FREE_THREADED, - KEEP_ALIVE_SHORT_LIVED, - KEEP_ALIVE_WHOLE_EXPRESSION, - MAX_SHORT_INT, -) +from mypyc.common import KEEP_ALIVE_SHORT_LIVED, KEEP_ALIVE_WHOLE_EXPRESSION, MAX_SHORT_INT from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD from mypyc.ir.ops import ( @@ -308,14 +303,16 @@ # - Native Final attributes are read-only at runtime, so they can never be reassigned. # - Vec-typed attributes require manual synchronization, so we borrow them liberally. can_borrow = builder.is_native_attr_ref(expr) and ( - not IS_FREE_THREADED or isinstance(rtype, RVec) or builder.is_final_native_attr_ref(expr) + not builder.options.target_python.free_threaded + or isinstance(rtype, RVec) + or builder.is_final_native_attr_ref(expr) ) obj = builder.accept(expr.expr, can_borrow=can_borrow) if ( is_object_rprimitive(obj.type) and expr.name == "__name__" - and builder.options.capi_version >= (3, 11) + and builder.options.target_python.version >= (3, 11) ): return builder.primitive_op(name_op, [obj], expr.line) @@ -963,9 +960,9 @@ base_type = builder.node_type(expr.base) # We can borrow a list item safely only if GIL is enabled. The vec type is optimized for # performance, so we'll do unsafe borrowing. - can_borrow = (is_list_rprimitive(base_type) and not IS_FREE_THREADED) or isinstance( - base_type, RVec - ) + can_borrow = ( + is_list_rprimitive(base_type) and not builder.options.target_python.free_threaded + ) or isinstance(base_type, RVec) can_borrow_base = ( is_list_rprimitive(base_type) or isinstance(base_type, RVec) ) and is_borrow_friendly_expr(builder, index)
diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index a3c7349..74db465 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py
@@ -28,7 +28,7 @@ Var, ) from mypy.types import LiteralType, TupleType, get_proper_type, get_proper_types -from mypyc.common import GENERATOR_HELPER_NAME, IS_FREE_THREADED +from mypyc.common import GENERATOR_HELPER_NAME from mypyc.ir.ops import ( ERR_NEVER, BasicBlock, @@ -871,7 +871,7 @@ # since we want to use __getitem__ if we don't have an unsafe version, # so we just check manually. if is_list_rprimitive(target.type): - if not IS_FREE_THREADED: + if not builder.options.target_python.free_threaded: return builder.primitive_op(list_get_item_unsafe_op, [target, index], line) else: return builder.primitive_op(list_get_item_int64_op, [target, index], line)
diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 61ed6d5..0adf7ff 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py
@@ -6,7 +6,6 @@ from __future__ import annotations -import sys from collections.abc import Callable, Iterator, Sequence from contextlib import contextmanager from typing import Final, TypeGuard, cast @@ -19,7 +18,6 @@ BITMAP_BITS, FAST_ISINSTANCE_MAX_SUBCLASSES, FAST_PREFIX, - IS_FREE_THREADED, KEEP_ALIVE_SHORT_LIVED, MAX_LITERAL_SHORT_INT, MAX_SHORT_INT, @@ -2520,6 +2518,8 @@ continue if desc.experimental and not self.options.experimental_features: continue + if desc.gil_only and self.options.target_python.free_threaded: + continue if all( # formal is not None and # TODO is_subtype(actual.type, formal, relaxed=not strict) @@ -2842,7 +2842,8 @@ def set_immortal_if_free_threaded(self, v: Value, line: int) -> None: """Make an object immortal on free-threaded builds (to avoid contention).""" - if IS_FREE_THREADED and sys.version_info >= (3, 14): + target = self.options.target_python + if target.free_threaded and target.version >= (3, 14): self.primitive_op(set_immortal_op, [v], line) # Internal helpers
diff --git a/mypyc/options.py b/mypyc/options.py index 8eb33b2..a116c54 100644 --- a/mypyc/options.py +++ b/mypyc/options.py
@@ -1,6 +1,108 @@ from __future__ import annotations +import re import sys +import sysconfig +from typing import Final, final + +from mypy.defaults import PYTHON3_VERSION_MIN +from mypyc.common import EXT_SUFFIX + +# Is the running interpreter a free-threaded build (GIL disabled)? Use +# TargetPython.host().free_threaded instead of this. +_HOST_FREE_THREADED: Final = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + + +@final +class TargetPython: + """The Python build that generated code targets. + + This defaults to the running interpreter, but can be overridden so that + the C code can be generated on a different Python than the one that will + compile and run it. + + This is currently mostly meant for users that bring their own monolithic + build system (e.g. Bazel or Buck2), which call mypyc_build() to only generate + C and then compile it themselves against the headers of the target Python. + It's not exposed through the mypyc command line or mypycify(). + + WARNING: Never compile the generated C for a different Python than the + target. setuptools-based builds always compile using the running interpreter, + so a different target there produces extension modules that fail to compile + or, worse, compile but don't work (e.g. code for a GIL build has data races + on a free-threaded build). + """ + + def __init__(self, version: tuple[int, int], free_threaded: bool = False) -> None: + if version[0] != 3 or version < PYTHON3_VERSION_MIN: + min_version = ".".join(str(v) for v in PYTHON3_VERSION_MIN) + raise ValueError( + f"Unsupported target Python {version[0]}.{version[1]} " + f"({min_version} or later is required)" + ) + if free_threaded and version < (3, 13): + raise ValueError( + f"Free-threaded builds require Python 3.13 or later " + f"(got {version[0]}.{version[1]})" + ) + # Python (C API) version, such as (3, 13) + self.version: Final = version + # Is this a free-threaded (GIL disabled) build? + self.free_threaded: Final = free_threaded + # Python 3.12 introduced immortal objects, specified via a special reference + # count value. The reference counts of immortal objects are normally not + # modified, but it's not strictly wrong to modify them. See PEP 683 for more + # information, but note that some details in the PEP are out of date. + self.have_immortal: Final = version >= (3, 12) + # File name suffix of extension modules, e.g. ".cpython-314t-x86_64-linux-gnu.so". + # Only the Python version (not the platform) can differ from the host. + is_host = version == sys.version_info[:2] and free_threaded == _HOST_FREE_THREADED + self.ext_suffix: Final = ( + EXT_SUFFIX + if is_host + else replace_ext_suffix_version(EXT_SUFFIX, version, free_threaded) + ) + + @staticmethod + def host() -> TargetPython: + """Target the running interpreter.""" + return TargetPython(sys.version_info[:2], _HOST_FREE_THREADED) + + @staticmethod + def parse(target: str) -> TargetPython: + """Parse a target such as "3.13" or "3.14t" ("t" means free-threaded).""" + m = re.fullmatch(r"(\d+)\.(\d+)(t?)", target.strip()) + if m is None: + raise ValueError(f'Invalid target Python "{target}" (expected e.g. "3.13" or "3.14t")') + return TargetPython((int(m.group(1)), int(m.group(2))), m.group(3) == "t") + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, TargetPython) + and self.version == other.version + and self.free_threaded == other.free_threaded + ) + + def __hash__(self) -> int: + return hash((self.version, self.free_threaded)) + + def __str__(self) -> str: + return f"{self.version[0]}.{self.version[1]}{'t' if self.free_threaded else ''}" + + def __repr__(self) -> str: + return f"TargetPython({str(self)!r})" + + +def replace_ext_suffix_version(suffix: str, version: tuple[int, int], free_threaded: bool) -> str: + """Replace the Python version tag in an extension suffix such as ".cp313-win_amd64.pyd".""" + tag = f"{version[0]}{version[1]}{'t' if free_threaded else ''}" + result, n = re.subn(r"(\.cpython-|\.cp)\d+t?(?=-)", rf"\g<1>{tag}", suffix, count=1) + if n == 0: + raise ValueError( + f'Can\'t target Python {version[0]}.{version[1]}{"t" if free_threaded else ""}: ' + f'unrecognized extension module suffix "{suffix}"' + ) + return result class CompilerOptions: @@ -12,14 +114,13 @@ separate: bool = False, target_dir: str | None = None, include_runtime_files: bool | None = None, - capi_version: tuple[int, int] | None = None, - python_version: tuple[int, int] | None = None, strict_dunder_typing: bool = False, group_name: str | None = None, log_trace: bool = False, depends_on_librt_internal: bool = False, experimental_features: bool = False, strict_traceback_checks: bool = False, + target_python: TargetPython | None = None, ) -> None: self.strip_asserts = strip_asserts self.multi_file = multi_file @@ -30,12 +131,10 @@ self.include_runtime_files = ( include_runtime_files if include_runtime_files is not None else not multi_file ) - # The target Python C API version. Overriding this is mostly - # useful in IR tests, since there's no guarantee that - # binaries are backward compatible even if no recent API - # features are used. - self.capi_version = capi_version or sys.version_info[:2] - self.python_version = python_version + # The Python build to generate code for (see TargetPython, including the + # warning there). This determines both the C API version and the Python + # version used for type checking. + self.target_python = target_python or TargetPython.host() # Make possible to inline dunder methods in the generated code. # Typically, the convention is the dunder methods can return `NotImplemented` # even when its return type is just `bool`.
diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index 8af7e9e..f5548e8 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py
@@ -2,7 +2,6 @@ from __future__ import annotations -from mypyc.common import IS_FREE_THREADED from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( bit_rprimitive, @@ -119,39 +118,42 @@ priority=5, ) -if not IS_FREE_THREADED: - # list[index] that produces a borrowed result - method_op( - name="__getitem__", - arg_types=[list_rprimitive, int_rprimitive], - return_type=object_rprimitive, - c_function_name="CPyList_GetItemBorrow", - error_kind=ERR_MAGIC, - is_borrowed=True, - priority=3, - ) +# list[index] that produces a borrowed result. Borrowed variants are unsafe on +# free-threaded builds (another thread could remove the item and free it). +method_op( + name="__getitem__", + arg_types=[list_rprimitive, int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemBorrow", + error_kind=ERR_MAGIC, + is_borrowed=True, + gil_only=True, + priority=3, +) - # list[index] that produces a borrowed result and index is known to be short - method_op( - name="__getitem__", - arg_types=[list_rprimitive, short_int_rprimitive], - return_type=object_rprimitive, - c_function_name="CPyList_GetItemShortBorrow", - error_kind=ERR_MAGIC, - is_borrowed=True, - priority=4, - ) +# list[index] that produces a borrowed result and index is known to be short +method_op( + name="__getitem__", + arg_types=[list_rprimitive, short_int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemShortBorrow", + error_kind=ERR_MAGIC, + is_borrowed=True, + gil_only=True, + priority=4, +) - # Version with native int index - method_op( - name="__getitem__", - arg_types=[list_rprimitive, int64_rprimitive], - return_type=object_rprimitive, - c_function_name="CPyList_GetItemInt64Borrow", - is_borrowed=True, - error_kind=ERR_MAGIC, - priority=6, - ) +# Version with native int index +method_op( + name="__getitem__", + arg_types=[list_rprimitive, int64_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemInt64Borrow", + is_borrowed=True, + gil_only=True, + error_kind=ERR_MAGIC, + priority=6, +) # This is unsafe because it assumes that the index is a non-negative integer # that is in-bounds for the list.
diff --git a/mypyc/primitives/registry.py b/mypyc/primitives/registry.py index 060af0f..51b5359 100644 --- a/mypyc/primitives/registry.py +++ b/mypyc/primitives/registry.py
@@ -108,6 +108,7 @@ is_pure: bool = False, experimental: bool = False, dependencies: list[Dependency] | None = None, + gil_only: bool = False, ) -> PrimitiveDescription: """Define a c function call op that replaces a method call. @@ -134,6 +135,7 @@ priority: if multiple ops match, the one with the highest priority is picked is_pure: if True, declare that the C function has no side effects, takes immutable arguments, and never raises an exception + gil_only: if True, don't use this op when targeting a free-threaded build """ if extra_int_constants is None: extra_int_constants = [] @@ -153,6 +155,7 @@ priority, is_pure=is_pure, experimental=experimental, + gil_only=gil_only, dependencies=dependencies, type_params=None, )
diff --git a/mypyc/test-data/irbuild-statements.test b/mypyc/test-data/irbuild-statements.test index 68e0e2d..40168ac 100644 --- a/mypyc/test-data/irbuild-statements.test +++ b/mypyc/test-data/irbuild-statements.test
@@ -314,7 +314,7 @@ L4: return y -[case testForList_nogil] +[case testForList_nogil_64bit] from typing import List def f(ls: List[int]) -> int: @@ -708,7 +708,7 @@ y = r9 return 1 -[case testMultipleAssignmentUnpackFromSequence_nogil] +[case testMultipleAssignmentUnpackFromSequence_nogil_64bit] from typing import List, Tuple def f(l: List[int], t: Tuple[int, ...]) -> None:
diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 718eabb..d9f96ea 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test
@@ -333,7 +333,7 @@ L4: return r1 -[case testVecI64FastComprehensionFromList_nogil] +[case testVecI64FastComprehensionFromList_nogil_64bit] from librt.vecs import vec from mypy_extensions import i64 from typing import List
diff --git a/mypyc/test/test_emit.py b/mypyc/test/test_emit.py index b2199b2..987724c 100644 --- a/mypyc/test/test_emit.py +++ b/mypyc/test/test_emit.py
@@ -3,7 +3,6 @@ import unittest from mypyc.codegen.emit import Emitter, EmitterContext -from mypyc.common import HAVE_IMMORTAL from mypyc.ir.class_ir import ClassIR from mypyc.ir.ops import BasicBlock, Register, Value from mypyc.ir.rtypes import ( @@ -19,6 +18,7 @@ ) from mypyc.irbuild.vtable import compute_vtable from mypyc.namegen import NameGenerator +from mypyc.options import TargetPython class TestEmitter(unittest.TestCase): @@ -107,14 +107,14 @@ def test_emit_inc_ref_list(self) -> None: self.emitter.emit_inc_ref("x", list_rprimitive) - if HAVE_IMMORTAL: + if TargetPython.host().have_immortal: self.assert_output("CPy_INCREF_NO_IMM(x);\n") else: self.assert_output("CPy_INCREF(x);\n") def test_emit_inc_ref_instance(self) -> None: self.emitter.emit_inc_ref("x", self.instance_a) - if HAVE_IMMORTAL: + if TargetPython.host().have_immortal: self.assert_output("CPy_INCREF_NO_IMM(x);\n") else: self.assert_output("CPy_INCREF(x);\n") @@ -144,24 +144,24 @@ def test_emit_dec_ref_list(self) -> None: self.emitter.emit_dec_ref("x", list_rprimitive) - if HAVE_IMMORTAL: + if TargetPython.host().have_immortal: self.assert_output("CPy_DECREF_NO_IMM(x);\n") else: self.assert_output("CPy_DECREF(x);\n") self.emitter.emit_dec_ref("x", list_rprimitive, is_xdec=True) - if HAVE_IMMORTAL: + if TargetPython.host().have_immortal: self.assert_output("CPy_XDECREF_NO_IMM(x);\n") else: self.assert_output("CPy_XDECREF(x);\n") def test_emit_dec_ref_instance(self) -> None: self.emitter.emit_dec_ref("x", self.instance_a) - if HAVE_IMMORTAL: + if TargetPython.host().have_immortal: self.assert_output("CPy_DECREF_NO_IMM(x);\n") else: self.assert_output("CPy_DECREF(x);\n") self.emitter.emit_dec_ref("x", self.instance_a, is_xdec=True) - if HAVE_IMMORTAL: + if TargetPython.host().have_immortal: self.assert_output("CPy_XDECREF_NO_IMM(x);\n") else: self.assert_output("CPy_XDECREF(x);\n")
diff --git a/mypyc/test/test_emitclass.py b/mypyc/test/test_emitclass.py index 4e34f35..a1cd2de 100644 --- a/mypyc/test/test_emitclass.py +++ b/mypyc/test/test_emitclass.py
@@ -11,10 +11,10 @@ setter_name, slot_key, ) -from mypyc.common import IS_FREE_THREADED from mypyc.ir.class_ir import ClassIR from mypyc.ir.rtypes import int32_rprimitive, object_rprimitive from mypyc.namegen import NameGenerator +from mypyc.options import TargetPython class TestEmitClass(unittest.TestCase): @@ -44,15 +44,16 @@ assert getter_name(cls, "down", generator) == "testing___SomeClass_get_down" - @unittest.skipUnless(IS_FREE_THREADED, "requires a free threaded build") def test_free_threaded_ref_attribute_getter_and_setter_use_owner(self) -> None: - # Note: We can't monkey patch IS_FREE_THREADED to test this on a build with - # the GIL enabled, since monkey patching doesn't work if mypyc is compiled. cl = ClassIR("A", "mod") cl.attributes = {"o": object_rprimitive} cl.deletable = ["o"] cl.mro = cl.base_mro = [cl] - emitter = Emitter(EmitterContext(NameGenerator([["mod"]]), True)) + emitter = Emitter( + EmitterContext( + NameGenerator([["mod"]]), True, target_python=TargetPython((3, 14), True) + ) + ) generate_getter(cl, "o", object_rprimitive, emitter) generate_setter(cl, "o", object_rprimitive, emitter)
diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index b6e8d13..05c3c4c 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py
@@ -5,7 +5,7 @@ from mypy.test.helpers import assert_string_arrays_equal from mypyc.codegen.emit import Emitter, EmitterContext from mypyc.codegen.emitfunc import FunctionEmitterVisitor, generate_native_function -from mypyc.common import HAVE_IMMORTAL, IS_FREE_THREADED, PLATFORM_SIZE +from mypyc.common import PLATFORM_SIZE from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import ( FUNC_CLASSMETHOD, @@ -77,6 +77,7 @@ ) from mypyc.irbuild.vtable import compute_vtable from mypyc.namegen import NameGenerator +from mypyc.options import TargetPython from mypyc.primitives.dict_ops import ( dict_get_item_op, dict_new_op, @@ -148,6 +149,11 @@ self.context = EmitterContext(NameGenerator([["mod"]]), True) + def use_free_threaded_target(self) -> None: + self.context = EmitterContext( + NameGenerator([["mod"]]), True, target_python=TargetPython((3, 14), True) + ) + def test_goto(self) -> None: self.assert_emit(Goto(BasicBlock(2)), "goto CPyL2;") @@ -539,10 +545,8 @@ """, ) - # Note: We can't monkey patch IS_FREE_THREADED to test this on a build with the - # GIL enabled, since monkey patching doesn't work if mypyc is compiled. - @unittest.skipUnless(IS_FREE_THREADED, "requires a free threaded build") def test_get_attr_ref_free_threaded(self) -> None: + self.use_free_threaded_target() self.assert_emit( GetAttr(self.r, "o", 1), """\ @@ -604,8 +608,8 @@ """, ) - @unittest.skipUnless(IS_FREE_THREADED, "requires a free threaded build") def test_set_attr_ref_free_threaded(self) -> None: + self.use_free_threaded_target() self.assert_emit( SetAttr(self.r, "o", self.o, 1), """\ @@ -1009,16 +1013,23 @@ def test_inc_ref_none(self) -> None: b = Box(self.none) - self.assert_emit([b, IncRef(b)], "" if HAVE_IMMORTAL else "CPy_INCREF(cpy_r_r0);") + self.assert_emit( + [b, IncRef(b)], "" if TargetPython.host().have_immortal else "CPy_INCREF(cpy_r_r0);" + ) def test_inc_ref_bool(self) -> None: b = Box(self.b) - self.assert_emit([b, IncRef(b)], "" if HAVE_IMMORTAL else "CPy_INCREF(cpy_r_r0);") + self.assert_emit( + [b, IncRef(b)], "" if TargetPython.host().have_immortal else "CPy_INCREF(cpy_r_r0);" + ) def test_inc_ref_int_literal(self) -> None: for x in -5, 0, 1, 5, 255, 256: b = LoadLiteral(x, object_rprimitive) - self.assert_emit([b, IncRef(b)], "" if HAVE_IMMORTAL else "CPy_INCREF(cpy_r_r0);") + self.assert_emit( + [b, IncRef(b)], + "" if TargetPython.host().have_immortal else "CPy_INCREF(cpy_r_r0);", + ) for x in -1123355, -6, 257, 123235345: b = LoadLiteral(x, object_rprimitive) self.assert_emit([b, IncRef(b)], "CPy_INCREF(cpy_r_r0);")
diff --git a/mypyc/test/test_exceptions.py b/mypyc/test/test_exceptions.py index d842ea1..ac1b577 100644 --- a/mypyc/test/test_exceptions.py +++ b/mypyc/test/test_exceptions.py
@@ -11,13 +11,15 @@ from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase from mypyc.analysis.blockfreq import frequently_executed_blocks -from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME +from mypyc.common import TOP_LEVEL_NAME from mypyc.ir.pprint import format_func +from mypyc.options import CompilerOptions from mypyc.test.testutil import ( ICODE_GEN_BUILTINS, MypycDataSuite, assert_test_output, build_ir_for_single_file, + infer_target_python_from_test_name, remove_comment_lines, use_custom_builtins, ) @@ -35,17 +37,11 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: """Perform a runtime checking transformation test case.""" - if "_withgil" in testcase.name and IS_FREE_THREADED: - # Test case should only run on a non-free-threaded build. - return - if "_nogil" in testcase.name and not IS_FREE_THREADED: - # Test case should only run on a free-threaded build. - return - + options = CompilerOptions(target_python=infer_target_python_from_test_name(testcase.name)) with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) try: - ir = build_ir_for_single_file(testcase.input) + ir = build_ir_for_single_file(testcase.input, options) except CompileError as e: actual = e.messages else:
diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index 1484d47..0dc19b6 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py
@@ -8,7 +8,7 @@ from mypy.errors import CompileError from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase -from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME +from mypyc.common import TOP_LEVEL_NAME from mypyc.ir.pprint import format_func from mypyc.test.testutil import ( ICODE_GEN_BUILTINS, @@ -88,12 +88,6 @@ if options is None: # Skipped test case return - if "_withgil" in testcase.name and IS_FREE_THREADED: - # Test case should only run on a non-free-threaded build. - return - if "_nogil" in testcase.name and not IS_FREE_THREADED: - # Test case should only run on a free-threaded build. - return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) expected_output = replace_word_size(expected_output)
diff --git a/mypyc/test/test_lowering.py b/mypyc/test/test_lowering.py index 75351cd..3eb4698 100644 --- a/mypyc/test/test_lowering.py +++ b/mypyc/test/test_lowering.py
@@ -7,7 +7,7 @@ from mypy.errors import CompileError from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase -from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME +from mypyc.common import TOP_LEVEL_NAME from mypyc.ir.pprint import format_func from mypyc.options import CompilerOptions from mypyc.test.testutil import ( @@ -36,12 +36,6 @@ if options is None: # Skipped test case return - if "_withgil" in testcase.name and IS_FREE_THREADED: - # Test case should only run on a non-free-threaded build. - return - if "_nogil" in testcase.name and not IS_FREE_THREADED: - # Test case should only run on a free-threaded build. - return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) expected_output = replace_word_size(expected_output)
diff --git a/mypyc/test/test_optimizations.py b/mypyc/test/test_optimizations.py index c6a40d0..b71071e 100644 --- a/mypyc/test/test_optimizations.py +++ b/mypyc/test/test_optimizations.py
@@ -7,7 +7,7 @@ from mypy.errors import CompileError from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase -from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME +from mypyc.common import TOP_LEVEL_NAME from mypyc.ir.func_ir import FuncIR from mypyc.ir.pprint import format_func from mypyc.options import CompilerOptions @@ -16,6 +16,7 @@ MypycDataSuite, assert_test_output, build_ir_for_single_file, + infer_target_python_from_test_name, remove_comment_lines, use_custom_builtins, ) @@ -33,16 +34,11 @@ base_path = test_temp_dir def run_case(self, testcase: DataDrivenTestCase) -> None: - if "_withgil" in testcase.name and IS_FREE_THREADED: - # Test case should only run on a non-free-threaded build. - return - if "_nogil" in testcase.name and not IS_FREE_THREADED: - # Test case should only run on a free-threaded build. - return + options = CompilerOptions(target_python=infer_target_python_from_test_name(testcase.name)) with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) try: - ir = build_ir_for_single_file(testcase.input) + ir = build_ir_for_single_file(testcase.input, options) except CompileError as e: actual = e.messages else:
diff --git a/mypyc/test/test_refcount.py b/mypyc/test/test_refcount.py index 896ba14..7efa861 100644 --- a/mypyc/test/test_refcount.py +++ b/mypyc/test/test_refcount.py
@@ -11,7 +11,7 @@ from mypy.errors import CompileError from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase -from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME +from mypyc.common import TOP_LEVEL_NAME from mypyc.ir.pprint import format_func from mypyc.test.testutil import ( ICODE_GEN_BUILTINS, @@ -40,12 +40,6 @@ if options is None: # Skipped test case return - if "_withgil" in testcase.name and IS_FREE_THREADED: - # Test case should only run on a non-free-threaded build. - return - if "_nogil" in testcase.name and not IS_FREE_THREADED: - # Test case should only run on a free-threaded build. - return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) expected_output = replace_word_size(expected_output)
diff --git a/mypyc/test/test_target_python.py b/mypyc/test/test_target_python.py new file mode 100644 index 0000000..1ac87d0 --- /dev/null +++ b/mypyc/test/test_target_python.py
@@ -0,0 +1,104 @@ +"""Tests for targeting a Python build other than the running one.""" + +from __future__ import annotations + +import os +import tempfile +import unittest + +from mypyc.build import get_mypy_config +from mypyc.codegen.emit import Emitter, EmitterContext +from mypyc.codegen.emitmodule import GroupGenerator +from mypyc.common import EXT_SUFFIX +from mypyc.ir.rtypes import RUnion, list_rprimitive, object_rprimitive +from mypyc.namegen import NameGenerator +from mypyc.options import CompilerOptions, TargetPython, replace_ext_suffix_version +from mypyc.test.testutil import infer_target_python_from_test_name + + +class TestTargetPython(unittest.TestCase): + def test_parse(self) -> None: + assert TargetPython.parse("3.13") == TargetPython((3, 13), False) + assert TargetPython.parse("3.14t") == TargetPython((3, 14), True) + assert TargetPython.parse("3.10") == TargetPython((3, 10), False) + + def test_parse_invalid(self) -> None: + for target in ["3", "3.13x", "313", "2.7", "4.0", "3.9", "3.12t", "t", "3.10t"]: + with self.assertRaises(ValueError, msg=target): + TargetPython.parse(target) + + def test_str(self) -> None: + assert str(TargetPython((3, 13), False)) == "3.13" + assert str(TargetPython((3, 14), True)) == "3.14t" + + def test_have_immortal(self) -> None: + assert not TargetPython((3, 11)).have_immortal + assert TargetPython((3, 12)).have_immortal + + def test_ext_suffix(self) -> None: + assert TargetPython.host().ext_suffix == EXT_SUFFIX + for host, target, expected in [ + (".cpython-313-x86_64-linux-gnu.so", "3.14t", ".cpython-314t-x86_64-linux-gnu.so"), + (".cpython-314t-darwin.so", "3.13", ".cpython-313-darwin.so"), + (".cp313-win_amd64.pyd", "3.14t", ".cp314t-win_amd64.pyd"), + ("_d.cp313t-win_amd64.pyd", "3.12", "_d.cp312-win_amd64.pyd"), + ]: + parsed = TargetPython.parse(target) + result = replace_ext_suffix_version(host, parsed.version, parsed.free_threaded) + assert result == expected + for suffix in [".so", ".abi3.so", ".pypy310-pp73-x86_64-linux-gnu.so"]: + with self.assertRaises(ValueError): + replace_ext_suffix_version(suffix, (3, 14), True) + + def test_may_be_immortal(self) -> None: + old, new = TargetPython((3, 11)), TargetPython((3, 12)) + assert not object_rprimitive.may_be_immortal(old) + assert object_rprimitive.may_be_immortal(new) + assert not list_rprimitive.may_be_immortal(new) + assert RUnion([list_rprimitive, object_rprimitive]).may_be_immortal(new) + + def test_emit_ref_count_ops(self) -> None: + def emit(target: TargetPython) -> str: + emitter = Emitter(EmitterContext(NameGenerator([["mod"]]), True, target_python=target)) + emitter.emit_inc_ref("x", list_rprimitive) + emitter.emit_dec_ref("x", list_rprimitive) + return "".join(emitter.fragments) + + assert emit(TargetPython((3, 11))) == "CPy_INCREF(x);\nCPy_DECREF(x);\n" + assert emit(TargetPython((3, 12))) == "CPy_INCREF_NO_IMM(x);\nCPy_DECREF_NO_IMM(x);\n" + + def test_invalid_target(self) -> None: + for version, free_threaded in [((3, 9), False), ((2, 7), False), ((3, 12), True)]: + with self.assertRaises(ValueError): + TargetPython(version, free_threaded) + + def test_infer_target_python_from_test_name(self) -> None: + assert infer_target_python_from_test_name("testFoo") == TargetPython((3, 10)) + assert infer_target_python_from_test_name("testFoo_withgil") == TargetPython((3, 10)) + assert infer_target_python_from_test_name("testFoo_nogil") == TargetPython((3, 13), True) + assert infer_target_python_from_test_name("testFoo_python3_14_nogil") == TargetPython( + (3, 14), True + ) + assert infer_target_python_from_test_name("testFoo_python3_12") == TargetPython((3, 12)) + + def test_module_def_slots(self) -> None: + def slots(target: TargetPython) -> str: + context = EmitterContext(NameGenerator([["mod"]]), True, target_python=target) + emitter = Emitter(context) + generator = GroupGenerator({}, {}, None, {}, context.names, CompilerOptions()) + generator.emit_module_def_slots(emitter, "prefix", "mod") + return "".join(emitter.fragments) + + assert "Py_mod_gil" not in slots(TargetPython((3, 12))) + assert "Py_mod_multiple_interpreters" in slots(TargetPython((3, 12))) + assert "Py_mod_gil" in slots(TargetPython((3, 13))) + + def test_mypy_python_version_follows_target(self) -> None: + # This is used by mypyc_build(), which is how a target is selected. + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "a.py") + with open(path, "w") as f: + f.write("x = 1\n") + options = CompilerOptions(target_python=TargetPython((3, 14), True)) + _, _, mypy_options = get_mypy_config([path], None, options, None) + assert mypy_options.python_version == (3, 14)
diff --git a/mypyc/test/testutil.py b/mypyc/test/testutil.py index 2d139ac..cf56d76 100644 --- a/mypyc/test/testutil.py +++ b/mypyc/test/testutil.py
@@ -10,6 +10,7 @@ from collections.abc import Callable, Iterator from mypy import build +from mypy.defaults import PYTHON3_VERSION_MIN from mypy.errors import CompileError from mypy.main import process_options from mypy.nodes import Expression, MypyFile @@ -25,7 +26,7 @@ from mypyc.ir.module_ir import ModuleIR from mypyc.irbuild.main import build_ir from mypyc.irbuild.mapper import Mapper -from mypyc.options import CompilerOptions +from mypyc.options import CompilerOptions, TargetPython from mypyc.test.config import test_data_prefix # The builtins stub used during icode generation test cases. @@ -107,7 +108,9 @@ # By default, generate IR compatible with the earliest supported Python C API. # If a test needs more recent API features, this should be overridden. - compiler_options = compiler_options or CompilerOptions(capi_version=(3, 10)) + compiler_options = compiler_options or CompilerOptions( + target_python=TargetPython(PYTHON3_VERSION_MIN) + ) if flags: flag_list = flags.group(1).split() _, options = process_options(flag_list, require_targets=False) @@ -117,7 +120,7 @@ options.hide_error_codes = True options.use_builtins_fixtures = True options.strict_optional = True - options.python_version = compiler_options.python_version or (3, 10) + options.python_version = compiler_options.target_python.version options.export_types = True options.preserve_asts = True options.allow_empty_bodies = True @@ -281,6 +284,8 @@ Run test caseonly on 32-bit platforms *_python3_10* (or for any Python version): Use Python 3.10+ C API features (default: lowest supported version) + *_nogil*: + Target a free-threaded build (default: build with the GIL) *StripAssert*: Don't generate code for assert statements """ @@ -290,22 +295,35 @@ if "_32bit" in name and not IS_32_BIT_PLATFORM: return None options = CompilerOptions( - strip_asserts="StripAssert" in name, capi_version=(3, 10), strict_traceback_checks=True + strip_asserts="StripAssert" in name, + strict_traceback_checks=True, + target_python=infer_target_python_from_test_name(name), ) - # A suffix like _python3_10 is used to set the target C API version. - m = re.search(r"_python([0-9]+)_([0-9]+)(_|\b)", name) - if m: - version = (int(m.group(1)), int(m.group(2))) - assert version >= (3, 10), f"Unsupported _python* suffix: {name}" - options.capi_version = version - options.python_version = options.capi_version - elif "_py" in name or "_Python" in name: - assert False, f"Invalid _py* suffix (should be _pythonX_Y): {name}" if has_test_name_tag(name, "experimental"): options.experimental_features = True return options +def infer_target_python_from_test_name(name: str) -> TargetPython: + """Infer the target Python of a test case from magic substrings in its name. + + A suffix like _python3_12 sets the version (default: lowest supported version). + A _nogil suffix targets a free-threaded build (default: build with the GIL), + and this defaults to Python 3.13, since free threading requires it. + + The running interpreter doesn't matter, so test output is the same everywhere. + """ + free_threaded = "_nogil" in name + m = re.search(r"_python([0-9]+)_([0-9]+)(_|\b)", name) + if m: + version = (int(m.group(1)), int(m.group(2))) + elif "_py" in name or "_Python" in name: + assert False, f"Invalid _py* suffix (should be _pythonX_Y): {name}" + else: + version = (3, 13) if free_threaded else PYTHON3_VERSION_MIN + return TargetPython(version, free_threaded) + + def has_test_name_tag(name: str, tag: str) -> bool: """Check if a test case name contains a tag token like ``_experimental``.