[mypyc] Call __index__ when converting objects to u32 and u64 (#22484)
Converting an object to `u32` or `u64` didn't call `__index__`, unlike
every other native int type. The unboxing slow paths
`CPyLong_AsUInt32_()` and `CPyLong_AsUInt64_()` use
`PyLong_AsUnsignedLong()` and `PyLong_AsUnsignedLongLong()`, which only
accept int objects, while the helpers for `u8`, `u16` and the signed
types use `PyLong_AsLongAndOverflow()`, which calls `__index__`.
This shows up in three places:
```python
from typing import Any
from mypy_extensions import u16, u64
import librt.strings
class Index:
def __init__(self, n: int) -> None:
self.n = n
def __index__(self) -> int:
return self.n
def take_u16(x: u16) -> u16: return x
def take_u64(x: u64) -> u64: return x
def coerce_u64(o: Any) -> u64:
x: u64 = o
return x
```
| | `u8`, `u16`, `i64` etc. | `u32`, `u64` before | `u32`, `u64` after |
|---|---|---|---|
| `take_u64(Index(5))` called from Python | `5` | `TypeError: u64 object
expected; got Index` | `5` |
| `coerce_u64(Index(5))` | `5` | `TypeError: an integer is required` |
`5` |
| `coerce_u64(1.0)` | `TypeError: 'float' object cannot be interpreted
as an integer` | `TypeError: an integer is required` | same as `u16` |
| `librt.strings.write_u64_le(w, Index(5))` | writes `5` | `TypeError:
an integer is required` | writes `5` |
Now the slow paths convert a non-int object with `PyNumber_Index()`
first. The fast path (a one-digit int) isn't changed, and an int still
goes straight to `PyLong_AsUnsigned*()`, so the only extra cost on the
slow path for ints is an incref/decref. Negative and too large values
(also when returned by `__index__`) still raise `ValueError: int too
large or small to convert to u64`, and the overlapping error value 239
is handled as before.
Tests: `testU32BasicOps` and `testU64BasicOps` now unbox objects with
`__index__` (including 239 and the maximum value, and -1 and too large
values, which raise `ValueError`), through both an implicit coercion and
a call from Python, and check the errors for a float.
`testLibrtStringsI8AndUnsigned_librt` writes `Index` objects with all
`write_u16/u32/u64_le/be` functions and checks the same errors. The
three tests fail without the change to `int_ops.c`.
diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c
index 36548e8..5bd6020 100644
--- a/mypyc/lib-rt/int_ops.c
+++ b/mypyc/lib-rt/int_ops.c
@@ -648,9 +648,24 @@
PyErr_SetString(PyExc_ValueError, "int too large to convert to i16");
}
+// Unlike PyLong_AsLongAndOverflow(), which the other native int types use,
+// PyLong_AsUnsignedLong() and PyLong_AsUnsignedLongLong() don't call __index__, so
+// convert the object first. Return a new reference, or NULL with an exception set.
+static PyObject *CPyLong_IndexForUnsigned(PyObject *o) {
+ if (likely(PyLong_Check(o))) {
+ return Py_NewRef(o);
+ }
+ return PyNumber_Index(o);
+}
+
// u64 unboxing slow path
uint64_t CPyLong_AsUInt64_(PyObject *o) {
- unsigned long long result = PyLong_AsUnsignedLongLong(o);
+ PyObject *index = CPyLong_IndexForUnsigned(o);
+ if (index == NULL) {
+ return CPY_LL_UINT_ERROR;
+ }
+ unsigned long long result = PyLong_AsUnsignedLongLong(index);
+ Py_DECREF(index);
if (result == (unsigned long long)-1 && PyErr_Occurred()) {
if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
PyErr_Clear();
@@ -667,7 +682,12 @@
// u32 unboxing slow path
uint32_t CPyLong_AsUInt32_(PyObject *o) {
- unsigned long result = PyLong_AsUnsignedLong(o);
+ PyObject *index = CPyLong_IndexForUnsigned(o);
+ if (index == NULL) {
+ return CPY_LL_UINT_ERROR;
+ }
+ unsigned long result = PyLong_AsUnsignedLong(index);
+ Py_DECREF(index);
if (result == (unsigned long)-1 && PyErr_Occurred()) {
if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
PyErr_Clear();
diff --git a/mypyc/test-data/run-librt-strings.test b/mypyc/test-data/run-librt-strings.test
index e48f506..0e703ee 100644
--- a/mypyc/test-data/run-librt-strings.test
+++ b/mypyc/test-data/run-librt-strings.test
@@ -1686,6 +1686,29 @@
with assertRaises(TypeError):
write_func(w)
+class Index:
+ def __init__(self, n: int) -> None:
+ self.n = n
+
+ def __index__(self) -> int:
+ return self.n
+
+def test_write_via_any_index_and_non_int() -> None:
+ for write_func, fmt, bits in WRITE_FUNCS:
+ values = [0, 5, 239, (1 << bits) - 1]
+ w: Any = BytesWriter()
+ for v in values:
+ write_func(w, Index(v))
+ assert w.getvalue() == b"".join(struct.pack(fmt, v) for v in values)
+
+ msg = f"int too large or small to convert to u{bits}"
+ with assertRaises(ValueError, msg):
+ write_func(w, Index(1 << bits))
+ with assertRaises(ValueError, msg):
+ write_func(w, Index(-1))
+ with assertRaises(TypeError, "'float' object cannot be interpreted as an integer"):
+ write_func(w, 1.0)
+
def test_read_via_any() -> None:
for read_func, fmt, bits in READ_FUNCS:
values = [0, 239, (1 << bits) - 1, 1 << (bits - 1)]
diff --git a/mypyc/test-data/run-u32.test b/mypyc/test-data/run-u32.test
index 847fcda..026d6fe 100644
--- a/mypyc/test-data/run-u32.test
+++ b/mypyc/test-data/run-u32.test
@@ -21,6 +21,33 @@
o = -1
x3: u32 = o
+class Index:
+ def __init__(self, n: int) -> None:
+ self.n = n
+
+ def __index__(self) -> int:
+ return self.n
+
+def take_u32(x: u32) -> u32:
+ return x
+
+def test_unbox_index_and_non_int() -> None:
+ f: Any = take_u32
+ for n in [0, 5, ERROR, 2**30, 4294967295]:
+ o: Any = Index(n)
+ x: u32 = o
+ assert x == n
+ assert f(o) == n
+ for n in [-1, 4294967296, 18446744073709551616]:
+ o = Index(n)
+ with assertRaises(ValueError, "int too large or small to convert to u32"):
+ x = o
+ o = 1.0
+ with assertRaises(TypeError, "'float' object cannot be interpreted as an integer"):
+ x = o
+ with assertRaises(TypeError, "u32 object expected; got float"):
+ f(o)
+
def div_by_7(x: u32) -> u32:
return x // 7
diff --git a/mypyc/test-data/run-u64.test b/mypyc/test-data/run-u64.test
index abb918a..d94a952 100644
--- a/mypyc/test-data/run-u64.test
+++ b/mypyc/test-data/run-u64.test
@@ -21,6 +21,33 @@
o = -1
x3: u64 = o
+class Index:
+ def __init__(self, n: int) -> None:
+ self.n = n
+
+ def __index__(self) -> int:
+ return self.n
+
+def take_u64(x: u64) -> u64:
+ return x
+
+def test_unbox_index_and_non_int() -> None:
+ f: Any = take_u64
+ for n in [0, 5, ERROR, 2**40, 18446744073709551615]:
+ o: Any = Index(n)
+ x: u64 = o
+ assert x == n
+ assert f(o) == n
+ for n in [-1, 18446744073709551616]:
+ o = Index(n)
+ with assertRaises(ValueError, "int too large or small to convert to u64"):
+ x = o
+ o = 1.0
+ with assertRaises(TypeError, "'float' object cannot be interpreted as an integer"):
+ x = o
+ with assertRaises(TypeError, "u64 object expected; got float"):
+ f(o)
+
def div_by_7(x: u64) -> u64:
return x // 7