[mypyc] Call __index__ when converting objects to u32 and u64 (#22484)

Converting an object to `u32` or `u64` didn't call `__index__`, unlike
every other native int type. The unboxing slow paths
`CPyLong_AsUInt32_()` and `CPyLong_AsUInt64_()` use
`PyLong_AsUnsignedLong()` and `PyLong_AsUnsignedLongLong()`, which only
accept int objects, while the helpers for `u8`, `u16` and the signed
types use `PyLong_AsLongAndOverflow()`, which calls `__index__`.

This shows up in three places:

```python
from typing import Any
from mypy_extensions import u16, u64
import librt.strings

class Index:
    def __init__(self, n: int) -> None:
        self.n = n
    def __index__(self) -> int:
        return self.n

def take_u16(x: u16) -> u16: return x
def take_u64(x: u64) -> u64: return x

def coerce_u64(o: Any) -> u64:
    x: u64 = o
    return x
```

| | `u8`, `u16`, `i64` etc. | `u32`, `u64` before | `u32`, `u64` after |
|---|---|---|---|
| `take_u64(Index(5))` called from Python | `5` | `TypeError: u64 object
expected; got Index` | `5` |
| `coerce_u64(Index(5))` | `5` | `TypeError: an integer is required` |
`5` |
| `coerce_u64(1.0)` | `TypeError: 'float' object cannot be interpreted
as an integer` | `TypeError: an integer is required` | same as `u16` |
| `librt.strings.write_u64_le(w, Index(5))` | writes `5` | `TypeError:
an integer is required` | writes `5` |

Now the slow paths convert a non-int object with `PyNumber_Index()`
first. The fast path (a one-digit int) isn't changed, and an int still
goes straight to `PyLong_AsUnsigned*()`, so the only extra cost on the
slow path for ints is an incref/decref. Negative and too large values
(also when returned by `__index__`) still raise `ValueError: int too
large or small to convert to u64`, and the overlapping error value 239
is handled as before.

Tests: `testU32BasicOps` and `testU64BasicOps` now unbox objects with
`__index__` (including 239 and the maximum value, and -1 and too large
values, which raise `ValueError`), through both an implicit coercion and
a call from Python, and check the errors for a float.
`testLibrtStringsI8AndUnsigned_librt` writes `Index` objects with all
`write_u16/u32/u64_le/be` functions and checks the same errors. The
three tests fail without the change to `int_ops.c`.
diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c
index 36548e8..5bd6020 100644
--- a/mypyc/lib-rt/int_ops.c
+++ b/mypyc/lib-rt/int_ops.c
@@ -648,9 +648,24 @@
     PyErr_SetString(PyExc_ValueError, "int too large to convert to i16");
 }
 
+// Unlike PyLong_AsLongAndOverflow(), which the other native int types use,
+// PyLong_AsUnsignedLong() and PyLong_AsUnsignedLongLong() don't call __index__, so
+// convert the object first. Return a new reference, or NULL with an exception set.
+static PyObject *CPyLong_IndexForUnsigned(PyObject *o) {
+    if (likely(PyLong_Check(o))) {
+        return Py_NewRef(o);
+    }
+    return PyNumber_Index(o);
+}
+
 // u64 unboxing slow path
 uint64_t CPyLong_AsUInt64_(PyObject *o) {
-    unsigned long long result = PyLong_AsUnsignedLongLong(o);
+    PyObject *index = CPyLong_IndexForUnsigned(o);
+    if (index == NULL) {
+        return CPY_LL_UINT_ERROR;
+    }
+    unsigned long long result = PyLong_AsUnsignedLongLong(index);
+    Py_DECREF(index);
     if (result == (unsigned long long)-1 && PyErr_Occurred()) {
         if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
             PyErr_Clear();
@@ -667,7 +682,12 @@
 
 // u32 unboxing slow path
 uint32_t CPyLong_AsUInt32_(PyObject *o) {
-    unsigned long result = PyLong_AsUnsignedLong(o);
+    PyObject *index = CPyLong_IndexForUnsigned(o);
+    if (index == NULL) {
+        return CPY_LL_UINT_ERROR;
+    }
+    unsigned long result = PyLong_AsUnsignedLong(index);
+    Py_DECREF(index);
     if (result == (unsigned long)-1 && PyErr_Occurred()) {
         if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
             PyErr_Clear();
diff --git a/mypyc/test-data/run-librt-strings.test b/mypyc/test-data/run-librt-strings.test
index e48f506..0e703ee 100644
--- a/mypyc/test-data/run-librt-strings.test
+++ b/mypyc/test-data/run-librt-strings.test
@@ -1686,6 +1686,29 @@
         with assertRaises(TypeError):
             write_func(w)
 
+class Index:
+    def __init__(self, n: int) -> None:
+        self.n = n
+
+    def __index__(self) -> int:
+        return self.n
+
+def test_write_via_any_index_and_non_int() -> None:
+    for write_func, fmt, bits in WRITE_FUNCS:
+        values = [0, 5, 239, (1 << bits) - 1]
+        w: Any = BytesWriter()
+        for v in values:
+            write_func(w, Index(v))
+        assert w.getvalue() == b"".join(struct.pack(fmt, v) for v in values)
+
+        msg = f"int too large or small to convert to u{bits}"
+        with assertRaises(ValueError, msg):
+            write_func(w, Index(1 << bits))
+        with assertRaises(ValueError, msg):
+            write_func(w, Index(-1))
+        with assertRaises(TypeError, "'float' object cannot be interpreted as an integer"):
+            write_func(w, 1.0)
+
 def test_read_via_any() -> None:
     for read_func, fmt, bits in READ_FUNCS:
         values = [0, 239, (1 << bits) - 1, 1 << (bits - 1)]
diff --git a/mypyc/test-data/run-u32.test b/mypyc/test-data/run-u32.test
index 847fcda..026d6fe 100644
--- a/mypyc/test-data/run-u32.test
+++ b/mypyc/test-data/run-u32.test
@@ -21,6 +21,33 @@
         o = -1
         x3: u32 = o
 
+class Index:
+    def __init__(self, n: int) -> None:
+        self.n = n
+
+    def __index__(self) -> int:
+        return self.n
+
+def take_u32(x: u32) -> u32:
+    return x
+
+def test_unbox_index_and_non_int() -> None:
+    f: Any = take_u32
+    for n in [0, 5, ERROR, 2**30, 4294967295]:
+        o: Any = Index(n)
+        x: u32 = o
+        assert x == n
+        assert f(o) == n
+    for n in [-1, 4294967296, 18446744073709551616]:
+        o = Index(n)
+        with assertRaises(ValueError, "int too large or small to convert to u32"):
+            x = o
+    o = 1.0
+    with assertRaises(TypeError, "'float' object cannot be interpreted as an integer"):
+        x = o
+    with assertRaises(TypeError, "u32 object expected; got float"):
+        f(o)
+
 def div_by_7(x: u32) -> u32:
     return x // 7
 
diff --git a/mypyc/test-data/run-u64.test b/mypyc/test-data/run-u64.test
index abb918a..d94a952 100644
--- a/mypyc/test-data/run-u64.test
+++ b/mypyc/test-data/run-u64.test
@@ -21,6 +21,33 @@
         o = -1
         x3: u64 = o
 
+class Index:
+    def __init__(self, n: int) -> None:
+        self.n = n
+
+    def __index__(self) -> int:
+        return self.n
+
+def take_u64(x: u64) -> u64:
+    return x
+
+def test_unbox_index_and_non_int() -> None:
+    f: Any = take_u64
+    for n in [0, 5, ERROR, 2**40, 18446744073709551615]:
+        o: Any = Index(n)
+        x: u64 = o
+        assert x == n
+        assert f(o) == n
+    for n in [-1, 18446744073709551616]:
+        o = Index(n)
+        with assertRaises(ValueError, "int too large or small to convert to u64"):
+            x = o
+    o = 1.0
+    with assertRaises(TypeError, "'float' object cannot be interpreted as an integer"):
+        x = o
+    with assertRaises(TypeError, "u64 object expected; got float"):
+        f(o)
+
 def div_by_7(x: u64) -> u64:
     return x // 7