Avoid upcalls for long and float objects where we can

Author: timfel

graalpython/com.oracle.graal.python.cext/include/graalpy/handles.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -45,12 +45,50 @@
 #include <object.h>
 
 #define MANAGED_REFCNT 10
-#define HANDLE_BASE 0x8000000000000000ULL
+
+/*
+ * We cannot do NaN tagging. Even if we rely on running systems that use at
+ * most 48 bits for their adresses (so not running with PML5/la57 enabled), we
+ * do not control all allocations, so PyObject* allocated somewhere else will
+ * not be tagged and thus are indistinguishable from subnormal floating point
+ * values or +0.0 for NULL values. Additionally, we know that at least cffi
+ * uses tagging of the low 3 alignment bits o PyObject* and thus those also
+ * need to be 0 for anything we hand out. So we can realistically only use the
+ * upper 16 and lower 45 bits, which isn't enough to do NaN tagging (unless we
+ * want to do weird stuff like disallow certain double bit patterns). So the
+ * thing we can do is to tag managed pointers, 32-bit integers, and double
+ * values that fit into single-precision floats without loss.
+ */
+
+// Aligned with the same constants in CApiTransitions. Update comment there if
+// you change or move these.
+#define HANDLE_TAG_BIT  ((uint64_t)(1ULL << 63))
+#define INTEGER_TAG_BIT ((uint64_t)(1ULL << 62))
+#define FLOAT_TAG_BIT   ((uint64_t)(1ULL << 61))
+
  // IMMORTAL_REFCNT value is aligned with include/object.h
 #define IMMORTAL_REFCNT 0xFFFFFFFFL
+#define _35_BIT_MASK (0xFFFFFFFFULL << 3)
+
+#define points_to_py_handle_space(PTR) (((uint64_t)(uintptr_t)(PTR)) & HANDLE_TAG_BIT)
+#define points_to_py_int_handle(PTR) (((uint64_t)(uintptr_t)(PTR)) & INTEGER_TAG_BIT)
+#define points_to_py_float_handle(PTR) (((uint64_t)(uintptr_t)(PTR)) & FLOAT_TAG_BIT)
+
+#define stub_to_pointer(STUB_PTR) ((uintptr_t)(((uint64_t)(uintptr_t)(PTR)) | HANDLE_TAG_BIT))
+#define int_to_pointer(INT)       ((uintptr_t)((((uint64_t)(uint32_t)(INT) << 3) & _35_BIT_MASK) | HANDLE_TAG_BIT | INTEGER_TAG_BIT))
+static inline PyObject* float_to_pointer(float dbl) {
+    uint32_t float_bits;
+    memcpy(&float_bits, &dbl, sizeof(float));
+    return (PyObject *)(uintptr_t)(((((uint64_t)float_bits) << 3) & _35_BIT_MASK) | HANDLE_TAG_BIT | FLOAT_TAG_BIT);
+}
 
-#define points_to_py_handle_space(PTR) ((((uintptr_t) (PTR)) & HANDLE_BASE) != 0)
-#define stub_to_pointer(STUB_PTR) (((uintptr_t) (STUB_PTR)) | HANDLE_BASE)
-#define pointer_to_stub(PTR) ((PyObject *)(((uintptr_t) (PTR)) & ~HANDLE_BASE))
+#define pointer_to_stub(PTR)  ((PyObject*)(((uint64_t)(uintptr_t)(PTR)) & ~HANDLE_TAG_BIT))
+#define pointer_to_long(PTR)   ((int64_t)(int32_t)(((uint64_t)(uintptr_t)(PTR)) >> 3))
+static inline double pointer_to_double(PyObject* ptr) {
+    uint32_t float_bits = (uint32_t)(((uint64_t)(uintptr_t)ptr) >> 3);
+    float value;
+    memcpy(&value, &float_bits, sizeof(float));
+    return (double)value;
+}
 
 #endif /* SRC_HANDLES_H_ */

graalpython/com.oracle.graal.python.cext/include/internal/pycore_gc.h

@@ -33,7 +33,7 @@ static inline PyGC_Head* _Py_AS_GC(PyObject *op) {
 #define _PyGC_Head_UNUSED PyGC_Head
 
 // GraalPy change
-#define _PyGCHead_UNTAG(PTR) ((PyGC_Head *)(((uintptr_t) (PTR)) & ~HANDLE_BASE))
+#define _PyGCHead_UNTAG(PTR) ((PyGC_Head *)(((uintptr_t) (PTR)) & ~HANDLE_TAG_BIT))
 
 /* True if the object is currently tracked by the GC. */
 static inline int _PyObject_GC_IS_TRACKED(PyObject *op) {

graalpython/com.oracle.graal.python.cext/src/floatobject.c

@@ -145,11 +145,20 @@ PyFloat_GetInfo(void)
 
     return floatinfo;
 }
+#endif
 
 PyObject *
 PyFloat_FromDouble(double fval)
 {
     PyFloatObject *op;
+    // GraalPy: different implementation
+    if (fval == (float)fval) {
+        return float_to_pointer(fval);
+    } else {
+        return GraalPyPrivate_Float_FromDouble(fval);
+    }
+}
+#if 0 // GraalPy change
 #if PyFloat_MAXFREELIST > 0
     struct _Py_float_state *state = get_float_state();
     op = state->free_list;
@@ -323,6 +332,9 @@ PyFloat_AsDouble(PyObject *op)
 
     // GraalPy change: upcall for managed
     if (points_to_py_handle_space(op)) {
+        if (points_to_py_int_handle(op)) {
+            return (double)pointer_to_long(op);
+        }
         return GraalPyPrivate_Float_AsDouble(op);
     }
 
@@ -2661,6 +2673,9 @@ PyFloat_Unpack8(const char *data, int le)
 
 double GraalPyFloat_AS_DOUBLE(PyObject *op) {
     if (points_to_py_handle_space(op)) {
+        if (points_to_py_float_handle(op)) {
+            return pointer_to_double(op);
+        }
         return ((GraalPyFloatObject*) pointer_to_stub(op))->ob_fval;
     } else {
         return _PyFloat_CAST(op)->ob_fval;

graalpython/com.oracle.graal.python.cext/src/longobject.c

@@ -337,7 +337,7 @@ PyLong_FromLong(long ival)
     if (IS_SMALL_INT(ival)) {
         return get_small_int((sdigit)ival);
     }
-    return GraalPyPrivate_Long_FromLongLong((long long) ival);
+    return PyLong_FromLongLong((long long) ival);
 }
 
 #if 0 // GraalPy change
@@ -477,6 +477,10 @@ PyLong_AsLongAndOverflow(PyObject *vv, int *overflow)
         PyErr_BadInternalCall();
         return -1;
     }
+    if (points_to_py_int_handle(vv)) {
+        *overflow = 0;
+        return pointer_to_long(vv);
+    }
     long result = (long) GraalPyPrivate_Long_AsPrimitive(vv, MODE_COERCE_SIGNED, sizeof(long));
     if (result == -1L && PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_OverflowError)) {
         PyErr_Clear();
@@ -494,6 +498,9 @@ long
 PyLong_AsLong(PyObject *obj)
 {
     // GraalPy change: different implementation
+    if (points_to_py_int_handle(obj)) {
+        return pointer_to_long(obj);
+    }
     return (long) GraalPyPrivate_Long_AsPrimitive(obj, MODE_COERCE_SIGNED, sizeof(long));
 }
 
@@ -520,6 +527,9 @@ _PyLong_AsInt(PyObject *obj)
 
 Py_ssize_t
 PyLong_AsSsize_t(PyObject *vv) {
+    if (points_to_py_int_handle(vv)) {
+        return pointer_to_long(vv);
+    }
     return (Py_ssize_t) GraalPyPrivate_Long_AsPrimitive(vv, MODE_PINT_SIGNED, sizeof(Py_ssize_t));
 }
 
@@ -534,6 +544,14 @@ PyLong_AsUnsignedLong(PyObject *vv)
         PyErr_BadInternalCall();
         return (unsigned long) -1;
     }
+    if (points_to_py_int_handle(vv)) {
+        long value = pointer_to_long(vv);
+        if (value < 0) {
+            PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned int");
+            return (unsigned long) -1;
+        }
+        return (unsigned long) value;
+    }
     return (unsigned long) GraalPyPrivate_Long_AsPrimitive(vv, MODE_PINT_UNSIGNED, sizeof(unsigned long));
 }
 
@@ -544,6 +562,14 @@ size_t
 PyLong_AsSize_t(PyObject *vv)
 {
     // GraalPy change: different implementation
+    if (points_to_py_int_handle(vv)) {
+        long value = pointer_to_long(vv);
+        if (value < 0) {
+            PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned int");
+            return (size_t) -1;
+        }
+        return (size_t) value;
+    }
     return (size_t) GraalPyPrivate_Long_AsPrimitive(vv, MODE_PINT_UNSIGNED, sizeof(size_t));
 }
 
@@ -588,6 +614,9 @@ PyLong_AsUnsignedLongMask(PyObject *op)
         PyErr_BadInternalCall();
         return (unsigned long) -1;
     }
+    if (points_to_py_int_handle(op)) {
+        return pointer_to_long(op);
+    }
     return (unsigned long) GraalPyPrivate_Long_AsPrimitive(op, MODE_COERCE_MASK, sizeof(unsigned long));
 }
 
@@ -957,7 +986,11 @@ PyLong_FromLongLong(long long ival)
     if (IS_SMALL_INT(ival)) {
         return get_small_int((sdigit)ival);
     }
-    return GraalPyPrivate_Long_FromLongLong(ival);
+    if ((int)ival == ival) {
+        return int_to_pointer(ival);
+    } else {
+        return GraalPyPrivate_Long_FromLongLong(ival);
+    }
 }
 
 /* Create a new int object from a C Py_ssize_t. */
@@ -969,7 +1002,7 @@ PyLong_FromSsize_t(Py_ssize_t ival)
     if (IS_SMALL_INT(ival)) {
         return get_small_int((sdigit)ival);
     }
-    return GraalPyPrivate_Long_FromLongLong((long long) ival);
+    return PyLong_FromLongLong((long long) ival);
 }
 
 /* Get a C long long int from an int object or any object that has an
@@ -983,6 +1016,9 @@ PyLong_AsLongLong(PyObject *vv)
         PyErr_BadInternalCall();
         return -1;
     }
+    if (points_to_py_int_handle(vv)) {
+        return pointer_to_long(vv);
+    }
     return (long long) GraalPyPrivate_Long_AsPrimitive(vv, MODE_COERCE_SIGNED, sizeof(long long));
 }
 
@@ -997,6 +1033,14 @@ PyLong_AsUnsignedLongLong(PyObject *vv)
         PyErr_BadInternalCall();
         return (unsigned long long)-1;
     }
+    if (points_to_py_int_handle(vv)) {
+        long value = pointer_to_long(vv);
+        if (value < 0) {
+            PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned int");
+            return (unsigned long long) -1;
+        }
+        return (unsigned long long) value;
+    }
     return (unsigned long long) GraalPyPrivate_Long_AsPrimitive(vv, MODE_PINT_UNSIGNED, sizeof(unsigned long long));
 }
 
@@ -1042,6 +1086,9 @@ PyLong_AsUnsignedLongLongMask(PyObject *op)
         PyErr_BadInternalCall();
         return (unsigned long long)-1;
     }
+    if (points_to_py_int_handle(op)) {
+        return pointer_to_long(op);
+    }
     return (unsigned long long) GraalPyPrivate_Long_AsPrimitive(op, MODE_COERCE_MASK, sizeof(unsigned long long));
 }
 
@@ -6102,12 +6149,31 @@ PyUnstable_Long_CompactValue(const PyLongObject* op) {
 
 #undef PyUnstable_Long_CompactValue
 
-Py_ssize_t
-PyUnstable_Long_CompactValue(const PyLongObject *op) {
+Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject *op) {
+    if (points_to_py_int_handle(op)) {
+        return pointer_to_long(op);
+    }
     return GraalPyPrivate_Long_AsPrimitive((PyObject*) op, MODE_PINT_SIGNED, sizeof(Py_ssize_t));
 }
 
 // GraalPy additions
 uintptr_t GraalPyPrivate_Long_lv_tag(const PyLongObject *op) {
+    if (points_to_py_int_handle(op)) {
+        int64_t t = pointer_to_long(op);
+        if (t == 0) {
+            return SIGN_ZERO;
+        }
+        int64_t sign = 0;
+        if (t < 0) {
+            sign = SIGN_NEGATIVE;
+            t = -t;
+        }
+        int64_t size = 0;
+        while (t != 0) {
+            ++size;
+            t >>= PYLONG_BITS_IN_DIGIT;
+        }
+        return (size << NON_SIZE_BITS) | sign;
+    }
     return GET_SLOT_SPECIAL(op, PyLongObject, long_value_lv_tag, long_value.lv_tag);
 }

graalpython/com.oracle.graal.python.cext/src/object.c

@@ -309,7 +309,7 @@ _Py_IncRef(PyObject *o)
 {
     // GraalPy change: different implementation
     const Py_ssize_t refcnt = Py_REFCNT(o);
-    if (refcnt != IMMORTAL_REFCNT)
+    if (refcnt != _Py_IMMORTAL_REFCNT)
     {
         Py_SET_REFCNT(o, refcnt + 1);
         if (refcnt == MANAGED_REFCNT) {
@@ -337,7 +337,7 @@ _Py_DecRef(PyObject *o)
         return;
     }
     const Py_ssize_t refcnt = Py_REFCNT(o);
-    if (refcnt != IMMORTAL_REFCNT) {
+    if (refcnt != _Py_IMMORTAL_REFCNT) {
         const Py_ssize_t updated_refcnt = refcnt - 1;
         Py_SET_REFCNT(o, updated_refcnt);
         if (updated_refcnt != 0) {
@@ -531,6 +531,9 @@ int
 _PyObject_IsFreed(PyObject *op)
 {
     if (points_to_py_handle_space(op)) {
+        if (points_to_py_float_handle(op) || points_to_py_int_handle(op)) {
+            return 0;
+        }
         return GraalPyPrivate_Object_IsFreed(op);
     }
 #if 0 // GraalPy change
@@ -2847,6 +2850,9 @@ Py_ssize_t Py_REFCNT(PyObject *obj) {
     Py_ssize_t res;
     if (points_to_py_handle_space(obj))
     {
+        if (points_to_py_int_handle(obj) || points_to_py_float_handle(obj)) {
+            return _Py_IMMORTAL_REFCNT;
+        }
         res = pointer_to_stub(obj)->ob_refcnt;
 #ifndef NDEBUG
         if (GraalPyPrivate_Debug_CAPI() && GraalPyPrivate_GET_PyObject_ob_refcnt(obj) != res)
@@ -2897,6 +2903,12 @@ PyTypeObject* GraalPy_TYPE(PyObject *a) {
     PyTypeObject *res;
     if (points_to_py_handle_space(a))
     {
+        if (points_to_py_float_handle(a)) {
+            return &PyFloat_Type;
+        }
+        if (points_to_py_int_handle(a)) {
+            return &PyLong_Type;
+        }
         res = pointer_to_stub(a)->ob_type;
 #ifndef NDEBUG
         if (GraalPyPrivate_Debug_CAPI() && GraalPyPrivate_GET_PyObject_ob_type(a) != res)

graalpython/com.oracle.graal.python.cext/src/typeobject.c

@@ -5418,7 +5418,7 @@ type_is_gc(PyTypeObject *type)
      * just test for 'points_to_py_handle_space' but this is not applicable for
      * 'PyTypeObject' because there we really allocate and fill a native
      * mirror. */
-    return type->tp_flags & Py_TPFLAGS_HEAPTYPE && Py_REFCNT(type) != IMMORTAL_REFCNT;
+    return type->tp_flags & Py_TPFLAGS_HEAPTYPE && Py_REFCNT(type) != _Py_IMMORTAL_REFCNT;
 }
 
 

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/nodes/MemMoveNodeTests.java

@@ -113,10 +113,10 @@ public Object execute(VirtualFrame frame) {
 
         }.getCallTarget().call();
 
-        Assert.assertEquals(1, GetItemScalarNode.executeUncached(storage, 0));
-        Assert.assertEquals(2, GetItemScalarNode.executeUncached(storage, 1));
-        Assert.assertEquals(2, GetItemScalarNode.executeUncached(storage, 2));
-        Assert.assertEquals(2, GetItemScalarNode.executeUncached(storage, 3));
-        Assert.assertEquals(3, GetItemScalarNode.executeUncached(storage, 4));
+        Assert.assertEquals(1L, ((Number) GetItemScalarNode.executeUncached(storage, 0)).longValue());
+        Assert.assertEquals(2L, ((Number) GetItemScalarNode.executeUncached(storage, 1)).longValue());
+        Assert.assertEquals(2L, ((Number) GetItemScalarNode.executeUncached(storage, 2)).longValue());
+        Assert.assertEquals(2L, ((Number) GetItemScalarNode.executeUncached(storage, 3)).longValue());
+        Assert.assertEquals(3L, ((Number) GetItemScalarNode.executeUncached(storage, 4)).longValue());
     }
 }