Expose buffer cache layout version (#20145)

This idea is quite simple: each time we change how an object is cached
in the buffer we bump this version (even if we don't change the ABI
version). Notes:
* This is *not* the same as ABI version, if the ABI version is
different, the capsule import will fail and you can't even use this
version of `librt`. We can however use buffer cache layout version to
abandon cache files that would result in reading garbage data.
* I can't actually start using this in cache metas in the same PR, to
avoid breaking self-check in CI. If there are no comments objections, I
will merge this soon. Then release a new `librt` version, and make
second PR that actually uses this feature.

While I am at it I fix a bug in the float deserialization (magic value
was not handled).

Author: ilevkivskyi

mypy/typeshed/stubs/librt/librt/internal.pyi

@@ -16,3 +16,4 @@ def write_int(data: Buffer, value: int) -> None: ...
 def read_int(data: Buffer) -> int: ...
 def write_tag(data: Buffer, value: u8) -> None: ...
 def read_tag(data: Buffer) -> u8: ...
+def cache_version() -> u8: ...

mypyc/lib-rt/librt_internal.c

@@ -656,6 +656,16 @@ write_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames
     return Py_None;
 }
 
+static uint8_t
+cache_version_internal(void) {
+    return 0;
+}
+
+static PyObject*
+cache_version(PyObject *self, PyObject *Py_UNUSED(ignored)) {
+    return PyLong_FromLong(cache_version_internal());
+}
+
 static PyMethodDef librt_internal_module_methods[] = {
     {"write_bool", (PyCFunction)write_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a bool")},
     {"read_bool", (PyCFunction)read_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a bool")},
@@ -669,6 +679,7 @@ static PyMethodDef librt_internal_module_methods[] = {
     {"read_int", (PyCFunction)read_int, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read an int")},
     {"write_tag", (PyCFunction)write_tag, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a short int")},
     {"read_tag", (PyCFunction)read_tag, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a short int")},
+    {"cache_version", (PyCFunction)cache_version, METH_NOARGS, PyDoc_STR("cache format version")},
     {NULL, NULL, 0, NULL}
 };
 
@@ -688,7 +699,7 @@ librt_internal_module_exec(PyObject *m)
     }
 
     // Export mypy internal C API, be careful with the order!
-    static void *NativeInternal_API[16] = {
+    static void *NativeInternal_API[17] = {
         (void *)Buffer_internal,
         (void *)Buffer_internal_empty,
         (void *)Buffer_getvalue_internal,
@@ -705,6 +716,7 @@ librt_internal_module_exec(PyObject *m)
         (void *)NativeInternal_ABI_Version,
         (void *)write_bytes_internal,
         (void *)read_bytes_internal,
+        (void *)cache_version_internal,
     };
     PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL);
     if (PyModule_Add(m, "_C_API", c_api_object) < 0) {

mypyc/lib-rt/librt_internal.h

@@ -21,6 +21,7 @@ static uint8_t read_tag_internal(PyObject *data);
 static int NativeInternal_ABI_Version(void);
 static char write_bytes_internal(PyObject *data, PyObject *value);
 static PyObject *read_bytes_internal(PyObject *data);
+static uint8_t cache_version_internal(void);
 
 #else
 
@@ -42,6 +43,7 @@ static void **NativeInternal_API;
 #define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[13])
 #define write_bytes_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[14])
 #define read_bytes_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[15])
+#define cache_version_internal (*(uint8_t (*)(void)) NativeInternal_API[16])
 
 static int
 import_librt_internal(void)

mypyc/primitives/misc_ops.py

@@ -422,7 +422,7 @@
     arg_types=[object_rprimitive],
     return_type=float_rprimitive,
     c_function_name="read_float_internal",
-    error_kind=ERR_MAGIC,
+    error_kind=ERR_MAGIC_OVERLAPPING,
 )
 
 function_op(
@@ -456,3 +456,11 @@
     c_function_name="read_tag_internal",
     error_kind=ERR_MAGIC_OVERLAPPING,
 )
+
+function_op(
+    name="librt.internal.cache_version",
+    arg_types=[],
+    return_type=uint8_rprimitive,
+    c_function_name="cache_version_internal",
+    error_kind=ERR_NEVER,
+)

mypyc/test-data/irbuild-classes.test

@@ -1455,6 +1455,7 @@ from mypy_extensions import u8
 from librt.internal import (
     Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float,
     write_int, read_int, write_tag, read_tag, write_bytes, read_bytes,
+    cache_version,
 )
 
 Tag = u8
@@ -1476,6 +1477,7 @@ def foo() -> None:
     z = read_float(b)
     t = read_int(b)
     u = read_tag(b)
+    v = cache_version()
 [out]
 def foo():
     r0, b :: librt.internal.Buffer
@@ -1490,7 +1492,7 @@ def foo():
     r13, y :: bool
     r14, z :: float
     r15, t :: int
-    r16, u :: u8
+    r16, u, r17, v :: u8
 L0:
     r0 = Buffer_internal_empty()
     b = r0
@@ -1517,6 +1519,8 @@ L0:
     t = r15
     r16 = read_tag_internal(b)
     u = r16
+    r17 = cache_version_internal()
+    v = r17
     return 1
 
 [case testEnumFastPath]

mypyc/test-data/run-classes.test

@@ -2715,7 +2715,8 @@ from typing import Final
 from mypy_extensions import u8
 from librt.internal import (
     Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float,
-    write_int, read_int, write_tag, read_tag, write_bytes, read_bytes
+    write_int, read_int, write_tag, read_tag, write_bytes, read_bytes,
+    cache_version,
 )
 
 Tag = u8
@@ -2724,6 +2725,7 @@ TAG_B: Final[Tag] = 255
 TAG_SPECIAL: Final[Tag] = 239
 
 def test_buffer_basic() -> None:
+    assert cache_version() == 0
     b = Buffer(b"foo")
     assert b.getvalue() == b"foo"
 
@@ -2739,6 +2741,7 @@ def test_buffer_roundtrip() -> None:
     write_bytes(b, b"a" * 127)
     write_bytes(b, b"a" * 128)
     write_float(b, 0.1)
+    write_float(b, -113.0)
     write_int(b, 0)
     write_int(b, 1)
     write_tag(b, TAG_A)
@@ -2763,6 +2766,7 @@ def test_buffer_roundtrip() -> None:
     assert read_bytes(b) == b"a" * 127
     assert read_bytes(b) == b"a" * 128
     assert read_float(b) == 0.1
+    assert read_float(b) == -113.0
     assert read_int(b) == 0
     assert read_int(b) == 1
     assert read_tag(b) == TAG_A