From fbd0e5449eef2378d518af41a18515ec5ce299a4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 21 Oct 2025 12:40:11 +0100 Subject: [PATCH 01/27] Move data type check to wrapper functions --- mypyc/lib-rt/librt_internal.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index eaf451eff22b..4d786b7a08cf 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -220,7 +220,6 @@ bool format: single byte static char read_bool_internal(PyObject *data) { - _CHECK_BUFFER(data, CPY_BOOL_ERROR) _CHECK_READ(data, 1, CPY_BOOL_ERROR) char res = _READ(data, char) if (unlikely((res != 0) & (res != 1))) { @@ -238,6 +237,7 @@ read_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_BUFFER(data, NULL) char res = read_bool_internal(data); if (unlikely(res == CPY_BOOL_ERROR)) return NULL; @@ -248,7 +248,6 @@ read_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames static char write_bool_internal(PyObject *data, char value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) _CHECK_SIZE(data, 1) _WRITE(data, char, value) ((BufferObject *)data)->end += 1; @@ -264,6 +263,7 @@ write_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_BUFFER(data, NULL) if (unlikely(!PyBool_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a bool"); return NULL; @@ -306,8 +306,6 @@ _read_short_int(PyObject *data, uint8_t first) { static PyObject* read_str_internal(PyObject *data) { - _CHECK_BUFFER(data, NULL) - // Read string length. _CHECK_READ(data, 1, NULL) uint8_t first = _READ(data, uint8_t) @@ -345,6 +343,7 @@ read_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_BUFFER(data, NULL) return read_str_internal(data); } @@ -379,8 +378,6 @@ _write_short_int(PyObject *data, Py_ssize_t real_value) { static char write_str_internal(PyObject *data, PyObject *value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - Py_ssize_t size; const char *chunk = PyUnicode_AsUTF8AndSize(value, &size); if (unlikely(chunk == NULL)) @@ -412,6 +409,7 @@ write_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_BUFFER(data, NULL) if (unlikely(!PyUnicode_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a str"); return NULL; @@ -429,8 +427,6 @@ bytes format: size as int (see below) followed by bytes static PyObject* read_bytes_internal(PyObject *data) { - _CHECK_BUFFER(data, NULL) - // Read length. _CHECK_READ(data, 1, NULL) uint8_t first = _READ(data, uint8_t) @@ -468,13 +464,12 @@ read_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_BUFFER(data, NULL) return read_bytes_internal(data); } static char write_bytes_internal(PyObject *data, PyObject *value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - const char *chunk = PyBytes_AsString(value); if (unlikely(chunk == NULL)) return CPY_NONE_ERROR; @@ -506,6 +501,7 @@ write_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnam if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_BUFFER(data, NULL) if (unlikely(!PyBytes_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a bytes object"); return NULL; @@ -524,7 +520,6 @@ float format: static double read_float_internal(PyObject *data) { - _CHECK_BUFFER(data, CPY_FLOAT_ERROR) _CHECK_READ(data, 8, CPY_FLOAT_ERROR) char *buf = ((BufferObject *)data)->buf; double res = PyFloat_Unpack8(buf + ((BufferObject *)data)->pos, 1); @@ -542,6 +537,7 @@ read_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_BUFFER(data, NULL) double retval = read_float_internal(data); if (unlikely(retval == CPY_FLOAT_ERROR && PyErr_Occurred())) { return NULL; @@ -551,7 +547,6 @@ read_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname static char write_float_internal(PyObject *data, double value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) _CHECK_SIZE(data, 8) char *buf = ((BufferObject *)data)->buf; int res = PyFloat_Pack8(value, buf + ((BufferObject *)data)->pos, 1); @@ -571,6 +566,7 @@ write_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnam if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_BUFFER(data, NULL) if (unlikely(!PyFloat_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a float"); return NULL; @@ -595,7 +591,6 @@ since negative integers are much more rare. static CPyTagged read_int_internal(PyObject *data) { - _CHECK_BUFFER(data, CPY_INT_TAG) _CHECK_READ(data, 1, CPY_INT_TAG) uint8_t first = _READ(data, uint8_t) @@ -645,6 +640,7 @@ read_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_BUFFER(data, NULL) CPyTagged retval = read_int_internal(data); if (unlikely(retval == CPY_INT_TAG)) { return NULL; @@ -731,8 +727,6 @@ _write_long_int(PyObject *data, CPyTagged value) { static char write_int_internal(PyObject *data, CPyTagged value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - if (likely((value & CPY_INT_TAG) == 0)) { Py_ssize_t real_value = CPyTagged_ShortAsSsize_t(value); if (likely(real_value >= MIN_FOUR_BYTES_INT && real_value <= MAX_FOUR_BYTES_INT)) { @@ -754,6 +748,7 @@ write_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_BUFFER(data, NULL) if (unlikely(!PyLong_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be an int"); return NULL; @@ -773,7 +768,6 @@ integer tag format (0 <= t <= 255): static uint8_t read_tag_internal(PyObject *data) { - _CHECK_BUFFER(data, CPY_LL_UINT_ERROR) _CHECK_READ(data, 1, CPY_LL_UINT_ERROR) uint8_t ret = _READ(data, uint8_t) return ret; @@ -787,6 +781,7 @@ read_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_BUFFER(data, NULL) uint8_t retval = read_tag_internal(data); if (unlikely(retval == CPY_LL_UINT_ERROR && PyErr_Occurred())) { return NULL; @@ -796,7 +791,6 @@ read_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) static char write_tag_internal(PyObject *data, uint8_t value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) _CHECK_SIZE(data, 1) _WRITE(data, uint8_t, value) ((BufferObject *)data)->end += 1; @@ -812,6 +806,7 @@ write_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_BUFFER(data, NULL) uint8_t unboxed = CPyLong_AsUInt8(value); if (unlikely(unboxed == CPY_LL_UINT_ERROR && PyErr_Occurred())) { CPy_TypeError("u8", value); From 934c091384e7d340efe0bc14fe1703bf23619538 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 21 Oct 2025 16:11:08 +0100 Subject: [PATCH 02/27] Update stubs --- mypy/typeshed/stubs/librt/librt/internal.pyi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypy/typeshed/stubs/librt/librt/internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi index 8654e31c100e..a164490c8679 100644 --- a/mypy/typeshed/stubs/librt/librt/internal.pyi +++ b/mypy/typeshed/stubs/librt/librt/internal.pyi @@ -1,7 +1,9 @@ from mypy_extensions import u8 -class Buffer: +class ReadBuffer: def __init__(self, source: bytes = ...) -> None: ... + +class WriteBuffer: def getvalue(self) -> bytes: ... def write_bool(data: Buffer, value: bool) -> None: ... From 2283ef99c6466a08c89edd20bd4c1cff0fd20db8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 21 Oct 2025 16:12:06 +0100 Subject: [PATCH 03/27] Split Buffer into WriteBuffer and ReadBuffer --- mypyc/ir/rtypes.py | 2 +- mypyc/lib-rt/librt_internal.c | 395 ++++++++++++++++++++++------------ mypyc/lib-rt/librt_internal.h | 15 +- mypyc/primitives/misc_ops.py | 23 +- 4 files changed, 280 insertions(+), 155 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 941670ab230d..66b98e5d6398 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -514,7 +514,7 @@ def __hash__(self) -> int: KNOWN_NATIVE_TYPES: Final = { name: RPrimitive(name, is_unboxed=False, is_refcounted=True) - for name in ["librt.internal.Buffer"] + for name in ["librt.internal.WriteBuffer", "librt.internal.ReadBuffer"] } diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 4d786b7a08cf..ed35b4823d79 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -30,18 +30,30 @@ #define CPY_NONE_ERROR 2 #define CPY_NONE 1 -#define _CHECK_BUFFER(data, err) if (unlikely(_check_buffer(data) == CPY_NONE_ERROR)) \ - return err; -#define _CHECK_SIZE(data, need) if (unlikely(_check_size((BufferObject *)data, need) == CPY_NONE_ERROR)) \ - return CPY_NONE_ERROR; -#define _CHECK_READ(data, size, err) if (unlikely(_check_read((BufferObject *)data, size) == CPY_NONE_ERROR)) \ - return err; - -#define _READ(data, type) *(type *)(((BufferObject *)data)->buf + ((BufferObject *)data)->pos); \ - ((BufferObject *)data)->pos += sizeof(type); - -#define _WRITE(data, type, v) *(type *)(((BufferObject *)data)->buf + ((BufferObject *)data)->pos) = v; \ - ((BufferObject *)data)->pos += sizeof(type); +#define _CHECK_READ_BUFFER(data, err) if (unlikely(_check_read_buffer(data) == CPY_NONE_ERROR)) \ + return err; +#define _CHECK_WRITE_BUFFER(data, err) if (unlikely(_check_write_buffer(data) == CPY_NONE_ERROR)) \ + return err; +#define _CHECK_WRITE(data, need) if (unlikely(_check_size((WriteBufferObject *)data, need) == CPY_NONE_ERROR)) \ + return CPY_NONE_ERROR; +#define _CHECK_READ(data, size, err) if (unlikely(_check_read((ReadBufferObject *)data, size) == CPY_NONE_ERROR)) \ + return err; + +#define _READ(result, data, type) \ + do { \ + *(result) = *(type *)(((ReadBufferObject *)data)->ptr); \ + ((ReadBufferObject *)data)->ptr += sizeof(type); \ + } while (0) + +#define _WRITE(data, type, v) \ + do { \ + *(type *)(((WriteBufferObject *)data)->ptr) = v; \ + ((WriteBufferObject *)data)->ptr += sizeof(type); \ + } while (0) + +// +// ReadBuffer +// #if PY_BIG_ENDIAN uint16_t reverse_16(uint16_t number) { @@ -55,65 +67,155 @@ uint32_t reverse_32(uint32_t number) { typedef struct { PyObject_HEAD - Py_ssize_t pos; - Py_ssize_t end; - Py_ssize_t size; - char *buf; -} BufferObject; + char *ptr; // Current read location in the buffer + char *end; // End of the buffer + PyObject *source; // The object that contains the buffer +} ReadBufferObject; -static PyTypeObject BufferType; +static PyTypeObject ReadBufferType; static PyObject* -Buffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +ReadBuffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - if (type != &BufferType) { - PyErr_SetString(PyExc_TypeError, "Buffer should not be subclassed"); + if (type != &ReadBufferType) { + PyErr_SetString(PyExc_TypeError, "ReadBuffer should not be subclassed"); return NULL; } - BufferObject *self = (BufferObject *)type->tp_alloc(type, 0); + ReadBufferObject *self = (ReadBufferObject *)type->tp_alloc(type, 0); if (self != NULL) { - self->pos = 0; - self->end = 0; - self->size = 0; - self->buf = NULL; + self->source = NULL; + self->ptr = NULL; + self->end = NULL; } return (PyObject *) self; } +static int +ReadBuffer_init_internal(ReadBufferObject *self, PyObject *source) { + if (!PyBytes_CheckExact(source)) { + PyErr_SetString(PyExc_TypeError, "source must be a bytes object"); + return -1; + } + self->source = Py_NewRef(source); + self->ptr = PyBytes_AsString(source); + self->end = self->ptr + PyBytes_GET_SIZE(source); + return 0; +} + +static PyObject* +ReadBuffer_internal(PyObject *source) { + ReadBufferObject *self = (ReadBufferObject *)ReadBufferType.tp_alloc(&ReadBufferType, 0); + if (self == NULL) + return NULL; + self->ptr = NULL; + self->end = NULL; + self->source = NULL; + if (ReadBuffer_init_internal(self, source) == -1) { + Py_DECREF(self); + return NULL; + } + return (PyObject *)self; +} + +static PyObject* +ReadBuffer_internal_empty(void) { + return ReadBuffer_internal(NULL); +} + +static int +ReadBuffer_init(ReadBufferObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"source", NULL}; + PyObject *source = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &source)) + return -1; + + return ReadBuffer_init_internal(self, source); +} + +static void +ReadBuffer_dealloc(ReadBufferObject *self) +{ + Py_CLEAR(self->source); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyMethodDef ReadBuffer_methods[] = { + {NULL} /* Sentinel */ +}; + +static PyTypeObject ReadBufferType = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "ReadBuffer", + .tp_doc = PyDoc_STR("Mypy cache buffer objects"), + .tp_basicsize = sizeof(ReadBufferObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = ReadBuffer_new, + .tp_init = (initproc) ReadBuffer_init, + .tp_dealloc = (destructor) ReadBuffer_dealloc, + .tp_methods = ReadBuffer_methods, +}; + +// +// WriteBuffer +// + +typedef struct { + PyObject_HEAD + char *buf; // Beginning of the buffer + char *ptr; // Current write location in the buffer + char *end; // End of the buffer +} WriteBufferObject; + +static PyTypeObject WriteBufferType; + +static PyObject* +WriteBuffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + if (type != &WriteBufferType) { + PyErr_SetString(PyExc_TypeError, "WriteBuffer cannot be subclassed"); + return NULL; + } + + WriteBufferObject *self = (WriteBufferObject *)type->tp_alloc(type, 0); + if (self != NULL) { + self->buf = NULL; + self->ptr = NULL; + self->end = NULL; + } + return (PyObject *)self; +} static int -Buffer_init_internal(BufferObject *self, PyObject *source) { +WriteBuffer_init_internal(WriteBufferObject *self, PyObject *source) { if (source) { - if (!PyBytes_Check(source)) { - PyErr_SetString(PyExc_TypeError, "source must be a bytes object"); - return -1; - } - self->end = PyBytes_GET_SIZE(source); - // Allocate at least one byte to simplify resizing logic. - // The original bytes buffer has last null byte, so this is safe. - self->size = self->end + 1; - // This returns a pointer to internal bytes data, so make our own copy. - char *buf = PyBytes_AsString(source); - self->buf = PyMem_Malloc(self->size); - memcpy(self->buf, buf, self->end); - } else { - self->buf = PyMem_Malloc(START_SIZE); - self->size = START_SIZE; + // TODO + PyErr_SetString(PyExc_TypeError, "source should not be provided"); + return -1; + } + Py_ssize_t size = START_SIZE; + self->buf = PyMem_Malloc(size + 1); + if (self->buf == NULL) { + // TODO + CPyError_OutOfMemory(); } + self->ptr = self->buf; + self->end = self->buf + size; return 0; } static PyObject* -Buffer_internal(PyObject *source) { - BufferObject *self = (BufferObject *)BufferType.tp_alloc(&BufferType, 0); +WriteBuffer_internal(PyObject *source) { + WriteBufferObject *self = (WriteBufferObject *)WriteBufferType.tp_alloc(&WriteBufferType, 0); if (self == NULL) return NULL; - self->pos = 0; - self->end = 0; - self->size = 0; self->buf = NULL; - if (Buffer_init_internal(self, source) == -1) { + self->ptr = NULL; + self->end = NULL; + if (WriteBuffer_init_internal(self, source) == -1) { Py_DECREF(self); return NULL; } @@ -121,12 +223,12 @@ Buffer_internal(PyObject *source) { } static PyObject* -Buffer_internal_empty(void) { - return Buffer_internal(NULL); +WriteBuffer_internal_empty(void) { + return WriteBuffer_internal(NULL); } static int -Buffer_init(BufferObject *self, PyObject *args, PyObject *kwds) +WriteBuffer_init(WriteBufferObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"source", NULL}; PyObject *source = NULL; @@ -134,53 +236,67 @@ Buffer_init(BufferObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &source)) return -1; - return Buffer_init_internal(self, source); + return WriteBuffer_init_internal(self, source); } static void -Buffer_dealloc(BufferObject *self) +WriteBuffer_dealloc(WriteBufferObject *self) { PyMem_Free(self->buf); Py_TYPE(self)->tp_free((PyObject *)self); } static PyObject* -Buffer_getvalue_internal(PyObject *self) +WriteBuffer_getvalue_internal(PyObject *self) { - return PyBytes_FromStringAndSize(((BufferObject *)self)->buf, ((BufferObject *)self)->end); + WriteBufferObject *obj = (WriteBufferObject *)self; + return PyBytes_FromStringAndSize(obj->buf, obj->ptr - obj->buf); } static PyObject* -Buffer_getvalue(BufferObject *self, PyObject *Py_UNUSED(ignored)) +WriteBuffer_getvalue(WriteBufferObject *self, PyObject *Py_UNUSED(ignored)) { - return PyBytes_FromStringAndSize(self->buf, self->end); + return PyBytes_FromStringAndSize(self->buf, self->ptr - self->buf); } -static PyMethodDef Buffer_methods[] = { - {"getvalue", (PyCFunction) Buffer_getvalue, METH_NOARGS, +static PyMethodDef WriteBuffer_methods[] = { + {"getvalue", (PyCFunction) WriteBuffer_getvalue, METH_NOARGS, "Return the buffer content as bytes object" }, {NULL} /* Sentinel */ }; -static PyTypeObject BufferType = { +static PyTypeObject WriteBufferType = { .ob_base = PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "Buffer", + .tp_name = "WriteBuffer", .tp_doc = PyDoc_STR("Mypy cache buffer objects"), - .tp_basicsize = sizeof(BufferObject), + .tp_basicsize = sizeof(WriteBufferObject), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = Buffer_new, - .tp_init = (initproc) Buffer_init, - .tp_dealloc = (destructor) Buffer_dealloc, - .tp_methods = Buffer_methods, + .tp_new = WriteBuffer_new, + .tp_init = (initproc) WriteBuffer_init, + .tp_dealloc = (destructor) WriteBuffer_dealloc, + .tp_methods = WriteBuffer_methods, }; +// ---------- + +static inline char +_check_read_buffer(PyObject *data) { + if (unlikely(Py_TYPE(data) != &ReadBufferType)) { + PyErr_Format( + PyExc_TypeError, "data must be a ReadBuffer object, got %s", Py_TYPE(data)->tp_name + ); + return CPY_NONE_ERROR; + } + return CPY_NONE; +} + static inline char -_check_buffer(PyObject *data) { - if (unlikely(Py_TYPE(data) != &BufferType)) { +_check_write_buffer(PyObject *data) { + if (unlikely(Py_TYPE(data) != &WriteBufferType)) { PyErr_Format( - PyExc_TypeError, "data must be a Buffer object, got %s", Py_TYPE(data)->tp_name + PyExc_TypeError, "data must be a WriteBuffer object, got %s", Py_TYPE(data)->tp_name ); return CPY_NONE_ERROR; } @@ -188,24 +304,28 @@ _check_buffer(PyObject *data) { } static inline char -_check_size(BufferObject *data, Py_ssize_t need) { - Py_ssize_t target = data->pos + need; - if (target <= data->size) +_check_size(WriteBufferObject *data, Py_ssize_t need) { + if (data->end - data->ptr >= need) return CPY_NONE; - do - data->size *= 2; - while (target >= data->size); - data->buf = PyMem_Realloc(data->buf, data->size); + Py_ssize_t index = data->ptr - data->buf; + Py_ssize_t target = index + need; + Py_ssize_t size = data->end - data->ptr; + do { + size *= 2; + } while (target >= size); + data->buf = PyMem_Realloc(data->buf, size); if (unlikely(data->buf == NULL)) { PyErr_NoMemory(); return CPY_NONE_ERROR; } + data->ptr = data->buf + index; + data->end = data->buf + size; return CPY_NONE; } static inline char -_check_read(BufferObject *data, Py_ssize_t need) { - if (unlikely(data->pos + need > data->end)) { +_check_read(ReadBufferObject *data, Py_ssize_t need) { + if (unlikely((data->end - data->ptr) < need)) { PyErr_SetString(PyExc_ValueError, "reading past the buffer end"); return CPY_NONE_ERROR; } @@ -221,7 +341,8 @@ bool format: single byte static char read_bool_internal(PyObject *data) { _CHECK_READ(data, 1, CPY_BOOL_ERROR) - char res = _READ(data, char) + char res; + _READ(&res, data, char); if (unlikely((res != 0) & (res != 1))) { PyErr_SetString(PyExc_ValueError, "invalid bool value"); return CPY_BOOL_ERROR; @@ -237,7 +358,7 @@ read_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } - _CHECK_BUFFER(data, NULL) + _CHECK_READ_BUFFER(data, NULL) char res = read_bool_internal(data); if (unlikely(res == CPY_BOOL_ERROR)) return NULL; @@ -248,9 +369,8 @@ read_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames static char write_bool_internal(PyObject *data, char value) { - _CHECK_SIZE(data, 1) - _WRITE(data, char, value) - ((BufferObject *)data)->end += 1; + _CHECK_WRITE(data, 1) + _WRITE(data, char, value); return CPY_NONE; } @@ -263,7 +383,7 @@ write_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } - _CHECK_BUFFER(data, NULL) + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyBool_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a bool"); return NULL; @@ -308,7 +428,8 @@ static PyObject* read_str_internal(PyObject *data) { // Read string length. _CHECK_READ(data, 1, NULL) - uint8_t first = _READ(data, uint8_t) + uint8_t first; + _READ(&first, data, uint8_t); if (unlikely(first == LONG_INT_TRAILER)) { // Fail fast for invalid/tampered data. PyErr_SetString(PyExc_ValueError, "invalid str size"); @@ -324,14 +445,12 @@ read_str_internal(PyObject *data) { } Py_ssize_t size = tagged_size >> 1; // Read string content. - char *buf = ((BufferObject *)data)->buf; + char *ptr = ((ReadBufferObject *)data)->ptr; _CHECK_READ(data, size, NULL) - PyObject *res = PyUnicode_FromStringAndSize( - buf + ((BufferObject *)data)->pos, (Py_ssize_t)size - ); + PyObject *res = PyUnicode_FromStringAndSize(ptr, (Py_ssize_t)size); if (unlikely(res == NULL)) return NULL; - ((BufferObject *)data)->pos += size; + ((ReadBufferObject *)data)->ptr += size; return res; } @@ -343,7 +462,7 @@ read_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } - _CHECK_BUFFER(data, NULL) + _CHECK_READ_BUFFER(data, NULL) return read_str_internal(data); } @@ -393,10 +512,9 @@ write_str_internal(PyObject *data, PyObject *value) { } // Write string content. _CHECK_SIZE(data, size) - char *buf = ((BufferObject *)data)->buf; - memcpy(buf + ((BufferObject *)data)->pos, chunk, size); - ((BufferObject *)data)->pos += size; - ((BufferObject *)data)->end += size; + char *ptr = ((WriteBufferObject *)data)->ptr; + memcpy(ptr, chunk, size); + ((WriteBufferObject *)data)->ptr += size; return CPY_NONE; } @@ -409,7 +527,7 @@ write_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } - _CHECK_BUFFER(data, NULL) + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyUnicode_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a str"); return NULL; @@ -429,7 +547,8 @@ static PyObject* read_bytes_internal(PyObject *data) { // Read length. _CHECK_READ(data, 1, NULL) - uint8_t first = _READ(data, uint8_t) + uint8_t first; + _READ(&first, data, uint8_t); if (unlikely(first == LONG_INT_TRAILER)) { // Fail fast for invalid/tampered data. PyErr_SetString(PyExc_ValueError, "invalid bytes size"); @@ -445,14 +564,12 @@ read_bytes_internal(PyObject *data) { } Py_ssize_t size = tagged_size >> 1; // Read bytes content. - char *buf = ((BufferObject *)data)->buf; + char *ptr = ((ReadBufferObject *)data)->ptr; _CHECK_READ(data, size, NULL) - PyObject *res = PyBytes_FromStringAndSize( - buf + ((BufferObject *)data)->pos, (Py_ssize_t)size - ); + PyObject *res = PyBytes_FromStringAndSize(ptr, (Py_ssize_t)size); if (unlikely(res == NULL)) return NULL; - ((BufferObject *)data)->pos += size; + ((ReadBufferObject *)data)->ptr += size; return res; } @@ -464,7 +581,7 @@ read_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } - _CHECK_BUFFER(data, NULL) + _CHECK_READ_BUFFER(data, NULL) return read_bytes_internal(data); } @@ -485,10 +602,9 @@ write_bytes_internal(PyObject *data, PyObject *value) { } // Write bytes content. _CHECK_SIZE(data, size) - char *buf = ((BufferObject *)data)->buf; - memcpy(buf + ((BufferObject *)data)->pos, chunk, size); - ((BufferObject *)data)->pos += size; - ((BufferObject *)data)->end += size; + char *ptr = ((WriteBufferObject *)data)->ptr; + memcpy(ptr, chunk, size); + ((WriteBufferObject *)data)->ptr += size; return CPY_NONE; } @@ -501,7 +617,7 @@ write_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnam if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } - _CHECK_BUFFER(data, NULL) + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyBytes_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a bytes object"); return NULL; @@ -521,11 +637,11 @@ float format: static double read_float_internal(PyObject *data) { _CHECK_READ(data, 8, CPY_FLOAT_ERROR) - char *buf = ((BufferObject *)data)->buf; - double res = PyFloat_Unpack8(buf + ((BufferObject *)data)->pos, 1); + char *ptr = ((BufferObject *)data)->ptr; + double res = PyFloat_Unpack8(ptr, 1); if (unlikely((res == -1.0) && PyErr_Occurred())) return CPY_FLOAT_ERROR; - ((BufferObject *)data)->pos += 8; + ((BufferObject *)data)->ptr += 8; return res; } @@ -537,7 +653,7 @@ read_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } - _CHECK_BUFFER(data, NULL) + _CHECK_READ_BUFFER(data, NULL) double retval = read_float_internal(data); if (unlikely(retval == CPY_FLOAT_ERROR && PyErr_Occurred())) { return NULL; @@ -548,12 +664,11 @@ read_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname static char write_float_internal(PyObject *data, double value) { _CHECK_SIZE(data, 8) - char *buf = ((BufferObject *)data)->buf; - int res = PyFloat_Pack8(value, buf + ((BufferObject *)data)->pos, 1); + char *ptr = ((BufferObject *)data)->ptr; + int res = PyFloat_Pack8(value, ptr, 1); if (unlikely(res == -1)) return CPY_NONE_ERROR; - ((BufferObject *)data)->pos += 8; - ((BufferObject *)data)->end += 8; + ((BufferObject *)data)->ptr += 8; return CPY_NONE; } @@ -566,7 +681,7 @@ write_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnam if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } - _CHECK_BUFFER(data, NULL) + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyFloat_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a float"); return NULL; @@ -593,7 +708,8 @@ static CPyTagged read_int_internal(PyObject *data) { _CHECK_READ(data, 1, CPY_INT_TAG) - uint8_t first = _READ(data, uint8_t) + uint8_t first; + _READ(&first, data, uint8_t); if (likely(first != LONG_INT_TRAILER)) { return _read_short_int(data, first); } @@ -602,7 +718,7 @@ read_int_internal(PyObject *data) { // Read byte length and sign. _CHECK_READ(data, 1, CPY_INT_TAG) - first = _READ(data, uint8_t) + _READ(&first, data, uint8_t); Py_ssize_t size_and_sign = _read_short_int(data, first); if (size_and_sign == CPY_INT_TAG) return CPY_INT_TAG; @@ -615,12 +731,11 @@ read_int_internal(PyObject *data) { // Construct an int object from the byte array. _CHECK_READ(data, size, CPY_INT_TAG) - char *buf = ((BufferObject *)data)->buf; - PyObject *num = _PyLong_FromByteArray( - (unsigned char *)(buf + ((BufferObject *)data)->pos), size, 1, 0); + char *ptr = ((BufferObject *)data)->ptr; + PyObject *num = _PyLong_FromByteArray((unsigned char *)ptr, size, 1, 0); if (num == NULL) return CPY_INT_TAG; - ((BufferObject *)data)->pos += size; + ((BufferObject *)data)->ptr += size; if (sign) { PyObject *old = num; num = PyNumber_Negative(old); @@ -640,7 +755,7 @@ read_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } - _CHECK_BUFFER(data, NULL) + _CHECK_READ_BUFFER(data, NULL) CPyTagged retval = read_int_internal(data); if (unlikely(retval == CPY_INT_TAG)) { return NULL; @@ -748,7 +863,7 @@ write_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } - _CHECK_BUFFER(data, NULL) + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyLong_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be an int"); return NULL; @@ -769,7 +884,8 @@ integer tag format (0 <= t <= 255): static uint8_t read_tag_internal(PyObject *data) { _CHECK_READ(data, 1, CPY_LL_UINT_ERROR) - uint8_t ret = _READ(data, uint8_t) + uint8_t ret; + _READ(&ret, data, uint8_t); return ret; } @@ -781,7 +897,7 @@ read_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } - _CHECK_BUFFER(data, NULL) + _CHECK_READ_BUFFER(data, NULL) uint8_t retval = read_tag_internal(data); if (unlikely(retval == CPY_LL_UINT_ERROR && PyErr_Occurred())) { return NULL; @@ -791,9 +907,8 @@ read_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) static char write_tag_internal(PyObject *data, uint8_t value) { - _CHECK_SIZE(data, 1) - _WRITE(data, uint8_t, value) - ((BufferObject *)data)->end += 1; + _CHECK_WRITE(data, 1) + _WRITE(data, uint8_t, value); return CPY_NONE; } @@ -806,7 +921,7 @@ write_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } - _CHECK_BUFFER(data, NULL) + _CHECK_WRITE_BUFFER(data, NULL) uint8_t unboxed = CPyLong_AsUInt8(value); if (unlikely(unboxed == CPY_LL_UINT_ERROR && PyErr_Occurred())) { CPy_TypeError("u8", value); @@ -854,18 +969,24 @@ NativeInternal_ABI_Version(void) { static int librt_internal_module_exec(PyObject *m) { - if (PyType_Ready(&BufferType) < 0) { + if (PyType_Ready(&ReadBufferType) < 0) { + return -1; + } + if (PyType_Ready(&WriteBufferType) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "ReadBuffer", (PyObject *) &ReadBufferType) < 0) { return -1; } - if (PyModule_AddObjectRef(m, "Buffer", (PyObject *) &BufferType) < 0) { + if (PyModule_AddObjectRef(m, "WriteBuffer", (PyObject *) &WriteBufferType) < 0) { return -1; } // Export mypy internal C API, be careful with the order! - static void *NativeInternal_API[17] = { - (void *)Buffer_internal, - (void *)Buffer_internal_empty, - (void *)Buffer_getvalue_internal, + static void *NativeInternal_API[LIBRT_INTERNAL_API_LEN] = { + (void *)ReadBuffer_internal, + (void *)WriteBuffer_internal_empty, + (void *)WriteBuffer_getvalue_internal, (void *)write_bool_internal, (void *)read_bool_internal, (void *)write_str_internal, diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index 1d16e1cb127f..c8d76dfeb8ad 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -2,12 +2,15 @@ #define LIBRT_INTERNAL_H #define LIBRT_INTERNAL_ABI_VERSION 0 +#define LIBRT_INTERNAL_API_LEN 16 #ifdef LIBRT_INTERNAL_MODULE -static PyObject *Buffer_internal(PyObject *source); -static PyObject *Buffer_internal_empty(void); -static PyObject *Buffer_getvalue_internal(PyObject *self); +static PyObject *ReadBuffer_internal(PyObject *source); +static PyObject *WriteBuffer_internal_empty(void); +static PyObject *WriteBuffer_getvalue_internal(PyObject *self); +static PyObject *ReadBuffer_internal(PyObject *source); +static PyObject *ReadBuffer_internal_empty(void); static char write_bool_internal(PyObject *data, char value); static char read_bool_internal(PyObject *data); static char write_str_internal(PyObject *data, PyObject *value); @@ -27,9 +30,9 @@ static uint8_t cache_version_internal(void); static void **NativeInternal_API; -#define Buffer_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[0]) -#define Buffer_internal_empty (*(PyObject* (*)(void)) NativeInternal_API[1]) -#define Buffer_getvalue_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[2]) +#define ReadBuffer_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[0]) +#define WriteBuffer_internal_empty (*(PyObject* (*)(void)) NativeInternal_API[1]) +#define WriteBuffer_getvalue_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[2]) #define write_bool_internal (*(char (*)(PyObject *source, char value)) NativeInternal_API[3]) #define read_bool_internal (*(char (*)(PyObject *source)) NativeInternal_API[4]) #define write_str_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[5]) diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 10f4bc001e29..6bf7a8ab3dee 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -333,31 +333,32 @@ error_kind=ERR_NEVER, ) -buffer_rprimitive = KNOWN_NATIVE_TYPES["librt.internal.Buffer"] +write_buffer_rprimitive = KNOWN_NATIVE_TYPES["librt.internal.WriteBuffer"] +read_buffer_rprimitive = KNOWN_NATIVE_TYPES["librt.internal.ReadBuffer"] -# Buffer(source) +# ReadBuffer(source) function_op( - name="librt.internal.Buffer", + name="librt.internal.ReadBuffer", arg_types=[bytes_rprimitive], - return_type=buffer_rprimitive, - c_function_name="Buffer_internal", + return_type=read_buffer_rprimitive, + c_function_name="ReadBuffer_internal", error_kind=ERR_MAGIC, ) -# Buffer() +# WriteBuffer() function_op( - name="librt.internal.Buffer", + name="librt.internal.WriteBuffer", arg_types=[], - return_type=buffer_rprimitive, - c_function_name="Buffer_internal_empty", + return_type=write_buffer_rprimitive, + c_function_name="WriteBuffer_internal_empty", error_kind=ERR_MAGIC, ) method_op( name="getvalue", - arg_types=[buffer_rprimitive], + arg_types=[write_buffer_rprimitive], return_type=bytes_rprimitive, - c_function_name="Buffer_getvalue_internal", + c_function_name="WriteBuffer_getvalue_internal", error_kind=ERR_MAGIC, ) From 94e116b858c02838f850a8146904e4381b1dcefc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 21 Oct 2025 16:12:21 +0100 Subject: [PATCH 04/27] Update test case --- mypyc/test-data/run-classes.test | 67 ++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 0805da184e1a..6ee3ab60f940 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2711,10 +2711,12 @@ from native import Player Player.MIN = [case testBufferRoundTrip_librt_internal] +from __future__ import annotations + from typing import Final from mypy_extensions import u8 from librt.internal import ( - Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, + ReadBuffer, WriteBuffer, , write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag, write_bytes, read_bytes, cache_version, ) @@ -2726,11 +2728,14 @@ TAG_SPECIAL: Final[Tag] = 239 def test_buffer_basic() -> None: assert cache_version() == 0 - b = Buffer(b"foo") - assert b.getvalue() == b"foo" + w = WriteBuffer() + write_str(w, "foo") + r = ReadBuffer(w.getvalue()) + assert read_str(r) == "foo" def test_buffer_roundtrip() -> None: - b = Buffer() + b: WriteBuffer | ReadBuffer + b = WriteBuffer() write_str(b, "foo") write_bool(b, True) write_str(b, "bar" * 1000) @@ -2757,7 +2762,7 @@ def test_buffer_roundtrip() -> None: write_int(b, 536860912) write_int(b, 1234567891) - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == "foo" assert read_bool(b) is True assert read_str(b) == "bar" * 1000 @@ -2785,23 +2790,24 @@ def test_buffer_roundtrip() -> None: assert read_int(b) == 1234567891 def test_buffer_int_size() -> None: + b: WriteBuffer | ReadBuffer for i in (-10, -9, 0, 116, 117): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 1 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i for i in (-100, -11, 118, 12344, 16283): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 2 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i for i in (-10000, 16284, 123456789): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 4 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i def test_buffer_int_powers() -> None: @@ -2838,18 +2844,19 @@ def test_negative_long_int_serialized_bytes() -> None: assert read_int(b) == n def test_buffer_str_size() -> None: + b: WriteBuffer | ReadBuffer for s in ("", "a", "a" * 117): - b = Buffer() + b = WriteBuffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 1 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == s for s in ("a" * 118, "a" * 16283): - b = Buffer() + b = WriteBuffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 2 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == s [file driver.py] @@ -2864,11 +2871,13 @@ test_positive_long_int_serialized_bytes() test_negative_long_int_serialized_bytes() def test_buffer_basic_interpreted() -> None: - b = Buffer(b"foo") - assert b.getvalue() == b"foo" + b = WriteBuffer() + write_str(b, "foo") + b = ReadBuffer(b.getvalue()) + assert read_str(b) == "foo" def test_buffer_roundtrip_interpreted() -> None: - b = Buffer() + b = WriteBuffer() write_str(b, "foo") write_bool(b, True) write_str(b, "bar" * 1000) @@ -2893,7 +2902,7 @@ def test_buffer_roundtrip_interpreted() -> None: write_int(b, 536860912) write_int(b, 1234567891) - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == "foo" assert read_bool(b) is True assert read_str(b) == "bar" * 1000 @@ -2920,22 +2929,22 @@ def test_buffer_roundtrip_interpreted() -> None: def test_buffer_int_size_interpreted() -> None: for i in (-10, -9, 0, 116, 117): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 1 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i for i in (-100, -11, 118, 12344, 16283): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 2 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i for i in (-10000, 16284, 123456789): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 4 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i def test_buffer_int_powers_interpreted() -> None: @@ -2950,17 +2959,17 @@ def test_buffer_int_powers_interpreted() -> None: def test_buffer_str_size_interpreted() -> None: for s in ("", "a", "a" * 117): - b = Buffer() + b = WriteBuffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 1 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == s for s in ("a" * 118, "a" * 16283): - b = Buffer() + b = WriteBuffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 2 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == s test_buffer_basic_interpreted() From 20e7df4970f5318bf4a96f2c4361ad5f315454d5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 21 Oct 2025 16:16:15 +0100 Subject: [PATCH 05/27] Update more tests --- mypyc/test-data/irbuild-classes.test | 42 ++++++++++++++-------------- mypyc/test-data/run-classes.test | 6 ++-- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index a8ee7213ef96..7707ef4f903d 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1453,7 +1453,7 @@ class TestOverload: from typing import Final from mypy_extensions import u8 from librt.internal import ( - Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, + WriteBuffer, ReadBuffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag, write_bytes, read_bytes, cache_version, ) @@ -1462,7 +1462,7 @@ Tag = u8 TAG: Final[Tag] = 1 def foo() -> None: - b = Buffer() + b = WriteBuffer() write_str(b, "foo") write_bytes(b, b"bar") write_bool(b, True) @@ -1470,23 +1470,23 @@ def foo() -> None: write_int(b, 1) write_tag(b, TAG) - b = Buffer(b.getvalue()) - x = read_str(b) - xb = read_bytes(b) - y = read_bool(b) - z = read_float(b) - t = read_int(b) - u = read_tag(b) + rb = ReadBuffer(b.getvalue()) + x = read_str(rb) + xb = read_bytes(rb) + y = read_bool(rb) + z = read_float(rb) + t = read_int(rb) + u = read_tag(rb) v = cache_version() [out] def foo(): - r0, b :: librt.internal.Buffer + r0, b :: librt.internal.WriteBuffer r1 :: str r2 :: None r3 :: bytes r4, r5, r6, r7, r8 :: None r9 :: bytes - r10 :: librt.internal.Buffer + r10, rb :: librt.internal.ReadBuffer r11, x :: str r12, xb :: bytes r13, y :: bool @@ -1494,7 +1494,7 @@ def foo(): r15, t :: int r16, u, r17, v :: u8 L0: - r0 = Buffer_internal_empty() + r0 = WriteBuffer_internal_empty() b = r0 r1 = 'foo' r2 = write_str_internal(b, r1) @@ -1504,20 +1504,20 @@ L0: r6 = write_float_internal(b, 0.1) r7 = write_int_internal(b, 2) r8 = write_tag_internal(b, 1) - r9 = Buffer_getvalue_internal(b) - r10 = Buffer_internal(r9) - b = r10 - r11 = read_str_internal(b) + r9 = WriteBuffer_getvalue_internal(b) + r10 = ReadBuffer_internal(r9) + rb = r10 + r11 = read_str_internal(rb) x = r11 - r12 = read_bytes_internal(b) + r12 = read_bytes_internal(rb) xb = r12 - r13 = read_bool_internal(b) + r13 = read_bool_internal(rb) y = r13 - r14 = read_float_internal(b) + r14 = read_float_internal(rb) z = r14 - r15 = read_int_internal(b) + r15 = read_int_internal(rb) t = r15 - r16 = read_tag_internal(b) + r16 = read_tag_internal(rb) u = r16 r17 = cache_version_internal() v = r17 diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 6ee3ab60f940..d5cca9f9b853 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2979,12 +2979,12 @@ test_buffer_str_size_interpreted() test_buffer_int_powers_interpreted() [case testBufferEmpty_librt_internal] -from librt.internal import Buffer, write_int, read_int +from librt.internal import WriteBuffer, ReadBuffer, write_int, read_int def test_empty() -> None: - b = Buffer(b"") + b = WriteBuffer() write_int(b, 42) - b1 = Buffer(b.getvalue()) + b1 = ReadBuffer(b.getvalue()) assert read_int(b1) == 42 [case testEnumMethodCalls] From a5069a8a4b5364be25f2a5b178fb83c073be27bc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 21 Oct 2025 16:17:29 +0100 Subject: [PATCH 06/27] Fix stubs --- mypy/typeshed/stubs/librt/librt/internal.pyi | 24 ++++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/mypy/typeshed/stubs/librt/librt/internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi index a164490c8679..8b92c7ea8209 100644 --- a/mypy/typeshed/stubs/librt/librt/internal.pyi +++ b/mypy/typeshed/stubs/librt/librt/internal.pyi @@ -6,16 +6,16 @@ class ReadBuffer: class WriteBuffer: def getvalue(self) -> bytes: ... -def write_bool(data: Buffer, value: bool) -> None: ... -def read_bool(data: Buffer) -> bool: ... -def write_str(data: Buffer, value: str) -> None: ... -def read_str(data: Buffer) -> str: ... -def write_bytes(data: Buffer, value: bytes) -> None: ... -def read_bytes(data: Buffer) -> bytes: ... -def write_float(data: Buffer, value: float) -> None: ... -def read_float(data: Buffer) -> float: ... -def write_int(data: Buffer, value: int) -> None: ... -def read_int(data: Buffer) -> int: ... -def write_tag(data: Buffer, value: u8) -> None: ... -def read_tag(data: Buffer) -> u8: ... +def write_bool(data: WriteBuffer, value: bool) -> None: ... +def read_bool(data: ReadBuffer) -> bool: ... +def write_str(data: WriteBuffer, value: str) -> None: ... +def read_str(data: ReadBuffer) -> str: ... +def write_bytes(data: WriteBuffer, value: bytes) -> None: ... +def read_bytes(data: ReadBuffer) -> bytes: ... +def write_float(data: WriteBuffer, value: float) -> None: ... +def read_float(data: ReadBuffer) -> float: ... +def write_int(data: WriteBuffer, value: int) -> None: ... +def read_int(data: ReadBuffer) -> int: ... +def write_tag(data: WriteBuffer, value: u8) -> None: ... +def read_tag(data: ReadBuffer) -> u8: ... def cache_version() -> u8: ... From 0eb8639d4a3f0f1293d8edceb1ac735a6c9f2a74 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 21 Oct 2025 17:02:43 +0100 Subject: [PATCH 07/27] Simlify --- mypyc/lib-rt/librt_internal.c | 35 ++++++++++++----------------------- mypyc/lib-rt/librt_internal.h | 4 ++-- mypyc/primitives/misc_ops.py | 2 +- 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index ed35b4823d79..55e5553efcff 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -118,11 +118,6 @@ ReadBuffer_internal(PyObject *source) { return (PyObject *)self; } -static PyObject* -ReadBuffer_internal_empty(void) { - return ReadBuffer_internal(NULL); -} - static int ReadBuffer_init(ReadBufferObject *self, PyObject *args, PyObject *kwds) { @@ -190,12 +185,7 @@ WriteBuffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -WriteBuffer_init_internal(WriteBufferObject *self, PyObject *source) { - if (source) { - // TODO - PyErr_SetString(PyExc_TypeError, "source should not be provided"); - return -1; - } +WriteBuffer_init_internal(WriteBufferObject *self) { Py_ssize_t size = START_SIZE; self->buf = PyMem_Malloc(size + 1); if (self->buf == NULL) { @@ -208,35 +198,34 @@ WriteBuffer_init_internal(WriteBufferObject *self, PyObject *source) { } static PyObject* -WriteBuffer_internal(PyObject *source) { +WriteBuffer_internal(void) { WriteBufferObject *self = (WriteBufferObject *)WriteBufferType.tp_alloc(&WriteBufferType, 0); if (self == NULL) return NULL; self->buf = NULL; self->ptr = NULL; self->end = NULL; - if (WriteBuffer_init_internal(self, source) == -1) { + if (WriteBuffer_init_internal(self) == -1) { Py_DECREF(self); return NULL; } return (PyObject *)self; } -static PyObject* -WriteBuffer_internal_empty(void) { - return WriteBuffer_internal(NULL); -} - static int WriteBuffer_init(WriteBufferObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"source", NULL}; - PyObject *source = NULL; + if (!PyArg_ParseTuple(args, "")) { + return -1; + } - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &source)) + if (kwds != NULL && PyDict_Size(kwds) > 0) { + PyErr_SetString(PyExc_TypeError, + "WriteBuffer() takes no keyword arguments"); return -1; + } - return WriteBuffer_init_internal(self, source); + return WriteBuffer_init_internal(self); } static void @@ -985,7 +974,7 @@ librt_internal_module_exec(PyObject *m) // Export mypy internal C API, be careful with the order! static void *NativeInternal_API[LIBRT_INTERNAL_API_LEN] = { (void *)ReadBuffer_internal, - (void *)WriteBuffer_internal_empty, + (void *)WriteBuffer_internal, (void *)WriteBuffer_getvalue_internal, (void *)write_bool_internal, (void *)read_bool_internal, diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index c8d76dfeb8ad..d95b0e421cb4 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -7,7 +7,7 @@ #ifdef LIBRT_INTERNAL_MODULE static PyObject *ReadBuffer_internal(PyObject *source); -static PyObject *WriteBuffer_internal_empty(void); +static PyObject *WriteBuffer_internal(void); static PyObject *WriteBuffer_getvalue_internal(PyObject *self); static PyObject *ReadBuffer_internal(PyObject *source); static PyObject *ReadBuffer_internal_empty(void); @@ -31,7 +31,7 @@ static uint8_t cache_version_internal(void); static void **NativeInternal_API; #define ReadBuffer_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[0]) -#define WriteBuffer_internal_empty (*(PyObject* (*)(void)) NativeInternal_API[1]) +#define WriteBuffer_internal (*(PyObject* (*)(void)) NativeInternal_API[1]) #define WriteBuffer_getvalue_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[2]) #define write_bool_internal (*(char (*)(PyObject *source, char value)) NativeInternal_API[3]) #define read_bool_internal (*(char (*)(PyObject *source)) NativeInternal_API[4]) diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 6bf7a8ab3dee..f685b1cfbcf5 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -350,7 +350,7 @@ name="librt.internal.WriteBuffer", arg_types=[], return_type=write_buffer_rprimitive, - c_function_name="WriteBuffer_internal_empty", + c_function_name="WriteBuffer_internal", error_kind=ERR_MAGIC, ) From 3118407ea54f41e1c515edcebb17df3f84ccdf16 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 21 Oct 2025 17:11:11 +0100 Subject: [PATCH 08/27] Fix growing the buffer --- mypyc/lib-rt/librt_internal.c | 2 +- mypyc/test-data/run-classes.test | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 55e5553efcff..ae7bc5a48f4f 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -298,7 +298,7 @@ _check_size(WriteBufferObject *data, Py_ssize_t need) { return CPY_NONE; Py_ssize_t index = data->ptr - data->buf; Py_ssize_t target = index + need; - Py_ssize_t size = data->end - data->ptr; + Py_ssize_t size = data->end - data->buf; do { size *= 2; } while (target >= size); diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index d5cca9f9b853..c9087fb3ae5c 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2721,6 +2721,8 @@ from librt.internal import ( cache_version, ) +from testutil import assertRaises + Tag = u8 TAG_A: Final[Tag] = 33 TAG_B: Final[Tag] = 255 @@ -2733,6 +2735,17 @@ def test_buffer_basic() -> None: r = ReadBuffer(w.getvalue()) assert read_str(r) == "foo" +def test_buffer_grow() -> None: + w = WriteBuffer() + n = 100 * 1000 + for i in range(n): + write_int(w, i & 63) + r = ReadBuffer(w.getvalue()) + for i in range(n): + assert read_int(r) == (i & 63) + with assertRaises(ValueError): + read_int(r) + def test_buffer_roundtrip() -> None: b: WriteBuffer | ReadBuffer b = WriteBuffer() @@ -2863,6 +2876,7 @@ def test_buffer_str_size() -> None: from native import * test_buffer_basic() +test_buffer_grow() test_buffer_roundtrip() test_buffer_int_size() test_buffer_str_size() From 31773bf4dbf5c0fa8f8ccbc47e573fe1df95b2c8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 21 Oct 2025 17:23:02 +0100 Subject: [PATCH 09/27] WIP benchmark --- mypyc/build.py | 1 + mypyc/test-data/run-classes.test | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/mypyc/build.py b/mypyc/build.py index 13648911c0b5..2387b8d97e6b 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -588,6 +588,7 @@ def mypycify( build_dir = compiler_options.target_dir cflags: list[str] = [] + opt_level = 3 if compiler.compiler_type == "unix": cflags += [ f"-O{opt_level}", diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index c9087fb3ae5c..d40396b21174 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -5450,3 +5450,34 @@ def test_read_corrupted_data() -> None: print("RANDOMIZED TEST FAILURE -- please open an issue with the following context:") print(">>>", e, data) raise + + + +[case testXXX_librt_internal] +from librt.internal import WriteBuffer, ReadBuffer, write_int, read_int + +import time + + +def bench(data: bytes, n: int) -> None: + for i in range(100 * 1000): + b = ReadBuffer(data) + for i in range(n): + read_int(b) + + +w = WriteBuffer() +N = 1000 +for i in range(N): + write_int(w, i & 63) + +d = w.getvalue() + +bench(d, N) + +def test_foo() -> None: + t0 = time.time() + bench(d, N) + print("===>", time.time() - t0) + print() + assert False From ad02c93d00e6734bcf6133e5306ccb31914d0692 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 21 Oct 2025 17:36:05 +0100 Subject: [PATCH 10/27] Optimize API calls --- mypyc/lib-rt/librt_internal.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index d95b0e421cb4..7734cfb9a9d9 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -28,7 +28,7 @@ static uint8_t cache_version_internal(void); #else -static void **NativeInternal_API; +static void *NativeInternal_API[LIBRT_INTERNAL_API_LEN]; #define ReadBuffer_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[0]) #define WriteBuffer_internal (*(PyObject* (*)(void)) NativeInternal_API[1]) @@ -55,9 +55,10 @@ import_librt_internal(void) if (mod == NULL) return -1; Py_DECREF(mod); // we import just for the side effect of making the below work. - NativeInternal_API = (void **)PyCapsule_Import("librt.internal._C_API", 0); - if (NativeInternal_API == NULL) + void *capsule = PyCapsule_Import("librt.internal._C_API", 0); + if (capsule == NULL) return -1; + memcpy(NativeInternal_API, capsule, sizeof(NativeInternal_API)); if (NativeInternal_ABI_Version() != LIBRT_INTERNAL_ABI_VERSION) { PyErr_SetString(PyExc_ValueError, "ABI version conflict for librt.internal"); return -1; From 4c3f2196d9c989954da7af11a3910d4881753bae Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 21 Oct 2025 17:37:13 +0100 Subject: [PATCH 11/27] Run 10x more iterations --- mypyc/test-data/run-classes.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index d40396b21174..0f93ac59df6f 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -5460,7 +5460,7 @@ import time def bench(data: bytes, n: int) -> None: - for i in range(100 * 1000): + for i in range(1000 * 1000): b = ReadBuffer(data) for i in range(n): read_int(b) From 7695140cc22d6f18fad4bd56027970a1a85c6951 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 21 Oct 2025 18:07:04 +0100 Subject: [PATCH 12/27] Update benchmark --- mypyc/test-data/run-classes.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 0f93ac59df6f..f32f119b9c74 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -5460,14 +5460,14 @@ import time def bench(data: bytes, n: int) -> None: - for i in range(1000 * 1000): + for i in range(10 * 1000): b = ReadBuffer(data) for i in range(n): read_int(b) w = WriteBuffer() -N = 1000 +N = 400 * 1000 for i in range(N): write_int(w, i & 63) From 1297eb3a11d1aacedef682fb8087229deaa9860c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 3 Nov 2025 14:05:17 +0000 Subject: [PATCH 13/27] Update API len --- mypyc/lib-rt/librt_internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index 7734cfb9a9d9..c60769ae7ace 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -2,7 +2,7 @@ #define LIBRT_INTERNAL_H #define LIBRT_INTERNAL_ABI_VERSION 0 -#define LIBRT_INTERNAL_API_LEN 16 +#define LIBRT_INTERNAL_API_LEN 17 #ifdef LIBRT_INTERNAL_MODULE From 45695d942ce67ec6ca45b8bca7c72735c18759f9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 3 Nov 2025 14:05:35 +0000 Subject: [PATCH 14/27] Fix after rebase --- mypyc/lib-rt/librt_internal.c | 48 +++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index ae7bc5a48f4f..b4c87bbf919f 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -398,14 +398,14 @@ _read_short_int(PyObject *data, uint8_t first) { } if ((first & FOUR_BYTES_INT_BIT) == 0) { _CHECK_READ(data, 1, CPY_INT_TAG) - second = _READ(data, uint8_t) + _READ(&second, data, uint8_t); return ((((Py_ssize_t)second) << 6) + (Py_ssize_t)(first >> 2) + MIN_TWO_BYTES_INT) << 1; } // The caller is responsible to verify this is called only for short ints. _CHECK_READ(data, 3, CPY_INT_TAG) // TODO: check if compilers emit optimal code for these two reads, and tweak if needed. - second = _READ(data, uint8_t) - two_more = _READ(data, uint16_t) + _READ(&second, data, uint8_t); + _READ(&two_more, data, uint16_t); #if PY_BIG_ENDIAN two_more = reverse_16(two_more); #endif @@ -459,27 +459,27 @@ read_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) static inline char _write_short_int(PyObject *data, Py_ssize_t real_value) { if (real_value >= MIN_ONE_BYTE_INT && real_value <= MAX_ONE_BYTE_INT) { - _CHECK_SIZE(data, 1) - _WRITE(data, uint8_t, (uint8_t)(real_value - MIN_ONE_BYTE_INT) << 1) - ((BufferObject *)data)->end += 1; + _CHECK_WRITE(data, 1) + _WRITE(data, uint8_t, (uint8_t)(real_value - MIN_ONE_BYTE_INT) << 1); + ((WriteBufferObject *)data)->ptr += 1; } else if (real_value >= MIN_TWO_BYTES_INT && real_value <= MAX_TWO_BYTES_INT) { - _CHECK_SIZE(data, 2) + _CHECK_WRITE(data, 2) #if PY_BIG_ENDIAN uint16_t to_write = ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT; _WRITE(data, uint16_t, reverse_16(to_write)) #else - _WRITE(data, uint16_t, ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT) + _WRITE(data, uint16_t, ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT); #endif - ((BufferObject *)data)->end += 2; + ((WriteBufferObject *)data)->ptr += 2; } else { - _CHECK_SIZE(data, 4) + _CHECK_WRITE(data, 4) #if PY_BIG_ENDIAN uint32_t to_write = ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER; _WRITE(data, uint32_t, reverse_32(to_write)) #else - _WRITE(data, uint32_t, ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER) + _WRITE(data, uint32_t, ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER); #endif - ((BufferObject *)data)->end += 4; + ((WriteBufferObject *)data)->ptr += 4; } return CPY_NONE; } @@ -500,7 +500,7 @@ write_str_internal(PyObject *data, PyObject *value) { return CPY_NONE_ERROR; } // Write string content. - _CHECK_SIZE(data, size) + _CHECK_WRITE(data, size) char *ptr = ((WriteBufferObject *)data)->ptr; memcpy(ptr, chunk, size); ((WriteBufferObject *)data)->ptr += size; @@ -590,7 +590,7 @@ write_bytes_internal(PyObject *data, PyObject *value) { return CPY_NONE_ERROR; } // Write bytes content. - _CHECK_SIZE(data, size) + _CHECK_WRITE(data, size) char *ptr = ((WriteBufferObject *)data)->ptr; memcpy(ptr, chunk, size); ((WriteBufferObject *)data)->ptr += size; @@ -626,11 +626,11 @@ float format: static double read_float_internal(PyObject *data) { _CHECK_READ(data, 8, CPY_FLOAT_ERROR) - char *ptr = ((BufferObject *)data)->ptr; + char *ptr = ((ReadBufferObject *)data)->ptr; double res = PyFloat_Unpack8(ptr, 1); if (unlikely((res == -1.0) && PyErr_Occurred())) return CPY_FLOAT_ERROR; - ((BufferObject *)data)->ptr += 8; + ((ReadBufferObject *)data)->ptr += 8; return res; } @@ -652,12 +652,12 @@ read_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname static char write_float_internal(PyObject *data, double value) { - _CHECK_SIZE(data, 8) - char *ptr = ((BufferObject *)data)->ptr; + _CHECK_WRITE(data, 8) + char *ptr = ((WriteBufferObject *)data)->ptr; int res = PyFloat_Pack8(value, ptr, 1); if (unlikely(res == -1)) return CPY_NONE_ERROR; - ((BufferObject *)data)->ptr += 8; + ((WriteBufferObject *)data)->ptr += 8; return CPY_NONE; } @@ -720,11 +720,11 @@ read_int_internal(PyObject *data) { // Construct an int object from the byte array. _CHECK_READ(data, size, CPY_INT_TAG) - char *ptr = ((BufferObject *)data)->ptr; + char *ptr = ((ReadBufferObject *)data)->ptr; PyObject *num = _PyLong_FromByteArray((unsigned char *)ptr, size, 1, 0); if (num == NULL) return CPY_INT_TAG; - ((BufferObject *)data)->ptr += size; + ((ReadBufferObject *)data)->ptr += size; if (sign) { PyObject *old = num; num = PyNumber_Negative(old); @@ -764,9 +764,9 @@ static inline int hex_to_int(char c) { static inline char _write_long_int(PyObject *data, CPyTagged value) { - _CHECK_SIZE(data, 1) - _WRITE(data, uint8_t, LONG_INT_TRAILER) - ((BufferObject *)data)->end += 1; + _CHECK_WRITE(data, 1) + _WRITE(data, uint8_t, LONG_INT_TRAILER); + ((WriteBufferObject *)data)->end += 1; PyObject *hex_str = NULL; PyObject* int_value = CPyTagged_AsObject(value); From f97cc7cd66d76ca50408b4b5355fb0e64662cc62 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 3 Nov 2025 14:08:27 +0000 Subject: [PATCH 15/27] Fix test case to use ReadBuffer --- mypyc/test-data/run-classes.test | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index f32f119b9c74..8843aff31228 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -5385,37 +5385,37 @@ test_deletable_attr() [case testBufferCorruptedData_librt_internal] from librt.internal import ( - Buffer, read_bool, read_str, read_float, read_int, read_tag, read_bytes + ReadBuffer, read_bool, read_str, read_float, read_int, read_tag, read_bytes ) from random import randbytes def check(data: bytes) -> None: - b = Buffer(data) + b = ReadBuffer(data) try: while True: read_bool(b) except ValueError: pass - b = Buffer(data) + b = ReadBuffer(data) read_tag(b) # Always succeeds try: while True: read_int(b) except ValueError: pass - b = Buffer(data) + b = ReadBuffer(data) try: while True: read_str(b) except ValueError: pass - b = Buffer(data) + b = ReadBuffer(data) try: while True: read_bytes(b) except ValueError: pass - b = Buffer(data) + b = ReadBuffer(data) try: while True: read_float(b) From 3eaa9898a2bda9dadafe0793b0534a9dcf386f63 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 3 Nov 2025 14:09:23 +0000 Subject: [PATCH 16/27] Fix irbuild test case to use WriteBuffer_internal --- mypyc/test-data/irbuild-classes.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 7707ef4f903d..0f8ec2b094f0 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1494,7 +1494,7 @@ def foo(): r15, t :: int r16, u, r17, v :: u8 L0: - r0 = WriteBuffer_internal_empty() + r0 = WriteBuffer_internal() b = r0 r1 = 'foo' r2 = write_str_internal(b, r1) From 4fb6b1f0d3b89700aed918fffda0196bfc44a212 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 3 Nov 2025 14:23:02 +0000 Subject: [PATCH 17/27] Fix run test to use WriteBuffer/ReadBuffer consistently --- mypyc/test-data/run-classes.test | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 8843aff31228..6056384bb5ad 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2716,7 +2716,7 @@ from __future__ import annotations from typing import Final from mypy_extensions import u8 from librt.internal import ( - ReadBuffer, WriteBuffer, , write_bool, read_bool, write_str, read_str, write_float, read_float, + ReadBuffer, WriteBuffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag, write_bytes, read_bytes, cache_version, ) @@ -2826,35 +2826,35 @@ def test_buffer_int_size() -> None: def test_buffer_int_powers() -> None: # 0, 1, 2 are tested above for p in range(2, 200): - b = Buffer() + b = WriteBuffer() write_int(b, 1 << p) write_int(b, (1 << p) - 1) write_int(b, -1 << p) write_int(b, (-1 << p) + 1) - b = Buffer(b.getvalue()) - assert read_int(b) == 1 << p - assert read_int(b) == (1 << p) - 1 - assert read_int(b) == -1 << p - assert read_int(b) == (-1 << p) + 1 + rb = ReadBuffer(b.getvalue()) + assert read_int(rb) == 1 << p + assert read_int(rb) == (1 << p) - 1 + assert read_int(rb) == -1 << p + assert read_int(rb) == (-1 << p) + 1 def test_positive_long_int_serialized_bytes() -> None: - b = Buffer() + b = WriteBuffer() n = 0x123456789ab write_int(b, n) x = b.getvalue() # Two prefix bytes, followed by little endian encoded integer in variable-length format assert x == b"\x0f\x2c\xab\x89\x67\x45\x23\x01" - b = Buffer(x) - assert read_int(b) == n + rb = ReadBuffer(x) + assert read_int(rb) == n def test_negative_long_int_serialized_bytes() -> None: - b = Buffer() + b = WriteBuffer() n = -0x123456789abcde write_int(b, n) x = b.getvalue() assert x == b"\x0f\x32\xde\xbc\x9a\x78\x56\x34\x12" - b = Buffer(x) - assert read_int(b) == n + rb = ReadBuffer(x) + assert read_int(rb) == n def test_buffer_str_size() -> None: b: WriteBuffer | ReadBuffer @@ -2964,10 +2964,10 @@ def test_buffer_int_size_interpreted() -> None: def test_buffer_int_powers_interpreted() -> None: # 0, 1, 2 are tested above for p in range(2, 9): - b = Buffer() + b = WriteBuffer() write_int(b, 1 << p) write_int(b, -1 << p) - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == 1 << p assert read_int(b) == -1 << p From e1311bece9452ba046e8efd8901b189d4a913ce1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 3 Nov 2025 14:23:31 +0000 Subject: [PATCH 18/27] Fix WriteBuffer writes in C lib --- mypyc/lib-rt/librt_internal.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index b4c87bbf919f..c2c524c27a63 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -461,7 +461,6 @@ _write_short_int(PyObject *data, Py_ssize_t real_value) { if (real_value >= MIN_ONE_BYTE_INT && real_value <= MAX_ONE_BYTE_INT) { _CHECK_WRITE(data, 1) _WRITE(data, uint8_t, (uint8_t)(real_value - MIN_ONE_BYTE_INT) << 1); - ((WriteBufferObject *)data)->ptr += 1; } else if (real_value >= MIN_TWO_BYTES_INT && real_value <= MAX_TWO_BYTES_INT) { _CHECK_WRITE(data, 2) #if PY_BIG_ENDIAN @@ -470,7 +469,6 @@ _write_short_int(PyObject *data, Py_ssize_t real_value) { #else _WRITE(data, uint16_t, ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT); #endif - ((WriteBufferObject *)data)->ptr += 2; } else { _CHECK_WRITE(data, 4) #if PY_BIG_ENDIAN @@ -479,7 +477,6 @@ _write_short_int(PyObject *data, Py_ssize_t real_value) { #else _WRITE(data, uint32_t, ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER); #endif - ((WriteBufferObject *)data)->ptr += 4; } return CPY_NONE; } @@ -766,7 +763,6 @@ static inline char _write_long_int(PyObject *data, CPyTagged value) { _CHECK_WRITE(data, 1) _WRITE(data, uint8_t, LONG_INT_TRAILER); - ((WriteBufferObject *)data)->end += 1; PyObject *hex_str = NULL; PyObject* int_value = CPyTagged_AsObject(value); From e4d0c0371ebb77d18c6514403dcda9c356034bf2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Nov 2025 14:24:15 +0000 Subject: [PATCH 19/27] Bump ABI version --- mypyc/lib-rt/librt_internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index c60769ae7ace..ec5e7822cf02 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -1,7 +1,7 @@ #ifndef LIBRT_INTERNAL_H #define LIBRT_INTERNAL_H -#define LIBRT_INTERNAL_ABI_VERSION 0 +#define LIBRT_INTERNAL_ABI_VERSION 1 #define LIBRT_INTERNAL_API_LEN 17 #ifdef LIBRT_INTERNAL_MODULE From 84b6a7013f7cfa2b7812499970f5e190b0b8a00c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Nov 2025 14:37:02 +0000 Subject: [PATCH 20/27] Provide access to the type objects via API --- mypyc/lib-rt/librt_internal.c | 12 ++++++++++++ mypyc/lib-rt/librt_internal.h | 15 ++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index c2c524c27a63..bfa065f380ab 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -929,6 +929,16 @@ cache_version(PyObject *self, PyObject *Py_UNUSED(ignored)) { return PyLong_FromLong(cache_version_internal()); } +static PyTypeObject * +ReadBuffer_type_internal(void) { + return &ReadBufferType; // Return borrowed reference +} + +static PyTypeObject * +WriteBuffer_type_internal(void) { + return &WriteBufferType; // Return borrowed reference +}; + static PyMethodDef librt_internal_module_methods[] = { {"write_bool", (PyCFunction)write_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a bool")}, {"read_bool", (PyCFunction)read_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a bool")}, @@ -986,6 +996,8 @@ librt_internal_module_exec(PyObject *m) (void *)write_bytes_internal, (void *)read_bytes_internal, (void *)cache_version_internal, + (void *)ReadBuffer_type_internal, + (void *)WriteBuffer_type_internal, }; PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index ec5e7822cf02..329a0fd68c11 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -2,7 +2,7 @@ #define LIBRT_INTERNAL_H #define LIBRT_INTERNAL_ABI_VERSION 1 -#define LIBRT_INTERNAL_API_LEN 17 +#define LIBRT_INTERNAL_API_LEN 19 #ifdef LIBRT_INTERNAL_MODULE @@ -25,6 +25,8 @@ static int NativeInternal_ABI_Version(void); static char write_bytes_internal(PyObject *data, PyObject *value); static PyObject *read_bytes_internal(PyObject *data); static uint8_t cache_version_internal(void); +static PyTypeObject *ReadBuffer_type_internal(void); +static PyTypeObject *WriteBuffer_type_internal(void); #else @@ -47,6 +49,8 @@ static void *NativeInternal_API[LIBRT_INTERNAL_API_LEN]; #define write_bytes_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[14]) #define read_bytes_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[15]) #define cache_version_internal (*(uint8_t (*)(void)) NativeInternal_API[16]) +#define ReadBuffer_type_internal (*(PyTypeObject* (*)(void)) NativeInternal_API[17]) +#define WriteBuffer_type_internal (*(PyTypeObject* (*)(void)) NativeInternal_API[18]) static int import_librt_internal(void) @@ -67,4 +71,13 @@ import_librt_internal(void) } #endif + +static inline bool CPyReadBuffer_Check(PyObject *obj) { + return Py_TYPE(obj) == ReadBuffer_type_internal(); +} + +static inline bool CPyWriteBuffer_Check(PyObject *obj) { + return Py_TYPE(obj) == WriteBuffer_type_internal(); +} + #endif // LIBRT_INTERNAL_H From c74f01d042d332e91cd15db67fe77fc07a5a501b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Nov 2025 15:33:49 +0000 Subject: [PATCH 21/27] Perform runtime type checks --- mypyc/codegen/emit.py | 14 +++++++++++++- mypyc/test-data/run-classes.test | 19 ++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 4ef53296ef0d..f2a2271e020e 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -705,13 +705,25 @@ def emit_cast( self.emit_lines(f" {dest} = {src};", "else {") self.emit_cast_error_handler(error, src, dest, typ, raise_exception) self.emit_line("}") - elif is_object_rprimitive(typ) or is_native_rprimitive(typ): + elif is_object_rprimitive(typ): if declare_dest: self.emit_line(f"PyObject *{dest};") self.emit_arg_check(src, dest, typ, "", optional) self.emit_line(f"{dest} = {src};") if optional: self.emit_line("}") + elif is_native_rprimitive(typ): + # Native primitive types have type check functions of form "CPy_Check(...)". + if declare_dest: + self.emit_line(f"PyObject *{dest};") + short_name = typ.name.rsplit(".", 1)[-1] + check = f"(CPy{short_name}_Check({src}))" + if likely: + check = f"(likely{check})" + self.emit_arg_check(src, dest, typ, check, optional) + self.emit_lines(f" {dest} = {src};", "else {") + self.emit_cast_error_handler(error, src, dest, typ, raise_exception) + self.emit_line("}") elif isinstance(typ, RUnion): self.emit_union_cast( src, dest, typ, declare_dest, error, optional, src_type, raise_exception diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 6056384bb5ad..58d89ef31a18 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2713,7 +2713,7 @@ Player.MIN = [case testBufferRoundTrip_librt_internal] from __future__ import annotations -from typing import Final +from typing import Final, Any from mypy_extensions import u8 from librt.internal import ( ReadBuffer, WriteBuffer, write_bool, read_bool, write_str, read_str, write_float, read_float, @@ -2746,6 +2746,23 @@ def test_buffer_grow() -> None: with assertRaises(ValueError): read_int(r) +def test_buffer_primitive_types() -> None: + a1: Any = WriteBuffer() + w: WriteBuffer = a1 + write_str(w, "foo") + data = w.getvalue() + assert read_str(ReadBuffer(data)) == "foo" + a2: Any = ReadBuffer(b"foo") + with assertRaises(TypeError): + w2: WriteBuffer = a2 + + a3: Any = ReadBuffer(data) + r: ReadBuffer = a3 + assert read_str(r) == "foo" + a4: Any = WriteBuffer() + with assertRaises(TypeError): + r2: ReadBuffer = a4 + def test_buffer_roundtrip() -> None: b: WriteBuffer | ReadBuffer b = WriteBuffer() From 24927124637bfd0084d1d0c2d1d1f188879c4443 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Nov 2025 15:39:08 +0000 Subject: [PATCH 22/27] Revert benchmark related changes --- mypyc/build.py | 1 - mypyc/test-data/run-classes.test | 31 ------------------------------- 2 files changed, 32 deletions(-) diff --git a/mypyc/build.py b/mypyc/build.py index 2387b8d97e6b..13648911c0b5 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -588,7 +588,6 @@ def mypycify( build_dir = compiler_options.target_dir cflags: list[str] = [] - opt_level = 3 if compiler.compiler_type == "unix": cflags += [ f"-O{opt_level}", diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 58d89ef31a18..9b5b1b698258 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -5467,34 +5467,3 @@ def test_read_corrupted_data() -> None: print("RANDOMIZED TEST FAILURE -- please open an issue with the following context:") print(">>>", e, data) raise - - - -[case testXXX_librt_internal] -from librt.internal import WriteBuffer, ReadBuffer, write_int, read_int - -import time - - -def bench(data: bytes, n: int) -> None: - for i in range(10 * 1000): - b = ReadBuffer(data) - for i in range(n): - read_int(b) - - -w = WriteBuffer() -N = 400 * 1000 -for i in range(N): - write_int(w, i & 63) - -d = w.getvalue() - -bench(d, N) - -def test_foo() -> None: - t0 = time.time() - bench(d, N) - print("===>", time.time() - t0) - print() - assert False From 906bd7133d9b5c38112480627ce616a117b00323 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Nov 2025 15:57:27 +0000 Subject: [PATCH 23/27] Test arg checking when calling via wrapper functions --- mypyc/test-data/run-classes.test | 47 ++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 9b5b1b698258..54c3268f038b 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2763,6 +2763,53 @@ def test_buffer_primitive_types() -> None: with assertRaises(TypeError): r2: ReadBuffer = a4 +def test_type_check_args_in_write_functions() -> None: + # Test calling wrapper functions with invalid arg types + from librt import internal + alias: Any = internal + w = WriteBuffer() + with assertRaises(TypeError): + alias.write_str(None, "foo") + with assertRaises(TypeError): + alias.write_str(w, None) + with assertRaises(TypeError): + alias.write_bool(None, True) + with assertRaises(TypeError): + alias.write_bool(w, None) + with assertRaises(TypeError): + alias.write_bytes(None, b"foo") + with assertRaises(TypeError): + alias.write_bytes(w, None) + with assertRaises(TypeError): + alias.write_float(None, 1.5) + with assertRaises(TypeError): + alias.write_float(w, None) + with assertRaises(TypeError): + alias.write_int(None, 15) + with assertRaises(TypeError): + alias.write_int(w, None) + with assertRaises(TypeError): + alias.write_tag(None, 15) + with assertRaises(TypeError): + alias.write_tag(w, None) + +def test_type_check_buffer_in_read_functions() -> None: + # Test calling wrapper functions with invalid arg types + from librt import internal + alias: Any = internal + with assertRaises(TypeError): + alias.read_str(None) + with assertRaises(TypeError): + alias.read_bool(None) + with assertRaises(TypeError): + alias.read_bytes(None) + with assertRaises(TypeError): + alias.read_float(None) + with assertRaises(TypeError): + alias.read_int(None) + with assertRaises(TypeError): + alias.read_tag(None) + def test_buffer_roundtrip() -> None: b: WriteBuffer | ReadBuffer b = WriteBuffer() From b1b1d9bcd2072bc772f91fd55e0e28bfa63d7b79 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Nov 2025 16:04:15 +0000 Subject: [PATCH 24/27] Fix stub --- mypy/typeshed/stubs/librt/librt/internal.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/typeshed/stubs/librt/librt/internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi index 8b92c7ea8209..2969ccfbadda 100644 --- a/mypy/typeshed/stubs/librt/librt/internal.pyi +++ b/mypy/typeshed/stubs/librt/librt/internal.pyi @@ -1,7 +1,7 @@ from mypy_extensions import u8 class ReadBuffer: - def __init__(self, source: bytes = ...) -> None: ... + def __init__(self, source: bytes) -> None: ... class WriteBuffer: def getvalue(self) -> bytes: ... From a568b9c23ed2268f350084c9c610bf82c2d4ca5c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Nov 2025 16:13:35 +0000 Subject: [PATCH 25/27] Hacky support in stub for both the old and new APIs Only one of these works at runtime, however. --- mypy/typeshed/stubs/librt/librt/internal.pyi | 30 ++++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/mypy/typeshed/stubs/librt/librt/internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi index 2969ccfbadda..78e7f9caa117 100644 --- a/mypy/typeshed/stubs/librt/librt/internal.pyi +++ b/mypy/typeshed/stubs/librt/librt/internal.pyi @@ -1,21 +1,27 @@ from mypy_extensions import u8 +# TODO: Remove Buffer -- right now we have hacky support for BOTH the old and new APIs + +class Buffer: + def __init__(self, source: bytes = ...) -> None: ... + def getvalue(self) -> bytes: ... + class ReadBuffer: def __init__(self, source: bytes) -> None: ... class WriteBuffer: def getvalue(self) -> bytes: ... -def write_bool(data: WriteBuffer, value: bool) -> None: ... -def read_bool(data: ReadBuffer) -> bool: ... -def write_str(data: WriteBuffer, value: str) -> None: ... -def read_str(data: ReadBuffer) -> str: ... -def write_bytes(data: WriteBuffer, value: bytes) -> None: ... -def read_bytes(data: ReadBuffer) -> bytes: ... -def write_float(data: WriteBuffer, value: float) -> None: ... -def read_float(data: ReadBuffer) -> float: ... -def write_int(data: WriteBuffer, value: int) -> None: ... -def read_int(data: ReadBuffer) -> int: ... -def write_tag(data: WriteBuffer, value: u8) -> None: ... -def read_tag(data: ReadBuffer) -> u8: ... +def write_bool(data: WriteBuffer | Buffer, value: bool) -> None: ... +def read_bool(data: ReadBuffer | Buffer) -> bool: ... +def write_str(data: WriteBuffer | Buffer, value: str) -> None: ... +def read_str(data: ReadBuffer | Buffer) -> str: ... +def write_bytes(data: WriteBuffer | Buffer, value: bytes) -> None: ... +def read_bytes(data: ReadBuffer | Buffer) -> bytes: ... +def write_float(data: WriteBuffer | Buffer, value: float) -> None: ... +def read_float(data: ReadBuffer | Buffer) -> float: ... +def write_int(data: WriteBuffer | Buffer, value: int) -> None: ... +def read_int(data: ReadBuffer | Buffer) -> int: ... +def write_tag(data: WriteBuffer | Buffer, value: u8) -> None: ... +def read_tag(data: ReadBuffer | Buffer) -> u8: ... def cache_version() -> u8: ... From 2e331472ad42cb6356e846bc4fe511180aaa8daf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Nov 2025 16:21:01 +0000 Subject: [PATCH 26/27] Minor fixes --- mypyc/lib-rt/librt_internal.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index bfa065f380ab..91ad1a3ad53f 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -189,8 +189,8 @@ WriteBuffer_init_internal(WriteBufferObject *self) { Py_ssize_t size = START_SIZE; self->buf = PyMem_Malloc(size + 1); if (self->buf == NULL) { - // TODO - CPyError_OutOfMemory(); + PyErr_NoMemory(); + return -1; } self->ptr = self->buf; self->end = self->buf + size; @@ -232,6 +232,7 @@ static void WriteBuffer_dealloc(WriteBufferObject *self) { PyMem_Free(self->buf); + self->buf = NULL; Py_TYPE(self)->tp_free((PyObject *)self); } From b9ffa18d48e362a55bd02c8b0baa8652a424fd3d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Nov 2025 16:56:00 +0000 Subject: [PATCH 27/27] Minor optimization --- mypyc/lib-rt/librt_internal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 91ad1a3ad53f..ada2dfeb39a5 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -98,7 +98,7 @@ ReadBuffer_init_internal(ReadBufferObject *self, PyObject *source) { return -1; } self->source = Py_NewRef(source); - self->ptr = PyBytes_AsString(source); + self->ptr = PyBytes_AS_STRING(source); self->end = self->ptr + PyBytes_GET_SIZE(source); return 0; }