Refactor PString native representation

Author: msimacek

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextSlotBuiltins.java

@@ -77,7 +77,6 @@
 import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.setter;
 import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.vectorcallfunc;
 import static com.oracle.graal.python.nodes.HiddenAttr.METHOD_DEF_PTR;
-import static com.oracle.graal.python.nodes.HiddenAttr.NATIVE_STORAGE;
 import static com.oracle.graal.python.nodes.HiddenAttr.PROMOTED_START;
 import static com.oracle.graal.python.nodes.HiddenAttr.PROMOTED_STEP;
 import static com.oracle.graal.python.nodes.HiddenAttr.PROMOTED_STOP;
@@ -111,7 +110,7 @@
 import com.oracle.graal.python.builtins.objects.object.PythonObject;
 import com.oracle.graal.python.builtins.objects.set.PBaseSet;
 import com.oracle.graal.python.builtins.objects.slice.PSlice;
-import com.oracle.graal.python.builtins.objects.str.NativeCharSequence;
+import com.oracle.graal.python.builtins.objects.str.NativeStringData;
 import com.oracle.graal.python.builtins.objects.str.PString;
 import com.oracle.graal.python.builtins.objects.str.StringNodes;
 import com.oracle.graal.python.builtins.objects.str.StringNodes.StringLenNode;
@@ -167,10 +166,12 @@ abstract static class GraalPyPrivate_Get_PyASCIIObject_state_ascii extends CApiU
         int get(PString object,
                         @Bind Node inliningTarget,
                         @Cached InlinedConditionProfile storageProfile,
+                        @Cached HiddenAttr.ReadNode readAttrNode,
                         @Cached TruffleString.GetCodeRangeNode getCodeRangeNode) {
             // important: avoid materialization of native sequences
-            if (storageProfile.profile(inliningTarget, object.isNativeCharSequence())) {
-                return object.getNativeCharSequence().isAsciiOnly() ? 1 : 0;
+            NativeStringData nativeData = object.getNativeStringData(inliningTarget, readAttrNode);
+            if (storageProfile.profile(inliningTarget, nativeData != null)) {
+                return nativeData.isAscii() ? 1 : 0;
             }
 
             TruffleString string = object.getMaterialized();
@@ -205,10 +206,12 @@ abstract static class GraalPyPrivate_Get_PyASCIIObject_state_kind extends CApiUn
         static int get(PString object,
                         @Bind Node inliningTarget,
                         @Cached InlinedConditionProfile storageProfile,
+                        @Cached HiddenAttr.ReadNode readAttrNode,
                         @Cached TruffleString.GetCodeRangeNode getCodeRangeNode) {
             // important: avoid materialization of native sequences
-            if (storageProfile.profile(inliningTarget, object.isNativeCharSequence())) {
-                return object.getNativeCharSequence().getElementSize() & 0b111;
+            NativeStringData nativeData = object.getNativeStringData(inliningTarget, readAttrNode);
+            if (storageProfile.profile(inliningTarget, nativeData != null)) {
+                return nativeData.getCharSize();
             }
             TruffleString string = object.getMaterialized();
             TruffleString.CodeRange range = getCodeRangeNode.execute(string, TS_ENCODING);
@@ -677,10 +680,12 @@ static Object get(PString object,
                         @Cached TruffleString.SwitchEncodingNode switchEncodingNode,
                         @Cached CStructAccess.AllocateNode allocateNode,
                         @Cached CStructAccess.WriteTruffleStringNode writeTruffleStringNode,
-                        @Cached HiddenAttr.WriteNode writeAttribute) {
-            if (object.isNativeCharSequence()) {
+                        @Cached HiddenAttr.ReadNode readAttrNode,
+                        @Cached HiddenAttr.WriteNode writeAttrNode) {
+            NativeStringData nativeData = object.getNativeStringData(inliningTarget, readAttrNode);
+            if (nativeData != null) {
                 // in this case, we can just return the pointer
-                return object.getNativeCharSequence().getPtr();
+                return nativeData.getPtr();
             }
             TruffleString string = object.getMaterialized();
             TruffleString.CodeRange range = getCodeRangeNode.execute(string, TS_ENCODING);
@@ -702,20 +707,14 @@ static Object get(PString object,
                 encoding = TruffleString.Encoding.UTF_32;
             }
             string = switchEncodingNode.execute(string, encoding);
-            int byteLength = string.byteLength(encoding) + /* null terminator */ charSize;
-            Object ptr = allocateNode.alloc(byteLength);
+            int byteLength = string.byteLength(encoding);
+            Object ptr = allocateNode.alloc(byteLength + /* null terminator */ charSize);
             writeTruffleStringNode.write(ptr, string, encoding);
             /*
-             * Set native char sequence, so we can just return the pointer the next time.
+             * Set native data, so we can just return the pointer the next time.
              */
-            NativeCharSequence nativeSequence = new NativeCharSequence(ptr, string.byteLength(encoding) / charSize, charSize, isAscii);
-            object.setNativeCharSequence(nativeSequence);
-            /*
-             * Create a native sequence storage to manage the lifetime of the native memory.
-             *
-             * TODO it would be nicer if the native char sequence could manage its own memory
-             */
-            writeAttribute.execute(inliningTarget, object, NATIVE_STORAGE, NativeByteSequenceStorage.create(ptr, byteLength, byteLength, true));
+            NativeStringData data = NativeStringData.create(charSize, isAscii, ptr, byteLength);
+            object.setNativeStringData(inliningTarget, writeAttrNode, data);
             return ptr;
         }
     }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextUnicodeBuiltins.java

@@ -124,7 +124,7 @@
 import com.oracle.graal.python.builtins.objects.dict.PDict;
 import com.oracle.graal.python.builtins.objects.ints.PInt;
 import com.oracle.graal.python.builtins.objects.memoryview.PMemoryView;
-import com.oracle.graal.python.builtins.objects.str.NativeCharSequence;
+import com.oracle.graal.python.builtins.objects.str.NativeStringData;
 import com.oracle.graal.python.builtins.objects.str.PString;
 import com.oracle.graal.python.builtins.objects.str.StringBuiltins;
 import com.oracle.graal.python.builtins.objects.str.StringBuiltins.EncodeNode;
@@ -812,12 +812,22 @@ static int doGeneric(Object type, long lindex,
         }
     }
 
-    @CApiBuiltin(ret = PyObjectTransfer, args = {Pointer, Py_ssize_t, Py_ssize_t, PY_UCS4}, call = Ignored)
+    @CApiBuiltin(ret = PyObjectTransfer, args = {Pointer, Py_ssize_t, Int, Int}, call = Ignored)
     abstract static class GraalPyPrivate_Unicode_New extends CApiQuaternaryBuiltinNode {
         @Specialization
-        static Object doGeneric(Object ptr, long elements, long elementSize, int isAscii,
-                        @Bind PythonLanguage language) {
-            return PFactory.createString(language, new NativeCharSequence(ptr, (int) elements, (int) elementSize, isAscii != 0));
+        static Object doGeneric(Object ptr, long elements, int charSize, int isAscii,
+                        @Bind Node inliningTarget,
+                        @Bind PythonLanguage language,
+                        @Cached HiddenAttr.WriteNode writeNode,
+                        @Cached PRaiseNode raiseNode) {
+            long size = elements * charSize;
+            if (!PInt.isIntRange(size)) {
+                throw raiseNode.raise(inliningTarget, MemoryError);
+            }
+            PString s = PFactory.createString(language, null);
+            NativeStringData data = NativeStringData.create(charSize, isAscii != 0, ptr, (int) size);
+            s.setNativeStringData(inliningTarget, writeNode, data);
+            return s;
         }
     }
 
@@ -1246,9 +1256,6 @@ abstract static class GraalPyPrivate_Unicode_IsMaterialized extends CApiUnaryBui
 
         @Specialization
         static int pstring(PString s) {
-            if (s.isNativeCharSequence()) {
-                return s.isNativeMaterialized() ? 1 : 0;
-            }
             return s.isMaterialized() ? 1 : 0;
         }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/str/NativeCharSequence.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.builtins.objects.str;
+
+import com.oracle.graal.python.runtime.sequence.storage.NativeByteSequenceStorage;
+import com.oracle.truffle.api.CompilerDirectives;
+import com.oracle.truffle.api.strings.TruffleString;
+
+public final class NativeStringData {
+    private static final byte KIND_ASCII = 0;
+    private static final byte KIND_1BYTE = 1;
+    private static final byte KIND_2BYTE = 2;
+    private static final byte KIND_4BYTE = 4;
+    private final byte kind;
+    // We need the storage object for memory management, don't inline its fields here
+    private final NativeByteSequenceStorage storage;
+
+    private NativeStringData(int charSize, boolean isAscii, NativeByteSequenceStorage storage) {
+        assert charSize == 1 || charSize == 2 || charSize == 4;
+        assert !isAscii || charSize == 1;
+        this.kind = isAscii ? KIND_ASCII : (byte) charSize;
+        this.storage = storage;
+    }
+
+    public static NativeStringData create(int charSize, boolean isAscii, Object ptr, int length) {
+        return new NativeStringData(charSize, isAscii, NativeByteSequenceStorage.create(ptr, length, length, true));
+    }
+
+    public boolean isAscii() {
+        return kind == KIND_ASCII;
+    }
+
+    public int getCharSize() {
+        return kind != 0 ? kind : KIND_1BYTE;
+    }
+
+    public Object getPtr() {
+        return storage.getPtr();
+    }
+
+    public int length() {
+        return storage.length();
+    }
+
+    public TruffleString toTruffleString(TruffleString.FromNativePointerNode fromNativePointerNode) {
+        TruffleString.Encoding encoding = switch (kind) {
+            case KIND_ASCII -> TruffleString.Encoding.US_ASCII;
+            case KIND_1BYTE -> TruffleString.Encoding.ISO_8859_1;
+            case KIND_2BYTE -> TruffleString.Encoding.UTF_16;
+            case KIND_4BYTE -> TruffleString.Encoding.UTF_32;
+            default -> throw CompilerDirectives.shouldNotReachHere();
+        };
+        // NativeByteSequenceStorage implements asPointer
+        return fromNativePointerNode.execute(storage, 0, storage.length(), encoding, false);
+    }
+}