Add writeUtf8Char

Author: e5l

core/commonMain/src/kotlinx/io/text/TextInputObsolete.kt

@@ -255,8 +255,8 @@ private val Utf8StateMachine = intArrayOf(
 
 private const val STATE_FINISH = -2
 //private const val Utf8_STATE_ASCII = -1
-private const val STATE_UTF_8 = 0
-private const val STATE_REJECT = 1
+internal const val STATE_UTF_8 = 0
+internal const val STATE_REJECT = 1
 
 private inline fun Input.decodeUtf8(consumer: (Int) -> Boolean) {
     val stateMachine = Utf8StateMachine

core/commonMain/src/kotlinx/io/text/TextOutputObsolete.kt

@@ -27,8 +27,9 @@ public fun Output.writeUtf8String(text: CharSequence, index: Int = 0, length: In
                     continue
                 }
 
-                if (textIndex == textEndIndex)
+                if (textIndex == textEndIndex) {
                     return@writeBuffer offset
+                }
 
                 // get next character
                 val character = text[textIndex++]
@@ -41,8 +42,8 @@ public fun Output.writeUtf8String(text: CharSequence, index: Int = 0, length: In
                 // fetch next code
                 val code = when {
                     character.isHighSurrogate() -> {
-                        if (textIndex == textEndIndex - 1) {
-                            throw MalformedInputException("Splitted surrogate character")
+                        if (textIndex == textEndIndex) {
+                            splittedSurogate()
                         }
                         codePoint(character, text[textIndex++])
                     }
@@ -51,7 +52,7 @@ public fun Output.writeUtf8String(text: CharSequence, index: Int = 0, length: In
 
                 // write Utf8 bytes to buffer or queue them for write in `bytes` if not enough space
                 when {
-                    code < 0x7ff -> {
+                    code <= 0x7ff -> {
                         buffer[offset++] = (0xc0 or ((code shr 6) and 0x1f)).toByte()
                         val byte1 = (code and 0x3f) or 0x80
                         if (offset < buffer.size) {
@@ -60,7 +61,7 @@ public fun Output.writeUtf8String(text: CharSequence, index: Int = 0, length: In
                             bytes = byte1
                         }
                     }
-                    code < 0xffff -> {
+                    code <= 0xffff -> {
                         buffer[offset++] = ((code shr 12) and 0x0f or 0xe0).toByte()
                         val byte1 = ((code shr 6) and 0x3f) or 0x80
                         val byte2 = (code and 0x3f) or 0x80
@@ -71,7 +72,7 @@ public fun Output.writeUtf8String(text: CharSequence, index: Int = 0, length: In
                             bytes = (byte2 shl 8) or byte1 // order is reversed for writes
                         }
                     }
-                    code < 0x10ffff -> {
+                    code <= 0x10ffff -> {
                         buffer[offset++] = ((code shr 18) and 0x07 or 0xf0).toByte()
                         val byte1 = ((code shr 12) and 0x3f) or 0x80
                         val byte2 = ((code shr 6) and 0x3f) or 0x80
@@ -93,6 +94,52 @@ public fun Output.writeUtf8String(text: CharSequence, index: Int = 0, length: In
     flush()
 }
 
+/**
+ * Write single UTF-8 [character] to [Output].
+ *
+ * @return count of written bytes
+ * @throws MalformedInputException if [character] is splitted surrogate or invalid.
+ */
+public fun Output.writeUtf8Char(character: Char): Int {
+    // ASCII character
+    if (character <= lastASCII) {
+        writeByte(character.toByte())
+        return 1
+    }
+
+    if (character.isHighSurrogate()) {
+        throw MalformedInputException("Splitted surrogate character: $character")
+    }
+
+    val code = character.toInt()
+
+    return when {
+        code <= 0x7ff -> {
+            val byte0 = (0xc0 or ((code shr 6) and 0x1f))
+            val byte1 = (code and 0x3f) or 0x80
+            writeShort(((byte0 shl 8) or byte1).toShort())
+            2
+        }
+        code <= 0xffff -> {
+            val byte0 = ((code shr 12) and 0x0f or 0xe0).toByte()
+            val byte1 = ((code shr 6) and 0x3f) or 0x80
+            val byte2 = (code and 0x3f) or 0x80
+            writeByte(byte0)
+            writeShort(((byte1 shl 8) or byte2).toShort())
+            3
+        }
+        code <= 0x10ffff -> {
+            val byte0 = ((code shr 18) and 0x07 or 0xf0)
+            val byte1 = ((code shr 12) and 0x3f) or 0x80
+            val byte2 = ((code shr 6) and 0x3f) or 0x80
+            val byte3 = (code and 0x3f) or 0x80
+            writeInt((byte0 shl 24) or (byte1 shl 16) or (byte2 shl 8) or byte3)
+            4
+        }
+        else -> malformedCodePoint(code)
+    }
+}
+
 internal fun codePoint(high: Char, low: Char): Int {
     check(high.isHighSurrogate())
     check(low.isLowSurrogate())
@@ -107,3 +154,7 @@ private fun malformedCodePoint(codePoint: Int): Nothing {
     // TODO: revise exceptions
     throw MalformedInputException("Malformed Utf8 code point $codePoint")
 }
+
+internal fun splittedSurogate(): Nothing {
+    throw MalformedInputException("Splitted surrogate character")
+}

core/commonTest/src/kotlinx/io/text/OutputStringTest.kt

@@ -69,6 +69,34 @@ open class OutputStringTest {
         assertEquals(expected.contentToString(), read.contentToString())
     }
 
+    @Test
+    fun testWriteUtf8Chars() = bufferSizes.forEach { size ->
+        val text = "file content with unicode  : здороваться : 여보세요 : 你好 : ñç."
+        // @formatter:off
+        val expected = ubyteArrayOf(
+            0x66u, 0x69u, 0x6cu, 0x65u, 0x20u, 0x63u, 0x6fu, 0x6eu, 0x74u, 0x65u, 0x6eu, 0x74u, 0x20u,
+            0x77u, 0x69u, 0x74u, 0x68u, 0x20u, 0x75u, 0x6eu, 0x69u, 0x63u, 0x6fu, 0x64u, 0x65u, 0x20u, // ascii ends
+            0x20u, 0x3au, 0x20u, 0xd0u, 0xb7u, 0xd0u, 0xb4u, 0xd0u, 0xbeu,
+            0xd1u, 0x80u, 0xd0u, 0xbeu, 0xd0u, 0xb2u, 0xd0u, 0xb0u, 0xd1u, 0x82u, 0xd1u, 0x8cu, 0xd1u,
+            0x81u, 0xd1u, 0x8fu, 0x20u, 0x3au, 0x20u, 0xecu, 0x97u, 0xacu, 0xebu, 0xb3u, 0xb4u, 0xecu,
+            0x84u, 0xb8u, 0xecu, 0x9au, 0x94u, 0x20u, 0x3au, 0x20u, 0xe4u, 0xbdu, 0xa0u,
+            0xe5u, 0xa5u, 0xbdu, 0x20u, 0x3au, 0x20u, 0xc3u, 0xb1u, 0xc3u, 0xa7u, 0x2eu
+        )
+        // @formatter:on
+
+        val bytes = buildBytes(size) {
+            text.forEach { writeUtf8Char(it) }
+        }
+
+        assertEquals(expected.size, bytes.size(), "Size $size")
+
+        val input = bytes.input()
+        val read = UByteArray(expected.size)
+        input.readByteArray(read)
+        assertTrue(input.exhausted(), "EOF")
+        assertEquals(expected.contentToString(), read.contentToString())
+    }
+
     @Test
     fun testWriteMultiByteAtEnd() {
         val input = buildBytes {
@@ -140,5 +168,32 @@ open class OutputStringTest {
         assertEquals("4444", input.readUtf8Line())
         assertTrue(input.exhausted(), "EOF")
     }
+
+    @Test
+    fun testWriteSingleUnicode() {
+        val text = """🤔"""
+        buildBytes {
+            writeUtf8String(text, 0, text.length)
+        }.useInput {
+            val actual = buildString {
+                readUtf8LineTo(this)
+            }
+            assertEquals(text, actual)
+        }
+    }
+
+    @Test
+    fun testParseGlyph() {
+        val text = """􏿿"""
+        buildBytes {
+            writeUtf8String(text, 0, text.length)
+        }.useInput {
+            val actual = buildString {
+                readUtf8LineTo(this)
+            }
+
+            assertEquals(text, actual)
+        }
+    }
 }