[stdlib] _CharacterRecognizer._firstBreak(inUncheckedUnsafeUTF8Buffer:startingAt:)

Author: lorentey

stdlib/public/core/StringGraphemeBreaking.swift

@@ -506,6 +506,44 @@ extension Unicode {
       _previous = next
       return r
     }
+
+    /// Decode the scalars in the given UTF-8 buffer and feed them to the
+    /// recognizer up to and including the scalar following the first grapheme
+    /// break. If the buffer contains a grapheme break, then this function
+    /// returns the index range of the scalar that follows the first one;
+    /// otherwise it returns `nil`.
+    ///
+    /// On return, the state of the recognizer is updated to reflect the scalars
+    /// up to and including the returned one. You can detect additional grapheme
+    /// breaks by feeding the recognizer subsequent data.
+    ///
+    /// - Parameter buffer: A buffer containing valid UTF-8 data, starting and
+    ///    ending on Unicode scalar boundaries.
+    ///
+    /// - Parameter start: A valid index into `buffer`, addressing the first
+    ///    code unit of a UTF-8 scalar in the buffer, or the end.
+    ///
+    /// - Returns: The index range of the scalar that follows the first grapheme
+    ///    break in the buffer, if there is one. If the buffer contains no
+    ///    grapheme breaks, then this function returns `nil`.
+    ///
+    /// - Warning: This function does not validate that the buffer contains
+    ///    valid UTF-8 data; its behavior is undefined if given invalid input.
+    @_effects(releasenone)
+    public mutating func _firstBreak(
+      inUncheckedUnsafeUTF8Buffer buffer: UnsafeBufferPointer<UInt8>,
+      startingAt start: Int = 0
+    ) -> Range<Int>? {
+      var i = start
+      while i < buffer.endIndex {
+        let (next, n) = _decodeScalar(buffer, startingAt: i)
+        if hasBreak(before: next) {
+          return Range(_uncheckedBounds: (i, i &+ n))
+        }
+        i &+= n
+      }
+      return nil
+    }
   }
 }
 

test/stdlib/CharacterRecognizer.swift

@@ -33,9 +33,50 @@ if #available(SwiftStdlib 5.8, *) {
   }
 }
 
+func scalars(in buffer: some Sequence<UInt8>) -> [Unicode.Scalar] {
+  var result: [Unicode.Scalar] = []
+  var it = buffer.makeIterator()
+  var utf8Decoder = UTF8()
+  while true {
+    switch utf8Decoder.decode(&it) {
+    case .scalarValue(let v): result.append(v)
+    case .emptyInput: return result
+    case .error: expectTrue(false, "Invalid scalar")
+    }
+  }
+}
+
 if #available(SwiftStdlib 5.8, *) {
-  suite.test("Consistency with Swift String's behavior") {
-    let sampleString = #"""
+  suite.test("Unicode test data/_firstBreak") {
+    for test in graphemeBreakTests {
+      var recognizer = Unicode._CharacterRecognizer()
+      var pieces: [[Unicode.Scalar]] = []
+      var str = test.string
+      str.withUTF8 { buffer in
+        var i = buffer.startIndex
+        var last = i
+        while i < buffer.endIndex {
+          guard let scalar = recognizer._firstBreak(
+            inUncheckedUnsafeUTF8Buffer: buffer, startingAt: i)
+          else { break }
+
+          if scalar.lowerBound > last {
+            pieces.append(scalars(in: buffer[last..<scalar.lowerBound]))
+          }
+
+          last = scalar.lowerBound
+          i = scalar.upperBound
+        }
+
+        pieces.append(scalars(in: buffer[last...]))
+      }
+      expectEqual(pieces, test.pieces,
+        "string: \(String(reflecting: test.string))")
+    }
+  }
+}
+
+let sampleString = #"""
     The powerful programming language that is also easy to learn.
     손쉽게 학습할 수 있는 강력한 프로그래밍 언어.
     🪙 A 🥞 short 🍰 piece 🫘 of 🌰 text 👨‍👨‍👧‍👧 with 👨‍👩‍👦 some 🚶🏽 emoji 🇺🇸🇨🇦 characters 🧈
@@ -52,6 +93,9 @@ if #available(SwiftStdlib 5.8, *) {
     e̶̢͕̦̜͔̘̘̝͈̪̖̺̥̺̹͉͎͈̫̯̯̻͑͑̿̽͂̀̽͋́̎̈́̈̿͆̿̒̈́̽̔̇͐͛̀̓͆̏̾̀̌̈́̆̽̕ͅ
     """#
 
+if #available(SwiftStdlib 5.8, *) {
+  suite.test("Consistency with Swift String's behavior/hasBreak") {
+
     let expectedBreaks = Array(sampleString.indices)
 
     let u = sampleString.unicodeScalars