Improve iterate for UTF8Str

Author: ScottPJones

src/support.jl

@@ -948,15 +948,15 @@ function repeat(ch::C, cnt::Integer) where {C<:Union{UCS2Chr,UTF32Chr}}
     cnt < 0 && repeaterr(cnt)
     if ch%UInt32 <= 0xff
         buf, pnt = _allocate(UInt8, cnt)
-        cnt == 1 && set_codepoint!(pnt, ch%UInt8) : _memset(pnt, ch%UInt8, cnt)
+        cnt == 1 && set_codeunit!(pnt, ch%UInt8) : _memset(pnt, ch%UInt8, cnt)
         Str(ifelse(ch%UInt8 <= 0x7f, ASCIICSE, LatinCSE), buf)
     elseif C == UCS2Chr || ch%UInt32 <= 0xffff
         buf, pnt = _allocate(UInt16, cnt)
-        cnt == 1 && set_codepoint!(pnt, ch%UInt16) : _aligned_set(pnt, ch%UInt16, cnt)
+        cnt == 1 && set_codeunit!(pnt, ch%UInt16) : _aligned_set(pnt, ch%UInt16, cnt)
         Str(UCS2CSE, buf)
     else
         buf, pnt = _allocate(UInt32, cnt)
-        cnt == 1 && set_codepoint!(pnt, ch%UInt32) : _aligned_set(pnt, ch%UInt32, cnt)
+        cnt == 1 && set_codeunit!(pnt, ch%UInt32) : _aligned_set(pnt, ch%UInt32, cnt)
         Str(UTF32CSE, buf)
     end
 end

src/utf8.jl

@@ -351,23 +351,23 @@ end
 end
 =#
 
+function _iterate_utf8(ch, str, pnt, pos)
+    if ch < 0xe0
+        ch < 0xc0 ? index_error(str, pos) : UTF32Chr(get_utf8_2byte(pnt + 1, ch)), pos + 2
+    elseif ch < 0xf0
+        UTF32Chr(get_utf8_3byte(pnt + 2, ch)), pos + 3
+    else
+        UTF32Chr(get_utf8_4byte(pnt + 3, ch)), pos + 4
+    end
+end
+
 @propagate_inbounds function iterate(str::MS_UTF8, pos::Integer=1)
     pos > ncodeunits(str) && return nothing
     @boundscheck pos <= 0 && boundserr(str, pos)
     @preserve str begin
         pnt = pointer(str) + pos - 1
         ch = get_codeunit(pnt)
-        if ch < 0x80
-            UTF32Chr(ch), pos + 1
-        elseif ch < 0xc0
-            index_error(str, pos)
-        elseif ch < 0xe0
-            UTF32Chr(get_utf8_2byte(pnt + 1, ch)), pos + 2
-        elseif ch < 0xf0
-            UTF32Chr(get_utf8_3byte(pnt + 2, ch)), pos + 3
-        else
-            UTF32Chr(get_utf8_4byte(pnt + 3, ch)), pos + 4
-        end
+        ch <= 0x7f ? (UTF32Chr(ch), pos + 1) : _iterate_utf8(ch, str, pnt, pos)
     end
 end