Don't strip any \0 bytes

Fix tests to handle inconsistencies with \0 termination between UTF8String, UTF16String, and UTF32String in Julia

Author: ScottPJones

src/iconv.jl

@@ -260,15 +260,7 @@ Convert an array of bytes `a` representing text in encoding `enc` to a string.
 function decode(a::Vector{UInt8}, enc::ASCIIString)
     b = IOBuffer(a)
     try
-        d = readbytes(StringDecoder(b, enc, "UTF-8"))
-        # Skip final null bytes if needed
-        # FIXME: find a better solution?
-        i = length(d)
-        while i >= 1
-            d[i] != 0 && break
-            i -= 1
-        end
-        UTF8String(d[1:i])
+        UTF8String(readbytes(StringDecoder(b, enc, "UTF-8")))
     finally
         close(b)
     end

test/runtests.jl

@@ -1,13 +1,16 @@
 using Base.Test
 using iconv
 
-for s in ("", "a", "café crème",
+for s in ("", "\0", "a", "café crème",
           "a"^(iconv.BUFSIZE-1) * "€ with an incomplete codepoint between two input buffer fills",
-          "a string € チャネルパートナーの選択")
+          "a string € チャネルパートナーの選択",
+          "a string \0€ チャネルパ\0ー\0トナーの選択 with embedded and trailing nuls\0")
     # Test round-trip to Unicode formats, checking against pure-Julia implementation
-    for T in (UTF8String, UTF16String, UTF32String)
+    for (T, nullen) in ((UTF8String, 0), (UTF16String, 2), (UTF32String, 4))
         enc = iconv.encoding_string(T)
         a = reinterpret(UInt8, T(s).data)
+        # Adjust for explicit \0 only for .data on UTF16String/UTF32String
+        a = a[1:end - nullen]
         @test decode(a, enc) == s
         @test decode(encode(s, enc), enc) == s
     end