Fix error when :trim_bom is used with :encoding

Signed-off-by: José Valim <jose.valim@plataformatec.com.br>

Author: lukaszsamson

lib/elixir/lib/file.ex

@@ -1512,7 +1512,8 @@ defmodule File do
   in raw mode for performance reasons. Therefore, Elixir **will** open
   streams in `:raw` mode with the `:read_ahead` option unless an encoding
   is specified. This means any data streamed into the file must be
-  converted to `t:iodata/0` type. If you pass `[:utf8]` in the modes parameter,
+  converted to `t:iodata/0` type. If you pass e.g. `[encoding: :utf8]`
+  or `[encoding: {:utf16, :little}]` in the modes parameter,
   the underlying stream will use `IO.write/2` and the `String.Chars` protocol
   to convert the data. See `IO.binwrite/2` and `IO.write/2` .
 
@@ -1524,6 +1525,9 @@ defmodule File do
   If you pass `:trim_bom` in the modes parameter, the stream will
   trim UTF-8, UTF-16 and UTF-32 byte order marks when reading from file.
 
+  Note that this function does not try to discover the file encoding basing
+  on BOM.
+
   ## Examples
 
       # Read in 2048 byte chunks rather than lines

lib/elixir/lib/file/stream.ex

@@ -140,12 +140,24 @@ defmodule File.Stream do
       end
     end
 
-    defp trim_bom(device, raw) do
-      header = if raw, do: IO.binread(device, 4), else: IO.read(device, 1)
-      {:ok, new_pos} = :file.position(device, bom_length(header))
+    defp trim_bom(device, true) do
+      bom_length = device |> IO.binread(4) |> bom_length()
+      {:ok, new_pos} = :file.position(device, bom_length)
       {device, new_pos}
     end
 
+    defp trim_bom(device, false) do
+      # Or we read the bom in the correct amount or it isn't there
+      case bom_length(IO.read(device, 1)) do
+        0 ->
+          {:ok, _} = :file.position(device, 0)
+          {device, 0}
+
+        _ ->
+          {device, 1}
+      end
+    end
+
     defp bom_length(<<239, 187, 191, _rest::binary>>), do: 3
     defp bom_length(<<254, 255, _rest::binary>>), do: 2
     defp bom_length(<<255, 254, _rest::binary>>), do: 2

lib/elixir/test/elixir/file_test.exs

@@ -1569,23 +1569,63 @@ defmodule FileTest do
       src = fixture_path("utf8_bom.txt")
 
       assert src
-             |> File.stream!([:utf8])
+             |> File.stream!([{:encoding, :utf8}])
              |> Enum.take(1) == [<<239, 187, 191>> <> "Русский\n"]
 
       assert src
-             |> File.stream!([:utf8], 1)
+             |> File.stream!([{:encoding, :utf8}], 1)
              |> Enum.take(9) == ["\uFEFF", "Р", "у", "с", "с", "к", "и", "й", "\n"]
     end
 
     test "trims BOM via option with utf8 encoding" do
       src = fixture_path("utf8_bom.txt")
 
       assert src
-             |> File.stream!([:utf8, :trim_bom])
+             |> File.stream!([{:encoding, :utf8}, :trim_bom])
              |> Enum.take(1) == ["Русский\n"]
 
       assert src
-             |> File.stream!([:utf8, :trim_bom], 1)
+             |> File.stream!([{:encoding, :utf8}, :trim_bom], 1)
+             |> Enum.take(8) == ["Р", "у", "с", "с", "к", "и", "й", "\n"]
+    end
+
+    test "keeps BOM with UTF16 BE" do
+      src = fixture_path("utf16_be_bom.txt")
+
+      assert src
+             |> File.stream!([{:encoding, {:utf16, :big}}])
+             |> Enum.take(1) == ["\uFEFFРусский\n"]
+    end
+
+    test "keeps BOM with UTF16 LE" do
+      src = fixture_path("utf16_le_bom.txt")
+
+      assert src
+             |> File.stream!([{:encoding, {:utf16, :little}}])
+             |> Enum.take(1) == ["\uFEFFРусский\n"]
+    end
+
+    test "trims BOM via option with utf16 BE encoding" do
+      src = fixture_path("utf16_be_bom.txt")
+
+      assert src
+             |> File.stream!([{:encoding, {:utf16, :big}}, :trim_bom])
+             |> Enum.take(1) == ["Русский\n"]
+
+      assert src
+             |> File.stream!([{:encoding, {:utf16, :big}}, :trim_bom], 1)
+             |> Enum.take(8) == ["Р", "у", "с", "с", "к", "и", "й", "\n"]
+    end
+
+    test "trims BOM via option with utf16 LE encoding" do
+      src = fixture_path("utf16_le_bom.txt")
+
+      assert src
+             |> File.stream!([{:encoding, {:utf16, :little}}, :trim_bom])
+             |> Enum.take(1) == ["Русский\n"]
+
+      assert src
+             |> File.stream!([{:encoding, {:utf16, :little}}, :trim_bom], 1)
              |> Enum.take(8) == ["Р", "у", "с", "с", "к", "и", "й", "\n"]
     end