Add :indentation to Code.string_to_quoted/2 (#14406)

Author: josevalim

lib/eex/lib/eex/compiler.ex

@@ -304,7 +304,7 @@ defmodule EEx.Compiler do
       source: source,
       line: line,
       quoted: [],
-      parser_options: parser_options,
+      parser_options: [indentation: indentation] ++ parser_options,
       indentation: indentation
     }
 

lib/eex/test/eex_test.exs

@@ -502,6 +502,59 @@ defmodule EExTest do
       end
     end
 
+    test "from Elixir parser" do
+      line = __ENV__.line + 6
+
+      message =
+        assert_raise TokenMissingError, fn ->
+          EEx.compile_string(
+            """
+            <li>
+              <strong>Some:</strong>
+              <%= true && @some[ %>
+            </li>
+            """,
+            file: __ENV__.file,
+            line: line,
+            indentation: 12
+          )
+        end
+
+      assert message |> Exception.message() |> strip_ansi() =~ """
+                  │
+              514 │                   true && @some[\s
+                  │                                │ └ missing closing delimiter (expected "]")
+                  │                                └ unclosed delimiter
+             """
+    end
+
+    test "from Elixir parser with line breaks" do
+      line = __ENV__.line + 6
+
+      message =
+        assert_raise TokenMissingError, fn ->
+          EEx.compile_string(
+            """
+            <li>
+              <strong>Some:</strong>
+              <%= true &&
+                @some[ %>
+            </li>
+            """,
+            file: __ENV__.file,
+            line: line,
+            indentation: 12
+          )
+        end
+
+      assert message |> Exception.message() |> strip_ansi() =~ """
+                  │
+              #{line + 3} │                 @some[\s
+                  │                      │ └ missing closing delimiter (expected "]")
+                  │                      └ unclosed delimiter
+             """
+    end
+
     test "honor line numbers" do
       assert_raise EEx.SyntaxError,
                    "nofile:100:6: expected closing '%>' for EEx expression",
@@ -948,6 +1001,12 @@ defmodule EExTest do
     end
   end
 
+  @strip_ansi [IO.ANSI.green(), IO.ANSI.red(), IO.ANSI.reset()]
+
+  defp strip_ansi(doc) do
+    String.replace(doc, @strip_ansi, "")
+  end
+
   defp assert_eval(expected, actual, binding \\ [], opts \\ []) do
     opts = Keyword.merge([file: __ENV__.file, engine: opts[:engine] || EEx.Engine], opts)
     result = EEx.eval_string(actual, binding, opts)

lib/elixir/lib/code.ex

@@ -1068,7 +1068,8 @@ defmodule Code do
   Macro arguments are typically transformed by unquoting them into the
   returned quoted expressions (instead of evaluated).
 
-  See `eval_string/3` for a description of `binding` and `opts`.
+  See `eval_string/3` for a description of arguments and return types.
+  The options are described under `env_for_eval/1`.
 
   ## Examples
 
@@ -1168,6 +1169,10 @@ defmodule Code do
     * `:column` - (since v1.11.0) the starting column of the string being parsed.
       Defaults to 1.
 
+    * `:indentation` - (since v1.19.0) the indentation for the string being parsed.
+      This is useful when the code parsed is embedded within another document.
+      Defaults to 0.
+
     * `:columns` - when `true`, attach a `:column` key to the quoted
       metadata. Defaults to `false`.
 
@@ -1360,13 +1365,11 @@ defmodule Code do
         {forms, comments}
 
       {:error, {location, error, token}} ->
-        :elixir_errors.parse_error(
-          location,
-          Keyword.get(opts, :file, "nofile"),
-          error,
-          token,
-          {charlist, Keyword.get(opts, :line, 1), Keyword.get(opts, :column, 1)}
-        )
+        file = Keyword.get(opts, :file, "nofile")
+        line = Keyword.get(opts, :line, 1)
+        column = Keyword.get(opts, :column, 1)
+        input = {charlist, line, column, Keyword.get(opts, :indentation, 0)}
+        :elixir_errors.parse_error(location, file, error, token, input)
     end
   end
 

lib/elixir/src/elixir.erl

@@ -490,10 +490,14 @@ parser_location(Meta) ->
         {ok, Forms} ->
           Forms;
         {error, {Meta, Error, Token}} ->
-          elixir_errors:parse_error(Meta, File, Error, Token, {String, StartLine, StartColumn})
+          Indentation = proplists:get_value(indentation, Opts, 0),
+          Input = {String, StartLine, StartColumn, Indentation},
+          elixir_errors:parse_error(Meta, File, Error, Token, Input)
       end;
     {error, {Meta, Error, Token}} ->
-      elixir_errors:parse_error(Meta, File, Error, Token, {String, StartLine, StartColumn})
+      Indentation = proplists:get_value(indentation, Opts, 0),
+      Input = {String, StartLine, StartColumn, Indentation},
+      elixir_errors:parse_error(Meta, File, Error, Token, Input)
   end.
 
 to_binary(List) when is_list(List) -> elixir_utils:characters_to_binary(List);

lib/elixir/src/elixir_errors.erl

@@ -446,11 +446,12 @@ cut_snippet(Location, Input) ->
       nil
   end.
 
-cut_snippet({InputString, StartLine, StartColumn}, Line, Span) ->
+cut_snippet({InputString, StartLine, StartColumn, Indentation}, Line, Span) ->
   %% In case the code is indented, we need to add the indentation back
   %% for the snippets to match the reported columns.
-  Indent = binary:copy(<<" ">>, StartColumn - 1),
-  Lines = string:split(InputString, "\n", all),
+  Prelude = lists:duplicate(max(StartColumn - Indentation - 1, 0), " "),
+  Lines = string:split(Prelude ++ InputString, "\n", all),
+  Indent = binary:copy(<<" ">>, Indentation),
   [Head | Tail] = lists:nthtail(Line - StartLine, Lines),
   IndentedTail = indent_n(Tail, Span - 1, <<"\n", Indent/binary>>),
   elixir_utils:characters_to_binary([Indent, Head, IndentedTail]).

lib/elixir/src/elixir_tokenizer.erl

@@ -130,9 +130,11 @@ tokenize(String, Line, Column, Opts) ->
         Acc#elixir_tokenizer{preserve_comments=PreserveComments};
       ({unescape, Unescape}, Acc) when is_boolean(Unescape) ->
         Acc#elixir_tokenizer{unescape=Unescape};
+      ({indentation, Indentation}, Acc) when Indentation >= 0 ->
+        Acc#elixir_tokenizer{column=Indentation+1};
       (_, Acc) ->
         Acc
-    end, #elixir_tokenizer{identifier_tokenizer=IdentifierTokenizer, column=Column}, Opts),
+    end, #elixir_tokenizer{identifier_tokenizer=IdentifierTokenizer}, Opts),
 
   tokenize(String, Line, Column, Scope, []).
 

lib/elixir/test/elixir/code_test.exs

@@ -392,7 +392,7 @@ defmodule CodeTest do
     Code.unrequire_files([fixture_path("code_sample.exs")])
   end
 
-  test "string_to_quoted!/2 errors take lines and columns into account" do
+  test "string_to_quoted!/2 errors take lines/columns/indentation into account" do
     assert_exception(
       SyntaxError,
       ["nofile:1:5:", "syntax error before:", "1 + * 3", "^"],
@@ -419,9 +419,9 @@ defmodule CodeTest do
 
     assert_exception(
       SyntaxError,
-      ["nofile:11:7:", "syntax error before:", "1 + * 3", "^"],
+      ["nofile:11:15:", "syntax error before:", "1 + * 3", "^"],
       fn ->
-        Code.string_to_quoted!(":ok\n1 + * 3", line: 10, column: 3)
+        Code.string_to_quoted!(":ok\n1 + * 3", line: 10, column: 3, indentation: 10)
       end
     )
   end

lib/iex/lib/iex/evaluator.ex

@@ -82,7 +82,7 @@ defmodule IEx.Evaluator do
       opts[:file],
       "incomplete expression",
       "",
-      {~c"", Keyword.get(opts, :line, 1), Keyword.get(opts, :column, 1)}
+      {~c"", Keyword.get(opts, :line, 1), Keyword.get(opts, :column, 1), 0}
     )
   end
 
@@ -119,7 +119,7 @@ defmodule IEx.Evaluator do
           file,
           error,
           token,
-          {charlist, line, column}
+          {charlist, line, column, 0}
         )
     end
   end