Richer parsing

Author: RoyalIcing

apps/components_guide_web/lib/components_guide_web/controllers/universal_modules_controller.ex

@@ -14,32 +14,53 @@ defmodule ComponentsGuideWeb.UniversalModulesController do
     const verbose = false;
     const alwaysNull = null;
 
+    const debug = verbose;
+
     export const dateFormat = "YYYY/MM/DD";
 
+    const exampleDotOrg = new URL("https://example.org");
+
     const array = [1, 2, 3];
     const arrayMultiline = [
       4,
       5,
       6
     ];
     export const flavors = ["vanilla", "chocolate", "caramel", "raspberry"];
+    export const flavorsSet = new Set(["vanilla", "chocolate", "caramel", "raspberry"]);
 
     const object = { "key": "value" };
 
     function hello() {}
-    function* gen() {}
+    function* gen() {
+      const a = 1;
+      yield [1, 2, 3];
+    }
     """
     # decoded = decode_module(source)
     decoded = Parser.decode(source)
+    identifiers = ComponentsGuideWeb.UniversalModulesInspector.list_identifiers(elem(decoded, 1))
 
     render(conn, "index.html",
       page_title: "Universal Modules",
       source: source,
-      decoded: inspect(decoded)
+      decoded: inspect(decoded),
+      identifiers: inspect(identifiers),
     )
   end
 end
 
+defmodule ComponentsGuideWeb.UniversalModulesInspector do
+  def is_identifier({:const, _, _}), do: true
+  def is_identifier({:function, _, _, _}), do: true
+  def is_identifier({:generator_function, _, _, _}), do: true
+  def is_identifier(_), do: false
+
+  def list_identifiers(module_body) do
+    for statement <- module_body, is_identifier(statement), do: statement
+  end
+end
+
 defmodule ComponentsGuideWeb.UniversalModulesParser do
   def compose(submodule, input) do
     mod = Module.concat(__MODULE__, submodule)
@@ -110,6 +131,8 @@ defmodule ComponentsGuideWeb.UniversalModulesParser do
   end
 
   defmodule Function do
+    defdelegate compose(submodule, input), to: ComponentsGuideWeb.UniversalModulesParser
+
     def decode(<<"function", rest::bitstring>>),
       do: decode(%{generator_mark: nil, name: nil, args: nil, body: nil}, rest)
 
@@ -136,10 +159,33 @@ defmodule ComponentsGuideWeb.UniversalModulesParser do
     defp decode(%{args: {:closed, args}, name: name, body: {:open, body_items}} = context, <<"}", rest::bitstring>>) do
       case context.generator_mark do
         true ->
-          {:ok, {:generator_function, name, args, body_items}, rest}
+          {:ok, {:generator_function, name, args, Enum.reverse(body_items)}, rest}
 
         nil ->
-          {:ok, {:function, name, args, body_items}, rest}
+          {:ok, {:function, name, args, Enum.reverse(body_items)}, rest}
+      end
+    end
+
+    defp decode(%{body: {:open, _}} = context, <<char::utf8, rest::bitstring>>) when char in [?\n, ?\t, ?;],
+      do: decode(context, rest)
+
+    defp decode(%{body: {:open, body_items}} = context, <<"const ", _::bitstring>> = input) do
+      case compose(Const, input) do
+        {:ok, term, rest} ->
+          decode(%{context | body: {:open, [term | body_items]}}, rest)
+
+        {:error, reason} ->
+          {:error, {reason, body_items}}
+      end
+    end
+
+    defp decode(%{body: {:open, body_items}} = context, <<"yield ", _::bitstring>> = input) do
+      case compose(Yield, input) do
+        {:ok, term, rest} ->
+          decode(%{context | body: {:open, [term | body_items]}}, rest)
+
+        {:error, reason} ->
+          {:error, {reason, body_items}}
       end
     end
 
@@ -168,6 +214,9 @@ defmodule ComponentsGuideWeb.UniversalModulesParser do
     def decode(<<"const ", rest::bitstring>>),
       do: decode({:expect_identifier, []}, rest)
 
+    def decode(<<_::bitstring>>),
+      do: {:error, :expected_const}
+
     defp decode({:expect_identifier, _} = context, <<" ", rest::bitstring>>),
       do: decode(context, rest)
 
@@ -199,16 +248,47 @@ defmodule ComponentsGuideWeb.UniversalModulesParser do
   end
 
   defmodule Expression do
+    import Unicode.Guards
+
     def decode(input), do: decode([], input)
 
+    defp finalize([{:found_identifier, reverse_identifier} | context_rest]) do
+      identifier = reverse_identifier |> Enum.reverse() |> :binary.list_to_bin()
+      [{:ref, identifier} | context_rest]
+    end
+
+    defp finalize(expression), do: expression
+
     defp decode(expression, <<";", rest::bitstring>>),
-      do: {:ok, expression, rest}
+      do: {:ok, finalize(expression), rest}
 
     defp decode([] = context, <<" ", rest::bitstring>>), do: decode(context, rest)
     defp decode([], <<"true", rest::bitstring>>), do: decode(true, rest)
     defp decode([], <<"false", rest::bitstring>>), do: decode(false, rest)
     defp decode([], <<"null", rest::bitstring>>), do: decode(nil, rest)
 
+    defp decode([], <<"new URL(", rest::bitstring>>) do
+      [encoded_json, rest] = String.split(rest, ");\n", parts: 2)
+      case Jason.decode(encoded_json) do
+        {:ok, value} ->
+          {:ok, {:url, value}, rest}
+
+        {:error, error} ->
+          {:error, error}
+      end
+    end
+
+    defp decode([], <<"new Set(", rest::bitstring>>) do
+      [encoded_json, rest] = String.split(rest, ");\n", parts: 2)
+      case Jason.decode(encoded_json) do
+        {:ok, value} ->
+          {:ok, {:set, value}, rest}
+
+        {:error, error} ->
+          {:error, error}
+      end
+    end
+
     # TODO: parse JSON by finding the end character followed by a semicolon + newline.
     # JSON strings cannoc contain literal newlines (it’s considered to be a control character),
     # so instead it must be encoded as "\n". So we can use this fast to know an actual newline is
@@ -245,6 +325,14 @@ defmodule ComponentsGuideWeb.UniversalModulesParser do
           decode(f, rest)
       end
     end
+
+    defp decode([], <<char::utf8, rest::bitstring>>) when is_lower(char) or is_upper(char),
+      do: decode([{:found_identifier, [char]}], rest)
+
+    defp decode([{:found_identifier, reverse_identifier} | context_rest], <<char::utf8, rest::bitstring>>)
+         when is_lower(char) or is_upper(char) or is_digit(char) do
+      decode([{:found_identifier, [char | reverse_identifier]} | context_rest], rest)
+    end
   end
 
   defmodule KnownIdentifier do

apps/components_guide_web/lib/components_guide_web/templates/universal_modules/index.html.eex

@@ -5,3 +5,9 @@
 <pre class="whitespace-pre-line">
   <%= @decoded %>
 </pre>
+
+<h2>Identifiers</h2>
+
+<pre class="whitespace-pre-line">
+  <%= @identifiers %>
+</pre>

apps/components_guide_web/mix.exs

@@ -47,6 +47,7 @@ defmodule ComponentsGuideWeb.MixProject do
       {:telemetry_metrics, "~> 0.4"},
       {:telemetry_poller, "~> 0.4"},
       {:gettext, "~> 0.11"},
+      {:unicode_guards, "~> 1.0"},
       {:components_guide, in_umbrella: true},
       {:jason, "~> 1.0"},
       {:plug_cowboy, "~> 2.0"},

mix.lock

@@ -54,6 +54,9 @@
   "telemetry_metrics": {:hex, :telemetry_metrics, "0.5.0", "1b796e74add83abf844e808564275dfb342bcc930b04c7577ab780e262b0d998", [:mix], [{:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31225e6ce7a37a421a0a96ec55244386aec1c190b22578bd245188a4a33298fd"},
   "telemetry_poller": {:hex, :telemetry_poller, "0.5.1", "21071cc2e536810bac5628b935521ff3e28f0303e770951158c73eaaa01e962a", [:rebar3], [{:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4cab72069210bc6e7a080cec9afffad1b33370149ed5d379b81c7c5f0c663fd4"},
   "tesla": {:hex, :tesla, "1.3.3", "26ae98627af5c406584aa6755ab5fc96315d70d69a24dd7f8369cfcb75094a45", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "2648f1c276102f9250299e0b7b57f3071c67827349d9173f34c281756a1b124c"},
+  "unicode": {:hex, :unicode, "1.13.1", "4ebb6dc60e91f04e0dd46dca93576416473bc22bae69484122c11682af48e8df", [:mix], [], "hexpm", "fe4fcc8e15f444cf07bfbca30ad726be99c12d45da06d70b0e4f0c49b0d7d5ce"},
+  "unicode_guards": {:hex, :unicode_guards, "1.0.0", "e0a71517486de1ac2a3bdc516da48d6f149f61873732d7a3d882e8951c7649f2", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:unicode_set, "~> 1.0", [hex: :unicode_set, repo: "hexpm", optional: false]}], "hexpm", "d8c91bf78ff9bd6a9053d4ad43f0ec4ff3ed15c36917fe36bb19dbf36650096c"},
+  "unicode_set": {:hex, :unicode_set, "1.1.0", "32971b9b8061f2b2f0c607ba588ad9a6202e78a0c577555df4da899fd4434f23", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:unicode, "~> 1.13", [hex: :unicode, repo: "hexpm", optional: false]}], "hexpm", "b416e7b18d1297bbdc25c05e56506291e8fd3f339b8386d4e1f6ddb57c047918"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
   "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"},
 }