Add floki instead of meeseeks, add mint

Author: RoyalIcing

apps/components_guide/lib/components_guide/research/spec.ex

@@ -1,32 +1,90 @@
 defmodule ComponentsGuide.Research.Spec do
   alias ComponentsGuide.HTTPClient
-  import Meeseeks.CSS
-  import Meeseeks.XPath
 
   @cache_name :research_spec_cache
 
   defp read_cache(key) do
-    value = Cachex.get(@cache_name, key)
+    tuple = {:ok, value} = Cachex.get(@cache_name, key)
     IO.puts("reading #{if value == nil, do: "nil", else: "present"}")
     IO.inspect(key)
-    value
+    tuple
+    # {:ok, nil}
   end
 
   defp write_cache(key, value) do
     Cachex.put(@cache_name, key, value)
   end
 
-  defp body({:fetch, url}) do
-    with {:ok, response} <- HTTPClient.get(url) do
-      html = response.body
-      {:ok, html}
+  defmodule Fetch do
+    defstruct done: false, request_ref: nil, body_list: [], status: nil, headers: []
+
+    defp apply_response({:status, _request_ref, status_code}, state = %Fetch{}) do
+      %{state | status: status_code}
+    end
+
+    defp apply_response({:headers, _request_ref, headers}, state = %Fetch{}) do
+      %{state | headers: headers}
+    end
+
+    defp apply_response({:data, _request_ref, chunk}, state = %Fetch{body_list: body_list}) do
+      %{state | body_list: [body_list | [chunk]]}
     end
+
+    defp apply_response({:done, _request_ref}, state = %Fetch{}) do
+      %{state | done: true}
+    end
+
+    defp receive_mint_response(state = %Fetch{}, conn, request_ref) do
+      receive do
+        message ->
+          case Mint.HTTP.stream(conn, message) do
+            :unknown ->
+              receive_mint_response(state, conn, request_ref)
+
+            {:error, _, e, _} ->
+              {:error, e}
+
+            {:ok, conn, responses} ->
+              state = Enum.reduce(responses, state, &apply_response/2)
+
+              if state.done do
+                {:ok, IO.iodata_to_binary(state.body_list)}
+              else
+                receive_mint_response(state, conn, request_ref)
+              end
+          end
+      end
+    end
+
+    def get(url) do
+      uri = URI.parse(url)
+
+      IO.puts("fetching URL #{uri} #{uri.host} #{uri.path}")
+      {:ok, conn} = Mint.HTTP.connect(:https, uri.host, 443)
+      {:ok, conn, request_ref} = Mint.HTTP.request(conn, "GET", uri.path, [], nil)
+
+      receive_mint_response(%Fetch{}, conn, request_ref)
+    end
+  end
+
+  defp body({:fetch, url}) do
+    IO.puts("fetching URL #{url}")
+    # with {:ok, response} <- HTTPClient.get(url) do
+    #   html = response.body
+    #   {:ok, html}
+    # end
+
+    Fetch.get(url)
+
+    # {:ok, response} = Mojito.request(method: :get, url: url, timeout: 50000)
+    # {:ok, response.body}
   end
 
   defp body({:html_document, url}) do
     {:ok, html} = read({:fetch, url})
-    document = Meeseeks.parse(html)
-    {:ok, document}
+    IO.puts("html length #{length(html)}")
+    tuple = {:ok, _document} = Floki.parse_document(html)
+    tuple
   end
 
   defp run(key) do
@@ -54,11 +112,24 @@ defmodule ComponentsGuide.Research.Spec do
 
   def search_for(:whatwg_html_spec, query) when is_binary(query) do
     IO.puts("searching whatwg html spec")
-    {:ok, document} = read({:html_document, "https://html.spec.whatwg.org/"})
+
+    url = "https://html.spec.whatwg.org/"
+    url = "https://html.spec.whatwg.org/dev/"
+
+    {:ok, html} = body({:fetch, url})
+    # {:ok, document} = read({:html_document, "https://html.spec.whatwg.org/"})
+
+    # IO.puts("document size #{:erts_debug.flat_size(document)}")
 
     # selector = css("#contents")
-    selector = xpath("//*[@id='contents']/following-sibling::ol[1]")
-    result = Meeseeks.one(document, selector)
-    Meeseeks.html(result)
+    # selector = xpath("//*[@id='contents']/following-sibling::ol[1]")
+    # result = Meeseeks.one(document, selector)
+    # Meeseeks.html(result)
+
+    # document
+    # |> Floki.find("#contents")
+    # |> Floki.raw_html()
+
+    html
   end
 end

apps/components_guide/mix.exs

@@ -41,7 +41,9 @@ defmodule ComponentsGuide.MixProject do
       {:httpotion, "~> 3.1.0"},
       {:jason, "~> 1.0"},
       {:tesla, "~> 1.3.0"},
-      {:meeseeks, "~> 0.15.0"},
+      {:mojito, "~> 0.6.1"},
+      {:mint, "~> 1.0"},
+      {:floki, "~> 0.26.0"},
     ]
   end
 

apps/components_guide_web/lib/components_guide_web.ex

@@ -42,7 +42,8 @@ defmodule ComponentsGuideWeb do
       import ComponentsGuideWeb.ErrorHelpers
       import ComponentsGuideWeb.Gettext
       alias ComponentsGuideWeb.Router.Helpers, as: Routes
-      import Phoenix.LiveView, only: [live_render: 2, live_render: 3]
+      import Phoenix.LiveView.Helpers
+      # import Phoenix.LiveView, only: [live_render: 2, live_render: 3]
       alias ComponentsGuideWeb.StylingHelpers, as: Styling
     end
   end

mix.lock

@@ -1,21 +1,26 @@
 %{
   "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm", "3ad58ae787e9c7c94dd7ceda3b587ec2c64604563e049b2a0e8baafae832addb"},
   "cachex": {:hex, :cachex, "3.1.3", "86ed0669ea4b2f3e3982dbb5c6ca9e0964e46738e572c9156f22ceb75f57c336", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "a0bd9cb3f85bd8e77dc44d5d11b009ba6b73e45da5bb6444b576efb541896f74"},
+  "castore": {:hex, :castore, "0.1.5", "591c763a637af2cc468a72f006878584bc6c306f8d111ef8ba1d4c10e0684010", [:mix], [], "hexpm", "6db356b2bc6cc22561e051ff545c20ad064af57647e436650aa24d7d06cd941a"},
   "cowboy": {:hex, :cowboy, "2.6.3", "99aa50e94e685557cad82e704457336a453d4abcb77839ad22dbe71f311fcc06", [:rebar3], [{:cowlib, "~> 2.7.3", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "e5580029080f3f1ad17436fb97b0d5ed2ed4e4815a96bac36b5a992e20f58db6"},
   "cowlib": {:hex, :cowlib, "2.7.3", "a7ffcd0917e6d50b4d5fb28e9e2085a0ceb3c97dea310505f7460ff5ed764ce9", [:rebar3], [], "hexpm", "1e1a3d176d52daebbecbbcdfd27c27726076567905c2a9d7398c54da9d225761"},
   "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm", "771ea78576e5fa505ad58a834f57915c7f5f9df11c87a598a01fdf6065ccfb5d"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
   "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "d3c38bf0887d44f442d3ebdcd71486d1eafcb3ffb7d3779c3428959c1360203d"},
   "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm", "b14f1dc204321429479c569cfbe8fb287541184ed040956c8862cb7a677b8406"},
   "file_system": {:hex, :file_system, "0.2.7", "e6f7f155970975789f26e77b8b8d8ab084c59844d8ecfaf58cbda31c494d14aa", [:mix], [], "hexpm", "b4cfa2d69c7f0b18fd06db222b2398abeef743a72504e6bd7df9c52f171b047f"},
+  "floki": {:hex, :floki, "0.26.0", "4df88977e2e357c6720e1b650f613444bfb48c5acfc6a0c646ab007d08ad13bf", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "e7b66ce7feef5518a9cd9fc7b52dd62a64028bd9cb6d6ad282a0f0fc90a4ae52"},
   "gettext": {:hex, :gettext, "0.16.1", "e2130b25eebcbe02bb343b119a07ae2c7e28bd4b146c4a154da2ffb2b3507af2", [:mix], [], "hexpm", "dd3a7ea5e3e87ee9df29452dd9560709b4c7cc8141537d0b070155038d92bdf1"},
+  "html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"},
   "httpotion": {:hex, :httpotion, "3.1.2", "50e3e559c2ffe8c8908c97e4ffb01efc1c18e8547cc7ce5dd173c9cf0a573a3b", [:mix], [{:ibrowse, "== 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: false]}], "hexpm", "45056b74ad83f89ebc3e746f7ee7b887ab2eb8cedbe5b80d8298d2681abe9c53"},
   "ibrowse": {:hex, :ibrowse, "4.4.0", "2d923325efe0d2cb09b9c6a047b2835a5eda69d8a47ed6ff8bc03628b764e991", [:rebar3], [], "hexpm", "6a8e5988872086f0506bef68311493551ac5beae7c06ba2a00d5e9f97a60f1c2"},
   "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fdf843bca858203ae1de16da2ee206f53416bbda5dc8c9e78f43243de4bc3afe"},
   "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"},
   "meeseeks": {:hex, :meeseeks, "0.15.0", "8ea95378a7cbaf8f17b7369c150bae6a8dc11c9793d9932deff0ce06c6a7e11a", [:mix], [{:meeseeks_html5ever, "~> 0.12.1", [hex: :meeseeks_html5ever, repo: "hexpm", optional: false]}], "hexpm", "3d3644ef0fd0eb4828f5d76fc8be413c1fa35ddbc22f2d424e99eea48a9b6013"},
   "meeseeks_html5ever": {:hex, :meeseeks_html5ever, "0.12.1", "718fab10d05b83204524a518b2b88caa37ba6a6e02f82e80d6a7bc47552fb54a", [:mix], [{:rustler, "~> 0.21.0", [hex: :rustler, repo: "hexpm", optional: false]}], "hexpm", "11489094637f49a26bad4610a9138352c8d229339d888169cb35b08cdfd8861a"},
   "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm", "6cbe761d6a0ca5a31a0931bf4c63204bceb64538e664a8ecf784a9a6f3b875f1"},
+  "mint": {:hex, :mint, "1.0.0", "ca5ab33497ba2bdcc42f6cdd3927420a6159116be87c8173658e93c8746703da", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "b8943ef1e630879538dd6620bfc189d4d75fab3ad39f3fe9c50539879f7efd84"},
+  "mojito": {:hex, :mojito, "0.6.3", "1d9af4bee09c5489d189263c4558c1d5b0cc2c2a6d94d2426590077a6b4287ef", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "99c40fe806bfc8bdecdd9b8ac111cc3834ae638afd3e233be554f3ed5811c1f9"},
   "phoenix": {:hex, :phoenix, "1.4.6", "8535f4a01291f0fbc2c30c78c4ca6a2eacc148db5178ad76e8b2fc976c590115", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "17aa6f4909e41eebfa7589b61c71f0ebe8fdea997194fd85596e629bbf8d3e15"},
   "phoenix_ecto": {:hex, :phoenix_ecto, "4.0.0", "c43117a136e7399ea04ecaac73f8f23ee0ffe3e07acfcb8062fe5f4c9f0f6531", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "fe15d9fee5b82f5e64800502011ffe530650d42e1710ae9b14bc4c9be38bf303"},
   "phoenix_html": {:hex, :phoenix_html, "2.13.2", "f5d27c9b10ce881a60177d2b5227314fc60881e6b66b41dfe3349db6ed06cf57", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "346764ebecf5e7d318ec64db92a2aa9ef17fcb2ae0b001791cd37a0223f724ba"},
@@ -25,6 +30,7 @@
   "plug": {:hex, :plug, "1.8.0", "9d2685cb007fe5e28ed9ac27af2815bc262b7817a00929ac10f56f169f43b977", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "4cc0099d5bffb938ee1ff78cc69f2a787d85f6c66b546f9ac2df7400b4ee1624"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.0.2", "6055f16868cc4882b24b6e1d63d2bada94fb4978413377a3b32ac16c18dffba2", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "44a58653dda2b8cf69757c7f14c0920d4df0f79305ed8571bb93b2ea0a31a895"},
   "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm", "73c1682f0e414cfb5d9b95c8e8cd6ffcfdae699e3b05e1db744e58b7be857759"},
+  "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"},
   "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"},
   "rustler": {:hex, :rustler, "0.21.0", "68cc4fc015d0b9541865ea78e78e9ef2dd91ee4be80bf543fd15791102a45aca", [:mix], [{:toml, "~> 0.5.2", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "e5429378c397f37f1091a35593b153aee1925e197c6842d04648d802edb52f80"},
   "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"},