feat: epmdless clustering (#205)

* feat: epmdless clustering

* docs: update epmd/distribution docs

* fix: check for epmd on expert startup

Author: doorgan

apps/engine/test/test_helper.exs

@@ -1,11 +1,11 @@
 Application.ensure_all_started(:snowflake)
 Application.ensure_all_started(:refactorex)
-{"", 0} = System.cmd("epmd", ~w(-daemon))
+
 random_number = :rand.uniform(500)
 
 with :nonode@nohost <- Node.self() do
   {:ok, _pid} =
-    :net_kernel.start(:"testing-#{random_number}@127.0.0.1", %{name_domain: :longnames})
+    Node.start(:"expert-manager-testing-#{random_number}@127.0.0.1", :longnames)
 end
 
 Engine.Module.Loader.start_link(nil)

apps/expert/lib/expert/application.ex

@@ -64,7 +64,10 @@ defmodule Expert.Application do
           System.halt(1)
       end
 
+    ensure_epmd_module!()
+
     children = [
+      {Forge.NodePortMapper, []},
       document_store_child_spec(),
       {DynamicSupervisor, Expert.Project.DynamicSupervisor.options()},
       {DynamicSupervisor, name: Expert.DynamicSupervisor},
@@ -89,4 +92,23 @@ defmodule Expert.Application do
   def document_store_child_spec do
     {Document.Store, derive: [analysis: &Forge.Ast.analyze/1]}
   end
+
+  def ensure_epmd_module! do
+    epmd_module = to_charlist(Forge.EPMD)
+
+    case :init.get_argument(:epmd_module) do
+      {:ok, [[^epmd_module]]} ->
+        :ok
+
+      _ ->
+        Application.put_env(:kernel, :epmd_module, Forge.EPMD, persistent: true)
+
+        # Note: this is a private API
+        if :net_kernel.epmd_module() != Forge.EPMD do
+          raise("""
+          you must set the environment variable ELIXIR_ERL_OPTIONS="-epmd_module #{Forge.EPMD}"
+          """)
+        end
+    end
+  end
 end

apps/expert/lib/expert/engine_node.ex

@@ -28,20 +28,39 @@ defmodule Expert.EngineNode do
     @dialyzer {:nowarn_function, start: 3}
 
     def start(%__MODULE__{} = state, paths, from) do
-      this_node = inspect(Node.self())
-
-      args = [
-        "--name",
-        Project.node_name(state.project),
-        "--cookie",
-        state.cookie,
-        "--no-halt",
-        "-e",
-        "Node.connect(#{this_node})"
-        | path_append_arguments(paths)
-      ]
-
-      case Expert.Port.open_elixir(state.project, args: args) do
+      this_node = to_string(Node.self())
+      dist_port = Forge.EPMD.dist_port()
+
+      args =
+        [
+          "--erl",
+          "-start_epmd false -epmd_module #{Forge.EPMD}",
+          "--cookie",
+          state.cookie,
+          "--no-halt",
+          "-e",
+          # We manually start distribution here instead of using --sname/--name
+          # because those options are not really compatible with `-epmd_module`.
+          # Apparently, passing the --name/-sname options causes the Erlang VM
+          # to start distribution right away before the modules in the code path
+          # are loaded, and it will crash because Forge.EPMD doesn't exist yet.
+          # If we start distribution manually after all the code is loaded,
+          # everything works fine.
+          """
+          {:ok, _} = Node.start(:"#{Project.node_name(state.project)}", :longnames)
+          #{Forge.NodePortMapper}.register()
+          IO.puts(\"ok\")
+          """
+          | path_append_arguments(paths)
+        ]
+
+      env =
+        [
+          {"EXPERT_PARENT_NODE", this_node},
+          {"EXPERT_PARENT_PORT", to_string(dist_port)}
+        ]
+
+      case Expert.Port.open_elixir(state.project, args: args, env: env) do
         {:error, :no_elixir, message} ->
           GenLSP.error(Expert.get_lsp(), message)
           Expert.terminate("Failed to find an elixir executable, shutting down", 1)
@@ -64,7 +83,7 @@ defmodule Expert.EngineNode do
     end
 
     def on_nodeup(%__MODULE__{} = state, node_name) do
-      if node_name == Project.node_name(state.project) do
+      if String.starts_with?(to_string(node_name), to_string(Project.node_name(state.project))) do
         {pid, _ref} = state.started_by
         Process.monitor(pid)
         GenServer.reply(state.started_by, :ok)
@@ -117,7 +136,6 @@ defmodule Expert.EngineNode do
   use GenServer
 
   def start(project) do
-    :ok = ensure_epmd_started()
     start_net_kernel(project)
 
     node_name = Project.node_name(project)
@@ -134,23 +152,13 @@ defmodule Expert.EngineNode do
 
   defp start_net_kernel(%Project{} = project) do
     manager = Project.manager_node_name(project)
-    :net_kernel.start(manager, %{name_domain: :longnames})
+    Node.start(manager, :longnames)
   end
 
   defp ensure_apps_started(node) do
     :rpc.call(node, Engine, :ensure_apps_started, [])
   end
 
-  defp ensure_epmd_started do
-    case System.cmd("epmd", ~w(-daemon)) do
-      {"", 0} ->
-        :ok
-
-      _ ->
-        {:error, :epmd_failed}
-    end
-  end
-
   if Mix.env() == :test do
     # In test environment, Expert depends on the Engine app, so we look for it
     # in the expert build path.
@@ -294,7 +302,7 @@ defmodule Expert.EngineNode do
 
   @impl true
   def handle_call({:start, paths}, from, %State{} = state) do
-    :ok = :net_kernel.monitor_nodes(true, node_type: :visible)
+    :ok = :net_kernel.monitor_nodes(true, node_type: :all)
     Process.send_after(self(), :maybe_start_timeout, @start_timeout)
 
     case State.start(state, paths, from) do
@@ -365,7 +373,8 @@ defmodule Expert.EngineNode do
   end
 
   @impl true
-  def handle_info({_port, {:data, _message}}, %State{} = state) do
+  def handle_info({_port, {:data, message}}, %State{} = state) do
+    Logger.debug("Node port message: #{to_string(message)}")
     {:noreply, state}
   end
 

apps/expert/lib/expert/port.ex

@@ -106,7 +106,7 @@ defmodule Expert.Port do
         opts
       end
 
-    Port.open({:spawn_executable, launcher}, opts)
+    Port.open({:spawn_executable, launcher}, [:stderr_to_stdout | opts])
   end
 
   @doc """

apps/expert/rel/remote.vm.args.eex

@@ -0,0 +1,13 @@
+## Customize flags given to the VM: https://www.erlang.org/doc/man/erl.html
+## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here
+
+## Increase number of concurrent ports/sockets
+##+Q 65536
+
+## Tweak GC to run more often
+##-env ERL_FULLSWEEP_AFTER 10
+
+## Enable deployment without epmd
+## (requires changing both vm.args and remote.vm.args)
+-epmd_module Elixir.XPForge.EPMD
+-start_epmd false

apps/expert/rel/vm.args.eex

@@ -0,0 +1,12 @@
+## Customize flags given to the VM: https://www.erlang.org/doc/man/erl.html
+## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here
+
+## Increase number of concurrent ports/sockets
+##+Q 65536
+
+## Tweak GC to run more often
+##-env ERL_FULLSWEEP_AFTER 10
+
+## Enable deployment without epmd
+## (requires changing both vm.args and remote.vm.args)
+-start_epmd false -epmd_module Elixir.XPForge.EPMD

apps/expert/test/engine/build_test.exs

@@ -42,6 +42,7 @@ defmodule Engine.BuildTest do
     |> Project.workspace_path()
     |> File.rm_rf()
 
+    {:ok, _} = start_supervised(Forge.NodePortMapper)
     {:ok, _} = start_supervised({EngineSupervisor, project})
     {:ok, _, _} = EngineNode.start(project)
     EngineApi.register_listener(project, self(), [:all])
@@ -650,6 +651,7 @@ defmodule Engine.BuildTest do
 
   describe ".exs files" do
     setup do
+      start_supervised!({Forge.NodePortMapper, []})
       start_supervised!(Engine.Dispatch)
       start_supervised!(Engine.ModuleMappings)
       start_supervised!(Build.CaptureServer)

apps/expert/test/engine/code_intelligence/definition_test.exs

@@ -43,6 +43,7 @@ defmodule Expert.Engine.CodeIntelligence.DefinitionTest do
   end
 
   setup_all do
+    {:ok, _} = start_supervised({Forge.NodePortMapper, []})
     project = project(:navigations)
     start_supervised!({Document.Store, derive: [analysis: &Forge.Ast.analyze/1]})
     {:ok, _} = start_supervised({EngineSupervisor, project})

apps/expert/test/engine/engine_test.exs

@@ -9,6 +9,7 @@ defmodule EngineTest do
   import Forge.Test.Fixtures
 
   def start_project(%Project{} = project) do
+    start_supervised!({Forge.NodePortMapper, []})
     start_supervised!({Expert.EngineSupervisor, project})
     assert {:ok, _, _} = EngineNode.start(project)
     :ok

apps/expert/test/expert/engine_node_test.exs

@@ -10,6 +10,7 @@ defmodule Expert.EngineNodeTest do
 
   setup do
     project = project()
+    start_supervised!({Forge.NodePortMapper, []})
     start_supervised!({EngineSupervisor, project})
     {:ok, %{project: project}}
   end