feat: epmdless clustering

doorgan · doorgan · commit bb5c93b3d003 · 2025-11-11T16:13:26.000-03:00
diff --git a/apps/engine/test/test_helper.exs b/apps/engine/test/test_helper.exs
@@ -1,11 +1,11 @@
 Application.ensure_all_started(:snowflake)
 Application.ensure_all_started(:refactorex)
-{"", 0} = System.cmd("epmd", ~w(-daemon))
+
 random_number = :rand.uniform(500)
 
 with :nonode@nohost <- Node.self() do
   {:ok, _pid} =
-    :net_kernel.start(:"testing-#{random_number}@127.0.0.1", %{name_domain: :longnames})
+    :net_kernel.start(:"expert-manager-testing-#{random_number}", %{name_domain: :shortnames})
 end
 
 Engine.Module.Loader.start_link(nil)
diff --git a/apps/expert/lib/expert/application.ex b/apps/expert/lib/expert/application.ex
@@ -65,6 +65,7 @@ defmodule Expert.Application do
       end
 
     children = [
+      {Forge.NodePortMapper, []},
       document_store_child_spec(),
       {DynamicSupervisor, Expert.Project.DynamicSupervisor.options()},
       {DynamicSupervisor, name: Expert.DynamicSupervisor},
diff --git a/apps/expert/lib/expert/engine_node.ex b/apps/expert/lib/expert/engine_node.ex
@@ -28,20 +28,30 @@ defmodule Expert.EngineNode do
     @dialyzer {:nowarn_function, start: 3}
 
     def start(%__MODULE__{} = state, paths, from) do
-      this_node = inspect(Node.self())
-
-      args = [
-        "--name",
-        Project.node_name(state.project),
-        "--cookie",
-        state.cookie,
-        "--no-halt",
-        "-e",
-        "Node.connect(#{this_node})"
-        | path_append_arguments(paths)
-      ]
-
-      case Expert.Port.open_elixir(state.project, args: args) do
+      this_node = to_string(Node.self())
+      dist_port = Forge.EPMD.dist_port()
+
+      args =
+        [
+          "--erl",
+          "-start_epmd false",
+          "--sname",
+          Project.node_name(state.project),
+          "--cookie",
+          state.cookie,
+          "--no-halt",
+          "-e",
+          "Forge.NodePortMapper.register(); IO.puts(\"ok\")"
+          | path_append_arguments(paths)
+        ]
+
+      env =
+        [
+          {"EXPERT_PARENT_NODE", this_node},
+          {"EXPERT_PARENT_PORT", to_string(dist_port)}
+        ]
+
+      case Expert.Port.open_elixir(state.project, args: args, env: env) do
         {:error, :no_elixir, message} ->
           GenLSP.error(Expert.get_lsp(), message)
           Expert.terminate("Failed to find an elixir executable, shutting down", 1)
@@ -134,7 +144,7 @@ defmodule Expert.EngineNode do
 
   defp start_net_kernel(%Project{} = project) do
     manager = Project.manager_node_name(project)
-    :net_kernel.start(manager, %{name_domain: :longnames})
+    :net_kernel.start(manager, %{name_domain: :shortnames})
   end
 
   defp ensure_apps_started(node) do
@@ -294,7 +304,7 @@ defmodule Expert.EngineNode do
 
   @impl true
   def handle_call({:start, paths}, from, %State{} = state) do
-    :ok = :net_kernel.monitor_nodes(true, node_type: :visible)
+    :ok = :net_kernel.monitor_nodes(true, node_type: :all)
     Process.send_after(self(), :maybe_start_timeout, @start_timeout)
 
     case State.start(state, paths, from) do
@@ -365,7 +375,8 @@ defmodule Expert.EngineNode do
   end
 
   @impl true
-  def handle_info({_port, {:data, _message}}, %State{} = state) do
+  def handle_info({_port, {:data, message}}, %State{} = state) do
+    Logger.debug("Node port message: #{to_string(message)}")
     {:noreply, state}
   end
 
diff --git a/apps/expert/rel/remote.vm.args.eex b/apps/expert/rel/remote.vm.args.eex
@@ -0,0 +1,13 @@
+## Customize flags given to the VM: https://www.erlang.org/doc/man/erl.html
+## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here
+
+## Increase number of concurrent ports/sockets
+##+Q 65536
+
+## Tweak GC to run more often
+##-env ERL_FULLSWEEP_AFTER 10
+
+## Enable deployment without epmd
+## (requires changing both vm.args and remote.vm.args)
+-epmd_module Elixir.XPForge.EPMD
+-start_epmd false
diff --git a/apps/expert/rel/vm.args.eex b/apps/expert/rel/vm.args.eex
@@ -0,0 +1,12 @@
+## Customize flags given to the VM: https://www.erlang.org/doc/man/erl.html
+## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here
+
+## Increase number of concurrent ports/sockets
+##+Q 65536
+
+## Tweak GC to run more often
+##-env ERL_FULLSWEEP_AFTER 10
+
+## Enable deployment without epmd
+## (requires changing both vm.args and remote.vm.args)
+-start_epmd false -epmd_module Elixir.XPForge.EPMD
diff --git a/apps/expert/test/engine/build_test.exs b/apps/expert/test/engine/build_test.exs
@@ -42,6 +42,7 @@ defmodule Engine.BuildTest do
     |> Project.workspace_path()
     |> File.rm_rf()
 
+    {:ok, _} = start_supervised(Forge.NodePortMapper)
     {:ok, _} = start_supervised({EngineSupervisor, project})
     {:ok, _, _} = EngineNode.start(project)
     EngineApi.register_listener(project, self(), [:all])
@@ -650,6 +651,7 @@ defmodule Engine.BuildTest do
 
   describe ".exs files" do
     setup do
+      start_supervised!({Forge.NodePortMapper, []})
       start_supervised!(Engine.Dispatch)
       start_supervised!(Engine.ModuleMappings)
       start_supervised!(Build.CaptureServer)
diff --git a/apps/expert/test/engine/code_intelligence/definition_test.exs b/apps/expert/test/engine/code_intelligence/definition_test.exs
@@ -43,6 +43,7 @@ defmodule Expert.Engine.CodeIntelligence.DefinitionTest do
   end
 
   setup_all do
+    {:ok, _} = start_supervised({Forge.NodePortMapper, []})
     project = project(:navigations)
     start_supervised!({Document.Store, derive: [analysis: &Forge.Ast.analyze/1]})
     {:ok, _} = start_supervised({EngineSupervisor, project})
diff --git a/apps/expert/test/engine/engine_test.exs b/apps/expert/test/engine/engine_test.exs
@@ -9,6 +9,7 @@ defmodule EngineTest do
   import Forge.Test.Fixtures
 
   def start_project(%Project{} = project) do
+    start_supervised!({Forge.NodePortMapper, []})
     start_supervised!({Expert.EngineSupervisor, project})
     assert {:ok, _, _} = EngineNode.start(project)
     :ok
diff --git a/apps/expert/test/expert/engine_node_test.exs b/apps/expert/test/expert/engine_node_test.exs
@@ -10,6 +10,7 @@ defmodule Expert.EngineNodeTest do
 
   setup do
     project = project()
+    start_supervised!({Forge.NodePortMapper, []})
     start_supervised!({EngineSupervisor, project})
     {:ok, %{project: project}}
   end
diff --git a/apps/expert/test/expert/project/node_test.exs b/apps/expert/test/expert/project/node_test.exs
@@ -11,6 +11,7 @@ defmodule Expert.Project.NodeTest do
   setup do
     project = project()
 
+    {:ok, _} = start_supervised({Forge.NodePortMapper, []})
     {:ok, _} = start_supervised({DynamicSupervisor, Expert.Project.DynamicSupervisor.options()})
     {:ok, _} = start_supervised({Expert.Project.Supervisor, project})
 
diff --git a/apps/expert/test/expert/provider/handlers/code_action_test.exs b/apps/expert/test/expert/provider/handlers/code_action_test.exs
@@ -12,6 +12,7 @@ defmodule Expert.Provider.Handlers.CodeActionTest do
   use ExUnit.Case, async: false
 
   setup_all do
+    start_supervised!({Forge.NodePortMapper, []})
     start_supervised!({Document.Store, derive: [analysis: &Forge.Ast.analyze/1]})
     project = project(:navigations)
 
diff --git a/apps/expert/test/expert/provider/handlers/code_lens_test.exs b/apps/expert/test/expert/provider/handlers/code_lens_test.exs
@@ -16,6 +16,7 @@ defmodule Expert.Provider.Handlers.CodeLensTest do
   use Patch
 
   setup_all do
+    start_supervised!({Forge.NodePortMapper, []})
     start_supervised(Document.Store)
     project = project(:umbrella)
 
diff --git a/apps/expert/test/expert/provider/handlers/formatting_test.exs b/apps/expert/test/expert/provider/handlers/formatting_test.exs
@@ -12,6 +12,7 @@ defmodule Expert.Provider.Handlers.FormattingTest do
   end
 
   def with_real_project(%{project: project}) do
+    {:ok, _} = start_supervised({Forge.NodePortMapper, []})
     {:ok, _} = start_supervised({Expert.EngineSupervisor, project})
     {:ok, _, _} = EngineNode.start(project)
     EngineApi.register_listener(project, self(), [:all])
diff --git a/apps/expert/test/expert/provider/handlers/go_to_definition_test.exs b/apps/expert/test/expert/provider/handlers/go_to_definition_test.exs
@@ -15,6 +15,7 @@ defmodule Expert.Provider.Handlers.GoToDefinitionTest do
   setup_all do
     project = project(:navigations)
 
+    start_supervised!({Forge.NodePortMapper, []})
     start_supervised!(Expert.Application.document_store_child_spec())
     start_supervised!({DynamicSupervisor, Expert.Project.DynamicSupervisor.options()})
     start_supervised!({Expert.Project.Supervisor, project})
diff --git a/apps/expert/test/expert/provider/handlers/hover_test.exs b/apps/expert/test/expert/provider/handlers/hover_test.exs
@@ -20,6 +20,7 @@ defmodule Expert.Provider.Handlers.HoverTest do
   setup_all do
     project = Fixtures.project()
 
+    start_supervised!({Forge.NodePortMapper, []})
     start_supervised!(Expert.Application.document_store_child_spec())
     start_supervised!({DynamicSupervisor, Expert.Project.DynamicSupervisor.options()})
     start_supervised!({Expert.Project.Supervisor, project})
diff --git a/apps/expert/test/support/test/completion_case.ex b/apps/expert/test/support/test/completion_case.ex
@@ -18,6 +18,7 @@ defmodule Expert.Test.Expert.CompletionCase do
   setup_all do
     project = project()
 
+    start_supervised!({Forge.NodePortMapper, []})
     start_supervised!({DynamicSupervisor, Expert.Project.DynamicSupervisor.options()})
     start_supervised!({Expert.Project.Supervisor, project})
 
diff --git a/apps/expert/test/test_helper.exs b/apps/expert/test/test_helper.exs
@@ -1,11 +1,11 @@
 Application.ensure_all_started(:snowflake)
 Application.ensure_all_started(:refactorex)
-{"", 0} = System.cmd("epmd", ~w(-daemon))
+
 random_number = :rand.uniform(500)
 
 with :nonode@nohost <- Node.self() do
   {:ok, _pid} =
-    :net_kernel.start(:"testing-#{random_number}@127.0.0.1", %{name_domain: :longnames})
+    :net_kernel.start(:"expert-manager-testing-#{random_number}", %{name_domain: :shortnames})
 end
 
 Engine.Module.Loader.start_link(nil)
diff --git a/apps/forge/lib/forge/epmd.ex b/apps/forge/lib/forge/epmd.ex
@@ -0,0 +1,100 @@
+defmodule Forge.EPMD do
+  @moduledoc false
+
+  # From Erlang/OTP 23+
+  @epmd_dist_version 6
+
+  @doc ~S"""
+  This is the distribution port of the current node.
+
+  The parent node must be named `expert-manager-*`.
+  The child node must be named `expert-project-*`.
+
+  When the parent boots the child, it must pass
+  its node name and port as the respective environment
+  variables `EXPERT_PARENT_NODE` and `EXPERT_PARENT_PORT`.
+
+  The parent must have this as a child in its supervision tree:
+
+      {Forge.NodePortMapper, []}
+
+  The child, in turn, must have this:
+
+      {Task, &Forge.NodePortMapper.register/0}
+
+  This will register the child within the parent, so they can
+  find each other.
+
+  ## Example
+
+  In order to manually simulate the connections, run `elixirc epmd.ex` to compile
+  this file and follow the steps below. Notice we call the functions in the
+  `Forge.NodePortMapper` module directly, while in practice they will be called
+  as part of the app's supervision tree.
+
+      # In one node
+      $ iex --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD" --sname expert-manager-foo
+      iex(expert_parent_foo@macstudio)> Forge.NodePortMapper.start_link([])
+      iex(expert_parent_foo@macstudio)> Forge.EPMD.dist_port()
+      52914
+
+  Get the port name from the step above and then, in another terminal, do:
+
+      $ EXPERT_PARENT_NODE=expert_parent_foo@macstudio EXPERT_PARENT_PORT=52914 \
+          iex --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD" --sname expert-project-bar
+      iex> Forge.NodePortMapper.register()
+
+  And in another terminal:
+
+      $ EXPERT_PARENT_NODE=expert_parent_foo@macstudio EXPERT_PARENT_PORT=52914 \
+          iex --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD -expert parent_port 52914" --sname expert-project-baz
+      iex> Forge.NodePortMapper.register()
+
+  If you try `Node.ping(:expert-project-bar@HOSTNAME)` from the last node, it should work.
+  The child nodes will find each other even without EPMD.
+  """
+  def dist_port do
+    :persistent_term.get(:expert_dist_port, nil)
+  end
+
+  # EPMD callbacks
+
+  def register_node(name, port), do: register_node(name, port, :inet)
+
+  def register_node(name, port, family) do
+    :persistent_term.put(:expert_dist_port, port)
+
+    # We don't care if EPMD is not running
+    case :erl_epmd.register_node(name, port, family) do
+      {:error, _} -> {:ok, -1}
+      {:ok, _} = ok -> ok
+    end
+  end
+
+  def port_please(name, host), do: port_please(name, host, :infinity)
+
+  def port_please(~c"expert-manager-" ++ _ = name, host, timeout) do
+    if port = System.get_env("EXPERT_PARENT_PORT") do
+      {:port, String.to_integer(port), @epmd_dist_version}
+    else
+      :erl_epmd.port_please(name, host, timeout)
+    end
+  end
+
+  def port_please(~c"expert-project-" ++ _ = name, host, timeout) do
+    if port = Forge.NodePortMapper.get_port(List.to_atom(name)) do
+      {:port, port, @epmd_dist_version}
+    else
+      :erl_epmd.port_please(name, host, timeout)
+    end
+  end
+
+  def port_please(name, host, timeout) do
+    :erl_epmd.port_please(name, host, timeout)
+  end
+
+  defdelegate start_link(), to: :erl_epmd
+  defdelegate listen_port_please(name, host), to: :erl_epmd
+  defdelegate address_please(name, host, family), to: :erl_epmd
+  defdelegate names(host_name), to: :erl_epmd
+end
diff --git a/apps/forge/lib/forge/node_port_mapper.ex b/apps/forge/lib/forge/node_port_mapper.ex
@@ -0,0 +1,42 @@
+defmodule Forge.NodePortMapper do
+  use GenServer
+
+  @name __MODULE__
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, :ok, name: @name)
+  end
+
+  defp parent_node do
+    if parent_node = System.get_env("EXPERT_PARENT_NODE") do
+      String.to_atom(parent_node)
+    else
+      node()
+    end
+  end
+
+  def register() do
+    GenServer.call({@name, parent_node()}, {:register, node(), Forge.EPMD.dist_port()})
+  end
+
+  def get_port(node) do
+    GenServer.call({@name, parent_node()}, {:get_port, node})
+  end
+
+  def init(:ok) do
+    {:ok, %{}}
+  end
+
+  def handle_call({:register, node, port}, _from, state) do
+    :erlang.monitor_node(node, true)
+    {:reply, :ok, Map.put(state, node, port)}
+  end
+
+  def handle_call({:get_port, node}, _from, state) do
+    {:reply, Map.get(state, node), state}
+  end
+
+  def handle_info({:nodedown, node}, state) do
+    {:noreply, Map.delete(state, node)}
+  end
+end
diff --git a/apps/forge/lib/forge/project.ex b/apps/forge/lib/forge/project.ex
@@ -76,7 +76,8 @@ defmodule Forge.Project do
   The project node's name
   """
   def node_name(%__MODULE__{} = project) do
-    :"project-#{name(project)}-#{entropy(project)}@127.0.0.1"
+    {:ok, host} = :inet.gethostname()
+    :"expert-project-#{name(project)}-#{entropy(project)}@#{host}"
   end
 
   def entropy(%__MODULE__{} = project) do
@@ -164,7 +165,7 @@ defmodule Forge.Project do
   end
 
   def manager_node_name(%__MODULE__{} = project) do
-    :"manager-#{name(project)}-#{entropy(project)}@127.0.0.1"
+    :"expert-manager-#{name(project)}-#{entropy(project)}"
   end
 
   @doc """
diff --git a/justfile b/justfile
