From ab6460786043aace6cb5a198f6b6392376d3422b Mon Sep 17 00:00:00 2001 From: doorgan Date: Mon, 10 Nov 2025 19:11:55 -0300 Subject: [PATCH 1/3] feat: epmdless clustering --- apps/engine/test/test_helper.exs | 4 +- apps/expert/lib/expert/application.ex | 1 + apps/expert/lib/expert/engine_node.ex | 77 +++++++++----- apps/expert/lib/expert/port.ex | 2 +- apps/expert/rel/remote.vm.args.eex | 13 +++ apps/expert/rel/vm.args.eex | 12 +++ apps/expert/test/engine/build_test.exs | 2 + .../code_intelligence/definition_test.exs | 1 + apps/expert/test/engine/engine_test.exs | 1 + apps/expert/test/expert/engine_node_test.exs | 1 + apps/expert/test/expert/project/node_test.exs | 1 + .../provider/handlers/code_action_test.exs | 1 + .../provider/handlers/code_lens_test.exs | 1 + .../provider/handlers/formatting_test.exs | 1 + .../handlers/go_to_definition_test.exs | 1 + .../expert/provider/handlers/hover_test.exs | 1 + .../test/support/test/completion_case.ex | 1 + apps/expert/test/test_helper.exs | 4 +- apps/forge/lib/forge/document/store.ex | 2 +- apps/forge/lib/forge/epmd.ex | 100 ++++++++++++++++++ apps/forge/lib/forge/node_port_mapper.ex | 44 ++++++++ apps/forge/lib/forge/project.ex | 4 +- justfile | 20 +++- 23 files changed, 256 insertions(+), 39 deletions(-) create mode 100644 apps/expert/rel/remote.vm.args.eex create mode 100644 apps/expert/rel/vm.args.eex create mode 100644 apps/forge/lib/forge/epmd.ex create mode 100644 apps/forge/lib/forge/node_port_mapper.ex diff --git a/apps/engine/test/test_helper.exs b/apps/engine/test/test_helper.exs index 8a213117..8c5f7020 100644 --- a/apps/engine/test/test_helper.exs +++ b/apps/engine/test/test_helper.exs @@ -1,11 +1,11 @@ Application.ensure_all_started(:snowflake) Application.ensure_all_started(:refactorex) -{"", 0} = System.cmd("epmd", ~w(-daemon)) + random_number = :rand.uniform(500) with :nonode@nohost <- Node.self() do {:ok, _pid} = - :net_kernel.start(:"testing-#{random_number}@127.0.0.1", %{name_domain: :longnames}) + Node.start(:"expert-manager-testing-#{random_number}@127.0.0.1", :longnames) end Engine.Module.Loader.start_link(nil) diff --git a/apps/expert/lib/expert/application.ex b/apps/expert/lib/expert/application.ex index f0678cba..17adeda0 100644 --- a/apps/expert/lib/expert/application.ex +++ b/apps/expert/lib/expert/application.ex @@ -65,6 +65,7 @@ defmodule Expert.Application do end children = [ + {Forge.NodePortMapper, []}, document_store_child_spec(), {DynamicSupervisor, Expert.Project.DynamicSupervisor.options()}, {DynamicSupervisor, name: Expert.DynamicSupervisor}, diff --git a/apps/expert/lib/expert/engine_node.ex b/apps/expert/lib/expert/engine_node.ex index 76d66d24..27024f87 100644 --- a/apps/expert/lib/expert/engine_node.ex +++ b/apps/expert/lib/expert/engine_node.ex @@ -28,20 +28,49 @@ defmodule Expert.EngineNode do @dialyzer {:nowarn_function, start: 3} def start(%__MODULE__{} = state, paths, from) do - this_node = inspect(Node.self()) - - args = [ - "--name", - Project.node_name(state.project), - "--cookie", - state.cookie, - "--no-halt", - "-e", - "Node.connect(#{this_node})" - | path_append_arguments(paths) - ] - - case Expert.Port.open_elixir(state.project, args: args) do + epmd_module = to_charlist(Forge.EPMD) + + case :init.get_argument(:epmd_module) do + {:ok, [[^epmd_module]]} -> + :ok + + _ -> + Application.put_env(:kernel, :epmd_module, Forge.EPMD, persistent: true) + + # Note: this is a private API + if :net_kernel.epmd_module() != Forge.EPMD do + raise(""" + you must set the environment variable ELIXIR_ERL_OPTIONS="-epmd_module #{Forge.EPMD}" + """) + end + end + + this_node = to_string(Node.self()) + dist_port = Forge.EPMD.dist_port() + + args = + [ + "--erl", + "-start_epmd false -epmd_module #{Forge.EPMD}", + "--cookie", + state.cookie, + "--no-halt", + "-e", + """ + {:ok, _} = Node.start(:"#{Project.node_name(state.project)}", :longnames) + #{Forge.NodePortMapper}.register() + IO.puts(\"ok\") + """ + | path_append_arguments(paths) + ] + + env = + [ + {"EXPERT_PARENT_NODE", this_node}, + {"EXPERT_PARENT_PORT", to_string(dist_port)} + ] + + case Expert.Port.open_elixir(state.project, args: args, env: env) do {:error, :no_elixir, message} -> GenLSP.error(Expert.get_lsp(), message) Expert.terminate("Failed to find an elixir executable, shutting down", 1) @@ -64,7 +93,7 @@ defmodule Expert.EngineNode do end def on_nodeup(%__MODULE__{} = state, node_name) do - if node_name == Project.node_name(state.project) do + if String.starts_with?(to_string(node_name), to_string(Project.node_name(state.project))) do {pid, _ref} = state.started_by Process.monitor(pid) GenServer.reply(state.started_by, :ok) @@ -117,7 +146,6 @@ defmodule Expert.EngineNode do use GenServer def start(project) do - :ok = ensure_epmd_started() start_net_kernel(project) node_name = Project.node_name(project) @@ -134,23 +162,13 @@ defmodule Expert.EngineNode do defp start_net_kernel(%Project{} = project) do manager = Project.manager_node_name(project) - :net_kernel.start(manager, %{name_domain: :longnames}) + Node.start(manager, :longnames) end defp ensure_apps_started(node) do :rpc.call(node, Engine, :ensure_apps_started, []) end - defp ensure_epmd_started do - case System.cmd("epmd", ~w(-daemon)) do - {"", 0} -> - :ok - - _ -> - {:error, :epmd_failed} - end - end - if Mix.env() == :test do # In test environment, Expert depends on the Engine app, so we look for it # in the expert build path. @@ -294,7 +312,7 @@ defmodule Expert.EngineNode do @impl true def handle_call({:start, paths}, from, %State{} = state) do - :ok = :net_kernel.monitor_nodes(true, node_type: :visible) + :ok = :net_kernel.monitor_nodes(true, node_type: :all) Process.send_after(self(), :maybe_start_timeout, @start_timeout) case State.start(state, paths, from) do @@ -365,7 +383,8 @@ defmodule Expert.EngineNode do end @impl true - def handle_info({_port, {:data, _message}}, %State{} = state) do + def handle_info({_port, {:data, message}}, %State{} = state) do + Logger.debug("Node port message: #{to_string(message)}") {:noreply, state} end diff --git a/apps/expert/lib/expert/port.ex b/apps/expert/lib/expert/port.ex index 99c8a650..6f4bb710 100644 --- a/apps/expert/lib/expert/port.ex +++ b/apps/expert/lib/expert/port.ex @@ -106,7 +106,7 @@ defmodule Expert.Port do opts end - Port.open({:spawn_executable, launcher}, opts) + Port.open({:spawn_executable, launcher}, [:stderr_to_stdout | opts]) end @doc """ diff --git a/apps/expert/rel/remote.vm.args.eex b/apps/expert/rel/remote.vm.args.eex new file mode 100644 index 00000000..d2f39ee6 --- /dev/null +++ b/apps/expert/rel/remote.vm.args.eex @@ -0,0 +1,13 @@ +## Customize flags given to the VM: https://www.erlang.org/doc/man/erl.html +## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here + +## Increase number of concurrent ports/sockets +##+Q 65536 + +## Tweak GC to run more often +##-env ERL_FULLSWEEP_AFTER 10 + +## Enable deployment without epmd +## (requires changing both vm.args and remote.vm.args) +-epmd_module Elixir.XPForge.EPMD +-start_epmd false diff --git a/apps/expert/rel/vm.args.eex b/apps/expert/rel/vm.args.eex new file mode 100644 index 00000000..0f7544a4 --- /dev/null +++ b/apps/expert/rel/vm.args.eex @@ -0,0 +1,12 @@ +## Customize flags given to the VM: https://www.erlang.org/doc/man/erl.html +## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here + +## Increase number of concurrent ports/sockets +##+Q 65536 + +## Tweak GC to run more often +##-env ERL_FULLSWEEP_AFTER 10 + +## Enable deployment without epmd +## (requires changing both vm.args and remote.vm.args) +-start_epmd false -epmd_module Elixir.XPForge.EPMD diff --git a/apps/expert/test/engine/build_test.exs b/apps/expert/test/engine/build_test.exs index ea7f74aa..29dbaa70 100644 --- a/apps/expert/test/engine/build_test.exs +++ b/apps/expert/test/engine/build_test.exs @@ -42,6 +42,7 @@ defmodule Engine.BuildTest do |> Project.workspace_path() |> File.rm_rf() + {:ok, _} = start_supervised(Forge.NodePortMapper) {:ok, _} = start_supervised({EngineSupervisor, project}) {:ok, _, _} = EngineNode.start(project) EngineApi.register_listener(project, self(), [:all]) @@ -650,6 +651,7 @@ defmodule Engine.BuildTest do describe ".exs files" do setup do + start_supervised!({Forge.NodePortMapper, []}) start_supervised!(Engine.Dispatch) start_supervised!(Engine.ModuleMappings) start_supervised!(Build.CaptureServer) diff --git a/apps/expert/test/engine/code_intelligence/definition_test.exs b/apps/expert/test/engine/code_intelligence/definition_test.exs index f0519ef9..3b44b964 100644 --- a/apps/expert/test/engine/code_intelligence/definition_test.exs +++ b/apps/expert/test/engine/code_intelligence/definition_test.exs @@ -43,6 +43,7 @@ defmodule Expert.Engine.CodeIntelligence.DefinitionTest do end setup_all do + {:ok, _} = start_supervised({Forge.NodePortMapper, []}) project = project(:navigations) start_supervised!({Document.Store, derive: [analysis: &Forge.Ast.analyze/1]}) {:ok, _} = start_supervised({EngineSupervisor, project}) diff --git a/apps/expert/test/engine/engine_test.exs b/apps/expert/test/engine/engine_test.exs index f4b0c5b3..3efeeedb 100644 --- a/apps/expert/test/engine/engine_test.exs +++ b/apps/expert/test/engine/engine_test.exs @@ -9,6 +9,7 @@ defmodule EngineTest do import Forge.Test.Fixtures def start_project(%Project{} = project) do + start_supervised!({Forge.NodePortMapper, []}) start_supervised!({Expert.EngineSupervisor, project}) assert {:ok, _, _} = EngineNode.start(project) :ok diff --git a/apps/expert/test/expert/engine_node_test.exs b/apps/expert/test/expert/engine_node_test.exs index c1fb7c28..623af7d1 100644 --- a/apps/expert/test/expert/engine_node_test.exs +++ b/apps/expert/test/expert/engine_node_test.exs @@ -10,6 +10,7 @@ defmodule Expert.EngineNodeTest do setup do project = project() + start_supervised!({Forge.NodePortMapper, []}) start_supervised!({EngineSupervisor, project}) {:ok, %{project: project}} end diff --git a/apps/expert/test/expert/project/node_test.exs b/apps/expert/test/expert/project/node_test.exs index 1d0bf196..1dd30372 100644 --- a/apps/expert/test/expert/project/node_test.exs +++ b/apps/expert/test/expert/project/node_test.exs @@ -11,6 +11,7 @@ defmodule Expert.Project.NodeTest do setup do project = project() + {:ok, _} = start_supervised({Forge.NodePortMapper, []}) {:ok, _} = start_supervised({DynamicSupervisor, Expert.Project.DynamicSupervisor.options()}) {:ok, _} = start_supervised({Expert.Project.Supervisor, project}) diff --git a/apps/expert/test/expert/provider/handlers/code_action_test.exs b/apps/expert/test/expert/provider/handlers/code_action_test.exs index f8882475..245a9d1b 100644 --- a/apps/expert/test/expert/provider/handlers/code_action_test.exs +++ b/apps/expert/test/expert/provider/handlers/code_action_test.exs @@ -12,6 +12,7 @@ defmodule Expert.Provider.Handlers.CodeActionTest do use ExUnit.Case, async: false setup_all do + start_supervised!({Forge.NodePortMapper, []}) start_supervised!({Document.Store, derive: [analysis: &Forge.Ast.analyze/1]}) project = project(:navigations) diff --git a/apps/expert/test/expert/provider/handlers/code_lens_test.exs b/apps/expert/test/expert/provider/handlers/code_lens_test.exs index d17cd605..d9a84d46 100644 --- a/apps/expert/test/expert/provider/handlers/code_lens_test.exs +++ b/apps/expert/test/expert/provider/handlers/code_lens_test.exs @@ -16,6 +16,7 @@ defmodule Expert.Provider.Handlers.CodeLensTest do use Patch setup_all do + start_supervised!({Forge.NodePortMapper, []}) start_supervised(Document.Store) project = project(:umbrella) diff --git a/apps/expert/test/expert/provider/handlers/formatting_test.exs b/apps/expert/test/expert/provider/handlers/formatting_test.exs index 9471c839..ee9ef6ed 100644 --- a/apps/expert/test/expert/provider/handlers/formatting_test.exs +++ b/apps/expert/test/expert/provider/handlers/formatting_test.exs @@ -12,6 +12,7 @@ defmodule Expert.Provider.Handlers.FormattingTest do end def with_real_project(%{project: project}) do + {:ok, _} = start_supervised({Forge.NodePortMapper, []}) {:ok, _} = start_supervised({Expert.EngineSupervisor, project}) {:ok, _, _} = EngineNode.start(project) EngineApi.register_listener(project, self(), [:all]) diff --git a/apps/expert/test/expert/provider/handlers/go_to_definition_test.exs b/apps/expert/test/expert/provider/handlers/go_to_definition_test.exs index e46184cb..1b8a16b7 100644 --- a/apps/expert/test/expert/provider/handlers/go_to_definition_test.exs +++ b/apps/expert/test/expert/provider/handlers/go_to_definition_test.exs @@ -15,6 +15,7 @@ defmodule Expert.Provider.Handlers.GoToDefinitionTest do setup_all do project = project(:navigations) + start_supervised!({Forge.NodePortMapper, []}) start_supervised!(Expert.Application.document_store_child_spec()) start_supervised!({DynamicSupervisor, Expert.Project.DynamicSupervisor.options()}) start_supervised!({Expert.Project.Supervisor, project}) diff --git a/apps/expert/test/expert/provider/handlers/hover_test.exs b/apps/expert/test/expert/provider/handlers/hover_test.exs index a4927e83..41868917 100644 --- a/apps/expert/test/expert/provider/handlers/hover_test.exs +++ b/apps/expert/test/expert/provider/handlers/hover_test.exs @@ -20,6 +20,7 @@ defmodule Expert.Provider.Handlers.HoverTest do setup_all do project = Fixtures.project() + start_supervised!({Forge.NodePortMapper, []}) start_supervised!(Expert.Application.document_store_child_spec()) start_supervised!({DynamicSupervisor, Expert.Project.DynamicSupervisor.options()}) start_supervised!({Expert.Project.Supervisor, project}) diff --git a/apps/expert/test/support/test/completion_case.ex b/apps/expert/test/support/test/completion_case.ex index 2b313f1f..e1383711 100644 --- a/apps/expert/test/support/test/completion_case.ex +++ b/apps/expert/test/support/test/completion_case.ex @@ -18,6 +18,7 @@ defmodule Expert.Test.Expert.CompletionCase do setup_all do project = project() + start_supervised!({Forge.NodePortMapper, []}) start_supervised!({DynamicSupervisor, Expert.Project.DynamicSupervisor.options()}) start_supervised!({Expert.Project.Supervisor, project}) diff --git a/apps/expert/test/test_helper.exs b/apps/expert/test/test_helper.exs index 15d06e94..05ac47b0 100644 --- a/apps/expert/test/test_helper.exs +++ b/apps/expert/test/test_helper.exs @@ -1,11 +1,11 @@ Application.ensure_all_started(:snowflake) Application.ensure_all_started(:refactorex) -{"", 0} = System.cmd("epmd", ~w(-daemon)) + random_number = :rand.uniform(500) with :nonode@nohost <- Node.self() do {:ok, _pid} = - :net_kernel.start(:"testing-#{random_number}@127.0.0.1", %{name_domain: :longnames}) + Node.start(:"expert-manager-testing-#{random_number}@127.0.0.1", :longnames) end Engine.Module.Loader.start_link(nil) diff --git a/apps/forge/lib/forge/document/store.ex b/apps/forge/lib/forge/document/store.ex index 70dfe144..34dee336 100644 --- a/apps/forge/lib/forge/document/store.ex +++ b/apps/forge/lib/forge/document/store.ex @@ -414,7 +414,7 @@ defmodule Forge.Document.Store do end def name do - {:via, :global, {__MODULE__, entropy()}} + {:global, {__MODULE__, entropy()}} end defp entropy_key do diff --git a/apps/forge/lib/forge/epmd.ex b/apps/forge/lib/forge/epmd.ex new file mode 100644 index 00000000..2ff65430 --- /dev/null +++ b/apps/forge/lib/forge/epmd.ex @@ -0,0 +1,100 @@ +defmodule Forge.EPMD do + @moduledoc false + + # From Erlang/OTP 23+ + @epmd_dist_version 6 + + @doc ~S""" + This is the distribution port of the current node. + + The parent node must be named `expert-manager-*`. + The child node must be named `expert-project-*`. + + When the parent boots the child, it must pass + its node name and port as the respective environment + variables `EXPERT_PARENT_NODE` and `EXPERT_PARENT_PORT`. + + The parent must have this as a child in its supervision tree: + + {Forge.NodePortMapper, []} + + The child, in turn, must have this: + + {Task, &Forge.NodePortMapper.register/0} + + This will register the child within the parent, so they can + find each other. + + ## Example + + In order to manually simulate the connections, run `elixirc epmd.ex` to compile + this file and follow the steps below. Notice we call the functions in the + `Forge.NodePortMapper` module directly, while in practice they will be called + as part of the app's supervision tree. + + # In one node + $ iex --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD" --sname expert-manager-foo + iex(expert_parent_foo@macstudio)> Forge.NodePortMapper.start_link([]) + iex(expert_parent_foo@macstudio)> Forge.EPMD.dist_port() + 52914 + + Get the port name from the step above and then, in another terminal, do: + + $ EXPERT_PARENT_NODE=expert_parent_foo@macstudio EXPERT_PARENT_PORT=52914 \ + iex --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD" --sname expert-project-bar + iex> Forge.NodePortMapper.register() + + And in another terminal: + + $ EXPERT_PARENT_NODE=expert_parent_foo@macstudio EXPERT_PARENT_PORT=52914 \ + iex --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD -expert parent_port 52914" --sname expert-project-baz + iex> Forge.NodePortMapper.register() + + If you try `Node.ping(:expert-project-bar@HOSTNAME)` from the last node, it should work. + The child nodes will find each other even without EPMD. + """ + def dist_port do + :persistent_term.get(:expert_dist_port, nil) + end + + # EPMD callbacks + + def register_node(name, port), do: register_node(name, port, :inet) + + def register_node(name, port, family) do + :persistent_term.put(:expert_dist_port, port) + + # We don't care if EPMD is not running + case :erl_epmd.register_node(name, port, family) do + {:error, _} -> {:ok, -1} + {:ok, _} = ok -> ok + end + end + + def port_please(name, host), do: port_please(name, host, :infinity) + + def port_please(~c"expert-manager-" ++ _ = name, host, timeout) do + if port = System.get_env("EXPERT_PARENT_PORT") do + {:port, String.to_integer(port), @epmd_dist_version} + else + :erl_epmd.port_please(name, host, timeout) + end + end + + def port_please(~c"expert-project-" ++ _ = name, host, timeout) do + if port = Forge.NodePortMapper.get_port(List.to_atom(name)) do + {:port, port, @epmd_dist_version} + else + :erl_epmd.port_please(name, host, timeout) + end + end + + def port_please(name, host, timeout) do + :erl_epmd.port_please(name, host, timeout) + end + + defdelegate start_link, to: :erl_epmd + defdelegate listen_port_please(name, host), to: :erl_epmd + defdelegate address_please(name, host, family), to: :erl_epmd + defdelegate names(host_name), to: :erl_epmd +end diff --git a/apps/forge/lib/forge/node_port_mapper.ex b/apps/forge/lib/forge/node_port_mapper.ex new file mode 100644 index 00000000..ad58ec41 --- /dev/null +++ b/apps/forge/lib/forge/node_port_mapper.ex @@ -0,0 +1,44 @@ +defmodule Forge.NodePortMapper do + use GenServer + require Logger + + @name __MODULE__ + + def start_link(_) do + Logger.info("Starting NodePortMapper on #{node()}") + GenServer.start_link(__MODULE__, :ok, name: @name) + end + + defp parent_node do + if parent_node = System.get_env("EXPERT_PARENT_NODE") do + String.to_atom(parent_node) + else + node() + end + end + + def register do + GenServer.call({@name, parent_node()}, {:register, node(), Forge.EPMD.dist_port()}) + end + + def get_port(node) do + GenServer.call({@name, parent_node()}, {:get_port, node}) + end + + def init(:ok) do + {:ok, %{}} + end + + def handle_call({:register, node, port}, _from, state) do + :erlang.monitor_node(node, true) + {:reply, :ok, Map.put(state, node, port)} + end + + def handle_call({:get_port, node}, _from, state) do + {:reply, Map.get(state, node), state} + end + + def handle_info({:nodedown, node}, state) do + {:noreply, Map.delete(state, node)} + end +end diff --git a/apps/forge/lib/forge/project.ex b/apps/forge/lib/forge/project.ex index 6b9e097b..7ce5645a 100644 --- a/apps/forge/lib/forge/project.ex +++ b/apps/forge/lib/forge/project.ex @@ -76,7 +76,7 @@ defmodule Forge.Project do The project node's name """ def node_name(%__MODULE__{} = project) do - :"project-#{name(project)}-#{entropy(project)}@127.0.0.1" + :"expert-project-#{name(project)}-#{entropy(project)}@127.0.0.1" end def entropy(%__MODULE__{} = project) do @@ -164,7 +164,7 @@ defmodule Forge.Project do end def manager_node_name(%__MODULE__{} = project) do - :"manager-#{name(project)}-#{entropy(project)}@127.0.0.1" + :"expert-manager-#{name(project)}-#{entropy(project)}@127.0.0.1" end @doc """ diff --git a/justfile b/justfile index 91ea097f..0b4cee93 100644 --- a/justfile +++ b/justfile @@ -32,11 +32,27 @@ mix project="all" *args="": case {{ project }} in all) for proj in {{ apps }}; do - (cd "apps/$proj" && mix {{args}}) + case $proj in + expert) + (cd "apps/$proj" && elixir --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD" -S mix {{args}}) + ;; + engine) + (cd "apps/$proj" && elixir --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD" -S mix {{args}}) + ;; + *) + (cd "apps/$proj" && mix {{args}}) + ;; + esac done ;; + expert) + (cd "apps/expert" && elixir --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD" -S mix {{args}}) + ;; + engine) + (cd "apps/engine" && elixir --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD" -S mix {{args}}) + ;; *) - (cd "apps/{{ project }}" && mix {{args}}) + (cd "apps/{{ project }}" && mix {{args}}) ;; esac From bd3839a817d66cbeddd8d8302130b4fa3dc2a9b2 Mon Sep 17 00:00:00 2001 From: doorgan Date: Fri, 14 Nov 2025 12:30:01 -0300 Subject: [PATCH 2/3] docs: update epmd/distribution docs --- apps/expert/lib/expert/engine_node.ex | 7 +++++++ apps/forge/lib/forge/epmd.ex | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/expert/lib/expert/engine_node.ex b/apps/expert/lib/expert/engine_node.ex index 27024f87..4427debc 100644 --- a/apps/expert/lib/expert/engine_node.ex +++ b/apps/expert/lib/expert/engine_node.ex @@ -56,6 +56,13 @@ defmodule Expert.EngineNode do state.cookie, "--no-halt", "-e", + # We manually start distribution here instead of using --sname/--name + # because those options are not really compatible with `-epmd_module`. + # Apparently, passing the --name/-sname options causes the Erlang VM + # to start distribution right away before the modules in the code path + # are loaded, and it will crash because Forge.EPMD doesn't exist yet. + # If we start distribution manually after all the code is loaded, + # everything works fine. """ {:ok, _} = Node.start(:"#{Project.node_name(state.project)}", :longnames) #{Forge.NodePortMapper}.register() diff --git a/apps/forge/lib/forge/epmd.ex b/apps/forge/lib/forge/epmd.ex index 2ff65430..55809cf5 100644 --- a/apps/forge/lib/forge/epmd.ex +++ b/apps/forge/lib/forge/epmd.ex @@ -47,7 +47,7 @@ defmodule Forge.EPMD do And in another terminal: $ EXPERT_PARENT_NODE=expert_parent_foo@macstudio EXPERT_PARENT_PORT=52914 \ - iex --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD -expert parent_port 52914" --sname expert-project-baz + iex --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD" --sname expert-project-baz iex> Forge.NodePortMapper.register() If you try `Node.ping(:expert-project-bar@HOSTNAME)` from the last node, it should work. From c7059584e61e8e369ea3cfd80daa366c12abadb9 Mon Sep 17 00:00:00 2001 From: doorgan Date: Sat, 15 Nov 2025 14:24:33 -0300 Subject: [PATCH 3/3] fix: check for epmd on expert startup --- apps/expert/lib/expert/application.ex | 21 +++++++++++++++++++++ apps/expert/lib/expert/engine_node.ex | 17 ----------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/apps/expert/lib/expert/application.ex b/apps/expert/lib/expert/application.ex index 17adeda0..fc9b81f1 100644 --- a/apps/expert/lib/expert/application.ex +++ b/apps/expert/lib/expert/application.ex @@ -64,6 +64,8 @@ defmodule Expert.Application do System.halt(1) end + ensure_epmd_module!() + children = [ {Forge.NodePortMapper, []}, document_store_child_spec(), @@ -90,4 +92,23 @@ defmodule Expert.Application do def document_store_child_spec do {Document.Store, derive: [analysis: &Forge.Ast.analyze/1]} end + + def ensure_epmd_module! do + epmd_module = to_charlist(Forge.EPMD) + + case :init.get_argument(:epmd_module) do + {:ok, [[^epmd_module]]} -> + :ok + + _ -> + Application.put_env(:kernel, :epmd_module, Forge.EPMD, persistent: true) + + # Note: this is a private API + if :net_kernel.epmd_module() != Forge.EPMD do + raise(""" + you must set the environment variable ELIXIR_ERL_OPTIONS="-epmd_module #{Forge.EPMD}" + """) + end + end + end end diff --git a/apps/expert/lib/expert/engine_node.ex b/apps/expert/lib/expert/engine_node.ex index 4427debc..a3ce8730 100644 --- a/apps/expert/lib/expert/engine_node.ex +++ b/apps/expert/lib/expert/engine_node.ex @@ -28,23 +28,6 @@ defmodule Expert.EngineNode do @dialyzer {:nowarn_function, start: 3} def start(%__MODULE__{} = state, paths, from) do - epmd_module = to_charlist(Forge.EPMD) - - case :init.get_argument(:epmd_module) do - {:ok, [[^epmd_module]]} -> - :ok - - _ -> - Application.put_env(:kernel, :epmd_module, Forge.EPMD, persistent: true) - - # Note: this is a private API - if :net_kernel.epmd_module() != Forge.EPMD do - raise(""" - you must set the environment variable ELIXIR_ERL_OPTIONS="-epmd_module #{Forge.EPMD}" - """) - end - end - this_node = to_string(Node.self()) dist_port = Forge.EPMD.dist_port()