Skip to content

Commit 4bffe9b

Browse files
committed
feat: epmdless clustering
1 parent c5ac441 commit 4bffe9b

File tree

11 files changed

+219
-22
lines changed

11 files changed

+219
-22
lines changed

apps/engine/lib/engine/bootstrap.ex

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ defmodule Engine.Bootstrap do
1212
require Logger
1313

1414
def init(%Project{} = project, document_store_entropy, app_configs) do
15+
Application.put_env(:kernel, :epmd_module, Forge.EPMD, persistent: true)
16+
Forge.NodePortMapper.register()
1517
Forge.Document.Store.set_entropy(document_store_entropy)
1618

1719
Application.put_all_env(app_configs)

apps/engine/test/test_helper.exs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
Application.ensure_all_started(:snowflake)
22
Application.ensure_all_started(:refactorex)
3-
{"", 0} = System.cmd("epmd", ~w(-daemon))
3+
44
random_number = :rand.uniform(500)
55

66
with :nonode@nohost <- Node.self() do
77
{:ok, _pid} =
8-
:net_kernel.start(:"testing-#{random_number}@127.0.0.1", %{name_domain: :longnames})
8+
:net_kernel.start(:"expert-manager-testing-#{random_number}", %{name_domain: :shortnames})
99
end
1010

1111
Engine.Module.Loader.start_link(nil)

apps/expert/lib/expert/application.ex

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ defmodule Expert.Application do
6565
end
6666

6767
children = [
68+
{Forge.NodePortMapper, []},
6869
document_store_child_spec(),
6970
{DynamicSupervisor, Expert.Project.DynamicSupervisor.options()},
7071
{DynamicSupervisor, name: Expert.DynamicSupervisor},

apps/expert/lib/expert/engine_node.ex

Lines changed: 26 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -29,19 +29,29 @@ defmodule Expert.EngineNode do
2929

3030
def start(%__MODULE__{} = state, paths, from) do
3131
this_node = inspect(Node.self())
32-
33-
args = [
34-
"--name",
35-
Project.node_name(state.project),
36-
"--cookie",
37-
state.cookie,
38-
"--no-halt",
39-
"-e",
40-
"Node.connect(#{this_node})"
41-
| path_append_arguments(paths)
42-
]
43-
44-
case Expert.Port.open_elixir(state.project, args: args) do
32+
dist_port = Forge.EPMD.dist_port()
33+
34+
args =
35+
[
36+
"--erl",
37+
"-start_epmd false -epmd_module #{Forge.EPMD}",
38+
"--sname",
39+
Project.node_name(state.project),
40+
"--cookie",
41+
state.cookie,
42+
"--no-halt",
43+
"-e",
44+
"IO.puts(\"ok\")"
45+
| path_append_arguments(paths)
46+
]
47+
48+
env =
49+
[
50+
{"EXPERT_PARENT_NODE", this_node},
51+
{"EXPERT_PARENT_PORT", to_string(dist_port)}
52+
]
53+
54+
case Expert.Port.open_elixir(state.project, args: args, env: env) do
4555
{:error, :no_elixir, message} ->
4656
GenLSP.error(Expert.get_lsp(), message)
4757
Expert.terminate("Failed to find an elixir executable, shutting down", 1)
@@ -134,7 +144,7 @@ defmodule Expert.EngineNode do
134144

135145
defp start_net_kernel(%Project{} = project) do
136146
manager = Project.manager_node_name(project)
137-
:net_kernel.start(manager, %{name_domain: :longnames})
147+
:net_kernel.start(manager, %{name_domain: :shortnames})
138148
end
139149

140150
defp ensure_apps_started(node) do
@@ -365,7 +375,8 @@ defmodule Expert.EngineNode do
365375
end
366376

367377
@impl true
368-
def handle_info({_port, {:data, _message}}, %State{} = state) do
378+
def handle_info({_port, {:data, message}}, %State{} = state) do
379+
Logger.debug("Node port message: #{to_string(message)}")
369380
{:noreply, state}
370381
end
371382

apps/expert/rel/remote.vm.args.eex

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
## Customize flags given to the VM: https://www.erlang.org/doc/man/erl.html
2+
## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here
3+
4+
## Increase number of concurrent ports/sockets
5+
##+Q 65536
6+
7+
## Tweak GC to run more often
8+
##-env ERL_FULLSWEEP_AFTER 10
9+
10+
## Enable deployment without epmd
11+
## (requires changing both vm.args and remote.vm.args)
12+
-epmd_module Elixir.XPForge.EPMD
13+
-start_epmd false -dist_listen false

apps/expert/rel/vm.args.eex

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
## Customize flags given to the VM: https://www.erlang.org/doc/man/erl.html
2+
## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here
3+
4+
## Increase number of concurrent ports/sockets
5+
##+Q 65536
6+
7+
## Tweak GC to run more often
8+
##-env ERL_FULLSWEEP_AFTER 10
9+
10+
## Enable deployment without epmd
11+
## (requires changing both vm.args and remote.vm.args)
12+
-start_epmd false -epmd_module Elixir.XPForge.EPMD

apps/expert/test/test_helper.exs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
Application.ensure_all_started(:snowflake)
22
Application.ensure_all_started(:refactorex)
3-
{"", 0} = System.cmd("epmd", ~w(-daemon))
3+
44
random_number = :rand.uniform(500)
55

66
with :nonode@nohost <- Node.self() do
77
{:ok, _pid} =
8-
:net_kernel.start(:"testing-#{random_number}@127.0.0.1", %{name_domain: :longnames})
8+
:net_kernel.start(:"expert-manager-testing-#{random_number}", %{name_domain: :shortnames})
99
end
1010

1111
Engine.Module.Loader.start_link(nil)

apps/forge/lib/forge/epmd.ex

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
defmodule Forge.EPMD do
2+
@moduledoc false
3+
4+
# From Erlang/OTP 23+
5+
@epmd_dist_version 6
6+
7+
@doc ~S"""
8+
This is the distribution port of the current node.
9+
10+
The parent node must be named `expert-manager-*`.
11+
The child node must be named `expert-project-*`.
12+
13+
When the parent boots the child, it must pass
14+
its node name and port as the respective environment
15+
variables `EXPERT_PARENT_NODE` and `EXPERT_PARENT_PORT`.
16+
17+
The parent must have this as a child in its supervision tree:
18+
19+
{Forge.NodePortMapper, []}
20+
21+
The child, in turn, must have this:
22+
23+
{Task, &Forge.NodePortMapper.register/0}
24+
25+
This will register the child within the parent, so they can
26+
find each other.
27+
28+
## Example
29+
30+
In order to manually simulate the connections, run `elixirc epmd.ex` to compile
31+
this file and follow the steps below. Notice we call the functions in the
32+
`Forge.NodePortMapper` module directly, while in practice they will be called
33+
as part of the app's supervision tree.
34+
35+
# In one node
36+
$ iex --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD" --sname expert-manager-foo
37+
iex(expert_parent_foo@macstudio)> Forge.NodePortMapper.start_link([])
38+
iex(expert_parent_foo@macstudio)> Forge.EPMD.dist_port()
39+
52914
40+
41+
Get the port name from the step above and then, in another terminal, do:
42+
43+
$ EXPERT_PARENT_NODE=expert_parent_foo@macstudio EXPERT_PARENT_PORT=52914 \
44+
iex --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD" --sname expert-project-bar
45+
iex> Forge.NodePortMapper.register()
46+
47+
And in another terminal:
48+
49+
$ EXPERT_PARENT_NODE=expert_parent_foo@macstudio EXPERT_PARENT_PORT=52914 \
50+
iex --erl "-start_epmd false -epmd_module Elixir.Forge.EPMD -expert parent_port 52914" --sname expert-project-baz
51+
iex> Forge.NodePortMapper.register()
52+
53+
If you try `Node.ping(:expert-project-bar@HOSTNAME)` from the last node, it should work.
54+
The child nodes will find each other even without EPMD.
55+
"""
56+
def dist_port do
57+
:persistent_term.get(:expert_dist_port, nil)
58+
end
59+
60+
# EPMD callbacks
61+
62+
def register_node(name, port), do: register_node(name, port, :inet)
63+
64+
def register_node(name, port, family) do
65+
:persistent_term.put(:expert_dist_port, port)
66+
67+
# We don't care if EPMD is not running
68+
case :erl_epmd.register_node(name, port, family) do
69+
{:error, _} -> {:ok, -1}
70+
{:ok, _} = ok -> ok
71+
end
72+
end
73+
74+
def port_please(name, host), do: port_please(name, host, :infinity)
75+
76+
def port_please(~c"expert-manager-" ++ _ = name, host, timeout) do
77+
if port = System.get_env("EXPERT_PARENT_PORT") do
78+
{:port, String.to_integer(port), @epmd_dist_version}
79+
else
80+
:erl_epmd.port_please(name, host, timeout)
81+
end
82+
end
83+
84+
def port_please(~c"expert-project-" ++ _ = name, host, timeout) do
85+
if port = Forge.NodePortMapper.get_port(List.to_atom(name)) do
86+
{:port, port, @epmd_dist_version}
87+
else
88+
:erl_epmd.port_please(name, host, timeout)
89+
end
90+
end
91+
92+
def port_please(name, host, timeout) do
93+
:erl_epmd.port_please(name, host, timeout)
94+
end
95+
96+
defdelegate start_link(), to: :erl_epmd
97+
defdelegate listen_port_please(name, host), to: :erl_epmd
98+
defdelegate address_please(name, host, family), to: :erl_epmd
99+
defdelegate names(host_name), to: :erl_epmd
100+
end
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
defmodule Forge.NodePortMapper do
2+
use GenServer
3+
4+
@name __MODULE__
5+
6+
def start_link(_) do
7+
GenServer.start_link(__MODULE__, :ok, name: @name)
8+
end
9+
10+
defp parent_node do
11+
if parent_node = System.get_env("EXPERT_PARENT_NODE") do
12+
String.to_atom(parent_node)
13+
else
14+
node()
15+
end
16+
end
17+
18+
def register() do
19+
GenServer.call({@name, parent_node()}, {:register, node(), Forge.EPMD.dist_port()})
20+
end
21+
22+
def get_port(node) do
23+
GenServer.call({@name, parent_node()}, {:get_port, node})
24+
end
25+
26+
def init(:ok) do
27+
{:ok, %{}}
28+
end
29+
30+
def handle_call({:register, node, port}, _from, state) do
31+
:erlang.monitor_node(node, true)
32+
{:reply, :ok, Map.put(state, node, port)}
33+
end
34+
35+
def handle_call({:get_port, node}, _from, state) do
36+
{:reply, Map.get(state, node), state}
37+
end
38+
39+
def handle_info({:nodedown, node}, state) do
40+
{:noreply, Map.delete(state, node)}
41+
end
42+
end

apps/forge/lib/forge/project.ex

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ defmodule Forge.Project do
164164
end
165165

166166
def manager_node_name(%__MODULE__{} = project) do
167-
:"manager-#{name(project)}-#{entropy(project)}@127.0.0.1"
167+
:"expert-manager-#{name(project)}-#{entropy(project)}@127.0.0.1"
168168
end
169169

170170
@doc """

0 commit comments

Comments
 (0)