From e1dfc852b7d1db6e290053ecd7e299f58259de28 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sun, 2 Nov 2025 19:50:23 -0500 Subject: [PATCH 01/17] Revert "fix: revert "feat: epmdless deployments (#167)" (#180)" This reverts commit 0f66faa317fdbefd3aed407ce46c294d1f6bdec2. --- apps/expert/lib/expert/engine_node.ex | 39 +++++++++++---------------- apps/expert/lib/expert/epmd.ex | 26 ++++++++++++++++++ apps/expert/rel/remote.vm.args.eex | 12 +++++++++ apps/expert/rel/vm.args.eex | 12 +++++++++ 4 files changed, 66 insertions(+), 23 deletions(-) create mode 100644 apps/expert/lib/expert/epmd.ex create mode 100644 apps/expert/rel/remote.vm.args.eex create mode 100644 apps/expert/rel/vm.args.eex diff --git a/apps/expert/lib/expert/engine_node.ex b/apps/expert/lib/expert/engine_node.ex index 7ed60742..c948d21e 100644 --- a/apps/expert/lib/expert/engine_node.ex +++ b/apps/expert/lib/expert/engine_node.ex @@ -29,17 +29,21 @@ defmodule Expert.EngineNode do def start(%__MODULE__{} = state, paths, from) do this_node = inspect(Node.self()) - - args = [ - "--name", - Project.node_name(state.project), - "--cookie", - state.cookie, - "--no-halt", - "-e", - "Node.connect(#{this_node})" - | path_append_arguments(paths) - ] + dist_port = Expert.EPMD.dist_port() + + args = + [ + "--erl", + "-start_epmd false -erl_epmd_port #{dist_port} -dist_listen false", + "--name", + Project.node_name(state.project), + "--cookie", + state.cookie, + "--no-halt", + "-e", + "Node.connect(#{this_node}); IO.puts(\"ok\")" + | path_append_arguments(paths) + ] case Expert.Port.open_elixir(state.project, args: args) do {:error, :no_elixir, message} -> @@ -117,7 +121,6 @@ defmodule Expert.EngineNode do use GenServer def start(project) do - :ok = ensure_epmd_started() start_net_kernel(project) node_name = Project.node_name(project) @@ -141,16 +144,6 @@ defmodule Expert.EngineNode do :rpc.call(node, Engine, :ensure_apps_started, []) end - defp ensure_epmd_started do - case System.cmd("epmd", ~w(-daemon)) do - {"", 0} -> - :ok - - _ -> - {:error, :epmd_failed} - end - end - if Mix.env() == :test do # In test environment, Expert depends on the Engine app, so we look for it # in the expert build path. @@ -291,7 +284,7 @@ defmodule Expert.EngineNode do @impl true def handle_call({:start, paths}, from, %State{} = state) do - :ok = :net_kernel.monitor_nodes(true, node_type: :visible) + :ok = :net_kernel.monitor_nodes(true, node_type: :all) Process.send_after(self(), :maybe_start_timeout, @start_timeout) case State.start(state, paths, from) do diff --git a/apps/expert/lib/expert/epmd.ex b/apps/expert/lib/expert/epmd.ex new file mode 100644 index 00000000..bff5030e --- /dev/null +++ b/apps/expert/lib/expert/epmd.ex @@ -0,0 +1,26 @@ +defmodule Expert.EPMD do + def dist_port do + :persistent_term.get(:expert_dist_port, nil) + end + + # EPMD callbacks + + def register_node(name, port), do: register_node(name, port, :inet) + + def register_node(name, port, family) do + :persistent_term.put(:expert_dist_port, port) + + # We don't care if EPMD is not running + case :erl_epmd.register_node(name, port, family) do + {:error, _} -> {:ok, -1} + {:ok, _} = ok -> ok + end + end + + defdelegate start_link(), to: :erl_epmd + defdelegate port_please(name, host), to: :erl_epmd + defdelegate port_please(name, host, timeout), to: :erl_epmd + defdelegate listen_port_please(name, host), to: :erl_epmd + defdelegate address_please(name, host, family), to: :erl_epmd + defdelegate names(host_name), to: :erl_epmd +end diff --git a/apps/expert/rel/remote.vm.args.eex b/apps/expert/rel/remote.vm.args.eex new file mode 100644 index 00000000..559a4767 --- /dev/null +++ b/apps/expert/rel/remote.vm.args.eex @@ -0,0 +1,12 @@ +## Customize flags given to the VM: https://www.erlang.org/doc/man/erl.html +## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here + +## Increase number of concurrent ports/sockets +##+Q 65536 + +## Tweak GC to run more often +##-env ERL_FULLSWEEP_AFTER 10 + +## Enable deployment without epmd +## (requires changing both vm.args and remote.vm.args) +-start_epmd false -dist_listen false diff --git a/apps/expert/rel/vm.args.eex b/apps/expert/rel/vm.args.eex new file mode 100644 index 00000000..c31a4837 --- /dev/null +++ b/apps/expert/rel/vm.args.eex @@ -0,0 +1,12 @@ +## Customize flags given to the VM: https://www.erlang.org/doc/man/erl.html +## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here + +## Increase number of concurrent ports/sockets +##+Q 65536 + +## Tweak GC to run more often +##-env ERL_FULLSWEEP_AFTER 10 + +## Enable deployment without epmd +## (requires changing both vm.args and remote.vm.args) +-start_epmd false -epmd_module Elixir.XPExpert.EPMD From 82fd238d4ec2c766dd3a7a03f76349c574138787 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sun, 2 Nov 2025 20:58:19 -0500 Subject: [PATCH 02/17] feat: real epmd-less --- apps/engine/lib/engine/search/store/backends/ets.ex | 2 +- apps/engine/mix.exs | 1 + apps/engine/mix.lock | 3 +++ apps/expert/lib/expert/state.ex | 2 -- apps/expert/mix.lock | 3 +++ apps/forge/lib/forge/document/store.ex | 2 +- apps/forge/mix.exs | 1 + 7 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/engine/lib/engine/search/store/backends/ets.ex b/apps/engine/lib/engine/search/store/backends/ets.ex index f5952ac3..7f16ebe1 100644 --- a/apps/engine/lib/engine/search/store/backends/ets.ex +++ b/apps/engine/lib/engine/search/store/backends/ets.ex @@ -182,7 +182,7 @@ defmodule Engine.Search.Store.Backends.Ets do end defp genserver_name(%Project{} = project) do - {:global, leader_name(project)} + {:via, :swarm, leader_name(project)} end defp become_leader(%Project{} = project) do diff --git a/apps/engine/mix.exs b/apps/engine/mix.exs index 9bf81c94..bfcc6a67 100644 --- a/apps/engine/mix.exs +++ b/apps/engine/mix.exs @@ -56,6 +56,7 @@ defmodule Engine.MixProject do {:path_glob, "~> 0.2"}, {:phoenix_live_view, "~> 1.0", only: [:test], runtime: false}, {:sourceror, "~> 1.9"}, + {:swarm, "~> 3.4"}, {:stream_data, "~> 1.1", only: [:test], runtime: false}, {:refactorex, "~> 0.1.52"} ] diff --git a/apps/engine/mix.lock b/apps/engine/mix.lock index 001928b2..cf4e5fc6 100644 --- a/apps/engine/mix.lock +++ b/apps/engine/mix.lock @@ -10,8 +10,10 @@ "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, "file_system": {:hex, :file_system, "1.1.0", "08d232062284546c6c34426997dd7ef6ec9f8bbd090eb91780283c9016840e8f", [:mix], [], "hexpm", "bfcf81244f416871f2a2e15c1b515287faa5db9c6bcf290222206d120b3d43f6"}, "gen_lsp": {:hex, :gen_lsp, "0.11.1", "395034f8d0f33cc76cbae85f480c827ae7bafa6b2436445501aaa582ba3195ff", [:mix], [{:jason, "~> 1.3", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.5 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:schematic, "~> 0.2.1", [hex: :schematic, repo: "hexpm", optional: false]}, {:typed_struct, "~> 0.3.0", [hex: :typed_struct, repo: "hexpm", optional: false]}], "hexpm", "78cd7994c0e46399c71e727fe29cfb8ff41e32711c1a30ad4b92203ee0d7920d"}, + "gen_state_machine": {:hex, :gen_state_machine, "2.1.0", "a38b0e53fad812d29ec149f0d354da5d1bc0d7222c3711f3a0bd5aa608b42992", [:mix], [], "hexpm", "ae367038808db25cee2f2c4b8d0531522ea587c4995eb6f96ee73410a60fa06b"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, + "libring": {:hex, :libring, "1.7.0", "4f245d2f1476cd7ed8f03740f6431acba815401e40299208c7f5c640e1883bda", [:mix], [], "hexpm", "070e3593cb572e04f2c8470dd0c119bc1817a7a0a7f88229f43cf0345268ec42"}, "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, @@ -31,6 +33,7 @@ "sourceror": {:hex, :sourceror, "1.10.0", "38397dedbbc286966ec48c7af13e228b171332be1ad731974438c77791945ce9", [:mix], [], "hexpm", "29dbdfc92e04569c9d8e6efdc422fc1d815f4bd0055dc7c51b8800fb75c4b3f1"}, "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"}, "stream_data": {:hex, :stream_data, "1.2.0", "58dd3f9e88afe27dc38bef26fce0c84a9e7a96772b2925c7b32cd2435697a52b", [:mix], [], "hexpm", "eb5c546ee3466920314643edf68943a5b14b32d1da9fe01698dc92b73f89a9ed"}, + "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm", "94884f84783fc1ba027aba8fe8a7dae4aad78c98e9f9c76667ec3471585c08c6"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, diff --git a/apps/expert/lib/expert/state.ex b/apps/expert/lib/expert/state.ex index 927345b3..36aa012e 100644 --- a/apps/expert/lib/expert/state.ex +++ b/apps/expert/lib/expert/state.ex @@ -138,11 +138,9 @@ defmodule Expert.State do case Document.Store.open(uri, text, version, language_id) do :ok -> - Logger.info("################### opened #{uri}") {:ok, state} error -> - Logger.error("################## Could not open #{uri} #{inspect(error)}") error end end diff --git a/apps/expert/mix.lock b/apps/expert/mix.lock index 9df9cca3..321b6a8b 100644 --- a/apps/expert/mix.lock +++ b/apps/expert/mix.lock @@ -9,8 +9,10 @@ "file_system": {:hex, :file_system, "1.1.0", "08d232062284546c6c34426997dd7ef6ec9f8bbd090eb91780283c9016840e8f", [:mix], [], "hexpm", "bfcf81244f416871f2a2e15c1b515287faa5db9c6bcf290222206d120b3d43f6"}, "finch": {:hex, :finch, "0.20.0", "5330aefb6b010f424dcbbc4615d914e9e3deae40095e73ab0c1bb0968933cadf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2658131a74d051aabfcba936093c903b8e89da9a1b63e430bee62045fa9b2ee2"}, "gen_lsp": {:hex, :gen_lsp, "0.11.1", "395034f8d0f33cc76cbae85f480c827ae7bafa6b2436445501aaa582ba3195ff", [:mix], [{:jason, "~> 1.3", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.5 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:schematic, "~> 0.2.1", [hex: :schematic, repo: "hexpm", optional: false]}, {:typed_struct, "~> 0.3.0", [hex: :typed_struct, repo: "hexpm", optional: false]}], "hexpm", "78cd7994c0e46399c71e727fe29cfb8ff41e32711c1a30ad4b92203ee0d7920d"}, + "gen_state_machine": {:hex, :gen_state_machine, "2.1.0", "a38b0e53fad812d29ec149f0d354da5d1bc0d7222c3711f3a0bd5aa608b42992", [:mix], [], "hexpm", "ae367038808db25cee2f2c4b8d0531522ea587c4995eb6f96ee73410a60fa06b"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, + "libring": {:hex, :libring, "1.7.0", "4f245d2f1476cd7ed8f03740f6431acba815401e40299208c7f5c640e1883bda", [:mix], [], "hexpm", "070e3593cb572e04f2c8470dd0c119bc1817a7a0a7f88229f43cf0345268ec42"}, "logger_file_backend": {:hex, :logger_file_backend, "0.0.14", "774bb661f1c3fed51b624d2859180c01e386eb1273dc22de4f4a155ef749a602", [:mix], [], "hexpm", "071354a18196468f3904ef09413af20971d55164267427f6257b52cfba03f9e6"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, @@ -24,6 +26,7 @@ "schematic": {:hex, :schematic, "0.2.1", "0b091df94146fd15a0a343d1bd179a6c5a58562527746dadd09477311698dbb1", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0b255d65921e38006138201cd4263fd8bb807d9dfc511074615cd264a571b3b1"}, "snowflake": {:hex, :snowflake, "1.0.4", "8433b4e04fbed19272c55e1b7de0f7a1ee1230b3ae31a813b616fd6ef279e87a", [:mix], [], "hexpm", "badb07ebb089a5cff737738297513db3962760b10fe2b158ae3bebf0b4d5be13"}, "sourceror": {:hex, :sourceror, "1.10.0", "38397dedbbc286966ec48c7af13e228b171332be1ad731974438c77791945ce9", [:mix], [], "hexpm", "29dbdfc92e04569c9d8e6efdc422fc1d815f4bd0055dc7c51b8800fb75c4b3f1"}, + "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm", "94884f84783fc1ba027aba8fe8a7dae4aad78c98e9f9c76667ec3471585c08c6"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, } diff --git a/apps/forge/lib/forge/document/store.ex b/apps/forge/lib/forge/document/store.ex index 70dfe144..94e806ee 100644 --- a/apps/forge/lib/forge/document/store.ex +++ b/apps/forge/lib/forge/document/store.ex @@ -414,7 +414,7 @@ defmodule Forge.Document.Store do end def name do - {:via, :global, {__MODULE__, entropy()}} + {:via, :swarm, {__MODULE__, entropy()}} end defp entropy_key do diff --git a/apps/forge/mix.exs b/apps/forge/mix.exs index 9b32fed3..24b1549d 100644 --- a/apps/forge/mix.exs +++ b/apps/forge/mix.exs @@ -40,6 +40,7 @@ defmodule Forge.MixProject do Mix.Dialyzer.dependency(), {:deps_nix, "~> 2.4", only: :dev}, {:gen_lsp, "~> 0.11"}, + {:swarm, "~> 3.4"}, {:snowflake, "~> 1.0"}, {:sourceror, "~> 1.9"}, {:stream_data, "~> 1.1", only: [:test], runtime: false}, From 7b9a0162b3e601381653f3dc3cd0fa9b5537dd07 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sun, 2 Nov 2025 22:04:17 -0500 Subject: [PATCH 03/17] test: get epmdless and swarm working --- .../lib/engine/search/store/backends/ets.ex | 2 +- apps/engine/test/test_helper.exs | 2 +- apps/expert/bin/debug_shell.sh | 2 +- apps/expert/lib/expert/engine_node.ex | 37 +++++++++++++------ apps/expert/lib/expert/port.ex | 7 +++- apps/expert/mix.exs | 2 +- apps/expert/test/test_helper.exs | 4 +- apps/forge/lib/forge/project.ex | 4 +- justfile | 4 +- 9 files changed, 40 insertions(+), 24 deletions(-) diff --git a/apps/engine/lib/engine/search/store/backends/ets.ex b/apps/engine/lib/engine/search/store/backends/ets.ex index 7f16ebe1..51bfd166 100644 --- a/apps/engine/lib/engine/search/store/backends/ets.ex +++ b/apps/engine/lib/engine/search/store/backends/ets.ex @@ -235,7 +235,7 @@ defmodule Engine.Search.Store.Backends.Ets do for {node_name_charlist, _port} <- node_and_port_list, node_name_string = List.to_string(node_name_charlist), String.contains?(node_name_string, project_substring) do - :"#{node_name_string}@127.0.0.1" + :"#{node_name_string}" end end diff --git a/apps/engine/test/test_helper.exs b/apps/engine/test/test_helper.exs index 8a213117..4740d84a 100644 --- a/apps/engine/test/test_helper.exs +++ b/apps/engine/test/test_helper.exs @@ -5,7 +5,7 @@ random_number = :rand.uniform(500) with :nonode@nohost <- Node.self() do {:ok, _pid} = - :net_kernel.start(:"testing-#{random_number}@127.0.0.1", %{name_domain: :longnames}) + Node.start(:"testing-#{random_number}", :shortnames) end Engine.Module.Loader.start_link(nil) diff --git a/apps/expert/bin/debug_shell.sh b/apps/expert/bin/debug_shell.sh index cbd73f28..033744dc 100755 --- a/apps/expert/bin/debug_shell.sh +++ b/apps/expert/bin/debug_shell.sh @@ -3,6 +3,6 @@ project_name=$1 node_name=$(epmd -names | grep manager-"$project_name" | awk '{print $2}') -iex --name "shell@127.0.0.1" \ +iex --sname "shell" \ --remsh "${node_name}" \ --cookie expert diff --git a/apps/expert/lib/expert/engine_node.ex b/apps/expert/lib/expert/engine_node.ex index c948d21e..0e1ade78 100644 --- a/apps/expert/lib/expert/engine_node.ex +++ b/apps/expert/lib/expert/engine_node.ex @@ -35,13 +35,13 @@ defmodule Expert.EngineNode do [ "--erl", "-start_epmd false -erl_epmd_port #{dist_port} -dist_listen false", - "--name", + "--sname", Project.node_name(state.project), "--cookie", state.cookie, "--no-halt", "-e", - "Node.connect(#{this_node}); IO.puts(\"ok\")" + "Node.connect(#{this_node}) |> dbg(); IO.puts(\"ok\")" | path_append_arguments(paths) ] @@ -52,6 +52,7 @@ defmodule Expert.EngineNode do {:error, :no_elixir} port -> + dbg(port) state = %{state | port: port, started_by: from} {:ok, state} end @@ -68,7 +69,10 @@ defmodule Expert.EngineNode do end def on_nodeup(%__MODULE__{} = state, node_name) do - if node_name == Project.node_name(state.project) do + dbg(node_name) + + if String.starts_with?(to_string(node_name), to_string(Project.node_name(state.project))) + |> dbg() do {pid, _ref} = state.started_by Process.monitor(pid) GenServer.reply(state.started_by, :ok) @@ -126,18 +130,23 @@ defmodule Expert.EngineNode do node_name = Project.node_name(project) bootstrap_args = [project, Document.Store.entropy(), all_app_configs()] + dbg(node_name) + with {:ok, node_pid} <- EngineSupervisor.start_project_node(project), {:ok, glob_paths} <- glob_paths(project), :ok <- start_node(project, glob_paths), - :ok <- :rpc.call(node_name, Engine.Bootstrap, :init, bootstrap_args), + dbg(Node.ping(node_name)), + dbg(Node.list(:hidden)), + :ok <- :rpc.call(node_name, Engine.Bootstrap, :init, bootstrap_args) |> dbg(), :ok <- ensure_apps_started(node_name) do {:ok, node_name, node_pid} end + |> dbg() end defp start_net_kernel(%Project{} = project) do manager = Project.manager_node_name(project) - :net_kernel.start(manager, %{name_domain: :longnames}) + Node.start(manager, :shortnames) |> dbg() end defp ensure_apps_started(node) do @@ -155,7 +164,7 @@ defmodule Expert.EngineNode do ["/**/priv" | app_globs] end - def glob_paths(_) do + defp glob_paths(_) do entries = for entry <- :code.get_path(), entry_string = List.to_string(entry), @@ -206,6 +215,7 @@ defmodule Expert.EngineNode do launcher = Expert.Port.path() GenLSP.info(lsp, "Finding or building engine for project #{project_name}") + dbg("building engine") with_progress(project, "Building engine for #{project_name}", fn -> port = @@ -265,15 +275,14 @@ defmodule Expert.EngineNode do def start_link(%Project{} = project) do state = State.new(project) - GenServer.start_link(__MODULE__, state, name: name(project)) + dbg(state) + GenServer.start_link(__MODULE__, state, name: name(project)) |> dbg() end - @start_timeout 3_000 + @start_timeout 30_000 defp start_node(project, paths) do - project - |> name() - |> GenServer.call({:start, paths}, @start_timeout + 500) + project |> name() |> GenServer.call({:start, paths}, @start_timeout + 500) |> dbg() end @impl GenServer @@ -287,8 +296,11 @@ defmodule Expert.EngineNode do :ok = :net_kernel.monitor_nodes(true, node_type: :all) Process.send_after(self(), :maybe_start_timeout, @start_timeout) + dbg("hi") + case State.start(state, paths, from) do {:ok, state} -> + dbg(state) {:noreply, state} {:error, :no_elixir} -> @@ -355,7 +367,8 @@ defmodule Expert.EngineNode do end @impl true - def handle_info({_port, {:data, _message}}, %State{} = state) do + def handle_info({_port, {:data, message}}, %State{} = state) do + dbg(message) {:noreply, state} end diff --git a/apps/expert/lib/expert/port.ex b/apps/expert/lib/expert/port.ex index 99c8a650..860fa248 100644 --- a/apps/expert/lib/expert/port.ex +++ b/apps/expert/lib/expert/port.ex @@ -80,9 +80,12 @@ defmodule Expert.Port do _ -> {path, 0} = - System.cmd(shell, ["-i", "-l", "-c", "cd #{directory} && echo $PATH"], env: env) + System.cmd(shell, ["-i", "-l", "-c", "cd #{directory} && echo $PATH"], + stderr_to_stdout: true, + env: env + ) - path + path |> String.trim() |> String.split("\n") |> List.last() end end diff --git a/apps/expert/mix.exs b/apps/expert/mix.exs index f117d11d..979fb235 100644 --- a/apps/expert/mix.exs +++ b/apps/expert/mix.exs @@ -77,7 +77,7 @@ defmodule Expert.MixProject do defp deps do [ - {:burrito, "~> 1.5", only: [:dev, :prod]}, + {:burrito, "~> 1.5"}, {:deps_nix, "~> 2.4", only: :dev}, Mix.Credo.dependency(), Mix.Dialyzer.dependency(), diff --git a/apps/expert/test/test_helper.exs b/apps/expert/test/test_helper.exs index 15d06e94..a53c27b0 100644 --- a/apps/expert/test/test_helper.exs +++ b/apps/expert/test/test_helper.exs @@ -1,11 +1,11 @@ Application.ensure_all_started(:snowflake) Application.ensure_all_started(:refactorex) -{"", 0} = System.cmd("epmd", ~w(-daemon)) +Application.ensure_all_started(:swarm) random_number = :rand.uniform(500) with :nonode@nohost <- Node.self() do {:ok, _pid} = - :net_kernel.start(:"testing-#{random_number}@127.0.0.1", %{name_domain: :longnames}) + Node.start(:"testing-#{random_number}", :shortnames) end Engine.Module.Loader.start_link(nil) diff --git a/apps/forge/lib/forge/project.ex b/apps/forge/lib/forge/project.ex index 6b9e097b..ab892a94 100644 --- a/apps/forge/lib/forge/project.ex +++ b/apps/forge/lib/forge/project.ex @@ -76,7 +76,7 @@ defmodule Forge.Project do The project node's name """ def node_name(%__MODULE__{} = project) do - :"project-#{name(project)}-#{entropy(project)}@127.0.0.1" + :"project-#{name(project)}-#{entropy(project)}" end def entropy(%__MODULE__{} = project) do @@ -164,7 +164,7 @@ defmodule Forge.Project do end def manager_node_name(%__MODULE__{} = project) do - :"manager-#{name(project)}-#{entropy(project)}@127.0.0.1" + :"manager-#{name(project)}-#{entropy(project)}" end @doc """ diff --git a/justfile b/justfile index 91ea097f..878365db 100644 --- a/justfile +++ b/justfile @@ -32,11 +32,11 @@ mix project="all" *args="": case {{ project }} in all) for proj in {{ apps }}; do - (cd "apps/$proj" && mix {{args}}) + (cd "apps/$proj" && elixir --erl "-start_epmd false -epmd_module Elixir.Expert.EPMD" -S mix {{args}}) done ;; *) - (cd "apps/{{ project }}" && mix {{args}}) + (cd "apps/{{ project }}" && elixir --erl "-start_epmd false -epmd_module Elixir.Expert.EPMD" -S mix {{args}}) ;; esac From 1e22d2546f791d5781acf346ef8f051cb8cedb1c Mon Sep 17 00:00:00 2001 From: doorgan Date: Tue, 4 Nov 2025 19:30:30 -0300 Subject: [PATCH 04/17] fix: include the hostname in calls using the project node name --- apps/expert/lib/expert/engine_api.ex | 2 +- apps/expert/lib/expert/engine_node.ex | 36 +++++++------------ apps/expert/test/expert/project/node_test.exs | 2 +- apps/forge/lib/forge/project.ex | 5 +++ apps/forge/lib/test/document_support.ex | 2 +- 5 files changed, 20 insertions(+), 27 deletions(-) diff --git a/apps/expert/lib/expert/engine_api.ex b/apps/expert/lib/expert/engine_api.ex index 46aa28e3..aeec56ef 100644 --- a/apps/expert/lib/expert/engine_api.ex +++ b/apps/expert/lib/expert/engine_api.ex @@ -13,7 +13,7 @@ defmodule Expert.EngineApi do def call(%Project{} = project, m, f, a \\ []) do project - |> Project.node_name() + |> Project.node() |> :erpc.call(m, f, a) end diff --git a/apps/expert/lib/expert/engine_node.ex b/apps/expert/lib/expert/engine_node.ex index 0e1ade78..8fd4976d 100644 --- a/apps/expert/lib/expert/engine_node.ex +++ b/apps/expert/lib/expert/engine_node.ex @@ -41,7 +41,7 @@ defmodule Expert.EngineNode do state.cookie, "--no-halt", "-e", - "Node.connect(#{this_node}) |> dbg(); IO.puts(\"ok\")" + "Node.connect(#{this_node}); IO.puts(\"ok\")" | path_append_arguments(paths) ] @@ -52,7 +52,6 @@ defmodule Expert.EngineNode do {:error, :no_elixir} port -> - dbg(port) state = %{state | port: port, started_by: from} {:ok, state} end @@ -69,10 +68,7 @@ defmodule Expert.EngineNode do end def on_nodeup(%__MODULE__{} = state, node_name) do - dbg(node_name) - - if String.starts_with?(to_string(node_name), to_string(Project.node_name(state.project))) - |> dbg() do + if String.starts_with?(to_string(node_name), to_string(Project.node_name(state.project))) do {pid, _ref} = state.started_by Process.monitor(pid) GenServer.reply(state.started_by, :ok) @@ -84,7 +80,7 @@ defmodule Expert.EngineNode do end def on_nodedown(%__MODULE__{} = state, node_name) do - if node_name == Project.node_name(state.project) do + if node_name == Project.node(state.project) do maybe_reply_to_stopper(state) {:shutdown, %{state | status: :stopped}} else @@ -115,7 +111,7 @@ defmodule Expert.EngineNode do defp project_rpc(%__MODULE__{} = state, module, function, args \\ []) do state.project - |> Project.node_name() + |> Project.node() |> :rpc.call(module, function, args) end end @@ -127,26 +123,22 @@ defmodule Expert.EngineNode do def start(project) do start_net_kernel(project) - node_name = Project.node_name(project) bootstrap_args = [project, Document.Store.entropy(), all_app_configs()] - dbg(node_name) + node = Project.node(project) with {:ok, node_pid} <- EngineSupervisor.start_project_node(project), {:ok, glob_paths} <- glob_paths(project), :ok <- start_node(project, glob_paths), - dbg(Node.ping(node_name)), - dbg(Node.list(:hidden)), - :ok <- :rpc.call(node_name, Engine.Bootstrap, :init, bootstrap_args) |> dbg(), - :ok <- ensure_apps_started(node_name) do - {:ok, node_name, node_pid} + :ok <- :rpc.call(node, Engine.Bootstrap, :init, bootstrap_args), + :ok <- ensure_apps_started(node) do + {:ok, node, node_pid} end - |> dbg() end defp start_net_kernel(%Project{} = project) do manager = Project.manager_node_name(project) - Node.start(manager, :shortnames) |> dbg() + Node.start(manager, :shortnames) end defp ensure_apps_started(node) do @@ -215,7 +207,6 @@ defmodule Expert.EngineNode do launcher = Expert.Port.path() GenLSP.info(lsp, "Finding or building engine for project #{project_name}") - dbg("building engine") with_progress(project, "Building engine for #{project_name}", fn -> port = @@ -275,14 +266,13 @@ defmodule Expert.EngineNode do def start_link(%Project{} = project) do state = State.new(project) - dbg(state) - GenServer.start_link(__MODULE__, state, name: name(project)) |> dbg() + GenServer.start_link(__MODULE__, state, name: name(project)) end @start_timeout 30_000 defp start_node(project, paths) do - project |> name() |> GenServer.call({:start, paths}, @start_timeout + 500) |> dbg() + project |> name() |> GenServer.call({:start, paths}, @start_timeout + 500) end @impl GenServer @@ -296,11 +286,8 @@ defmodule Expert.EngineNode do :ok = :net_kernel.monitor_nodes(true, node_type: :all) Process.send_after(self(), :maybe_start_timeout, @start_timeout) - dbg("hi") - case State.start(state, paths, from) do {:ok, state} -> - dbg(state) {:noreply, state} {:error, :no_elixir} -> @@ -368,6 +355,7 @@ defmodule Expert.EngineNode do @impl true def handle_info({_port, {:data, message}}, %State{} = state) do + Logger.debug("Received port message: #{inspect(message)}") dbg(message) {:noreply, state} end diff --git a/apps/expert/test/expert/project/node_test.exs b/apps/expert/test/expert/project/node_test.exs index 1d0bf196..6e7f6ad5 100644 --- a/apps/expert/test/expert/project/node_test.exs +++ b/apps/expert/test/expert/project/node_test.exs @@ -42,7 +42,7 @@ defmodule Expert.Project.NodeTest do end test "the node restarts when the supervisor pid is killed", %{project: project} do - node_name = EngineNode.node_name(project) + node_name = Forge.Project.node(project) supervisor_pid = EngineApi.call(project, Process, :whereis, [Engine.Supervisor]) assert is_pid(supervisor_pid) diff --git a/apps/forge/lib/forge/project.ex b/apps/forge/lib/forge/project.ex index ab892a94..0172db56 100644 --- a/apps/forge/lib/forge/project.ex +++ b/apps/forge/lib/forge/project.ex @@ -79,6 +79,11 @@ defmodule Forge.Project do :"project-#{name(project)}-#{entropy(project)}" end + def node(%__MODULE__{} = project) do + {:ok, hostname} = :inet.gethostname() + :"#{node_name(project)}@#{hostname}" + end + def entropy(%__MODULE__{} = project) do project.entropy end diff --git a/apps/forge/lib/test/document_support.ex b/apps/forge/lib/test/document_support.ex index 2ff235d4..e1ae05a2 100644 --- a/apps/forge/lib/test/document_support.ex +++ b/apps/forge/lib/test/document_support.ex @@ -3,7 +3,7 @@ defmodule Forge.Test.DocumentSupport do use ExUnit.CaseTemplate setup do - {:ok, _store} = start_supervised(Document.Store) + start_supervised(Document.Store) :ok end From d49317ef1246aa74e4a35859a700b29d1a378593 Mon Sep 17 00:00:00 2001 From: doorgan Date: Tue, 4 Nov 2025 20:12:25 -0300 Subject: [PATCH 05/17] fix: start swarm for engine tests --- apps/engine/test/test_helper.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/engine/test/test_helper.exs b/apps/engine/test/test_helper.exs index 4740d84a..ce3a950b 100644 --- a/apps/engine/test/test_helper.exs +++ b/apps/engine/test/test_helper.exs @@ -1,6 +1,6 @@ Application.ensure_all_started(:snowflake) Application.ensure_all_started(:refactorex) -{"", 0} = System.cmd("epmd", ~w(-daemon)) +Application.ensure_all_started(:swarm) random_number = :rand.uniform(500) with :nonode@nohost <- Node.self() do From ee01317985f01e87aba67e110fca23f801757167 Mon Sep 17 00:00:00 2001 From: doorgan Date: Tue, 4 Nov 2025 21:03:19 -0300 Subject: [PATCH 06/17] fix: use :global for ETS store name Using `:swarm` causes the store to not be discoverable by the GenServer apis. I'm not sure this is the right way to go, but it seems to work when I build a release and a lot of tests got fixed. --- apps/engine/lib/engine/search/store/backends/ets.ex | 2 +- apps/expert/lib/expert/engine_node.ex | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/engine/lib/engine/search/store/backends/ets.ex b/apps/engine/lib/engine/search/store/backends/ets.ex index 51bfd166..222283e9 100644 --- a/apps/engine/lib/engine/search/store/backends/ets.ex +++ b/apps/engine/lib/engine/search/store/backends/ets.ex @@ -182,7 +182,7 @@ defmodule Engine.Search.Store.Backends.Ets do end defp genserver_name(%Project{} = project) do - {:via, :swarm, leader_name(project)} + {:via, :global, leader_name(project)} end defp become_leader(%Project{} = project) do diff --git a/apps/expert/lib/expert/engine_node.ex b/apps/expert/lib/expert/engine_node.ex index 8fd4976d..b069fbeb 100644 --- a/apps/expert/lib/expert/engine_node.ex +++ b/apps/expert/lib/expert/engine_node.ex @@ -356,7 +356,6 @@ defmodule Expert.EngineNode do @impl true def handle_info({_port, {:data, message}}, %State{} = state) do Logger.debug("Received port message: #{inspect(message)}") - dbg(message) {:noreply, state} end From e7d0d8603de7942d7b0daa32c756b542be943dd4 Mon Sep 17 00:00:00 2001 From: doorgan Date: Wed, 5 Nov 2025 17:07:28 -0300 Subject: [PATCH 07/17] fix: use a swarm fork that compiles on otp 28 --- apps/engine/mix.exs | 2 +- apps/engine/mix.lock | 2 +- apps/expert/mix.lock | 2 +- apps/forge/mix.exs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/engine/mix.exs b/apps/engine/mix.exs index bfcc6a67..ea468ef6 100644 --- a/apps/engine/mix.exs +++ b/apps/engine/mix.exs @@ -56,7 +56,7 @@ defmodule Engine.MixProject do {:path_glob, "~> 0.2"}, {:phoenix_live_view, "~> 1.0", only: [:test], runtime: false}, {:sourceror, "~> 1.9"}, - {:swarm, "~> 3.4"}, + {:swarm, github: "doorgan/swarm", ref: "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"}, {:stream_data, "~> 1.1", only: [:test], runtime: false}, {:refactorex, "~> 0.1.52"} ] diff --git a/apps/engine/mix.lock b/apps/engine/mix.lock index cf4e5fc6..b83c2d73 100644 --- a/apps/engine/mix.lock +++ b/apps/engine/mix.lock @@ -33,7 +33,7 @@ "sourceror": {:hex, :sourceror, "1.10.0", "38397dedbbc286966ec48c7af13e228b171332be1ad731974438c77791945ce9", [:mix], [], "hexpm", "29dbdfc92e04569c9d8e6efdc422fc1d815f4bd0055dc7c51b8800fb75c4b3f1"}, "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"}, "stream_data": {:hex, :stream_data, "1.2.0", "58dd3f9e88afe27dc38bef26fce0c84a9e7a96772b2925c7b32cd2435697a52b", [:mix], [], "hexpm", "eb5c546ee3466920314643edf68943a5b14b32d1da9fe01698dc92b73f89a9ed"}, - "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm", "94884f84783fc1ba027aba8fe8a7dae4aad78c98e9f9c76667ec3471585c08c6"}, + "swarm": {:git, "https://github.com/doorgan/swarm.git", "0fff93ab51d28783b8eab22f2a2bff859c7e0b69", [ref: "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"]}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, diff --git a/apps/expert/mix.lock b/apps/expert/mix.lock index 321b6a8b..79769816 100644 --- a/apps/expert/mix.lock +++ b/apps/expert/mix.lock @@ -26,7 +26,7 @@ "schematic": {:hex, :schematic, "0.2.1", "0b091df94146fd15a0a343d1bd179a6c5a58562527746dadd09477311698dbb1", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0b255d65921e38006138201cd4263fd8bb807d9dfc511074615cd264a571b3b1"}, "snowflake": {:hex, :snowflake, "1.0.4", "8433b4e04fbed19272c55e1b7de0f7a1ee1230b3ae31a813b616fd6ef279e87a", [:mix], [], "hexpm", "badb07ebb089a5cff737738297513db3962760b10fe2b158ae3bebf0b4d5be13"}, "sourceror": {:hex, :sourceror, "1.10.0", "38397dedbbc286966ec48c7af13e228b171332be1ad731974438c77791945ce9", [:mix], [], "hexpm", "29dbdfc92e04569c9d8e6efdc422fc1d815f4bd0055dc7c51b8800fb75c4b3f1"}, - "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm", "94884f84783fc1ba027aba8fe8a7dae4aad78c98e9f9c76667ec3471585c08c6"}, + "swarm": {:git, "https://github.com/doorgan/swarm.git", "0fff93ab51d28783b8eab22f2a2bff859c7e0b69", [ref: "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"]}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, } diff --git a/apps/forge/mix.exs b/apps/forge/mix.exs index 24b1549d..677c6acc 100644 --- a/apps/forge/mix.exs +++ b/apps/forge/mix.exs @@ -40,7 +40,7 @@ defmodule Forge.MixProject do Mix.Dialyzer.dependency(), {:deps_nix, "~> 2.4", only: :dev}, {:gen_lsp, "~> 0.11"}, - {:swarm, "~> 3.4"}, + {:swarm, github: "doorgan/swarm", ref: "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"}, {:snowflake, "~> 1.0"}, {:sourceror, "~> 1.9"}, {:stream_data, "~> 1.1", only: [:test], runtime: false}, From eb36c86f7242980819426a740e85f33f7a66945a Mon Sep 17 00:00:00 2001 From: doorgan Date: Thu, 6 Nov 2025 15:15:20 -0300 Subject: [PATCH 08/17] fix(engine): update build exceptions tests --- apps/engine/test/engine/build/error_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/engine/test/engine/build/error_test.exs b/apps/engine/test/engine/build/error_test.exs index 5dfad106..5a8114e0 100644 --- a/apps/engine/test/engine/build/error_test.exs +++ b/apps/engine/test/engine/build/error_test.exs @@ -451,7 +451,7 @@ defmodule Engine.Build.ErrorTest do # used a regex here because the error changed in elixir 1.18 assert diagnostic.message =~ - ~r[protocol Enumerable not implemented for( 1 of)? type Integer] + ~r[protocol Enumerable not implemented for( 1 of type)? Integer] assert decorate(document_text, diagnostic.position) =~ "«for i <- 1, do: i\n»" end @@ -468,7 +468,7 @@ defmodule Engine.Build.ErrorTest do # used a regex here because the error changed in elixir 1.18 assert diagnostic.message =~ - ~r[protocol Enumerable not implemented for( 1 of)? type Integer] + ~r[protocol Enumerable not implemented for( 1 of type)? Integer] assert decorate(document_text, diagnostic.position) =~ "«for i <- 1, do: i\n»" end From d02f52f6d995478df003d6c064dd0467ae8ef263 Mon Sep 17 00:00:00 2001 From: doorgan Date: Thu, 6 Nov 2025 16:28:02 -0300 Subject: [PATCH 09/17] chore: update forge's lock file --- apps/forge/mix.lock | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/forge/mix.lock b/apps/forge/mix.lock index 629b4e66..b85b7ec5 100644 --- a/apps/forge/mix.lock +++ b/apps/forge/mix.lock @@ -8,8 +8,10 @@ "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, "file_system": {:hex, :file_system, "1.1.0", "08d232062284546c6c34426997dd7ef6ec9f8bbd090eb91780283c9016840e8f", [:mix], [], "hexpm", "bfcf81244f416871f2a2e15c1b515287faa5db9c6bcf290222206d120b3d43f6"}, "gen_lsp": {:hex, :gen_lsp, "0.11.1", "395034f8d0f33cc76cbae85f480c827ae7bafa6b2436445501aaa582ba3195ff", [:mix], [{:jason, "~> 1.3", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.5 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:schematic, "~> 0.2.1", [hex: :schematic, repo: "hexpm", optional: false]}, {:typed_struct, "~> 0.3.0", [hex: :typed_struct, repo: "hexpm", optional: false]}], "hexpm", "78cd7994c0e46399c71e727fe29cfb8ff41e32711c1a30ad4b92203ee0d7920d"}, + "gen_state_machine": {:hex, :gen_state_machine, "2.1.0", "a38b0e53fad812d29ec149f0d354da5d1bc0d7222c3711f3a0bd5aa608b42992", [:mix], [], "hexpm", "ae367038808db25cee2f2c4b8d0531522ea587c4995eb6f96ee73410a60fa06b"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, + "libring": {:hex, :libring, "1.7.0", "4f245d2f1476cd7ed8f03740f6431acba815401e40299208c7f5c640e1883bda", [:mix], [], "hexpm", "070e3593cb572e04f2c8470dd0c119bc1817a7a0a7f88229f43cf0345268ec42"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "patch": {:hex, :patch, "0.15.0", "947dd6a8b24a2d2d1137721f20bb96a8feb4f83248e7b4ad88b4871d52807af5", [:mix], [], "hexpm", "e8dadf9b57b30e92f6b2b1ce2f7f57700d14c66d4ed56ee27777eb73fb77e58d"}, @@ -18,6 +20,7 @@ "sourceror": {:hex, :sourceror, "1.9.0", "3bf5fe2d017aaabe3866d8a6da097dd7c331e0d2d54e59e21c2b066d47f1e08e", [:mix], [], "hexpm", "d20a9dd5efe162f0d75a307146faa2e17b823ea4f134f662358d70f0332fed82"}, "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"}, "stream_data": {:hex, :stream_data, "1.1.3", "15fdb14c64e84437901258bb56fc7d80aaf6ceaf85b9324f359e219241353bfb", [:mix], [], "hexpm", "859eb2be72d74be26c1c4f272905667672a52e44f743839c57c7ee73a1a66420"}, + "swarm": {:git, "https://github.com/doorgan/swarm.git", "0fff93ab51d28783b8eab22f2a2bff859c7e0b69", [ref: "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"]}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, } From 9cd1c861469a7e1eea20b17d0739a51833305371 Mon Sep 17 00:00:00 2001 From: doorgan Date: Thu, 6 Nov 2025 16:29:12 -0300 Subject: [PATCH 10/17] fix: use the right node name in test --- apps/expert/test/expert/project/node_test.exs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/expert/test/expert/project/node_test.exs b/apps/expert/test/expert/project/node_test.exs index 6e7f6ad5..9aade5e0 100644 --- a/apps/expert/test/expert/project/node_test.exs +++ b/apps/expert/test/expert/project/node_test.exs @@ -1,6 +1,5 @@ defmodule Expert.Project.NodeTest do alias Expert.EngineApi - alias Expert.Project.Node, as: EngineNode import Forge.Test.Fixtures import Forge.EngineApi.Messages @@ -30,7 +29,7 @@ defmodule Expert.Project.NodeTest do end test "the node is restarted when it goes down", %{project: project} do - node_name = EngineNode.node_name(project) + node_name = Forge.Project.node(project) old_pid = node_pid(project) :ok = EngineApi.stop(project) From 31ad440f87c3271dec3f0db552fbde936b3ba453 Mon Sep 17 00:00:00 2001 From: doorgan Date: Thu, 6 Nov 2025 17:34:07 -0300 Subject: [PATCH 11/17] fix: make `just test` use the right erl flags per project --- justfile | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/justfile b/justfile index 878365db..7fd85dac 100644 --- a/justfile +++ b/justfile @@ -32,11 +32,27 @@ mix project="all" *args="": case {{ project }} in all) for proj in {{ apps }}; do - (cd "apps/$proj" && elixir --erl "-start_epmd false -epmd_module Elixir.Expert.EPMD" -S mix {{args}}) + case $proj in + expert) + (cd "apps/$proj" && elixir --erl "-start_epmd false -epmd_module Elixir.Expert.EPMD" -S mix {{args}}) + ;; + engine) + (cd "apps/$proj" && elixir --erl "-start_epmd false -dist_listen false" -S mix {{args}}) + ;; + *) + (cd "apps/$proj" && mix {{args}}) + ;; + esac done ;; + expert) + (cd "apps/expert" && elixir --erl "-start_epmd false -epmd_module Elixir.Expert.EPMD" -S mix {{args}}) + ;; + engine) + (cd "apps/engine" && elixir --erl "-start_epmd false -dist_listen false" -S mix {{args}}) + ;; *) - (cd "apps/{{ project }}" && elixir --erl "-start_epmd false -epmd_module Elixir.Expert.EPMD" -S mix {{args}}) + (cd "apps/{{ project }}" && mix {{args}}) ;; esac From f1fded34d91a11042475dafd3638f5c70d278088 Mon Sep 17 00:00:00 2001 From: doorgan Date: Thu, 6 Nov 2025 23:10:53 -0300 Subject: [PATCH 12/17] fix: don't use swarm in tests --- apps/forge/lib/forge/document/store.ex | 17 ++++++++++++++++- apps/forge/test/forge/document/store_test.exs | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/apps/forge/lib/forge/document/store.ex b/apps/forge/lib/forge/document/store.ex index 94e806ee..51d9b703 100644 --- a/apps/forge/lib/forge/document/store.ex +++ b/apps/forge/lib/forge/document/store.ex @@ -413,8 +413,23 @@ defmodule Forge.Document.Store do end end + # NOTE(dorgan): In dev/prod we use Swarm for distribution, + # mainly to avoid using EPMD. This works well in practice, + # but in tests Swarm becomes incredibly noisy and causes + # lots of timing issues and introduces lots of flakiness. + # So in tests we use :global instead. + if Mix.env() == :test do + def clustering_method do + :global + end + else + def clustering_method do + :swarm + end + end + def name do - {:via, :swarm, {__MODULE__, entropy()}} + {:via, clustering_method(), {__MODULE__, entropy()}} end defp entropy_key do diff --git a/apps/forge/test/forge/document/store_test.exs b/apps/forge/test/forge/document/store_test.exs index f3f6b623..d55491b5 100644 --- a/apps/forge/test/forge/document/store_test.exs +++ b/apps/forge/test/forge/document/store_test.exs @@ -8,7 +8,7 @@ defmodule Forge.Document.StoreTest do def with_store(%{} = context) do store_opts = Map.get(context, :store, []) - {:ok, _} = start_supervised({Document.Store, store_opts}) + {:ok, _pid} = start_supervised({Document.Store, store_opts}) :ok end From 302fa2560e8d7d854d97e1ef337633e4c006e8db Mon Sep 17 00:00:00 2001 From: doorgan Date: Fri, 7 Nov 2025 03:24:38 -0300 Subject: [PATCH 13/17] fix: update error tests --- apps/engine/test/engine/build/error_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/engine/test/engine/build/error_test.exs b/apps/engine/test/engine/build/error_test.exs index 5a8114e0..4fa312f8 100644 --- a/apps/engine/test/engine/build/error_test.exs +++ b/apps/engine/test/engine/build/error_test.exs @@ -451,7 +451,7 @@ defmodule Engine.Build.ErrorTest do # used a regex here because the error changed in elixir 1.18 assert diagnostic.message =~ - ~r[protocol Enumerable not implemented for( 1 of type)? Integer] + ~r[protocol Enumerable not implemented for( 1 of)?( type)? Integer] assert decorate(document_text, diagnostic.position) =~ "«for i <- 1, do: i\n»" end @@ -468,7 +468,7 @@ defmodule Engine.Build.ErrorTest do # used a regex here because the error changed in elixir 1.18 assert diagnostic.message =~ - ~r[protocol Enumerable not implemented for( 1 of type)? Integer] + ~r[protocol Enumerable not implemented for( 1 of)?( type)? Integer] assert decorate(document_text, diagnostic.position) =~ "«for i <- 1, do: i\n»" end From 85b5726591af444ceadf81cc33bdb43f3b7754ac Mon Sep 17 00:00:00 2001 From: doorgan Date: Fri, 7 Nov 2025 03:33:48 -0300 Subject: [PATCH 14/17] chore: log when the engine node is stopped --- apps/expert/lib/expert/engine_node.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/expert/lib/expert/engine_node.ex b/apps/expert/lib/expert/engine_node.ex index b069fbeb..893eb8fc 100644 --- a/apps/expert/lib/expert/engine_node.ex +++ b/apps/expert/lib/expert/engine_node.ex @@ -297,6 +297,7 @@ defmodule Expert.EngineNode do @impl true def handle_call({:stop, stop_timeout}, from, %State{} = state) do + Logger.info("Stopping engine node for project #{Project.name(state.project)}") state = State.stop(state, from, stop_timeout) {:noreply, state, stop_timeout} end From 0c440857d7a61aac70743a6ab60cbe2e0f420f3b Mon Sep 17 00:00:00 2001 From: doorgan Date: Fri, 7 Nov 2025 03:34:19 -0300 Subject: [PATCH 15/17] chore: update nix deps --- apps/engine/deps.nix | 61 +++++++++++++++++++++++++++++++++-- apps/expert/deps.nix | 65 +++++++++++++++++++++++++++++++++++--- apps/expert_credo/deps.nix | 57 +++++++++++++++++++++++++++++++++ apps/expert_credo/mix.lock | 3 ++ apps/forge/deps.nix | 61 +++++++++++++++++++++++++++++++++-- 5 files changed, 239 insertions(+), 8 deletions(-) diff --git a/apps/engine/deps.nix b/apps/engine/deps.nix index b15396f9..8f8ba891 100644 --- a/apps/engine/deps.nix +++ b/apps/engine/deps.nix @@ -165,7 +165,7 @@ let gen_lsp = let - version = "0.11.0"; + version = "0.11.1"; drv = buildMix { inherit version; name = "gen_lsp"; @@ -174,7 +174,7 @@ let src = fetchHex { inherit version; pkg = "gen_lsp"; - sha256 = "d67c20650a5290a02f7bac53083ac4487d3c6b461f35a8b14c5d2d7638c20d26"; + sha256 = "78cd7994c0e46399c71e727fe29cfb8ff41e32711c1a30ad4b92203ee0d7920d"; }; beamDeps = [ @@ -187,6 +187,23 @@ let in drv; + gen_state_machine = + let + version = "2.1.0"; + drv = buildMix { + inherit version; + name = "gen_state_machine"; + appConfigPath = ./config; + + src = fetchHex { + inherit version; + pkg = "gen_state_machine"; + sha256 = "ae367038808db25cee2f2c4b8d0531522ea587c4995eb6f96ee73410a60fa06b"; + }; + }; + in + drv; + jason = let version = "1.4.4"; @@ -204,6 +221,23 @@ let in drv; + libring = + let + version = "1.7.0"; + drv = buildMix { + inherit version; + name = "libring"; + appConfigPath = ./config; + + src = fetchHex { + inherit version; + pkg = "libring"; + sha256 = "070e3593cb572e04f2c8470dd0c119bc1817a7a0a7f88229f43cf0345268ec42"; + }; + }; + in + drv; + nimble_options = let version = "1.1.1"; @@ -335,6 +369,29 @@ let in drv; + swarm = + let + version = "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"; + drv = buildMix { + inherit version; + name = "swarm"; + appConfigPath = ./config; + + src = pkgs.fetchFromGitHub { + owner = "doorgan"; + repo = "swarm"; + rev = "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"; + hash = "sha256-GpgapXxf0LTTD0PqrACBhX42dtQqRxZiOV43iuXzjqo="; + }; + + beamDeps = [ + libring + gen_state_machine + ]; + }; + in + drv; + telemetry = let version = "1.3.0"; diff --git a/apps/expert/deps.nix b/apps/expert/deps.nix index 205cc84a..6a25173c 100644 --- a/apps/expert/deps.nix +++ b/apps/expert/deps.nix @@ -147,7 +147,7 @@ let burrito = let - version = "1.4.0"; + version = "1.5.0"; drv = buildMix { inherit version; name = "burrito"; @@ -156,7 +156,7 @@ let src = fetchHex { inherit version; pkg = "burrito"; - sha256 = "0fa052e6f446cd3e5ff7e00813452b09eeadeddb5ec5174c2976eb0e4ad88765"; + sha256 = "3861abda7bffa733862b48da3e03df0b4cd41abf6fd24b91745f5c16d971e5fa"; }; beamDeps = [ @@ -195,7 +195,7 @@ let gen_lsp = let - version = "0.11.0"; + version = "0.11.1"; drv = buildMix { inherit version; name = "gen_lsp"; @@ -204,7 +204,7 @@ let src = fetchHex { inherit version; pkg = "gen_lsp"; - sha256 = "d67c20650a5290a02f7bac53083ac4487d3c6b461f35a8b14c5d2d7638c20d26"; + sha256 = "78cd7994c0e46399c71e727fe29cfb8ff41e32711c1a30ad4b92203ee0d7920d"; }; beamDeps = [ @@ -217,6 +217,23 @@ let in drv; + gen_state_machine = + let + version = "2.1.0"; + drv = buildMix { + inherit version; + name = "gen_state_machine"; + appConfigPath = ./config; + + src = fetchHex { + inherit version; + pkg = "gen_state_machine"; + sha256 = "ae367038808db25cee2f2c4b8d0531522ea587c4995eb6f96ee73410a60fa06b"; + }; + }; + in + drv; + hpax = let version = "1.0.3"; @@ -251,6 +268,23 @@ let in drv; + libring = + let + version = "1.7.0"; + drv = buildMix { + inherit version; + name = "libring"; + appConfigPath = ./config; + + src = fetchHex { + inherit version; + pkg = "libring"; + sha256 = "070e3593cb572e04f2c8470dd0c119bc1817a7a0a7f88229f43cf0345268ec42"; + }; + }; + in + drv; + logger_file_backend = let version = "0.0.14"; @@ -456,6 +490,29 @@ let in drv; + swarm = + let + version = "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"; + drv = buildMix { + inherit version; + name = "swarm"; + appConfigPath = ./config; + + src = pkgs.fetchFromGitHub { + owner = "doorgan"; + repo = "swarm"; + rev = "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"; + hash = "sha256-GpgapXxf0LTTD0PqrACBhX42dtQqRxZiOV43iuXzjqo="; + }; + + beamDeps = [ + libring + gen_state_machine + ]; + }; + in + drv; + telemetry = let version = "1.3.0"; diff --git a/apps/expert_credo/deps.nix b/apps/expert_credo/deps.nix index dddc486d..d1794fb8 100644 --- a/apps/expert_credo/deps.nix +++ b/apps/expert_credo/deps.nix @@ -169,6 +169,23 @@ let in drv; + gen_state_machine = + let + version = "2.1.0"; + drv = buildMix { + inherit version; + name = "gen_state_machine"; + appConfigPath = ./config; + + src = fetchHex { + inherit version; + pkg = "gen_state_machine"; + sha256 = "ae367038808db25cee2f2c4b8d0531522ea587c4995eb6f96ee73410a60fa06b"; + }; + }; + in + drv; + jason = let version = "1.4.4"; @@ -186,6 +203,23 @@ let in drv; + libring = + let + version = "1.7.0"; + drv = buildMix { + inherit version; + name = "libring"; + appConfigPath = ./config; + + src = fetchHex { + inherit version; + pkg = "libring"; + sha256 = "070e3593cb572e04f2c8470dd0c119bc1817a7a0a7f88229f43cf0345268ec42"; + }; + }; + in + drv; + nimble_options = let version = "1.1.1"; @@ -258,6 +292,29 @@ let in drv; + swarm = + let + version = "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"; + drv = buildMix { + inherit version; + name = "swarm"; + appConfigPath = ./config; + + src = pkgs.fetchFromGitHub { + owner = "doorgan"; + repo = "swarm"; + rev = "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"; + hash = "sha256-GpgapXxf0LTTD0PqrACBhX42dtQqRxZiOV43iuXzjqo="; + }; + + beamDeps = [ + libring + gen_state_machine + ]; + }; + in + drv; + telemetry = let version = "1.3.0"; diff --git a/apps/expert_credo/mix.lock b/apps/expert_credo/mix.lock index e7c80dfe..7875f471 100644 --- a/apps/expert_credo/mix.lock +++ b/apps/expert_credo/mix.lock @@ -8,8 +8,10 @@ "ex_doc": {:hex, :ex_doc, "0.37.2", "2a3aa7014094f0e4e286a82aa5194a34dd17057160988b8509b15aa6c292720c", [:mix], [{:earmark_parser, "~> 1.4.42", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "4dfa56075ce4887e4e8b1dcc121cd5fcb0f02b00391fd367ff5336d98fa49049"}, "file_system": {:hex, :file_system, "1.1.0", "08d232062284546c6c34426997dd7ef6ec9f8bbd090eb91780283c9016840e8f", [:mix], [], "hexpm", "bfcf81244f416871f2a2e15c1b515287faa5db9c6bcf290222206d120b3d43f6"}, "gen_lsp": {:hex, :gen_lsp, "0.11.0", "9eda4d2fcaff94d9b3062e322fcf524c176db1502f584a3cff6135088b46084b", [:mix], [{:jason, "~> 1.3", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.5 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:schematic, "~> 0.2.1", [hex: :schematic, repo: "hexpm", optional: false]}, {:typed_struct, "~> 0.3.0", [hex: :typed_struct, repo: "hexpm", optional: false]}], "hexpm", "d67c20650a5290a02f7bac53083ac4487d3c6b461f35a8b14c5d2d7638c20d26"}, + "gen_state_machine": {:hex, :gen_state_machine, "2.1.0", "a38b0e53fad812d29ec149f0d354da5d1bc0d7222c3711f3a0bd5aa608b42992", [:mix], [], "hexpm", "ae367038808db25cee2f2c4b8d0531522ea587c4995eb6f96ee73410a60fa06b"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, + "libring": {:hex, :libring, "1.7.0", "4f245d2f1476cd7ed8f03740f6431acba815401e40299208c7f5c640e1883bda", [:mix], [], "hexpm", "070e3593cb572e04f2c8470dd0c119bc1817a7a0a7f88229f43cf0345268ec42"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, @@ -19,6 +21,7 @@ "schematic": {:hex, :schematic, "0.2.1", "0b091df94146fd15a0a343d1bd179a6c5a58562527746dadd09477311698dbb1", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0b255d65921e38006138201cd4263fd8bb807d9dfc511074615cd264a571b3b1"}, "snowflake": {:hex, :snowflake, "1.0.4", "8433b4e04fbed19272c55e1b7de0f7a1ee1230b3ae31a813b616fd6ef279e87a", [:mix], [], "hexpm", "badb07ebb089a5cff737738297513db3962760b10fe2b158ae3bebf0b4d5be13"}, "sourceror": {:hex, :sourceror, "1.9.0", "3bf5fe2d017aaabe3866d8a6da097dd7c331e0d2d54e59e21c2b066d47f1e08e", [:mix], [], "hexpm", "d20a9dd5efe162f0d75a307146faa2e17b823ea4f134f662358d70f0332fed82"}, + "swarm": {:git, "https://github.com/doorgan/swarm.git", "0fff93ab51d28783b8eab22f2a2bff859c7e0b69", [ref: "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"]}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, } diff --git a/apps/forge/deps.nix b/apps/forge/deps.nix index dddc486d..651f42b0 100644 --- a/apps/forge/deps.nix +++ b/apps/forge/deps.nix @@ -147,7 +147,7 @@ let gen_lsp = let - version = "0.11.0"; + version = "0.11.1"; drv = buildMix { inherit version; name = "gen_lsp"; @@ -156,7 +156,7 @@ let src = fetchHex { inherit version; pkg = "gen_lsp"; - sha256 = "d67c20650a5290a02f7bac53083ac4487d3c6b461f35a8b14c5d2d7638c20d26"; + sha256 = "78cd7994c0e46399c71e727fe29cfb8ff41e32711c1a30ad4b92203ee0d7920d"; }; beamDeps = [ @@ -169,6 +169,23 @@ let in drv; + gen_state_machine = + let + version = "2.1.0"; + drv = buildMix { + inherit version; + name = "gen_state_machine"; + appConfigPath = ./config; + + src = fetchHex { + inherit version; + pkg = "gen_state_machine"; + sha256 = "ae367038808db25cee2f2c4b8d0531522ea587c4995eb6f96ee73410a60fa06b"; + }; + }; + in + drv; + jason = let version = "1.4.4"; @@ -186,6 +203,23 @@ let in drv; + libring = + let + version = "1.7.0"; + drv = buildMix { + inherit version; + name = "libring"; + appConfigPath = ./config; + + src = fetchHex { + inherit version; + pkg = "libring"; + sha256 = "070e3593cb572e04f2c8470dd0c119bc1817a7a0a7f88229f43cf0345268ec42"; + }; + }; + in + drv; + nimble_options = let version = "1.1.1"; @@ -258,6 +292,29 @@ let in drv; + swarm = + let + version = "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"; + drv = buildMix { + inherit version; + name = "swarm"; + appConfigPath = ./config; + + src = pkgs.fetchFromGitHub { + owner = "doorgan"; + repo = "swarm"; + rev = "0fff93ab51d28783b8eab22f2a2bff859c7e0b69"; + hash = "sha256-GpgapXxf0LTTD0PqrACBhX42dtQqRxZiOV43iuXzjqo="; + }; + + beamDeps = [ + libring + gen_state_machine + ]; + }; + in + drv; + telemetry = let version = "1.3.0"; From 3b79a52327c30a3c4300aefdf9b2768c24bd2659 Mon Sep 17 00:00:00 2001 From: doorgan Date: Fri, 7 Nov 2025 03:43:49 -0300 Subject: [PATCH 16/17] fix: wait a bit before bringing down the project node --- apps/expert/lib/expert/engine_node.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/expert/lib/expert/engine_node.ex b/apps/expert/lib/expert/engine_node.ex index 893eb8fc..f4bf2210 100644 --- a/apps/expert/lib/expert/engine_node.ex +++ b/apps/expert/lib/expert/engine_node.ex @@ -58,10 +58,16 @@ defmodule Expert.EngineNode do end def stop(%__MODULE__{} = state, from, stop_timeout) do - project_rpc(state, System, :stop) + # Wait a little bit before stopping the node to allow the reply + # to be sent back to the caller before the node goes down. + :timer.apply_after(50, __MODULE__, :perform_stop, [state]) %{state | stopped_by: from, stop_timeout: stop_timeout, status: :stopping} end + def perform_stop(%__MODULE__{} = state) do + project_rpc(state, System, :stop) + end + def halt(%__MODULE__{} = state) do project_rpc(state, System, :halt) %{state | status: :stopped} From 51295811fcd1e39d47fc5ea6d1e1b2e876fc078e Mon Sep 17 00:00:00 2001 From: doorgan Date: Fri, 7 Nov 2025 04:13:26 -0300 Subject: [PATCH 17/17] refactor: make the clustering method a configuration Forge gets compiled as :dev even in tests for both engine and expert, so we need to make the clustering method configurable. Furthermore, the engine and expert have different requirements for how the store is clustered, so we need to set different values for both. --- apps/engine/config/test.exs | 3 +++ apps/expert/config/test.exs | 6 ++++++ apps/forge/config/config.exs | 8 ++++++++ apps/forge/lib/forge/document/store.ex | 17 ++--------------- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/apps/engine/config/test.exs b/apps/engine/config/test.exs index a30e8851..0021d566 100644 --- a/apps/engine/config/test.exs +++ b/apps/engine/config/test.exs @@ -2,6 +2,9 @@ import Config config :logger, level: :none +config :forge, + document_store_clustering: :global + config :engine, edit_window_millis: 10, modules_cache_expiry: {50, :millisecond}, diff --git a/apps/expert/config/test.exs b/apps/expert/config/test.exs index becde769..40e846a5 100644 --- a/apps/expert/config/test.exs +++ b/apps/expert/config/test.exs @@ -1 +1,7 @@ import Config + +config :forge, + # Expert does need proper clustering even in tests, + # since a lot of tests actually rely on actual nodes + # being started and needing proper distribution. + document_store_clustering: :swarm diff --git a/apps/forge/config/config.exs b/apps/forge/config/config.exs index eece3a80..0d2be09f 100644 --- a/apps/forge/config/config.exs +++ b/apps/forge/config/config.exs @@ -4,3 +4,11 @@ config :snowflake, machine_id: 1, # First second of 2024 epoch: 1_704_070_800_000 + +config :forge, + # NOTE(dorgan): In dev/prod we use Swarm for distribution, + # mainly to avoid using EPMD. This works well in practice, + # but in tests Swarm becomes incredibly noisy and causes + # lots of timing issues and introduces lots of flakiness. + # So in tests we use :global instead. + document_store_clustering: :global diff --git a/apps/forge/lib/forge/document/store.ex b/apps/forge/lib/forge/document/store.ex index 51d9b703..19e5f1bf 100644 --- a/apps/forge/lib/forge/document/store.ex +++ b/apps/forge/lib/forge/document/store.ex @@ -413,23 +413,10 @@ defmodule Forge.Document.Store do end end - # NOTE(dorgan): In dev/prod we use Swarm for distribution, - # mainly to avoid using EPMD. This works well in practice, - # but in tests Swarm becomes incredibly noisy and causes - # lots of timing issues and introduces lots of flakiness. - # So in tests we use :global instead. - if Mix.env() == :test do - def clustering_method do - :global - end - else - def clustering_method do - :swarm - end - end + @clustering_method Application.compile_env(:forge, :document_store_clustering, :swarm) def name do - {:via, clustering_method(), {__MODULE__, entropy()}} + {:via, @clustering_method, {__MODULE__, entropy()}} end defp entropy_key do