From 002f90eb0912c528228a25ccc2e582e549770637 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 7 Apr 2025 18:33:00 +0000 Subject: [PATCH 01/53] change location of secrets files --- docker-compose-dev.yaml | 1 + docker-compose-prod.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index 4765b09..327e941 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -51,6 +51,7 @@ services: - "./mirror-sync-scheduler/storage:/storage" - "./mirror-sync-scheduler/configs:/mirror/configs:ro" - "./mirror-sync-scheduler/scripts:/mirror/scripts:ro" + - "./mirror-sync-scheduler/secrets:/mirror/secrets:ro" networks: - mirror depends_on: diff --git a/docker-compose-prod.yaml b/docker-compose-prod.yaml index 3e5f490..56dcea3 100644 --- a/docker-compose-prod.yaml +++ b/docker-compose-prod.yaml @@ -71,6 +71,8 @@ services: - "/storage:/storage" - "./configs:/mirror/configs:ro" - "./scripts:/mirror/scripts:ro" + - "./secrets:/mirror/secrets:ro" + networks: - mirror depends_on: From 27b57ec61a7fd61afddbe7e5919ac2937931b13f Mon Sep 17 00:00:00 2001 From: carykees98 Date: Wed, 9 Apr 2025 22:13:20 +0000 Subject: [PATCH 02/53] add secrets directory to gitignore --- .gitignore | 3 ++- mirror-sync-scheduler/.gitignore | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a12064f..893d2bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .vscode build/* -GeoLite2City \ No newline at end of file +GeoLite2City +secrets diff --git a/mirror-sync-scheduler/.gitignore b/mirror-sync-scheduler/.gitignore index 83ae538..f407a01 100644 --- a/mirror-sync-scheduler/.gitignore +++ b/mirror-sync-scheduler/.gitignore @@ -2,3 +2,4 @@ build *.secret .idea +secrets From 57a4b92ff799e194abf87c951212e5c3c9054557 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 01:45:11 +0000 Subject: [PATCH 03/53] fixed typo in JobManager --- mirror-sync-scheduler/src/JobManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index d6049ca..4074dcd 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -229,7 +229,7 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> auto JobManager::interrupt_job(const ::pid_t processID) -> void { // Interrupt child processes recursively. Starts with the grandest child and - // works its way back up to the direct decendant of the sync scheduler + // works its way back up to the direct descendant of the sync scheduler // // Base case: process with no children. `get_child_process_ids` will be an // empty collection meaning nothing to iterate over From 5177906aaac1ad0a47fd7584c83acd05adceb444 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 01:46:24 +0000 Subject: [PATCH 04/53] updating SyncDetails to work with new mirrors.json schema --- .../mirror/sync_scheduler/SyncDetails.hpp | 15 ++- mirror-sync-scheduler/src/SyncDetails.cpp | 108 ++++++++++++------ 2 files changed, 81 insertions(+), 42 deletions(-) diff --git a/mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp b/mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp index 3e0fae5..133634c 100644 --- a/mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp +++ b/mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp @@ -40,7 +40,7 @@ class SyncDetails } [[nodiscard]] - auto get_commands() const -> std::vector + auto get_commands() const -> std::vector> { return m_Commands; } @@ -48,18 +48,23 @@ class SyncDetails private: // Methods [[nodiscard]] static auto compose_rsync_commands(const nlohmann::json& rsyncConfig) + -> std::vector>; + + static auto handle_rsync_options_array(const nlohmann::json& optionsArray) + -> std::vector>; + static auto + handle_rsync_options_strings(const nlohmann::json& optionsString) -> std::vector; auto handle_sync_config(const nlohmann::json& project) -> void; auto handle_rsync_config(const nlohmann::json& project) -> void; - auto handle_script_config(const nlohmann::json& project) -> void; private: // Members - std::size_t m_SyncsPerDay; - std::optional m_PasswordFile; - std::vector m_Commands; + std::size_t m_SyncsPerDay; + std::optional m_PasswordFile; + std::vector> m_Commands; }; struct static_project_exception : std::runtime_error diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index 34fed04..9ce3f95 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -25,10 +25,12 @@ SyncDetails::SyncDetails(const nlohmann::json& project) { if (project.contains("static")) { - throw static_project_exception(std::format( - "Project '{}' uses a static sync. Project not created!", - project.value("name", "") - )); + throw static_project_exception( + std::format( + "Project '{}' uses a static sync. Project not created!", + project.value("name", "") + ) + ); } const bool isRsyncOrScript @@ -36,19 +38,21 @@ SyncDetails::SyncDetails(const nlohmann::json& project) if (!isRsyncOrScript) { - throw std::runtime_error(std::format( - "Project '{}' is missing a sync type!", - project.value("name", "") - )); + throw std::runtime_error( + std::format( + "Project '{}' is missing a sync type!", + project.value("name", "") + ) + ); } SyncDetails::handle_sync_config(project); } auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) - -> std::vector + -> std::vector> { - std::vector commands; + std::vector> commands; using namespace std::string_view_literals; constexpr auto RSYNC_EXECUTABLE = "/usr/bin/rsync"sv; @@ -58,44 +62,72 @@ auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) const std::string src = rsyncConfig.value("src", ""); const std::string dest = rsyncConfig.value("dest", ""); - for (const std::string options : rsyncConfig.at("options")) + for (const nlohmann::json& options : rsyncConfig.at("options")) { if (options.empty()) { - throw std::runtime_error(std::format( - "Project {} contains invalid rsync config options", - rsyncConfig.value("name", "") - )); + throw std::runtime_error( + std::format( + "Project {} contains invalid rsync config options", + rsyncConfig.value("name", "") + ) + ); } - if (!user.empty()) + if (options.is_array()) { - commands.emplace_back(std::format( - "{} {} {}@{}::{} {}", - RSYNC_EXECUTABLE, - options, - user, - host, - src, - dest - )); + commands = handle_rsync_options_array(options); } - else + else if (options.is_string()) { - commands.emplace_back(std::format( - "{} {} {}::{} {}", - RSYNC_EXECUTABLE, - options, - host, - src, - dest - )); + commands.emplace_back(handle_rsync_options_strings(options)); + } + + for (auto& command : commands) + { + if (!user.empty()) + { + command.emplace_back(std::format("{}@{}::{}", user, host, src)); + } + else + { + command.emplace_back(std::format("{}::{}", host, src)); + } + + command.emplace_back(dest); } } return commands; } +auto SyncDetails::handle_rsync_options_array(const nlohmann::json& optionsArray) + -> std::vector> +{ + std::vector> toReturn; + + for (const nlohmann::json& optionsStrings : optionsArray) + { + toReturn.emplace_back(handle_rsync_options_strings(optionsStrings)); + } + + return toReturn; +} + +auto SyncDetails::handle_rsync_options_strings( + const nlohmann::json& optionsStrings +) -> std::vector +{ + std::vector toReturn = { "/usr/bin/rsync" }; + + for (const std::string option : optionsStrings) + { + toReturn.emplace_back(option); + } + + return toReturn; +} + auto SyncDetails::handle_sync_config(const nlohmann::json& project) -> void { if (project.contains("rsync")) @@ -130,15 +162,17 @@ auto SyncDetails::handle_script_config(const nlohmann::json& project) -> void { m_SyncsPerDay = project.at("script").at("syncs_per_day").get(); - std::string command = project.at("script").at("command"); + std::vector toReturn = { "/usr/bin/sh", "-c" }; + toReturn.emplace_back(project.at("script").at("command")); + const std::vector arguments = project.at("script").value("arguments", std::vector {}); for (const std::string& arg : arguments) { - command = std::format("{} {}", command, arg); + toReturn.emplace_back(arg); } - m_Commands.emplace_back(command); + m_Commands.emplace_back(toReturn); } } // namespace mirror::sync_scheduler From ca303ea2c8bba3750c044064f0a97ca8c8114ed0 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 01:52:55 +0000 Subject: [PATCH 05/53] updated format for SyncScheduer.cpp --- mirror-sync-scheduler/src/SyncScheduler.cpp | 34 ++++++++++++--------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/mirror-sync-scheduler/src/SyncScheduler.cpp b/mirror-sync-scheduler/src/SyncScheduler.cpp index e6ed6d7..0c8b5e9 100644 --- a/mirror-sync-scheduler/src/SyncScheduler.cpp +++ b/mirror-sync-scheduler/src/SyncScheduler.cpp @@ -40,10 +40,12 @@ namespace mirror::sync_scheduler { SyncScheduler::SyncScheduler() try // Function try block my beloved - : m_ProjectCatalogue(SyncScheduler::generate_project_catalogue( - SyncScheduler::load_json_config("configs/mirrors.json") - .value("mirrors", nlohmann::json()) - )), + : m_ProjectCatalogue( + SyncScheduler::generate_project_catalogue( + SyncScheduler::load_json_config("configs/mirrors.json") + .value("mirrors", nlohmann::json()) + ) + ), m_Schedule(m_ProjectCatalogue), m_DryRun(false) { @@ -90,12 +92,14 @@ auto SyncScheduler::load_json_config(const std::filesystem::path& file) { std::string errorMessage(BUFSIZ, '\0'); - throw std::runtime_error(std::format( - "Failed to load config file {}! OS Error: {}", - file.filename().string(), - // NOLINTNEXTLINE(*-include-cleaner) - ::strerror_r(errno, errorMessage.data(), errorMessage.size()) - )); + throw std::runtime_error( + std::format( + "Failed to load config file {}! OS Error: {}", + file.filename().string(), + // NOLINTNEXTLINE(*-include-cleaner) + ::strerror_r(errno, errorMessage.data(), errorMessage.size()) + ) + ); } return nlohmann::json::parse(mirrorsConfigFile); @@ -214,10 +218,12 @@ auto SyncScheduler::manual_sync_loop() -> void spdlog::error("Manual sync for {} failed", projectName); socket.send( - zmq::message_t(std::format( - "FAILURE: Failed to start sync for {}", - projectName - )), + zmq::message_t( + std::format( + "FAILURE: Failed to start sync for {}", + projectName + ) + ), zmq::send_flags::none ); continue; From a0601c4cb08b891f2e43a5b458a502cdf94c5944 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 02:24:44 +0000 Subject: [PATCH 06/53] updated JobManager to work with mirrors.json change --- .../mirror/sync_scheduler/JobManager.hpp | 2 +- mirror-sync-scheduler/src/JobManager.cpp | 22 +++++++++---------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp index 8c8d814..874942c 100644 --- a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp +++ b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp @@ -38,7 +38,7 @@ class JobManager public: // Methods auto start_job( const std::string& jobName, - std::string command, + std::vector command, const std::filesystem::path& passwordFile ) -> bool; diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index 4074dcd..cb2dc6b 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -384,7 +384,7 @@ auto JobManager::deregister_jobs(const std::vector<::pid_t>& completedJobs) // NOLINTBEGIN(*-easily-swappable-parameters) auto JobManager::start_job( const std::string& jobName, - std::string command, + std::vector command, const std::filesystem::path& passwordFile ) -> bool // NOLINTEND(*-easily-swappable-parameters) @@ -469,17 +469,15 @@ auto JobManager::start_job( ); } - //! ::strdup allocates memory with ::malloc. Typically this memory - //! should be ::free'd. However, argv is supposed to have the same - //! lifetime as the process it belongs to, therefore the memory should - //! never be freed and we do not need to maintain a copy of the pointer - //! to free it at a later time - // NOLINTBEGIN(*-include-cleaner) - const std::array argv { ::strdup("/bin/sh"), - ::strdup("-c"), - ::strdup(command.data()), - nullptr }; - // NOLINTEND(*-include-cleaner) + std::vector argv; + argv.resize(command.size()); + + std::transform( + std::begin(command), + std::end(command), + std::begin(argv), + [](std::string& str) { return std::data(str); } + ); spdlog::debug("Setting process group ID"); ::setpgid(0, 0); From bed3549f3518ac4320ae8523150accbc1d64c66b Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 02:40:03 +0000 Subject: [PATCH 07/53] trying to figure out what is causing an error --- mirror-sync-scheduler/src/SyncDetails.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index 9ce3f95..6840f31 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -17,6 +17,7 @@ // Third Party Includes #include +#include namespace mirror::sync_scheduler { @@ -120,9 +121,20 @@ auto SyncDetails::handle_rsync_options_strings( { std::vector toReturn = { "/usr/bin/rsync" }; - for (const std::string option : optionsStrings) + try { - toReturn.emplace_back(option); + for (const std::string option : optionsStrings) + { + toReturn.emplace_back(option); + } + } + catch (nlohmann::json::exception& jsone) + { + spdlog::error( + "Failed to handle options strings {}. Error: {}", + optionsStrings.dump(), + jsone.what() + ); } return toReturn; From 26f1bc33127e37cb3ab7c933aaafd25aaf17e159 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 02:52:20 +0000 Subject: [PATCH 08/53] more debug --- mirror-sync-scheduler/src/SyncDetails.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index 6840f31..2509c7e 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -125,6 +125,7 @@ auto SyncDetails::handle_rsync_options_strings( { for (const std::string option : optionsStrings) { + spdlog::trace("Option: {}", option); toReturn.emplace_back(option); } } From 9c9b699e29063569ffdc3b840170ce453796d74e Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 03:01:17 +0000 Subject: [PATCH 09/53] debug again --- mirror-sync-scheduler/src/SyncDetails.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index 2509c7e..3b0595a 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -18,6 +18,8 @@ // Third Party Includes #include #include +// NOLINTNEXTLINE(misc-include-cleaner) Required to print a range in a log +#include namespace mirror::sync_scheduler { @@ -125,7 +127,6 @@ auto SyncDetails::handle_rsync_options_strings( { for (const std::string option : optionsStrings) { - spdlog::trace("Option: {}", option); toReturn.emplace_back(option); } } @@ -138,6 +139,8 @@ auto SyncDetails::handle_rsync_options_strings( ); } + spdlog::trace("Options: {{ {} }}", fmt::join(toReturn, ", ")); + return toReturn; } From b373b8f8e4a96da906da57a713ba9b36dcf33aa5 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 03:18:57 +0000 Subject: [PATCH 10/53] maybe the fix? --- .../mirror/sync_scheduler/SyncDetails.hpp | 3 -- mirror-sync-scheduler/src/SyncDetails.cpp | 42 +++++-------------- 2 files changed, 10 insertions(+), 35 deletions(-) diff --git a/mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp b/mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp index 133634c..06916cb 100644 --- a/mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp +++ b/mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp @@ -51,9 +51,6 @@ class SyncDetails -> std::vector>; static auto handle_rsync_options_array(const nlohmann::json& optionsArray) - -> std::vector>; - static auto - handle_rsync_options_strings(const nlohmann::json& optionsString) -> std::vector; auto handle_sync_config(const nlohmann::json& project) -> void; diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index 3b0595a..70cd3e5 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -79,11 +79,16 @@ auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) if (options.is_array()) { - commands = handle_rsync_options_array(options); + commands.emplace_back(handle_rsync_options_array(options)); } else if (options.is_string()) { - commands.emplace_back(handle_rsync_options_strings(options)); + if (commands.empty()) + { + commands.resize(1); + } + + commands.front().emplace_back(options); } for (auto& command : commands) @@ -105,41 +110,14 @@ auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) } auto SyncDetails::handle_rsync_options_array(const nlohmann::json& optionsArray) - -> std::vector> -{ - std::vector> toReturn; - - for (const nlohmann::json& optionsStrings : optionsArray) - { - toReturn.emplace_back(handle_rsync_options_strings(optionsStrings)); - } - - return toReturn; -} - -auto SyncDetails::handle_rsync_options_strings( - const nlohmann::json& optionsStrings -) -> std::vector + -> std::vector { std::vector toReturn = { "/usr/bin/rsync" }; - try + for (const std::string option : optionsArray) { - for (const std::string option : optionsStrings) - { - toReturn.emplace_back(option); - } + toReturn.emplace_back(option); } - catch (nlohmann::json::exception& jsone) - { - spdlog::error( - "Failed to handle options strings {}. Error: {}", - optionsStrings.dump(), - jsone.what() - ); - } - - spdlog::trace("Options: {{ {} }}", fmt::join(toReturn, ", ")); return toReturn; } From 3989f30208ca0d9086fe2e347f268af6e086633b Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 03:24:42 +0000 Subject: [PATCH 11/53] more debug (wooo) --- mirror-sync-scheduler/src/SyncDetails.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index 70cd3e5..a7d92dd 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -119,6 +119,8 @@ auto SyncDetails::handle_rsync_options_array(const nlohmann::json& optionsArray) toReturn.emplace_back(option); } + spdlog::trace("Options: {{ {} }}", fmt::join(toReturn, ", ")); + return toReturn; } From 784e5f6c884d85688e91e44677549cf8746e22d4 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 03:33:26 +0000 Subject: [PATCH 12/53] moved command logging --- mirror-sync-scheduler/src/SyncDetails.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index a7d92dd..92d32fd 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -103,6 +103,8 @@ auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) } command.emplace_back(dest); + + spdlog::trace("Options: {{ {} }}", fmt::join(command, ", ")); } } @@ -119,8 +121,6 @@ auto SyncDetails::handle_rsync_options_array(const nlohmann::json& optionsArray) toReturn.emplace_back(option); } - spdlog::trace("Options: {{ {} }}", fmt::join(toReturn, ", ")); - return toReturn; } From 5d6ce6b995c00de9790a569429621f1fd4781e70 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 21:30:00 +0000 Subject: [PATCH 13/53] debug and formatting update --- mirror-sync-scheduler/src/SyncDetails.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index 92d32fd..7a650fa 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -77,9 +77,13 @@ auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) ); } + spdlog::trace("Options: {{ {} }}", fmt::join(options, ", ")); + if (options.is_array()) { - commands.emplace_back(handle_rsync_options_array(options)); + commands.emplace_back( + SyncDetails::handle_rsync_options_array(options) + ); } else if (options.is_string()) { @@ -103,8 +107,6 @@ auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) } command.emplace_back(dest); - - spdlog::trace("Options: {{ {} }}", fmt::join(command, ", ")); } } From e15aa75f044230de21b1d3f96f0de5e3c79f4a04 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 21:32:34 +0000 Subject: [PATCH 14/53] libfmt cant format json, using dump instead --- mirror-sync-scheduler/src/SyncDetails.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index 7a650fa..b31ea99 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -77,7 +77,7 @@ auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) ); } - spdlog::trace("Options: {{ {} }}", fmt::join(options, ", ")); + spdlog::trace("Options: {}", options.dump()); if (options.is_array()) { From 48de63f84728376ab361609ff9bbac0ffa6d2ad3 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 21:35:33 +0000 Subject: [PATCH 15/53] adding host, source, and dest shouldn't have been a nested loop --- mirror-sync-scheduler/src/SyncDetails.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index b31ea99..f668272 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -94,20 +94,20 @@ auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) commands.front().emplace_back(options); } + } - for (auto& command : commands) + for (auto& command : commands) + { + if (!user.empty()) { - if (!user.empty()) - { - command.emplace_back(std::format("{}@{}::{}", user, host, src)); - } - else - { - command.emplace_back(std::format("{}::{}", host, src)); - } - - command.emplace_back(dest); + command.emplace_back(std::format("{}@{}::{}", user, host, src)); } + else + { + command.emplace_back(std::format("{}::{}", host, src)); + } + + command.emplace_back(dest); } return commands; From e7425fe1c727a3eadd6de63108cf312f4af638d8 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 21:38:46 +0000 Subject: [PATCH 16/53] debug print to see if fix worked --- mirror-sync-scheduler/src/SyncDetails.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index f668272..0ecf43b 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -90,6 +90,8 @@ auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) if (commands.empty()) { commands.resize(1); + + commands.front().emplace_back("/usr/bin/rsync"); } commands.front().emplace_back(options); @@ -108,6 +110,8 @@ auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) } command.emplace_back(dest); + + spdlog::trace("Options: {{ {} }}", fmt::join(command, ", ")); } return commands; From 20a49bb99c35db7238c2f212c6f9c6c3eb879a94 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 22:21:09 +0000 Subject: [PATCH 17/53] log stderr for failing rsync processes --- docker-compose-dev.yaml | 1 + docker-compose-prod.yaml | 1 + .../mirror/sync_scheduler/JobManager.hpp | 2 + mirror-sync-scheduler/src/JobManager.cpp | 43 ++++++++++++++++++- 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index 327e941..d86732f 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -49,6 +49,7 @@ services: - 9281 volumes: - "./mirror-sync-scheduler/storage:/storage" + - "./mirror-sync-scheduler/error-logs:/mirror/error-logs" - "./mirror-sync-scheduler/configs:/mirror/configs:ro" - "./mirror-sync-scheduler/scripts:/mirror/scripts:ro" - "./mirror-sync-scheduler/secrets:/mirror/secrets:ro" diff --git a/docker-compose-prod.yaml b/docker-compose-prod.yaml index 56dcea3..1e48631 100644 --- a/docker-compose-prod.yaml +++ b/docker-compose-prod.yaml @@ -69,6 +69,7 @@ services: - configs/sync-scheduler.env volumes: - "/storage:/storage" + - "./sync-error-logs:/mirror/error-logs" - "./configs:/mirror/configs:ro" - "./scripts:/mirror/scripts:ro" - "./secrets:/mirror/secrets:ro" diff --git a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp index 874942c..75c1964 100644 --- a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp +++ b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp @@ -53,6 +53,8 @@ class JobManager auto kill_all_jobs() -> void; auto reap_processes() -> std::vector<::pid_t>; auto deregister_jobs(const std::vector<::pid_t>& completedJobs) -> void; + auto JobManager::write_stderr_to_file(const ::pid_t processID) + -> std::string; auto process_reaper(const std::stop_token& stopToken) -> void; private: // Static Methods diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index cb2dc6b..a93b01a 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -8,6 +8,7 @@ #include // System Includes +#include #include #include #include @@ -26,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -196,9 +198,14 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> } else if (isKnownJob && exitStatus != EXIT_SUCCESS) { + const auto logFileName + = this->write_stderr_to_file(childProcessID); + spdlog::warn( - "Project {} failed to sync! Exit code: {} (pid: {})", + "Project {} failed to sync! The contents of STDERR for the " + "process have been written to {}! Exit code: {} (pid: {})", m_ActiveJobs.at(childProcessID).jobName, + logFileName, exitStatus, childProcessID ); @@ -225,6 +232,40 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> return completedJobs; } +auto JobManager::write_stderr_to_file(const ::pid_t processID) -> std::string +{ + static std::random_device randomDevice; + static std::mt19937 randomGenerator(randomDevice()); + static std::uniform_int_distribution distribution(1, 10000); + + const auto logNumber = distribution(randomGenerator); + const auto errorLogPath = std::filesystem::relative("error-logs"); + + auto logFileName = std::format( + "{}-{}-stderr.log", + m_ActiveJobs.at(processID).jobName, + logNumber + ); + + std::ofstream logFile(errorLogPath / logFileName); + + int bytesAvailable = 0; + ::ioctl(m_ActiveJobs.at(processID).stderrPipe, FIONREAD, &bytesAvailable); + + std::string stderrContents; + stderrContents.resize(bytesAvailable + 1); + + ::read( + m_ActiveJobs.at(processID).stderrPipe, + stderrContents.data(), + bytesAvailable + ); + + logFile << stderrContents; + + return logFileName; +} + // NOLINTNEXTLINE(*-no-recursion) auto JobManager::interrupt_job(const ::pid_t processID) -> void { From 981b5889b165c9a63e1cff843189be461db2be91 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 22:23:49 +0000 Subject: [PATCH 18/53] close stdout and stderr pipes before removing job from active jobs map --- mirror-sync-scheduler/src/JobManager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index a93b01a..69de8a3 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -418,6 +418,9 @@ auto JobManager::deregister_jobs(const std::vector<::pid_t>& completedJobs) const std::lock_guard jobLock(m_JobMutex); for (const auto& job : completedJobs) { + ::close(m_ActiveJobs.at(job).stderrPipe); + ::close(m_ActiveJobs.at(job).stdoutPipe); + m_ActiveJobs.erase(job); } } From 635c809a3417adbb7d1a4b6c44267f0f7d3a7d53 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 22:35:04 +0000 Subject: [PATCH 19/53] removed extra qualification from write_stderr_to_file --- .../include/mirror/sync_scheduler/JobManager.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp index 75c1964..006c3a8 100644 --- a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp +++ b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp @@ -53,8 +53,7 @@ class JobManager auto kill_all_jobs() -> void; auto reap_processes() -> std::vector<::pid_t>; auto deregister_jobs(const std::vector<::pid_t>& completedJobs) -> void; - auto JobManager::write_stderr_to_file(const ::pid_t processID) - -> std::string; + auto write_stderr_to_file(const ::pid_t processID) -> std::string; auto process_reaper(const std::stop_token& stopToken) -> void; private: // Static Methods From 7e9fbf3d1f2f9d2886aaa5f42259593293866676 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 22:54:16 +0000 Subject: [PATCH 20/53] write stdout and stderr instead of just stderr --- .../mirror/sync_scheduler/JobManager.hpp | 3 +- mirror-sync-scheduler/src/JobManager.cpp | 77 ++++++++++++++----- 2 files changed, 60 insertions(+), 20 deletions(-) diff --git a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp index 006c3a8..dd848cd 100644 --- a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp +++ b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp @@ -53,7 +53,8 @@ class JobManager auto kill_all_jobs() -> void; auto reap_processes() -> std::vector<::pid_t>; auto deregister_jobs(const std::vector<::pid_t>& completedJobs) -> void; - auto write_stderr_to_file(const ::pid_t processID) -> std::string; + auto write_streams_to_file(const ::pid_t processID) + -> std::pair; auto process_reaper(const std::stop_token& stopToken) -> void; private: // Static Methods diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index 69de8a3..fd10245 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include // Third Party Includes @@ -198,14 +199,16 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> } else if (isKnownJob && exitStatus != EXIT_SUCCESS) { - const auto logFileName - = this->write_stderr_to_file(childProcessID); + const auto logFileNames + = this->write_streams_to_file(childProcessID); spdlog::warn( - "Project {} failed to sync! The contents of STDERR for the " - "process have been written to {}! Exit code: {} (pid: {})", + "Project {} failed to sync! The contents of the process' " + "stream have been written to {} and {}! Exit code: {} " + "(pid: {})", m_ActiveJobs.at(childProcessID).jobName, - logFileName, + logFileNames.first, + logFileNames.second, exitStatus, childProcessID ); @@ -232,7 +235,8 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> return completedJobs; } -auto JobManager::write_stderr_to_file(const ::pid_t processID) -> std::string +auto JobManager::write_streams_to_file(const ::pid_t processID) + -> std::pair { static std::random_device randomDevice; static std::mt19937 randomGenerator(randomDevice()); @@ -241,29 +245,64 @@ auto JobManager::write_stderr_to_file(const ::pid_t processID) -> std::string const auto logNumber = distribution(randomGenerator); const auto errorLogPath = std::filesystem::relative("error-logs"); - auto logFileName = std::format( + const auto stdoutLogFileName = std::format( + "{}-{}-stdout.log", + m_ActiveJobs.at(processID).jobName, + logNumber + ); + + const auto stderrLogFileName = std::format( "{}-{}-stderr.log", m_ActiveJobs.at(processID).jobName, logNumber ); - std::ofstream logFile(errorLogPath / logFileName); + int bytesAvailable = 0; + std::string streamContents; - int bytesAvailable = 0; - ::ioctl(m_ActiveJobs.at(processID).stderrPipe, FIONREAD, &bytesAvailable); + // STDOUT + { + // NOLINTNEXTLINE(*-include-cleaner, *-vararg) + ::ioctl( + m_ActiveJobs.at(processID).stdoutPipe, + FIONREAD, + &bytesAvailable + ); - std::string stderrContents; - stderrContents.resize(bytesAvailable + 1); + streamContents.resize(bytesAvailable + 1); - ::read( - m_ActiveJobs.at(processID).stderrPipe, - stderrContents.data(), - bytesAvailable - ); + ::read( + m_ActiveJobs.at(processID).stdoutPipe, + streamContents.data(), + bytesAvailable + ); - logFile << stderrContents; + std::ofstream stdoutLogFile(errorLogPath / stdoutLogFileName); + stdoutLogFile << streamContents; + } + + // STDERR + { + // NOLINTNEXTLINE(*-include-cleaner, *-vararg) + ::ioctl( + m_ActiveJobs.at(processID).stderrPipe, + FIONREAD, + &bytesAvailable + ); + + streamContents.resize(bytesAvailable + 1); + + ::read( + m_ActiveJobs.at(processID).stderrPipe, + streamContents.data(), + bytesAvailable + ); + + std::ofstream stderrLogFile(errorLogPath / stderrLogFileName); + stderrLogFile << streamContents; + } - return logFileName; + return { stdoutLogFileName, stderrLogFileName }; } // NOLINTNEXTLINE(*-no-recursion) From d47be4e8c39a411f93756970ad87d6eae4dad471 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 23:05:23 +0000 Subject: [PATCH 21/53] don't use ioctl, just read into a buffer of size bufsiz --- mirror-sync-scheduler/src/JobManager.cpp | 55 ++++++++++-------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index fd10245..ba927e6 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -8,6 +8,7 @@ #include // System Includes +#include #include #include #include @@ -203,7 +204,8 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> = this->write_streams_to_file(childProcessID); spdlog::warn( - "Project {} failed to sync! The contents of the process' " + "Project {} failed to sync! The contents of the " + "process' " "stream have been written to {} and {}! Exit code: {} " "(pid: {})", m_ActiveJobs.at(childProcessID).jobName, @@ -217,14 +219,16 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> else if (!isKnownJob && exitStatus == EXIT_SUCCESS) { spdlog::info( - "Reaped successful unregistered child process with pid {}", + "Reaped successful unregistered child process with pid " + "{}", childProcessID ); } else { spdlog::warn( - "Reaped unsuccessful unregistered child process with pid " + "Reaped unsuccessful unregistered child process with " + "pid " "{}", childProcessID ); @@ -257,24 +261,16 @@ auto JobManager::write_streams_to_file(const ::pid_t processID) logNumber ); - int bytesAvailable = 0; - std::string streamContents; + static std::string streamContents; + + streamContents.resize(BUFSIZ); // STDOUT { - // NOLINTNEXTLINE(*-include-cleaner, *-vararg) - ::ioctl( - m_ActiveJobs.at(processID).stdoutPipe, - FIONREAD, - &bytesAvailable - ); - - streamContents.resize(bytesAvailable + 1); - ::read( m_ActiveJobs.at(processID).stdoutPipe, streamContents.data(), - bytesAvailable + streamContents.size() ); std::ofstream stdoutLogFile(errorLogPath / stdoutLogFileName); @@ -283,19 +279,10 @@ auto JobManager::write_streams_to_file(const ::pid_t processID) // STDERR { - // NOLINTNEXTLINE(*-include-cleaner, *-vararg) - ::ioctl( - m_ActiveJobs.at(processID).stderrPipe, - FIONREAD, - &bytesAvailable - ); - - streamContents.resize(bytesAvailable + 1); - ::read( m_ActiveJobs.at(processID).stderrPipe, streamContents.data(), - bytesAvailable + streamContents.size() ); std::ofstream stderrLogFile(errorLogPath / stderrLogFileName); @@ -308,11 +295,12 @@ auto JobManager::write_streams_to_file(const ::pid_t processID) // NOLINTNEXTLINE(*-no-recursion) auto JobManager::interrupt_job(const ::pid_t processID) -> void { - // Interrupt child processes recursively. Starts with the grandest child and - // works its way back up to the direct descendant of the sync scheduler + // Interrupt child processes recursively. Starts with the grandest child + // and works its way back up to the direct descendant of the sync + // scheduler // - // Base case: process with no children. `get_child_process_ids` will be an - // empty collection meaning nothing to iterate over + // Base case: process with no children. `get_child_process_ids` will be + // an empty collection meaning nothing to iterate over for (const ::pid_t childProcessID : JobManager::get_child_process_ids()) { JobManager::interrupt_job(childProcessID); @@ -492,7 +480,8 @@ auto JobManager::start_job( std::string errorMessage(BUFSIZ, '\0'); spdlog::warn( - "Failed to create pipe for child stdout while syncing project {}! " + "Failed to create pipe for child stdout while syncing project " + "{}! " "Error message: {}", jobName, // NOLINTNEXTLINE(*-include-cleaner) @@ -507,7 +496,8 @@ auto JobManager::start_job( std::string errorMessage(BUFSIZ, '\0'); spdlog::warn( - "Failed to create pipe for child stderr while syncing project {}! " + "Failed to create pipe for child stderr while syncing project " + "{}! " "Error message: {}", jobName, // NOLINTNEXTLINE(*-include-cleaner) @@ -572,7 +562,8 @@ auto JobManager::start_job( std::string errorMessage(BUFSIZ, '\0'); spdlog::error( - "Call to execv() failed while trying to sync {}! Error message: {}", + "Call to execv() failed while trying to sync {}! Error " + "message: {}", jobName, // NOLINTNEXTLINE(*-include-cleaner) ::strerror_r(errno, errorMessage.data(), errorMessage.size()) From 98dc3fd41f11831325629068c6bec3fb68246d23 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 23:22:44 +0000 Subject: [PATCH 22/53] had been registering closed write end of pipe as read end --- mirror-sync-scheduler/src/JobManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index ba927e6..ffc31ce 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -592,7 +592,7 @@ auto JobManager::start_job( // Close write end of the stderr pipe in the parent process ::close(stderrPipes.at(1)); - this->register_job(jobName, pid, stdoutPipes.at(1), stderrPipes.at(1)); + this->register_job(jobName, pid, stdoutPipes.at(0), stderrPipes.at(0)); return true; } } // namespace mirror::sync_scheduler From 24fe5d425eeea05a91afd3485e7978f04d5f3a35 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 23:34:49 +0000 Subject: [PATCH 23/53] last item in argv has to be nullptr --- mirror-sync-scheduler/src/JobManager.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index ffc31ce..e3a7447 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -277,6 +277,8 @@ auto JobManager::write_streams_to_file(const ::pid_t processID) stdoutLogFile << streamContents; } + streamContents.clear(); + // STDERR { ::read( @@ -543,7 +545,7 @@ auto JobManager::start_job( } std::vector argv; - argv.resize(command.size()); + argv.resize(command.size() + 1); std::transform( std::begin(command), @@ -552,6 +554,9 @@ auto JobManager::start_job( [](std::string& str) { return std::data(str); } ); + // Last item in argv has to be a nullptr; + argv.emplace_back(nullptr); + spdlog::debug("Setting process group ID"); ::setpgid(0, 0); From c6a59cc89d562a39a698456805d526f54b742c0d Mon Sep 17 00:00:00 2001 From: carykees98 Date: Mon, 14 Apr 2025 23:41:39 +0000 Subject: [PATCH 24/53] updated warning for failed sync --- mirror-sync-scheduler/src/JobManager.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index e3a7447..fc72c6d 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -204,10 +204,8 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> = this->write_streams_to_file(childProcessID); spdlog::warn( - "Project {} failed to sync! The contents of the " - "process' " - "stream have been written to {} and {}! Exit code: {} " - "(pid: {})", + "Project {} failed to sync! STDOUT and STDERR been written " + "to {} and {}! Exit code: {} (pid: {})", m_ActiveJobs.at(childProcessID).jobName, logFileNames.first, logFileNames.second, From fe9ae41be2ababee8aa34cfcd808276bc95df9fd Mon Sep 17 00:00:00 2001 From: carykees98 Date: Wed, 16 Apr 2025 08:55:02 +0000 Subject: [PATCH 25/53] resize stream contents string after clearing it --- mirror-sync-scheduler/src/JobManager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index fc72c6d..b6dae34 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -74,7 +74,7 @@ auto JobManager::process_reaper(const std::stop_token& stopToken) -> void return; } - auto completedJobs = reap_processes(); + auto completedJobs = this->reap_processes(); this->deregister_jobs(completedJobs); completedJobs.clear(); @@ -276,6 +276,7 @@ auto JobManager::write_streams_to_file(const ::pid_t processID) } streamContents.clear(); + streamContents.resize(BUFSIZ); // STDERR { From 7068c5f8036adfdaf33a146f4325a9dd51187d22 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Wed, 16 Apr 2025 09:35:48 +0000 Subject: [PATCH 26/53] update compose files --- docker-compose-dev.yaml | 4 ---- docker-compose-prod.yaml | 5 ----- 2 files changed, 9 deletions(-) diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index d86732f..b2a6525 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -55,10 +55,6 @@ services: - "./mirror-sync-scheduler/secrets:/mirror/secrets:ro" networks: - mirror - depends_on: - - log-server - stdin_open: true - tty: true # --- Torrent handler --- # torrent-handler: diff --git a/docker-compose-prod.yaml b/docker-compose-prod.yaml index 1e48631..4743a27 100644 --- a/docker-compose-prod.yaml +++ b/docker-compose-prod.yaml @@ -73,13 +73,8 @@ services: - "./configs:/mirror/configs:ro" - "./scripts:/mirror/scripts:ro" - "./secrets:/mirror/secrets:ro" - networks: - mirror - depends_on: - - log-server - stdin_open: true - tty: true # --- Torrent handler --- # torrent-handler: From 308b28dd81d38d661f9be1715034f645f5f9e471 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Fri, 18 Apr 2025 06:59:46 +0000 Subject: [PATCH 27/53] fixed bug where all syncs would be interrupted if one timed out --- mirror-sync-scheduler/src/JobManager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index b6dae34..83e8631 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -302,7 +302,8 @@ auto JobManager::interrupt_job(const ::pid_t processID) -> void // // Base case: process with no children. `get_child_process_ids` will be // an empty collection meaning nothing to iterate over - for (const ::pid_t childProcessID : JobManager::get_child_process_ids()) + for (const ::pid_t childProcessID : + JobManager::get_child_process_ids(processID)) { JobManager::interrupt_job(childProcessID); } From 0d0d9ea5a2af760565e619eea25532a954886c86 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Thu, 24 Apr 2025 18:54:59 +0000 Subject: [PATCH 28/53] added zsh to sync scheduler docker file to run fedora sync --- mirror-sync-scheduler/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirror-sync-scheduler/Dockerfile b/mirror-sync-scheduler/Dockerfile index 99008de..ef96d08 100644 --- a/mirror-sync-scheduler/Dockerfile +++ b/mirror-sync-scheduler/Dockerfile @@ -14,7 +14,7 @@ RUN cmake --build build --target all # Run FROM ubuntu:24.04 -RUN apt update && apt install -y python3 curl wget libzmq3-dev rsync +RUN apt update && apt install -y python3 curl wget libzmq3-dev rsync zsh WORKDIR /mirror COPY --from=builder /mirror/build/src/sync_scheduler . RUN chmod 744 sync_scheduler From 464e2f3a7bb0e930eee3069dfb2b43e6da6a4e1d Mon Sep 17 00:00:00 2001 From: carykees98 Date: Thu, 24 Apr 2025 18:55:22 +0000 Subject: [PATCH 29/53] updated how stderr and stdout are written to files --- .../mirror/sync_scheduler/JobManager.hpp | 4 + mirror-sync-scheduler/src/JobManager.cpp | 83 ++++++++++++------- 2 files changed, 56 insertions(+), 31 deletions(-) diff --git a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp index dd848cd..f51fcf6 100644 --- a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp +++ b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp @@ -62,6 +62,10 @@ class JobManager -> std::vector<::pid_t>; static auto interrupt_job(const ::pid_t processID) -> void; static auto kill_job(const ::pid_t processID) -> void; + static auto write_stream_to_file( + const std::string& logfileName, + const int pipeFileDescriptor + ) -> bool; private: // Members std::map<::pid_t, SyncJob> m_ActiveJobs; diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index 83e8631..e0224cc 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -9,7 +9,6 @@ // System Includes #include -#include #include #include #include @@ -20,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -192,25 +193,23 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> if (isKnownJob && exitStatus == EXIT_SUCCESS) { spdlog::info( - "Project {} successfully synced! (pid: {})", - m_ActiveJobs.at(childProcessID).jobName, - childProcessID + "Project {} successfully synced!", + m_ActiveJobs.at(childProcessID).jobName ); completedJobs.emplace_back(childProcessID); } else if (isKnownJob && exitStatus != EXIT_SUCCESS) { - const auto logFileNames + const auto [stdoutLogfile, stderrLogfile] = this->write_streams_to_file(childProcessID); spdlog::warn( - "Project {} failed to sync! STDOUT and STDERR been written " - "to {} and {}! Exit code: {} (pid: {})", + "Project {} failed to sync! Attempted to write sync output " + "to {} and {}. Exit code: {}", m_ActiveJobs.at(childProcessID).jobName, - logFileNames.first, - logFileNames.second, - exitStatus, - childProcessID + stdoutLogfile, + stderrLogfile, + exitStatus ); completedJobs.emplace_back(childProcessID); } @@ -244,8 +243,7 @@ auto JobManager::write_streams_to_file(const ::pid_t processID) static std::mt19937 randomGenerator(randomDevice()); static std::uniform_int_distribution distribution(1, 10000); - const auto logNumber = distribution(randomGenerator); - const auto errorLogPath = std::filesystem::relative("error-logs"); + const auto logNumber = distribution(randomGenerator); const auto stdoutLogFileName = std::format( "{}-{}-stdout.log", @@ -259,38 +257,61 @@ auto JobManager::write_streams_to_file(const ::pid_t processID) logNumber ); - static std::string streamContents; + const bool stdoutSuccess = write_stream_to_file( + stdoutLogFileName, + m_ActiveJobs.at(processID).stdoutPipe + ); - streamContents.resize(BUFSIZ); + const bool stderrSuccess = write_stream_to_file( + stderrLogFileName, + m_ActiveJobs.at(processID).stderrPipe + ); + + return { (stdoutSuccess ? stdoutLogFileName : ""), + (stderrSuccess ? stderrLogFileName : "") }; +} + +auto JobManager::write_stream_to_file( + const std::string& logfileName, + const int pipeFileDescriptor +) -> bool +{ + const auto errorLogPath = std::filesystem::relative("error-logs"); + std::ofstream logfileStream(errorLogPath / logfileName); - // STDOUT + if (!logfileStream.good()) { - ::read( - m_ActiveJobs.at(processID).stdoutPipe, - streamContents.data(), - streamContents.size() - ); + spdlog::error("Failed to open log file {}!", logfileName); - std::ofstream stdoutLogFile(errorLogPath / stdoutLogFileName); - stdoutLogFile << streamContents; + return false; } - streamContents.clear(); + std::string streamContents; streamContents.resize(BUFSIZ); - // STDERR + std::ptrdiff_t bytesRead = 1; + + while (bytesRead != 0 && bytesRead != -1) { - ::read( - m_ActiveJobs.at(processID).stderrPipe, + bytesRead = ::read( + pipeFileDescriptor, streamContents.data(), streamContents.size() ); - std::ofstream stderrLogFile(errorLogPath / stderrLogFileName); - stderrLogFile << streamContents; + // The buffer was filled + if (bytesRead == streamContents.size()) + { + logfileStream << streamContents; + } + else // Fewer characters were read than the buffer could fit + { + streamContents.resize(bytesRead); + logfileStream << streamContents; + } } - return { stdoutLogFileName, stderrLogFileName }; + return true; } // NOLINTNEXTLINE(*-no-recursion) @@ -558,7 +579,7 @@ auto JobManager::start_job( argv.emplace_back(nullptr); spdlog::debug("Setting process group ID"); - ::setpgid(0, 0); + //::setpgid(0, 0); spdlog::trace("Calling exec"); ::execv(argv.at(0), argv.data()); From 0fa2877b37c25f1b8237db6650128068ec487e96 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Fri, 25 Apr 2025 03:55:26 +0000 Subject: [PATCH 30/53] added ability to manual sync all projects --- mirror-sync-scheduler/src/SyncScheduler.cpp | 38 +++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/mirror-sync-scheduler/src/SyncScheduler.cpp b/mirror-sync-scheduler/src/SyncScheduler.cpp index 0c8b5e9..3a593f0 100644 --- a/mirror-sync-scheduler/src/SyncScheduler.cpp +++ b/mirror-sync-scheduler/src/SyncScheduler.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -179,6 +180,8 @@ auto SyncScheduler::start_sync(const std::string& projectName) -> bool auto SyncScheduler::manual_sync_loop() -> void { + using namespace std::string_view_literals; + zmq::context_t socketContext {}; zmq::socket_t socket { socketContext, zmq::socket_type::rep }; @@ -228,6 +231,41 @@ auto SyncScheduler::manual_sync_loop() -> void ); continue; } + else if (projectName == "all_projects") + { + bool allSyncsStarted = true; + + for (const auto& [project, _] : m_ProjectCatalogue) + { + const bool syncStarted = SyncScheduler::start_sync(projectName); + allSyncsStarted = allSyncsStarted && syncStarted; + + if (!syncStarted) + { + spdlog::error( + "Failed to start manual sync for {}", + projectName + ); + } + } + + if (allSyncsStarted) + { + socket.send( + zmq::message_t("SUCCESS: started syncing all projects"sv), + zmq::send_flags::none + ); + continue; + } + + socket.send( + zmq::message_t( + "FAILURE: Failed to start sync for some projects"sv + ), + zmq::send_flags::none + ); + continue; + } spdlog::error("Project {} not found!", projectName); socket.send( From 2a52db2f405cdef27a5503e555c69eb23f41ad1b Mon Sep 17 00:00:00 2001 From: carykees98 Date: Fri, 25 Apr 2025 04:00:26 +0000 Subject: [PATCH 31/53] passed the wrong variable to start_sync when syncing all projects --- mirror-sync-scheduler/src/SyncScheduler.cpp | 30 ++++++++++++--------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/mirror-sync-scheduler/src/SyncScheduler.cpp b/mirror-sync-scheduler/src/SyncScheduler.cpp index 3a593f0..c7ab81f 100644 --- a/mirror-sync-scheduler/src/SyncScheduler.cpp +++ b/mirror-sync-scheduler/src/SyncScheduler.cpp @@ -202,40 +202,43 @@ auto SyncScheduler::manual_sync_loop() -> void [[maybe_unused]] auto result = socket.recv(syncRequest, zmq::recv_flags::none); - const std::string projectName = syncRequest.to_string(); + const std::string requestedProjectName = syncRequest.to_string(); - spdlog::info("Manual sync requested for {}", projectName); + spdlog::info("Manual sync requested for {}", requestedProjectName); - if (m_ProjectCatalogue.contains(projectName)) + if (m_ProjectCatalogue.contains(requestedProjectName)) { - if (SyncScheduler::start_sync(projectName)) + if (SyncScheduler::start_sync(requestedProjectName)) { socket.send( zmq::message_t( - std::format("SUCCESS: started sync for {}", projectName) + std::format( + "SUCCESS: started sync for {}", + requestedProjectName + ) ), zmq::send_flags::none ); continue; } - spdlog::error("Manual sync for {} failed", projectName); + spdlog::error("Manual sync for {} failed", requestedProjectName); socket.send( zmq::message_t( std::format( "FAILURE: Failed to start sync for {}", - projectName + requestedProjectName ) ), zmq::send_flags::none ); continue; } - else if (projectName == "all_projects") + else if (requestedProjectName == "all_projects") { bool allSyncsStarted = true; - for (const auto& [project, _] : m_ProjectCatalogue) + for (const auto& [projectName, _] : m_ProjectCatalogue) { const bool syncStarted = SyncScheduler::start_sync(projectName); allSyncsStarted = allSyncsStarted && syncStarted; @@ -244,7 +247,7 @@ auto SyncScheduler::manual_sync_loop() -> void { spdlog::error( "Failed to start manual sync for {}", - projectName + requestedProjectName ); } } @@ -267,10 +270,13 @@ auto SyncScheduler::manual_sync_loop() -> void continue; } - spdlog::error("Project {} not found!", projectName); + spdlog::error("Project {} not found!", requestedProjectName); socket.send( zmq::message_t( - std::format("FAILURE: Project {} not found!", projectName) + std::format( + "FAILURE: Project {} not found!", + requestedProjectName + ) ), zmq::send_flags::none ); From 048af47e257f59e84c1a5afb1c7f5ffbfd6902a9 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Fri, 25 Apr 2025 05:21:55 +0000 Subject: [PATCH 32/53] fixed script commands not being properly composed --- mirror-sync-scheduler/src/SyncDetails.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index 0ecf43b..ed582c1 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -111,7 +111,7 @@ auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) command.emplace_back(dest); - spdlog::trace("Options: {{ {} }}", fmt::join(command, ", ")); + spdlog::trace("Sync Command: {{ {} }}", fmt::join(command, ", ")); } return commands; @@ -162,19 +162,21 @@ auto SyncDetails::handle_rsync_config(const nlohmann::json& project) -> void auto SyncDetails::handle_script_config(const nlohmann::json& project) -> void { - m_SyncsPerDay = project.at("script").at("syncs_per_day").get(); + const auto& scriptBlock = project.at("script"); - std::vector toReturn = { "/usr/bin/sh", "-c" }; - toReturn.emplace_back(project.at("script").at("command")); + m_SyncsPerDay = scriptBlock.at("syncs_per_day").get(); - const std::vector arguments - = project.at("script").value("arguments", std::vector {}); + std::vector toReturn = { "/usr/bin/bash", "-c" }; - for (const std::string& arg : arguments) + toReturn.emplace_back(scriptBlock.at("command")); + + for (const auto& arg : scriptBlock.at("arguments")) { - toReturn.emplace_back(arg); + toReturn.back() = std::format("{} {}", toReturn.back(), arg.dump()); } + spdlog::trace("Sync Command: {{ {} }}", fmt::join(toReturn, ", ")); + m_Commands.emplace_back(toReturn); } } // namespace mirror::sync_scheduler From 9bfa6646d6559104b8e68c2adca864767d115d0b Mon Sep 17 00:00:00 2001 From: carykees98 Date: Fri, 25 Apr 2025 05:42:02 +0000 Subject: [PATCH 33/53] tty true for colored logs --- docker-compose-dev.yaml | 2 +- docker-compose-prod.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index b2a6525..788d06b 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -55,7 +55,7 @@ services: - "./mirror-sync-scheduler/secrets:/mirror/secrets:ro" networks: - mirror - + tty: true # --- Torrent handler --- # torrent-handler: # build: ./mirror-torrent-handler diff --git a/docker-compose-prod.yaml b/docker-compose-prod.yaml index 4743a27..6db580c 100644 --- a/docker-compose-prod.yaml +++ b/docker-compose-prod.yaml @@ -75,6 +75,7 @@ services: - "./secrets:/mirror/secrets:ro" networks: - mirror + tty: true # --- Torrent handler --- # torrent-handler: From 52c3d87dae1e7bc56612bf70c101c6ca7f628a8b Mon Sep 17 00:00:00 2001 From: carykees98 Date: Fri, 16 May 2025 23:37:49 +0000 Subject: [PATCH 34/53] log project name instead of "all_projects" when failing to start a manual all project sync --- mirror-sync-scheduler/src/SyncScheduler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirror-sync-scheduler/src/SyncScheduler.cpp b/mirror-sync-scheduler/src/SyncScheduler.cpp index c7ab81f..a0e97e5 100644 --- a/mirror-sync-scheduler/src/SyncScheduler.cpp +++ b/mirror-sync-scheduler/src/SyncScheduler.cpp @@ -247,7 +247,7 @@ auto SyncScheduler::manual_sync_loop() -> void { spdlog::error( "Failed to start manual sync for {}", - requestedProjectName + projectName ); } } From d7a847def544fbaa3e9842b50af8e8e8d155643b Mon Sep 17 00:00:00 2001 From: carykees98 Date: Sat, 17 May 2025 01:23:44 +0000 Subject: [PATCH 35/53] updated formatting for script command string --- mirror-sync-scheduler/src/SyncDetails.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index ed582c1..7123fc6 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -170,11 +170,15 @@ auto SyncDetails::handle_script_config(const nlohmann::json& project) -> void toReturn.emplace_back(scriptBlock.at("command")); + toReturn.back() = "\""; + for (const auto& arg : scriptBlock.at("arguments")) { toReturn.back() = std::format("{} {}", toReturn.back(), arg.dump()); } + toReturn.back() = "\""; + spdlog::trace("Sync Command: {{ {} }}", fmt::join(toReturn, ", ")); m_Commands.emplace_back(toReturn); From 33604fa2e754dee13cad2453368ba15f38c5167c Mon Sep 17 00:00:00 2001 From: carykees98 Date: Sat, 17 May 2025 01:32:18 +0000 Subject: [PATCH 36/53] treat argument as string instead of auto --- mirror-sync-scheduler/src/SyncDetails.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index 7123fc6..80bea04 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -172,9 +172,9 @@ auto SyncDetails::handle_script_config(const nlohmann::json& project) -> void toReturn.back() = "\""; - for (const auto& arg : scriptBlock.at("arguments")) + for (const std::string arg : scriptBlock.at("arguments")) { - toReturn.back() = std::format("{} {}", toReturn.back(), arg.dump()); + toReturn.back() = std::format("{} '{}'", toReturn.back(), arg); } toReturn.back() = "\""; From e53ba35a5bd3effa68e1e62c61db0da20cd27d1e Mon Sep 17 00:00:00 2001 From: carykees98 Date: Sat, 17 May 2025 01:55:29 +0000 Subject: [PATCH 37/53] more formatting --- mirror-sync-scheduler/src/SyncDetails.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index 80bea04..52435b4 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -168,16 +168,16 @@ auto SyncDetails::handle_script_config(const nlohmann::json& project) -> void std::vector toReturn = { "/usr/bin/bash", "-c" }; - toReturn.emplace_back(scriptBlock.at("command")); + toReturn.emplace_back("\""); - toReturn.back() = "\""; + toReturn.back() += scriptBlock.at("command"); for (const std::string arg : scriptBlock.at("arguments")) { toReturn.back() = std::format("{} '{}'", toReturn.back(), arg); } - toReturn.back() = "\""; + toReturn.back() += "\""; spdlog::trace("Sync Command: {{ {} }}", fmt::join(toReturn, ", ")); From a11d3f63300eee957d6770a471be961e3f3b603c Mon Sep 17 00:00:00 2001 From: carykees98 Date: Sat, 17 May 2025 02:18:27 +0000 Subject: [PATCH 38/53] back to what it was? --- mirror-sync-scheduler/src/JobManager.cpp | 3 +-- mirror-sync-scheduler/src/SyncDetails.cpp | 8 ++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index e0224cc..d25be0e 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -172,8 +172,7 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> spdlog::warn( "Project {} has been syncing for at least {} hour{}. " - "Process " - "may be hanging, attempting to send SIGTERM. (pid: {})", + "Process may be hanging, attempting to send SIGTERM. (pid: {})", m_ActiveJobs.at(childProcessID).jobName, JOB_TIMEOUT.count(), // if not one hour, plural diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index 52435b4..c649a39 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -168,17 +168,13 @@ auto SyncDetails::handle_script_config(const nlohmann::json& project) -> void std::vector toReturn = { "/usr/bin/bash", "-c" }; - toReturn.emplace_back("\""); - - toReturn.back() += scriptBlock.at("command"); + toReturn.emplace_back(scriptBlock.at("command")); for (const std::string arg : scriptBlock.at("arguments")) { - toReturn.back() = std::format("{} '{}'", toReturn.back(), arg); + toReturn.back() = std::format("{} {}", toReturn.back(), arg); } - toReturn.back() += "\""; - spdlog::trace("Sync Command: {{ {} }}", fmt::join(toReturn, ", ")); m_Commands.emplace_back(toReturn); From 83383bc9bc478d470ef9e9545625bc902d7edd7e Mon Sep 17 00:00:00 2001 From: carykees98 Date: Sat, 17 May 2025 02:38:54 +0000 Subject: [PATCH 39/53] add shebang to kicad sync script --- scripts/kicad/kicad_sync.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/scripts/kicad/kicad_sync.py b/scripts/kicad/kicad_sync.py index 6916b2c..46d3f78 100644 --- a/scripts/kicad/kicad_sync.py +++ b/scripts/kicad/kicad_sync.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import subprocess from time import sleep import os @@ -8,38 +10,46 @@ # Path of download root (ex. "/storage/kicad/") DOWNLOAD_PREFIX = "/storage/kicad/" + def runCommandWithOutput(args: list[str]) -> str: return subprocess.run(args, stdout=subprocess.PIPE).stdout.decode() + def runCommand(args: list[str]) -> str: subprocess.run(args) + def isFile(path: str): return not (path.endswith("/") or path.endswith("index.html") or path.endswith("list.js") or path.endswith("favicon.ico")) + def isValue(line: str): return not line.startswith("<") + def getNextMarker(xml: str): - lines: list[str] = xml.replace("", "\n").replace("", "\n").splitlines() + lines: list[str] = xml.replace("", "\n").replace( + "", "\n").splitlines() markers: list[str] = list(filter(lambda x: isValue(x), lines)) - if(len(markers) > 0): + if (len(markers) > 0): return markers[0] return "" + def getFilePaths(bucket_url: str) -> list[str]: paths: list[str] = [] suffix: str = "" i: int = 0 - while(True): + while (True): print("----- Fetching block {} of index... -----".format(i)) full_url: str = "{}{}".format(bucket_url, suffix) xml: str = runCommandWithOutput(["curl", "-s", full_url]) - lines: list[str] = xml.replace("", "\n").replace("", "\n").splitlines() + lines: list[str] = xml.replace("", "\n").replace( + "", "\n").splitlines() paths.extend(list(filter(lambda x: isValue(x) and isFile(x), lines))) next_marker: str = getNextMarker(xml) - if(next_marker == ""): + if (next_marker == ""): print("----- Done fetching index. -----") break suffix = "?marker={}".format(next_marker) @@ -47,13 +57,15 @@ def getFilePaths(bucket_url: str) -> list[str]: sleep(0.25) return paths + def downloadFile(bucket_url: str, dl_prefix: str, path: str): full_url: str = bucket_url + "/" + path - if(os.path.exists("{}{}".format(dl_prefix, path))): + if (os.path.exists("{}{}".format(dl_prefix, path))): print("Skipping this file (already exists).") else: runCommand(["wget", "-c", "-nH", "-x", "-P", dl_prefix, full_url]) + def main(): print("----- Fetching index... -----") paths: list[str] = getFilePaths(BUCKET_URL) @@ -65,9 +77,10 @@ def main(): downloadFile(BUCKET_URL, DOWNLOAD_PREFIX, path) i += 1 -if(__name__ == "__main__"): + +if (__name__ == "__main__"): try: main() - except(KeyboardInterrupt): + except (KeyboardInterrupt): print("CTRL-C") pass From 2194572d050574105692847a650855c3c85faf8b Mon Sep 17 00:00:00 2001 From: Cary Keesler <44330591+carykees98@users.noreply.github.com> Date: Wed, 18 Jun 2025 18:20:53 -0400 Subject: [PATCH 40/53] Update docker-compose-prod.yaml --- docker-compose-prod.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose-prod.yaml b/docker-compose-prod.yaml index 6db580c..21107ed 100644 --- a/docker-compose-prod.yaml +++ b/docker-compose-prod.yaml @@ -76,6 +76,10 @@ services: networks: - mirror tty: true + logging: + options: + max-size: 1g + max-file: 5 # --- Torrent handler --- # torrent-handler: From 967b1cec46f366d321f50f57d00a3db37dbe9a80 Mon Sep 17 00:00:00 2001 From: Cary Keesler <44330591+carykees98@users.noreply.github.com> Date: Wed, 18 Jun 2025 18:26:39 -0400 Subject: [PATCH 41/53] Update SyncScheduler.cpp --- mirror-sync-scheduler/src/SyncScheduler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirror-sync-scheduler/src/SyncScheduler.cpp b/mirror-sync-scheduler/src/SyncScheduler.cpp index a0e97e5..0ffc42a 100644 --- a/mirror-sync-scheduler/src/SyncScheduler.cpp +++ b/mirror-sync-scheduler/src/SyncScheduler.cpp @@ -1,4 +1,4 @@ -/** +a/** * @file SyncScheduler.cpp * @author Cary Keesler * @brief @@ -144,7 +144,7 @@ auto SyncScheduler::start_sync(const std::string& projectName) -> bool return true; } - spdlog::info("Attempting sync for {}", projectName); + spdlog::debug("Attempting sync for {}", projectName); const auto& syncDetails = m_ProjectCatalogue.at(projectName); bool startSuccessful = false; From 859e3d4bc6cae599dfb18a2a2d85d4ff5b7ce2ac Mon Sep 17 00:00:00 2001 From: Cary Keesler <44330591+carykees98@users.noreply.github.com> Date: Wed, 18 Jun 2025 18:36:48 -0400 Subject: [PATCH 42/53] :expressionless: --- mirror-sync-scheduler/src/SyncScheduler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirror-sync-scheduler/src/SyncScheduler.cpp b/mirror-sync-scheduler/src/SyncScheduler.cpp index 0ffc42a..99f0d2e 100644 --- a/mirror-sync-scheduler/src/SyncScheduler.cpp +++ b/mirror-sync-scheduler/src/SyncScheduler.cpp @@ -1,4 +1,4 @@ -a/** +/** * @file SyncScheduler.cpp * @author Cary Keesler * @brief From bb15affdf69cc94cc5d57d71a98d7d6b746fa4b2 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Sat, 5 Jul 2025 20:12:29 +0000 Subject: [PATCH 43/53] default argument defined in the wrong place --- .../include/mirror/sync_scheduler/JobManager.hpp | 2 +- mirror-sync-scheduler/src/JobManager.cpp | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp index f51fcf6..1fbdb0d 100644 --- a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp +++ b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp @@ -58,7 +58,7 @@ class JobManager auto process_reaper(const std::stop_token& stopToken) -> void; private: // Static Methods - static auto get_child_process_ids(const ::pid_t processID) + static auto get_child_process_ids(const ::pid_t processID = ::getpid()) -> std::vector<::pid_t>; static auto interrupt_job(const ::pid_t processID) -> void; static auto kill_job(const ::pid_t processID) -> void; diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index d25be0e..b7c07ae 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -82,7 +82,7 @@ auto JobManager::process_reaper(const std::stop_token& stopToken) -> void } } -auto JobManager::get_child_process_ids(const ::pid_t processID = ::getpid()) +auto JobManager::get_child_process_ids(const ::pid_t processID) -> std::vector<::pid_t> { static const std::filesystem::path taskDirectory @@ -123,8 +123,6 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> { std::string errorMessage(BUFSIZ, '\0'); - static const ::pid_t syncSchedulerProcessID = ::getpid(); - std::vector<::pid_t> completedJobs; completedJobs.reserve(m_ActiveJobs.size()); @@ -138,7 +136,7 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> int status = 0; const bool isKnownJob = m_ActiveJobs.contains(childProcessID); - constexpr auto JOB_TIMEOUT = std::chrono::hours(6); + constexpr auto JOB_TIMEOUT = std::chrono::hours(1); std::chrono::hours syncDuration; // NOLINTNEXTLINE(misc-include-cleaner) From c80b8a99bba6f567b32163f8e11170ea521c1c6b Mon Sep 17 00:00:00 2001 From: carykees98 Date: Sat, 5 Jul 2025 20:17:42 +0000 Subject: [PATCH 44/53] updated format string --- mirror-sync-scheduler/src/JobManager.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index b7c07ae..fd52399 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -516,8 +516,7 @@ auto JobManager::start_job( std::string errorMessage(BUFSIZ, '\0'); spdlog::warn( - "Failed to create pipe for child stderr while syncing project " - "{}! " + "Failed to create pipe for child stderr while syncing project {}! " "Error message: {}", jobName, // NOLINTNEXTLINE(*-include-cleaner) From 0be5f664322458444e1a2e30249dd4eae88662f2 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Sat, 5 Jul 2025 20:47:47 +0000 Subject: [PATCH 45/53] added debug messages --- mirror-sync-scheduler/src/JobManager.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index fd52399..f066c3b 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -89,6 +89,12 @@ auto JobManager::get_child_process_ids(const ::pid_t processID) = std::filesystem::absolute(std::format("/proc/{}/task/", processID)); std::vector<::pid_t> childProcesses = {}; + + spdlog::trace( + "Gathering child process ids for process with pid {}", + processID + ); + for (const auto& taskEntry : std::filesystem::directory_iterator(taskDirectory)) { @@ -320,6 +326,8 @@ auto JobManager::interrupt_job(const ::pid_t processID) -> void // // Base case: process with no children. `get_child_process_ids` will be // an empty collection meaning nothing to iterate over + spdlog::trace("Interrupting job with pid {}", processID); + for (const ::pid_t childProcessID : JobManager::get_child_process_ids(processID)) { From bcb6fdeec4ebe64bd766027c7b0026ceb3670548 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Sat, 5 Jul 2025 21:01:42 +0000 Subject: [PATCH 46/53] more debug --- mirror-sync-scheduler/src/JobManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index f066c3b..95850f7 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -326,11 +326,11 @@ auto JobManager::interrupt_job(const ::pid_t processID) -> void // // Base case: process with no children. `get_child_process_ids` will be // an empty collection meaning nothing to iterate over - spdlog::trace("Interrupting job with pid {}", processID); for (const ::pid_t childProcessID : JobManager::get_child_process_ids(processID)) { + spdlog::trace("Interrupting job with pid {}", childProcessID); JobManager::interrupt_job(childProcessID); } From 707a259ecd58f9d41bea6d76f9ebda01323fbf38 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Sat, 5 Jul 2025 21:10:23 +0000 Subject: [PATCH 47/53] static --- mirror-sync-scheduler/src/JobManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index 95850f7..28eed26 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -85,7 +85,7 @@ auto JobManager::process_reaper(const std::stop_token& stopToken) -> void auto JobManager::get_child_process_ids(const ::pid_t processID) -> std::vector<::pid_t> { - static const std::filesystem::path taskDirectory + const std::filesystem::path taskDirectory = std::filesystem::absolute(std::format("/proc/{}/task/", processID)); std::vector<::pid_t> childProcesses = {}; From 6f70daede211a9e3ca252355b52812ee5631dd84 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Sat, 5 Jul 2025 21:31:31 +0000 Subject: [PATCH 48/53] dont reap processes in interrupt_process as they will likely be reaped by their own parent --- mirror-sync-scheduler/src/JobManager.cpp | 36 +++++++++++++----------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index 28eed26..6c87069 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -353,29 +353,29 @@ auto JobManager::interrupt_job(const ::pid_t processID) -> void spdlog::debug("Successfully sent process {} a SIGTERM", processID); - constexpr auto SIGTERM_TIMEOUT = std::chrono::seconds(30); - const auto start = std::chrono::system_clock::now(); - auto now = start; + /* constexpr auto SIGTERM_TIMEOUT = std::chrono::seconds(30); + const auto start = std::chrono::system_clock::now(); + auto now = start; - while ((now - start) < SIGTERM_TIMEOUT) - { - const int waitReturn = ::waitpid(processID, nullptr, WNOHANG); - - if (waitReturn == processID) + while ((now - start) < SIGTERM_TIMEOUT) { - spdlog::trace("Process {} successfully reaped", processID); - return; - } + const int waitReturn = ::waitpid(processID, nullptr, WNOHANG); - constexpr auto CHECK_INTERVAL = std::chrono::milliseconds(100); - std::this_thread::sleep_for(CHECK_INTERVAL); + if (waitReturn == processID) + { + spdlog::trace("Process {} successfully reaped", processID); + return; + } - now = std::chrono::system_clock::now(); - } + constexpr auto CHECK_INTERVAL = std::chrono::milliseconds(100); + std::this_thread::sleep_for(CHECK_INTERVAL); + + now = std::chrono::system_clock::now(); + } - spdlog::error("Failed to terminate process {} with SIGTERM", processID); + spdlog::error("Failed to terminate process {} with SIGTERM", processID); - JobManager::kill_job(processID); + JobManager::kill_job(processID); */ } auto JobManager::job_is_running(const std::string& jobName) -> bool @@ -409,6 +409,8 @@ auto JobManager::kill_job(const ::pid_t processID) -> void // NOLINTNEXTLINE(*-include-cleaner) ::strerror_r(errno, errorMessage.data(), errorMessage.size()) ); + + return; } const int waitReturn = ::waitpid(processID, nullptr, 0); From 285ab632ba3c79ca435c7a82056c843c1e8b5297 Mon Sep 17 00:00:00 2001 From: carykees98 Date: Sat, 5 Jul 2025 21:47:14 +0000 Subject: [PATCH 49/53] added todo --- mirror-sync-scheduler/src/JobManager.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index 6c87069..e0c4055 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -353,6 +353,8 @@ auto JobManager::interrupt_job(const ::pid_t processID) -> void spdlog::debug("Successfully sent process {} a SIGTERM", processID); + // TODO: Handle asynchronously instead of commenting out. Also find some way + // to account for process possibly being reaped by its parent /* constexpr auto SIGTERM_TIMEOUT = std::chrono::seconds(30); const auto start = std::chrono::system_clock::now(); auto now = start; From 8d039e538cb72c255799d294d5e8a41776072b0c Mon Sep 17 00:00:00 2001 From: Cary Keesler <44330591+carykees98@users.noreply.github.com> Date: Tue, 8 Jul 2025 21:40:37 -0400 Subject: [PATCH 50/53] Increased sync time limit from 1 hour to 3 --- mirror-sync-scheduler/src/JobManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index e0c4055..f8a169c 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -142,7 +142,7 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> int status = 0; const bool isKnownJob = m_ActiveJobs.contains(childProcessID); - constexpr auto JOB_TIMEOUT = std::chrono::hours(1); + constexpr auto JOB_TIMEOUT = std::chrono::hours(3); std::chrono::hours syncDuration; // NOLINTNEXTLINE(misc-include-cleaner) From 91d222468ec1f65c5ff4015ed2e458d5853ad903 Mon Sep 17 00:00:00 2001 From: Brendan Connelly Date: Tue, 26 Aug 2025 15:50:54 -0400 Subject: [PATCH 51/53] sync-scheduler: Improve logging --- mirror-sync-scheduler/src/SyncScheduler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirror-sync-scheduler/src/SyncScheduler.cpp b/mirror-sync-scheduler/src/SyncScheduler.cpp index 99f0d2e..d2631d3 100644 --- a/mirror-sync-scheduler/src/SyncScheduler.cpp +++ b/mirror-sync-scheduler/src/SyncScheduler.cpp @@ -125,11 +125,11 @@ auto SyncScheduler::generate_project_catalogue(const nlohmann::json& mirrors) } catch (static_project_exception& spe) { - spdlog::trace(spe.what()); + spdlog::trace(std::format("Project {}: {}", name, spe.what())); } catch (std::runtime_error& re) { - spdlog::error(re.what()); + spdlog::error(std::format("Project {}: {}", name, re.what())); } } From d7d26f5523ed5365b6d9a017180117e9a4db1836 Mon Sep 17 00:00:00 2001 From: Brendan Connelly Date: Tue, 26 Aug 2025 16:12:11 -0400 Subject: [PATCH 52/53] sync-scheduler: Added catch clause during project cataloging --- mirror-sync-scheduler/src/SyncScheduler.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mirror-sync-scheduler/src/SyncScheduler.cpp b/mirror-sync-scheduler/src/SyncScheduler.cpp index d2631d3..bde16a6 100644 --- a/mirror-sync-scheduler/src/SyncScheduler.cpp +++ b/mirror-sync-scheduler/src/SyncScheduler.cpp @@ -125,11 +125,15 @@ auto SyncScheduler::generate_project_catalogue(const nlohmann::json& mirrors) } catch (static_project_exception& spe) { - spdlog::trace(std::format("Project {}: {}", name, spe.what())); + spdlog::trace("{}: {}", name, spe.what()); } catch (std::runtime_error& re) { - spdlog::error(std::format("Project {}: {}", name, re.what())); + spdlog::error("{}: {}", name, re.what()); + } + catch (std::exception& e) + { + spdlog::error("{}: {}", name, e.what()); } } From 33d14511c193500aa8e6c959ddadc242acf660bd Mon Sep 17 00:00:00 2001 From: Brendan Connelly Date: Thu, 28 Aug 2025 11:52:42 -0400 Subject: [PATCH 53/53] sync-scheduler: Change dev config for raspbian mirror to reflect new schema --- mirror-sync-scheduler/configs/mirrors.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mirror-sync-scheduler/configs/mirrors.json b/mirror-sync-scheduler/configs/mirrors.json index a079c17..0f765ac 100644 --- a/mirror-sync-scheduler/configs/mirrors.json +++ b/mirror-sync-scheduler/configs/mirrors.json @@ -572,10 +572,8 @@ "name": "Raspbian", "page": "Distributions", "script": { - "env": { - "mirror": "mirror.umd.edu" - }, - "command": "python3 raspbmirror.py --tmpdir /storage/raspbian-tmp/ --sourcepool /storage/debian/pool http://${mirror}/raspbian http://${mirror}/raspbian http://snapshot.raspbian.org/hashpool", + "command": "python3", + "arguments": ["raspbmirror.py", "--tmpdir", "/storage/raspbian-tmp/", "--sourcepool", "/storage/debian/pool", "http://mirror.umd.edu/raspbian", "http://mirror.umd.edu/raspbian", "http://snapshot.raspbian.org/hashpool"], "syncs_per_day": 3 }, "official": false,