From 32c9ffc85e8d64361eb96ada1015e1890ace53cb Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 16:45:04 -0500 Subject: [PATCH 01/39] introduces a new function to handle RabbitMQ version upgrades --- kubernetes/update-stack.sh | 288 +++++++++++++++++++++++++++++++++++++ 1 file changed, 288 insertions(+) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index e167e79..6034427 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -183,6 +183,293 @@ update_wes_plugins() { # 2. extract and reshape relevant parts into generated configs # 3. load generated configs / secrets using kustomize # if we buy into just using kustomize from the get go, then mostly only need step 3. + +# determine_rabbitmq_upgrade_path: Determines the upgrade path between two RabbitMQ versions +# Returns a space-separated list of intermediate versions, or empty string if direct upgrade +determine_rabbitmq_upgrade_path() { + local current_ver="$1" + local target_ver="$2" + + # Define supported upgrade paths based on RabbitMQ documentation + # https://www.rabbitmq.com/docs/upgrade + # From -> To (only one hop is supported) + declare -A supported_paths + supported_paths["3.7.18"]="3.8.x" + supported_paths["3.8.x"]="3.9.x" + supported_paths["3.9.x"]="3.10.x" + supported_paths["3.10.x"]="3.11.x" + supported_paths["3.11.18"]="3.12.x" + supported_paths["3.12.x"]="3.13.x" + supported_paths["3.13.x"]="4.0.x" + supported_paths["3.13.x"]="4.1.x" + supported_paths["4.0.x"]="4.1.x" + + # Determine current version pattern + local current_pattern="" + if [[ "$current_ver" =~ ^3\.7\. ]]; then + current_pattern="3.7.18" + elif [[ "$current_ver" =~ ^3\.8\. ]]; then + current_pattern="3.8.x" + elif [[ "$current_ver" =~ ^3\.9\. ]]; then + current_pattern="3.9.x" + elif [[ "$current_ver" =~ ^3\.10\. ]]; then + current_pattern="3.10.x" + elif [[ "$current_ver" =~ ^3\.11\. ]]; then + current_pattern="3.11.18" + elif [[ "$current_ver" =~ ^3\.12\. ]]; then + current_pattern="3.12.x" + elif [[ "$current_ver" =~ ^3\.13\. ]]; then + current_pattern="3.13.x" + elif [[ "$current_ver" =~ ^4\.0\. ]]; then + current_pattern="4.0.x" + fi + + # Check if direct upgrade is supported + if [ -n "$current_pattern" ] && [ -n "${supported_paths[$current_pattern]}" ]; then + local target_pattern="${supported_paths[$current_pattern]}" + + # Check if target version matches the supported upgrade path + if [[ "$target_ver" =~ ^${target_pattern//x/} ]]; then + # Direct upgrade is supported + echo "" + return 0 + fi + + # Need to find intermediate path + local intermediate_versions="" + local current_step="$current_pattern" + + while [ -n "$current_step" ] && [ "$current_step" != "$target_pattern" ]; do + local next_step="${supported_paths[$current_step]}" + if [ -n "$next_step" ]; then + # Convert pattern to major.minor version for intermediate step + if [[ "$next_step" =~ ^3\. ]]; then + case "$next_step" in + "3.8.x") intermediate_versions="$intermediate_versions 3.8" ;; + "3.9.x") intermediate_versions="$intermediate_versions 3.9" ;; + "3.10.x") intermediate_versions="$intermediate_versions 3.10" ;; + "3.11.x") intermediate_versions="$intermediate_versions 3.11" ;; + "3.12.x") intermediate_versions="$intermediate_versions 3.12" ;; + "3.13.x") intermediate_versions="$intermediate_versions 3.13" ;; + esac + elif [[ "$next_step" =~ ^4\. ]]; then + case "$next_step" in + "4.0.x") intermediate_versions="$intermediate_versions 4.0" ;; + "4.1.x") intermediate_versions="$intermediate_versions 4.1" ;; + esac + fi + + # Check if we've reached the target pattern + if [[ "$target_ver" =~ ^${next_step//x/} ]]; then + break + fi + + current_step="$next_step" + else + break + fi + done + + # Return intermediate versions (trimmed) + echo "$intermediate_versions" | sed 's/^ *//;s/ *$//' + return 0 + fi + + # No supported path found + return 1 +} + +# upgrade_rabbitmq_to_version: Upgrades RabbitMQ to a specific version +upgrade_rabbitmq_to_version() { + local target_ver="$1" + + echo "Upgrading to version $target_ver..." + waggle_log info "RabbitMQ upgrading to version $target_ver" + + kubectl set image statefulset/wes-rabbitmq wes-rabbitmq="rabbitmq:${target_ver}-management-alpine" + echo "Waiting for $target_ver rollout to complete..." + + if ! kubectl rollout status statefulset/wes-rabbitmq; then + echo "Error: $target_ver rollout failed" + waggle_log err "RabbitMQ $target_ver rollout failed" + return 1 + fi + + # Wait for RabbitMQ to be ready + echo "Waiting for RabbitMQ to be ready after $target_ver upgrade..." + if ! kubectl wait --for=condition=ready pod/wes-rabbitmq-0 --timeout=300s; then + echo "Error: RabbitMQ not ready after $target_ver upgrade" + waggle_log err "RabbitMQ not ready after $target_ver upgrade" + return 1 + fi + + # Enable feature flags for next upgrade + echo "Enabling feature flags for next upgrade..." + kubectl exec wes-rabbitmq-0 -- rabbitmqctl enable_feature_flag all || true + + echo "Successfully upgraded to $target_ver" + return 0 +} + +# update_rabbitmq_version: Main function to handle RabbitMQ version upgrades +# This function handles: +# - Upgrades following RabbitMQ's official supported upgrade paths +# - Only one-hop upgrades (e.g., 3.8.x -> 3.9.x, 3.13.x -> 4.0.x) +# - Automatic backup creation before major upgrades +# - Feature flag enabling for compatibility +# - Rollout verification and health checks +# - Comprehensive logging via waggle_log +update_rabbitmq_version() { + echo "checking if RabbitMQ version upgrade is needed..." + waggle_log info "starting RabbitMQ version upgrade check" + + # Get current running version + if ! current_version=$(kubectl get statefulset wes-rabbitmq -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null); then + echo "RabbitMQ StatefulSet not found, skipping version upgrade" + waggle_log info "RabbitMQ StatefulSet not found, skipping version upgrade" + return 0 + fi + + # Extract version from image (e.g., "rabbitmq:3.8.11-management-alpine" -> "3.8.11") + current_ver=$(echo "$current_version" | sed 's/rabbitmq://' | sed 's/-management-alpine//') + + # Validate current version format + if ! echo "$current_ver" | grep -qE '^[0-9]+\.[0-9]+(\.[0-9]+)?$'; then + echo "Error: Invalid current version format: $current_ver" + waggle_log err "Invalid RabbitMQ current version format: $current_ver" + return 1 + fi + + # Get target version from the YAML file + target_ver=$(grep "image: rabbitmq:" wes-rabbitmq.yaml | sed 's/.*rabbitmq://' | sed 's/-management-alpine//') + + # Validate target version format + if ! echo "$target_ver" | grep -qE '^[0-9]+\.[0-9]+(\.[0-9]+)?$'; then + echo "Error: Invalid target version format: $target_ver" + waggle_log err "Invalid RabbitMQ target version format: $target_ver" + return 1 + fi + + if [ "$current_ver" = "$target_ver" ]; then + echo "RabbitMQ is already at target version $target_ver" + waggle_log info "RabbitMQ already at target version $target_ver" + return 0 + fi + + echo "RabbitMQ version upgrade needed: $current_ver -> $target_ver" + waggle_log info "RabbitMQ version upgrade needed: $current_ver -> $target_ver" + + # Check if this is a downgrade (not supported) + local current_major=$(echo "$current_ver" | cut -d. -f1) + local current_minor=$(echo "$current_ver" | cut -d. -f2) + local target_major=$(echo "$target_ver" | cut -d. -f1) + local target_minor=$(echo "$target_ver" | cut -d. -f2) + + if [ "$current_major" -gt "$target_major" ] || ([ "$current_major" -eq "$target_major" ] && [ "$current_minor" -gt "$target_minor" ]); then + echo "Error: Downgrading RabbitMQ from $current_ver to $target_ver is not supported" + waggle_log err "RabbitMQ downgrade not supported: $current_ver -> $target_ver" + return 1 + fi + + # Ensure RabbitMQ is running before attempting upgrade + echo "Ensuring RabbitMQ is running..." + if ! kubectl wait --for=condition=ready pod/wes-rabbitmq-0 --timeout=60s; then + echo "Error: RabbitMQ is not ready, cannot proceed with upgrade" + waggle_log err "RabbitMQ is not ready, cannot proceed with upgrade" + return 1 + fi + + # Backup data before major upgrades + echo "Creating backup of RabbitMQ data..." + waggle_log info "Creating backup of RabbitMQ data" + + # Check available disk space (need at least 1GB free) + available_space=$(df . | awk 'NR==2 {print $4}') + if [ "$available_space" -lt 1048576 ]; then # 1GB in KB + echo "Warning: Low disk space available (${available_space}KB), backup may fail" + waggle_log warn "Low disk space for RabbitMQ backup: ${available_space}KB" + fi + + backup_file="/var/backups/rabbitmq-data-$(date +%F).tar.gz" + if ! kubectl exec wes-rabbitmq-0 -- tar czf /tmp/rabbitmq-data.tar.gz /var/lib/rabbitmq/mnesia; then + echo "Error: Failed to create backup, stopping upgrade..." + waggle_log err "Failed to create RabbitMQ backup, stopping upgrade" + return 1 + fi + + if ! kubectl cp wes-rabbitmq-0:/tmp/rabbitmq-data.tar.gz "$backup_file"; then + echo "Error: Failed to copy backup to host, stopping upgrade..." + waggle_log err "Failed to copy RabbitMQ backup to host, stopping upgrade" + return 1 + fi + + echo "Backup created: $backup_file" + waggle_log info "RabbitMQ backup created: $backup_file" + + # Enable feature flags for upgrade + echo "Enabling feature flags for upgrade..." + if ! kubectl exec wes-rabbitmq-0 -- rabbitmqctl enable_feature_flag all; then + echo "Warning: Failed to enable feature flags, continuing with upgrade..." + waggle_log warn "Failed to enable RabbitMQ feature flags, continuing with upgrade" + fi + + # Verify feature flags are enabled + echo "Verifying feature flags..." + kubectl exec wes-rabbitmq-0 -- rabbitmqctl -q --formatter pretty_table list_feature_flags || true + + # Determine upgrade path + echo "Determining upgrade path..." + upgrade_path=$(determine_rabbitmq_upgrade_path "$current_ver" "$target_ver") + + if [ $? -ne 0 ]; then + echo "Error: No supported upgrade path found from $current_ver to $target_ver" + waggle_log err "No supported RabbitMQ upgrade path found: $current_ver -> $target_ver" + return 1 + fi + + # Execute upgrades + if [ -n "$upgrade_path" ]; then + echo "Will upgrade through intermediate versions: $upgrade_path" + waggle_log info "RabbitMQ will upgrade through intermediate versions: $upgrade_path" + + # Upgrade through intermediate versions + for version in $upgrade_path; do + if ! upgrade_rabbitmq_to_version "$version"; then + echo "Error: Failed to upgrade to intermediate version $version" + return 1 + fi + done + else + echo "Direct upgrade path available" + waggle_log info "RabbitMQ direct upgrade path available" + fi + + # Finally upgrade to target version + if ! upgrade_rabbitmq_to_version "$target_ver"; then + echo "Error: Failed to upgrade to target version $target_ver" + return 1 + fi + + # Verify the upgrade was successful + echo "Verifying upgrade was successful..." + if ! new_version=$(kubectl get statefulset wes-rabbitmq -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null); then + echo "Warning: Could not verify new version" + waggle_log warn "Could not verify RabbitMQ new version" + else + new_ver=$(echo "$new_version" | sed 's/rabbitmq://' | sed 's/-management-alpine//') + if [ "$new_ver" = "$target_ver" ]; then + echo "Upgrade verification successful: RabbitMQ is now running version $new_ver" + waggle_log info "RabbitMQ upgrade verification successful: now running version $new_ver" + else + echo "Warning: Upgrade verification failed. Expected: $target_ver, Got: $new_ver" + waggle_log warn "RabbitMQ upgrade verification failed. Expected: $target_ver, Got: $new_ver" + fi + fi + + echo "RabbitMQ version upgrade completed successfully" + waggle_log info "RabbitMQ version upgrade completed successfully: $current_ver -> $target_ver" +} + update_wes() { echo "updating wes" @@ -782,6 +1069,7 @@ update_wes_tools update_node_secrets update_node_manifest_v2 update_data_config +update_rabbitmq_version update_wes_plugins update_wes update_influxdb_retention From 61a09704fdc00ba302873647aa3e41ae10f233d6 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 16:52:58 -0500 Subject: [PATCH 02/39] refactor RabbitMQ upgrade path logi --- kubernetes/update-stack.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 6034427..014b289 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -239,7 +239,7 @@ determine_rabbitmq_upgrade_path() { local intermediate_versions="" local current_step="$current_pattern" - while [ -n "$current_step" ] && [ "$current_step" != "$target_pattern" ]; do + while [ -n "$current_step" ]; do local next_step="${supported_paths[$current_step]}" if [ -n "$next_step" ]; then # Convert pattern to major.minor version for intermediate step @@ -261,6 +261,8 @@ determine_rabbitmq_upgrade_path() { # Check if we've reached the target pattern if [[ "$target_ver" =~ ^${next_step//x/} ]]; then + # We've found the target pattern, stop here + # Don't add this step to intermediate versions since it's the target break fi From 771e5336774b0077f2c145e431315ebc2d647acb Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 17:02:04 -0500 Subject: [PATCH 03/39] update RabbitMQ readiness check to wait for running state instead of ready --- kubernetes/update-stack.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 014b289..0a3df91 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -297,11 +297,11 @@ upgrade_rabbitmq_to_version() { return 1 fi - # Wait for RabbitMQ to be ready - echo "Waiting for RabbitMQ to be ready after $target_ver upgrade..." - if ! kubectl wait --for=condition=ready pod/wes-rabbitmq-0 --timeout=300s; then - echo "Error: RabbitMQ not ready after $target_ver upgrade" - waggle_log err "RabbitMQ not ready after $target_ver upgrade" + # Wait for RabbitMQ to be running + echo "Waiting for RabbitMQ to be running after $target_ver upgrade..." + if ! kubectl wait --for=condition=running pod/wes-rabbitmq-0 --timeout=600s; then + echo "Error: RabbitMQ not running after $target_ver upgrade" + waggle_log err "RabbitMQ not running after $target_ver upgrade" return 1 fi From e0839794089c9be2f75e2b4f64750d543ae85a4f Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 17:17:47 -0500 Subject: [PATCH 04/39] update RabbitMQ readiness check to wait for 'Ready' state and add a delay before enabling feature flags --- kubernetes/update-stack.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 0a3df91..c6dadd9 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -299,13 +299,14 @@ upgrade_rabbitmq_to_version() { # Wait for RabbitMQ to be running echo "Waiting for RabbitMQ to be running after $target_ver upgrade..." - if ! kubectl wait --for=condition=running pod/wes-rabbitmq-0 --timeout=600s; then - echo "Error: RabbitMQ not running after $target_ver upgrade" - waggle_log err "RabbitMQ not running after $target_ver upgrade" + if ! kubectl wait --for=condition=Ready pod/wes-rabbitmq-0 --timeout=600s; then + echo "Error: RabbitMQ not ready after $target_ver upgrade" + waggle_log err "RabbitMQ not ready after $target_ver upgrade" return 1 fi # Enable feature flags for next upgrade + sleep 1m echo "Enabling feature flags for next upgrade..." kubectl exec wes-rabbitmq-0 -- rabbitmqctl enable_feature_flag all || true From 457cc9c12dfeb522948e1bc6dd16a3deca1af896 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 17:32:48 -0500 Subject: [PATCH 05/39] Add debug logging to determine_rabbitmq_upgrade_path for better traceability --- kubernetes/update-stack.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index c6dadd9..594d46d 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -190,6 +190,8 @@ determine_rabbitmq_upgrade_path() { local current_ver="$1" local target_ver="$2" + echo "DEBUG: determine_rabbitmq_upgrade_path called with current_ver=$current_ver, target_ver=$target_ver" + # Define supported upgrade paths based on RabbitMQ documentation # https://www.rabbitmq.com/docs/upgrade # From -> To (only one hop is supported) @@ -224,13 +226,17 @@ determine_rabbitmq_upgrade_path() { current_pattern="4.0.x" fi + echo "DEBUG: current_pattern determined as: $current_pattern" + # Check if direct upgrade is supported if [ -n "$current_pattern" ] && [ -n "${supported_paths[$current_pattern]}" ]; then local target_pattern="${supported_paths[$current_pattern]}" + echo "DEBUG: target_pattern from supported_paths: $target_pattern" # Check if target version matches the supported upgrade path if [[ "$target_ver" =~ ^${target_pattern//x/} ]]; then # Direct upgrade is supported + echo "DEBUG: Direct upgrade supported from $current_pattern to $target_pattern" echo "" return 0 fi @@ -238,9 +244,12 @@ determine_rabbitmq_upgrade_path() { # Need to find intermediate path local intermediate_versions="" local current_step="$current_pattern" + echo "DEBUG: Starting path building from current_step: $current_step" while [ -n "$current_step" ]; do + echo "DEBUG: Loop iteration - current_step: $current_step" local next_step="${supported_paths[$current_step]}" + echo "DEBUG: next_step from supported_paths: $next_step" if [ -n "$next_step" ]; then # Convert pattern to major.minor version for intermediate step if [[ "$next_step" =~ ^3\. ]]; then @@ -273,11 +282,14 @@ determine_rabbitmq_upgrade_path() { done # Return intermediate versions (trimmed) - echo "$intermediate_versions" | sed 's/^ *//;s/ *$//' + local final_result=$(echo "$intermediate_versions" | sed 's/^ *//;s/ *$//') + echo "DEBUG: Final intermediate_versions result: '$final_result'" + echo "$final_result" return 0 fi # No supported path found + echo "DEBUG: No supported path found for current_ver=$current_ver, target_ver=$target_ver" return 1 } From 59d8f301b5bc71522f004cc2e8137184c0430b47 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 17:37:31 -0500 Subject: [PATCH 06/39] Add additional debug logging in update_rabbitmq_version for enhanced error tracking --- kubernetes/update-stack.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 594d46d..7057c82 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -431,12 +431,17 @@ update_rabbitmq_version() { # Verify feature flags are enabled echo "Verifying feature flags..." kubectl exec wes-rabbitmq-0 -- rabbitmqctl -q --formatter pretty_table list_feature_flags || true + echo "DEBUG: Feature flags verification completed" # Determine upgrade path echo "Determining upgrade path..." + echo "DEBUG: About to call determine_rabbitmq_upgrade_path with current_ver=$current_ver, target_ver=$target_ver" upgrade_path=$(determine_rabbitmq_upgrade_path "$current_ver" "$target_ver") + echo "DEBUG: determine_rabbitmq_upgrade_path returned: $upgrade_path" - if [ $? -ne 0 ]; then + local exit_code=$? + echo "DEBUG: determine_rabbitmq_upgrade_path exit code: $exit_code" + if [ $exit_code -ne 0 ]; then echo "Error: No supported upgrade path found from $current_ver to $target_ver" waggle_log err "No supported RabbitMQ upgrade path found: $current_ver -> $target_ver" return 1 From c84ece1d4e55764efc1362e69fe8b273fd6a9937 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 17:38:47 -0500 Subject: [PATCH 07/39] Add debug logging in determine_rabbitmq_upgrade_path for improved traceability --- kubernetes/update-stack.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 7057c82..c157a1b 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -187,6 +187,7 @@ update_wes_plugins() { # determine_rabbitmq_upgrade_path: Determines the upgrade path between two RabbitMQ versions # Returns a space-separated list of intermediate versions, or empty string if direct upgrade determine_rabbitmq_upgrade_path() { + echo "DEBUG: ENTERING determine_rabbitmq_upgrade_path function" local current_ver="$1" local target_ver="$2" From a30d975e2cb77f9d85634de817185b1819928fe9 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 17:40:11 -0500 Subject: [PATCH 08/39] debug --- kubernetes/update-stack.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index c157a1b..f78d22b 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -437,8 +437,15 @@ update_rabbitmq_version() { # Determine upgrade path echo "Determining upgrade path..." echo "DEBUG: About to call determine_rabbitmq_upgrade_path with current_ver=$current_ver, target_ver=$target_ver" - upgrade_path=$(determine_rabbitmq_upgrade_path "$current_ver" "$target_ver") - echo "DEBUG: determine_rabbitmq_upgrade_path returned: $upgrade_path" + echo "DEBUG: Testing if function exists..." + if declare -f determine_rabbitmq_upgrade_path > /dev/null; then + echo "DEBUG: Function exists, calling it..." + upgrade_path=$(determine_rabbitmq_upgrade_path "$current_ver" "$target_ver") + echo "DEBUG: determine_rabbitmq_upgrade_path returned: $upgrade_path" + else + echo "DEBUG: Function does not exist!" + return 1 + fi local exit_code=$? echo "DEBUG: determine_rabbitmq_upgrade_path exit code: $exit_code" From acab64a37c912f15fc462ee55168611ec7120971 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 17:42:05 -0500 Subject: [PATCH 09/39] debug --- kubernetes/update-stack.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index f78d22b..bfe6b84 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -440,6 +440,9 @@ update_rabbitmq_version() { echo "DEBUG: Testing if function exists..." if declare -f determine_rabbitmq_upgrade_path > /dev/null; then echo "DEBUG: Function exists, calling it..." + echo "DEBUG: Function definition:" + declare -f determine_rabbitmq_upgrade_path + echo "DEBUG: Now calling the function..." upgrade_path=$(determine_rabbitmq_upgrade_path "$current_ver" "$target_ver") echo "DEBUG: determine_rabbitmq_upgrade_path returned: $upgrade_path" else From f19d7f5d8909ef185f57fcbb4c68c1e1a51d16a3 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 17:44:18 -0500 Subject: [PATCH 10/39] debug --- kubernetes/update-stack.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index bfe6b84..0b7a564 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -443,6 +443,8 @@ update_rabbitmq_version() { echo "DEBUG: Function definition:" declare -f determine_rabbitmq_upgrade_path echo "DEBUG: Now calling the function..." + echo "DEBUG: Calling function directly..." + determine_rabbitmq_upgrade_path "$current_ver" "$target_ver" upgrade_path=$(determine_rabbitmq_upgrade_path "$current_ver" "$target_ver") echo "DEBUG: determine_rabbitmq_upgrade_path returned: $upgrade_path" else From 1cdcedb32cc440af0e6e8d2cb3f07c21dda9497f Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 17:51:46 -0500 Subject: [PATCH 11/39] found bug within regex --- kubernetes/update-stack.sh | 45 +++++++++----------------------------- 1 file changed, 10 insertions(+), 35 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 0b7a564..203e1a3 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -187,12 +187,9 @@ update_wes_plugins() { # determine_rabbitmq_upgrade_path: Determines the upgrade path between two RabbitMQ versions # Returns a space-separated list of intermediate versions, or empty string if direct upgrade determine_rabbitmq_upgrade_path() { - echo "DEBUG: ENTERING determine_rabbitmq_upgrade_path function" local current_ver="$1" local target_ver="$2" - echo "DEBUG: determine_rabbitmq_upgrade_path called with current_ver=$current_ver, target_ver=$target_ver" - # Define supported upgrade paths based on RabbitMQ documentation # https://www.rabbitmq.com/docs/upgrade # From -> To (only one hop is supported) @@ -209,35 +206,33 @@ determine_rabbitmq_upgrade_path() { # Determine current version pattern local current_pattern="" - if [[ "$current_ver" =~ ^3\.7\. ]]; then + if [[ "$current_ver" =~ ^3\.7(\.|$) ]]; then current_pattern="3.7.18" - elif [[ "$current_ver" =~ ^3\.8\. ]]; then + elif [[ "$current_ver" =~ ^3\.8(\.|$) ]]; then current_pattern="3.8.x" - elif [[ "$current_ver" =~ ^3\.9\. ]]; then + elif [[ "$current_ver" =~ ^3\.9(\.|$) ]]; then current_pattern="3.9.x" - elif [[ "$current_ver" =~ ^3\.10\. ]]; then + elif [[ "$current_ver" =~ ^3\.10(\.|$) ]]; then current_pattern="3.10.x" - elif [[ "$current_ver" =~ ^3\.11\. ]]; then + elif [[ "$current_ver" =~ ^3\.11(\.|$) ]]; then current_pattern="3.11.18" - elif [[ "$current_ver" =~ ^3\.12\. ]]; then + elif [[ "$current_ver" =~ ^3\.12(\.|$) ]]; then current_pattern="3.12.x" - elif [[ "$current_ver" =~ ^3\.13\. ]]; then + elif [[ "$current_ver" =~ ^3\.13(\.|$) ]]; then current_pattern="3.13.x" - elif [[ "$current_ver" =~ ^4\.0\. ]]; then + elif [[ "$current_ver" =~ ^4\.0(\.|$) ]]; then current_pattern="4.0.x" fi - echo "DEBUG: current_pattern determined as: $current_pattern" + # Check if direct upgrade is supported if [ -n "$current_pattern" ] && [ -n "${supported_paths[$current_pattern]}" ]; then local target_pattern="${supported_paths[$current_pattern]}" - echo "DEBUG: target_pattern from supported_paths: $target_pattern" # Check if target version matches the supported upgrade path if [[ "$target_ver" =~ ^${target_pattern//x/} ]]; then # Direct upgrade is supported - echo "DEBUG: Direct upgrade supported from $current_pattern to $target_pattern" echo "" return 0 fi @@ -245,12 +240,9 @@ determine_rabbitmq_upgrade_path() { # Need to find intermediate path local intermediate_versions="" local current_step="$current_pattern" - echo "DEBUG: Starting path building from current_step: $current_step" while [ -n "$current_step" ]; do - echo "DEBUG: Loop iteration - current_step: $current_step" local next_step="${supported_paths[$current_step]}" - echo "DEBUG: next_step from supported_paths: $next_step" if [ -n "$next_step" ]; then # Convert pattern to major.minor version for intermediate step if [[ "$next_step" =~ ^3\. ]]; then @@ -284,13 +276,11 @@ determine_rabbitmq_upgrade_path() { # Return intermediate versions (trimmed) local final_result=$(echo "$intermediate_versions" | sed 's/^ *//;s/ *$//') - echo "DEBUG: Final intermediate_versions result: '$final_result'" echo "$final_result" return 0 fi # No supported path found - echo "DEBUG: No supported path found for current_ver=$current_ver, target_ver=$target_ver" return 1 } @@ -432,25 +422,10 @@ update_rabbitmq_version() { # Verify feature flags are enabled echo "Verifying feature flags..." kubectl exec wes-rabbitmq-0 -- rabbitmqctl -q --formatter pretty_table list_feature_flags || true - echo "DEBUG: Feature flags verification completed" # Determine upgrade path echo "Determining upgrade path..." - echo "DEBUG: About to call determine_rabbitmq_upgrade_path with current_ver=$current_ver, target_ver=$target_ver" - echo "DEBUG: Testing if function exists..." - if declare -f determine_rabbitmq_upgrade_path > /dev/null; then - echo "DEBUG: Function exists, calling it..." - echo "DEBUG: Function definition:" - declare -f determine_rabbitmq_upgrade_path - echo "DEBUG: Now calling the function..." - echo "DEBUG: Calling function directly..." - determine_rabbitmq_upgrade_path "$current_ver" "$target_ver" - upgrade_path=$(determine_rabbitmq_upgrade_path "$current_ver" "$target_ver") - echo "DEBUG: determine_rabbitmq_upgrade_path returned: $upgrade_path" - else - echo "DEBUG: Function does not exist!" - return 1 - fi + upgrade_path=$(determine_rabbitmq_upgrade_path "$current_ver" "$target_ver") local exit_code=$? echo "DEBUG: determine_rabbitmq_upgrade_path exit code: $exit_code" From 8e2059b9825f65fb04d15cc3057902028bf25d4f Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 17:55:23 -0500 Subject: [PATCH 12/39] Refactor determine_rabbitmq_upgrade_path --- kubernetes/update-stack.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 203e1a3..e514d92 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -244,6 +244,13 @@ determine_rabbitmq_upgrade_path() { while [ -n "$current_step" ]; do local next_step="${supported_paths[$current_step]}" if [ -n "$next_step" ]; then + # Check if we've reached the target pattern BEFORE adding to intermediate versions + if [[ "$target_ver" =~ ^${next_step//x/} ]]; then + # We've found the target pattern, stop here + # Don't add this step to intermediate versions since it's the target + break + fi + # Convert pattern to major.minor version for intermediate step if [[ "$next_step" =~ ^3\. ]]; then case "$next_step" in @@ -261,13 +268,6 @@ determine_rabbitmq_upgrade_path() { esac fi - # Check if we've reached the target pattern - if [[ "$target_ver" =~ ^${next_step//x/} ]]; then - # We've found the target pattern, stop here - # Don't add this step to intermediate versions since it's the target - break - fi - current_step="$next_step" else break @@ -428,7 +428,6 @@ update_rabbitmq_version() { upgrade_path=$(determine_rabbitmq_upgrade_path "$current_ver" "$target_ver") local exit_code=$? - echo "DEBUG: determine_rabbitmq_upgrade_path exit code: $exit_code" if [ $exit_code -ne 0 ]; then echo "Error: No supported upgrade path found from $current_ver to $target_ver" waggle_log err "No supported RabbitMQ upgrade path found: $current_ver -> $target_ver" From 1b7cc8e4d1b3eaf240fa50779a67d2c8a3d41a4a Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 17:57:49 -0500 Subject: [PATCH 13/39] debug --- kubernetes/update-stack.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index e514d92..92f6f36 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -224,8 +224,6 @@ determine_rabbitmq_upgrade_path() { current_pattern="4.0.x" fi - - # Check if direct upgrade is supported if [ -n "$current_pattern" ] && [ -n "${supported_paths[$current_pattern]}" ]; then local target_pattern="${supported_paths[$current_pattern]}" @@ -245,11 +243,14 @@ determine_rabbitmq_upgrade_path() { local next_step="${supported_paths[$current_step]}" if [ -n "$next_step" ]; then # Check if we've reached the target pattern BEFORE adding to intermediate versions + echo "DEBUG: Checking if target_ver=$target_ver matches next_step=$next_step (pattern: ^${next_step//x/})" if [[ "$target_ver" =~ ^${next_step//x/} ]]; then + echo "DEBUG: Target pattern matched! Breaking without adding $next_step" # We've found the target pattern, stop here # Don't add this step to intermediate versions since it's the target break fi + echo "DEBUG: Target pattern did not match, will add $next_step to intermediate versions" # Convert pattern to major.minor version for intermediate step if [[ "$next_step" =~ ^3\. ]]; then From e00697126f7d1cd3e2a90bbce22aa83b3fa20759 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 18:09:58 -0500 Subject: [PATCH 14/39] add to valid --- kubernetes/update-stack.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 92f6f36..2dfc838 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -699,6 +699,14 @@ EOF "destination_type": "queue", "routing_key": "*.ansible", "arguments": {} + }, + { + "source": "to-validator", + "vhost": "/", + "destination": "to-validator", + "destination_type": "queue", + "routing_key": "*.to-validator", + "arguments": {} } ], "parameters": [ From 0ffc04e51856d7948c8077634c8633a2d8bd49d9 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Thu, 21 Aug 2025 18:13:39 -0500 Subject: [PATCH 15/39] remove debug --- kubernetes/update-stack.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 2dfc838..0b57bcb 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -243,14 +243,11 @@ determine_rabbitmq_upgrade_path() { local next_step="${supported_paths[$current_step]}" if [ -n "$next_step" ]; then # Check if we've reached the target pattern BEFORE adding to intermediate versions - echo "DEBUG: Checking if target_ver=$target_ver matches next_step=$next_step (pattern: ^${next_step//x/})" if [[ "$target_ver" =~ ^${next_step//x/} ]]; then - echo "DEBUG: Target pattern matched! Breaking without adding $next_step" # We've found the target pattern, stop here # Don't add this step to intermediate versions since it's the target break fi - echo "DEBUG: Target pattern did not match, will add $next_step to intermediate versions" # Convert pattern to major.minor version for intermediate step if [[ "$next_step" =~ ^3\. ]]; then From 2fa8ab7ef8716b9acbceb943755c79bed5fd37a8 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Fri, 22 Aug 2025 11:23:12 -0500 Subject: [PATCH 16/39] debug --- kubernetes/update-stack.sh | 61 +++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 0b57bcb..21c3114 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -256,7 +256,12 @@ determine_rabbitmq_upgrade_path() { "3.9.x") intermediate_versions="$intermediate_versions 3.9" ;; "3.10.x") intermediate_versions="$intermediate_versions 3.10" ;; "3.11.x") intermediate_versions="$intermediate_versions 3.11" ;; - "3.12.x") intermediate_versions="$intermediate_versions 3.12" ;; + "3.12.x") + # Special case: only add 3.12 if we're coming from 3.11.18 + if [ "$current_step" = "3.11.18" ]; then + intermediate_versions="$intermediate_versions 3.12" + fi + ;; "3.13.x") intermediate_versions="$intermediate_versions 3.13" ;; esac elif [[ "$next_step" =~ ^4\. ]]; then @@ -437,42 +442,42 @@ update_rabbitmq_version() { echo "Will upgrade through intermediate versions: $upgrade_path" waggle_log info "RabbitMQ will upgrade through intermediate versions: $upgrade_path" - # Upgrade through intermediate versions - for version in $upgrade_path; do - if ! upgrade_rabbitmq_to_version "$version"; then - echo "Error: Failed to upgrade to intermediate version $version" - return 1 - fi - done + # # Upgrade through intermediate versions + # for version in $upgrade_path; do + # if ! upgrade_rabbitmq_to_version "$version"; then + # echo "Error: Failed to upgrade to intermediate version $version" + # return 1 + # fi + # done else echo "Direct upgrade path available" waggle_log info "RabbitMQ direct upgrade path available" fi # Finally upgrade to target version - if ! upgrade_rabbitmq_to_version "$target_ver"; then - echo "Error: Failed to upgrade to target version $target_ver" - return 1 - fi + # if ! upgrade_rabbitmq_to_version "$target_ver"; then + # echo "Error: Failed to upgrade to target version $target_ver" + # return 1 + # fi # Verify the upgrade was successful - echo "Verifying upgrade was successful..." - if ! new_version=$(kubectl get statefulset wes-rabbitmq -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null); then - echo "Warning: Could not verify new version" - waggle_log warn "Could not verify RabbitMQ new version" - else - new_ver=$(echo "$new_version" | sed 's/rabbitmq://' | sed 's/-management-alpine//') - if [ "$new_ver" = "$target_ver" ]; then - echo "Upgrade verification successful: RabbitMQ is now running version $new_ver" - waggle_log info "RabbitMQ upgrade verification successful: now running version $new_ver" - else - echo "Warning: Upgrade verification failed. Expected: $target_ver, Got: $new_ver" - waggle_log warn "RabbitMQ upgrade verification failed. Expected: $target_ver, Got: $new_ver" - fi - fi + # echo "Verifying upgrade was successful..." + # if ! new_version=$(kubectl get statefulset wes-rabbitmq -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null); then + # echo "Warning: Could not verify new version" + # waggle_log warn "Could not verify RabbitMQ new version" + # else + # new_ver=$(echo "$new_version" | sed 's/rabbitmq://' | sed 's/-management-alpine//') + # if [ "$new_ver" = "$target_ver" ]; then + # echo "Upgrade verification successful: RabbitMQ is now running version $new_ver" + # waggle_log info "RabbitMQ upgrade verification successful: now running version $new_ver" + # else + # echo "Warning: Upgrade verification failed. Expected: $target_ver, Got: $new_ver" + # waggle_log warn "RabbitMQ upgrade verification failed. Expected: $target_ver, Got: $new_ver" + # fi + # fi - echo "RabbitMQ version upgrade completed successfully" - waggle_log info "RabbitMQ version upgrade completed successfully: $current_ver -> $target_ver" + # echo "RabbitMQ version upgrade completed successfully" + # waggle_log info "RabbitMQ version upgrade completed successfully: $current_ver -> $target_ver" } update_wes() { From 4309e93e3f39adece4ed09c96bdd68e223bb018d Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Fri, 22 Aug 2025 11:33:49 -0500 Subject: [PATCH 17/39] debug --- kubernetes/update-stack.sh | 123 ++++++++++++------------------------- 1 file changed, 39 insertions(+), 84 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 21c3114..6f6154d 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -189,102 +189,57 @@ update_wes_plugins() { determine_rabbitmq_upgrade_path() { local current_ver="$1" local target_ver="$2" - - # Define supported upgrade paths based on RabbitMQ documentation - # https://www.rabbitmq.com/docs/upgrade - # From -> To (only one hop is supported) + declare -A supported_paths - supported_paths["3.7.18"]="3.8.x" + supported_paths["3.7.x"]="3.8.x" supported_paths["3.8.x"]="3.9.x" supported_paths["3.9.x"]="3.10.x" supported_paths["3.10.x"]="3.11.x" - supported_paths["3.11.18"]="3.12.x" + supported_paths["3.11.x"]="3.12.x" supported_paths["3.12.x"]="3.13.x" supported_paths["3.13.x"]="4.0.x" - supported_paths["3.13.x"]="4.1.x" supported_paths["4.0.x"]="4.1.x" - - # Determine current version pattern + + # Map specific versions to generalized patterns local current_pattern="" - if [[ "$current_ver" =~ ^3\.7(\.|$) ]]; then - current_pattern="3.7.18" - elif [[ "$current_ver" =~ ^3\.8(\.|$) ]]; then - current_pattern="3.8.x" - elif [[ "$current_ver" =~ ^3\.9(\.|$) ]]; then - current_pattern="3.9.x" - elif [[ "$current_ver" =~ ^3\.10(\.|$) ]]; then - current_pattern="3.10.x" - elif [[ "$current_ver" =~ ^3\.11(\.|$) ]]; then - current_pattern="3.11.18" - elif [[ "$current_ver" =~ ^3\.12(\.|$) ]]; then - current_pattern="3.12.x" - elif [[ "$current_ver" =~ ^3\.13(\.|$) ]]; then - current_pattern="3.13.x" - elif [[ "$current_ver" =~ ^4\.0(\.|$) ]]; then - current_pattern="4.0.x" + if [[ "$current_ver" =~ ^3\.7 ]]; then current_pattern="3.7.x" + elif [[ "$current_ver" =~ ^3\.8 ]]; then current_pattern="3.8.x" + elif [[ "$current_ver" =~ ^3\.9 ]]; then current_pattern="3.9.x" + elif [[ "$current_ver" =~ ^3\.10 ]]; then current_pattern="3.10.x" + elif [[ "$current_ver" =~ ^3\.11 ]]; then current_pattern="3.11.x" + elif [[ "$current_ver" =~ ^3\.12 ]]; then current_pattern="3.12.x" + elif [[ "$current_ver" =~ ^3\.13 ]]; then current_pattern="3.13.x" + elif [[ "$current_ver" =~ ^4\.0 ]]; then current_pattern="4.0.x" + elif [[ "$current_ver" =~ ^4\.1 ]]; then current_pattern="4.1.x" + else + echo "Unsupported current version: $current_ver" >&2 + return 1 fi - - # Check if direct upgrade is supported - if [ -n "$current_pattern" ] && [ -n "${supported_paths[$current_pattern]}" ]; then - local target_pattern="${supported_paths[$current_pattern]}" - - # Check if target version matches the supported upgrade path - if [[ "$target_ver" =~ ^${target_pattern//x/} ]]; then - # Direct upgrade is supported - echo "" - return 0 + + local intermediate_versions=() + local step="$current_pattern" + + while [ -n "${supported_paths[$step]}" ]; do + step="${supported_paths[$step]}" + local actual_version="${step//.x/}" # remove .x suffix + + # Add step to the list + intermediate_versions+=("$actual_version") + + # Stop if this step matches the target + if [[ "$target_ver" =~ ^$actual_version(\.|$) ]]; then + break fi - - # Need to find intermediate path - local intermediate_versions="" - local current_step="$current_pattern" - - while [ -n "$current_step" ]; do - local next_step="${supported_paths[$current_step]}" - if [ -n "$next_step" ]; then - # Check if we've reached the target pattern BEFORE adding to intermediate versions - if [[ "$target_ver" =~ ^${next_step//x/} ]]; then - # We've found the target pattern, stop here - # Don't add this step to intermediate versions since it's the target - break - fi - - # Convert pattern to major.minor version for intermediate step - if [[ "$next_step" =~ ^3\. ]]; then - case "$next_step" in - "3.8.x") intermediate_versions="$intermediate_versions 3.8" ;; - "3.9.x") intermediate_versions="$intermediate_versions 3.9" ;; - "3.10.x") intermediate_versions="$intermediate_versions 3.10" ;; - "3.11.x") intermediate_versions="$intermediate_versions 3.11" ;; - "3.12.x") - # Special case: only add 3.12 if we're coming from 3.11.18 - if [ "$current_step" = "3.11.18" ]; then - intermediate_versions="$intermediate_versions 3.12" - fi - ;; - "3.13.x") intermediate_versions="$intermediate_versions 3.13" ;; - esac - elif [[ "$next_step" =~ ^4\. ]]; then - case "$next_step" in - "4.0.x") intermediate_versions="$intermediate_versions 4.0" ;; - "4.1.x") intermediate_versions="$intermediate_versions 4.1" ;; - esac - fi - - current_step="$next_step" - else - break - fi - done - - # Return intermediate versions (trimmed) - local final_result=$(echo "$intermediate_versions" | sed 's/^ *//;s/ *$//') - echo "$final_result" + done + + # Final validation: make sure path reached target major.minor + if [[ "${intermediate_versions[-1]}" =~ ^${target_ver%.*} ]]; then + echo "${intermediate_versions[*]}" return 0 + else + echo "No valid upgrade path from $current_ver to $target_ver" >&2 + return 1 fi - - # No supported path found - return 1 } # upgrade_rabbitmq_to_version: Upgrades RabbitMQ to a specific version From fc798ec9b8f3a82f29ce2cf31d52e05bf2c25c2f Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Fri, 22 Aug 2025 11:41:55 -0500 Subject: [PATCH 18/39] commented out upgrade func --- kubernetes/update-stack.sh | 54 +++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 6f6154d..78dbb72 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -397,42 +397,42 @@ update_rabbitmq_version() { echo "Will upgrade through intermediate versions: $upgrade_path" waggle_log info "RabbitMQ will upgrade through intermediate versions: $upgrade_path" - # # Upgrade through intermediate versions - # for version in $upgrade_path; do - # if ! upgrade_rabbitmq_to_version "$version"; then - # echo "Error: Failed to upgrade to intermediate version $version" - # return 1 - # fi - # done + # Upgrade through intermediate versions + for version in $upgrade_path; do + if ! upgrade_rabbitmq_to_version "$version"; then + echo "Error: Failed to upgrade to intermediate version $version" + return 1 + fi + done else echo "Direct upgrade path available" waggle_log info "RabbitMQ direct upgrade path available" fi # Finally upgrade to target version - # if ! upgrade_rabbitmq_to_version "$target_ver"; then - # echo "Error: Failed to upgrade to target version $target_ver" - # return 1 - # fi + if ! upgrade_rabbitmq_to_version "$target_ver"; then + echo "Error: Failed to upgrade to target version $target_ver" + return 1 + fi # Verify the upgrade was successful - # echo "Verifying upgrade was successful..." - # if ! new_version=$(kubectl get statefulset wes-rabbitmq -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null); then - # echo "Warning: Could not verify new version" - # waggle_log warn "Could not verify RabbitMQ new version" - # else - # new_ver=$(echo "$new_version" | sed 's/rabbitmq://' | sed 's/-management-alpine//') - # if [ "$new_ver" = "$target_ver" ]; then - # echo "Upgrade verification successful: RabbitMQ is now running version $new_ver" - # waggle_log info "RabbitMQ upgrade verification successful: now running version $new_ver" - # else - # echo "Warning: Upgrade verification failed. Expected: $target_ver, Got: $new_ver" - # waggle_log warn "RabbitMQ upgrade verification failed. Expected: $target_ver, Got: $new_ver" - # fi - # fi + echo "Verifying upgrade was successful..." + if ! new_version=$(kubectl get statefulset wes-rabbitmq -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null); then + echo "Warning: Could not verify new version" + waggle_log warn "Could not verify RabbitMQ new version" + else + new_ver=$(echo "$new_version" | sed 's/rabbitmq://' | sed 's/-management-alpine//') + if [ "$new_ver" = "$target_ver" ]; then + echo "Upgrade verification successful: RabbitMQ is now running version $new_ver" + waggle_log info "RabbitMQ upgrade verification successful: now running version $new_ver" + else + echo "Warning: Upgrade verification failed. Expected: $target_ver, Got: $new_ver" + waggle_log warn "RabbitMQ upgrade verification failed. Expected: $target_ver, Got: $new_ver" + fi + fi - # echo "RabbitMQ version upgrade completed successfully" - # waggle_log info "RabbitMQ version upgrade completed successfully: $current_ver -> $target_ver" + echo "RabbitMQ version upgrade completed successfully" + waggle_log info "RabbitMQ version upgrade completed successfully: $current_ver -> $target_ver" } update_wes() { From 243be2d6da7bf6eaf30147cd584bb4039916eef3 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Fri, 22 Aug 2025 11:58:27 -0500 Subject: [PATCH 19/39] added missing definitions --- kubernetes/update-stack.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 78dbb72..7c51a6e 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -584,6 +584,20 @@ EOF "durable": true, "auto_delete": false, "arguments": {} + }, + { + "name": "ansible", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "to-validator", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} } ], "exchanges": [ @@ -622,6 +636,15 @@ EOF "auto_delete": false, "internal": false, "arguments": {} + }, + { + "name": "to-validator", + "vhost": "/", + "type": "topic", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} } ], "bindings": [ From a1e5bc6840cc9adbd1164c3d7172baa3a8bd6f9d Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Fri, 22 Aug 2025 12:02:14 -0500 Subject: [PATCH 20/39] Enable deprecated features for management metrics collection --- kubernetes/update-stack.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 7c51a6e..e697513 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -486,6 +486,9 @@ log.console = true # mqtt config for lorawan mqtt.default_user = service mqtt.default_pass = service + +# enable deprecated features for management metrics collection +deprecated_features.permit.management_metrics_collection = true EOF WAGGLE_BEEHIVE_RABBITMQ_HOST=$(get_configmap_field waggle-config WAGGLE_BEEHIVE_RABBITMQ_HOST) From e3af0fa6e8d6a7bfbc8a1a21c60c9cce00bd4553 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Fri, 22 Aug 2025 13:47:44 -0500 Subject: [PATCH 21/39] Update RabbitMQ image version to 3.13.7 in wes-rabbitmq.yaml --- kubernetes/wes-rabbitmq.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/wes-rabbitmq.yaml b/kubernetes/wes-rabbitmq.yaml index 0438ade..55f98d9 100644 --- a/kubernetes/wes-rabbitmq.yaml +++ b/kubernetes/wes-rabbitmq.yaml @@ -39,7 +39,7 @@ spec: node-role.kubernetes.io/master: "true" containers: - name: wes-rabbitmq - image: rabbitmq:3.8.11-management-alpine + image: rabbitmq:3.13.7-management-alpine imagePullPolicy: IfNotPresent resources: limits: From 9b10b366d9a02617e35fad142d44e857f2fb0480 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Fri, 22 Aug 2025 13:52:23 -0500 Subject: [PATCH 22/39] Increase sleep duration to 3 minutes before enabling feature flags for RabbitMQ upgrade --- kubernetes/update-stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index e697513..789563b 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -267,7 +267,7 @@ upgrade_rabbitmq_to_version() { fi # Enable feature flags for next upgrade - sleep 1m + sleep 3m echo "Enabling feature flags for next upgrade..." kubectl exec wes-rabbitmq-0 -- rabbitmqctl enable_feature_flag all || true From 270448b3dde5a352deb34a4f4aea4cb7860c7198 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Tue, 26 Aug 2025 08:31:00 -0500 Subject: [PATCH 23/39] Refactor RabbitMQ upgrade functions: rename functions for consistency and add enable_rmq_ft_flags to handle feature flag enabling with retry logic. --- kubernetes/update-stack.sh | 53 +++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 22f74ad..86e7945 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -185,9 +185,9 @@ update_wes_plugins() { # 3. load generated configs / secrets using kustomize # if we buy into just using kustomize from the get go, then mostly only need step 3. -# determine_rabbitmq_upgrade_path: Determines the upgrade path between two RabbitMQ versions +# determine_rmq_upgrade_path: Determines the upgrade path between two RabbitMQ versions # Returns a space-separated list of intermediate versions, or empty string if direct upgrade -determine_rabbitmq_upgrade_path() { +determine_rmq_upgrade_path() { local current_ver="$1" local target_ver="$2" @@ -243,6 +243,36 @@ determine_rabbitmq_upgrade_path() { fi } +# enable_rmq_ft_flags: Enables feature flags for RabbitMQ +enable_rmq_ft_flags() { + local max_retries=5 + local retry_count=0 + local retry_delay=30 # Start with 30 seconds + while [ $retry_count -lt $max_retries ]; do + echo "Attempt $((retry_count + 1))/$max_retries to enable feature flags..." + + if kubectl exec wes-rabbitmq-0 -- rabbitmqctl enable_feature_flag all; then + echo "Successfully enabled feature flags" + echo "feature flags enabled:" + kubectl exec wes-rabbitmq-0 -- rabbitmqctl -q --formatter pretty_table list_feature_flags || true + break + else + retry_count=$((retry_count + 1)) + + if [ $retry_count -lt $max_retries ]; then + echo "Failed to enable feature flags (attempt $retry_count/$max_retries)" + echo "Retrying in ${retry_delay} seconds..." + waggle_log warn "Feature flag enable failed, retrying in ${retry_delay}s (attempt $retry_count/$max_retries)" + sleep $retry_delay + retry_delay=$((retry_delay * 2)) # Exponential backoff + else + echo "Warning: Failed to enable feature flags after $max_retries attempts" + waggle_log warn "Failed to enable RabbitMQ feature flags after $max_retries attempts" + fi + fi + done +} + # upgrade_rabbitmq_to_version: Upgrades RabbitMQ to a specific version upgrade_rabbitmq_to_version() { local target_ver="$1" @@ -270,13 +300,13 @@ upgrade_rabbitmq_to_version() { # Enable feature flags for next upgrade sleep 3m echo "Enabling feature flags for next upgrade..." - kubectl exec wes-rabbitmq-0 -- rabbitmqctl enable_feature_flag all || true + enable_rmq_ft_flags echo "Successfully upgraded to $target_ver" return 0 } -# update_rabbitmq_version: Main function to handle RabbitMQ version upgrades +# update_rmq_version: Main function to handle RabbitMQ version upgrades # This function handles: # - Upgrades following RabbitMQ's official supported upgrade paths # - Only one-hop upgrades (e.g., 3.8.x -> 3.9.x, 3.13.x -> 4.0.x) @@ -284,7 +314,7 @@ upgrade_rabbitmq_to_version() { # - Feature flag enabling for compatibility # - Rollout verification and health checks # - Comprehensive logging via waggle_log -update_rabbitmq_version() { +update_rmq_version() { echo "checking if RabbitMQ version upgrade is needed..." waggle_log info "starting RabbitMQ version upgrade check" @@ -373,18 +403,11 @@ update_rabbitmq_version() { # Enable feature flags for upgrade echo "Enabling feature flags for upgrade..." - if ! kubectl exec wes-rabbitmq-0 -- rabbitmqctl enable_feature_flag all; then - echo "Warning: Failed to enable feature flags, continuing with upgrade..." - waggle_log warn "Failed to enable RabbitMQ feature flags, continuing with upgrade" - fi - - # Verify feature flags are enabled - echo "Verifying feature flags..." - kubectl exec wes-rabbitmq-0 -- rabbitmqctl -q --formatter pretty_table list_feature_flags || true + enable_rmq_ft_flags # Determine upgrade path echo "Determining upgrade path..." - upgrade_path=$(determine_rabbitmq_upgrade_path "$current_ver" "$target_ver") + upgrade_path=$(determine_rmq_upgrade_path "$current_ver" "$target_ver") local exit_code=$? if [ $exit_code -ne 0 ]; then @@ -1075,7 +1098,7 @@ update_wes_tools update_node_secrets update_node_manifest_v2 update_data_config -update_rabbitmq_version +update_rmq_version update_wes_plugins update_wes update_influxdb_retention From 0cca3692d68c1ae3f4173bfc98f63b257a5ab9ea Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Tue, 26 Aug 2025 09:04:33 -0500 Subject: [PATCH 24/39] Fix logging format in enable_rmq_ft_flags function --- kubernetes/update-stack.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 86e7945..23810a6 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -266,8 +266,8 @@ enable_rmq_ft_flags() { sleep $retry_delay retry_delay=$((retry_delay * 2)) # Exponential backoff else - echo "Warning: Failed to enable feature flags after $max_retries attempts" - waggle_log warn "Failed to enable RabbitMQ feature flags after $max_retries attempts" + echo "Warning: Failed to enable feature flags after ${max_retries} attempts" + waggle_log warn "Failed to enable RabbitMQ feature flags after ${max_retries} attempts" fi fi done From d789bac0aa5d27c13ce1a03aa52b2f0ae31300dc Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Tue, 26 Aug 2025 09:40:24 -0500 Subject: [PATCH 25/39] remove duplicate logs --- kubernetes/update-stack.sh | 50 +++++++++----------------------------- 1 file changed, 11 insertions(+), 39 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 23810a6..d3a4d69 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -252,21 +252,17 @@ enable_rmq_ft_flags() { echo "Attempt $((retry_count + 1))/$max_retries to enable feature flags..." if kubectl exec wes-rabbitmq-0 -- rabbitmqctl enable_feature_flag all; then - echo "Successfully enabled feature flags" - echo "feature flags enabled:" + waggle_log info "Successfully enabled RabbitMQ feature flags" kubectl exec wes-rabbitmq-0 -- rabbitmqctl -q --formatter pretty_table list_feature_flags || true break else retry_count=$((retry_count + 1)) if [ $retry_count -lt $max_retries ]; then - echo "Failed to enable feature flags (attempt $retry_count/$max_retries)" - echo "Retrying in ${retry_delay} seconds..." waggle_log warn "Feature flag enable failed, retrying in ${retry_delay}s (attempt $retry_count/$max_retries)" sleep $retry_delay retry_delay=$((retry_delay * 2)) # Exponential backoff else - echo "Warning: Failed to enable feature flags after ${max_retries} attempts" waggle_log warn "Failed to enable RabbitMQ feature flags after ${max_retries} attempts" fi fi @@ -277,32 +273,29 @@ enable_rmq_ft_flags() { upgrade_rabbitmq_to_version() { local target_ver="$1" - echo "Upgrading to version $target_ver..." waggle_log info "RabbitMQ upgrading to version $target_ver" kubectl set image statefulset/wes-rabbitmq wes-rabbitmq="rabbitmq:${target_ver}-management-alpine" - echo "Waiting for $target_ver rollout to complete..." + waggle_log info "Waiting for $target_ver rollout to complete..." if ! kubectl rollout status statefulset/wes-rabbitmq; then - echo "Error: $target_ver rollout failed" waggle_log err "RabbitMQ $target_ver rollout failed" return 1 fi # Wait for RabbitMQ to be running - echo "Waiting for RabbitMQ to be running after $target_ver upgrade..." + waggle_log info "Waiting for RabbitMQ to be running after $target_ver upgrade..." if ! kubectl wait --for=condition=Ready pod/wes-rabbitmq-0 --timeout=600s; then - echo "Error: RabbitMQ not ready after $target_ver upgrade" waggle_log err "RabbitMQ not ready after $target_ver upgrade" return 1 fi # Enable feature flags for next upgrade sleep 3m - echo "Enabling feature flags for next upgrade..." + waggle_log info "Enabling feature flags for next upgrade..." enable_rmq_ft_flags - echo "Successfully upgraded to $target_ver" + waggle_log info "Successfully upgraded to $target_ver" return 0 } @@ -315,12 +308,10 @@ upgrade_rabbitmq_to_version() { # - Rollout verification and health checks # - Comprehensive logging via waggle_log update_rmq_version() { - echo "checking if RabbitMQ version upgrade is needed..." waggle_log info "starting RabbitMQ version upgrade check" # Get current running version if ! current_version=$(kubectl get statefulset wes-rabbitmq -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null); then - echo "RabbitMQ StatefulSet not found, skipping version upgrade" waggle_log info "RabbitMQ StatefulSet not found, skipping version upgrade" return 0 fi @@ -330,7 +321,6 @@ update_rmq_version() { # Validate current version format if ! echo "$current_ver" | grep -qE '^[0-9]+\.[0-9]+(\.[0-9]+)?$'; then - echo "Error: Invalid current version format: $current_ver" waggle_log err "Invalid RabbitMQ current version format: $current_ver" return 1 fi @@ -340,18 +330,15 @@ update_rmq_version() { # Validate target version format if ! echo "$target_ver" | grep -qE '^[0-9]+\.[0-9]+(\.[0-9]+)?$'; then - echo "Error: Invalid target version format: $target_ver" waggle_log err "Invalid RabbitMQ target version format: $target_ver" return 1 fi if [ "$current_ver" = "$target_ver" ]; then - echo "RabbitMQ is already at target version $target_ver" waggle_log info "RabbitMQ already at target version $target_ver" return 0 fi - echo "RabbitMQ version upgrade needed: $current_ver -> $target_ver" waggle_log info "RabbitMQ version upgrade needed: $current_ver -> $target_ver" # Check if this is a downgrade (not supported) @@ -361,101 +348,86 @@ update_rmq_version() { local target_minor=$(echo "$target_ver" | cut -d. -f2) if [ "$current_major" -gt "$target_major" ] || ([ "$current_major" -eq "$target_major" ] && [ "$current_minor" -gt "$target_minor" ]); then - echo "Error: Downgrading RabbitMQ from $current_ver to $target_ver is not supported" waggle_log err "RabbitMQ downgrade not supported: $current_ver -> $target_ver" return 1 fi # Ensure RabbitMQ is running before attempting upgrade - echo "Ensuring RabbitMQ is running..." + waggle_log info "Ensuring RabbitMQ is running..." if ! kubectl wait --for=condition=ready pod/wes-rabbitmq-0 --timeout=60s; then - echo "Error: RabbitMQ is not ready, cannot proceed with upgrade" waggle_log err "RabbitMQ is not ready, cannot proceed with upgrade" return 1 fi # Backup data before major upgrades - echo "Creating backup of RabbitMQ data..." waggle_log info "Creating backup of RabbitMQ data" # Check available disk space (need at least 1GB free) available_space=$(df . | awk 'NR==2 {print $4}') if [ "$available_space" -lt 1048576 ]; then # 1GB in KB - echo "Warning: Low disk space available (${available_space}KB), backup may fail" waggle_log warn "Low disk space for RabbitMQ backup: ${available_space}KB" fi backup_file="/var/backups/rabbitmq-data-$(date +%F).tar.gz" if ! kubectl exec wes-rabbitmq-0 -- tar czf /tmp/rabbitmq-data.tar.gz /var/lib/rabbitmq/mnesia; then - echo "Error: Failed to create backup, stopping upgrade..." waggle_log err "Failed to create RabbitMQ backup, stopping upgrade" return 1 fi if ! kubectl cp wes-rabbitmq-0:/tmp/rabbitmq-data.tar.gz "$backup_file"; then - echo "Error: Failed to copy backup to host, stopping upgrade..." waggle_log err "Failed to copy RabbitMQ backup to host, stopping upgrade" return 1 fi - echo "Backup created: $backup_file" waggle_log info "RabbitMQ backup created: $backup_file" # Enable feature flags for upgrade - echo "Enabling feature flags for upgrade..." + waggle_log info "Enabling feature flags for upgrade..." enable_rmq_ft_flags # Determine upgrade path - echo "Determining upgrade path..." + waggle_log info "Determining upgrade path..." upgrade_path=$(determine_rmq_upgrade_path "$current_ver" "$target_ver") local exit_code=$? if [ $exit_code -ne 0 ]; then - echo "Error: No supported upgrade path found from $current_ver to $target_ver" waggle_log err "No supported RabbitMQ upgrade path found: $current_ver -> $target_ver" return 1 fi # Execute upgrades if [ -n "$upgrade_path" ]; then - echo "Will upgrade through intermediate versions: $upgrade_path" waggle_log info "RabbitMQ will upgrade through intermediate versions: $upgrade_path" # Upgrade through intermediate versions for version in $upgrade_path; do if ! upgrade_rabbitmq_to_version "$version"; then - echo "Error: Failed to upgrade to intermediate version $version" + waggle_log err "Failed to upgrade to intermediate version $version" return 1 fi done else - echo "Direct upgrade path available" waggle_log info "RabbitMQ direct upgrade path available" fi # Finally upgrade to target version if ! upgrade_rabbitmq_to_version "$target_ver"; then - echo "Error: Failed to upgrade to target version $target_ver" + waggle_log err "Failed to upgrade to target version $target_ver" return 1 fi # Verify the upgrade was successful - echo "Verifying upgrade was successful..." + waggle_log info "Verifying upgrade was successful..." if ! new_version=$(kubectl get statefulset wes-rabbitmq -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null); then - echo "Warning: Could not verify new version" waggle_log warn "Could not verify RabbitMQ new version" else new_ver=$(echo "$new_version" | sed 's/rabbitmq://' | sed 's/-management-alpine//') if [ "$new_ver" = "$target_ver" ]; then - echo "Upgrade verification successful: RabbitMQ is now running version $new_ver" waggle_log info "RabbitMQ upgrade verification successful: now running version $new_ver" else - echo "Warning: Upgrade verification failed. Expected: $target_ver, Got: $new_ver" waggle_log warn "RabbitMQ upgrade verification failed. Expected: $target_ver, Got: $new_ver" fi fi - - echo "RabbitMQ version upgrade completed successfully" waggle_log info "RabbitMQ version upgrade completed successfully: $current_ver -> $target_ver" } From f2c64c78b8a079df2017713a591e66decf012988 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Tue, 26 Aug 2025 09:58:52 -0500 Subject: [PATCH 26/39] reload RabbitMQ definitions after each update --- kubernetes/update-stack.sh | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index d3a4d69..f4e64c3 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -269,6 +269,31 @@ enable_rmq_ft_flags() { done } +# reload_rmq_definitions: Reloads RabbitMQ definitions +reload_rmq_definitions() { + local max_retries=5 + local retry_count=0 + local retry_delay=30 # Start with 30 seconds + + while [ $retry_count -lt $max_retries ]; do + echo "Attempt $((retry_count + 1))/$max_retries to reload RabbitMQ definitions..." + + if kubectl exec wes-rabbitmq-0 -- rabbitmqctl eval 'rabbit_definitions:load(<<"/etc/rabbitmq/definitions.json">>)'; then + waggle_log info "Successfully reloaded RabbitMQ definitions" + else + retry_count=$((retry_count + 1)) + + if [ $retry_count -lt $max_retries ]; then + waggle_log warn "Definitions reload failed, retrying in ${retry_delay}s (attempt $retry_count/$max_retries)" + sleep $retry_delay + retry_delay=$((retry_delay * 2)) # Exponential backoff + else + waggle_log warn "Failed to reload RabbitMQ definitions after ${max_retries} attempts" + fi + fi + done +} + # upgrade_rabbitmq_to_version: Upgrades RabbitMQ to a specific version upgrade_rabbitmq_to_version() { local target_ver="$1" @@ -290,8 +315,14 @@ upgrade_rabbitmq_to_version() { return 1 fi - # Enable feature flags for next upgrade + # wait for startup sleep 3m + + # reload definitions for new version + waggle_log info "Reloading RabbitMQ definitions..." + reload_rmq_definitions + + # Enable feature flags for next upgrade waggle_log info "Enabling feature flags for next upgrade..." enable_rmq_ft_flags From a9fb9323da81a6fb41f64c1c9839b09b0ec8ab1e Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Tue, 26 Aug 2025 10:14:53 -0500 Subject: [PATCH 27/39] order of operations was wrong --- kubernetes/update-stack.sh | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index f4e64c3..47e4773 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -269,31 +269,6 @@ enable_rmq_ft_flags() { done } -# reload_rmq_definitions: Reloads RabbitMQ definitions -reload_rmq_definitions() { - local max_retries=5 - local retry_count=0 - local retry_delay=30 # Start with 30 seconds - - while [ $retry_count -lt $max_retries ]; do - echo "Attempt $((retry_count + 1))/$max_retries to reload RabbitMQ definitions..." - - if kubectl exec wes-rabbitmq-0 -- rabbitmqctl eval 'rabbit_definitions:load(<<"/etc/rabbitmq/definitions.json">>)'; then - waggle_log info "Successfully reloaded RabbitMQ definitions" - else - retry_count=$((retry_count + 1)) - - if [ $retry_count -lt $max_retries ]; then - waggle_log warn "Definitions reload failed, retrying in ${retry_delay}s (attempt $retry_count/$max_retries)" - sleep $retry_delay - retry_delay=$((retry_delay * 2)) # Exponential backoff - else - waggle_log warn "Failed to reload RabbitMQ definitions after ${max_retries} attempts" - fi - fi - done -} - # upgrade_rabbitmq_to_version: Upgrades RabbitMQ to a specific version upgrade_rabbitmq_to_version() { local target_ver="$1" @@ -318,10 +293,6 @@ upgrade_rabbitmq_to_version() { # wait for startup sleep 3m - # reload definitions for new version - waggle_log info "Reloading RabbitMQ definitions..." - reload_rmq_definitions - # Enable feature flags for next upgrade waggle_log info "Enabling feature flags for next upgrade..." enable_rmq_ft_flags @@ -1101,9 +1072,9 @@ update_wes_tools update_node_secrets update_node_manifest_v2 update_data_config -update_rmq_version update_wes_plugins update_wes +update_rmq_version update_influxdb_retention clean_manifestv2_cm clean_slash_run_tempfs From 421d96bce43356c9c3177704b364d5d36e027a0f Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Tue, 26 Aug 2025 10:48:05 -0500 Subject: [PATCH 28/39] Add separate RabbitMQ secrets management. This change improves rmq configurations during updates. --- kubernetes/update-stack.sh | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 47e4773..a06aba0 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -787,6 +787,26 @@ data: WAGGLE_NODE_ID: "${WAGGLE_NODE_ID}" WAGGLE_NODE_VSN: "${WAGGLE_NODE_VSN}" EOF + + # Create wes-rabbitmq-secrets.yaml file to be able to apply separately + cat > wes-rabbitmq-secrets.yaml < Date: Tue, 26 Aug 2025 10:53:44 -0500 Subject: [PATCH 29/39] remove whitespace --- kubernetes/update-stack.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index a06aba0..40f51df 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -788,11 +788,10 @@ data: WAGGLE_NODE_VSN: "${WAGGLE_NODE_VSN}" EOF - # Create wes-rabbitmq-secrets.yaml file to be able to apply separately + # seperate rmq secrets to be able to apply separately cat > wes-rabbitmq-secrets.yaml < Date: Tue, 26 Aug 2025 13:52:05 -0500 Subject: [PATCH 30/39] had wrong flag --- kubernetes/update-stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 40f51df..050b131 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -804,7 +804,7 @@ secretGenerator: EOF # if rabbitmq version is updated, update version - kubectl apply -f wes-rabbitmq-secrets.yaml + kubectl apply -k wes-rabbitmq-secrets.yaml update_rmq_version # HACK(sean) at some point, kustomize deprecated env: for envs: in the configmap / secret generators. From 428c8f11be251364e0f4de66aca38e1cb4fac79c Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Tue, 26 Aug 2025 13:56:57 -0500 Subject: [PATCH 31/39] Refactor RabbitMQ secrets management by organizing files into a dedicated directory --- kubernetes/update-stack.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 050b131..5e09efc 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -789,7 +789,8 @@ data: EOF # seperate rmq secrets to be able to apply separately - cat > wes-rabbitmq-secrets.yaml < wes-rabbitmq-secrets/kustomization.yaml < Date: Tue, 26 Aug 2025 13:58:21 -0500 Subject: [PATCH 32/39] Update RabbitMQ configuration file paths --- kubernetes/update-stack.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 5e09efc..00b3c33 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -796,12 +796,12 @@ kind: Kustomization secretGenerator: - name: wes-rabbitmq-config files: - - configs/rabbitmq/rabbitmq.conf - - configs/rabbitmq/definitions.json - - configs/rabbitmq/enabled_plugins - - configs/rabbitmq/cacert.pem - - configs/rabbitmq/cert.pem - - configs/rabbitmq/key.pem + - ../configs/rabbitmq/rabbitmq.conf + - ../configs/rabbitmq/definitions.json + - ../configs/rabbitmq/enabled_plugins + - ../configs/rabbitmq/cacert.pem + - ../configs/rabbitmq/cert.pem + - ../configs/rabbitmq/key.pem EOF # if rabbitmq version is updated, update version From 9ed0ae89d79aeb5369559c82e03921910b732159 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Tue, 26 Aug 2025 14:01:06 -0500 Subject: [PATCH 33/39] Organize RabbitMQ secrets into a structured directory --- kubernetes/update-stack.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 00b3c33..1563082 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -789,19 +789,20 @@ data: EOF # seperate rmq secrets to be able to apply separately - mkdir -p wes-rabbitmq-secrets + mkdir -p wes-rabbitmq-secrets/configs/rabbitmq + cp configs/rabbitmq/* wes-rabbitmq-secrets/configs/rabbitmq/ cat > wes-rabbitmq-secrets/kustomization.yaml < Date: Tue, 26 Aug 2025 14:43:01 -0500 Subject: [PATCH 34/39] revert back kustomize files and change update rmq logic --- kubernetes/update-stack.sh | 48 +++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 1563082..0e6621c 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -302,6 +302,7 @@ upgrade_rabbitmq_to_version() { } # update_rmq_version: Main function to handle RabbitMQ version upgrades +# takes current version as an argument # This function handles: # - Upgrades following RabbitMQ's official supported upgrade paths # - Only one-hop upgrades (e.g., 3.8.x -> 3.9.x, 3.13.x -> 4.0.x) @@ -310,11 +311,11 @@ upgrade_rabbitmq_to_version() { # - Rollout verification and health checks # - Comprehensive logging via waggle_log update_rmq_version() { + local current_version="$1" waggle_log info "starting RabbitMQ version upgrade check" - - # Get current running version - if ! current_version=$(kubectl get statefulset wes-rabbitmq -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null); then - waggle_log info "RabbitMQ StatefulSet not found, skipping version upgrade" + + if [ -z "$current_version" ]; then + waggle_log err "RabbitMQ current version not found, skipping version upgrade" return 0 fi @@ -787,27 +788,6 @@ data: WAGGLE_NODE_ID: "${WAGGLE_NODE_ID}" WAGGLE_NODE_VSN: "${WAGGLE_NODE_VSN}" EOF - - # seperate rmq secrets to be able to apply separately - mkdir -p wes-rabbitmq-secrets/configs/rabbitmq - cp configs/rabbitmq/* wes-rabbitmq-secrets/configs/rabbitmq/ - cat > wes-rabbitmq-secrets/kustomization.yaml </dev/null); then + waggle_log info "RabbitMQ StatefulSet not found" + fi + echo "performing one-time operation - clean old chirpstack" # NOTE(Joe) this is a work-around to remove old deployments when converting to statefulsets # this can safely be removed after all nodes have moved to the "statefulset" postgresql and redis setups @@ -930,6 +921,9 @@ EOF # wait a moment before checking for images sleep 10 k3s crictl images | awk '$2 ~ // {print $3}' | xargs -r k3s crictl rmi || true + + # if rabbitmq version is updated, update version + update_rmq_version "${rmq_current_version}" } get_configmap_field() { From 66fffe635b455fda77c686fbb9a956e3c94198c3 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Tue, 26 Aug 2025 14:49:14 -0500 Subject: [PATCH 35/39] Rename upgrade function to update for clarity and ensure current version is updated before proceeding with RabbitMQ upgrades. --- kubernetes/update-stack.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 0e6621c..4f6ab37 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -270,7 +270,7 @@ enable_rmq_ft_flags() { } # upgrade_rabbitmq_to_version: Upgrades RabbitMQ to a specific version -upgrade_rabbitmq_to_version() { +update_rabbitmq_to_version() { local target_ver="$1" waggle_log info "RabbitMQ upgrading to version $target_ver" @@ -354,6 +354,12 @@ update_rmq_version() { waggle_log err "RabbitMQ downgrade not supported: $current_ver -> $target_ver" return 1 fi + + # make sure to start from current version + if ! update_rabbitmq_to_version "$current_ver"; then + waggle_log err "Failed to update to current version $current_ver" + return 1 + fi # Ensure RabbitMQ is running before attempting upgrade waggle_log info "Ensuring RabbitMQ is running..." @@ -404,7 +410,7 @@ update_rmq_version() { # Upgrade through intermediate versions for version in $upgrade_path; do - if ! upgrade_rabbitmq_to_version "$version"; then + if ! update_rabbitmq_to_version "$version"; then waggle_log err "Failed to upgrade to intermediate version $version" return 1 fi @@ -414,7 +420,7 @@ update_rmq_version() { fi # Finally upgrade to target version - if ! upgrade_rabbitmq_to_version "$target_ver"; then + if ! update_rabbitmq_to_version "$target_ver"; then waggle_log err "Failed to upgrade to target version $target_ver" return 1 fi From 9b86d42e6625e3ebeec7a503dce24a464b66b009 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Tue, 26 Aug 2025 14:54:01 -0500 Subject: [PATCH 36/39] changed logs for update_rabbitmq_to_version --- kubernetes/update-stack.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 4f6ab37..2493d41 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -273,7 +273,7 @@ enable_rmq_ft_flags() { update_rabbitmq_to_version() { local target_ver="$1" - waggle_log info "RabbitMQ upgrading to version $target_ver" + waggle_log info "RabbitMQ updating to version $target_ver" kubectl set image statefulset/wes-rabbitmq wes-rabbitmq="rabbitmq:${target_ver}-management-alpine" waggle_log info "Waiting for $target_ver rollout to complete..." @@ -284,9 +284,9 @@ update_rabbitmq_to_version() { fi # Wait for RabbitMQ to be running - waggle_log info "Waiting for RabbitMQ to be running after $target_ver upgrade..." + waggle_log info "Waiting for RabbitMQ to be running after $target_ver update..." if ! kubectl wait --for=condition=Ready pod/wes-rabbitmq-0 --timeout=600s; then - waggle_log err "RabbitMQ not ready after $target_ver upgrade" + waggle_log err "RabbitMQ not ready after $target_ver update" return 1 fi @@ -297,7 +297,7 @@ update_rabbitmq_to_version() { waggle_log info "Enabling feature flags for next upgrade..." enable_rmq_ft_flags - waggle_log info "Successfully upgraded to $target_ver" + waggle_log info "Successfully updated to $target_ver" return 0 } From 7209385d32d99eb2cf224cd0bbad68047a40e9d9 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Tue, 26 Aug 2025 15:16:25 -0500 Subject: [PATCH 37/39] Refactor RabbitMQ upgrade process by consolidating configuration management and ensuring version checks are performed before applying updates. --- kubernetes/update-stack.sh | 343 ++++++++++++++++++++----------------- 1 file changed, 182 insertions(+), 161 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index 2493d41..e86491b 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -311,156 +311,13 @@ update_rabbitmq_to_version() { # - Rollout verification and health checks # - Comprehensive logging via waggle_log update_rmq_version() { - local current_version="$1" - waggle_log info "starting RabbitMQ version upgrade check" - if [ -z "$current_version" ]; then - waggle_log err "RabbitMQ current version not found, skipping version upgrade" - return 0 - fi - - # Extract version from image (e.g., "rabbitmq:3.8.11-management-alpine" -> "3.8.11") - current_ver=$(echo "$current_version" | sed 's/rabbitmq://' | sed 's/-management-alpine//') - - # Validate current version format - if ! echo "$current_ver" | grep -qE '^[0-9]+\.[0-9]+(\.[0-9]+)?$'; then - waggle_log err "Invalid RabbitMQ current version format: $current_ver" - return 1 - fi - - # Get target version from the YAML file - target_ver=$(grep "image: rabbitmq:" wes-rabbitmq.yaml | sed 's/.*rabbitmq://' | sed 's/-management-alpine//') - - # Validate target version format - if ! echo "$target_ver" | grep -qE '^[0-9]+\.[0-9]+(\.[0-9]+)?$'; then - waggle_log err "Invalid RabbitMQ target version format: $target_ver" - return 1 - fi - - if [ "$current_ver" = "$target_ver" ]; then - waggle_log info "RabbitMQ already at target version $target_ver" - return 0 - fi - - waggle_log info "RabbitMQ version upgrade needed: $current_ver -> $target_ver" - - # Check if this is a downgrade (not supported) - local current_major=$(echo "$current_ver" | cut -d. -f1) - local current_minor=$(echo "$current_ver" | cut -d. -f2) - local target_major=$(echo "$target_ver" | cut -d. -f1) - local target_minor=$(echo "$target_ver" | cut -d. -f2) - - if [ "$current_major" -gt "$target_major" ] || ([ "$current_major" -eq "$target_major" ] && [ "$current_minor" -gt "$target_minor" ]); then - waggle_log err "RabbitMQ downgrade not supported: $current_ver -> $target_ver" - return 1 - fi + echo "generating rabbitmq configs" - # make sure to start from current version - if ! update_rabbitmq_to_version "$current_ver"; then - waggle_log err "Failed to update to current version $current_ver" - return 1 - fi - - # Ensure RabbitMQ is running before attempting upgrade - waggle_log info "Ensuring RabbitMQ is running..." - if ! kubectl wait --for=condition=ready pod/wes-rabbitmq-0 --timeout=60s; then - waggle_log err "RabbitMQ is not ready, cannot proceed with upgrade" - return 1 - fi - - # Backup data before major upgrades - waggle_log info "Creating backup of RabbitMQ data" - - # Check available disk space (need at least 1GB free) - available_space=$(df . | awk 'NR==2 {print $4}') - if [ "$available_space" -lt 1048576 ]; then # 1GB in KB - waggle_log warn "Low disk space for RabbitMQ backup: ${available_space}KB" - fi - - backup_file="/var/backups/rabbitmq-data-$(date +%F).tar.gz" - if ! kubectl exec wes-rabbitmq-0 -- tar czf /tmp/rabbitmq-data.tar.gz /var/lib/rabbitmq/mnesia; then - waggle_log err "Failed to create RabbitMQ backup, stopping upgrade" - return 1 - fi - - if ! kubectl cp wes-rabbitmq-0:/tmp/rabbitmq-data.tar.gz "$backup_file"; then - waggle_log err "Failed to copy RabbitMQ backup to host, stopping upgrade" - return 1 - fi - - waggle_log info "RabbitMQ backup created: $backup_file" - - # Enable feature flags for upgrade - waggle_log info "Enabling feature flags for upgrade..." - enable_rmq_ft_flags - - # Determine upgrade path - waggle_log info "Determining upgrade path..." - upgrade_path=$(determine_rmq_upgrade_path "$current_ver" "$target_ver") - - local exit_code=$? - if [ $exit_code -ne 0 ]; then - waggle_log err "No supported RabbitMQ upgrade path found: $current_ver -> $target_ver" - return 1 - fi - - # Execute upgrades - if [ -n "$upgrade_path" ]; then - waggle_log info "RabbitMQ will upgrade through intermediate versions: $upgrade_path" - - # Upgrade through intermediate versions - for version in $upgrade_path; do - if ! update_rabbitmq_to_version "$version"; then - waggle_log err "Failed to upgrade to intermediate version $version" - return 1 - fi - done - else - waggle_log info "RabbitMQ direct upgrade path available" - fi - - # Finally upgrade to target version - if ! update_rabbitmq_to_version "$target_ver"; then - waggle_log err "Failed to upgrade to target version $target_ver" - return 1 - fi - - # Verify the upgrade was successful - waggle_log info "Verifying upgrade was successful..." - if ! new_version=$(kubectl get statefulset wes-rabbitmq -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null); then - waggle_log warn "Could not verify RabbitMQ new version" - else - new_ver=$(echo "$new_version" | sed 's/rabbitmq://' | sed 's/-management-alpine//') - if [ "$new_ver" = "$target_ver" ]; then - waggle_log info "RabbitMQ upgrade verification successful: now running version $new_ver" - else - waggle_log warn "RabbitMQ upgrade verification failed. Expected: $target_ver, Got: $new_ver" - fi - fi - waggle_log info "RabbitMQ version upgrade completed successfully: $current_ver -> $target_ver" -} - -update_wes() { - echo "updating wes" - - echo "generating wes configs" - - mkdir -p configs configs/rabbitmq configs/upload-agent + mkdir -p configs configs/rabbitmq # make all config directories private find configs -type d | xargs -r chmod 700 - # generate identity config for kustomize - # NOTE we are ignoring the WAGGLE_NODE_ID in waggle-config and creating from local file - WAGGLE_NODE_ID=$(node_id) - WAGGLE_NODE_VSN=$(node_vsn) - cat > configs/wes-identity.env < wes-rabbitmq-config/kustomization.yaml </dev/null); then + waggle_log info "RabbitMQ StatefulSet not found" + fi + + if [ -z "$current_version" ]; then + waggle_log err "RabbitMQ current version not found, skipping version upgrade" + kubectl apply -k wes-rabbitmq-config + return 0 + fi + + # Extract version from image (e.g., "rabbitmq:3.8.11-management-alpine" -> "3.8.11") + current_ver=$(echo "$current_version" | sed 's/rabbitmq://' | sed 's/-management-alpine//') + + # Validate current version format + if ! echo "$current_ver" | grep -qE '^[0-9]+\.[0-9]+(\.[0-9]+)?$'; then + waggle_log err "Invalid RabbitMQ current version format: $current_ver" + kubectl apply -k wes-rabbitmq-config + return 1 + fi + + # Get target version from the YAML file + target_ver=$(grep "image: rabbitmq:" wes-rabbitmq.yaml | sed 's/.*rabbitmq://' | sed 's/-management-alpine//') + + # Validate target version format + if ! echo "$target_ver" | grep -qE '^[0-9]+\.[0-9]+(\.[0-9]+)?$'; then + waggle_log err "Invalid RabbitMQ target version format: $target_ver" + kubectl apply -k wes-rabbitmq-config + return 1 + fi + + if [ "$current_ver" = "$target_ver" ]; then + waggle_log info "RabbitMQ already at target version $target_ver" + kubectl apply -k wes-rabbitmq-config + return 0 + fi + + waggle_log info "RabbitMQ version upgrade needed: $current_ver -> $target_ver" + + # Check if this is a downgrade (not supported) + local current_major=$(echo "$current_ver" | cut -d. -f1) + local current_minor=$(echo "$current_ver" | cut -d. -f2) + local target_major=$(echo "$target_ver" | cut -d. -f1) + local target_minor=$(echo "$target_ver" | cut -d. -f2) + + if [ "$current_major" -gt "$target_major" ] || ([ "$current_major" -eq "$target_major" ] && [ "$current_minor" -gt "$target_minor" ]); then + waggle_log err "RabbitMQ downgrade not supported: $current_ver -> $target_ver" + return 1 + fi + + # make sure to start from current version + if ! update_rabbitmq_to_version "$current_ver"; then + waggle_log err "Failed to update to current version $current_ver" + return 1 + fi + + # Ensure RabbitMQ is running before attempting upgrade + waggle_log info "Ensuring RabbitMQ is running..." + if ! kubectl wait --for=condition=ready pod/wes-rabbitmq-0 --timeout=60s; then + waggle_log err "RabbitMQ is not ready, cannot proceed with upgrade" + return 1 + fi + + # Backup data before major upgrades + waggle_log info "Creating backup of RabbitMQ data" + + # Check available disk space (need at least 1GB free) + available_space=$(df . | awk 'NR==2 {print $4}') + if [ "$available_space" -lt 1048576 ]; then # 1GB in KB + waggle_log warn "Low disk space for RabbitMQ backup: ${available_space}KB" + fi + + backup_file="/var/backups/rabbitmq-data-$(date +%F).tar.gz" + if ! kubectl exec wes-rabbitmq-0 -- tar czf /tmp/rabbitmq-data.tar.gz /var/lib/rabbitmq/mnesia; then + waggle_log err "Failed to create RabbitMQ backup, stopping upgrade" + return 1 + fi + + if ! kubectl cp wes-rabbitmq-0:/tmp/rabbitmq-data.tar.gz "$backup_file"; then + waggle_log err "Failed to copy RabbitMQ backup to host, stopping upgrade" + return 1 + fi + + waggle_log info "RabbitMQ backup created: $backup_file" + + # Enable feature flags for upgrade + waggle_log info "Enabling feature flags for upgrade..." + enable_rmq_ft_flags + + # Determine upgrade path + waggle_log info "Determining upgrade path..." + upgrade_path=$(determine_rmq_upgrade_path "$current_ver" "$target_ver") + + local exit_code=$? + if [ $exit_code -ne 0 ]; then + waggle_log err "No supported RabbitMQ upgrade path found: $current_ver -> $target_ver" + return 1 + fi + + # Execute upgrades + if [ -n "$upgrade_path" ]; then + waggle_log info "RabbitMQ will upgrade through intermediate versions: $upgrade_path" + + # Upgrade through intermediate versions + for version in $upgrade_path; do + if ! update_rabbitmq_to_version "$version"; then + waggle_log err "Failed to upgrade to intermediate version $version" + return 1 + fi + done + else + waggle_log info "RabbitMQ direct upgrade path available" + fi + + # Finally upgrade to target version + kubectl apply -k wes-rabbitmq-config + if ! update_rabbitmq_to_version "$target_ver"; then + waggle_log err "Failed to upgrade to target version $target_ver" + return 1 + fi + + # Verify the upgrade was successful + waggle_log info "Verifying upgrade was successful..." + if ! new_version=$(kubectl get statefulset wes-rabbitmq -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null); then + waggle_log warn "Could not verify RabbitMQ new version" + else + new_ver=$(echo "$new_version" | sed 's/rabbitmq://' | sed 's/-management-alpine//') + if [ "$new_ver" = "$target_ver" ]; then + waggle_log info "RabbitMQ upgrade verification successful: now running version $new_ver" + else + waggle_log warn "RabbitMQ upgrade verification failed. Expected: $target_ver, Got: $new_ver" + fi + fi + waggle_log info "RabbitMQ version upgrade completed successfully: $current_ver -> $target_ver" +} + +update_wes() { + echo "updating wes" + + echo "generating wes configs" + + mkdir -p configs configs/upload-agent + # make all config directories private + find configs -type d | xargs -r chmod 700 + + # generate identity config for kustomize + # NOTE we are ignoring the WAGGLE_NODE_ID in waggle-config and creating from local file + WAGGLE_NODE_ID=$(node_id) + WAGGLE_NODE_VSN=$(node_vsn) + cat > configs/wes-identity.env < configs/rabbitmq/cacert.pem get_secret_field wes-beehive-rabbitmq-tls cert.pem > configs/rabbitmq/cert.pem @@ -819,14 +854,6 @@ configMapGenerator: files: - configs/${NODE_MANIFEST_V2} secretGenerator: - - name: wes-rabbitmq-config - files: - - configs/rabbitmq/rabbitmq.conf - - configs/rabbitmq/definitions.json - - configs/rabbitmq/enabled_plugins - - configs/rabbitmq/cacert.pem - - configs/rabbitmq/cert.pem - - configs/rabbitmq/key.pem - name: wes-upload-agent-config files: - configs/upload-agent/ca.pub @@ -851,7 +878,6 @@ resources: - wes-device-labeler.yaml - wes-audio-server.yaml - wes-data-sharing-service.yaml - - wes-rabbitmq.yaml - wes-upload-agent.yaml - wes-metrics-agent.yaml - wes-plugin-scheduler.yaml @@ -879,11 +905,6 @@ EOF } ' - # Get current rabbitmq running version, before applying kustomize files - if ! rmq_current_version=$(kubectl get statefulset wes-rabbitmq -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null); then - waggle_log info "RabbitMQ StatefulSet not found" - fi - echo "performing one-time operation - clean old chirpstack" # NOTE(Joe) this is a work-around to remove old deployments when converting to statefulsets # this can safely be removed after all nodes have moved to the "statefulset" postgresql and redis setups @@ -902,6 +923,8 @@ EOF kubectl delete -f nvidia-device-plugin.yaml || true echo "deploying wes stack" + # if rabbitmq version is updated, update version + update_rmq_version # NOTE(sean) this is split as its own thing as the version of kubectl (v1.20.2+k3s1) we were using # when this was added didn't seem to support nesting other kustomization dirs as resources. # i'm deploying this first, to ensure to influxdb pvc issue doesn't stop this from running @@ -928,8 +951,6 @@ EOF sleep 10 k3s crictl images | awk '$2 ~ // {print $3}' | xargs -r k3s crictl rmi || true - # if rabbitmq version is updated, update version - update_rmq_version "${rmq_current_version}" } get_configmap_field() { From eb1f7396d32ccd8717080fc3af86062735837888 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Tue, 26 Aug 2025 17:38:12 -0500 Subject: [PATCH 38/39] Add version check for RabbitMQ upgrade path to ensure target major and minor versions are reached before proceeding with updates. --- kubernetes/update-stack.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index e86491b..a5b857f 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -701,6 +701,15 @@ EOF # Upgrade through intermediate versions for version in $upgrade_path; do + # break if target version major & minor is in the version + local target_major=$(echo "$target_ver" | cut -d. -f1) + local target_minor=$(echo "$target_ver" | cut -d. -f2) + local version_major=$(echo "$version" | cut -d. -f1) + local version_minor=$(echo "$version" | cut -d. -f2) + if [ "$version_major" -eq "$target_major" ] && [ "$version_minor" -eq "$target_minor" ]; then + waggle_log info "RabbitMQ upgrade path reached target major & minor version, $target_ver" + break + fi if ! update_rabbitmq_to_version "$version"; then waggle_log err "Failed to upgrade to intermediate version $version" return 1 From fdbd494b8f007edf0901eb651b6eb1495137e059 Mon Sep 17 00:00:00 2001 From: FranciscoLozCoding Date: Tue, 26 Aug 2025 17:40:17 -0500 Subject: [PATCH 39/39] Remove redundant check for current RabbitMQ version before upgrade process in update-stack.sh --- kubernetes/update-stack.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/kubernetes/update-stack.sh b/kubernetes/update-stack.sh index a5b857f..ec629c9 100755 --- a/kubernetes/update-stack.sh +++ b/kubernetes/update-stack.sh @@ -645,12 +645,6 @@ EOF waggle_log err "RabbitMQ downgrade not supported: $current_ver -> $target_ver" return 1 fi - - # make sure to start from current version - if ! update_rabbitmq_to_version "$current_ver"; then - waggle_log err "Failed to update to current version $current_ver" - return 1 - fi # Ensure RabbitMQ is running before attempting upgrade waggle_log info "Ensuring RabbitMQ is running..."