✨ Add Prometheus alerts tracking to memory profiling

Enhance memory profiling plugin to capture and analyze Prometheus alerts
during e2e test execution:

1. Set E2E_SUMMARY_OUTPUT environment variable during test runs
   - Captures prometheus alerts, test failures, and other metrics
   - Saved to e2e-summary.json in the profile directory

2. Updated analyze-profiles.sh to extract and display alerts
   - Parses e2e-summary.json using jq (if available)
   - Shows alert names, severities, and descriptions
   - Includes test failures in the analysis report

3. Updated compare-profiles.sh to compare alerts between runs
   - Shows alert counts for both tests
   - Lists alerts detected in each test
   - Helps identify if optimizations introduced new alerts

This allows correlating memory usage with system health metrics,
making it easier to identify if memory optimizations have any
negative side effects on system stability.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

Author: tmshort

hack/tools/memory-profiling/analyze-profiles.sh

@@ -208,6 +208,61 @@ EOF
 EOF
 fi
 
+# Prometheus Alerts Analysis
+if [ -f "${OUTPUT_DIR}/e2e-summary.json" ]; then
+    log_info "Analyzing Prometheus alerts..."
+    cat >> "${REPORT_FILE}" << 'EOF'
+
+## Prometheus Alerts
+
+EOF
+
+    # Check if jq is available for JSON parsing
+    if command -v jq >/dev/null 2>&1; then
+        # Extract alerts from e2e-summary.json
+        ALERTS=$(jq -r '.prometheusAlerts // [] | .[] | "- **\(.labels.alertname)** (Severity: \(.labels.severity // "unknown"))\n  - \(.annotations.description // .annotations.message // "No description")"' "${OUTPUT_DIR}/e2e-summary.json" 2>/dev/null || echo "")
+
+        if [ -n "${ALERTS}" ]; then
+            cat >> "${REPORT_FILE}" << EOF
+### Alerts Detected
+
+${ALERTS}
+
+EOF
+        else
+            cat >> "${REPORT_FILE}" << 'EOF'
+No Prometheus alerts detected during test execution.
+
+EOF
+        fi
+
+        # Extract test failures if any
+        TEST_FAILURES=$(jq -r '.testFailures // [] | .[] | "- \(.test): \(.message)"' "${OUTPUT_DIR}/e2e-summary.json" 2>/dev/null || echo "")
+
+        if [ -n "${TEST_FAILURES}" ]; then
+            cat >> "${REPORT_FILE}" << EOF
+### Test Failures
+
+${TEST_FAILURES}
+
+EOF
+        fi
+    else
+        cat >> "${REPORT_FILE}" << 'EOF'
+E2E summary available at `e2e-summary.json` (install `jq` for detailed alert parsing)
+
+EOF
+    fi
+else
+    cat >> "${REPORT_FILE}" << 'EOF'
+
+## Prometheus Alerts
+
+E2E summary not available. Set `E2E_SUMMARY_OUTPUT` environment variable when running tests to capture alerts.
+
+EOF
+fi
+
 # Recommendations
 cat >> "${REPORT_FILE}" << 'EOF'
 

hack/tools/memory-profiling/compare-profiles.sh

@@ -281,6 +281,88 @@ cat >> "${REPORT_FILE}" << 'EOF'
 
 ---
 
+## Prometheus Alerts Comparison
+
+EOF
+
+# Compare prometheus alerts if available
+if [ -f "${TEST1_DIR}/e2e-summary.json" ] || [ -f "${TEST2_DIR}/e2e-summary.json" ]; then
+    if command -v jq >/dev/null 2>&1; then
+        # Test 1 alerts
+        if [ -f "${TEST1_DIR}/e2e-summary.json" ]; then
+            ALERTS1=$(jq -r '.prometheusAlerts // [] | length' "${TEST1_DIR}/e2e-summary.json" 2>/dev/null || echo "0")
+            ALERTS1_NAMES=$(jq -r '.prometheusAlerts // [] | .[].labels.alertname' "${TEST1_DIR}/e2e-summary.json" 2>/dev/null | sort | uniq || echo "")
+        else
+            ALERTS1="N/A"
+            ALERTS1_NAMES=""
+        fi
+
+        # Test 2 alerts
+        if [ -f "${TEST2_DIR}/e2e-summary.json" ]; then
+            ALERTS2=$(jq -r '.prometheusAlerts // [] | length' "${TEST2_DIR}/e2e-summary.json" 2>/dev/null || echo "0")
+            ALERTS2_NAMES=$(jq -r '.prometheusAlerts // [] | .[].labels.alertname' "${TEST2_DIR}/e2e-summary.json" 2>/dev/null | sort | uniq || echo "")
+        else
+            ALERTS2="N/A"
+            ALERTS2_NAMES=""
+        fi
+
+        cat >> "${REPORT_FILE}" << EOF
+### Alert Summary
+
+| Metric | ${TEST1} | ${TEST2} |
+|--------|----------|----------|
+| Alert Count | ${ALERTS1} | ${ALERTS2} |
+
+EOF
+
+        if [ -n "${ALERTS1_NAMES}" ] || [ -n "${ALERTS2_NAMES}" ]; then
+            cat >> "${REPORT_FILE}" << 'EOF'
+### Alerts by Test
+
+EOF
+
+            if [ -n "${ALERTS1_NAMES}" ]; then
+                cat >> "${REPORT_FILE}" << EOF
+**${TEST1}:**
+EOF
+                echo "${ALERTS1_NAMES}" | while read -r alert; do
+                    [ -n "${alert}" ] && echo "- ${alert}" >> "${REPORT_FILE}"
+                done
+                echo "" >> "${REPORT_FILE}"
+            fi
+
+            if [ -n "${ALERTS2_NAMES}" ]; then
+                cat >> "${REPORT_FILE}" << EOF
+**${TEST2}:**
+EOF
+                echo "${ALERTS2_NAMES}" | while read -r alert; do
+                    [ -n "${alert}" ] && echo "- ${alert}" >> "${REPORT_FILE}"
+                done
+                echo "" >> "${REPORT_FILE}"
+            fi
+        else
+            cat >> "${REPORT_FILE}" << 'EOF'
+No alerts detected in either test.
+
+EOF
+        fi
+    else
+        cat >> "${REPORT_FILE}" << 'EOF'
+E2E summaries available but `jq` not installed for parsing.
+
+EOF
+    fi
+else
+    cat >> "${REPORT_FILE}" << 'EOF'
+E2E summary not available for comparison.
+
+EOF
+fi
+
+cat >> "${REPORT_FILE}" << 'EOF'
+
+---
+
 ## Key Findings
 
 **Memory Impact:**

hack/tools/memory-profiling/run-profiled-test.sh

@@ -82,7 +82,8 @@ log_info "Output directory: ${OUTPUT_DIR}"
 
 # Start the e2e test
 log_info "Starting e2e test (${TEST_TARGET})..."
-make "${TEST_TARGET}" > "${OUTPUT_DIR}/test.log" 2>&1 &
+# Set E2E_SUMMARY_OUTPUT to capture prometheus alerts and other test metrics
+E2E_SUMMARY_OUTPUT="${OUTPUT_DIR}/e2e-summary.json" make "${TEST_TARGET}" > "${OUTPUT_DIR}/test.log" 2>&1 &
 TEST_PID=$!
 log_info "Test started (PID: ${TEST_PID})"