Fix success rate calculation to exclude skipped tests (#1136)

Co-authored-by: openhands <openhands@all-hands.dev>

Author: xingyaoww

tests/integration/schemas.py

@@ -88,7 +88,11 @@ def from_eval_outputs(
         total_tests = len(test_instances)
         successful_tests = sum(1 for t in test_instances if t.test_result.success)
         skipped_tests = sum(1 for t in test_instances if t.test_result.skipped)
-        success_rate = successful_tests / total_tests if total_tests > 0 else 0.0
+        # Exclude skipped tests from success rate calculation
+        non_skipped_tests = total_tests - skipped_tests
+        success_rate = (
+            successful_tests / non_skipped_tests if non_skipped_tests > 0 else 0.0
+        )
         total_cost = sum(t.cost for t in test_instances)
 
         return cls(
@@ -131,8 +135,13 @@ def from_model_results(
         # Calculate overall statistics
         total_tests_all = sum(r.total_tests for r in model_results)
         total_successful_all = sum(r.successful_tests for r in model_results)
+        total_skipped_all = sum(r.skipped_tests for r in model_results)
+        # Exclude skipped tests from overall success rate calculation
+        non_skipped_tests_all = total_tests_all - total_skipped_all
         overall_success_rate = (
-            total_successful_all / total_tests_all if total_tests_all > 0 else 0.0
+            total_successful_all / non_skipped_tests_all
+            if non_skipped_tests_all > 0
+            else 0.0
         )
         total_cost_all_models = sum(r.total_cost for r in model_results)
 

tests/integration/utils/generate_markdown_report.py

@@ -24,7 +24,8 @@ def generate_model_summary_table(model_results: list[ModelTestResults]) -> str:
 
     for result in model_results:
         success_rate = f"{result.success_rate:.1%}"
-        tests_passed = f"{result.successful_tests}/{result.total_tests}"
+        non_skipped = result.total_tests - result.skipped_tests
+        tests_passed = f"{result.successful_tests}/{non_skipped}"
         skipped = f"{result.skipped_tests}"
         cost = format_cost(result.total_cost)
 
@@ -45,11 +46,12 @@ def generate_detailed_results(model_results: list[ModelTestResults]) -> str:
     sections = []
 
     for result in model_results:
+        non_skipped = result.total_tests - result.skipped_tests
         section_lines = [
             f"### {result.model_name}",
             "",
             f"- **Success Rate**: {result.success_rate:.1%} "
-            f"({result.successful_tests}/{result.total_tests})",
+            f"({result.successful_tests}/{non_skipped})",
             f"- **Total Cost**: {format_cost(result.total_cost)}",
             f"- **Run Suffix**: `{result.run_suffix}`",
         ]