Fix unordered mu diffs when using scores

Author: vivekjoshy

benchmark/benchmark.py

@@ -9,7 +9,7 @@
 - BradleyTerryFull
 - BradleyTerryPart
 
-Each system is tested with margin=0.0 and margin=2.0 to evaluate the impact on prediction accuracy.
+Each system is tested with margin=0.0 and margin=1.0 to evaluate the impact on prediction accuracy.
 """
 
 import gc
@@ -79,7 +79,7 @@ def __init__(
         }
 
         # Define margins to test
-        self.margins = [0.0, 2.0]
+        self.margins = [0.0, 1.0]
 
     def load_data(self) -> None:
         """
@@ -391,7 +391,7 @@ def display_results(
         table.add_column("Avg Time (s)", style="yellow")
 
         # Add a separate table for margin difference analysis
-        diff_table = Table(title="Margin Impact Analysis (2.0 vs 0.0)")
+        diff_table = Table(title="Margin Impact Analysis")
         diff_table.add_column("Model", style="cyan")
         diff_table.add_column("Accuracy Difference", style="magenta")
         diff_table.add_column("Speed Difference", style="yellow")
@@ -412,19 +412,27 @@ def display_results(
                     f"{metrics[3]:.2f}",
                 )
 
-            # Calculate the difference between margin=2.0 and margin=0.0
-            if 0.0 in metrics_by_margin and 2.0 in metrics_by_margin:
-                m0 = metrics_by_margin[0.0]
-                m2 = metrics_by_margin[2.0]
+            # Calculate the difference between margins automatically
+            if len(self.margins) >= 2:
+                # Sort margins to get consistent comparison (highest vs lowest)
+                sorted_margins = sorted(self.margins)
+                low_margin = sorted_margins[0]
+                high_margin = sorted_margins[-1]
 
-                acc_diff = m2[2] - m0[2]
-                time_diff = (m2[3] / m0[3] - 1.0) * 100
+                if low_margin in metrics_by_margin and high_margin in metrics_by_margin:
+                    m_low = metrics_by_margin[low_margin]
+                    m_high = metrics_by_margin[high_margin]
 
-                diff_table.add_row(
-                    model_name,
-                    f"{acc_diff:.2f}% {'better' if acc_diff > 0 else 'worse'}",
-                    f"{abs(time_diff):.2f}% {'slower' if time_diff > 0 else 'faster'}",
-                )
+                    acc_diff = m_high[2] - m_low[2]
+                    time_diff = (
+                        (m_high[3] / m_low[3] - 1.0) * 100 if m_low[3] > 0 else 0
+                    )
+
+                    diff_table.add_row(
+                        model_name,
+                        f"{acc_diff:+.2f}% ({'better' if acc_diff > 0 else 'worse' if acc_diff < 0 else 'same'})",
+                        f"{abs(time_diff):.2f}% {'slower' if time_diff > 0 else 'faster' if time_diff < 0 else 'same'}",
+                    )
 
         # Print tables
         self.console.print(table)

openskill/models/weng_lin/bradley_terry_full.py

@@ -607,7 +607,7 @@ def rate(
 
         # Convert Score to Ranks
         if not ranks and scores:
-            ranks = [-s for s in scores]
+            ranks = list(map(lambda s: -s, scores))
             ranks = self._calculate_rankings(teams, ranks)
 
         # Normalize Weights
@@ -1056,8 +1056,7 @@ def _calculate_rankings(
         Calculates the rankings based on the scores or ranks of the teams.
 
         It assigns a rank to each team based on their score, with the team with
-        the highest score being ranked first. Ties are broken by a team's prior
-        averaged :math:`mu` values.
+        the highest score being ranked first.
 
         :param game: A list of teams, where teams are lists of
                      :class:`BradleyTerryFullRating` objects.
@@ -1069,36 +1068,16 @@ def _calculate_rankings(
         if not game:
             return []
 
-        if ranks is None:
-            return list(range(len(game)))
-
-        team_data = []
-        for index, rank in enumerate(ranks):
-            team = game[index]
-            average_ordinal = sum(player.ordinal() for player in team) / len(team)
-            team_data.append((rank, average_ordinal, index))
-
-        # Lower Ranks (Better), Higher Skill (Breaks Ties)
-        team_data.sort(key=lambda data: (data[0], -data[1]))  # Rank, -(Average Skill)
-
-        # Assign Final Ranks: Preserve Ranks for Identical Skill
-        final_ranks = [0.0] * len(game)
-        current_rank = 0
-
-        for team_index, (
-            original_rank,
-            average_ordinal,
-            original_team_index,
-        ) in enumerate(team_data):
-            if team_index > 0:
-                preceding_original_rank, preceding_average_ordinal, _ = team_data[
-                    team_index - 1
-                ]
-                # Advance Rank: Performance Changes OR Rank Changes
-                if (
-                    original_rank != preceding_original_rank
-                    or average_ordinal != preceding_average_ordinal
-                ):
-                    current_rank = original_team_index
-            final_ranks[original_team_index] = float(current_rank)
-        return final_ranks
+        if ranks:
+            team_scores = [ranks[i] or i for i, _ in enumerate(game)]
+        else:
+            team_scores = [i for i, _ in enumerate(game)]
+
+        output_ranks: dict[int, float] = {}
+        s = 0
+        for index, value in enumerate(team_scores):
+            if index > 0:
+                if team_scores[index - 1] < team_scores[index]:
+                    s = index
+            output_ranks[index] = s
+        return list(output_ranks.values())

openskill/models/weng_lin/bradley_terry_part.py

@@ -617,7 +617,7 @@ def rate(
 
         # Convert Score to Ranks
         if not ranks and scores:
-            ranks = [-s for s in scores]
+            ranks = list(map(lambda s: -s, scores))
             ranks = self._calculate_rankings(teams, ranks)
 
         # Normalize Weights
@@ -1052,8 +1052,7 @@ def _calculate_rankings(
         Calculates the rankings based on the scores or ranks of the teams.
 
         It assigns a rank to each team based on their score, with the team with
-        the highest score being ranked first. Ties are broken by a team's prior
-        averaged :math:`mu` values.
+        the highest score being ranked first.
 
         :param game: A list of teams, where teams are lists of
                      :class:`BradleyTerryPartRating` objects.
@@ -1065,36 +1064,16 @@ def _calculate_rankings(
         if not game:
             return []
 
-        if ranks is None:
-            return list(range(len(game)))
-
-        team_data = []
-        for index, rank in enumerate(ranks):
-            team = game[index]
-            average_ordinal = sum(player.ordinal() for player in team) / len(team)
-            team_data.append((rank, average_ordinal, index))
-
-        # Lower Ranks (Better), Higher Skill (Breaks Ties)
-        team_data.sort(key=lambda data: (data[0], -data[1]))  # Rank, -(Average Skill)
-
-        # Assign Final Ranks: Preserve Ranks for Identical Skill
-        final_ranks = [0.0] * len(game)
-        current_rank = 0
-
-        for team_index, (
-            original_rank,
-            average_ordinal,
-            original_team_index,
-        ) in enumerate(team_data):
-            if team_index > 0:
-                preceding_original_rank, preceding_average_ordinal, _ = team_data[
-                    team_index - 1
-                ]
-                # Advance Rank: Performance Changes OR Rank Changes
-                if (
-                    original_rank != preceding_original_rank
-                    or average_ordinal != preceding_average_ordinal
-                ):
-                    current_rank = original_team_index
-            final_ranks[original_team_index] = float(current_rank)
-        return final_ranks
+        if ranks:
+            team_scores = [ranks[i] or i for i, _ in enumerate(game)]
+        else:
+            team_scores = [i for i, _ in enumerate(game)]
+
+        output_ranks: dict[int, float] = {}
+        s = 0
+        for index, value in enumerate(team_scores):
+            if index > 0:
+                if team_scores[index - 1] < team_scores[index]:
+                    s = index
+            output_ranks[index] = s
+        return list(output_ranks.values())