refactor: added task id, refactor of storing task_input variables

Author: MagdalenaKotynia

src/rai_bench/rai_bench/test_models.py

@@ -26,15 +26,19 @@
 import rai_bench.manipulation_o3de as manipulation_o3de
 import rai_bench.tool_calling_agent as tool_calling_agent
 import rai_bench.vlm_benchmark as vlm_benchmark
-from rai_bench.base_benchmark import ModelSummary, RunSummary
-from rai_bench.results_processing.data_loading import SUMMARY_FILE_NAME
+from rai_bench.base_benchmark import ModelSummary, RunSummary, TasksSummary
+from rai_bench.results_processing.data_loading import (
+    DETAILED_FILE_NAME,
+    SUMMARY_FILE_NAME,
+)
 from rai_bench.utils import (
     define_benchmark_logger,
     get_llm_for_benchmark,
     get_llm_model_name,
 )
 
 REPEATS_SUMMARY_FILE_NAME = "repeats_summary.csv"
+TASKS_SUMMARY_FILE_NAME = "tasks_summary.csv"
 BENCHMARK_SUMMARY = "benchmark_summary.csv"
 
 
@@ -151,7 +155,7 @@ def merge_model_repeats_summary(
 
     merged_file = model_dir / REPEATS_SUMMARY_FILE_NAME
     with open(merged_file, "w", newline="") as f:
-        writer = csv.DictWriter(f, fieldnames=RunSummary.model_fields.keys())
+        writer = csv.DictWriter(f, fieldnames=ModelSummary.model_fields.keys())
         writer.writeheader()
         writer.writerow(merged_summary.model_dump())
 
@@ -174,7 +178,7 @@ def merge_benchmark_summary(
     if not bench_dir.exists():
         return
 
-    all_summaries: List[RunSummary] = []
+    all_summaries: List[ModelSummary] = []
     for model_name in model_names:
         model_dir = bench_dir / model_name
         merged_file = model_dir / REPEATS_SUMMARY_FILE_NAME
@@ -183,19 +187,89 @@ def merge_benchmark_summary(
             with open(merged_file, "r") as f:
                 reader = csv.DictReader(f)
                 for row in reader:
-                    all_summaries.append(RunSummary.model_validate(row))
+                    all_summaries.append(ModelSummary.model_validate(row))
 
     if not all_summaries:
         return
 
     benchmark_summary_file = bench_dir / BENCHMARK_SUMMARY
     with open(benchmark_summary_file, "w", newline="") as f:
-        writer = csv.DictWriter(f, fieldnames=RunSummary.model_fields.keys())
+        writer = csv.DictWriter(f, fieldnames=ModelSummary.model_fields.keys())
         writer.writeheader()
         for summary in all_summaries:
             writer.writerow(summary.model_dump())
 
 
+def merge_tasks_summary(bench_name: str, model_name: str, run_dir: Path) -> None:
+    """Merge task results across all repeats for a single model, aggregating by task.
+
+    Parameters
+    ----------
+    bench_name : str
+        Name of the benchmark
+    model_name : str
+        Name of the model
+    run_dir : Path
+        Directory containing the benchmark run results
+    """
+    model_dir = run_dir / bench_name / model_name
+    if not model_dir.exists():
+        return
+
+    # Collect all task results from all repeats
+    task_data_by_prompt: Dict[str, Dict[str, List[float]]] = {}
+
+    for repeat_dir in model_dir.iterdir():
+        if repeat_dir.is_dir() and repeat_dir.name.isdigit():
+            results_file = repeat_dir / DETAILED_FILE_NAME
+            if results_file.exists():
+                # Read detailed results from this repeat
+                with open(results_file, "r") as f:
+                    reader = csv.DictReader(f)
+                    for row in reader:
+                        task_prompt = row["task_prompt"]
+                        score = float(row["score"])
+                        total_time = float(row["total_time"])
+
+                        if task_prompt not in task_data_by_prompt:
+                            task_data_by_prompt[task_prompt] = {
+                                "scores": [],
+                                "times": [],
+                            }
+
+                        task_data_by_prompt[task_prompt]["scores"].append(score)
+                        task_data_by_prompt[task_prompt]["times"].append(total_time)
+
+    if not task_data_by_prompt:
+        return
+
+    # Calculate statistics for each task
+    task_summaries: List[TasksSummary] = []
+    for task_prompt, data in task_data_by_prompt.items():
+        scores = np.array(data["scores"])
+        times = np.array(data["times"])
+
+        task_summary = TasksSummary(
+            model_name=model_name,
+            task_prompt=task_prompt,
+            avg_success_rate=round(float(scores.mean()), 3),
+            std_success_rate=round(float(scores.std()), 3),
+            avg_time=round(float(times.mean()), 3),
+            std_time=round(float(times.std()), 3),
+            repeats=len(scores),  # TODO (mkotynia) (extract repeats in another way)
+        )
+        task_summaries.append(task_summary)
+
+    # Save task summaries to CSV
+    tasks_summary_file = model_dir / TASKS_SUMMARY_FILE_NAME
+    with open(tasks_summary_file, "w", newline="") as f:
+        if task_summaries:
+            writer = csv.DictWriter(f, fieldnames=TasksSummary.model_fields.keys())
+            writer.writeheader()
+            for task_summary in task_summaries:
+                writer.writerow(task_summary.model_dump())
+
+
 def test_dual_agents(
     multimodal_llms: List[BaseChatModel],
     tool_calling_models: List[BaseChatModel],
@@ -351,6 +425,7 @@ def test_models(
 
             for model_name in model_names:
                 merge_model_repeats_summary(bench_conf.name, model_name, run_dir)
+                merge_tasks_summary(bench_conf.name, model_name, run_dir)
 
             merge_benchmark_summary(bench_conf.name, run_dir, model_names)
 

src/rai_bench/rai_bench/vlm_benchmark/benchmark.py

@@ -133,14 +133,15 @@ def run_next(self, agent: CompiledStateGraph, experiment_id: uuid.UUID) -> None:
             score = task.validate(output=structured_output)
         else:
             errors.append(f"Not valid structured output: {type(structured_output)}")
-            score = False
+            score = 0
 
         te = time.perf_counter()
         total_time = te - ts
 
         self.logger.info(f"TASK SCORE: {score}, TOTAL TIME: {total_time:.3f}")
 
         task_result = TaskResult(
+            task_id=task.task_id,
             task_prompt=task.get_prompt(),
             system_prompt=task.get_system_prompt(),
             type=task.type,

src/rai_bench/rai_bench/vlm_benchmark/interfaces.py

@@ -12,13 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import hashlib
 import logging
 from abc import ABC, abstractmethod
 from typing import Any, Generic, List, Literal, Optional, TypeVar
 
 from langchain_core.messages import BaseMessage
 from langchain_core.runnables.config import DEFAULT_RECURSION_LIMIT
-from pydantic import BaseModel, ConfigDict, Field, ValidationError
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, computed_field
 
 loggers_type = logging.Logger
 
@@ -47,6 +48,13 @@ class ImageReasoningTaskInput(BaseModel, Generic[AnswerT]):
         ..., description="The expected answer to the question."
     )
 
+    @computed_field
+    @property
+    def task_id(self) -> str:
+        """Unique identifier for the task based on question and image paths."""
+        content = f"{self.question}|{sorted(self.images_paths)}"
+        return hashlib.sha256(content.encode()).hexdigest()
+
 
 class ImageReasoningAnswer(BaseModel, Generic[AnswerT]):
     """Base answer for an image reasoning task."""
@@ -84,6 +92,7 @@ class ImageReasoningTask(ABC, Generic[AnswerT]):
 
     def __init__(
         self,
+        task_input: ImageReasoningTaskInput[AnswerT],
         logger: loggers_type | None = None,
     ) -> None:
         """
@@ -101,9 +110,28 @@ def __init__(
             self.logger = logger
         else:
             self.logger = logging.getLogger(__name__)
-        self.question: str
-        self.images_paths: List[str]
-        # TODO move here task input
+
+        self._task_input = task_input
+
+    @property
+    def question(self) -> str:
+        """The question to be answered."""
+        return self._task_input.question
+
+    @property
+    def images_paths(self) -> List[str]:
+        """List of image file paths."""
+        return self._task_input.images_paths
+
+    @property
+    def expected_answer(self) -> AnswerT:
+        """The expected answer to the question."""
+        return self._task_input.expected_answer
+
+    @property
+    def task_id(self) -> str:
+        """Unique identifier for the task."""
+        return self._task_input.task_id
 
     def set_logger(self, logger: loggers_type):
         self.logger = logger

src/rai_bench/rai_bench/vlm_benchmark/results_tracking.py

@@ -19,6 +19,7 @@
 
 
 class TaskResult(BaseModel):
+    task_id: str = Field(..., description="Unique identifier for the task object.")
     task_prompt: str = Field(..., description="The task prompt.")
     system_prompt: str = Field(..., description="The system prompt.")
     complexity: str = Field(..., description="Complexity of the task.")

src/rai_bench/rai_bench/vlm_benchmark/tasks/tasks.py

@@ -66,11 +66,9 @@ def __init__(
         logger: loggers_type | None = None,
     ) -> None:
         super().__init__(
+            task_input=task_input,
             logger=logger,
         )
-        self.question = task_input.question
-        self.images_paths = task_input.images_paths
-        self.expected_answer = task_input.expected_answer
 
     @property
     def structured_output(self) -> type[BoolAnswerWithJustification]:
@@ -101,10 +99,7 @@ def __init__(
         task_input: QuantityImageTaskInput,
         logger: loggers_type | None = None,
     ) -> None:
-        super().__init__(logger=logger)
-        self.question = task_input.question
-        self.images_paths = task_input.images_paths
-        self.expected_answer = task_input.expected_answer
+        super().__init__(task_input=task_input, logger=logger)
 
     @property
     def type(self) -> str:
@@ -135,11 +130,8 @@ def __init__(
         task_input: MultipleChoiceImageTaskInput,
         logger: loggers_type | None = None,
     ) -> None:
-        super().__init__(logger=logger)
-        self.question = task_input.question
-        self.images_paths = task_input.images_paths
+        super().__init__(task_input=task_input, logger=logger)
         self.options = task_input.options
-        self.expected_answer = task_input.expected_answer
 
     @property
     def type(self) -> str: