|
| 1 | +"""Base class for collections metrics with modern component validation.""" |
| 2 | + |
| 3 | +import asyncio |
| 4 | +import typing as t |
| 5 | + |
| 6 | +from ragas.embeddings.base import BaseRagasEmbedding |
| 7 | +from ragas.llms.base import InstructorBaseRagasLLM |
| 8 | +from ragas.metrics.base import SimpleBaseMetric |
| 9 | +from ragas.metrics.result import MetricResult |
| 10 | +from ragas.metrics.validators import NumericValidator |
| 11 | + |
| 12 | + |
| 13 | +class BaseMetric(SimpleBaseMetric, NumericValidator): |
| 14 | + """ |
| 15 | + Base class for metrics collections with modern component validation. |
| 16 | +
|
| 17 | + This class inherits from SimpleBaseMetric and NumericValidator to provide: |
| 18 | + - All the base metric functionality (ascore, abatch_score, score, batch_score) |
| 19 | + - Numeric validation with configurable ranges |
| 20 | + - Modern LLM and embedding component validation (when defined by subclass) |
| 21 | + - Rejection of legacy wrappers with helpful error messages |
| 22 | + - Consistent error handling and type safety |
| 23 | +
|
| 24 | + Attributes: |
| 25 | + name: The metric name |
| 26 | + allowed_values: Score range for numeric validation (tuple of min, max) |
| 27 | +
|
| 28 | + Note: Subclasses define llm and/or embeddings fields only if they need them. |
| 29 | + The base classes handle all the core metric functionality - we just add modern component validation. |
| 30 | + """ |
| 31 | + |
| 32 | + def __init__( |
| 33 | + self, |
| 34 | + name: str = "base_metric", |
| 35 | + allowed_values: t.Tuple[float, float] = (0.0, 1.0), |
| 36 | + **kwargs, |
| 37 | + ): |
| 38 | + """Initialize the base metric with validation.""" |
| 39 | + super().__init__(name=name, allowed_values=allowed_values) |
| 40 | + |
| 41 | + # Validate components only if the metric defines them |
| 42 | + # Check if this instance has these attributes after initialization |
| 43 | + if hasattr(self, "llm"): |
| 44 | + self._validate_llm() |
| 45 | + if hasattr(self, "embeddings"): |
| 46 | + self._validate_embeddings() |
| 47 | + |
| 48 | + async def ascore(self, **kwargs) -> MetricResult: |
| 49 | + """ |
| 50 | + Default async scoring method - subclasses should override this. |
| 51 | +
|
| 52 | + This base implementation just returns a placeholder result. |
| 53 | + Subclasses should override this method with their specific logic. |
| 54 | +
|
| 55 | + The base class handles component validation in __post_init__. |
| 56 | + """ |
| 57 | + return MetricResult( |
| 58 | + value=0.0, reason="Base metric placeholder - override ascore() in subclass" |
| 59 | + ) |
| 60 | + |
| 61 | + def score(self, **kwargs) -> MetricResult: |
| 62 | + """ |
| 63 | + Synchronous scoring method that wraps ascore(). |
| 64 | +
|
| 65 | + This is a convenience method for backward compatibility and sync usage. |
| 66 | + For better performance, prefer using ascore() directly in async contexts. |
| 67 | +
|
| 68 | + Returns: |
| 69 | + MetricResult object |
| 70 | + """ |
| 71 | + try: |
| 72 | + # Check if we're already in an async context |
| 73 | + asyncio.get_running_loop() |
| 74 | + # If we get here, there's already a running loop |
| 75 | + raise RuntimeError( |
| 76 | + "Cannot call sync score() from an async context. Use ascore() instead." |
| 77 | + ) |
| 78 | + except RuntimeError as e: |
| 79 | + if "Use ascore() instead" in str(e): |
| 80 | + raise # Re-raise our custom error |
| 81 | + # No running loop found, safe to use asyncio.run() |
| 82 | + return asyncio.run(self.ascore(**kwargs)) |
| 83 | + |
| 84 | + def batch_score( |
| 85 | + self, |
| 86 | + inputs: t.List[t.Dict[str, t.Any]], |
| 87 | + ) -> t.List[MetricResult]: |
| 88 | + """ |
| 89 | + Synchronous batch scoring that wraps abatch_score(). |
| 90 | +
|
| 91 | + This is a convenience method for backward compatibility and sync usage. |
| 92 | + For better performance, prefer using abatch_score() directly in async contexts. |
| 93 | +
|
| 94 | + Args: |
| 95 | + inputs: List of input dictionaries for scoring |
| 96 | +
|
| 97 | + Returns: |
| 98 | + List of MetricResult objects |
| 99 | + """ |
| 100 | + try: |
| 101 | + # Check if we're already in an async context |
| 102 | + asyncio.get_running_loop() |
| 103 | + # If we get here, there's already a running loop |
| 104 | + raise RuntimeError( |
| 105 | + "Cannot call sync batch_score() from an async context. Use abatch_score() instead." |
| 106 | + ) |
| 107 | + except RuntimeError as e: |
| 108 | + if "Use abatch_score() instead" in str(e): |
| 109 | + raise # Re-raise our custom error |
| 110 | + # No running loop found, safe to use asyncio.run() |
| 111 | + return asyncio.run(self.abatch_score(inputs)) |
| 112 | + |
| 113 | + def _validate_llm(self): |
| 114 | + """Validate that a modern InstructorLLM is provided.""" |
| 115 | + llm = getattr(self, "llm", None) |
| 116 | + |
| 117 | + if not isinstance(llm, InstructorBaseRagasLLM): |
| 118 | + raise ValueError( |
| 119 | + f"Collections metrics only support modern InstructorLLM. Found: {type(llm).__name__}. " |
| 120 | + f"Use: instructor_llm_factory('openai', model='gpt-4o-mini', client=openai_client)" |
| 121 | + ) |
| 122 | + |
| 123 | + def _validate_embeddings(self): |
| 124 | + """Validate that modern embeddings are provided.""" |
| 125 | + embeddings = getattr(self, "embeddings", None) |
| 126 | + |
| 127 | + if not isinstance(embeddings, BaseRagasEmbedding): |
| 128 | + raise ValueError( |
| 129 | + f"Collections metrics only support modern embeddings. Found: {type(embeddings).__name__}. " |
| 130 | + f"Use: embedding_factory('openai', model='text-embedding-ada-002', client=openai_client, interface='modern')" |
| 131 | + ) |
0 commit comments