From e898ea194677621713b5b53637427f9869084f92 Mon Sep 17 00:00:00 2001 From: Colin Zuo Date: Tue, 21 Oct 2025 18:44:21 +0800 Subject: [PATCH 1/3] add custom processing logic for MetricGrouping --- src/lighteval/tasks/lighteval_task.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/lighteval/tasks/lighteval_task.py b/src/lighteval/tasks/lighteval_task.py index b84d421a6..72d344234 100644 --- a/src/lighteval/tasks/lighteval_task.py +++ b/src/lighteval/tasks/lighteval_task.py @@ -32,7 +32,7 @@ from lighteval.metrics.metrics import Metrics from lighteval.metrics.metrics_sample import SamplingMetric -from lighteval.metrics.utils.metric_utils import Metric +from lighteval.metrics.utils.metric_utils import Metric, MetricGrouping from lighteval.tasks.prompt_manager import FewShotSampler from lighteval.tasks.requests import ( Doc, @@ -167,7 +167,21 @@ def __str__(self, lite: bool = False): continue if k == "metrics": for ix, metrics in enumerate(v): + is_metric_grouping = False + if isinstance(getattr(self, k)[ix], MetricGrouping): + is_metric_grouping = True for metric_k, metric_v in metrics.items(): + if is_metric_grouping and isinstance(metric_v, dict): + for metric_sub_k, metric_sub_v in metric_v.items(): + if isinstance(metric_sub_v, Callable): + repr_v = metric_sub_v.__name__ + elif isinstance(metric_sub_v, Metric.get_allowed_types_for_metrics()): + repr_v = str(metric_sub_v) + else: + repr_v = repr(metric_sub_v) + values.append([f"{k} {ix}: {metric_k}: {metric_sub_k}", repr_v]) + continue + if isinstance(metric_v, Callable): repr_v = metric_v.__name__ elif isinstance(metric_v, Metric.get_allowed_types_for_metrics()): From 09372b8a4d3f34386c4860e33740670713347bf0 Mon Sep 17 00:00:00 2001 From: Colin Zuo Date: Wed, 5 Nov 2025 18:06:08 +0800 Subject: [PATCH 2/3] change to match linter preference --- src/lighteval/tasks/lighteval_task.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/lighteval/tasks/lighteval_task.py b/src/lighteval/tasks/lighteval_task.py index 72d344234..54440be24 100644 --- a/src/lighteval/tasks/lighteval_task.py +++ b/src/lighteval/tasks/lighteval_task.py @@ -180,15 +180,14 @@ def __str__(self, lite: bool = False): else: repr_v = repr(metric_sub_v) values.append([f"{k} {ix}: {metric_k}: {metric_sub_k}", repr_v]) - continue - - if isinstance(metric_v, Callable): - repr_v = metric_v.__name__ - elif isinstance(metric_v, Metric.get_allowed_types_for_metrics()): - repr_v = str(metric_v) else: - repr_v = repr(metric_v) - values.append([f"{k} {ix}: {metric_k}", repr_v]) + if isinstance(metric_v, Callable): + repr_v = metric_v.__name__ + elif isinstance(metric_v, Metric.get_allowed_types_for_metrics()): + repr_v = str(metric_v) + else: + repr_v = repr(metric_v) + values.append([f"{k} {ix}: {metric_k}", repr_v]) else: if isinstance(v, Callable): From 3eeb60995fe7c83fe7e6523209c0c5a12d734dbb Mon Sep 17 00:00:00 2001 From: Colin Zuo Date: Thu, 6 Nov 2025 09:56:55 +0800 Subject: [PATCH 3/3] extract metrics item part to separate function to meet linter complexity requirement --- src/lighteval/tasks/lighteval_task.py | 53 ++++++++++++++++----------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/src/lighteval/tasks/lighteval_task.py b/src/lighteval/tasks/lighteval_task.py index 54440be24..bc1771ade 100644 --- a/src/lighteval/tasks/lighteval_task.py +++ b/src/lighteval/tasks/lighteval_task.py @@ -167,28 +167,8 @@ def __str__(self, lite: bool = False): continue if k == "metrics": for ix, metrics in enumerate(v): - is_metric_grouping = False - if isinstance(getattr(self, k)[ix], MetricGrouping): - is_metric_grouping = True - for metric_k, metric_v in metrics.items(): - if is_metric_grouping and isinstance(metric_v, dict): - for metric_sub_k, metric_sub_v in metric_v.items(): - if isinstance(metric_sub_v, Callable): - repr_v = metric_sub_v.__name__ - elif isinstance(metric_sub_v, Metric.get_allowed_types_for_metrics()): - repr_v = str(metric_sub_v) - else: - repr_v = repr(metric_sub_v) - values.append([f"{k} {ix}: {metric_k}: {metric_sub_k}", repr_v]) - else: - if isinstance(metric_v, Callable): - repr_v = metric_v.__name__ - elif isinstance(metric_v, Metric.get_allowed_types_for_metrics()): - repr_v = str(metric_v) - else: - repr_v = repr(metric_v) - values.append([f"{k} {ix}: {metric_k}", repr_v]) - + item_str_list = self._build_metrics_item_str_list(ix, metrics) + values.extend(item_str_list) else: if isinstance(v, Callable): values.append([k, v.__name__]) @@ -199,6 +179,35 @@ def __str__(self, lite: bool = False): return md_writer.dumps() + def _build_metrics_item_str_list(self, ix: int, metrics: dict) -> list: + values = [] + + is_metric_grouping = False + + if isinstance(getattr(self, "metrics")[ix], MetricGrouping): + is_metric_grouping = True + + for metric_k, metric_v in metrics.items(): + if is_metric_grouping and isinstance(metric_v, dict): + for metric_sub_k, metric_sub_v in metric_v.items(): + if isinstance(metric_sub_v, Callable): + repr_v = metric_sub_v.__name__ + elif isinstance(metric_sub_v, Metric.get_allowed_types_for_metrics()): + repr_v = str(metric_sub_v) + else: + repr_v = repr(metric_sub_v) + values.append([f"metrics {ix}: {metric_k}: {metric_sub_k}", repr_v]) + else: + if isinstance(metric_v, Callable): + repr_v = metric_v.__name__ + elif isinstance(metric_v, Metric.get_allowed_types_for_metrics()): + repr_v = str(metric_v) + else: + repr_v = repr(metric_v) + values.append([f"metrics {ix}: {metric_k}", repr_v]) + + return values + def print(self, lite: bool = False): print(str(self, lite))