From 9dfa556f5c421753b62897ef7e112266ce8be59b Mon Sep 17 00:00:00 2001 From: kumar shivam ranjan Date: Wed, 20 Nov 2024 20:06:23 +0530 Subject: [PATCH 1/2] Create eval bugfix --- ads/aqua/evaluation/entities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ads/aqua/evaluation/entities.py b/ads/aqua/evaluation/entities.py index 20c3c01da..f6c02f3ce 100644 --- a/ads/aqua/evaluation/entities.py +++ b/ads/aqua/evaluation/entities.py @@ -83,7 +83,7 @@ class CreateAquaEvaluationDetails(Serializable): ocpus: Optional[float] = None log_group_id: Optional[str] = None log_id: Optional[str] = None - metrics: Optional[List[str]] = None + metrics: Optional[List] = None force_overwrite: Optional[bool] = False class Config: @@ -140,7 +140,7 @@ class AquaEvaluationCommands(Serializable): evaluation_id: str evaluation_target_id: str input_data: Dict[str, Any] - metrics: List[str] + metrics: List output_dir: str params: Dict[str, Any] From 2cbe2813701db4284d52e8b6da14110b1a9dcf7a Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Wed, 20 Nov 2024 10:21:34 -0800 Subject: [PATCH 2/2] Fix the eval for chat models --- ads/aqua/evaluation/entities.py | 4 ++-- ads/aqua/evaluation/evaluation.py | 8 ++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ads/aqua/evaluation/entities.py b/ads/aqua/evaluation/entities.py index f6c02f3ce..ad49c88ae 100644 --- a/ads/aqua/evaluation/entities.py +++ b/ads/aqua/evaluation/entities.py @@ -83,7 +83,7 @@ class CreateAquaEvaluationDetails(Serializable): ocpus: Optional[float] = None log_group_id: Optional[str] = None log_id: Optional[str] = None - metrics: Optional[List] = None + metrics: Optional[List[Dict[str, Any]]] = None force_overwrite: Optional[bool] = False class Config: @@ -140,7 +140,7 @@ class AquaEvaluationCommands(Serializable): evaluation_id: str evaluation_target_id: str input_data: Dict[str, Any] - metrics: List + metrics: List[Dict[str, Any]] output_dir: str params: Dict[str, Any] diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index 8cc799d18..cfd0029e6 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -159,7 +159,8 @@ def create( create_aqua_evaluation_details = CreateAquaEvaluationDetails(**kwargs) except Exception as ex: custom_errors = { - ".".join(map(str, e["loc"])): e["msg"] for e in json.loads(ex.json()) + ".".join(map(str, e["loc"])): e["msg"] + for e in json.loads(ex.json()) } raise AquaValueError( f"Invalid create evaluation parameters. Error details: {custom_errors}." @@ -619,11 +620,6 @@ def _build_launch_cmd( evaluation_id=evaluation_id, evaluation_target_id=evaluation_source_id, input_data={ - "columns": { - "prompt": "prompt", - "completion": "completion", - "category": "category", - }, "format": Path(dataset_path).suffix, "url": dataset_path, },