[AQUA][Evaluate] Externalize Supported Shapes List to Global Config. (#942)

Author: mrDzurb

ads/aqua/config/config.py

@@ -3,19 +3,15 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
-from datetime import datetime, timedelta
 from typing import Optional
 
-from cachetools import TTLCache, cached
-
 from ads.aqua.common.entities import ContainerSpec
 from ads.aqua.common.utils import get_container_config
 from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig
 
 DEFAULT_EVALUATION_CONTAINER = "odsc-llm-evaluate"
 
 
-@cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=5), timer=datetime.now))
 def evaluation_service_config(
     container: Optional[str] = DEFAULT_EVALUATION_CONTAINER,
 ) -> EvaluationServiceConfig:
@@ -27,6 +23,7 @@ def evaluation_service_config(
     EvaluationServiceConfig: The evaluation common config.
     """
 
+    container = container or DEFAULT_EVALUATION_CONTAINER
     return EvaluationServiceConfig(
         **get_container_config()
         .get(ContainerSpec.CONTAINER_SPEC, {})

ads/aqua/config/evaluation/evaluation_service_config.py

@@ -233,7 +233,7 @@ class EvaluationServiceConfig(Serializable):
     """
 
     version: Optional[str] = "1.0"
-    kind: Optional[str] = "evaluation"
+    kind: Optional[str] = "evaluation_service_config"
     report_params: Optional[ReportParams] = Field(default_factory=ReportParams)
     inference_params: Optional[InferenceParamsConfig] = Field(
         default_factory=InferenceParamsConfig

ads/aqua/evaluation/entities.py

@@ -102,6 +102,7 @@ class ModelParams(DataClassSerializable):
     presence_penalty: Optional[float] = 0.0
     frequency_penalty: Optional[float] = 0.0
     stop: Optional[Union[str, List[str]]] = field(default_factory=list)
+    model: Optional[str] = "odsc-llm"
 
 
 @dataclass(repr=False)

ads/aqua/evaluation/evaluation.py

@@ -11,7 +11,7 @@
 from datetime import datetime, timedelta
 from pathlib import Path
 from threading import Lock
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List, Optional, Union
 
 import oci
 from cachetools import TTLCache
@@ -46,6 +46,7 @@
     upload_local_to_os,
 )
 from ads.aqua.config.config import evaluation_service_config
+from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig
 from ads.aqua.constants import (
     CONSOLE_LINK_RESOURCE_TYPE_MAPPING,
     EVALUATION_REPORT,
@@ -171,8 +172,19 @@ def create(
                 f"Invalid evaluation source {create_aqua_evaluation_details.evaluation_source_id}. "
                 "Specify either a model or model deployment id."
             )
+
+        # The model to evaluate
         evaluation_source = None
-        eval_inference_configuration = None
+        # The evaluation service config
+        evaluation_config: EvaluationServiceConfig = evaluation_service_config()
+        # The evaluation inference configuration. The inference configuration will be extracted
+        # based on the inferencing container family.
+        eval_inference_configuration: Dict = {}
+        # The evaluation inference model sampling params. The system parameters that will not be
+        # visible for user, but will be applied implicitly for evaluation. The service model params
+        # will be extracted based on the container family and version.
+        eval_inference_service_model_params: Dict = {}
+
         if (
             DataScienceResource.MODEL_DEPLOYMENT
             in create_aqua_evaluation_details.evaluation_source_id
@@ -188,17 +200,32 @@ def create(
                     runtime = ModelDeploymentContainerRuntime.from_dict(
                         evaluation_source.runtime.to_dict()
                     )
-                    inference_config = AquaContainerConfig.from_container_index_json(
+                    container_config = AquaContainerConfig.from_container_index_json(
                         enable_spec=True
-                    ).inference
-                    for container in inference_config.values():
-                        if container.name == runtime.image[: runtime.image.rfind(":")]:
+                    )
+                    for (
+                        inference_container_family,
+                        inference_container_info,
+                    ) in container_config.inference.items():
+                        if (
+                            inference_container_info.name
+                            == runtime.image[: runtime.image.rfind(":")]
+                        ):
                             eval_inference_configuration = (
-                                container.spec.evaluation_configuration
+                                evaluation_config.get_merged_inference_params(
+                                    inference_container_family
+                                ).to_dict()
+                            )
+                            eval_inference_service_model_params = (
+                                evaluation_config.get_merged_inference_model_params(
+                                    inference_container_family,
+                                    inference_container_info.version,
+                                )
                             )
+
             except Exception:
                 logger.debug(
-                    f"Could not load inference config details for the evaluation id: "
+                    f"Could not load inference config details for the evaluation source id: "
                     f"{create_aqua_evaluation_details.evaluation_source_id}. Please check if the container"
                     f" runtime has the correct SMC image information."
                 )
@@ -415,13 +442,12 @@ def create(
                 container_image=container_image,
                 dataset_path=evaluation_dataset_path,
                 report_path=create_aqua_evaluation_details.report_path,
-                model_parameters=create_aqua_evaluation_details.model_parameters,
+                model_parameters={
+                    **eval_inference_service_model_params,
+                    **create_aqua_evaluation_details.model_parameters,
+                },
                 metrics=create_aqua_evaluation_details.metrics,
-                inference_configuration=(
-                    eval_inference_configuration.to_filtered_dict()
-                    if eval_inference_configuration
-                    else {}
-                ),
+                inference_configuration=eval_inference_configuration or {},
             )
         ).create(**kwargs)  ## TODO: decide what parameters will be needed
         logger.debug(
@@ -1188,45 +1214,24 @@ def _delete_job_and_model(job, model):
                 f"Exception message: {ex}"
             )
 
-    def load_evaluation_config(self):
+    def load_evaluation_config(self, container: Optional[str] = None) -> Dict:
         """Loads evaluation config."""
+
+        # retrieve the evaluation config by container family name
+        evaluation_config = evaluation_service_config(container)
+
+        # convert the new config representation to the old one
         return {
-            "model_params": {
-                "max_tokens": 500,
-                "temperature": 0.7,
-                "top_p": 1.0,
-                "top_k": 50,
-                "presence_penalty": 0.0,
-                "frequency_penalty": 0.0,
-                "stop": [],
-            },
+            "model_params": evaluation_config.ui_config.model_params.default,
             "shape": {
-                "VM.Standard.E3.Flex": {
-                    "ocpu": 8,
-                    "memory_in_gbs": 128,
-                    "block_storage_size": 200,
-                },
-                "VM.Standard.E4.Flex": {
-                    "ocpu": 8,
-                    "memory_in_gbs": 128,
-                    "block_storage_size": 200,
-                },
-                "VM.Standard3.Flex": {
-                    "ocpu": 8,
-                    "memory_in_gbs": 128,
-                    "block_storage_size": 200,
-                },
-                "VM.Optimized3.Flex": {
-                    "ocpu": 8,
-                    "memory_in_gbs": 128,
-                    "block_storage_size": 200,
-                },
-            },
-            "default": {
-                "ocpu": 8,
-                "memory_in_gbs": 128,
-                "block_storage_size": 200,
+                shape.name: shape.to_dict()
+                for shape in evaluation_config.ui_config.shapes
             },
+            "default": (
+                evaluation_config.ui_config.shapes[0].to_dict()
+                if len(evaluation_config.ui_config.shapes) > 0
+                else {}
+            ),
         }
 
     def _get_attribute_from_model_metadata(

ads/aqua/extension/evaluation_handler.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
+from typing import Optional
 from urllib.parse import urlparse
 
 from tornado.web import HTTPError
@@ -30,7 +31,7 @@ def get(self, eval_id=""):
         return self.read(eval_id)
 
     @handle_exceptions
-    def post(self, *args, **kwargs):
+    def post(self, *args, **kwargs):  # noqa
         """Handles post request for the evaluation APIs
 
         Raises
@@ -117,10 +118,10 @@ class AquaEvaluationConfigHandler(AquaAPIhandler):
     """Handler for Aqua Evaluation Config REST APIs."""
 
     @handle_exceptions
-    def get(self, model_id):
+    def get(self, container: Optional[str] = None, **kwargs):  # noqa
         """Handle GET request."""
 
-        return self.finish(AquaEvaluationApp().load_evaluation_config(model_id))
+        return self.finish(AquaEvaluationApp().load_evaluation_config(container))
 
 
 __handlers__ = [

ads/aqua/ui.py

@@ -84,9 +84,6 @@ class AquaContainerConfigSpec(DataClassSerializable):
     health_check_port: str = None
     env_vars: List[dict] = None
     restricted_params: List[str] = None
-    evaluation_configuration: AquaContainerEvaluationConfig = field(
-        default_factory=AquaContainerEvaluationConfig
-    )
 
 
 @dataclass(repr=False)
@@ -184,32 +181,37 @@ def from_container_index_json(
                         family=container_type,
                         platforms=platforms,
                         model_formats=model_formats,
-                        spec=AquaContainerConfigSpec(
-                            cli_param=container_spec.get(ContainerSpec.CLI_PARM, ""),
-                            server_port=container_spec.get(
-                                ContainerSpec.SERVER_PORT, ""
-                            ),
-                            health_check_port=container_spec.get(
-                                ContainerSpec.HEALTH_CHECK_PORT, ""
-                            ),
-                            env_vars=container_spec.get(ContainerSpec.ENV_VARS, []),
-                            restricted_params=container_spec.get(
-                                ContainerSpec.RESTRICTED_PARAMS, []
-                            ),
-                            evaluation_configuration=AquaContainerEvaluationConfig.from_config(
-                                container_spec.get(
-                                    ContainerSpec.EVALUATION_CONFIGURATION, {}
-                                )
-                            ),
-                        )
-                        if container_spec
-                        else None,
+                        spec=(
+                            AquaContainerConfigSpec(
+                                cli_param=container_spec.get(
+                                    ContainerSpec.CLI_PARM, ""
+                                ),
+                                server_port=container_spec.get(
+                                    ContainerSpec.SERVER_PORT, ""
+                                ),
+                                health_check_port=container_spec.get(
+                                    ContainerSpec.HEALTH_CHECK_PORT, ""
+                                ),
+                                env_vars=container_spec.get(ContainerSpec.ENV_VARS, []),
+                                restricted_params=container_spec.get(
+                                    ContainerSpec.RESTRICTED_PARAMS, []
+                                ),
+                            )
+                            if container_spec
+                            else None
+                        ),
                     )
                     if container.get("type") == "inference":
                         inference_items[container_type] = container_item
-                    elif container_type == "odsc-llm-fine-tuning":
+                    elif (
+                        container.get("type") == "fine-tune"
+                        or container_type == "odsc-llm-fine-tuning"
+                    ):
                         finetune_items[container_type] = container_item
-                    elif container_type == "odsc-llm-evaluate":
+                    elif (
+                        container.get("type") == "evaluate"
+                        or container_type == "odsc-llm-evaluate"
+                    ):
                         evaluate_items[container_type] = container_item
 
         return AquaContainerConfig(

tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json

@@ -102,7 +102,7 @@
       "inference_timeout": 120
     }
   },
-  "kind": "evaluation",
+  "kind": "evaluation_service_config",
   "report_params": {
     "default": {}
   },

tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json

@@ -7,7 +7,7 @@
     "containers": [],
     "default": {}
   },
-  "kind": "evaluation",
+  "kind": "evaluation_service_config",
   "report_params": {
     "default": {}
   },