ODSC-61986: Get evaluation shapes list form the service config.

mrDzurb · mrDzurb · commit 9f69f375625c · 2024-09-05T17:07:36.000-07:00
diff --git a/ads/aqua/config/config.py b/ads/aqua/config/config.py
@@ -15,7 +15,7 @@
 DEFAULT_EVALUATION_CONTAINER = "odsc-llm-evaluate"
 
 
-@cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=5), timer=datetime.now))
+@cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=1), timer=datetime.now))
 def evaluation_service_config(
     container: Optional[str] = DEFAULT_EVALUATION_CONTAINER,
 ) -> EvaluationServiceConfig:
@@ -27,6 +27,7 @@ def evaluation_service_config(
     EvaluationServiceConfig: The evaluation common config.
     """
 
+    container = container or DEFAULT_EVALUATION_CONTAINER
     return EvaluationServiceConfig(
         **get_container_config()
         .get(ContainerSpec.CONTAINER_SPEC, {})
diff --git a/ads/aqua/config/evaluation/evaluation_service_config.py b/ads/aqua/config/evaluation/evaluation_service_config.py
@@ -250,7 +250,7 @@ class EvaluationServiceConfig(Serializable):
     """
 
     version: Optional[str] = "1.0"
-    kind: Optional[str] = "evaluation"
+    kind: Optional[str] = "evaluation_service_config"
     report_params: Optional[ReportParams] = Field(default_factory=ReportParams)
     inference_params: Optional[InferenceParamsConfig] = Field(
         default_factory=InferenceParamsConfig
diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py
@@ -11,7 +11,7 @@
 from datetime import datetime, timedelta
 from pathlib import Path
 from threading import Lock
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List, Optional, Union
 
 import oci
 from cachetools import TTLCache
@@ -45,6 +45,7 @@
     is_valid_ocid,
     upload_local_to_os,
 )
+from ads.aqua.config.config import evaluation_service_config
 from ads.aqua.constants import (
     CONSOLE_LINK_RESOURCE_TYPE_MAPPING,
     EVALUATION_REPORT,
@@ -171,7 +172,7 @@ def create(
                 "Specify either a model or model deployment id."
             )
         evaluation_source = None
-        eval_inference_configuration = None
+        eval_inference_configuration: Dict = {}
         if (
             DataScienceResource.MODEL_DEPLOYMENT
             in create_aqua_evaluation_details.evaluation_source_id
@@ -187,17 +188,26 @@ def create(
                     runtime = ModelDeploymentContainerRuntime.from_dict(
                         evaluation_source.runtime.to_dict()
                     )
-                    inference_config = AquaContainerConfig.from_container_index_json(
+                    container_config = AquaContainerConfig.from_container_index_json(
                         enable_spec=True
-                    ).inference
-                    for container in inference_config.values():
-                        if container.name == runtime.image[:runtime.image.rfind(":")]:
+                    )
+                    for (
+                        inference_container_family,
+                        inference_container_info,
+                    ) in container_config.inference.items():
+                        if (
+                            inference_container_info.name
+                            == runtime.image[: runtime.image.rfind(":")]
+                        ):
                             eval_inference_configuration = (
-                                container.spec.evaluation_configuration
+                                evaluation_service_config()
+                                .get_merged_inference_params(inference_container_family)
+                                .to_dict()
                             )
+
             except Exception:
                 logger.debug(
-                    f"Could not load inference config details for the evaluation id: "
+                    f"Could not load inference config details for the evaluation source id: "
                     f"{create_aqua_evaluation_details.evaluation_source_id}. Please check if the container"
                     f" runtime has the correct SMC image information."
                 )
@@ -416,9 +426,7 @@ def create(
                 report_path=create_aqua_evaluation_details.report_path,
                 model_parameters=create_aqua_evaluation_details.model_parameters,
                 metrics=create_aqua_evaluation_details.metrics,
-                inference_configuration=eval_inference_configuration.to_filtered_dict()
-                if eval_inference_configuration
-                else {},
+                inference_configuration=eval_inference_configuration or {},
             )
         ).create(**kwargs)  ## TODO: decide what parameters will be needed
         logger.debug(
@@ -1225,45 +1233,24 @@ def _delete_job_and_model(job, model):
                 f"Exception message: {ex}"
             )
 
-    def load_evaluation_config(self, eval_id):
+    def load_evaluation_config(self, container: Optional[str] = None) -> Dict:
         """Loads evaluation config."""
+
+        # retrieve the evaluation config by container family name
+        evaluation_config = evaluation_service_config(container)
+
+        # convert the new config representation to the old one
         return {
-            "model_params": {
-                "max_tokens": 500,
-                "temperature": 0.7,
-                "top_p": 1.0,
-                "top_k": 50,
-                "presence_penalty": 0.0,
-                "frequency_penalty": 0.0,
-                "stop": [],
-            },
+            "model_params": evaluation_config.ui_config.model_params.default,
             "shape": {
-                "VM.Standard.E3.Flex": {
-                    "ocpu": 8,
-                    "memory_in_gbs": 128,
-                    "block_storage_size": 200,
-                },
-                "VM.Standard.E4.Flex": {
-                    "ocpu": 8,
-                    "memory_in_gbs": 128,
-                    "block_storage_size": 200,
-                },
-                "VM.Standard3.Flex": {
-                    "ocpu": 8,
-                    "memory_in_gbs": 128,
-                    "block_storage_size": 200,
-                },
-                "VM.Optimized3.Flex": {
-                    "ocpu": 8,
-                    "memory_in_gbs": 128,
-                    "block_storage_size": 200,
-                },
-            },
-            "default": {
-                "ocpu": 8,
-                "memory_in_gbs": 128,
-                "block_storage_size": 200,
+                shape.name: shape.to_dict()
+                for shape in evaluation_config.ui_config.shapes
             },
+            "default": (
+                evaluation_config.ui_config.shapes[0].to_dict()
+                if len(evaluation_config.ui_config.shapes) > 0
+                else {}
+            ),
         }
 
     def _get_attribute_from_model_metadata(
diff --git a/ads/aqua/extension/evaluation_handler.py b/ads/aqua/extension/evaluation_handler.py
@@ -2,6 +2,7 @@
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
+from typing import Optional
 from urllib.parse import urlparse
 
 from tornado.web import HTTPError
@@ -30,7 +31,7 @@ def get(self, eval_id=""):
         return self.read(eval_id)
 
     @handle_exceptions
-    def post(self, *args, **kwargs):
+    def post(self, *args, **kwargs):  # noqa
         """Handles post request for the evaluation APIs
 
         Raises
@@ -117,10 +118,10 @@ class AquaEvaluationConfigHandler(AquaAPIhandler):
     """Handler for Aqua Evaluation Config REST APIs."""
 
     @handle_exceptions
-    def get(self, model_id):
+    def get(self, container: Optional[str] = None, **kwargs):  # noqa
         """Handle GET request."""
 
-        return self.finish(AquaEvaluationApp().load_evaluation_config(model_id))
+        return self.finish(AquaEvaluationApp().load_evaluation_config(container))
 
 
 __handlers__ = [
diff --git a/ads/aqua/ui.py b/ads/aqua/ui.py
@@ -84,9 +84,6 @@ class AquaContainerConfigSpec(DataClassSerializable):
     health_check_port: str = None
     env_vars: List[dict] = None
     restricted_params: List[str] = None
-    evaluation_configuration: AquaContainerEvaluationConfig = field(
-        default_factory=AquaContainerEvaluationConfig
-    )
 
 
 @dataclass(repr=False)
@@ -184,32 +181,37 @@ def from_container_index_json(
                         family=container_type,
                         platforms=platforms,
                         model_formats=model_formats,
-                        spec=AquaContainerConfigSpec(
-                            cli_param=container_spec.get(ContainerSpec.CLI_PARM, ""),
-                            server_port=container_spec.get(
-                                ContainerSpec.SERVER_PORT, ""
-                            ),
-                            health_check_port=container_spec.get(
-                                ContainerSpec.HEALTH_CHECK_PORT, ""
-                            ),
-                            env_vars=container_spec.get(ContainerSpec.ENV_VARS, []),
-                            restricted_params=container_spec.get(
-                                ContainerSpec.RESTRICTED_PARAMS, []
-                            ),
-                            evaluation_configuration=AquaContainerEvaluationConfig.from_config(
-                                container_spec.get(
-                                    ContainerSpec.EVALUATION_CONFIGURATION, {}
-                                )
-                            ),
-                        )
-                        if container_spec
-                        else None,
+                        spec=(
+                            AquaContainerConfigSpec(
+                                cli_param=container_spec.get(
+                                    ContainerSpec.CLI_PARM, ""
+                                ),
+                                server_port=container_spec.get(
+                                    ContainerSpec.SERVER_PORT, ""
+                                ),
+                                health_check_port=container_spec.get(
+                                    ContainerSpec.HEALTH_CHECK_PORT, ""
+                                ),
+                                env_vars=container_spec.get(ContainerSpec.ENV_VARS, []),
+                                restricted_params=container_spec.get(
+                                    ContainerSpec.RESTRICTED_PARAMS, []
+                                ),
+                            )
+                            if container_spec
+                            else None
+                        ),
                     )
                     if container.get("type") == "inference":
                         inference_items[container_type] = container_item
-                    elif container_type == "odsc-llm-fine-tuning":
+                    elif (
+                        container.get("type") == "fine-tune"
+                        or container_type == "odsc-llm-fine-tuning"
+                    ):
                         finetune_items[container_type] = container_item
-                    elif container_type == "odsc-llm-evaluate":
+                    elif (
+                        container.get("type") == "evaluate"
+                        or container_type == "odsc-llm-evaluate"
+                    ):
                         evaluate_items[container_type] = container_item
 
         return AquaContainerConfig(
diff --git a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json
@@ -102,7 +102,7 @@
       "inference_timeout": 120
     }
   },
-  "kind": "evaluation",
+  "kind": "evaluation_service_config",
   "report_params": {
     "default": {}
   },
diff --git a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json
@@ -14,7 +14,7 @@
       "inference_timeout": 120
     }
   },
-  "kind": "evaluation",
+  "kind": "evaluation_service_config",
   "report_params": {
     "default": {}
   },
diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py
@@ -22,6 +22,12 @@
     AquaMissingKeyError,
     AquaRuntimeError,
 )
+from ads.aqua.config.evaluation.evaluation_service_config import (
+    EvaluationServiceConfig,
+    ModelParamsConfig,
+    ShapeConfig,
+    UIConfig,
+)
 from ads.aqua.constants import EVALUATION_REPORT_JSON, EVALUATION_REPORT_MD, UNKNOWN
 from ads.aqua.evaluation import AquaEvaluationApp
 from ads.aqua.evaluation.entities import (
@@ -887,16 +893,85 @@ def test_get_supported_metrics(self):
         for metric in response:
             assert check(metric_schema, metric)
 
-    def test_load_evaluation_config(self):
-        """Tests loading default config for evaluation.
+    @patch("ads.aqua.evaluation.evaluation.evaluation_service_config")
+    def test_load_evaluation_config(self, mock_evaluation_service_config):
+        """
+        Tests loading default config for evaluation.
         This method currently hardcoded the return value.
         """
-        from .utils import EvaluationConfigFormat as config_schema
-        from .utils import check
 
-        response = self.app.load_evaluation_config(eval_id=TestDataset.EVAL_ID)
+        test_evaluation_service_config = EvaluationServiceConfig(
+            ui_config=UIConfig(
+                model_params=ModelParamsConfig(
+                    **{
+                        "default": {
+                            "model": "odsc-llm",
+                            "max_tokens": 500,
+                            "temperature": 0.7,
+                            "top_p": 0.9,
+                            "top_k": 50,
+                            "presence_penalty": 0.0,
+                            "frequency_penalty": 0.0,
+                            "stop": [],
+                        }
+                    }
+                ),
+                shapes=[
+                    ShapeConfig(
+                        **{
+                            "name": "VM.Standard.E3.Flex",
+                            "ocpu": 8,
+                            "memory_in_gbs": 128,
+                            "block_storage_size": 200,
+                            "filter": {
+                                "evaluation_container": ["odsc-llm-evaluate"],
+                                "evaluation_target": ["datasciencemodeldeployment"],
+                            },
+                        }
+                    )
+                ],
+            )
+        )
+        mock_evaluation_service_config.return_value = test_evaluation_service_config
+
+        expected_result = {
+            "model_params": {
+                "model": "odsc-llm",
+                "max_tokens": 500,
+                "temperature": 0.7,
+                "top_p": 0.9,
+                "top_k": 50,
+                "presence_penalty": 0.0,
+                "frequency_penalty": 0.0,
+                "stop": [],
+            },
+            "shape": {
+                "VM.Standard.E3.Flex": {
+                    "name": "VM.Standard.E3.Flex",
+                    "ocpu": 8,
+                    "memory_in_gbs": 128,
+                    "block_storage_size": 200,
+                    "filter": {
+                        "evaluation_container": ["odsc-llm-evaluate"],
+                        "evaluation_target": ["datasciencemodeldeployment"],
+                    },
+                }
+            },
+            "default": {
+                "name": "VM.Standard.E3.Flex",
+                "ocpu": 8,
+                "memory_in_gbs": 128,
+                "block_storage_size": 200,
+                "filter": {
+                    "evaluation_container": ["odsc-llm-evaluate"],
+                    "evaluation_target": ["datasciencemodeldeployment"],
+                },
+            },
+        }
+
+        response = self.app.load_evaluation_config()
         assert isinstance(response, dict)
-        assert check(config_schema, response)
+        assert response == expected_result
 
 
 class TestAquaEvaluationList(unittest.TestCase):
diff --git a/tests/unitary/with_extras/aqua/test_ui.py b/tests/unitary/with_extras/aqua/test_ui.py
diff --git a/tests/unitary/with_extras/aqua/utils.py b/tests/unitary/with_extras/aqua/utils.py

Original file line number	Diff line number	Diff line change
`@@ -102,7 +102,7 @@`
`102`	`102`	`"inference_timeout": 120`
`103`	`103`	`}`
`104`	`104`	`},`
`105`		`- "kind": "evaluation",`
	`105`	`+ "kind": "evaluation_service_config",`
`106`	`106`	`"report_params": {`
`107`	`107`	`"default": {}`
`108`	`108`	`},`
Original file line number	Diff line number	Diff line change
`@@ -14,7 +14,7 @@`
`14`	`14`	`"inference_timeout": 120`
`15`	`15`	`}`
`16`	`16`	`},`
`17`		`- "kind": "evaluation",`
	`17`	`+ "kind": "evaluation_service_config",`
`18`	`18`	`"report_params": {`
`19`	`19`	`"default": {}`
`20`	`20`	`},`