add evaluation config params

Author: VipulMascarenhas

ads/aqua/constants.py

@@ -15,7 +15,6 @@
 EVALUATION_REPORT_JSON = "report.json"
 EVALUATION_REPORT_MD = "report.md"
 EVALUATION_REPORT = "report.html"
-EVALUATION_INFERENCE_DEFAULT_THREADS = 10
 UNKNOWN_JSON_STR = "{}"
 FINE_TUNING_RUNTIME_CONTAINER = "iad.ocir.io/ociodscdev/aqua_ft_cuda121:0.3.17.20"
 DEFAULT_FT_BLOCK_STORAGE_SIZE = 750

ads/aqua/evaluation/entities.py

@@ -64,8 +64,6 @@ class CreateAquaEvaluationDetails(DataClassSerializable):
         The metrics for the evaluation.
     force_overwrite: (bool, optional). Defaults to `False`.
         Whether to force overwrite the existing file in object storage.
-    inference_max_threads: (int, optional). Defaults to None
-        Set the value of concurrent requests to be made to the inference endpoint during evaluation.
     """
 
     evaluation_source_id: str
@@ -87,7 +85,6 @@ class CreateAquaEvaluationDetails(DataClassSerializable):
     log_id: Optional[str] = None
     metrics: Optional[List] = None
     force_overwrite: Optional[bool] = False
-    inference_max_threads: Optional[int] = None
 
 
 @dataclass(repr=False)
@@ -144,7 +141,6 @@ class AquaEvaluationCommands(DataClassSerializable):
     metrics: list
     output_dir: str
     params: dict
-    inference_max_threads: int
 
 
 @dataclass(repr=False)

ads/aqua/evaluation/evaluation.py

@@ -13,9 +13,15 @@
 from threading import Lock
 from typing import Any, Dict, List, Union
 
+import oci
 from cachetools import TTLCache
+from oci.data_science.models import (
+    JobRun,
+    Metadata,
+    UpdateModelDetails,
+    UpdateModelProvenanceDetails,
+)
 
-import oci
 from ads.aqua import logger
 from ads.aqua.app import AquaApp
 from ads.aqua.common import utils
@@ -41,7 +47,6 @@
 )
 from ads.aqua.constants import (
     CONSOLE_LINK_RESOURCE_TYPE_MAPPING,
-    EVALUATION_INFERENCE_DEFAULT_THREADS,
     EVALUATION_REPORT,
     EVALUATION_REPORT_JSON,
     EVALUATION_REPORT_MD,
@@ -97,12 +102,6 @@
 )
 from ads.model.model_version_set import ModelVersionSet
 from ads.telemetry import telemetry
-from oci.data_science.models import (
-    JobRun,
-    Metadata,
-    UpdateModelDetails,
-    UpdateModelProvenanceDetails,
-)
 
 
 class AquaEvaluationApp(AquaApp):
@@ -171,6 +170,7 @@ def create(
                 "Specify either a model or model deployment id."
             )
         evaluation_source = None
+        eval_inference_configuration = None
         if (
             DataScienceResource.MODEL_DEPLOYMENT
             in create_aqua_evaluation_details.evaluation_source_id
@@ -182,29 +182,14 @@ def create(
                 runtime = ModelDeploymentContainerRuntime.from_dict(
                     evaluation_source.runtime.to_dict()
                 )
-                container_config = AquaContainerConfig.from_container_index_json(
+                inference_config = AquaContainerConfig.from_container_index_json(
                     enable_spec=True
-                )
-                for container in container_config.inference.values():
+                ).inference
+                for container in inference_config.values():
                     if container.name == runtime.image.split(":")[0]:
-                        max_threads = container.spec.evaluation_configuration.evaluation_max_threads
-                        if (
-                            max_threads
-                            and create_aqua_evaluation_details.inference_max_threads
-                            and max_threads
-                            < create_aqua_evaluation_details.inference_max_threads
-                        ):
-                            raise AquaValueError(
-                                f"Invalid inference max threads. The maximum allowed value for {runtime.image} is {max_threads}."
-                            )
-                        if not create_aqua_evaluation_details.inference_max_threads:
-                            create_aqua_evaluation_details.inference_max_threads = container.spec.evaluation_configuration.evaluation_default_threads
-                        break
-                if not create_aqua_evaluation_details.inference_max_threads:
-                    create_aqua_evaluation_details.inference_max_threads = (
-                        EVALUATION_INFERENCE_DEFAULT_THREADS
-                    )
-
+                        eval_inference_configuration = (
+                            container.spec.evaluation_configuration
+                        )
         elif (
             DataScienceResource.MODEL
             in create_aqua_evaluation_details.evaluation_source_id
@@ -420,7 +405,9 @@ def create(
                 report_path=create_aqua_evaluation_details.report_path,
                 model_parameters=create_aqua_evaluation_details.model_parameters,
                 metrics=create_aqua_evaluation_details.metrics,
-                inference_max_threads=create_aqua_evaluation_details.inference_max_threads,
+                inference_configuration=eval_inference_configuration.to_filtered_dict()
+                if eval_inference_configuration
+                else {},
             )
         ).create(**kwargs)  ## TODO: decide what parameters will be needed
         logger.debug(
@@ -542,7 +529,7 @@ def _build_evaluation_runtime(
         report_path: str,
         model_parameters: dict,
         metrics: List = None,
-        inference_max_threads: int = None,
+        inference_configuration: dict = None,
     ) -> Runtime:
         """Builds evaluation runtime for Job."""
         # TODO the image name needs to be extracted from the mapping index.json file.
@@ -552,17 +539,19 @@ def _build_evaluation_runtime(
             .with_environment_variable(
                 **{
                     "AIP_SMC_EVALUATION_ARGUMENTS": json.dumps(
-                        asdict(
-                            self._build_launch_cmd(
-                                evaluation_id=evaluation_id,
-                                evaluation_source_id=evaluation_source_id,
-                                dataset_path=dataset_path,
-                                report_path=report_path,
-                                model_parameters=model_parameters,
-                                metrics=metrics,
-                                inference_max_threads=inference_max_threads,
-                            )
-                        )
+                        {
+                            **asdict(
+                                self._build_launch_cmd(
+                                    evaluation_id=evaluation_id,
+                                    evaluation_source_id=evaluation_source_id,
+                                    dataset_path=dataset_path,
+                                    report_path=report_path,
+                                    model_parameters=model_parameters,
+                                    metrics=metrics,
+                                ),
+                            ),
+                            **inference_configuration,
+                        },
                     ),
                     "CONDA_BUCKET_NS": CONDA_BUCKET_NS,
                 },
@@ -620,7 +609,6 @@ def _build_launch_cmd(
         report_path: str,
         model_parameters: dict,
         metrics: List = None,
-        inference_max_threads: int = None,
     ):
         return AquaEvaluationCommands(
             evaluation_id=evaluation_id,
@@ -637,7 +625,6 @@ def _build_launch_cmd(
             metrics=metrics,
             output_dir=report_path,
             params=model_parameters,
-            inference_max_threads=inference_max_threads,
         )
 
     @telemetry(entry_point="plugin=evaluation&action=get", name="aqua")
@@ -1227,7 +1214,7 @@ def _delete_job_and_model(job, model):
                 f"Exception message: {ex}"
             )
 
-    def load_evaluation_config(self, _):
+    def load_evaluation_config(self, eval_id):
         """Loads evaluation config."""
         return {
             "model_params": {

ads/aqua/ui.py

@@ -2,21 +2,22 @@
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 import concurrent.futures
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, fields
 from datetime import datetime, timedelta
 from enum import Enum
 from threading import Lock
 from typing import Dict, List, Optional
 
 from cachetools import TTLCache
+from oci.exceptions import ServiceError
+from oci.identity.models import Compartment
 
 from ads.aqua import logger
 from ads.aqua.app import AquaApp
 from ads.aqua.common.entities import ContainerSpec
 from ads.aqua.common.enums import Tags
 from ads.aqua.common.errors import AquaResourceAccessError, AquaValueError
 from ads.aqua.common.utils import get_container_config, load_config, sanitize_response
-from ads.aqua.constants import EVALUATION_INFERENCE_DEFAULT_THREADS
 from ads.common import oci_client as oc
 from ads.common.auth import default_signer
 from ads.common.object_storage_details import ObjectStorageDetails
@@ -29,8 +30,6 @@
     TENANCY_OCID,
 )
 from ads.telemetry import telemetry
-from oci.exceptions import ServiceError
-from oci.identity.models import Compartment
 
 
 class ModelFormat(Enum):
@@ -47,25 +46,36 @@ def to_dict(self):
 
 
 @dataclass(repr=False)
-class AquaContainerEvaluationConfiguration(DataClassSerializable):
+class AquaContainerEvaluationConfig(DataClassSerializable):
     """
     Represents the evaluation configuration for the container.
     """
 
-    evaluation_max_threads: Optional[int] = None
-    evaluation_default_threads: int = field(
-        default=EVALUATION_INFERENCE_DEFAULT_THREADS
-    )
+    inference_max_threads: Optional[int] = None
+    inference_rps: Optional[int] = None
+    inference_timeout: Optional[int] = None
+    inference_retries: Optional[int] = None
+    inference_backoff_factor: Optional[int] = None
+    inference_delay: Optional[int] = None
 
     @classmethod
-    def from_config(cls, config: dict) -> "AquaContainerEvaluationConfiguration":
+    def from_config(cls, config: dict) -> "AquaContainerEvaluationConfig":
         return cls(
-            evaluation_max_threads=config.get("MAX_THREADS"),
-            evaluation_default_threads=config.get(
-                "DEFAULT_THREADS", EVALUATION_INFERENCE_DEFAULT_THREADS
-            ),
+            inference_max_threads=config.get("inference_max_threads"),
+            inference_rps=config.get("inference_rps"),
+            inference_timeout=config.get("inference_timeout"),
+            inference_retries=config.get("inference_retries"),
+            inference_backoff_factor=config.get("inference_backoff_factor"),
+            inference_delay=config.get("inference_delay"),
         )
 
+    def to_filtered_dict(self):
+        return {
+            field.name: getattr(self, field.name)
+            for field in fields(self)
+            if getattr(self, field.name) is not None
+        }
+
 
 @dataclass(repr=False)
 class AquaContainerConfigSpec(DataClassSerializable):
@@ -74,8 +84,8 @@ class AquaContainerConfigSpec(DataClassSerializable):
     health_check_port: str = None
     env_vars: List[dict] = None
     restricted_params: List[str] = None
-    evaluation_configuration: AquaContainerEvaluationConfiguration = field(
-        default_factory=AquaContainerEvaluationConfiguration
+    evaluation_configuration: AquaContainerEvaluationConfig = field(
+        default_factory=AquaContainerEvaluationConfig
     )
 
 
@@ -186,7 +196,7 @@ def from_container_index_json(
                             restricted_params=container_spec.get(
                                 ContainerSpec.RESTRICTED_PARAMS, []
                             ),
-                            evaluation_configuration=AquaContainerEvaluationConfiguration.from_config(
+                            evaluation_configuration=AquaContainerEvaluationConfig.from_config(
                                 container_spec.get(
                                     ContainerSpec.EVALUATION_CONFIGURATION, {}
                                 )

tests/unitary/with_extras/aqua/test_data/ui/container_index.json

@@ -19,6 +19,10 @@
           "HEALTH_CHECK_PORT": "8080"
         }
       ],
+      "evaluationConfiguration": {
+        "inference_delay": 1,
+        "inference_max_threads": 1
+      },
       "healthCheckPort": "8080",
       "restrictedParams": [],
       "serverPort": "8080"
@@ -81,7 +85,7 @@
       "modelFormats": [
         "GGUF"
       ],
-      "name": "iad.ocir.io/ociodscdev/odsc-llama-cpp-python-aio-linux_arm64_v8",
+      "name": "dsmc://odsc-llama-cpp-python-aio-linux_arm64_v8",
       "platforms": [
         "ARM_CPU"
       ],

tests/unitary/with_extras/aqua/test_ui.py

@@ -484,7 +484,7 @@ def test_list_containers(self, mock_get_container_config):
             ],
             "inference": [
                 {
-                    "name": "iad.ocir.io/ociodscdev/odsc-llama-cpp-python-aio-linux_arm64_v8",
+                    "name": "dsmc://odsc-llama-cpp-python-aio-linux_arm64_v8",
                     "version": "0.2.75.5",
                     "display_name": "LLAMA-CPP:0.2.75",
                     "family": "odsc-llama-cpp-serving",
@@ -502,6 +502,14 @@ def test_list_containers(self, mock_get_container_config):
                         "health_check_port": "8080",
                         "restricted_params": [],
                         "server_port": "8080",
+                        "evaluation_configuration": {
+                            "inference_max_threads": 1,
+                            "inference_rps": None,
+                            "inference_timeout": None,
+                            "inference_backoff_factor": None,
+                            "inference_delay": 1,
+                            "inference_retries": None,
+                        },
                     },
                 },
                 {
@@ -528,6 +536,14 @@ def test_list_containers(self, mock_get_container_config):
                             "--trust-remote-code",
                         ],
                         "server_port": "8080",
+                        "evaluation_configuration": {
+                            "inference_max_threads": None,
+                            "inference_rps": None,
+                            "inference_timeout": None,
+                            "inference_backoff_factor": None,
+                            "inference_delay": None,
+                            "inference_retries": None,
+                        },
                     },
                 },
                 {
@@ -553,6 +569,14 @@ def test_list_containers(self, mock_get_container_config):
                             "--seed",
                         ],
                         "server_port": "8080",
+                        "evaluation_configuration": {
+                            "inference_max_threads": None,
+                            "inference_rps": None,
+                            "inference_timeout": None,
+                            "inference_backoff_factor": None,
+                            "inference_delay": None,
+                            "inference_retries": None,
+                        },
                     },
                 },
             ],