[ODSC-60256] llama cpp evaluation support (#909)

Author: VipulMascarenhas

ads/aqua/common/entities.py

@@ -14,3 +14,4 @@ class ContainerSpec:
     HEALTH_CHECK_PORT = "healthCheckPort"
     ENV_VARS = "envVars"
     RESTRICTED_PARAMS = "restrictedParams"
+    EVALUATION_CONFIGURATION = "evaluationConfiguration"

ads/aqua/common/utils.py

@@ -249,7 +249,7 @@ def list_os_files_with_extension(oss_path: str, extension: str) -> [str]:
     files: List[ObjectSummary] = oss_client.list_objects().objects
 
     return [
-        file.name[len(oss_client.filepath) :]
+        file.name[len(oss_client.filepath) :].lstrip("/")
         for file in files
         if file.name.endswith(extension)
     ]

ads/aqua/evaluation/entities.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 

ads/aqua/evaluation/evaluation.py

@@ -76,6 +76,7 @@
     ModelParams,
 )
 from ads.aqua.evaluation.errors import EVALUATION_JOB_EXIT_CODE_MESSAGE
+from ads.aqua.ui import AquaContainerConfig
 from ads.common.auth import default_signer
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.common.utils import get_console_link, get_files, get_log_links
@@ -90,7 +91,9 @@
 from ads.jobs.builders.runtimes.base import Runtime
 from ads.jobs.builders.runtimes.container_runtime import ContainerRuntime
 from ads.model.datascience_model import DataScienceModel
+from ads.model.deployment import ModelDeploymentContainerRuntime
 from ads.model.deployment.model_deployment import ModelDeployment
+from ads.model.generic_model import ModelDeploymentRuntimeType
 from ads.model.model_metadata import (
     MetadataTaxonomyKeys,
     ModelCustomMetadata,
@@ -166,15 +169,27 @@ def create(
                 f"Invalid evaluation source {create_aqua_evaluation_details.evaluation_source_id}. "
                 "Specify either a model or model deployment id."
             )
-
         evaluation_source = None
+        eval_inference_configuration = None
         if (
             DataScienceResource.MODEL_DEPLOYMENT
             in create_aqua_evaluation_details.evaluation_source_id
         ):
             evaluation_source = ModelDeployment.from_id(
                 create_aqua_evaluation_details.evaluation_source_id
             )
+            if evaluation_source.runtime.type == ModelDeploymentRuntimeType.CONTAINER:
+                runtime = ModelDeploymentContainerRuntime.from_dict(
+                    evaluation_source.runtime.to_dict()
+                )
+                inference_config = AquaContainerConfig.from_container_index_json(
+                    enable_spec=True
+                ).inference
+                for container in inference_config.values():
+                    if container.name == runtime.image.split(":")[0]:
+                        eval_inference_configuration = (
+                            container.spec.evaluation_configuration
+                        )
         elif (
             DataScienceResource.MODEL
             in create_aqua_evaluation_details.evaluation_source_id
@@ -390,6 +405,9 @@ def create(
                 report_path=create_aqua_evaluation_details.report_path,
                 model_parameters=create_aqua_evaluation_details.model_parameters,
                 metrics=create_aqua_evaluation_details.metrics,
+                inference_configuration=eval_inference_configuration.to_filtered_dict()
+                if eval_inference_configuration
+                else {},
             )
         ).create(**kwargs)  ## TODO: decide what parameters will be needed
         logger.debug(
@@ -511,6 +529,7 @@ def _build_evaluation_runtime(
         report_path: str,
         model_parameters: dict,
         metrics: List = None,
+        inference_configuration: dict = None,
     ) -> Runtime:
         """Builds evaluation runtime for Job."""
         # TODO the image name needs to be extracted from the mapping index.json file.
@@ -520,16 +539,19 @@ def _build_evaluation_runtime(
             .with_environment_variable(
                 **{
                     "AIP_SMC_EVALUATION_ARGUMENTS": json.dumps(
-                        asdict(
-                            self._build_launch_cmd(
-                                evaluation_id=evaluation_id,
-                                evaluation_source_id=evaluation_source_id,
-                                dataset_path=dataset_path,
-                                report_path=report_path,
-                                model_parameters=model_parameters,
-                                metrics=metrics,
-                            )
-                        )
+                        {
+                            **asdict(
+                                self._build_launch_cmd(
+                                    evaluation_id=evaluation_id,
+                                    evaluation_source_id=evaluation_source_id,
+                                    dataset_path=dataset_path,
+                                    report_path=report_path,
+                                    model_parameters=model_parameters,
+                                    metrics=metrics,
+                                ),
+                            ),
+                            **inference_configuration,
+                        },
                     ),
                     "CONDA_BUCKET_NS": CONDA_BUCKET_NS,
                 },

ads/aqua/ui.py

@@ -2,7 +2,7 @@
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 import concurrent.futures
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, fields
 from datetime import datetime, timedelta
 from enum import Enum
 from threading import Lock
@@ -45,13 +45,48 @@ def to_dict(self):
 #   within ads.aqua.common.entities. In that case, check for circular imports due to usage of get_container_config.
 
 
+@dataclass(repr=False)
+class AquaContainerEvaluationConfig(DataClassSerializable):
+    """
+    Represents the evaluation configuration for the container.
+    """
+
+    inference_max_threads: Optional[int] = None
+    inference_rps: Optional[int] = None
+    inference_timeout: Optional[int] = None
+    inference_retries: Optional[int] = None
+    inference_backoff_factor: Optional[int] = None
+    inference_delay: Optional[int] = None
+
+    @classmethod
+    def from_config(cls, config: dict) -> "AquaContainerEvaluationConfig":
+        return cls(
+            inference_max_threads=config.get("inference_max_threads"),
+            inference_rps=config.get("inference_rps"),
+            inference_timeout=config.get("inference_timeout"),
+            inference_retries=config.get("inference_retries"),
+            inference_backoff_factor=config.get("inference_backoff_factor"),
+            inference_delay=config.get("inference_delay"),
+        )
+
+    def to_filtered_dict(self):
+        return {
+            field.name: getattr(self, field.name)
+            for field in fields(self)
+            if getattr(self, field.name) is not None
+        }
+
+
 @dataclass(repr=False)
 class AquaContainerConfigSpec(DataClassSerializable):
     cli_param: str = None
     server_port: str = None
     health_check_port: str = None
     env_vars: List[dict] = None
     restricted_params: List[str] = None
+    evaluation_configuration: AquaContainerEvaluationConfig = field(
+        default_factory=AquaContainerEvaluationConfig
+    )
 
 
 @dataclass(repr=False)
@@ -161,6 +196,11 @@ def from_container_index_json(
                             restricted_params=container_spec.get(
                                 ContainerSpec.RESTRICTED_PARAMS, []
                             ),
+                            evaluation_configuration=AquaContainerEvaluationConfig.from_config(
+                                container_spec.get(
+                                    ContainerSpec.EVALUATION_CONFIGURATION, {}
+                                )
+                            ),
                         )
                         if container_spec
                         else None,

tests/unitary/with_extras/aqua/test_data/ui/container_index.json

@@ -19,6 +19,10 @@
           "HEALTH_CHECK_PORT": "8080"
         }
       ],
+      "evaluationConfiguration": {
+        "inference_delay": 1,
+        "inference_max_threads": 1
+      },
       "healthCheckPort": "8080",
       "restrictedParams": [],
       "serverPort": "8080"
@@ -81,7 +85,7 @@
       "modelFormats": [
         "GGUF"
       ],
-      "name": "iad.ocir.io/ociodscdev/odsc-llama-cpp-python-aio-linux_arm64_v8",
+      "name": "dsmc://odsc-llama-cpp-python-aio-linux_arm64_v8",
       "platforms": [
         "ARM_CPU"
       ],

tests/unitary/with_extras/aqua/test_ui.py

@@ -484,7 +484,7 @@ def test_list_containers(self, mock_get_container_config):
             ],
             "inference": [
                 {
-                    "name": "iad.ocir.io/ociodscdev/odsc-llama-cpp-python-aio-linux_arm64_v8",
+                    "name": "dsmc://odsc-llama-cpp-python-aio-linux_arm64_v8",
                     "version": "0.2.75.5",
                     "display_name": "LLAMA-CPP:0.2.75",
                     "family": "odsc-llama-cpp-serving",
@@ -502,6 +502,14 @@ def test_list_containers(self, mock_get_container_config):
                         "health_check_port": "8080",
                         "restricted_params": [],
                         "server_port": "8080",
+                        "evaluation_configuration": {
+                            "inference_max_threads": 1,
+                            "inference_rps": None,
+                            "inference_timeout": None,
+                            "inference_backoff_factor": None,
+                            "inference_delay": 1,
+                            "inference_retries": None,
+                        },
                     },
                 },
                 {
@@ -528,6 +536,14 @@ def test_list_containers(self, mock_get_container_config):
                             "--trust-remote-code",
                         ],
                         "server_port": "8080",
+                        "evaluation_configuration": {
+                            "inference_max_threads": None,
+                            "inference_rps": None,
+                            "inference_timeout": None,
+                            "inference_backoff_factor": None,
+                            "inference_delay": None,
+                            "inference_retries": None,
+                        },
                     },
                 },
                 {
@@ -553,6 +569,14 @@ def test_list_containers(self, mock_get_container_config):
                             "--seed",
                         ],
                         "server_port": "8080",
+                        "evaluation_configuration": {
+                            "inference_max_threads": None,
+                            "inference_rps": None,
+                            "inference_timeout": None,
+                            "inference_backoff_factor": None,
+                            "inference_delay": None,
+                            "inference_retries": None,
+                        },
                     },
                 },
             ],