add support to manage concurrent eval requests

Author: VipulMascarenhas

ads/aqua/constants.py

@@ -15,6 +15,7 @@
 EVALUATION_REPORT_JSON = "report.json"
 EVALUATION_REPORT_MD = "report.md"
 EVALUATION_REPORT = "report.html"
+EVALUATION_INFERENCE_DEFAULT_THREADS = 10
 UNKNOWN_JSON_STR = "{}"
 FINE_TUNING_RUNTIME_CONTAINER = "iad.ocir.io/ociodscdev/aqua_ft_cuda121:0.3.17.20"
 DEFAULT_FT_BLOCK_STORAGE_SIZE = 750

ads/aqua/evaluation/entities.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
@@ -13,6 +12,7 @@
 from dataclasses import dataclass, field
 from typing import List, Optional, Union
 
+from ads.aqua.constants import EVALUATION_INFERENCE_DEFAULT_THREADS
 from ads.aqua.data import AquaResourceIdentifier
 from ads.common.serializer import DataClassSerializable
 
@@ -65,6 +65,8 @@ class CreateAquaEvaluationDetails(DataClassSerializable):
         The metrics for the evaluation.
     force_overwrite: (bool, optional). Defaults to `False`.
         Whether to force overwrite the existing file in object storage.
+    inference_max_threads: (int, optional). Defaults to None
+        Set the value of concurrent requests to be made to the inference endpoint during evaluation.
     """
 
     evaluation_source_id: str
@@ -86,6 +88,7 @@ class CreateAquaEvaluationDetails(DataClassSerializable):
     log_id: Optional[str] = None
     metrics: Optional[List] = None
     force_overwrite: Optional[bool] = False
+    inference_max_threads: Optional[int] = EVALUATION_INFERENCE_DEFAULT_THREADS
 
 
 @dataclass(repr=False)
@@ -142,6 +145,7 @@ class AquaEvaluationCommands(DataClassSerializable):
     metrics: list
     output_dir: str
     params: dict
+    inference_max_threads: int
 
 
 @dataclass(repr=False)

ads/aqua/evaluation/evaluation.py

@@ -390,6 +390,7 @@ def create(
                 report_path=create_aqua_evaluation_details.report_path,
                 model_parameters=create_aqua_evaluation_details.model_parameters,
                 metrics=create_aqua_evaluation_details.metrics,
+                inference_max_threads=create_aqua_evaluation_details.inference_max_threads,
             )
         ).create(**kwargs)  ## TODO: decide what parameters will be needed
         logger.debug(
@@ -511,6 +512,7 @@ def _build_evaluation_runtime(
         report_path: str,
         model_parameters: dict,
         metrics: List = None,
+        inference_max_threads: int = None,
     ) -> Runtime:
         """Builds evaluation runtime for Job."""
         # TODO the image name needs to be extracted from the mapping index.json file.
@@ -528,6 +530,7 @@ def _build_evaluation_runtime(
                                 report_path=report_path,
                                 model_parameters=model_parameters,
                                 metrics=metrics,
+                                inference_max_threads=inference_max_threads,
                             )
                         )
                     ),
@@ -587,6 +590,7 @@ def _build_launch_cmd(
         report_path: str,
         model_parameters: dict,
         metrics: List = None,
+        inference_max_threads: int = None,
     ):
         return AquaEvaluationCommands(
             evaluation_id=evaluation_id,
@@ -603,6 +607,7 @@ def _build_launch_cmd(
             metrics=metrics,
             output_dir=report_path,
             params=model_parameters,
+            inference_max_threads=inference_max_threads,
         )
 
     @telemetry(entry_point="plugin=evaluation&action=get", name="aqua")

ads/aqua/modeldeployment/deployment.py

@@ -167,6 +167,7 @@ def create(
             Tags.AQUA_SERVICE_MODEL_TAG,
             Tags.AQUA_FINE_TUNED_MODEL_TAG,
             Tags.AQUA_TAG,
+            Tags.MODEL_FORMAT,
         ]:
             if tag in aqua_model.freeform_tags:
                 tags[tag] = aqua_model.freeform_tags[tag]