Merge branch 'ODSC-61884/global_evaluation_config' of https://github.com/oracle/accelerated-data-science into ODSC-61883/evaluation_supported_metrics

Author: mrDzurb

ads/aqua/config/config.py

@@ -4,16 +4,21 @@
 
 
 from datetime import datetime, timedelta
+from typing import Optional
 
 from cachetools import TTLCache, cached
 
-from ads.aqua.common.utils import service_config_path
+from ads.aqua.common.entities import ContainerSpec
+from ads.aqua.common.utils import get_container_config
 from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig
-from ads.aqua.constants import EVALUATION_SERVICE_CONFIG
+
+DEFAULT_EVALUATION_CONTAINER = "odsc-llm-evaluate"
 
 
 @cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=5), timer=datetime.now))
-def evaluation_service_config() -> EvaluationServiceConfig:
+def evaluation_service_config(
+    container: Optional[str] = DEFAULT_EVALUATION_CONTAINER,
+) -> EvaluationServiceConfig:
     """
     Retrieves the common evaluation configuration.
 
@@ -22,8 +27,10 @@ def evaluation_service_config() -> EvaluationServiceConfig:
     EvaluationServiceConfig: The evaluation common config.
     """
 
-    return EvaluationServiceConfig.from_json(
-        uri=f"{service_config_path()}/{EVALUATION_SERVICE_CONFIG}"
+    return EvaluationServiceConfig(
+        **get_container_config()
+        .get(ContainerSpec.CONTAINER_SPEC, {})
+        .get(container, {})
     )
 
 

ads/aqua/config/evaluation/evaluation_service_config.py

@@ -4,7 +4,7 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 from copy import deepcopy
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
 
 from pydantic import Field
 
@@ -19,17 +19,6 @@
 INFERENCE_DELAY = 0
 
 
-class ModelParamItem(Serializable):
-    """Represents min, max, and default values for a model parameter."""
-
-    min: Optional[Union[int, float]] = None
-    max: Optional[Union[int, float]] = None
-    default: Optional[Union[int, float]] = None
-
-    class Config:
-        extra = "ignore"
-
-
 class ModelParamsOverrides(Serializable):
     """Defines overrides for model parameters, including exclusions and additional inclusions."""
 
@@ -51,28 +40,11 @@ class Config:
         extra = "ignore"
 
 
-class ModelDefaultParams(Serializable):
-    """Defines default parameters for a model within a specific framework."""
-
-    model: Optional[str] = None
-    max_tokens: Optional[ModelParamItem] = Field(default_factory=ModelParamItem)
-    temperature: Optional[ModelParamItem] = Field(default_factory=ModelParamItem)
-    top_p: Optional[ModelParamItem] = Field(default_factory=ModelParamItem)
-    top_k: Optional[ModelParamItem] = Field(default_factory=ModelParamItem)
-    presence_penalty: Optional[ModelParamItem] = Field(default_factory=ModelParamItem)
-    frequency_penalty: Optional[ModelParamItem] = Field(default_factory=ModelParamItem)
-    stop: List[str] = Field(default_factory=list)
+class ModelParamsContainer(Serializable):
+    """Represents a container's model configuration, including tasks, defaults, and versions."""
 
-    class Config:
-        extra = "allow"
-
-
-class ModelFramework(Serializable):
-    """Represents a framework's model configuration, including tasks, defaults, and versions."""
-
-    framework: Optional[str] = None
-    task: Optional[List[str]] = Field(default_factory=list)
-    default: Optional[ModelDefaultParams] = Field(default_factory=ModelDefaultParams)
+    name: Optional[str] = None
+    default: Optional[Dict[str, Any]] = Field(default_factory=dict)
     versions: Optional[Dict[str, ModelParamsVersion]] = Field(default_factory=dict)
 
     class Config:
@@ -93,10 +65,10 @@ class Config:
         extra = "allow"
 
 
-class InferenceFramework(Serializable):
-    """Represents the inference parameters specific to a framework."""
+class InferenceContainer(Serializable):
+    """Represents the inference parameters specific to a container."""
 
-    framework: Optional[str] = None
+    name: Optional[str] = None
     params: Optional[Dict[str, Any]] = Field(default_factory=dict)
 
     class Config:
@@ -113,70 +85,66 @@ class Config:
 
 
 class InferenceParamsConfig(Serializable):
-    """Combines default inference parameters with framework-specific configurations."""
+    """Combines default inference parameters with container-specific configurations."""
 
     default: Optional[InferenceParams] = Field(default_factory=InferenceParams)
-    frameworks: Optional[List[InferenceFramework]] = Field(default_factory=list)
+    containers: Optional[List[InferenceContainer]] = Field(default_factory=list)
 
-    def get_merged_params(self, framework_name: str) -> InferenceParams:
+    def get_merged_params(self, container_name: str) -> InferenceParams:
         """
-        Merges default inference params with those specific to the given framework.
+        Merges default inference params with those specific to the given container.
 
         Parameters
         ----------
-        framework_name (str): The name of the framework.
+        container_name (str): The name of the container.
 
         Returns
         -------
         InferenceParams: The merged inference parameters.
         """
         merged_params = self.default.to_dict()
-        for framework in self.frameworks:
-            if framework.framework.lower() == framework_name.lower():
-                merged_params.update(framework.params or {})
+        for containers in self.containers:
+            if containers.name.lower() == container_name.lower():
+                merged_params.update(containers.params or {})
                 break
         return InferenceParams(**merged_params)
 
     class Config:
         extra = "ignore"
 
 
-class ModelParamsConfig(Serializable):
-    """Encapsulates the model parameters for different frameworks."""
+class InferenceModelParamsConfig(Serializable):
+    """Encapsulates the model parameters for different containers."""
 
     default: Optional[Dict[str, Any]] = Field(default_factory=dict)
-    frameworks: Optional[List[ModelFramework]] = Field(default_factory=list)
+    containers: Optional[List[ModelParamsContainer]] = Field(default_factory=list)
 
-    def get_model_params(
+    def get_merged_model_params(
         self,
-        framework_name: str,
+        container_name: str,
         version: Optional[str] = None,
-        task: Optional[str] = None,
     ) -> Dict[str, Any]:
         """
-        Gets the model parameters for a given framework, version, and tasks,
+        Gets the model parameters for a given container, version,
         merged with the defaults.
 
         Parameters
         ----------
-        framework_name (str): The name of the framework.
-        version (Optional[str]): The specific version of the framework.
-        task (Optional[str]): The specific task.
+        container_name (str): The name of the container.
+        version (Optional[str]): The specific version of the container.
 
         Returns
         -------
         Dict[str, Any]: The merged model parameters.
         """
         params = deepcopy(self.default)
 
-        for framework in self.frameworks:
-            if framework.framework.lower() == framework_name.lower() and (
-                not task or task.lower() in framework.task
-            ):
-                params.update(framework.default.to_dict())
+        for container in self.containers:
+            if container.name.lower() == container_name.lower():
+                params.update(container.default)
 
-                if version and version in framework.versions:
-                    version_overrides = framework.versions[version].overrides
+                if version and version in container.versions:
+                    version_overrides = container.versions[version].overrides
                     if version_overrides:
                         if version_overrides.include:
                             params.update(version_overrides.include)
@@ -228,59 +196,17 @@ class Config:
         extra = "ignore"
 
 
-class EvaluationServiceConfig(Serializable):
-    """
-    Root configuration class for evaluation setup including model,
-    inference, and shape configurations.
-    """
+class ModelParamsConfig(Serializable):
+    """Encapsulates the default model parameters."""
 
-    version: Optional[str] = "1.0"
-    kind: Optional[str] = "evaluation"
-    report_params: Optional[ReportParams] = Field(default_factory=ReportParams)
-    inference_params: Optional[InferenceParamsConfig] = Field(
-        default_factory=InferenceParamsConfig
-    )
+    default: Optional[Dict[str, Any]] = Field(default_factory=dict)
+
+
+class UIConfig(Serializable):
     model_params: Optional[ModelParamsConfig] = Field(default_factory=ModelParamsConfig)
     shapes: List[ShapeConfig] = Field(default_factory=list)
     metrics: List[MetricConfig] = Field(default_factory=list)
 
-    def get_merged_inference_params(self, framework_name: str) -> InferenceParams:
-        """
-        Merges default inference params with those specific to the given framework.
-
-        Params
-        ------
-        framework_name (str): The name of the framework.
-
-        Returns
-        -------
-        InferenceParams: The merged inference parameters.
-        """
-        return self.inference_params.get_merged_params(framework_name=framework_name)
-
-    def get_merged_model_params(
-        self,
-        framework_name: str,
-        version: Optional[str] = None,
-        task: Optional[str] = None,
-    ) -> Dict[str, Any]:
-        """
-        Gets the model parameters for a given framework, version, and task, merged with the defaults.
-
-        Parameters
-        ----------
-        framework_name (str): The name of the framework.
-        version (Optional[str]): The specific version of the framework.
-        task (Optional[str]): The task.
-
-        Returns
-        -------
-        Dict[str, Any]: The merged model parameters.
-        """
-        return self.model_params.get_model_params(
-            framework_name=framework_name, version=version, task=task
-        )
-
     def search_shapes(
         self,
         evaluation_container: Optional[str] = None,
@@ -315,3 +241,59 @@ def search_shapes(
 
     class Config:
         extra = "ignore"
+
+
+class EvaluationServiceConfig(Serializable):
+    """
+    Root configuration class for evaluation setup including model,
+    inference, and shape configurations.
+    """
+
+    version: Optional[str] = "1.0"
+    kind: Optional[str] = "evaluation"
+    report_params: Optional[ReportParams] = Field(default_factory=ReportParams)
+    inference_params: Optional[InferenceParamsConfig] = Field(
+        default_factory=InferenceParamsConfig
+    )
+    inference_model_params: Optional[InferenceModelParamsConfig] = Field(
+        default_factory=InferenceModelParamsConfig
+    )
+    ui_config: Optional[UIConfig] = Field(default_factory=UIConfig)
+
+    def get_merged_inference_params(self, container_name: str) -> InferenceParams:
+        """
+        Merges default inference params with those specific to the given container.
+
+        Params
+        ------
+        container_name (str): The name of the container.
+
+        Returns
+        -------
+        InferenceParams: The merged inference parameters.
+        """
+        return self.inference_params.get_merged_params(container_name=container_name)
+
+    def get_merged_inference_model_params(
+        self,
+        container_name: str,
+        version: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """
+        Gets the model parameters for a given container, version, and task, merged with the defaults.
+
+        Parameters
+        ----------
+        container_name (str): The name of the container.
+        version (Optional[str]): The specific version of the container.
+
+        Returns
+        -------
+        Dict[str, Any]: The merged model parameters.
+        """
+        return self.inference_model_params.get_merged_model_params(
+            container_name=container_name, version=version
+        )
+
+    class Config:
+        extra = "ignore"

ads/aqua/constants.py

@@ -15,7 +15,6 @@
 EVALUATION_REPORT_JSON = "report.json"
 EVALUATION_REPORT_MD = "report.md"
 EVALUATION_REPORT = "report.html"
-EVALUATION_SERVICE_CONFIG = "evaluation.json"
 UNKNOWN_JSON_STR = "{}"
 FINE_TUNING_RUNTIME_CONTAINER = "iad.ocir.io/ociodscdev/aqua_ft_cuda121:0.3.17.20"
 DEFAULT_FT_BLOCK_STORAGE_SIZE = 750

tests/unitary/with_extras/aqua/test_config.py

@@ -2,28 +2,38 @@
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
+import json
+import os
+from unittest.mock import patch
 
-from unittest.mock import MagicMock, patch
-
-from ads.aqua.common.utils import service_config_path
-from ads.aqua.config.config import evaluation_config
-from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig
-from ads.aqua.constants import EVALUATION_SERVICE_CONFIG
+from ads.aqua.common.entities import ContainerSpec
+from ads.aqua.config.config import evaluation_service_config
 
 
 class TestConfig:
     """Unit tests for AQUA common configurations."""
 
-    @patch.object(EvaluationServiceConfig, "from_json")
-    def test_evaluation_service_config(self, mock_from_json):
-        """Ensures that the common evaluation configuration can be successfully retrieved."""
-
-        expected_result = MagicMock()
-        mock_from_json.return_value = expected_result
+    def setup_class(cls):
+        cls.curr_dir = os.path.dirname(os.path.abspath(__file__))
+        cls.artifact_dir = os.path.join(cls.curr_dir, "test_data", "config")
 
-        test_result = evaluation_config()
+    @patch("ads.aqua.config.config.get_container_config")
+    def test_evaluation_service_config(self, mock_get_container_config):
+        """Ensures that the common evaluation configuration can be successfully retrieved."""
 
-        mock_from_json.assert_called_with(
-            uri=f"{service_config_path()}/{EVALUATION_SERVICE_CONFIG}"
+        with open(
+            os.path.join(
+                self.artifact_dir, "evaluation_config_with_default_params.json"
+            )
+        ) as file:
+            expected_result = {
+                ContainerSpec.CONTAINER_SPEC: {"test_container": json.load(file)}
+            }
+
+        mock_get_container_config.return_value = expected_result
+
+        test_result = evaluation_service_config(container="test_container")
+        assert (
+            test_result.to_dict()
+            == expected_result[ContainerSpec.CONTAINER_SPEC]["test_container"]
         )
-        assert test_result == expected_result