Merge branch 'main' of https://github.com/oracle/accelerated-data-science into feature/model_group

Author: lu-ohai

.github/workflows/run-forecast-explainer-tests.yml

@@ -54,6 +54,6 @@ jobs:
           $CONDA/bin/conda init
           source /home/runner/.bashrc
           pip install -r test-requirements-operators.txt
-          pip install "oracle-automlx[forecasting]>=25.1.1"
+          pip install "oracle-automlx[forecasting]>=25.3.0"
           pip install pandas>=2.2.0
           python -m pytest -v -p no:warnings --durations=5 tests/operators/forecast/test_explainers.py

.github/workflows/run-forecast-unit-tests.yml

@@ -56,6 +56,6 @@ jobs:
           $CONDA/bin/conda init
           source /home/runner/.bashrc
           pip install -r test-requirements-operators.txt
-          pip install "oracle-automlx[forecasting]>=25.1.1"
+          pip install "oracle-automlx[forecasting]>=25.3.0"
           pip install pandas>=2.2.0
           python -m pytest -v -p no:warnings --durations=5 tests/operators/forecast --ignore=tests/operators/forecast/test_explainers.py

ads/aqua/cli.py

@@ -14,6 +14,7 @@
 from ads.aqua.finetuning import AquaFineTuningApp
 from ads.aqua.model import AquaModelApp
 from ads.aqua.modeldeployment import AquaDeploymentApp
+from ads.aqua.verify_policies import AquaVerifyPoliciesApp
 from ads.common.utils import LOG_LEVELS
 
 
@@ -29,6 +30,7 @@ class AquaCommand:
     fine_tuning = AquaFineTuningApp
     deployment = AquaDeploymentApp
     evaluation = AquaEvaluationApp
+    verify_policies = AquaVerifyPoliciesApp
 
     def __init__(
         self,

ads/aqua/config/container_config.py

@@ -191,21 +191,28 @@ def from_service_config(
                     additional_configurations.get("modelFormats")
                 )
 
-                # Parse environment variables from `additional_configurations`.
-                # Only keys present in the configuration will be added to the result.
-                config_keys = {
-                    "MODEL_DEPLOY_PREDICT_ENDPOINT": UNKNOWN,
-                    "MODEL_DEPLOY_HEALTH_ENDPOINT": UNKNOWN,
-                    "PORT": UNKNOWN,
-                    "HEALTH_CHECK_PORT": UNKNOWN,
-                    "VLLM_USE_V1": UNKNOWN,
-                }
-
-                env_vars = [
-                    {key: additional_configurations.get(key, default)}
-                    for key, default in config_keys.items()
-                    if key in additional_configurations
-                ]
+                # TODO: Remove the else condition once SMC env variable config is updated everywhere
+                if additional_configurations.get("env_vars", None):
+                    env_vars_dict = json.loads(
+                        additional_configurations.get("env_vars") or "{}"
+                    )
+                    env_vars = [
+                        {key: str(value)} for key, value in env_vars_dict.items()
+                    ]
+                else:
+                    config_keys = {
+                        "MODEL_DEPLOY_PREDICT_ENDPOINT": UNKNOWN,
+                        "MODEL_DEPLOY_HEALTH_ENDPOINT": UNKNOWN,
+                        "PORT": UNKNOWN,
+                        "HEALTH_CHECK_PORT": UNKNOWN,
+                        "VLLM_USE_V1": UNKNOWN,
+                    }
+
+                    env_vars = [
+                        {key: additional_configurations.get(key, default)}
+                        for key, default in config_keys.items()
+                        if key in additional_configurations
+                    ]
 
                 # Build container spec
                 container_item.spec = AquaContainerConfigSpec(

ads/aqua/extension/model_handler.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
+import json
 from typing import Optional
 from urllib.parse import urlparse
 
@@ -341,9 +341,13 @@ def get(self, model_id):
         ):
             try:
                 oci_data_science_model = OCIDataScienceModel.from_id(model_id)
+                chat_template = oci_data_science_model.get_custom_metadata_artifact("chat_template")
+                chat_template = chat_template.decode("utf-8")
+
+                return self.finish(json.dumps({"chat_template": chat_template}))
+
             except Exception as e:
-                raise HTTPError(404, f"Model not found for id: {model_id}. Details: {str(e)}")
-            return self.finish(oci_data_science_model.get_custom_metadata_artifact("chat_template"))
+                raise HTTPError(404, f"Failed to fetch chat template for model_id={model_id}. Details: {str(e)}")
 
         raise HTTPError(400, f"The request {self.request.path} is invalid.")
 

ads/aqua/modeldeployment/deployment.py

@@ -1128,7 +1128,6 @@ def _create_deployment(
         logger.info(
             f"Aqua model deployment {deployment_id} created for model {aqua_model_id}. Work request Id is {deployment.dsc_model_deployment.workflow_req_id}"
         )
-        status_list = []
 
         progress_thread = threading.Thread(
             target=self.get_deployment_status,
@@ -1228,14 +1227,17 @@ def list(self, **kwargs) -> List["AquaDeployment"]:
                 aqua_model_id = model_deployment.freeform_tags.get(
                     Tags.AQUA_MODEL_ID_TAG, UNKNOWN
                 )
-                if "datasciencemodelgroup" in aqua_model_id:
+                if (
+                    "datasciencemodelgroup" in aqua_model_id
+                    or model_deployment.model_deployment_configuration_details.deployment_type
+                    == "UNKNOWN_ENUM_VALUE"
+                ):
                     continue
                 results.append(
                     AquaDeployment.from_oci_model_deployment(
                         model_deployment, self.region
                     )
                 )
-
                 # log telemetry if MD is in active or failed state
                 deployment_id = model_deployment.id
                 state = model_deployment.lifecycle_state.upper()
@@ -1686,45 +1688,24 @@ def get_deployment_status(
                 poll_interval=DEFAULT_POLL_INTERVAL,
             )
         except Exception:
-            status = ""
-            logs = deployment.show_logs().sort_values(by="time", ascending=False)
-
-            if logs and len(logs) > 0:
-                status = logs.iloc[0]["message"]
-
-            status = re.sub(r"[^a-zA-Z0-9]", " ", status)
-
             if data_science_work_request._error_message:
                 error_str = ""
                 for error in data_science_work_request._error_message:
                     error_str = error_str + " " + error.message
 
                 error_str = re.sub(r"[^a-zA-Z0-9]", " ", error_str)
-                telemetry_kwargs = {
-                    "ocid": ocid,
-                    "model_name": model_name,
-                    "work_request_error": error_str,
-                    "status": status,
-                }
 
-                self.telemetry.record_event(
-                    category=f"aqua/{model_type}/deployment/status",
-                    action="FAILED",
-                    **telemetry_kwargs,
-                )
-            else:
                 telemetry_kwargs = {
                     "ocid": ocid,
                     "model_name": model_name,
-                    "status": status,
+                    "work_request_error": error_str,
                 }
 
                 self.telemetry.record_event(
                     category=f"aqua/{model_type}/deployment/status",
                     action="FAILED",
                     **telemetry_kwargs,
                 )
-
         else:
             telemetry_kwargs = {"ocid": ocid, "model_name": model_name}
             self.telemetry.record_event(

ads/aqua/verify_policies/__init__.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+from ads.aqua.verify_policies.verify import AquaVerifyPoliciesApp
+
+__all__ = ["AquaVerifyPoliciesApp"]

ads/aqua/verify_policies/constants.py

@@ -0,0 +1,13 @@
+OBS_MANAGE_TEST_FILE = "AQUA Policy Verification - OBJECT STORAGE"
+TEST_MODEL_NAME="AQUA Policy Verification -  Model"
+TEST_MD_NAME="AQUA Policy Verification - Model Deployment"
+TEST_JOB_NAME="AQUA Policy Verification - Job"
+TEST_JOB_RUN_NAME="AQUA Policy Verification - Job Run"
+TEST_MVS_NAME="AQUA Policy Verification - Model Version Set"
+TEST_VM_SHAPE="VM.Standard.E4.Flex"
+TEST_DEFAULT_JOB_SHAPE = "VM.Standard.E3.Flex"
+TEST_LIMIT_NAME = "ds-gpu-a10-count"
+DUMMY_TEST_BYTE = b"7IV6cktUGcHIhur4bXTv"
+POLICY_HELP_LINK = "https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/ai-quick-actions/policies/README.md"
+
+

ads/aqua/verify_policies/entities.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+from ads.common.extended_enum import ExtendedEnum
+from ads.common.serializer import DataClassSerializable
+
+
+class PolicyStatus(ExtendedEnum):
+    SUCCESS = "SUCCESS"
+    FAILURE = "FAILURE"
+    UNVERIFIED = "UNVERIFIED"
+
+
+@dataclass(repr=False)
+class OperationResultSuccess(DataClassSerializable):
+    operation: str
+    status: PolicyStatus = PolicyStatus.SUCCESS
+
+
+@dataclass(repr=False)
+class OperationResultFailure(DataClassSerializable):
+    operation: str
+    error: str
+    policy_hint: str
+    status: PolicyStatus = PolicyStatus.FAILURE
+
+
+@dataclass(repr=False)
+class CommonSettings(DataClassSerializable):
+    compartment_id: str
+    project_id: str

ads/aqua/verify_policies/messages.py

@@ -0,0 +1,101 @@
+from ads.aqua.verify_policies.utils import VerifyPoliciesUtils
+
+utils = VerifyPoliciesUtils()
+operation_messages = {
+    utils.list_compartments.__name__: {
+        "name": "List Compartments",
+        "error": "Unable to retrieve the list of compartments. Please verify that you have the required permissions to list compartments in your tenancy. ",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to inspect compartments in tenancy"
+    },
+    utils.list_models.__name__: {
+        "name": "List Models",
+        "error": "Failed to fetch available models. Ensure that the policies allow you to list models from the Model Catalog in the selected compartment.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-models in compartment <your-compartment-name>"
+    },
+    utils.list_project.__name__: {
+        "name": "List Projects",
+        "error": "Failed to list Data Science projects. Verify that you have the appropriate permission to access projects in the selected compartment.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-projects in compartment <your-compartment-name>"
+    },
+    utils.list_model_version_sets.__name__: {
+        "name": "List Model Version Sets",
+        "error": "Unable to load model version sets. Check your access rights to list model version sets in the selected compartment.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-modelversionsets in compartment <your-compartment-name>"
+    },
+    utils.list_jobs.__name__: {
+        "name": "List Jobs",
+        "error": "Job list could not be retrieved. Please confirm that you have the necessary permissions to view jobs in the compartment.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-jobs in compartment <your-compartment-name>"
+    },
+    utils.list_job_runs.__name__: {
+        "name": "List Job Runs",
+        "error": "Job Runs list could not be retrieved. Please confirm that you have the necessary permissions to view job runs in the compartme",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-job-runs in compartment <your-compartment-name>"
+    },
+    utils.list_buckets.__name__: {
+        "name": "List Object Storage Buckets",
+        "error": "Cannot fetch Object Storage buckets. Verify that you have access to list buckets within the specified compartment.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to read objectstorage-namespaces in compartment <your-compartment-name>\nAllow dynamic-group aqua-dynamic-group to read buckets in compartment <your-compartment-name>"
+    },
+    utils.manage_bucket.__name__: {
+        "name": "Object Storage Access",
+        "error": "Failed to access the Object Storage bucket. Verify that the bucket exists and you have write permissions.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage object-family in compartment <your-compartment-name> where any {target.bucket.name='<your-bucket-name>'}"
+    },
+    utils.list_log_groups.__name__: {
+        "name": "List Log Groups",
+        "error": "Log groups or logs could not be retrieved. Please confirm you have logging read access for the selected compartment.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to use logging-family in compartment <your-compartment-name>"
+    },
+    utils.get_resource_availability.__name__: {
+        "name": "Verify Shape Limits",
+        "error": "Failed to retrieve shape limits for your compartment. Make sure the required policies are in place to read shape and quota data.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to read resource-availability in compartment <your-compartment-name>"
+    },
+    utils.register_model.__name__: {
+        "name": "Register Model",
+        "error": "Model registration failed. Ensure you have the correct permissions to write to the Model Catalog and access Object Storage.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-models in compartment <your-compartment-name>"
+    },
+    utils.aqua_model.delete_model.__name__: {
+        "name": "Delete Model",
+        "error": "Could not delete model. Please confirm you have delete permissions for Model Catalog resources in the compartment.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-models in compartment <your-compartment-name>"
+    },
+    utils.create_job.__name__: {
+        "name": "Create Job",
+        "error": "Job could not be created. Please check if you have permissions to create Data Science jobs.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-jobs in compartment <your-compartment-name>"
+    },
+    utils.create_job_run.__name__: {
+        "name": "Create Job Run",
+        "error": "Job Run could not be created. Confirm that you are allowed to run jobs in the selected compartment.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-job-runs in compartment <your-compartment-name>"
+    },
+    "delete_job": {
+        "name": "Delete Job",
+        "error": "Job could not be deleted. Please check if you have permissions to delete Data Science jobs.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-jobs in compartment <your-compartment-name>"
+    },
+    utils.aqua_model.create_model_version_set.__name__: {
+        "name": "Create Model Version Set",
+        "error": "Unable to create a model version set for storing evaluation results. Ensure that required Model Catalog permissions are set.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-modelversionsets in compartment <your-compartment-name>"
+    },
+    utils.aqua_model.ds_client.delete_model_version_set.__name__: {
+        "name": "Delete Model Version Set",
+        "error": "Unable to delete a model version. Ensure that required Model Catalog permissions are set.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-modelversionsets in compartment <your-compartment-name>"
+    },
+    utils.create_model_deployment.__name__: {
+        "name": "Create Model Deployment",
+        "error": "Model deployment could not be created. Confirm you have correct permissions to deploy models to the Model Deployment service.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-model-deployments in compartment <your-compartment-name>"
+    },
+    utils.aqua_model.ds_client.delete_model_deployment.__name__: {
+        "name": "Delete Model Deployment",
+        "error": "Unable to delete the model deployment. Please check if you have appropriate permissions to manage deployments.",
+        "policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-model-deployments in compartment <your-compartment-name>"
+    }
+
+}