[ODSC-59697] Llama cpp deployment support (#902)

Author: VipulMascarenhas

ads/aqua/extension/common_ws_msg_handler.py

@@ -1,18 +1,25 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
 import json
+from importlib import metadata
 from typing import List, Union
 
 from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID, fetch_service_compartment
 from ads.aqua.common.decorator import handle_exceptions
 from ads.aqua.common.errors import AquaResourceAccessError
 from ads.aqua.common.utils import known_realm
 from ads.aqua.extension.aqua_ws_msg_handler import AquaWSMsgHandler
-from ads.aqua.extension.models.ws_models import RequestResponseType, AdsVersionResponse, AdsVersionRequest, \
-    CompatibilityCheckResponse
-from importlib import metadata
+from ads.aqua.extension.models.ws_models import (
+    AdsVersionResponse,
+    CompatibilityCheckResponse,
+    RequestResponseType,
+)
 
 
 class AquaCommonWsMsgHandler(AquaWSMsgHandler):
-
     @staticmethod
     def get_message_types() -> List[RequestResponseType]:
         return [RequestResponseType.AdsVersion, RequestResponseType.CompatibilityCheck]
@@ -21,25 +28,30 @@ def __init__(self, message: Union[str, bytes]):
         super().__init__(message)
 
     @handle_exceptions
-    def process(self) -> AdsVersionResponse | CompatibilityCheckResponse:
+    def process(self) -> Union[AdsVersionResponse, CompatibilityCheckResponse]:
         request = json.loads(self.message)
-        if request.get('kind') == 'AdsVersion':
+        if request.get("kind") == "AdsVersion":
             version = metadata.version("oracle_ads")
             response = AdsVersionResponse(
                 message_id=request.get("message_id"),
                 kind=RequestResponseType.AdsVersion,
-                data=version)
+                data=version,
+            )
             return response
-        if request.get('kind') == 'CompatibilityCheck':
+        if request.get("kind") == "CompatibilityCheck":
             if ODSC_MODEL_COMPARTMENT_OCID or fetch_service_compartment():
-                return CompatibilityCheckResponse(message_id=request.get("message_id"),
-                                                  kind=RequestResponseType.CompatibilityCheck,
-                                                  data={'status': 'ok'})
+                return CompatibilityCheckResponse(
+                    message_id=request.get("message_id"),
+                    kind=RequestResponseType.CompatibilityCheck,
+                    data={"status": "ok"},
+                )
             elif known_realm():
-                return CompatibilityCheckResponse(message_id=request.get("message_id"),
-                                                  kind=RequestResponseType.CompatibilityCheck,
-                                                  data={'status': 'compatible'})
+                return CompatibilityCheckResponse(
+                    message_id=request.get("message_id"),
+                    kind=RequestResponseType.CompatibilityCheck,
+                    data={"status": "compatible"},
+                )
             else:
                 raise AquaResourceAccessError(
-                    f"The AI Quick actions extension is not compatible in the given region."
+                    "The AI Quick actions extension is not compatible in the given region."
                 )

ads/aqua/extension/deployment_handler.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
@@ -8,8 +7,8 @@
 from tornado.web import HTTPError
 
 from ads.aqua.common.decorator import handle_exceptions
-from ads.aqua.extension.errors import Errors
 from ads.aqua.extension.base_handler import AquaAPIhandler
+from ads.aqua.extension.errors import Errors
 from ads.aqua.modeldeployment import AquaDeploymentApp, MDInferenceResponse
 from ads.aqua.modeldeployment.entities import ModelParams
 from ads.config import COMPARTMENT_OCID, PROJECT_OCID
@@ -66,8 +65,8 @@ def post(self, *args, **kwargs):
         """
         try:
             input_data = self.get_json_body()
-        except Exception:
-            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT)
+        except Exception as ex:
+            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT) from ex
 
         if not input_data:
             raise HTTPError(400, Errors.NO_INPUT_DATA)
@@ -100,6 +99,8 @@ def post(self, *args, **kwargs):
         health_check_port = input_data.get("health_check_port")
         env_var = input_data.get("env_var")
         container_family = input_data.get("container_family")
+        ocpus = input_data.get("ocpus")
+        memory_in_gbs = input_data.get("memory_in_gbs")
 
         self.finish(
             AquaDeploymentApp().create(
@@ -119,6 +120,8 @@ def post(self, *args, **kwargs):
                 health_check_port=health_check_port,
                 env_var=env_var,
                 container_family=container_family,
+                ocpus=ocpus,
+                memory_in_gbs=memory_in_gbs,
             )
         )
 
@@ -153,9 +156,7 @@ def validate_predict_url(endpoint):
                 return False
             if not url.netloc:
                 return False
-            if not url.path.endswith("/predict"):
-                return False
-            return True
+            return url.path.endswith("/predict")
         except Exception:
             return False
 
@@ -170,8 +171,8 @@ def post(self, *args, **kwargs):
         """
         try:
             input_data = self.get_json_body()
-        except Exception:
-            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT)
+        except Exception as ex:
+            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT) from ex
 
         if not input_data:
             raise HTTPError(400, Errors.NO_INPUT_DATA)
@@ -192,10 +193,10 @@ def post(self, *args, **kwargs):
         )
         try:
             model_params_obj = ModelParams(**model_params)
-        except:
+        except Exception as ex:
             raise HTTPError(
                 400, Errors.INVALID_INPUT_DATA_FORMAT.format("model_params")
-            )
+            ) from ex
 
         return self.finish(
             MDInferenceResponse(prompt, model_params_obj).get_model_deployment_response(
@@ -236,8 +237,8 @@ def post(self, *args, **kwargs):
         """
         try:
             input_data = self.get_json_body()
-        except Exception:
-            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT)
+        except Exception as ex:
+            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT) from ex
 
         if not input_data:
             raise HTTPError(400, Errors.NO_INPUT_DATA)

ads/aqua/extension/deployment_ws_msg_handler.py

@@ -1,25 +1,35 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
 import json
 from typing import List, Union
 
 from ads.aqua.common.decorator import handle_exceptions
 from ads.aqua.extension.aqua_ws_msg_handler import AquaWSMsgHandler
-from ads.aqua.extension.models.ws_models import RequestResponseType, ListDeploymentResponse, ListDeploymentRequest, \
-    ModelDeploymentDetailsResponse
+from ads.aqua.extension.models.ws_models import (
+    ListDeploymentResponse,
+    ModelDeploymentDetailsResponse,
+    RequestResponseType,
+)
 from ads.aqua.modeldeployment import AquaDeploymentApp
 from ads.config import COMPARTMENT_OCID
 
 
 class AquaDeploymentWSMsgHandler(AquaWSMsgHandler):
-
     def __init__(self, message: Union[str, bytes]):
         super().__init__(message)
 
     @staticmethod
     def get_message_types() -> List[RequestResponseType]:
-        return [RequestResponseType.ListDeployments, RequestResponseType.DeploymentDetails]
+        return [
+            RequestResponseType.ListDeployments,
+            RequestResponseType.DeploymentDetails,
+        ]
 
     @handle_exceptions
-    def process(self) -> ListDeploymentResponse | ModelDeploymentDetailsResponse:
+    def process(self) -> Union[ListDeploymentResponse, ModelDeploymentDetailsResponse]:
         request = json.loads(self.message)
         if request.get("kind") == "ListDeployments":
             deployment_list = AquaDeploymentApp().list(
@@ -33,8 +43,12 @@ def process(self) -> ListDeploymentResponse | ModelDeploymentDetailsResponse:
             )
             return response
         elif request.get("kind") == "DeploymentDetails":
-            deployment_details = AquaDeploymentApp().get(request.get("model_deployment_id"))
-            response = ModelDeploymentDetailsResponse(message_id=request.get("message_id"),
-                                                      kind=RequestResponseType.DeploymentDetails,
-                                                      data=deployment_details)
+            deployment_details = AquaDeploymentApp().get(
+                request.get("model_deployment_id")
+            )
+            response = ModelDeploymentDetailsResponse(
+                message_id=request.get("message_id"),
+                kind=RequestResponseType.DeploymentDetails,
+                data=deployment_details,
+            )
             return response

ads/aqua/extension/evaluation_ws_msg_handler.py

@@ -29,7 +29,7 @@ def __init__(self, message: Union[str, bytes]):
         super().__init__(message)
 
     @handle_exceptions
-    def process(self) -> ListEvaluationsResponse | EvaluationDetailsResponse:
+    def process(self) -> Union[ListEvaluationsResponse, EvaluationDetailsResponse]:
         request = json.loads(self.message)
         if request["kind"] == "ListEvaluations":
             return self.list_evaluations(request)

ads/aqua/extension/models_ws_msg_handler.py

@@ -1,41 +1,49 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
 import json
 from typing import List, Union
 
 from ads.aqua.common.decorator import handle_exceptions
 from ads.aqua.extension.aqua_ws_msg_handler import AquaWSMsgHandler
-from ads.aqua.extension.models.ws_models import RequestResponseType, ListModelsResponse, ListModelsRequest, \
-    ModelDetailsResponse
+from ads.aqua.extension.models.ws_models import (
+    ListModelsResponse,
+    ModelDetailsResponse,
+    RequestResponseType,
+)
 from ads.aqua.model import AquaModelApp
 
 
 class AquaModelWSMsgHandler(AquaWSMsgHandler):
-
     def __init__(self, message: Union[str, bytes]):
         super().__init__(message)
 
     @staticmethod
     def get_message_types() -> List[RequestResponseType]:
-        return [RequestResponseType.ListModels,RequestResponseType.ModelDetails]
+        return [RequestResponseType.ListModels, RequestResponseType.ModelDetails]
 
     @handle_exceptions
-    def process(self) -> ListModelsResponse | ModelDetailsResponse:
+    def process(self) -> Union[ListModelsResponse, ModelDetailsResponse]:
         request = json.loads(self.message)
-        if request.get('kind') == 'ListModels':
+        if request.get("kind") == "ListModels":
             models_list = AquaModelApp().list(
                 compartment_id=request.get("compartment_id"),
                 project_id=request.get("project_id"),
-                model_type=request.get("model_type")
+                model_type=request.get("model_type"),
             )
             response = ListModelsResponse(
                 message_id=request.get("message_id"),
                 kind=RequestResponseType.ListModels,
                 data=models_list,
             )
             return response
-        elif request.get('kind') == 'ModelDetails':
-            model_id=request.get("model_id")
-            response=AquaModelApp().get(model_id)
-            return ModelDetailsResponse(message_id=request.get("message_id"),
-                                        kind=RequestResponseType.ModelDetails,
-                                        data=response)
-
+        elif request.get("kind") == "ModelDetails":
+            model_id = request.get("model_id")
+            response = AquaModelApp().get(model_id)
+            return ModelDetailsResponse(
+                message_id=request.get("message_id"),
+                kind=RequestResponseType.ModelDetails,
+                data=response,
+            )

ads/aqua/modeldeployment/deployment.py

@@ -3,7 +3,7 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import logging
-from typing import Dict, List, Union
+from typing import Dict, List, Optional, Union
 
 from ads.aqua.app import AquaApp, logger
 from ads.aqua.common.enums import (
@@ -102,6 +102,8 @@ def create(
         health_check_port: int = None,
         env_var: Dict = None,
         container_family: str = None,
+        memory_in_gbs: Optional[float] = None,
+        ocpus: Optional[float] = None,
     ) -> "AquaDeployment":
         """
         Creates a new Aqua deployment
@@ -142,6 +144,10 @@ def create(
             Environment variable for the deployment, by default None.
         container_family: str
             The image family of model deployment container runtime. Required for unverified Aqua models.
+        memory_in_gbs: float
+            The memory in gbs for the shape selected.
+        ocpus: float
+            The ocpu count for the shape selected.
         Returns
         -------
         AquaDeployment
@@ -325,6 +331,11 @@ def create(
                 log_id=predict_log_id,
             )
         )
+        if memory_in_gbs and ocpus and infrastructure.shape_name.endswith("Flex"):
+            infrastructure.with_shape_config_details(
+                ocpus=ocpus,
+                memory_in_gbs=memory_in_gbs,
+            )
         # configure model deployment runtime
         container_runtime = (
             ModelDeploymentContainerRuntime()
@@ -338,6 +349,7 @@ def create(
             .with_overwrite_existing_artifact(True)
             .with_remove_existing_artifact(True)
         )
+
         # configure model deployment and deploy model on container runtime
         deployment = (
             ModelDeployment()

tests/unitary/with_extras/aqua/test_data/deployment/aqua_create_gguf_deployment.yaml

@@ -0,0 +1,40 @@
+kind: deployment
+spec:
+  createdBy: ocid1.user.oc1..<OCID>
+  displayName: model-deployment-name
+  freeformTags:
+    OCI_AQUA: active
+    aqua_model_name: model-name
+  id: "ocid1.datasciencemodeldeployment.oc1.<region>.<MD_OCID>"
+  infrastructure:
+    kind: infrastructure
+    spec:
+      bandwidthMbps: 10
+      compartmentId: ocid1.compartment.oc1..<OCID>
+      deploymentType: SINGLE_MODEL
+      policyType: FIXED_SIZE
+      projectId: ocid1.datascienceproject.oc1.iad.<OCID>
+      replica: 1
+      shapeName: "VM.Standard.A1.Flex"
+      shapeConfigDetails:
+        memoryInGBs: 60.0
+        ocpus: 10
+    type: datascienceModelDeployment
+  lifecycleState: CREATING
+  modelDeploymentUrl: "https://modeldeployment.customer-oci.com/ocid1.datasciencemodeldeployment.oc1.<region>.<MD_OCID>"
+  runtime:
+    kind: runtime
+    spec:
+      env:
+        BASE_MODEL: service_models/model-name/artifact
+        BASE_MODEL_FILE: model-name.gguf
+        MODEL_DEPLOY_ENABLE_STREAMING: 'true'
+        MODEL_DEPLOY_PREDICT_ENDPOINT: /v1/completions
+        MODEL_DEPLOY_HEALTH_ENDPOINT: /v1/models
+      healthCheckPort: 8080
+      image: "dsmc://image-name:1.0.0.0"
+      modelUri: "ocid1.datasciencemodeldeployment.oc1.<region>.<MODEL_OCID>"
+      serverPort: 8080
+    type: container
+  timeCreated: 2024-01-01T00:00:00.000000+00:00
+type: modelDeployment