From 356cd3a9e34ecfd05f4fbb04713ee5777f5ca8b5 Mon Sep 17 00:00:00 2001 From: Vipul Date: Wed, 16 Oct 2024 14:11:28 -0700 Subject: [PATCH 1/8] add minor aqua features --- ads/aqua/config/config.py | 1 + .../config/deployment_config_defaults.json | 1 + ads/aqua/config/resource_limit_names.json | 1 + ads/aqua/modeldeployment/deployment.py | 101 +++++++++--------- 4 files changed, 56 insertions(+), 48 deletions(-) diff --git a/ads/aqua/config/config.py b/ads/aqua/config/config.py index 1cabc203c..1ccf2c703 100644 --- a/ads/aqua/config/config.py +++ b/ads/aqua/config/config.py @@ -40,6 +40,7 @@ def get_finetuning_config_defaults(): "VM.GPU.A10.2": {"batch_size": 1, "replica": "1-10"}, "BM.GPU.A10.4": {"batch_size": 1, "replica": 1}, "BM.GPU4.8": {"batch_size": 4, "replica": 1}, + "BM.GPU.L40S-NC.4": {"batch_size": 4, "replica": 1}, "BM.GPU.A100-v2.8": {"batch_size": 6, "replica": 1}, "BM.GPU.H100.8": {"batch_size": 6, "replica": 1}, } diff --git a/ads/aqua/config/deployment_config_defaults.json b/ads/aqua/config/deployment_config_defaults.json index 21572ee99..9caa8ef11 100644 --- a/ads/aqua/config/deployment_config_defaults.json +++ b/ads/aqua/config/deployment_config_defaults.json @@ -30,6 +30,7 @@ "VM.GPU.A10.2", "BM.GPU.A10.4", "BM.GPU4.8", + "BM.GPU.L40S-NC.4", "BM.GPU.A100-v2.8", "BM.GPU.H100.8", "VM.Standard.A1.Flex" diff --git a/ads/aqua/config/resource_limit_names.json b/ads/aqua/config/resource_limit_names.json index d3e23370e..3aabcfaee 100644 --- a/ads/aqua/config/resource_limit_names.json +++ b/ads/aqua/config/resource_limit_names.json @@ -2,6 +2,7 @@ "BM.GPU.A10.4": "ds-gpu-a10-count", "BM.GPU.A100-v2.8": "ds-gpu-a100-v2-count", "BM.GPU.H100.8": "ds-gpu-h100-count", + "BM.GPU.L40S-NC.4": "ds-gpu-l40s-nc-count", "BM.GPU4.8": "ds-gpu4-count", "VM.GPU.A10.1": "ds-gpu-a10-count", "VM.GPU.A10.2": "ds-gpu-a10-count" diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index 654e00dc8..8ce6ab320 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -46,7 +46,6 @@ from ads.config import ( AQUA_CONFIG_FOLDER, AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME, - AQUA_DEPLOYMENT_CONTAINER_OVERRIDE_FLAG_METADATA_NAME, AQUA_MODEL_DEPLOYMENT_CONFIG, AQUA_MODEL_DEPLOYMENT_CONFIG_DEFAULTS, COMPARTMENT_OCID, @@ -87,26 +86,27 @@ class AquaDeploymentApp(AquaApp): @telemetry(entry_point="plugin=deployment&action=create", name="aqua") def create( - self, - model_id: str, - instance_shape: str, - display_name: str, - instance_count: int = None, - log_group_id: str = None, - access_log_id: str = None, - predict_log_id: str = None, - compartment_id: str = None, - project_id: str = None, - description: str = None, - bandwidth_mbps: int = None, - web_concurrency: int = None, - server_port: int = None, - health_check_port: int = None, - env_var: Dict = None, - container_family: str = None, - memory_in_gbs: Optional[float] = None, - ocpus: Optional[float] = None, - model_file: Optional[str] = None, + self, + model_id: str, + instance_shape: str, + display_name: str, + instance_count: int = None, + log_group_id: str = None, + access_log_id: str = None, + predict_log_id: str = None, + compartment_id: str = None, + project_id: str = None, + description: str = None, + bandwidth_mbps: int = None, + web_concurrency: int = None, + server_port: int = None, + health_check_port: int = None, + env_var: Dict = None, + container_family: str = None, + memory_in_gbs: Optional[float] = None, + ocpus: Optional[float] = None, + model_file: Optional[str] = None, + cmd_var: List[str] = None, ) -> "AquaDeployment": """ Creates a new Aqua deployment @@ -153,6 +153,8 @@ def create( The ocpu count for the shape selected. model_file: str The file used for model deployment. + cmd_var: List[str] + The cmd of model deployment container runtime. Returns ------- AquaDeployment @@ -231,8 +233,7 @@ def create( env_var.update({"FT_MODEL": f"{fine_tune_output_path}"}) container_type_key = self._get_container_type_key( - model=aqua_model, - container_family=container_family + model=aqua_model, container_family=container_family ) # fetch image name from config @@ -248,7 +249,11 @@ def create( model_format = model_formats_str.split(",") # Figure out a better way to handle this in future release - if ModelFormat.GGUF.value in model_format and container_type_key.lower() == InferenceContainerTypeFamily.AQUA_LLAMA_CPP_CONTAINER_FAMILY: + if ( + ModelFormat.GGUF.value in model_format + and container_type_key.lower() + == InferenceContainerTypeFamily.AQUA_LLAMA_CPP_CONTAINER_FAMILY + ): if model_file is not None: logger.info( f"Overriding {model_file} as model_file for model {aqua_model.id}." @@ -299,8 +304,8 @@ def create( if user_params: # todo: remove this check in the future version, logic to be moved to container_index if ( - container_type_key.lower() - == InferenceContainerTypeFamily.AQUA_LLAMA_CPP_CONTAINER_FAMILY + container_type_key.lower() + == InferenceContainerTypeFamily.AQUA_LLAMA_CPP_CONTAINER_FAMILY ): # AQUA_LLAMA_CPP_CONTAINER_FAMILY container uses uvicorn that required model/server params # to be set as env vars @@ -369,6 +374,8 @@ def create( .with_overwrite_existing_artifact(True) .with_remove_existing_artifact(True) ) + if cmd_var: + container_runtime.with_cmd(cmd_var) # configure model deployment and deploy model on container runtime deployment = ( @@ -422,9 +429,8 @@ def _get_container_type_key(model: DataScienceModel, container_family: str) -> s f"for model {model.id}. For unverified Aqua models, {AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME} should be" f"set and value can be one of {', '.join(InferenceContainerTypeFamily.values())}." ) from err - + return container_type_key - @telemetry(entry_point="plugin=deployment&action=list", name="aqua") def list(self, **kwargs) -> List["AquaDeployment"]: @@ -453,8 +459,8 @@ def list(self, **kwargs) -> List["AquaDeployment"]: for model_deployment in model_deployments: oci_aqua = ( ( - Tags.AQUA_TAG in model_deployment.freeform_tags - or Tags.AQUA_TAG.lower() in model_deployment.freeform_tags + Tags.AQUA_TAG in model_deployment.freeform_tags + or Tags.AQUA_TAG.lower() in model_deployment.freeform_tags ) if model_deployment.freeform_tags else False @@ -508,8 +514,8 @@ def get(self, model_deployment_id: str, **kwargs) -> "AquaDeploymentDetail": oci_aqua = ( ( - Tags.AQUA_TAG in model_deployment.freeform_tags - or Tags.AQUA_TAG.lower() in model_deployment.freeform_tags + Tags.AQUA_TAG in model_deployment.freeform_tags + or Tags.AQUA_TAG.lower() in model_deployment.freeform_tags ) if model_deployment.freeform_tags else False @@ -526,8 +532,8 @@ def get(self, model_deployment_id: str, **kwargs) -> "AquaDeploymentDetail": log_group_name = "" logs = ( - model_deployment.category_log_details.access - or model_deployment.category_log_details.predict + model_deployment.category_log_details.access + or model_deployment.category_log_details.predict ) if logs: log_id = logs.log_id @@ -582,9 +588,9 @@ def get_deployment_config(self, model_id: str) -> Dict: return config def get_deployment_default_params( - self, - model_id: str, - instance_shape: str, + self, + model_id: str, + instance_shape: str, ) -> List[str]: """Gets the default params set in the deployment configs for the given model and instance shape. @@ -616,8 +622,8 @@ def get_deployment_default_params( ) if ( - container_type_key - and container_type_key in InferenceContainerTypeFamily.values() + container_type_key + and container_type_key in InferenceContainerTypeFamily.values() ): deployment_config = self.get_deployment_config(model_id) config_params = ( @@ -640,10 +646,10 @@ def get_deployment_default_params( return default_params def validate_deployment_params( - self, - model_id: str, - params: List[str] = None, - container_family: str = None, + self, + model_id: str, + params: List[str] = None, + container_family: str = None, ) -> Dict: """Validate if the deployment parameters passed by the user can be overridden. Parameter values are not validated, only param keys are validated. @@ -666,8 +672,7 @@ def validate_deployment_params( if params: model = DataScienceModel.from_id(model_id) container_type_key = self._get_container_type_key( - model=model, - container_family=container_family + model=model, container_family=container_family ) container_config = get_container_config() @@ -689,9 +694,9 @@ def validate_deployment_params( @staticmethod def _find_restricted_params( - default_params: Union[str, List[str]], - user_params: Union[str, List[str]], - container_family: str, + default_params: Union[str, List[str]], + user_params: Union[str, List[str]], + container_family: str, ) -> List[str]: """Returns a list of restricted params that user chooses to override when creating an Aqua deployment. The default parameters coming from the container index json file cannot be overridden. From 458d0cda860dcb6884ff227dd8c4fda658b52ec4 Mon Sep 17 00:00:00 2001 From: Vipul Date: Wed, 16 Oct 2024 14:13:47 -0700 Subject: [PATCH 2/8] updated release notes and version --- docs/source/release_notes.rst | 6 ++++++ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index c600dee05..5d61acc16 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -2,6 +2,12 @@ Release Notes ============= +2.12.2 +------- +Release date: October 16, 2024 + +* Introduced enhancements for AI Quick Actions. + 2.12.1 ------- Release date: October 10, 2024 diff --git a/pyproject.toml b/pyproject.toml index 1b8f15ade..d1bd85131 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ build-backend = "flit_core.buildapi" # Required name = "oracle_ads" # the install (PyPI) name; name for local build in [tool.flit.module] section below -version = "2.12.1" +version = "2.12.2" # Optional description = "Oracle Accelerated Data Science SDK" From 22ae99cba82c3335866da5eef4bb9ad393d30cf1 Mon Sep 17 00:00:00 2001 From: Vipul Date: Thu, 17 Oct 2024 18:06:37 -0700 Subject: [PATCH 3/8] register and deployment changes for byoc --- ads/aqua/common/enums.py | 9 +++++ ads/aqua/common/utils.py | 33 ++++++++++++++++- ads/aqua/constants.py | 1 + ads/aqua/model/entities.py | 1 + ads/aqua/model/model.py | 49 +++++++++++++++++++++----- ads/aqua/modeldeployment/deployment.py | 45 +++++++++++++++++++---- ads/config.py | 5 +-- 7 files changed, 126 insertions(+), 17 deletions(-) diff --git a/ads/aqua/common/enums.py b/ads/aqua/common/enums.py index 9bdc0be4a..4a423788d 100644 --- a/ads/aqua/common/enums.py +++ b/ads/aqua/common/enums.py @@ -52,6 +52,7 @@ class InferenceContainerTypeFamily(str, metaclass=ExtendedEnumMeta): AQUA_VLLM_CONTAINER_FAMILY = "odsc-vllm-serving" AQUA_TGI_CONTAINER_FAMILY = "odsc-tgi-serving" AQUA_LLAMA_CPP_CONTAINER_FAMILY = "odsc-llama-cpp-serving" + AQUA_TEI_CONTAINER_FAMILY = "odsc-tei-serving" class InferenceContainerParamType(str, metaclass=ExtendedEnumMeta): @@ -80,3 +81,11 @@ class RqsAdditionalDetails(str, metaclass=ExtendedEnumMeta): MODEL_VERSION_SET_NAME = "modelVersionSetName" PROJECT_ID = "projectId" VERSION_LABEL = "versionLabel" + + +class TextEmbeddingInferenceContainerParams(str, metaclass=ExtendedEnumMeta): + """Contains a subset of params that are required for enabling model deployment in OCI Data Science. More options + are available at https://huggingface.co/docs/text-embeddings-inference/en/cli_arguments""" + + MODEL_ID = "model-id" + PORT = "port" diff --git a/ads/aqua/common/utils.py b/ads/aqua/common/utils.py index ede4ddb88..7aea22797 100644 --- a/ads/aqua/common/utils.py +++ b/ads/aqua/common/utils.py @@ -35,6 +35,7 @@ InferenceContainerParamType, InferenceContainerType, RqsAdditionalDetails, + TextEmbeddingInferenceContainerParams, ) from ads.aqua.common.errors import ( AquaFileNotFoundError, @@ -51,6 +52,7 @@ MODEL_BY_REFERENCE_OSS_PATH_KEY, SERVICE_MANAGED_CONTAINER_URI_SCHEME, SUPPORTED_FILE_FORMATS, + TEI_CONTAINER_DEFAULT_HOST, TGI_INFERENCE_RESTRICTED_PARAMS, UNKNOWN, UNKNOWN_JSON_STR, @@ -63,7 +65,12 @@ from ads.common.object_storage_details import ObjectStorageDetails from ads.common.oci_resource import SEARCH_TYPE, OCIResource from ads.common.utils import copy_file, get_console_link, upload_to_os -from ads.config import AQUA_SERVICE_MODELS_BUCKET, CONDA_BUCKET_NS, TENANCY_OCID +from ads.config import ( + AQUA_MODEL_DEPLOYMENT_FOLDER, + AQUA_SERVICE_MODELS_BUCKET, + CONDA_BUCKET_NS, + TENANCY_OCID, +) from ads.model import DataScienceModel, ModelVersionSet logger = logging.getLogger("ads.aqua") @@ -1079,3 +1086,27 @@ def list_hf_models(query: str) -> List[str]: return [model.id for model in models if model.disabled is None] except HfHubHTTPError as err: raise format_hf_custom_error_message(err) from err + + +def generate_tei_cmd_vars(os_path: str) -> List[str]: + """This utility functions generates CMD params for Text Embedding Inference container. Only the + essential parameters for OCI model deployment are added, defaults are used for the rest. + Parameters + ---------- + os_path: str + OCI bucket path where the model artifacts are uploaded - oci://bucket@namespace/prefix + + Returns + ------- + List of command line arguments + """ + + cmd_prefix = "--" + cmd_vars = [ + cmd_prefix + TextEmbeddingInferenceContainerParams.MODEL_ID, + f"{AQUA_MODEL_DEPLOYMENT_FOLDER}{ObjectStorageDetails.from_path(os_path.rstrip('/')).filepath}/", + cmd_prefix + TextEmbeddingInferenceContainerParams.PORT, + TEI_CONTAINER_DEFAULT_HOST, + ] + + return cmd_vars diff --git a/ads/aqua/constants.py b/ads/aqua/constants.py index 958b161bd..f63f88db9 100644 --- a/ads/aqua/constants.py +++ b/ads/aqua/constants.py @@ -79,3 +79,4 @@ "--port", "--host", } +TEI_CONTAINER_DEFAULT_HOST = "8080" diff --git a/ads/aqua/model/entities.py b/ads/aqua/model/entities.py index 31125644f..1b73382e3 100644 --- a/ads/aqua/model/entities.py +++ b/ads/aqua/model/entities.py @@ -287,6 +287,7 @@ class ImportModelDetails(CLIBuilderMixin): compartment_id: Optional[str] = None project_id: Optional[str] = None model_file: Optional[str] = None + inference_container_uri: Optional[str] = None def __post_init__(self): self._command = "model register" diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py index 4a1879d83..4a5d9c6ac 100644 --- a/ads/aqua/model/model.py +++ b/ads/aqua/model/model.py @@ -14,13 +14,14 @@ from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID, logger from ads.aqua.app import AquaApp -from ads.aqua.common.enums import Tags +from ads.aqua.common.enums import InferenceContainerTypeFamily, Tags from ads.aqua.common.errors import AquaRuntimeError, AquaValueError from ads.aqua.common.utils import ( LifecycleStatus, _build_resource_identifier, copy_model_config, create_word_icon, + generate_tei_cmd_vars, get_artifact_path, get_hf_model_info, list_os_files_with_extension, @@ -67,7 +68,9 @@ from ads.common.oci_resource import SEARCH_TYPE, OCIResource from ads.common.utils import get_console_link from ads.config import ( + AQUA_DEPLOYMENT_CONTAINER_CMD_VAR, AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME, + AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME, AQUA_EVALUATION_CONTAINER_METADATA_NAME, AQUA_FINETUNING_CONTAINER_METADATA_NAME, COMPARTMENT_OCID, @@ -629,6 +632,7 @@ def _create_model_catalog_entry( validation_result: ModelValidationResult, compartment_id: Optional[str], project_id: Optional[str], + inference_container_uri: Optional[str], ) -> DataScienceModel: """Create model by reference from the object storage path @@ -640,6 +644,7 @@ def _create_model_catalog_entry( verified_model (DataScienceModel): If set, then copies all the tags and custom metadata information from the service verified model compartment_id (Optional[str]): Compartment Id of the compartment where the model has to be created project_id (Optional[str]): Project id of the project where the model has to be created + inference_container_uri (Optional[str]): Inference container uri for BYOC Returns: DataScienceModel: Returns Datascience model instance. @@ -685,6 +690,36 @@ def _create_model_catalog_entry( raise AquaRuntimeError( f"Require Inference container information. Model: {model_name} does not have associated inference container defaults. Check docs for more information on how to pass inference container." ) + metadata.add( + key=AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME, + value=inference_container, + description=f"Inference container mapping for {model_name}", + category="Other", + ) + if ( + inference_container + == InferenceContainerTypeFamily.AQUA_TEI_CONTAINER_FAMILY + ): + if not inference_container_uri: + logger.warn( + f"Proceeding with model registration without the inference container URI for " + f"{inference_container}. You can still add this configuration during model deployment." + ) + else: + metadata.add( + key=AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME, + value=inference_container_uri, + description=f"Inference container URI for {model_name}", + category="Other", + ) + + cmd_vars = generate_tei_cmd_vars(os_path) + metadata.add( + key=AQUA_DEPLOYMENT_CONTAINER_CMD_VAR, + value=",".join(cmd_vars), + description=f"Inference container cmd vars for {model_name}", + category="Other", + ) if finetuning_container: tags[Tags.READY_TO_FINE_TUNE] = "true" metadata.add( @@ -706,12 +741,6 @@ def _create_model_catalog_entry( category="Other", ) - metadata.add( - key=AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME, - value=inference_container, - description=f"Inference container mapping for {model_name}", - category="Other", - ) metadata.add( key=AQUA_EVALUATION_CONTAINER_METADATA_NAME, value="odsc-llm-evaluate", @@ -1012,7 +1041,10 @@ def _validate_model( AQUA_MODEL_TYPE_CUSTOM ) elif model_format == ModelFormat.GGUF and len(gguf_model_files) > 0: - if import_model_details.finetuning_container and not safetensors_model_files: + if ( + import_model_details.finetuning_container + and not safetensors_model_files + ): raise AquaValueError( "Fine-tuning is currently not supported with GGUF model format." ) @@ -1193,6 +1225,7 @@ def register( validation_result=validation_result, compartment_id=import_model_details.compartment_id, project_id=import_model_details.project_id, + inference_container_uri=import_model_details.inference_container_uri, ) # registered model will always have inference and evaluation container, but # fine-tuning container may be not set diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index 8ce6ab320..665a71dcf 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -46,6 +46,7 @@ from ads.config import ( AQUA_CONFIG_FOLDER, AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME, + AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME, AQUA_MODEL_DEPLOYMENT_CONFIG, AQUA_MODEL_DEPLOYMENT_CONFIG_DEFAULTS, COMPARTMENT_OCID, @@ -106,6 +107,7 @@ def create( memory_in_gbs: Optional[float] = None, ocpus: Optional[float] = None, model_file: Optional[str] = None, + container_image_uri: Optional[None] = None, cmd_var: List[str] = None, ) -> "AquaDeployment": """ @@ -153,6 +155,9 @@ def create( The ocpu count for the shape selected. model_file: str The file used for model deployment. + container_image_uri: str + The image of model deployment container runtime, ignored for service managed containers. + Required parameter for BYOC based deployments if this parameter was not set during model registration. cmd_var: List[str] The cmd of model deployment container runtime. Returns @@ -198,9 +203,11 @@ def create( f"from custom metadata for the model {config_source_id}" ) from err - # set up env vars + # set up env and cmd var if not env_var: env_var = {} + if not cmd_var: + cmd_var = {} try: model_path_prefix = aqua_model.custom_metadata_list.get( @@ -236,11 +243,37 @@ def create( model=aqua_model, container_family=container_family ) - # fetch image name from config - container_image = get_container_image(container_type=container_type_key) + # todo: revisit this when TEI is added to SMC list. Currently, container_image_uri is ignored if container + # family is SMC. + if container_type_key == InferenceContainerTypeFamily.AQUA_TEI_CONTAINER_FAMILY: + if not container_image_uri: + try: + container_image_uri = aqua_model.custom_metadata_list.get( + AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME + ).value + except ValueError as err: + raise AquaValueError( + f"{AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME} key is not available in the custom metadata " + f"field. Either re-register the model with custom container URI, or set container_image_uri " + f"parameter when creating this deployment." + ) from err + + try: + cmd_var_string = aqua_model.custom_metadata_list.get( + AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME + ).value + cmd_var.append(cmd_var_string.split(",")) + except ValueError as err: + raise AquaValueError( + f"{AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME} key is not available in the custom metadata " + f"field. Please check if the model was registered with {container_type_key} inference container." + ) from err + else: + # fetch image name from config + container_image_uri = get_container_image(container_type=container_type_key) logging.info( - f"Aqua Image used for deploying {aqua_model.id} : {container_image}" + f"Aqua Image used for deploying {aqua_model.id} : {container_image_uri}" ) model_formats_str = aqua_model.freeform_tags.get( @@ -310,7 +343,7 @@ def create( # AQUA_LLAMA_CPP_CONTAINER_FAMILY container uses uvicorn that required model/server params # to be set as env vars raise AquaValueError( - f"Currently, parameters cannot be overridden for the container: {container_image}. Please proceed " + f"Currently, parameters cannot be overridden for the container: {container_image_uri}. Please proceed " f"with deployment without parameter overrides." ) @@ -364,7 +397,7 @@ def create( # configure model deployment runtime container_runtime = ( ModelDeploymentContainerRuntime() - .with_image(container_image) + .with_image(container_image_uri) .with_server_port(server_port) .with_health_check_port(health_check_port) .with_env(env_var) diff --git a/ads/config.py b/ads/config.py index cb4d7df10..ceeb3f9ff 100644 --- a/ads/config.py +++ b/ads/config.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8; -*- # Copyright (c) 2020, 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ @@ -66,6 +65,8 @@ AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME = "deployment-container" AQUA_FINETUNING_CONTAINER_METADATA_NAME = "finetune-container" AQUA_EVALUATION_CONTAINER_METADATA_NAME = "evaluation-container" +AQUA_DEPLOYMENT_CONTAINER_CMD_VAR = "container_cmd_var" +AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME = "deployment-container-uri" AQUA_DEPLOYMENT_CONTAINER_OVERRIDE_FLAG_METADATA_NAME = "deployment-container-custom" AQUA_FINETUNING_CONTAINER_OVERRIDE_FLAG_METADATA_NAME = "finetune-container-custom" AQUA_MODEL_DEPLOYMENT_FOLDER = "/opt/ds/model/deployed_model/" @@ -212,7 +213,7 @@ def open( frame.f_globals.pop("config", None) # Restores original globals - for key in defined_globals.keys(): + for key in defined_globals: frame.f_globals[key] = defined_globals[key] # Saving config if it necessary From 0ab772af2c46c9f84b0db6373f18923ea65adf28 Mon Sep 17 00:00:00 2001 From: Vipul Date: Thu, 17 Oct 2024 21:39:17 -0700 Subject: [PATCH 4/8] update cmd var --- ads/aqua/model/model.py | 4 ++-- ads/aqua/modeldeployment/deployment.py | 7 ++++--- ads/config.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py index 4a5d9c6ac..c328fe8df 100644 --- a/ads/aqua/model/model.py +++ b/ads/aqua/model/model.py @@ -68,7 +68,7 @@ from ads.common.oci_resource import SEARCH_TYPE, OCIResource from ads.common.utils import get_console_link from ads.config import ( - AQUA_DEPLOYMENT_CONTAINER_CMD_VAR, + AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME, AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME, AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME, AQUA_EVALUATION_CONTAINER_METADATA_NAME, @@ -715,7 +715,7 @@ def _create_model_catalog_entry( cmd_vars = generate_tei_cmd_vars(os_path) metadata.add( - key=AQUA_DEPLOYMENT_CONTAINER_CMD_VAR, + key=AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME, value=",".join(cmd_vars), description=f"Inference container cmd vars for {model_name}", category="Other", diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index 665a71dcf..f69b72a98 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -45,6 +45,7 @@ from ads.common.utils import get_log_links from ads.config import ( AQUA_CONFIG_FOLDER, + AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME, AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME, AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME, AQUA_MODEL_DEPLOYMENT_CONFIG, @@ -207,7 +208,7 @@ def create( if not env_var: env_var = {} if not cmd_var: - cmd_var = {} + cmd_var = [] try: model_path_prefix = aqua_model.custom_metadata_list.get( @@ -260,12 +261,12 @@ def create( try: cmd_var_string = aqua_model.custom_metadata_list.get( - AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME + AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME ).value cmd_var.append(cmd_var_string.split(",")) except ValueError as err: raise AquaValueError( - f"{AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME} key is not available in the custom metadata " + f"{AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME} key is not available in the custom metadata " f"field. Please check if the model was registered with {container_type_key} inference container." ) from err else: diff --git a/ads/config.py b/ads/config.py index ceeb3f9ff..97fb7108d 100644 --- a/ads/config.py +++ b/ads/config.py @@ -65,7 +65,7 @@ AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME = "deployment-container" AQUA_FINETUNING_CONTAINER_METADATA_NAME = "finetune-container" AQUA_EVALUATION_CONTAINER_METADATA_NAME = "evaluation-container" -AQUA_DEPLOYMENT_CONTAINER_CMD_VAR = "container_cmd_var" +AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME = "container_cmd_var" AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME = "deployment-container-uri" AQUA_DEPLOYMENT_CONTAINER_OVERRIDE_FLAG_METADATA_NAME = "deployment-container-custom" AQUA_FINETUNING_CONTAINER_OVERRIDE_FLAG_METADATA_NAME = "finetune-container-custom" From 679d44faa93bd616945ffc50e809eb168f5489bf Mon Sep 17 00:00:00 2001 From: Vipul Date: Thu, 17 Oct 2024 22:36:38 -0700 Subject: [PATCH 5/8] clean up --- ads/aqua/common/utils.py | 9 +++++---- ads/aqua/model/model.py | 4 ++-- ads/aqua/modeldeployment/deployment.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ads/aqua/common/utils.py b/ads/aqua/common/utils.py index 7aea22797..8b68afe95 100644 --- a/ads/aqua/common/utils.py +++ b/ads/aqua/common/utils.py @@ -1088,7 +1088,7 @@ def list_hf_models(query: str) -> List[str]: raise format_hf_custom_error_message(err) from err -def generate_tei_cmd_vars(os_path: str) -> List[str]: +def generate_tei_cmd_var(os_path: str) -> List[str]: """This utility functions generates CMD params for Text Embedding Inference container. Only the essential parameters for OCI model deployment are added, defaults are used for the rest. Parameters @@ -1098,15 +1098,16 @@ def generate_tei_cmd_vars(os_path: str) -> List[str]: Returns ------- - List of command line arguments + cmd_var: + List of command line arguments """ cmd_prefix = "--" - cmd_vars = [ + cmd_var = [ cmd_prefix + TextEmbeddingInferenceContainerParams.MODEL_ID, f"{AQUA_MODEL_DEPLOYMENT_FOLDER}{ObjectStorageDetails.from_path(os_path.rstrip('/')).filepath}/", cmd_prefix + TextEmbeddingInferenceContainerParams.PORT, TEI_CONTAINER_DEFAULT_HOST, ] - return cmd_vars + return cmd_var diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py index c328fe8df..4bdd4f0c6 100644 --- a/ads/aqua/model/model.py +++ b/ads/aqua/model/model.py @@ -21,7 +21,7 @@ _build_resource_identifier, copy_model_config, create_word_icon, - generate_tei_cmd_vars, + generate_tei_cmd_var, get_artifact_path, get_hf_model_info, list_os_files_with_extension, @@ -713,7 +713,7 @@ def _create_model_catalog_entry( category="Other", ) - cmd_vars = generate_tei_cmd_vars(os_path) + cmd_vars = generate_tei_cmd_var(os_path) metadata.add( key=AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME, value=",".join(cmd_vars), diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index f69b72a98..68c3a81c2 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -263,7 +263,7 @@ def create( cmd_var_string = aqua_model.custom_metadata_list.get( AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME ).value - cmd_var.append(cmd_var_string.split(",")) + cmd_var.extend(cmd_var_string.split(",")) except ValueError as err: raise AquaValueError( f"{AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME} key is not available in the custom metadata " From 2b5cd841aa7dc6c1cc3ffa8b99699c6188d92296 Mon Sep 17 00:00:00 2001 From: Vipul Date: Wed, 23 Oct 2024 15:44:24 -0700 Subject: [PATCH 6/8] changes for tei byoc deployment --- ads/aqua/common/utils.py | 44 +++++ ads/aqua/model/model.py | 10 +- ads/aqua/modeldeployment/deployment.py | 6 +- ads/aqua/modeldeployment/entities.py | 5 +- ads/config.py | 2 +- .../aqua_create_embedding_deployment.yaml | 34 ++++ .../aqua_tei_byoc_embedding_model.yaml | 88 ++++++++++ .../with_extras/aqua/test_deployment.py | 161 ++++++++++++++++++ tests/unitary/with_extras/aqua/test_model.py | 86 ++++++++++ 9 files changed, 430 insertions(+), 6 deletions(-) create mode 100644 tests/unitary/with_extras/aqua/test_data/deployment/aqua_create_embedding_deployment.yaml create mode 100644 tests/unitary/with_extras/aqua/test_data/deployment/aqua_tei_byoc_embedding_model.yaml diff --git a/ads/aqua/common/utils.py b/ads/aqua/common/utils.py index 8b68afe95..a7b60ad8a 100644 --- a/ads/aqua/common/utils.py +++ b/ads/aqua/common/utils.py @@ -1111,3 +1111,47 @@ def generate_tei_cmd_var(os_path: str) -> List[str]: ] return cmd_var + + +def parse_cmd_var(cmd_list: List[str]) -> dict: + """Helper functions that parses a list into a key-value dictionary. The list contains keys separated by the prefix + '--' and the value of the key is the subsequent element. + """ + it = iter(cmd_list) + return { + key: next(it, None) if not key.startswith("--") else next(it, None) + for key in it + if key.startswith("--") + } + + +def validate_cmd_var(cmd_var: List[str], overrides: List[str]) -> List[str]: + """This function accepts two lists of parameters and combines them. If the second list shares the common parameter + names/keys, then it raises an error. + Parameters + ---------- + cmd_var: List[str] + Default list of parameters + overrides: List[str] + List of parameters to override + Returns + ------- + List[str] of combined parameters + """ + cmd_var = [str(x) for x in cmd_var] + if not overrides: + return cmd_var + overrides = [str(x) for x in overrides] + + cmd_dict = parse_cmd_var(cmd_var) + overrides_dict = parse_cmd_var(overrides) + + # check for conflicts + common_keys = set(cmd_dict.keys()) & set(overrides_dict.keys()) + if common_keys: + raise AquaValueError( + f"The following keys cannot be overridden: {', '.join(common_keys)}" + ) + + combined_cmd_var = cmd_var + overrides + return combined_cmd_var diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py index 4bdd4f0c6..2ed6a94e6 100644 --- a/ads/aqua/model/model.py +++ b/ads/aqua/model/model.py @@ -720,6 +720,7 @@ def _create_model_catalog_entry( description=f"Inference container cmd vars for {model_name}", category="Other", ) + if finetuning_container: tags[Tags.READY_TO_FINE_TUNE] = "true" metadata.add( @@ -1232,9 +1233,12 @@ def register( inference_container = ds_model.custom_metadata_list.get( ModelCustomMetadataFields.DEPLOYMENT_CONTAINER ).value - evaluation_container = ds_model.custom_metadata_list.get( - ModelCustomMetadataFields.EVALUATION_CONTAINER, - ).value + try: + evaluation_container = ds_model.custom_metadata_list.get( + ModelCustomMetadataFields.EVALUATION_CONTAINER, + ).value + except Exception: + evaluation_container = None try: finetuning_container = ds_model.custom_metadata_list.get( ModelCustomMetadataFields.FINETUNE_CONTAINER, diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index 68c3a81c2..54819fe00 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -24,6 +24,7 @@ get_resource_name, get_restricted_params_by_container, load_config, + validate_cmd_var, ) from ads.aqua.constants import ( AQUA_MODEL_ARTIFACT_FILE, @@ -263,12 +264,15 @@ def create( cmd_var_string = aqua_model.custom_metadata_list.get( AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME ).value - cmd_var.extend(cmd_var_string.split(",")) except ValueError as err: raise AquaValueError( f"{AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME} key is not available in the custom metadata " f"field. Please check if the model was registered with {container_type_key} inference container." ) from err + default_cmd_var = cmd_var_string.split(",") + if default_cmd_var: + cmd_var = validate_cmd_var(default_cmd_var, cmd_var) + logging.info(f"CMD used for deploying {aqua_model.id} :{cmd_var}") else: # fetch image name from config container_image_uri = get_container_image(container_type=container_type_key) diff --git a/ads/aqua/modeldeployment/entities.py b/ads/aqua/modeldeployment/entities.py index 411c68529..b5295cb56 100644 --- a/ads/aqua/modeldeployment/entities.py +++ b/ads/aqua/modeldeployment/entities.py @@ -3,7 +3,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ from dataclasses import dataclass, field -from typing import Union +from typing import List, Union from oci.data_science.models import ( ModelDeployment, @@ -52,6 +52,7 @@ class AquaDeployment(DataClassSerializable): shape_info: field(default_factory=ShapeInfo) = None tags: dict = None environment_variables: dict = None + cmd: List[str] = None @classmethod def from_oci_model_deployment( @@ -80,6 +81,7 @@ def from_oci_model_deployment( ) instance_count = oci_model_deployment.model_deployment_configuration_details.model_configuration_details.scaling_policy.instance_count environment_variables = oci_model_deployment.model_deployment_configuration_details.environment_configuration_details.environment_variables + cmd = oci_model_deployment.model_deployment_configuration_details.environment_configuration_details.cmd shape_info = ShapeInfo( instance_shape=instance_configuration.instance_shape_name, instance_count=instance_count, @@ -120,6 +122,7 @@ def from_oci_model_deployment( ), tags=freeform_tags, environment_variables=environment_variables, + cmd=cmd, ) diff --git a/ads/config.py b/ads/config.py index 97fb7108d..567fce784 100644 --- a/ads/config.py +++ b/ads/config.py @@ -65,7 +65,7 @@ AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME = "deployment-container" AQUA_FINETUNING_CONTAINER_METADATA_NAME = "finetune-container" AQUA_EVALUATION_CONTAINER_METADATA_NAME = "evaluation-container" -AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME = "container_cmd_var" +AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME = "container-cmd-var" AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME = "deployment-container-uri" AQUA_DEPLOYMENT_CONTAINER_OVERRIDE_FLAG_METADATA_NAME = "deployment-container-custom" AQUA_FINETUNING_CONTAINER_OVERRIDE_FLAG_METADATA_NAME = "finetune-container-custom" diff --git a/tests/unitary/with_extras/aqua/test_data/deployment/aqua_create_embedding_deployment.yaml b/tests/unitary/with_extras/aqua/test_data/deployment/aqua_create_embedding_deployment.yaml new file mode 100644 index 000000000..c55460140 --- /dev/null +++ b/tests/unitary/with_extras/aqua/test_data/deployment/aqua_create_embedding_deployment.yaml @@ -0,0 +1,34 @@ +kind: deployment +spec: + createdBy: ocid1.user.oc1.. + displayName: model-deployment-name + freeformTags: + OCI_AQUA: active + aqua_model_name: model-name + id: "ocid1.datasciencemodeldeployment.oc1.." + infrastructure: + kind: infrastructure + spec: + bandwidthMbps: 10 + compartmentId: ocid1.compartment.oc1.. + deploymentType: SINGLE_MODEL + policyType: FIXED_SIZE + projectId: ocid1.datascienceproject.oc1.iad. + replica: 1 + shapeName: "VM.GPU.A10.1" + type: datascienceModelDeployment + lifecycleState: CREATING + modelDeploymentUrl: "https://modeldeployment.customer-oci.com/ocid1.datasciencemodeldeployment.oc1.." + runtime: + kind: runtime + spec: + env: + BASE_MODEL: service_models/model-name/artifact + MODEL_DEPLOY_PREDICT_ENDPOINT: /v1/embeddings + healthCheckPort: 8080 + image: "dsmc://image-name:1.0.0.0" + modelUri: "ocid1.datasciencemodeldeployment.oc1.." + serverPort: 8080 + type: container + timeCreated: 2024-01-01T00:00:00.000000+00:00 +type: modelDeployment diff --git a/tests/unitary/with_extras/aqua/test_data/deployment/aqua_tei_byoc_embedding_model.yaml b/tests/unitary/with_extras/aqua/test_data/deployment/aqua_tei_byoc_embedding_model.yaml new file mode 100644 index 000000000..350a925bc --- /dev/null +++ b/tests/unitary/with_extras/aqua/test_data/deployment/aqua_tei_byoc_embedding_model.yaml @@ -0,0 +1,88 @@ +kind: datascienceModel +spec: + artifact: oci://service-managed-models@namespace/service_models/model-name/artifact + compartmentId: ocid1.compartment.oc1.. + customMetadataList: + data: + - category: Other + description: artifact location + key: artifact_location + value: service_models/model-name/artifact + - category: Other + description: model by reference flag + key: modelDescription + value: true + - category: Other + description: Deployment container mapping for model model-name + key: deployment-container + value: odsc-tei-serving + - category: Other + description: Inference container URI for model model-name + key: deployment-container-uri + value: region.ocir.io/tenancy/image_name:tag + - category: Other + description: Inference container cmd vars for model-name + key: container-cmd-var + value: --model-id,/opt/ds/model/deployed_model/service_models/model-name/artifact/,--port,8080 + definedTags: {} + description: Mock model description + displayName: model-name + freeformTags: + OCI_AQUA: active + license: License + organization: Organization + ready_to_fine_tune: false + task: text_embedding + id: ocid1.datasciencemodel.oc1.iad. + lifecycleState: ACTIVE + modelDescription: + models: + - bucketName: service-managed-models + namespace: namespace + objects: + - name: service_models/model-name/artifact/README.md + sizeInBytes: 10317 + version: 450a8124-f5ca-4ee6-b4cf-c1dc05b13d46 + - name: service_models/model-name/artifact/config.json + sizeInBytes: 950 + version: 3ace781b-4a48-4e89-88b6-61f0db6d51ad + - name: service_models/model-name/artifact/configuration_RW.py + sizeInBytes: 2607 + version: ba1df5b6-7546-42e5-964e-63cd013e988c + - name: service_models/model-name/artifact/generation_config.json + sizeInBytes: 111 + version: e23a04c8-9725-4f20-8bb1-f455129e2a4e + - name: service_models/model-name/artifact/modelling_RW.py + sizeInBytes: 47560 + version: a584c221-afab-441f-901d-fbe8251dccf6 + - name: service_models/model-name/artifact/pytorch_model-00001-of-00002.bin + sizeInBytes: 9951028193 + version: e919676e-48dd-4bea-af82-14b5f3eb2b9b + - name: service_models/model-name/artifact/pytorch_model-00002-of-00002.bin + sizeInBytes: 4483421659 + version: d6255d3e-bd91-4c05-b3ca-fc1be576ee10 + - name: service_models/model-name/artifact/pytorch_model.bin.index.json + sizeInBytes: 16924 + version: 0419428c-2a7b-45d9-bb78-142fe0630017 + - name: service_models/model-name/artifact/special_tokens_map.json + sizeInBytes: 281 + version: 5569231a-a526-4881-8945-a94a1bb59b2e + - name: service_models/model-name/artifact/tokenizer.json + sizeInBytes: 2734130 + version: d3a8a00a-de79-4d80-aa69-d8f68ee800ec + - name: service_models/model-name/artifact/tokenizer_config.json + sizeInBytes: 220 + version: 84eed6ff-c1ed-4641-8c10-e6a49364d7dd + prefix: service_models/model-name/artifact + type: modelOSSReferenceDescription + version: '1.0' + projectId: ocid1.datascienceproject.oc1.iad. + provenanceMetadata: + artifact_dir: null + git_branch: null + git_commit: 123456 + repository_url: https://model-name-url.com + training_id: null + training_script_path: null + timeCreated: 2024-01-01T00:00:00.000000+00:00 +type: dataScienceModel diff --git a/tests/unitary/with_extras/aqua/test_deployment.py b/tests/unitary/with_extras/aqua/test_deployment.py index 7ef73c25f..3b2fe6652 100644 --- a/tests/unitary/with_extras/aqua/test_deployment.py +++ b/tests/unitary/with_extras/aqua/test_deployment.py @@ -171,6 +171,78 @@ class TestDataset: } ] + model_deployment_object_tei_byoc = [ + { + "category_log_details": oci.data_science.models.CategoryLogDetails( + **{ + "access": oci.data_science.models.LogDetails( + **{ + "log_group_id": "ocid1.loggroup.oc1..", + "log_id": "ocid1.log.oc1..", + } + ), + "predict": oci.data_science.models.LogDetails( + **{ + "log_group_id": "ocid1.loggroup.oc1..", + "log_id": "ocid1.log.oc1..", + } + ), + } + ), + "compartment_id": "ocid1.compartment.oc1..", + "created_by": "ocid1.user.oc1..", + "defined_tags": {}, + "description": "Mock description", + "display_name": "model-deployment-name", + "freeform_tags": {"OCI_AQUA": "active", "aqua_model_name": "model-name"}, + "id": "ocid1.datasciencemodeldeployment.oc1..", + "lifecycle_state": "ACTIVE", + "model_deployment_configuration_details": oci.data_science.models.SingleModelDeploymentConfigurationDetails( + **{ + "deployment_type": "SINGLE_MODEL", + "environment_configuration_details": oci.data_science.models.OcirModelDeploymentEnvironmentConfigurationDetails( + **{ + "cmd": [ + "--model-id", + "/opt/ds/model/deployed_model/service_models/model-name/artifact/", + "--port", + "8080", + ], + "entrypoint": [], + "environment_configuration_type": "OCIR_CONTAINER", + "environment_variables": { + "BASE_MODEL": "service_models/model-name/artifact", + "MODEL_DEPLOY_PREDICT_ENDPOINT": "/v1/embeddings", + }, + "health_check_port": 8080, + "image": "dsmc://image-name:1.0.0.0", + "image_digest": "sha256:mock22373c16f2015f6f33c5c8553923cf8520217da0bd9504471c5e53cbc9d", + "server_port": 8080, + } + ), + "model_configuration_details": oci.data_science.models.ModelConfigurationDetails( + **{ + "bandwidth_mbps": 10, + "instance_configuration": oci.data_science.models.InstanceConfiguration( + **{ + "instance_shape_name": DEPLOYMENT_SHAPE_NAME, + "model_deployment_instance_shape_config_details": null, + } + ), + "model_id": "ocid1.datasciencemodel.oc1..", + "scaling_policy": oci.data_science.models.FixedSizeScalingPolicy( + **{"instance_count": 1, "policy_type": "FIXED_SIZE"} + ), + } + ), + } + ), + "model_deployment_url": MODEL_DEPLOYMENT_URL, + "project_id": "ocid1.datascienceproject.oc1..", + "time_created": "2024-01-01T00:00:00.000000+00:00", + } + ] + aqua_deployment_object = { "id": "ocid1.datasciencemodeldeployment.oc1..", "display_name": "model-deployment-name", @@ -235,6 +307,18 @@ class TestDataset: "top_k": 10, } + aqua_deployment_tei_byoc_embeddings_env_vars = { + "BASE_MODEL": "service_models/model-name/artifact", + "MODEL_DEPLOY_PREDICT_ENDPOINT": "/v1/embeddings", + } + + aqua_deployment_tei_byoc_embeddings_shape_info = { + "instance_shape": DEPLOYMENT_SHAPE_NAME, + "instance_count": 1, + "ocpus": None, + "memory_in_gbs": None, + } + class TestAquaDeployment(unittest.TestCase): def setUp(self): @@ -563,6 +647,83 @@ def test_create_deployment_for_gguf_model( ) assert actual_attributes == expected_result + @patch("ads.aqua.modeldeployment.deployment.get_container_config") + @patch("ads.aqua.model.AquaModelApp.create") + @patch("ads.aqua.modeldeployment.deployment.get_container_image") + @patch("ads.model.deployment.model_deployment.ModelDeployment.deploy") + def test_create_deployment_for_tei_byoc_embedding_model( + self, + mock_deploy, + mock_get_container_image, + mock_create, + mock_get_container_config, + ): + """Test to create a deployment for fine-tuned model""" + aqua_model = os.path.join( + self.curr_dir, "test_data/deployment/aqua_tei_byoc_embedding_model.yaml" + ) + datascience_model = DataScienceModel.from_yaml(uri=aqua_model) + mock_create.return_value = datascience_model + + config_json = os.path.join( + self.curr_dir, "test_data/deployment/deployment_config.json" + ) + with open(config_json, "r") as _file: + config = json.load(_file) + + self.app.get_deployment_config = MagicMock(return_value=config) + + container_index_json = os.path.join( + self.curr_dir, "test_data/ui/container_index.json" + ) + with open(container_index_json, "r") as _file: + container_index_config = json.load(_file) + mock_get_container_config.return_value = container_index_config + + mock_get_container_image.return_value = TestDataset.DEPLOYMENT_IMAGE_NAME + aqua_deployment = os.path.join( + self.curr_dir, "test_data/deployment/aqua_create_embedding_deployment.yaml" + ) + model_deployment_obj = ModelDeployment.from_yaml(uri=aqua_deployment) + model_deployment_dsc_obj = copy.deepcopy( + TestDataset.model_deployment_object_tei_byoc[0] + ) + model_deployment_dsc_obj["lifecycle_state"] = "CREATING" + model_deployment_obj.dsc_model_deployment = ( + oci.data_science.models.ModelDeploymentSummary(**model_deployment_dsc_obj) + ) + mock_deploy.return_value = model_deployment_obj + + result = self.app.create( + model_id=TestDataset.MODEL_ID, + instance_shape=TestDataset.DEPLOYMENT_SHAPE_NAME, + display_name="model-deployment-name", + log_group_id="ocid1.loggroup.oc1..", + access_log_id="ocid1.log.oc1..", + predict_log_id="ocid1.log.oc1..", + container_family="odsc-tei-serving", + cmd_var=[], + ) + + mock_create.assert_called_with( + model_id=TestDataset.MODEL_ID, compartment_id=None, project_id=None + ) + mock_get_container_image.assert_not_called() + mock_deploy.assert_called() + + expected_attributes = set(AquaDeployment.__annotations__.keys()) + actual_attributes = asdict(result) + assert set(actual_attributes) == set(expected_attributes), "Attributes mismatch" + expected_result = copy.deepcopy(TestDataset.aqua_deployment_object) + expected_result["state"] = "CREATING" + expected_result["shape_info"] = ( + TestDataset.aqua_deployment_tei_byoc_embeddings_shape_info + ) + expected_result["environment_variables"] = ( + TestDataset.aqua_deployment_tei_byoc_embeddings_env_vars + ) + assert actual_attributes == expected_result + @parameterized.expand( [ ( diff --git a/tests/unitary/with_extras/aqua/test_model.py b/tests/unitary/with_extras/aqua/test_model.py index c0386fc69..46baaa3b9 100644 --- a/tests/unitary/with_extras/aqua/test_model.py +++ b/tests/unitary/with_extras/aqua/test_model.py @@ -1012,6 +1012,83 @@ def test_import_any_model_smc_container( assert model.ready_to_deploy is True assert model.ready_to_finetune is True + @pytest.mark.parametrize( + "download_from_hf", + [True, False], + ) + @patch.object(AquaModelApp, "_find_matching_aqua_model") + @patch("ads.common.object_storage_details.ObjectStorageDetails.list_objects") + @patch("ads.aqua.common.utils.load_config", return_value={}) + @patch("huggingface_hub.snapshot_download") + @patch("subprocess.check_call") + def test_import_tei_model_byoc( + self, + mock_subprocess, + mock_snapshot_download, + mock_load_config, + mock_list_objects, + mock__find_matching_aqua_model, + download_from_hf, + mock_get_hf_model_info, + ): + ObjectStorageDetails.is_bucket_versioned = MagicMock(return_value=True) + ads.common.oci_datascience.OCIDataScienceMixin.init_client = MagicMock() + DataScienceModel.upload_artifact = MagicMock() + DataScienceModel.sync = MagicMock() + OCIDataScienceModel.create = MagicMock() + + mock_list_objects.return_value = MagicMock(objects=[]) + ds_model = DataScienceModel() + os_path = "oci://aqua-bkt@aqua-ns/prefix/path" + model_name = "oracle/aqua-1t-mega-model" + ds_freeform_tags = { + "OCI_AQUA": "ACTIVE", + "license": "aqua-license", + "organization": "oracle", + "task": "text_embedding", + } + ds_model = ( + ds_model.with_compartment_id("test_model_compartment_id") + .with_project_id("test_project_id") + .with_display_name(model_name) + .with_description("test_description") + .with_model_version_set_id("test_model_version_set_id") + .with_freeform_tags(**ds_freeform_tags) + .with_version_id("ocid1.version.id") + ) + custom_metadata_list = ModelCustomMetadata() + custom_metadata_list.add( + **{"key": "deployment-container", "value": "odsc-tei-serving"} + ) + ds_model.with_custom_metadata_list(custom_metadata_list) + ds_model.set_spec(ds_model.CONST_MODEL_FILE_DESCRIPTION, {}) + DataScienceModel.from_id = MagicMock(return_value=ds_model) + mock__find_matching_aqua_model.return_value = None + reload(ads.aqua.model.model) + app = AquaModelApp() + + if download_from_hf: + with tempfile.TemporaryDirectory() as tmpdir: + model: AquaModel = app.register( + model=model_name, + os_path=os_path, + local_dir=str(tmpdir), + download_from_hf=True, + inference_container="odsc-tei-serving", + inference_container_uri="region.ocir.io/your_tenancy/your_image", + ) + else: + model: AquaModel = app.register( + model="ocid1.datasciencemodel.xxx.xxxx.", + os_path=os_path, + download_from_hf=False, + inference_container="odsc-tei-serving", + inference_container_uri="region.ocir.io/your_tenancy/your_image", + ) + assert model.inference_container == "odsc-tei-serving" + assert model.ready_to_deploy is True + assert model.ready_to_finetune is False + @pytest.mark.parametrize( "data, expected_output", [ @@ -1047,6 +1124,15 @@ def test_import_any_model_smc_container( }, "ads aqua model register --model oracle/oracle-1it --os_path oci://aqua-bkt@aqua-ns/path --download_from_hf True --model_file test_model_file", ), + ( + { + "os_path": "oci://aqua-bkt@aqua-ns/path", + "model": "oracle/oracle-1it", + "inference_container": "odsc-tei-serving", + "inference_container_uri": ".ocir.io//", + }, + "ads aqua model register --model oracle/oracle-1it --os_path oci://aqua-bkt@aqua-ns/path --download_from_hf True --inference_container odsc-tei-serving --inference_container_uri .ocir.io//", + ), ], ) def test_import_cli(self, data, expected_output): From e0808227f68c396e3b921c41f85d910dda95b7b7 Mon Sep 17 00:00:00 2001 From: Vipul Date: Wed, 23 Oct 2024 16:15:37 -0700 Subject: [PATCH 7/8] fix tests --- tests/unitary/with_extras/aqua/test_deployment.py | 9 +++++++++ tests/unitary/with_extras/aqua/test_model.py | 10 +++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/unitary/with_extras/aqua/test_deployment.py b/tests/unitary/with_extras/aqua/test_deployment.py index 3b2fe6652..74efb33c4 100644 --- a/tests/unitary/with_extras/aqua/test_deployment.py +++ b/tests/unitary/with_extras/aqua/test_deployment.py @@ -259,6 +259,7 @@ class TestDataset: "MODEL_DEPLOY_PREDICT_ENDPOINT": "/v1/completions", "PARAMS": "--served-model-name odsc-llm --seed 42", }, + "cmd": [], "console_link": "https://cloud.oracle.com/data-science/model-deployments/ocid1.datasciencemodeldeployment.oc1..?region=region-name", "lifecycle_details": "", "shape_info": { @@ -319,6 +320,13 @@ class TestDataset: "memory_in_gbs": None, } + aqua_deployment_tei_byoc_embeddings_cmd = [ + "--model-id", + "/opt/ds/model/deployed_model/service_models/model-name/artifact/", + "--port", + "8080", + ] + class TestAquaDeployment(unittest.TestCase): def setUp(self): @@ -719,6 +727,7 @@ def test_create_deployment_for_tei_byoc_embedding_model( expected_result["shape_info"] = ( TestDataset.aqua_deployment_tei_byoc_embeddings_shape_info ) + expected_result["cmd"] = TestDataset.aqua_deployment_tei_byoc_embeddings_cmd expected_result["environment_variables"] = ( TestDataset.aqua_deployment_tei_byoc_embeddings_env_vars ) diff --git a/tests/unitary/with_extras/aqua/test_model.py b/tests/unitary/with_extras/aqua/test_model.py index 46baaa3b9..efbfbba2d 100644 --- a/tests/unitary/with_extras/aqua/test_model.py +++ b/tests/unitary/with_extras/aqua/test_model.py @@ -64,7 +64,7 @@ def mock_get_container_config(): yield mock_config -@pytest.fixture(autouse=True, scope="class") +@pytest.fixture(autouse=True, scope="function") def mock_get_hf_model_info(): with patch.object(HfApi, "model_info") as mock_get_hf_model_info: test_hf_model_info = ModelInfo( @@ -933,7 +933,7 @@ def test_import_model_with_missing_config( app.list = MagicMock(return_value=[]) if download_from_hf: - with pytest.raises(AquaValueError): + with pytest.raises(AquaRuntimeError): mock_get_hf_model_info.return_value.siblings = [] with tempfile.TemporaryDirectory() as tmpdir: model: AquaModel = app.register( @@ -1037,7 +1037,11 @@ def test_import_tei_model_byoc( DataScienceModel.sync = MagicMock() OCIDataScienceModel.create = MagicMock() - mock_list_objects.return_value = MagicMock(objects=[]) + artifact_path = "service_models/model-name/commit-id/artifact" + obj1 = MagicMock(etag="12345-1234-1234-1234-123456789", size=150) + obj1.name = f"{artifact_path}/config.json" + objects = [obj1] + mock_list_objects.return_value = MagicMock(objects=objects) ds_model = DataScienceModel() os_path = "oci://aqua-bkt@aqua-ns/prefix/path" model_name = "oracle/aqua-1t-mega-model" From 07483da9b8e9661ac2dbbfe471c884455b325a45 Mon Sep 17 00:00:00 2001 From: Vipul Date: Thu, 24 Oct 2024 13:40:36 -0700 Subject: [PATCH 8/8] review comments --- ads/aqua/common/utils.py | 8 ++- ads/aqua/modeldeployment/deployment.py | 52 +++++++++---------- .../with_extras/aqua/test_deployment.py | 2 +- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/ads/aqua/common/utils.py b/ads/aqua/common/utils.py index 670208670..8ef9c0405 100644 --- a/ads/aqua/common/utils.py +++ b/ads/aqua/common/utils.py @@ -576,15 +576,13 @@ def get_container_image( A dict of allowed configs. """ + container_image = UNKNOWN config = config_file_name or get_container_config() config_file_name = service_config_path() if container_type not in config: - raise AquaValueError( - f"{config_file_name} does not have config details for model: {container_type}" - ) + return UNKNOWN - container_image = None mapping = config[container_type] versions = [obj["version"] for obj in mapping] # assumes numbered versions, update if `latest` is used @@ -1149,7 +1147,7 @@ def validate_cmd_var(cmd_var: List[str], overrides: List[str]) -> List[str]: common_keys = set(cmd_dict.keys()) & set(overrides_dict.keys()) if common_keys: raise AquaValueError( - f"The following keys cannot be overridden: {', '.join(common_keys)}" + f"The following CMD input cannot be overridden for model deployment: {', '.join(common_keys)}" ) combined_cmd_var = cmd_var + overrides diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index 1aeaef5cb..69491e2e5 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -246,41 +246,41 @@ def create( model=aqua_model, container_family=container_family ) - # todo: revisit this when TEI is added to SMC list. Currently, container_image_uri is ignored if container - # family is SMC. - if container_type_key == InferenceContainerTypeFamily.AQUA_TEI_CONTAINER_FAMILY: - if not container_image_uri: - try: - container_image_uri = aqua_model.custom_metadata_list.get( - AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME - ).value - except ValueError as err: - raise AquaValueError( - f"{AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME} key is not available in the custom metadata " - f"field. Either re-register the model with custom container URI, or set container_image_uri " - f"parameter when creating this deployment." - ) from err - + container_image_uri = container_image_uri or get_container_image( + container_type=container_type_key + ) + if not container_image_uri: try: - cmd_var_string = aqua_model.custom_metadata_list.get( - AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME + container_image_uri = aqua_model.custom_metadata_list.get( + AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME ).value except ValueError as err: raise AquaValueError( - f"{AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME} key is not available in the custom metadata " - f"field. Please check if the model was registered with {container_type_key} inference container." + f"{AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME} key is not available in the custom metadata " + f"field. Either re-register the model with custom container URI, or set container_image_uri " + f"parameter when creating this deployment." ) from err + logging.info( + f"Aqua Image used for deploying {aqua_model.id} : {container_image_uri}" + ) + + try: + cmd_var_string = aqua_model.custom_metadata_list.get( + AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME + ).value default_cmd_var = cmd_var_string.split(",") if default_cmd_var: cmd_var = validate_cmd_var(default_cmd_var, cmd_var) logging.info(f"CMD used for deploying {aqua_model.id} :{cmd_var}") - else: - # fetch image name from config - container_image_uri = get_container_image(container_type=container_type_key) - - logging.info( - f"Aqua Image used for deploying {aqua_model.id} : {container_image_uri}" - ) + except ValueError: + logging.debug( + f"CMD will be ignored for this deployment as {AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME} " + f"key is not available in the custom metadata field for this model." + ) + except Exception as e: + logging.error( + f"There was an issue processing CMD arguments. Error: {str(e)}" + ) model_formats_str = aqua_model.freeform_tags.get( Tags.MODEL_FORMAT, ModelFormat.SAFETENSORS.value diff --git a/tests/unitary/with_extras/aqua/test_deployment.py b/tests/unitary/with_extras/aqua/test_deployment.py index 74efb33c4..2ca916a1e 100644 --- a/tests/unitary/with_extras/aqua/test_deployment.py +++ b/tests/unitary/with_extras/aqua/test_deployment.py @@ -716,7 +716,7 @@ def test_create_deployment_for_tei_byoc_embedding_model( mock_create.assert_called_with( model_id=TestDataset.MODEL_ID, compartment_id=None, project_id=None ) - mock_get_container_image.assert_not_called() + mock_get_container_image.assert_called() mock_deploy.assert_called() expected_attributes = set(AquaDeployment.__annotations__.keys())