Merge branch 'main' into operators/auto_select_tests

Author: ahosler

.github/workflows/run-operators-unit-tests.yml

@@ -27,7 +27,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/run-unittests-default_setup.yml

@@ -1,4 +1,4 @@
-name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"
+name: "[Py3.8-3.11] - Default Tests"
 
 on:
   workflow_dispatch:
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/run-unittests-py38-cov-report.yml

@@ -1,4 +1,4 @@
-name: "[Py3.8][COV REPORT] tests/unitary/**"
+name: "[Py3.8][COV REPORT] - All Unit Tests"
 
 on:
   workflow_dispatch:
@@ -44,6 +44,7 @@ jobs:
             ignore-path: |
               --ignore tests/unitary/with_extras/model \
               --ignore tests/unitary/with_extras/feature_store \
+              --ignore  tests/unitary/with_extras/operator/feature-store \
               --ignore tests/unitary/with_extras/operator/forecast \
               --ignore tests/unitary/with_extras/hpo
           - name: "slow_tests"
@@ -188,6 +189,7 @@ jobs:
           fi
 
       - name: "Add comment with cov diff to PR"
+        continue-on-error: true
         uses: actions/github-script@v7
         if: github.event_name == 'pull_request'
         with:

.github/workflows/run-unittests-py39-py310.yml

@@ -1,4 +1,4 @@
-name: "[Py3.9][Py3.10] - tests/unitary/**"
+name: "[Py3.9-3.11] - All Unit Tests"
 
 on:
   workflow_dispatch:
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
         name: ["unitary", "slow_tests"]
         include:
           - name: "unitary"
@@ -46,6 +46,7 @@ jobs:
             ignore-path: |
               --ignore tests/unitary/with_extras/model \
               --ignore tests/unitary/with_extras/feature_store \
+              --ignore tests/unitary/with_extras/operator/feature-store \
               --ignore tests/unitary/with_extras/operator/forecast \
               --ignore tests/unitary/with_extras/operator/pii \
               --ignore tests/unitary/with_extras/hpo

.github/workflows/test-env-setup/action.yml

@@ -7,6 +7,11 @@ runs:
   using: composite
   steps:
     - shell: bash
+      env:
+        # torch > v2.1.0 brings nvidia dependency by default, we want to install torch for cpu for tests.
+        # Fot that --index-url https://download.pytorch.org/whl/cpu needed - https://pytorch.org/get-started/locally/.
+        # Setting env variable here instead of flag --index-url (docs: https://pip.pypa.io/en/stable/cli/pip_install/):
+        PIP_EXTRA_INDEX_URL: "https://download.pytorch.org/whl/cpu"
       run: |
         set -x # print commands that are executed
 

.pre-commit-config.yaml

@@ -1,7 +1,19 @@
 repos:
+    # ruff
+    - repo: https://github.com/astral-sh/ruff-pre-commit
+      rev: v0.5.0
+      hooks:
+          - id: ruff
+            types_or: [ python, pyi, jupyter ]
+            args: [ --fix ]
+            files: ^ads
+            exclude: ^docs/
+          - id: ruff-format
+            types_or: [ python, pyi, jupyter ]
+            exclude: ^docs/
     # Standard hooks
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.4.0
+      rev: v4.6.0
       hooks:
           - id: check-ast
             exclude: ^docs/
@@ -20,12 +32,6 @@ repos:
           - id: trailing-whitespace
             args: [--markdown-linebreak-ext=md]
             exclude: ^docs/
-    # Black, the code formatter, natively supports pre-commit
-    - repo: https://github.com/psf/black
-      rev: 23.3.0
-      hooks:
-          - id: black
-            exclude: ^docs/
     # Regex based rst files common mistakes detector
     - repo: https://github.com/pre-commit/pygrep-hooks
       rev: v1.10.0
@@ -36,7 +42,7 @@ repos:
             files: ^docs/
     # Hardcoded secrets and ocids detector
     - repo: https://github.com/gitleaks/gitleaks
-      rev: v8.17.0
+      rev: v8.18.4
       hooks:
           - id: gitleaks
             exclude: .github/workflows/reusable-actions/set-dummy-conf.yml

ads/aqua/app.py

@@ -303,7 +303,15 @@ def get_config(self, model_id: str, config_file_name: str) -> Dict:
                 config_file_name=config_file_name,
             )
         except:
-            pass
+            # todo: temp fix for issue related to config load for byom models, update logic to choose the right path
+            try:
+                config_path = f"{artifact_path.rstrip('/')}/config/"
+                config = load_config(
+                    config_path,
+                    config_file_name=config_file_name,
+                )
+            except:
+                pass
 
         if not config:
             logger.error(

ads/aqua/common/utils.py

@@ -28,10 +28,11 @@
 from ads.aqua.constants import *
 from ads.aqua.data import AquaResourceIdentifier
 from ads.common.auth import default_signer
+from ads.common.decorator.threaded import threaded
 from ads.common.extended_enum import ExtendedEnumMeta
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.common.oci_resource import SEARCH_TYPE, OCIResource
-from ads.common.utils import get_console_link, upload_to_os, copy_file
+from ads.common.utils import copy_file, get_console_link, upload_to_os
 from ads.config import AQUA_SERVICE_MODELS_BUCKET, CONDA_BUCKET_NS, TENANCY_OCID
 from ads.model import DataScienceModel, ModelVersionSet
 
@@ -195,6 +196,7 @@ def read_file(file_path: str, **kwargs) -> str:
         return UNKNOWN
 
 
+@threaded()
 def load_config(file_path: str, config_file_name: str, **kwargs) -> dict:
     artifact_path = f"{file_path.rstrip('/')}/{config_file_name}"
     if artifact_path.startswith("oci://"):
@@ -540,8 +542,10 @@ def get_container_image(
 
 
 def fetch_service_compartment() -> Union[str, None]:
-    """Loads the compartment mapping json from service bucket. This json file has a service-model-compartment key which
-    contains a dictionary of namespaces and the compartment OCID of the service models in that namespace.
+    """
+    Loads the compartment mapping json from service bucket.
+    This json file has a service-model-compartment key which contains a dictionary of namespaces
+    and the compartment OCID of the service models in that namespace.
     """
     config_file_name = (
         f"oci://{AQUA_SERVICE_MODELS_BUCKET}@{CONDA_BUCKET_NS}/service_models/config"
@@ -554,8 +558,8 @@ def fetch_service_compartment() -> Union[str, None]:
         )
     except Exception as e:
         logger.debug(
-            f"Config file {config_file_name}/{CONTAINER_INDEX} to fetch service compartment OCID could not be found. "
-            f"\n{str(e)}."
+            f"Config file {config_file_name}/{CONTAINER_INDEX} to fetch service compartment OCID "
+            f"could not be found. \n{str(e)}."
         )
         return
     compartment_mapping = config.get(COMPARTMENT_MAPPING_KEY)
@@ -664,23 +668,26 @@ def get_model_by_reference_paths(model_file_description: dict):
     fine_tune_output_path = UNKNOWN
     models = model_file_description["models"]
 
-    for model in models:
-        namespace, bucket_name, prefix = (
-            model["namespace"],
-            model["bucketName"],
-            model["prefix"],
+    if not models:
+        raise AquaValueError(
+            f"Model path is not available in the model json artifact. "
+            f"Please check if the model created by reference has the correct artifact."
         )
-        bucket_uri = f"oci://{bucket_name}@{namespace}/{prefix}".rstrip("/")
-        if bucket_name == AQUA_SERVICE_MODELS_BUCKET:
-            base_model_path = bucket_uri
-        else:
-            fine_tune_output_path = bucket_uri
 
-    if not base_model_path:
-        raise AquaValueError(
-            f"Base Model should come from the bucket {AQUA_SERVICE_MODELS_BUCKET}. "
-            f"Other paths are not supported by Aqua."
+    if len(models) > 0:
+        # since the model_file_description json does not have a flag to identify the base model, we consider
+        # the first instance to be the base model.
+        base_model_artifact = models[0]
+        base_model_path = f"oci://{base_model_artifact['bucketName']}@{base_model_artifact['namespace']}/{base_model_artifact['prefix']}".rstrip(
+            "/"
+        )
+    if len(models) > 1:
+        # second model is considered as fine-tuned model
+        ft_model_artifact = models[1]
+        fine_tune_output_path = f"oci://{ft_model_artifact['bucketName']}@{ft_model_artifact['namespace']}/{ft_model_artifact['prefix']}".rstrip(
+            "/"
         )
+
     return base_model_path, fine_tune_output_path
 
 

ads/aqua/finetuning/finetuning.py

@@ -109,11 +109,13 @@ def create(
                 )
 
         source = self.get_source(create_fine_tuning_details.ft_source_id)
-        if source.compartment_id != ODSC_MODEL_COMPARTMENT_OCID:
-            raise AquaValueError(
-                f"Fine tuning is only supported for Aqua service models in {ODSC_MODEL_COMPARTMENT_OCID}. "
-                "Use a valid Aqua service model id instead."
-            )
+
+        # todo: revisit validation for fine tuned models
+        # if source.compartment_id != ODSC_MODEL_COMPARTMENT_OCID:
+        #     raise AquaValueError(
+        #         f"Fine tuning is only supported for Aqua service models in {ODSC_MODEL_COMPARTMENT_OCID}. "
+        #         "Use a valid Aqua service model id instead."
+        #     )
 
         target_compartment = (
             create_fine_tuning_details.compartment_id or COMPARTMENT_OCID
@@ -364,6 +366,7 @@ def create(
         source_freeform_tags.pop(Tags.LICENSE, None)
         source_freeform_tags.update({Tags.READY_TO_FINE_TUNE: "false"})
         source_freeform_tags.update({Tags.AQUA_TAG: UNKNOWN})
+        source_freeform_tags.pop(Tags.BASE_MODEL_CUSTOM, None)
 
         self.update_model(
             model_id=ft_model.id,

ads/aqua/model/model.py

@@ -648,15 +648,19 @@ def _create_model_catalog_entry(
             copy_model_config(
                 artifact_path=artifact_path, os_path=os_path, auth=default_signer()
             )
-
         except:
-            # Add artifact from user bucket
-            metadata.add(
-                key=MODEL_BY_REFERENCE_OSS_PATH_KEY,
-                value=os_path,
-                description="artifact location",
-                category="Other",
+            logger.debug(
+                f"Proceeding with model registration without copying model config files at {os_path}. "
+                f"Default configuration will be used for deployment and fine-tuning."
             )
+        # Set artifact location to user bucket, and replace existing key if present.
+        metadata.add(
+            key=MODEL_BY_REFERENCE_OSS_PATH_KEY,
+            value=os_path,
+            description="artifact location",
+            category="Other",
+            replace=True,
+        )
 
         model = (
             model.with_custom_metadata_list(metadata)