Add GGUF files validation and files endpoint (#900)

Author: harsh97

.pre-commit-config.yaml

@@ -45,7 +45,7 @@ repos:
       rev: v8.18.4
       hooks:
           - id: gitleaks
-            exclude: .github/workflows/reusable-actions/set-dummy-conf.yml
+            exclude: .github/workflows/reusable-actions/set-dummy-conf.yml|./tests/operators/common/test_load_data.py
     # Oracle copyright checker
     - repo: https://github.com/oracle-samples/oci-data-science-ai-samples/
       rev: 1bc5270a443b791c62f634233c0f4966dfcc0dd6

ads/aqua/common/enums.py

@@ -28,7 +28,6 @@ class Tags(str, metaclass=ExtendedEnumMeta):
     TASK = "task"
     LICENSE = "license"
     ORGANIZATION = "organization"
-    PLATFORM = "platform"
     AQUA_TAG = "OCI_AQUA"
     AQUA_SERVICE_MODEL_TAG = "aqua_service_model"
     AQUA_FINE_TUNED_MODEL_TAG = "aqua_fine_tuned_model"
@@ -39,6 +38,7 @@ class Tags(str, metaclass=ExtendedEnumMeta):
     READY_TO_IMPORT = "ready_to_import"
     BASE_MODEL_CUSTOM = "aqua_custom_base_model"
     AQUA_EVALUATION_MODEL_ID = "evaluation_model_id"
+    MODEL_FORMAT = "model_format"
 
 
 class InferenceContainerType(str, metaclass=ExtendedEnumMeta):

ads/aqua/common/utils.py

@@ -10,15 +10,16 @@
 import os
 import random
 import re
+from datetime import datetime, timedelta
 from functools import wraps
 from pathlib import Path
 from string import Template
 from typing import List, Union
 
 import fsspec
-import oci
-from oci.data_science.models import JobRun, Model
+from cachetools import TTLCache, cached
 
+import oci
 from ads.aqua.common.enums import (
     InferenceContainerParamType,
     InferenceContainerType,
@@ -52,6 +53,8 @@
 from ads.common.utils import copy_file, get_console_link, upload_to_os
 from ads.config import AQUA_SERVICE_MODELS_BUCKET, CONDA_BUCKET_NS, TENANCY_OCID
 from ads.model import DataScienceModel, ModelVersionSet
+from oci.data_science.models import JobRun, Model
+from oci.object_storage.models import ObjectSummary
 
 logger = logging.getLogger("ads.aqua")
 
@@ -228,6 +231,32 @@ def load_config(file_path: str, config_file_name: str, **kwargs) -> dict:
     return config
 
 
+def list_os_files_with_extension(oss_path: str, extension: str) -> [str]:
+    """
+    List files in the specified directory with the given extension.
+
+    Parameters:
+    - oss_path: The path to the directory where files are located.
+    - extension: The file extension to filter by (e.g., 'txt' for text files).
+
+    Returns:
+    - A list of file paths matching the specified extension.
+    """
+
+    oss_client = ObjectStorageDetails.from_path(oss_path)
+
+    # Ensure the extension is prefixed with a dot if not already
+    if not extension.startswith("."):
+        extension = "." + extension
+    files: List[ObjectSummary] = oss_client.list_objects().objects
+
+    return [
+        file.name
+        for file in files
+        if file.name.endswith(extension) and "/" not in file.name
+    ]
+
+
 def is_valid_ocid(ocid: str) -> bool:
     """Checks if the given ocid is valid.
 
@@ -503,6 +532,7 @@ def container_config_path():
     return f"oci://{AQUA_SERVICE_MODELS_BUCKET}@{CONDA_BUCKET_NS}/service_models/config"
 
 
+@cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=5), timer=datetime.now))
 def get_container_config():
     config = load_config(
         file_path=container_config_path(),

ads/aqua/config/deployment_config_defaults.json

@@ -1,9 +1,36 @@
 {
+  "configuration": {
+    "VM.Standard.A1.Flex": {
+      "parameters": {},
+      "shape_info": {
+        "configs": [
+          {
+            "memory_in_gbs": 128,
+            "ocpu": 32
+          },
+          {
+            "memory_in_gbs": 256,
+            "ocpu": 64
+          },
+          {
+            "memory_in_gbs": 512,
+            "ocpu": 128
+          },
+          {
+            "memory_in_gbs": 1024,
+            "ocpu": 256
+          }
+        ],
+        "type": "CPU"
+      }
+    }
+  },
   "shape": [
     "VM.GPU.A10.1",
     "VM.GPU.A10.2",
     "BM.GPU.A10.4",
     "BM.GPU4.8",
-    "BM.GPU.A100-v2.8"
+    "BM.GPU.A100-v2.8",
+    "VM.Standard.A1.Flex"
   ]
 }

ads/aqua/constants.py

@@ -21,8 +21,6 @@
 DEFAULT_FT_REPLICA = 1
 DEFAULT_FT_BATCH_SIZE = 1
 DEFAULT_FT_VALIDATION_SET_SIZE = 0.1
-ARM_CPU="arm_cpu"
-NVIDIA_GPU="nvidia_gpu"
 MAXIMUM_ALLOWED_DATASET_IN_BYTE = 52428800  # 1024 x 1024 x 50 = 50MB
 JOB_INFRASTRUCTURE_TYPE_DEFAULT_NETWORKING = "ME_STANDALONE"
 NB_SESSION_IDENTIFIER = "NB_SESSION_OCID"
@@ -35,6 +33,7 @@
 AQUA_MODEL_ARTIFACT_CONFIG = "config.json"
 AQUA_MODEL_ARTIFACT_CONFIG_MODEL_NAME = "_name_or_path"
 AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE = "model_type"
+AQUA_MODEL_ARTIFACT_FILE = "model_file"
 
 TRAINING_METRICS_FINAL = "training_metrics_final"
 VALIDATION_METRICS_FINAL = "validation_metrics_final"

ads/aqua/extension/errors.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
@@ -8,3 +7,4 @@ class Errors(str):
     INVALID_INPUT_DATA_FORMAT = "Invalid format of input data."
     NO_INPUT_DATA = "No input data provided."
     MISSING_REQUIRED_PARAMETER = "Missing required parameter: '{}'"
+    MISSING_ONEOF_REQUIRED_PARAMETER = "Either '{}' or '{}' is required."

ads/aqua/extension/model_handler.py

@@ -1,27 +1,50 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-import re
-from typing import Optional
 from urllib.parse import urlparse
 
 from tornado.web import HTTPError
-from ads.aqua.extension.errors import Errors
+
 from ads.aqua.common.decorator import handle_exceptions
+from ads.aqua.common.errors import AquaValueError
 from ads.aqua.extension.base_handler import AquaAPIhandler
+from ads.aqua.extension.errors import Errors
 from ads.aqua.model import AquaModelApp
+from ads.aqua.ui import ModelFormat
 
 
 class AquaModelHandler(AquaAPIhandler):
     """Handler for Aqua Model REST APIs."""
 
     @handle_exceptions
-    def get(self, model_id=""):
+    def get(
+        self,
+        model_id="",
+    ):
         """Handle GET request."""
-        if not model_id:
+        url_parse = urlparse(self.request.path)
+        paths = url_parse.path.strip("/")
+        if paths.startswith("aqua/model/files"):
+            os_path = self.get_argument("os_path")
+            if not os_path:
+                raise HTTPError(
+                    400, Errors.MISSING_REQUIRED_PARAMETER.format("os_path")
+                )
+            model_format = self.get_argument("model_format")
+            if not model_format:
+                raise HTTPError(
+                    400, Errors.MISSING_REQUIRED_PARAMETER.format("model_format")
+                )
+            try:
+                model_format = ModelFormat(model_format.upper())
+            except ValueError:
+                raise AquaValueError(f"Invalid model format: {model_format}")
+            else:
+                return self.finish(AquaModelApp.get_model_files(os_path, model_format))
+        elif not model_id:
             return self.list()
+
         return self.read(model_id)
 
     def read(self, model_id):
@@ -81,6 +104,7 @@ def post(self, *args, **kwargs):
         finetuning_container = input_data.get("finetuning_container")
         compartment_id = input_data.get("compartment_id")
         project_id = input_data.get("project_id")
+        model_file = input_data.get("model_file")
 
         return self.finish(
             AquaModelApp().register(
@@ -90,6 +114,7 @@ def post(self, *args, **kwargs):
                 finetuning_container=finetuning_container,
                 compartment_id=compartment_id,
                 project_id=project_id,
+                model_file=model_file,
             )
         )
 

ads/aqua/model/entities.py

@@ -14,14 +14,14 @@
 from typing import List, Optional
 
 import oci
-
 from ads.aqua import logger
 from ads.aqua.app import CLIBuilderMixin
 from ads.aqua.common import utils
 from ads.aqua.constants import LIFECYCLE_DETAILS_MISSING_JOBRUN, UNKNOWN_VALUE
 from ads.aqua.data import AquaResourceIdentifier
 from ads.aqua.model.enums import FineTuningDefinedMetadata
 from ads.aqua.training.exceptions import exit_code_dict
+from ads.aqua.ui import ModelFormat
 from ads.common.serializer import DataClassSerializable
 from ads.common.utils import get_log_links
 from ads.model.datascience_model import DataScienceModel
@@ -41,6 +41,12 @@ class AquaFineTuneValidation(DataClassSerializable):
     value: str = ""
 
 
+class ModelValidationResult:
+    model_file: Optional[str] = None
+    model_format: ModelFormat = None
+    telemetry_model_name: str = None
+
+
 @dataclass(repr=False)
 class AquaFineTuningMetric(DataClassSerializable):
     name: str = field(default_factory=str)
@@ -76,7 +82,9 @@ class AquaModelSummary(DataClassSerializable):
     ready_to_deploy: bool = True
     ready_to_finetune: bool = False
     ready_to_import: bool = False
-    platform: List[str] = field(default_factory=lambda: ["nvidia_gpu"])
+    nvidia_gpu_supported: bool = False
+    arm_cpu_supported: bool = False
+    model_format: ModelFormat = ModelFormat.UNKNOWN
 
 
 @dataclass(repr=False)
@@ -260,6 +268,7 @@ class ImportModelDetails(CLIBuilderMixin):
     finetuning_container: Optional[str] = None
     compartment_id: Optional[str] = None
     project_id: Optional[str] = None
+    model_file: Optional[str] = None
 
     def __post_init__(self):
         self._command = "model register"