Merge branch 'main' of github.com:oracle/accelerated-data-science into bug_fix

Author: Ziqun Ye

.github/workflows/run-unittests-py38-cov-report.yml

@@ -57,17 +57,9 @@ jobs:
       - uses: ./.github/workflows/set-dummy-conf
         name: "Test config setup"
 
-      - name: "Test env setup"
+      - uses: ./.github/workflows/test-env-setup
+        name: "Test env setup"
         timeout-minutes: 20
-        shell: bash
-        run: |
-          set -x # print commands that are executed
-
-          sudo apt-get install libkrb5-dev graphviz
-          $CONDA/bin/conda init
-          source /home/runner/.bashrc
-
-          pip install -r dev-requirements.txt
 
       - name: "Run unitary tests folder with maximum ADS dependencies"
         timeout-minutes: 60

.github/workflows/run-unittests-py39-py310.yml

@@ -58,17 +58,9 @@ jobs:
       - uses: ./.github/workflows/set-dummy-conf
         name: "Test config setup"
 
-      - name: "Test env setup"
+      - uses: ./.github/workflows/test-env-setup
+        name: "Test env setup"
         timeout-minutes: 20
-        shell: bash
-        run: |
-          set -x # print commands that are executed
-
-          sudo apt-get install libkrb5-dev graphviz
-          $CONDA/bin/conda init
-          source /home/runner/.bashrc
-
-          pip install -r dev-requirements.txt
 
       - name: "Run unitary tests folder with maximum ADS dependencies"
         timeout-minutes: 60

.github/workflows/test-env-setup/action.yml

@@ -0,0 +1,18 @@
+# This composite action is to avoid duplicating code across test actions.
+# Ref: https://docs.github.com/en/actions/creating-actions/creating-a-composite-action
+
+name: "Test env setup"
+description: "Creates conda test env with optional dependencies to run all unit tests"
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      run: |
+        set -x # print commands that are executed
+
+        sudo apt update
+        sudo apt-get install libkrb5-dev graphviz
+        $CONDA/bin/conda init
+        source /home/runner/.bashrc
+
+        pip install -r dev-requirements.txt

ads/common/oci_logging.py

@@ -533,7 +533,7 @@ def _search_and_format(
         time_end: datetime.datetime = None,
         limit: int = LOG_RECORDS_LIMIT,
         sort_by: str = "datetime",
-        sort_order: str = SortOrder.DESC,
+        sort_order: str = SortOrder.ASC,
         log_filter: str = None,
     ):
         """Returns the formatted log records.
@@ -550,7 +550,7 @@ def _search_and_format(
             Maximum number of records to be returned.
         sort_by: (str, optional). Defaults to "datetime"
             The field for sorting the logs.
-        sort_order: (str, optional). Defaults to "DESC".
+        sort_order: (str, optional). Defaults to "ASC".
             The sort order for the log records. Can be "ASC" or "DESC".
         log_filter : (str, optional). Defaults to None.
             Expression for filtering the logs.
@@ -574,8 +574,8 @@ def _search_and_format(
             sort_order=sort_order,
             log_filter=log_filter,
         )
-        logs = sorted((log.data for log in logs), key=lambda x: x.get("datetime"))
-        logs = [log.get("logContent", {}) for log in logs]
+
+        logs = [log.data.get("logContent", {}) for log in logs]
         return [
             {
                 "id": log.get("id"),
@@ -617,7 +617,7 @@ def tail(
         return self._search_and_format(
             source=source,
             limit=limit,
-            sort_order=SortOrder.DESC,
+            sort_order=SortOrder.ASC,
             time_start=time_start,
             log_filter=log_filter,
         )
@@ -808,7 +808,7 @@ def stream(
                 logs = self._search_and_format(
                     source=source,
                     limit=None,
-                    sort_order=SortOrder.DESC,
+                    sort_order=SortOrder.ASC,
                     time_start=time_start,
                     log_filter=log_filter,
                 )
@@ -858,7 +858,7 @@ def tail(
             self._search_and_format(
                 source=source,
                 limit=limit,
-                sort_order=SortOrder.DESC,
+                sort_order=SortOrder.ASC,
                 time_start=time_start,
                 log_filter=log_filter,
             )
@@ -927,7 +927,7 @@ def search(
         time_end: datetime.datetime = None,
         limit: int = None,
         sort_by: str = "datetime",
-        sort_order: str = SortOrder.DESC,
+        sort_order: str = SortOrder.ASC,
         log_filter: str = None,
     ) -> List[oci.loggingsearch.models.SearchResult]:
         """Searches raw logs.
@@ -951,7 +951,7 @@ def search(
             Defaults to "datetime"
         sort_order : str, optional.
             The sort order for the log records. Can be "ASC" or "DESC".
-            Defaults to "DESC".
+            Defaults to "ASC".
         log_filter : str, optional
             Expression for filtering the logs. This will be the WHERE clause of the query.
             Defaults to None.
@@ -979,7 +979,7 @@ def _search_and_format(
         time_end: datetime.datetime = None,
         limit: int = LOG_RECORDS_LIMIT,
         sort_by: str = "datetime",
-        sort_order: str = SortOrder.DESC,
+        sort_order: str = SortOrder.ASC,
         log_filter: str = None,
         need_format: bool = True,
     ) -> List[Union[oci.loggingsearch.models.SearchResult, dict]]:
@@ -1005,7 +1005,7 @@ def _search_and_format(
             Defaults to "datetime"
         sort_order : str, optional.
             The sort order for the log records. Can be "ASC" or "DESC".
-            Defaults to "DESC".
+            Defaults to "ASC".
         log_filter : str, optional
             Expression for filtering the logs. This will be the WHERE clause of the query.
             Defaults to None.
@@ -1039,12 +1039,6 @@ def _search_and_format(
                 )
             )
 
-        # _collect_logs returns a list of either dict or oci.loggingsearch.models.SearchResult
-        # objects based on `need_format` parameter, so below there are two cases for log sorting.
-        if need_format:
-            batch_logs.sort(key=lambda x: x.get("datetime"))
-        else:
-            batch_logs.sort(key=lambda x: x.data.get("datetime"))
         if limit and len(batch_logs) > limit:
             batch_logs = batch_logs[:limit]
         return batch_logs
@@ -1057,7 +1051,7 @@ def _collect_logs(
         time_end: datetime.datetime = None,
         limit: int = LOG_RECORDS_LIMIT,
         sort_by: str = "datetime",
-        sort_order: str = SortOrder.DESC,
+        sort_order: str = SortOrder.ASC,
         log_filter: str = None,
         need_format: bool = True,
     ) -> List[Union[oci.loggingsearch.models.SearchResult, dict]]:
@@ -1085,7 +1079,7 @@ def _collect_logs(
             Defaults to "datetime"
         sort_order : str, optional.
             The sort order for the log records. Can be "ASC" or "DESC".
-            Defaults to "DESC".
+            Defaults to "ASC".
         log_filter : str, optional
             Expression for filtering the logs. This will be the WHERE clause of the query.
             Defaults to None.

ads/common/utils.py

@@ -71,7 +71,7 @@
 MAX_DISPLAY_VALUES = 10
 
 # par link of the index json file.
-PAR_LINK = "https://objectstorage.us-ashburn-1.oraclecloud.com/p/Ri7zFc_h91sxMdgnza9Qnqw3Ina8hf8wzDvEpAnUXMDOnUR1U1fpsaBUjUfgPgIq/n/ociodscdev/b/service-conda-packs/o/service_pack/index.json"
+PAR_LINK = "https://objectstorage.us-ashburn-1.oraclecloud.com/p/WyjtfVIG0uda-P3-2FmAfwaLlXYQZbvPZmfX1qg0-sbkwEQO6jpwabGr2hMDBmBp/n/ociodscdev/b/service-conda-packs/o/service_pack/index.json"
 
 random_state = 42
 test_size = 0.3

ads/jobs/serializer.py

@@ -12,11 +12,9 @@
 import yaml
 from ads.common.auth import default_signer
 
+# Special type to represent the current enclosed class.
+# This type is used by factory class method or when a method returns ``self``.
 Self = TypeVar("Self", bound="Serializable")
-"""Special type to represent the current enclosed class.
-
-This type is used by factory class method or when a method returns ``self``.
-"""
 
 
 class Serializable(ABC):
@@ -72,6 +70,14 @@ def _write_to_file(s: str, uri: str, **kwargs) -> None:
                     "if you wish to overwrite."
                 )
 
+        # Add default signer if the uri is an object storage uri, and
+        # the user does not specify config or signer.
+        if (
+            uri.startswith("oci://")
+            and "config" not in kwargs
+            and "signer" not in kwargs
+        ):
+            kwargs.update(default_signer())
         with fsspec.open(uri, "w", **kwargs) as f:
             f.write(s)
 

ads/model/runtime/utils.py

@@ -111,7 +111,7 @@ def _get_index_json_through_bucket(
     bucketname: str
         The Object Storage bucketname.
     auth: (Dict, optional). Defaults to None.
-        The default authetication is set using `ads.set_auth` API. If you need to override the
+        The default authentication is set using `ads.set_auth` API. If you need to override the
         default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
         authentication signer and kwargs required to instantiate IdentityClient object.
 
@@ -147,7 +147,7 @@ def get_service_packs(
     bucketname: str
         bucketname of the service pack.
     auth: (Dict, optional). Defaults to None.
-        The default authetication is set using `ads.set_auth` API. If you need to override the
+        The default authentication is set using `ads.set_auth` API. If you need to override the
         default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
         authentication signer and kwargs required to instantiate IdentityClient object.
 

ads/opctl/config/merger.py

@@ -117,7 +117,7 @@ def _fill_config_with_defaults(self, ads_config_path: str) -> None:
             else:
                 self.config["execution"]["auth"] = AuthType.API_KEY
         # determine profile
-        if self.config["execution"]["auth"] == AuthType.RESOURCE_PRINCIPAL:
+        if self.config["execution"]["auth"] != AuthType.API_KEY:
             profile = self.config["execution"]["auth"].upper()
             exec_config.pop("oci_profile", None)
             self.config["execution"]["oci_profile"] = None
@@ -202,20 +202,23 @@ def _get_service_config(self, oci_profile: str, ads_config_folder: str) -> Dict:
     def _config_flex_shape_details(self):
         infrastructure = self.config["infrastructure"]
         backend = self.config["execution"].get("backend", None)
-        if backend == BACKEND_NAME.JOB.value or backend == BACKEND_NAME.MODEL_DEPLOYMENT.value:
+        if (
+            backend == BACKEND_NAME.JOB.value
+            or backend == BACKEND_NAME.MODEL_DEPLOYMENT.value
+        ):
             shape_name = infrastructure.get("shape_name", "")
             if shape_name.endswith(".Flex"):
                 if (
-                    "ocpus" not in infrastructure or 
-                    "memory_in_gbs" not in infrastructure
+                    "ocpus" not in infrastructure
+                    or "memory_in_gbs" not in infrastructure
                 ):
                     raise ValueError(
                         "Parameters `ocpus` and `memory_in_gbs` must be provided for using flex shape. "
                         "Call `ads opctl config` to specify."
                     )
                 infrastructure["shape_config_details"] = {
                     "ocpus": infrastructure.pop("ocpus"),
-                    "memory_in_gbs": infrastructure.pop("memory_in_gbs")
+                    "memory_in_gbs": infrastructure.pop("memory_in_gbs"),
                 }
         elif backend == BACKEND_NAME.DATAFLOW.value:
             executor_shape = infrastructure.get("executor_shape", "")
@@ -224,7 +227,7 @@ def _config_flex_shape_details(self):
                 "driver_shape_memory_in_gbs",
                 "driver_shape_ocpus",
                 "executor_shape_memory_in_gbs",
-                "executor_shape_ocpus"
+                "executor_shape_ocpus",
             ]
             # executor_shape and driver_shape must be the same shape family
             if executor_shape.endswith(".Flex") or driver_shape.endswith(".Flex"):
@@ -236,9 +239,9 @@ def _config_flex_shape_details(self):
                         )
                 infrastructure["driver_shape_config"] = {
                     "ocpus": infrastructure.pop("driver_shape_ocpus"),
-                    "memory_in_gbs": infrastructure.pop("driver_shape_memory_in_gbs")
+                    "memory_in_gbs": infrastructure.pop("driver_shape_memory_in_gbs"),
                 }
                 infrastructure["executor_shape_config"] = {
                     "ocpus": infrastructure.pop("executor_shape_ocpus"),
-                    "memory_in_gbs": infrastructure.pop("executor_shape_memory_in_gbs")
+                    "memory_in_gbs": infrastructure.pop("executor_shape_memory_in_gbs"),
                 }

ads/opctl/utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2022 Oracle and/or its affiliates.
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
@@ -89,9 +89,8 @@ def get_namespace(auth: dict) -> str:
 
 
 def get_region_key(auth: dict) -> str:
-    if len(auth["config"]) > 0:
-        tenancy = auth["config"]["tenancy"]
-    else:
+    tenancy = auth["config"].get("tenancy")
+    if not tenancy:
         tenancy = auth["signer"].tenancy_id
     client = OCIClientFactory(**auth).identity
     return client.get_tenancy(tenancy).data.home_region_key