Merge branch 'main' of github.com:oracle/accelerated-data-science into fix_doc

Author: Ziqun Ye

.github/workflows/run-unittests-default_setup.yml

@@ -2,12 +2,10 @@ name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"
 
 on:
   workflow_dispatch:
+  push:
   pull_request:
-    paths:
-      - "ads/**"
-      - pyproject.toml
-      - "**requirements.txt"
-      - ".github/workflows/run-unittests*.yml"
+    types: [opened, reopened]
+    branches: [ "main" ]
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value

.github/workflows/run-unittests-py38-cov-report.yml

@@ -2,12 +2,10 @@ name: "[Py3.8][COV REPORT] tests/unitary/**"
 
 on:
   workflow_dispatch:
+  push:
   pull_request:
-    paths:
-      - "ads/**"
-      - pyproject.toml
-      - "**requirements.txt"
-      - ".github/workflows/run-unittests*.yml"
+    types: [opened, reopened]
+    branches: [ "main" ]
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -71,7 +69,9 @@ jobs:
       - name: "Install Forecasting dependencies"
         run: |
           pip install -e ".[forecast]"
-
+      - name: "Install featurestore marketplace dependencies"
+        run: |
+          pip install -e ".[feature-store-marketplace]"
       - name: "Run unitary tests folder with maximum ADS dependencies"
         timeout-minutes: 40
         shell: bash
@@ -114,7 +114,7 @@ jobs:
     runs-on: ubuntu-latest
     continue-on-error: true
     needs: test
-    if: ${{ success() }} && && github.event_name == 'pull_request'
+    if: ${{ success() }} && github.event_name == 'pull_request'
     env:
       COMPARE_BRANCH: main
 
@@ -153,7 +153,6 @@ jobs:
           fi
 
       - name: "Calculate coverage diff"
-        if: always()
         run: |
           set -x # print commands that are executed
 
@@ -184,7 +183,7 @@ jobs:
 
       - name: "Add comment with cov diff to PR"
         uses: actions/github-script@v6
-        if: always()
+        if: github.event_name == 'pull_request'
         with:
           github-token: ${{ github.token }}
           script: |

.github/workflows/run-unittests-py39-py310.yml

@@ -2,12 +2,10 @@ name: "[Py3.9][Py3.10] - tests/unitary/**"
 
 on:
   workflow_dispatch:
+  push:
   pull_request:
-    paths:
-      - "ads/**"
-      - pyproject.toml
-      - "**requirements.txt"
-      - ".github/workflows/run-unittests*.yml"
+    types: [opened, reopened]
+    branches: [ "main" ]
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value

.pre-commit-config.yaml

@@ -1,52 +1,52 @@
 repos:
-# Standard hooks
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
-    hooks:
-    -   id: check-ast
-        exclude: ^docs/
-    -   id: check-docstring-first
-        exclude: ^(docs/|tests/)
-    -   id: check-json
-    -   id: check-merge-conflict
-    -   id: check-yaml
-        exclude: feature_store_*.yaml
-        args: ['--allow-multiple-documents']
-    -   id: detect-private-key
-    -   id: end-of-file-fixer
-        exclude: '\.ipynb?$'
-    -   id: pretty-format-json
-        args: ['--autofix']
-    -   id: trailing-whitespace
-        args: [--markdown-linebreak-ext=md]
-        exclude: ^docs/
-# Black, the code formatter, natively supports pre-commit
--   repo: https://github.com/psf/black
-    rev: 23.3.0
-    hooks:
-    -   id: black
-        exclude: ^docs/
-# Regex based rst files common mistakes detector
--   repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.10.0
-    hooks:
-    -   id: rst-backticks
-        files: ^docs/
-    -   id: rst-inline-touching-normal
-        files: ^docs/
-# Hardcoded secrets and ocids detector
--   repo: https://github.com/gitleaks/gitleaks
-    rev: v8.17.0
-    hooks:
-    -   id: gitleaks
-        exclude: .github/workflows/reusable-actions/set-dummy-conf.yml
-# Oracle copyright checker
--   repo: https://github.com/oracle-samples/oci-data-science-ai-samples/
-    rev: cbe0136f7aaffe463b31ddf3f34b0e16b4b124ff
-    hooks:
-    -   id: check-copyright
-        name: check-copyright
-        entry: .pre-commit-scripts/check-copyright.py
-        language: script
-        types_or: ['python', 'shell', 'bash']
-        exclude: ^docs/
+    # Standard hooks
+    - repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v4.4.0
+      hooks:
+          - id: check-ast
+            exclude: ^docs/
+          - id: check-docstring-first
+            exclude: ^(docs/|tests/)
+          - id: check-json
+          - id: check-merge-conflict
+          - id: check-yaml
+            exclude: feature_store_*.yaml
+            args: ["--allow-multiple-documents"]
+          - id: detect-private-key
+          - id: end-of-file-fixer
+            exclude: '\.ipynb?$'
+          - id: pretty-format-json
+            args: ["--autofix"]
+          - id: trailing-whitespace
+            args: [--markdown-linebreak-ext=md]
+            exclude: ^docs/
+    # Black, the code formatter, natively supports pre-commit
+    - repo: https://github.com/psf/black
+      rev: 23.3.0
+      hooks:
+          - id: black
+            exclude: ^docs/
+    # Regex based rst files common mistakes detector
+    - repo: https://github.com/pre-commit/pygrep-hooks
+      rev: v1.10.0
+      hooks:
+          - id: rst-backticks
+            files: ^docs/
+          - id: rst-inline-touching-normal
+            files: ^docs/
+    # Hardcoded secrets and ocids detector
+    - repo: https://github.com/gitleaks/gitleaks
+      rev: v8.17.0
+      hooks:
+          - id: gitleaks
+            exclude: .github/workflows/reusable-actions/set-dummy-conf.yml
+    # Oracle copyright checker
+    - repo: https://github.com/oracle-samples/oci-data-science-ai-samples/
+      rev: 1bc5270a443b791c62f634233c0f4966dfcc0dd6
+      hooks:
+          - id: check-copyright
+            name: check-copyright
+            entry: .pre-commit-scripts/check-copyright.py
+            language: script
+            types_or: ["python", "shell", "bash"]
+            exclude: ^docs/

ads/common/decorator/runtime_dependency.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 """
@@ -65,6 +65,7 @@ class OptionalDependency:
     SPARK = "oracle-ads[spark]"
     HUGGINGFACE = "oracle-ads[huggingface]"
     FORECAST = "oracle-ads[forecast]"
+    FEATURE_STORE_MARKETPLACE = "oracle-ads[feature-store-marketplace]"
     PII = "oracle-ads[pii]"
     FEATURE_STORE = "oracle-ads[feature-store]"
     GRAPHVIZ = "oracle-ads[graphviz]"

ads/common/dsc_file_system.py

@@ -265,8 +265,6 @@ def update_from_dsc_model(cls, dsc_model) -> dict:
 
 class DSCFileSystemManager:
 
-    storage_mount_dest = set()
-
     @classmethod
     def initialize(cls, arguments: dict) -> dict:
         """Initialize and update arguments to dsc model.
@@ -286,12 +284,6 @@ def initialize(cls, arguments: dict) -> dict:
                 "Parameter `dest` is required for mounting file storage system."
             )
 
-        if arguments["dest"] in cls.storage_mount_dest:
-            raise ValueError(
-                "Duplicate `dest` found. Please specify different `dest` for each file system to be mounted."
-            )
-        cls.storage_mount_dest.add(arguments["dest"])
-
         # case oci://bucket@namespace/prefix
         if arguments["src"].startswith("oci://") and "@" in arguments["src"]:
             return OCIObjectStorage(**arguments).update_to_dsc_model()

ads/common/extended_enum.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2022 Oracle and/or its affiliates.
+# Copyright (c) 2022, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
@@ -70,4 +70,4 @@ def values(cls):
 
     @classmethod
     def keys(cls):
-        return sorted(map(lambda c: c.name, cls))
+        return sorted(map(lambda c: c.name, cls))

ads/common/oci_client.py

@@ -1,18 +1,21 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import logging
 
+import oci.artifacts
 from oci.ai_language import AIServiceLanguageClient
+from oci.artifacts import ArtifactsClient
 from oci.data_catalog import DataCatalogClient
 from oci.data_flow import DataFlowClient
 from oci.data_labeling_service import DataLabelingManagementClient
 from oci.data_labeling_service_dataplane import DataLabelingClient
 from oci.data_science import DataScienceClient
 from oci.identity import IdentityClient
+from oci.marketplace import MarketplaceClient
 from oci.object_storage import ObjectStorageClient
 from oci.resource_search import ResourceSearchClient
 from oci.secrets import SecretsClient
@@ -25,7 +28,7 @@ class OCIClientFactory:
 
     """
     A factory class to create OCI client objects. The constructor takes in config, signer and client_kwargs. `client_kwargs` is passed
-    to the client constructor as key word argutments.
+    to the client constructor as key word arguments.
 
     Examples
     --------
@@ -48,12 +51,15 @@ class OCIClientFactory:
     oc.OCIClientFactory(**auth).object_storage # Creates Object storage client using instance principal authentication
     """
 
-    def __init__(self, config={}, signer=None, client_kwargs=None):
+    def __init__(self, config=None, signer=None, client_kwargs=None):
+        if not config:
+            config = {}
         self.config = config
         self.signer = signer
         self.client_kwargs = client_kwargs
 
-    def _client_impl(self, client):
+    @staticmethod
+    def _client_impl(client):
         client_map = {
             "object_storage": ObjectStorageClient,
             "data_science": DataScienceClient,
@@ -66,21 +72,17 @@ def _client_impl(self, client):
             "data_labeling_cp": DataLabelingManagementClient,
             "resource_search": ResourceSearchClient,
             "data_catalog": DataCatalogClient,
+            "marketplace": MarketplaceClient,
+            "artifacts": ArtifactsClient,
         }
-        try:
-            from oci.feature_store import FeatureStoreClient
-
-            client_map["feature_store"] = FeatureStoreClient
-        except ImportError:
-            logger.debug("OCI SDK with feature store support is not installed")
-            pass
 
         assert (
             client in client_map
         ), f"Invalid client name. Client name not found in {client_map.keys()}"
         return client_map[client]
 
-    def _validate_auth_param(self, auth):
+    @staticmethod
+    def _validate_auth_param(auth):
         if not isinstance(auth, dict):
             raise ValueError("auth parameter should be of type dictionary")
         if "config" in auth and not isinstance(auth["config"], dict):
@@ -135,10 +137,6 @@ def ai_language(self):
     def data_labeling_cp(self):
         return self.create_client("data_labeling_cp")
 
-    @property
-    def feature_store(self):
-        return self.create_client("feature_store")
-
     @property
     def data_labeling_dp(self):
         return self.create_client("data_labeling_dp")
@@ -150,3 +148,11 @@ def resource_search(self):
     @property
     def data_catalog(self):
         return self.create_client("data_catalog")
+
+    @property
+    def marketplace(self):
+        return self.create_client("marketplace")
+
+    @property
+    def artifacts(self) -> oci.artifacts.ArtifactsClient:
+        return self.create_client("artifacts")

ads/common/oci_datascience.py

@@ -1,17 +1,28 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
+import os
+
 import oci.data_science
-from ads.common.oci_mixin import OCIModelMixin
+
 from ads.common.decorator.utils import class_or_instance_method
+from ads.common.oci_mixin import OCIModelMixin
+
+ENV_VAR_OCI_ODSC_SERVICE_ENDPOINT = "OCI_ODSC_SERVICE_ENDPOINT"
 
 
 class OCIDataScienceMixin(OCIModelMixin):
     @class_or_instance_method
     def init_client(cls, **kwargs) -> oci.data_science.DataScienceClient:
+        client_kwargs = kwargs.get("client_kwargs", {})
+        if os.environ.get(ENV_VAR_OCI_ODSC_SERVICE_ENDPOINT):
+            client_kwargs.update(
+                dict(service_endpoint=os.environ.get(ENV_VAR_OCI_ODSC_SERVICE_ENDPOINT))
+            )
+            kwargs.update(client_kwargs)
         return cls._init_client(client=oci.data_science.DataScienceClient, **kwargs)
 
     @property

ads/common/oci_logging.py

@@ -907,8 +907,12 @@ def _print_log_annotation_message(self) -> None:
 
     def _print(self, logs: List[Dict]) -> None:
         self._print_log_annotation_message()
+        printed = set()
         for log in logs:
-            self._print_log_details(log)
+            log_id = log.get("id")
+            if log_id not in printed:
+                self._print_log_details(log)
+                printed.add(log_id)
 
     @staticmethod
     def _print_log_details(log) -> None: