ODSC-37692:[MLFlow][Projects] Improve ADS OPCTL to be able to generat… (#161)

Author: mrDzurb

.pre-commit-config.yaml

@@ -0,0 +1,27 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    -   id: check-ast
+    -   id: check-docstring-first
+        exclude: ^tests/
+    -   id: check-json
+    -   id: check-merge-conflict
+    -   id: check-yaml
+        args: ['--allow-multiple-documents']
+    -   id: detect-private-key
+    -   id: end-of-file-fixer
+    -   id: pretty-format-json
+        args: ['--autofix']
+    -   id: trailing-whitespace
+-   repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+    -   id: black
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.9.0
+    hooks:
+    -   id: rst-backticks
+    -   id: rst-inline-touching-normal
+
+exclude: ^(docs/)

ads/__init__.py

@@ -122,13 +122,8 @@ def _set_test_mode(mode=False):
 
 
 def hello():
-    """
-    Imports Pandas, sets the documentation mode, and prints a fancy "Hello".
-    """
-    import pandas
-
-    global documentation_mode
-    global debug_mode
+    import oci
+    import ocifs
 
     print(
         f"""
@@ -139,11 +134,11 @@ def hello():
 |   ||  /     |
 o   oo-o  o--o
 
-ADS SDK version: {__version__}
-Pandas version: {pandas.__version__}
-Debug mode: {debug_mode}
-"""
-    )
+ads v{__version__}
+oci v{oci.__version__}
+ocifs v{ocifs.__version__}
+
+""")
 
 
 configure_plotting()

ads/cli.py

@@ -3,17 +3,23 @@
 
 # Copyright (c) 2021, 2022 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+from ads.common import logger
+import traceback
+
 try:
     import click
     import ads.opctl.cli
     import ads.jobs.cli
     import ads.pipeline.cli
     import os
     import json
-except:
+except Exception as ex:
     print(
-        "Please run `pip install oracle-ads[opctl]` to install the required dependencies for ADS CLI"
+        "Please run `pip install oracle-ads[opctl]` to install "
+        "the required dependencies for ADS CLI."
     )
+    logger.debug(ex)
+    logger.debug(traceback.format_exc())
     exit()
 
 

ads/common/extended_enum.py

@@ -6,6 +6,7 @@
 
 
 from abc import ABCMeta
+from enum import Enum
 
 
 class ExtendedEnumMeta(ABCMeta):
@@ -15,13 +16,20 @@ class ExtendedEnumMeta(ABCMeta):
     -------
     values(cls) -> list:
         Gets the list of class attributes.
+
+    Examples
+    --------
+    >>> class TestEnum(str, metaclass=ExtendedEnumMeta):
+    ...    KEY1 = "value1"
+    ...    KEY2 = "value2"
+    >>> print(TestEnum.KEY1) # "value1"
     """
 
     def __contains__(cls, value):
         return value and value.lower() in tuple(value.lower() for value in cls.values())
 
     def values(cls) -> list:
-        """Gets the list of class attributes.
+        """Gets the list of class attributes values.
 
         Returns
         -------
@@ -31,3 +39,35 @@ def values(cls) -> list:
         return tuple(
             value for key, value in cls.__dict__.items() if not key.startswith("_")
         )
+
+    def keys(cls) -> list:
+        """Gets the list of class attributes names.
+
+        Returns
+        -------
+        list
+            The list of class attributes names.
+        """
+        return tuple(
+            key for key, value in cls.__dict__.items() if not key.startswith("_")
+        )
+
+
+class ExtendedEnum(Enum):
+    """The base class to extend functionality of a generic Enum.
+
+    Examples
+    --------
+    >>> class TestEnum(ExtendedEnumMeta):
+    ...    KEY1 = "value1"
+    ...    KEY2 = "value2"
+    >>> print(TestEnum.KEY1.value) # "value1"
+    """
+
+    @classmethod
+    def values(cls):
+        return sorted(map(lambda c: c.value, cls))
+
+    @classmethod
+    def keys(cls):
+        return sorted(map(lambda c: c.name, cls))

ads/common/serializer.py

@@ -4,14 +4,16 @@
 # Copyright (c) 2021, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
+import dataclasses
 import json
 from abc import ABC, abstractmethod
-import dataclasses
-from typing import Dict, Union, Optional, List
 from enum import Enum
+from typing import Dict, Optional, Union
+from urllib.parse import urlparse
 
 import fsspec
 import yaml
+
 from ads.common import logger
 
 try:
@@ -53,15 +55,19 @@ class Serializable(ABC):
     """
 
     @abstractmethod
-    def to_dict(self, **kwargs) -> dict:
-        """Serializes instance of class into a dictionary.
+    def to_dict(self, **kwargs: Dict) -> Dict:
+        """Serializes an instance of class into a dictionary.
+
+        Parameters
+        ----------
+        **kwargs: Dict
+            Additional arguments.
 
         Returns
         -------
         Dict
-            A dictionary.
+            The result dictionary.
         """
-        pass
 
     @classmethod
     @abstractmethod
@@ -90,17 +96,25 @@ def _write_to_file(s: str, uri: str, **kwargs) -> None:
             content
         uri: (string)
             URI location to save string s
-
-        kwargs
-        ------
-        keyword arguments to be passed into fsspec.open(). For OCI object storage, this should be config="path/to/.oci/config".
-            For other storage connections consider e.g. host, port, username, password, etc.
+        kwargs : dict
+            keyword arguments to be passed into fsspec.open().
+            For OCI object storage, this can be config="path/to/.oci/config".
 
         Returns
         -------
         None
-            Nothing.
+            Nothing
         """
+
+        overwrite = kwargs.pop("overwrite", True)
+        if not overwrite:
+            dst_path_scheme = urlparse(uri).scheme or "file"
+            if fsspec.filesystem(dst_path_scheme, **kwargs).exists(uri):
+                raise FileExistsError(
+                    f"The `{uri}` is already exists. Set `overwrite` to True "
+                    "if you wish to overwrite."
+                )
+
         with fsspec.open(uri, "w", **kwargs) as f:
             f.write(s)
 
@@ -112,15 +126,13 @@ def _read_from_file(uri: str, **kwargs) -> str:
         ----------
         uri: (string)
             URI location
-
-        kwargs
-        ------
-        keyword arguments to be passed into fsspec.open(). For OCI object storage, this should be config="path/to/.oci/config".
-            For other storage connections consider e.g. host, port, username, password, etc.
+        kwargs : dict
+            keyword arguments to be passed into fsspec.open().
+            For OCI object storage, this can be config="path/to/.oci/config".
 
         Returns
         -------
-            string: Contents in file specified by URI
+        string: Contents in file specified by URI
         """
         with fsspec.open(uri, "r", **kwargs) as f:
             return f.read()
@@ -139,15 +151,23 @@ def to_json(
 
         kwargs
         ------
-        keyword arguments to be passed into fsspec.open(). For OCI object storage, this should be config="path/to/.oci/config".
-            For other storage connections consider e.g. host, port, username, password, etc.
+        overwrite: (bool, optional). Defaults to True.
+            Whether to overwrite existing file or not.
 
-        Returns:
-            string: Serialized version of object
+        keyword arguments to be passed into fsspec.open().
+        For OCI object storage, this could be config="path/to/.oci/config".
+        For other storage connections consider e.g. host, port, username, password, etc.
+
+        Returns
+        -------
+        Union[str, None]
+            Serialized version of object.
+            `None` in case when `uri` provided.
         """
         json_string = json.dumps(self.to_dict(**kwargs), cls=encoder)
         if uri:
             self._write_to_file(s=json_string, uri=uri, **kwargs)
+            return None
         return json_string
 
     @classmethod
@@ -190,31 +210,40 @@ def from_json(
             return cls.from_dict(json_dict)
         raise ValueError("Must provide either JSON string or URI location")
 
-    def to_yaml(self, uri: str = None, dumper: callable = dumper, **kwargs) -> str:
+    def to_yaml(
+        self, uri: str = None, dumper: callable = yaml.SafeDumper, **kwargs
+    ) -> Union[str, None]:
         """Returns object serialized as a YAML string
 
         Parameters
         ----------
-        uri: (string, optional)
-            URI location to save the YAML string. Defaults to None.
-        dumper: (callable, optional)
-            Custom YAML Dumper. Defaults to CDumper/SafeDumper.
-
-        kwargs
-        ------
-        side_effect: Optional[SideEffect]
-            side effect to take on the dictionary. The side effect can be either
-            convert the dictionary keys to "lower" (SideEffect.CONVERT_KEYS_TO_LOWER.value)
-            or "upper"(SideEffect.CONVERT_KEYS_TO_UPPER.value) cases.
-        keyword arguments to be passed into fsspec.open(). For OCI object storage, this should be config="path/to/.oci/config".
-            For other storage connections consider e.g. host, port, username, password, etc.
+        uri : str, optional
+            URI location to save the YAML string, by default None
+        dumper : callable, optional
+            Custom YAML Dumper, by default yaml.SafeDumper
+        kwargs : dict
+            overwrite: (bool, optional). Defaults to True.
+                Whether to overwrite existing file or not.
+            note: (str, optional)
+                The note that needs to be added in the beginning of the YAML.
+                It will be added as is without any formatting.
+            side_effect: Optional[SideEffect]
+                side effect to take on the dictionary. The side effect can be either
+                convert the dictionary keys to "lower" (SideEffect.CONVERT_KEYS_TO_LOWER.value)
+                or "upper"(SideEffect.CONVERT_KEYS_TO_UPPER.value) cases.
+
+            The other keyword arguments to be passed into fsspec.open().
+            For OCI object storage, this could be config="path/to/.oci/config".
 
-        Returns:
-            Union[str, None]
-                Serialized version of object.
-                None in case when `uri` provided.
+        Returns
+        -------
+        Union[str, None]
+            Serialized version of object.
+            `None` in case when `uri` provided.
         """
-        yaml_string = yaml.dump(self.to_dict(**kwargs), Dumper=dumper)
+        note = kwargs.pop("note", "")
+
+        yaml_string = f"{note}\n" + yaml.dump(self.to_dict(**kwargs), Dumper=dumper)
         if uri:
             self._write_to_file(s=yaml_string, uri=uri, **kwargs)
             return None

ads/jobs/ads_job.py

@@ -3,7 +3,7 @@
 
 # Copyright (c) 2021, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-from typing import List, Union
+from typing import List, Union, Dict
 from urllib.parse import urlparse
 
 import fsspec
@@ -359,6 +359,8 @@ def with_name(self, name: str) -> "Job":
 
     def build(self) -> "Job":
         """Load default values from the environment for the job infrastructure."""
+        super().build()
+
         build_method = getattr(self.infrastructure, "build", None)
         if build_method and callable(build_method):
             build_method()
@@ -457,19 +459,27 @@ def status(self) -> str:
         """
         return getattr(self.infrastructure, "status", None)
 
-    def to_dict(self) -> dict:
+    def to_dict(self, **kwargs: Dict) -> Dict:
         """Serialize the job specifications to a dictionary.
 
+        Parameters
+        ----------
+        **kwargs: Dict
+            The additional arguments.
+            - filter_by_attribute_map: bool
+                If True, then in the result will be included only the fields
+                presented in the `attribute_map`.
+
         Returns
         -------
-        dict
+        Dict
             A dictionary containing job specifications.
         """
         spec = {"name": self.name}
         if self.runtime:
-            spec["runtime"] = self.runtime.to_dict()
+            spec["runtime"] = self.runtime.to_dict(**kwargs)
         if self.infrastructure:
-            spec["infrastructure"] = self.infrastructure.to_dict()
+            spec["infrastructure"] = self.infrastructure.to_dict(**kwargs)
         if self.id:
             spec["id"] = self.id
         return {
@@ -495,7 +505,7 @@ def from_dict(cls, config: dict) -> "Job":
         Raises
         ------
         NotImplementedError
-            If the type of the intrastructure or runtime is not supported.
+            If the type of the infrastructure or runtime is not supported.
         """
         if not isinstance(config, dict):
             raise ValueError("The config data for initializing the job is invalid.")