Merge branch 'develop' of https://github.com/oracle/accelerated-data-science into ODSC-37692/opctl_yaml_generator

Author: mrDzurb

.github/workflows/run-unittests-default_setup.yml

@@ -4,12 +4,14 @@ on:
   pull_request:
     branches:
       - main
-      - 'release/**'
+      - "release/**"
       - develop
     paths:
-      - 'ads/**'
+      - "ads/**"
       - setup.py
-      - '**requirements.txt'
+      - "**requirements.txt"
+      - .github/workflows/run-unittests.yml
+      - .github/workflows/run-unittests-default_setup.yml
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -74,7 +76,6 @@ jobs:
         timeout-minutes: 15
         shell: bash
         env:
-          NB_SESSION_COMPARTMENT_OCID: ocid1.compartment.oc1.<unique_ocid>
           NoDependency: True
         run: |
           set -x # print commands that are executed

.github/workflows/run-unittests.yml

@@ -4,12 +4,14 @@ on:
   pull_request:
     branches:
       - main
-      - 'release/**'
+      - "release/**"
       - develop
     paths:
-      - 'ads/**'
+      - "ads/**"
       - setup.py
-      - '**requirements.txt'
+      - "**requirements.txt"
+      - .github/workflows/run-unittests.yml
+      - .github/workflows/run-unittests-default_setup.yml
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -85,22 +87,21 @@ jobs:
         shell: bash
         run: |
           set -x # print commands that are executed
-          
+
           sudo apt-get install libkrb5-dev graphviz
           $CONDA/bin/conda init
           source /home/runner/.bashrc
-          
+
           pip install -r dev-requirements.txt
 
       - name: "Run unitary tests folder with maximum ADS dependencies"
         timeout-minutes: 60
         shell: bash
         env:
-          NB_SESSION_COMPARTMENT_OCID: ocid1.compartment.oc1.<unique_ocid>
           CONDA_PREFIX: /usr/share/miniconda
         run: |
           set -x # print commands that are executed
-          
+
           # Setup project and tests folder for cov reports to not be overwritten by another parallel step
           if [[ ! -z "${{ matrix.cov-reports }}" ]]; then
             mkdir -p cov-${{ matrix.name }}
@@ -109,7 +110,7 @@ jobs:
             ln -s ../ads ads
             ln -s ../.coveragerc .coveragerc
           fi
-          
+
           # Run tests
           python -m pytest -v -p no:warnings --durations=5 \
           -n auto --dist loadfile ${{ matrix.cov-reports }} \
@@ -145,18 +146,18 @@ jobs:
       - name: "Calculate overall coverage"
         run: |
           set -x # print commands that are executed
-          
+
           # Prepare default cov body text 
           COV_BODY_INTRO="📌 Overall coverage:\n\n"
           echo COV_BODY="$COV_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
-          
+
           # Combine coverage files
           pip install coverage
           coverage combine cov-reports-unitary/.coverage cov-reports-model/.coverage
-          
+
           # Make html report
           coverage html
-          
+
           # Calculate overall coverage and update body message
           COV=$(grep -E 'pc_cov' htmlcov/index.html | cut -d'>' -f 2 | cut -d'%' -f 1)
           if [[ ! -z $COV ]]; then 
@@ -168,18 +169,18 @@ jobs:
         if: always()
         run: |
           set -x # print commands that are executed
-          
+
           # Prepare default diff body text 
           DIFF_BODY_INTRO="📌 Cov diff with **${{ env.COMPARE_BRANCH }}**:\n\n"
           echo DIFF_BODY="$DIFF_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
-          
+
           # Prepare file paths to coverage xml files
           # Filenames taken from job.test last step with name - "Save coverage files"
           FILE1="cov-reports-unitary/coverage.xml"; [[ ! -f $FILE1 ]] && FILE1="" 
           FILE2="cov-reports-model/coverage.xml"; [[ ! -f $FILE2 ]] && FILE2=""
           echo "FILE1=$FILE1" >> $GITHUB_ENV
           echo "FILE2=$FILE2" >> $GITHUB_ENV
-          
+
           # Calculate coverage diff and update body message
           pip install diff_cover
           diff-cover $FILE1 $FILE2 --compare-branch=origin/${{ env.COMPARE_BRANCH }} 

ads/ads

@@ -1,2 +1,2 @@
 export OCI_PYTHON_SDK_NO_SERVICE_IMPORTS=1
-python -m ads.cli "$@"
+python3 -m ads.cli "$@"

ads/jobs/ads_job.py

@@ -74,6 +74,9 @@ class Job(Builder):
                 .with_python_path("other_packages")
                 # Copy files in "code_dir/output" to object storage after job finishes.
                 .with_output("output", "oci://bucket_name@namespace/path/to/dir")
+                # Tags
+                .with_freeform_tag(my_tag="my_value")
+                .with_defined_tag(**{"Operations": {"CostCenter": "42"}})
             )
         )
         # Create and Run the job

ads/jobs/builders/infrastructure/base.py

@@ -61,6 +61,7 @@ def run(
         args: str = None,
         env_var: dict = None,
         freeform_tags: dict = None,
+        defined_tags: dict = None,
         wait: bool = False,
     ):
         """Runs a job on the infrastructure.
@@ -72,9 +73,11 @@ def run(
         args : str, optional
             Command line arguments for the job run, by default None.
         env_var : dict, optional
-            Environment variable for the job run, by default None
+            Environment variable for the job run, by default None.
         freeform_tags : dict, optional
-            Freeform tags for the job run, by default None
+            Freeform tags for the job run, by default None.
+        defined_tags : dict, optional
+            Defined tags for the job run, by default None.
         wait : bool, optional
             Indicate if this method should wait for the run to finish before it returns, by default False.
         """

ads/jobs/builders/infrastructure/dsc_job.py

@@ -35,10 +35,7 @@
 from ads.jobs.builders.runtimes.container_runtime import ContainerRuntime
 from ads.jobs.builders.runtimes.python_runtime import GitPythonRuntime
 
-from ads.common.dsc_file_system import (
-    OCIFileStorage,
-    DSCFileSystemManager
-)
+from ads.common.dsc_file_system import OCIFileStorage, DSCFileSystemManager
 
 logger = logging.getLogger(__name__)
 
@@ -445,6 +442,8 @@ def run(self, **kwargs) -> DataScienceJobRun:
             * command_line_arguments: str
             * maximum_runtime_in_minutes: int
             * display_name: str
+            * freeform_tags: dict(str, str)
+            * defined_tags: dict(str, dict(str, object))
 
         If display_name is not specified, it will be generated as "<JOB_NAME>-run-<TIMESTAMP>".
 
@@ -845,9 +844,12 @@ class DataScienceJob(Infrastructure):
             .with_storage_mount(
                 {
                     "src" : "<mount_target_ip_address>:<export_path>",
-	            "dest" : "<destination_directory_name>"
+                    "dest" : "<destination_directory_name>"
                 }
             )
+            # Tags
+            .with_freeform_tag(my_tag="my_value")
+            .with_defined_tag(**{"Operations": {"CostCenter": "42"}})
         )
 
     """
@@ -866,6 +868,8 @@ class DataScienceJob(Infrastructure):
     CONST_LOG_ID = "logId"
     CONST_LOG_GROUP_ID = "logGroupId"
     CONST_STORAGE_MOUNT = "storageMount"
+    CONST_FREEFORM_TAGS = "freeformTags"
+    CONST_DEFINED_TAGS = "definedTags"
 
     attribute_map = {
         CONST_PROJECT_ID: "project_id",
@@ -880,6 +884,8 @@ class DataScienceJob(Infrastructure):
         CONST_LOG_ID: "log_id",
         CONST_LOG_GROUP_ID: "log_group_id",
         CONST_STORAGE_MOUNT: "storage_mount",
+        CONST_FREEFORM_TAGS: "freeform_tags",
+        CONST_DEFINED_TAGS: "defined_tags",
     }
 
     shape_config_details_attribute_map = {
@@ -1235,9 +1241,7 @@ def log_group_id(self) -> str:
         """
         return self.get_spec(self.CONST_LOG_GROUP_ID)
 
-    def with_storage_mount(
-        self, *storage_mount: List[dict]
-    ) -> DataScienceJob:
+    def with_storage_mount(self, *storage_mount: List[dict]) -> DataScienceJob:
         """Sets the file systems to be mounted for the data science job.
         A maximum number of 5 file systems are allowed to be mounted for a single data science job.
 
@@ -1275,6 +1279,36 @@ def storage_mount(self) -> List[dict]:
         """
         return self.get_spec(self.CONST_STORAGE_MOUNT, [])
 
+    def with_freeform_tag(self, **kwargs) -> DataScienceJob:
+        """Sets freeform tags
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+        """
+        return self.set_spec(self.CONST_FREEFORM_TAGS, kwargs)
+
+    def with_defined_tag(self, **kwargs) -> DataScienceJob:
+        """Sets defined tags
+
+        Returns
+        -------
+        DataScienceJob
+            The DataScienceJob instance (self)
+        """
+        return self.set_spec(self.CONST_DEFINED_TAGS, kwargs)
+
+    @property
+    def freeform_tags(self) -> dict:
+        """Freeform tags"""
+        return self.get_spec(self.CONST_FREEFORM_TAGS, {})
+
+    @property
+    def defined_tags(self) -> dict:
+        """Defined tags"""
+        return self.get_spec(self.CONST_DEFINED_TAGS, {})
+
     def _prepare_log_config(self) -> dict:
         if not self.log_group_id and not self.log_id:
             return None
@@ -1429,7 +1463,8 @@ def _update_job_infra(self, dsc_job: DSCJob) -> DataScienceJob:
                     "Storage mount hasn't been supported in the current OCI SDK installed."
                 )
             dsc_job.job_storage_mount_configuration_details_list = [
-                DSCFileSystemManager.initialize(file_system) for file_system in self.storage_mount
+                DSCFileSystemManager.initialize(file_system)
+                for file_system in self.storage_mount
             ]
         return self
 
@@ -1486,6 +1521,10 @@ def create(self, runtime, **kwargs) -> DataScienceJob:
 
         payload["display_name"] = display_name
         payload["job_log_configuration_details"] = self._prepare_log_config()
+        if not payload.get("freeform_tags"):
+            payload["freeform_tags"] = self.freeform_tags
+        if not payload.get("defined_tags"):
+            payload["defined_tags"] = self.defined_tags
 
         self.dsc_job = DSCJob(**payload)
         # Set Job infra to user values after DSCJob initialized the defaults
@@ -1496,7 +1535,13 @@ def create(self, runtime, **kwargs) -> DataScienceJob:
         return self
 
     def run(
-        self, name=None, args=None, env_var=None, freeform_tags=None, wait=False
+        self,
+        name=None,
+        args=None,
+        env_var=None,
+        freeform_tags=None,
+        defined_tags=None,
+        wait=False,
     ) -> DataScienceJobRun:
         """Runs a job on OCI Data Science job
 
@@ -1510,6 +1555,8 @@ def run(
             Environment variable for the job run, by default None
         freeform_tags : dict, optional
             Freeform tags for the job run, by default None
+        defined_tags : dict, optional
+            Defined tags for the job run, by default None
         wait : bool, optional
             Indicate if this method should wait for the run to finish before it returns, by default False.
 
@@ -1524,11 +1571,18 @@ def run(
             raise RuntimeError(
                 "Job is not created. Call create() to create the job first."
             )
-        tags = self.runtime.freeform_tags
-        if not tags:
-            tags = {}
-        if freeform_tags:
-            tags.update(freeform_tags)
+
+        if not freeform_tags:
+            freeform_tags = {}
+        runtime_freeform_tags = self.runtime.freeform_tags
+        if runtime_freeform_tags:
+            freeform_tags.update(runtime_freeform_tags)
+
+        if not defined_tags:
+            defined_tags = {}
+        runtime_defined_tags = self.runtime.defined_tags
+        if runtime_defined_tags:
+            defined_tags.update(runtime_defined_tags)
 
         if name:
             envs = self.runtime.envs
@@ -1540,7 +1594,8 @@ def run(
             display_name=name,
             command_line_arguments=args,
             environment_variables=env_var,
-            freeform_tags=tags,
+            freeform_tags=freeform_tags,
+            defined_tags=defined_tags,
             wait=wait,
         )
 

ads/jobs/builders/infrastructure/dsc_job_runtime.py

@@ -38,7 +38,7 @@
 from ads.jobs.builders.infrastructure.utils import get_value
 
 
-class IncompatibleRuntime(Exception):   # pragma: no cover
+class IncompatibleRuntime(Exception):  # pragma: no cover
     """Represents an exception when runtime is not compatible with the OCI data science job configuration.
     This exception is designed to be raised during the extraction of a runtime from OCI data science job.
     The data science job does not explicitly contain information of the type of the ADS runtime.
@@ -104,6 +104,8 @@ def translate(self, runtime: Runtime) -> dict:
         payload["job_configuration_details"] = self._translate_config(runtime)
         if runtime.freeform_tags:
             payload["freeform_tags"] = runtime.freeform_tags
+        if runtime.defined_tags:
+            payload["defined_tags"] = runtime.defined_tags
         self.data_science_job.runtime = runtime
         return payload
 
@@ -353,10 +355,14 @@ def _extract_tags(self, dsc_job):
         dict
             A runtime specification dictionary for initializing a runtime.
         """
+        tags = {}
         value = get_value(dsc_job, "freeform_tags")
         if value:
-            return {Runtime.CONST_TAG: value}
-        return {}
+            tags[Runtime.CONST_FREEFORM_TAGS] = value
+        value = get_value(dsc_job, "defined_tags")
+        if value:
+            tags[Runtime.CONST_DEFINED_TAGS] = value
+        return tags
 
     def _extract_artifact(self, dsc_job):
         """Extract the job artifact from data science job.