Merge branch 'develop' into develop

Author: lu-ohai

ads/jobs/builders/infrastructure/dataflow.py

@@ -436,7 +436,7 @@ def _load_default_properties() -> dict:
                 nb_session = dsc_client.get_notebook_session(
                     os.environ["NB_SESSION_OCID"]
                 ).data
-                nb_config = nb_session.notebook_session_configuration_details
+                nb_config = nb_session.notebook_session_config_details
                 defaults["driver_shape"] = nb_config.shape
                 logger.debug(f"Set driver shape to {nb_config.shape}")
                 defaults["executor_shape"] = nb_config.shape

ads/jobs/extension.py

@@ -1,26 +1,23 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2022 Oracle and/or its affiliates.
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
-import os
-import tempfile
 import json
+import os
 import shlex
+import tempfile
+import warnings
 from getopt import gnu_getopt
 
-from ads.opctl.constants import (
-    ADS_DATAFLOW_CONFIG_FILE_NAME,
-    DEFAULT_ADS_CONFIG_FOLDER,
-)
-from ads.jobs import Job, DataFlow, DataFlowRuntime, DataFlowRun
+from ads.common.decorator.runtime_dependency import (OptionalDependency,
+                                                     runtime_dependency)
+from ads.jobs import DataFlow, DataFlowRun, DataFlowRuntime, Job
 from ads.jobs.utils import get_dataflow_config
-from ads.common.decorator.runtime_dependency import (
-    runtime_dependency,
-    OptionalDependency,
-)
+from ads.opctl.constants import (ADS_DATAFLOW_CONFIG_FILE_NAME,
+                                 DEFAULT_ADS_CONFIG_FOLDER)
 
 
 def dataflow(line, cell=None):
@@ -91,6 +88,8 @@ def dataflow_run(options, args, cell):
         archive_name = options["-a"]
     elif "--archive" in options:
         archive_name = options["--archive"]
+    elif hasattr(dataflow_config, "archive_uri") and dataflow_config.archive_uri:
+        archive_name = dataflow_config.archive_uri
     else:
         archive_name = None
     with tempfile.TemporaryDirectory() as td:
@@ -105,7 +104,11 @@ def dataflow_run(options, args, cell):
             rt_spec["args"] = args[1:]
         if archive_name:
             rt_spec["archiveUri"] = archive_name
-            rt_spec["archiveBucket"] = dataflow_config.pop("archive_bucket")
+            rt_spec["archiveBucket"] = dataflow_config.pop("archive_bucket", None)
+            if not archive_name.startswith("oci://") and not rt_spec["archiveBucket"]:
+                raise ValueError(
+                    "`archiveBucket` has to be set in the config if `archive` is a local path."
+                )
         rt = DataFlowRuntime(rt_spec)
         infra = DataFlow(spec=dataflow_config)
         if "-o" in options or "--overwrite" in options:

ads/jobs/utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2022 Oracle and/or its affiliates.
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
@@ -43,7 +43,7 @@ def get_dataflow_config(path=None, oci_profile=None):
             )
         return config
     else:
-        logger.error(f"{dataflow_config_file_path} not found.")
+        logger.warning(f"{dataflow_config_file_path} not found. Follow this link https://accelerated-data-science.readthedocs.io/en/latest/user_guide/apachespark/dataflow.html to set up the config.")
         return {}
 
 

ads/model/deployment/model_deployment.py

@@ -1532,9 +1532,15 @@ def _build_model_deployment_configuration_details(self) -> Dict:
                 infrastructure.CONST_MEMORY_IN_GBS: infrastructure.shape_config_details.get(
                     "memory_in_gbs", None
                 )
+                or infrastructure.shape_config_details.get(
+                    "memoryInGBs", None
+                )
                 or MODEL_DEPLOYMENT_INSTANCE_MEMORY_IN_GBS,
             }
 
+        if infrastructure.subnet_id:
+            instance_configuration[infrastructure.CONST_SUBNET_ID] = infrastructure.subnet_id
+
         scaling_policy = {
             infrastructure.CONST_POLICY_TYPE: "FIXED_SIZE",
             infrastructure.CONST_INSTANCE_COUNT: infrastructure.replica

ads/model/deployment/model_deployment_infrastructure.py

@@ -39,6 +39,8 @@ class ModelDeploymentInfrastructure(Builder):
         The access and predict log id of model deployment
     web_concurrency: int
         The web concurrency of model deployment
+    subnet_id: str
+        The subnet id of model deployment
 
     Methods
     -------
@@ -64,6 +66,8 @@ class ModelDeploymentInfrastructure(Builder):
         Sets the access and predict log id of model deployment
     with_web_concurrency(web_concurrency)
         Sets the web concurrency of model deployment
+    with_subnet_id(subnet_id) 
+        Sets the subnet id of model deployment
 
     Example
     -------
@@ -79,6 +83,7 @@ class ModelDeploymentInfrastructure(Builder):
     ...        .with_replica(1)
     ...        .with_bandwidth_mbps(10)
     ...        .with_web_concurrency(10)
+    ...        .with_subnet_id(<subnet_id>)
     ...        .with_access_log(
     ...            log_group_id=<log_group_id>,
     ...            log_id=<log_id>
@@ -121,6 +126,7 @@ class ModelDeploymentInfrastructure(Builder):
     CONST_LOG_GROUP_ID = "logGroupId"
     CONST_WEB_CONCURRENCY = "webConcurrency"
     CONST_STREAM_CONFIG_DETAILS = "streamConfigurationDetails"
+    CONST_SUBNET_ID = "subnetId"
 
     attribute_map = {
         CONST_PROJECT_ID: "project_id",
@@ -136,6 +142,7 @@ class ModelDeploymentInfrastructure(Builder):
         CONST_LOG_ID: "log_id",
         CONST_LOG_GROUP_ID: "log_group_id",
         CONST_WEB_CONCURRENCY: "web_concurrency",
+        CONST_SUBNET_ID: "subnet_id"
     }
 
     shape_config_details_attribute_map = {
@@ -162,6 +169,7 @@ class ModelDeploymentInfrastructure(Builder):
         CONST_COMPARTMENT_ID: "compartment_id",
         CONST_SHAPE_NAME: f"{MODEL_CONFIG_DETAILS_PATH}.instance_configuration.instance_shape_name",
         CONST_SHAPE_CONFIG_DETAILS: f"{MODEL_CONFIG_DETAILS_PATH}.instance_configuration.model_deployment_instance_shape_config_details",
+        CONST_SUBNET_ID: f"{MODEL_CONFIG_DETAILS_PATH}.instance_configuration.subnet_id",
         CONST_REPLICA: f"{MODEL_CONFIG_DETAILS_PATH}.scaling_policy.instance_count",
         CONST_BANDWIDTH_MBPS: f"{MODEL_CONFIG_DETAILS_PATH}.bandwidth_mbps",
         CONST_ACCESS_LOG: "category_log_details.access",
@@ -518,3 +526,29 @@ def with_web_concurrency(
             The ModelDeploymentInfrastructure instance (self).
         """
         return self.set_spec(self.CONST_WEB_CONCURRENCY, web_concurrency)
+
+    def with_subnet_id(self, subnet_id: str) -> "ModelDeploymentInfrastructure":
+        """Sets the subnet id of model deployment.
+
+        Parameters
+        ----------
+        subnet_id : str
+            The subnet id of model deployment.
+
+        Returns
+        -------
+        ModelDeploymentInfrastructure
+            The ModelDeploymentInfrastructure instance (self).
+        """
+        return self.set_spec(self.CONST_SUBNET_ID, subnet_id)
+
+    @property
+    def subnet_id(self) -> str:
+        """The model deployment subnet id.
+        
+        Returns
+        -------
+        str
+            The model deployment subnet id.
+        """
+        return self.get_spec(self.CONST_SUBNET_ID, None)

ads/model/generic_model.py

@@ -1902,6 +1902,7 @@ def deploy(
         display_name: Optional[str] = None,
         description: Optional[str] = None,
         deployment_instance_shape: Optional[str] = None,
+        deployment_instance_subnet_id: Optional[str] = None,
         deployment_instance_count: Optional[int] = None,
         deployment_bandwidth_mbps: Optional[int] = None,
         deployment_log_group_id: Optional[str] = None,
@@ -1950,6 +1951,8 @@ def deploy(
             The description of the model.
         deployment_instance_shape: (str, optional). Default to `VM.Standard2.1`.
             The shape of the instance used for deployment.
+        deployment_instance_subnet_id: (str, optional). Default to None.
+            The subnet id of the instance used for deployment.
         deployment_instance_count: (int, optional). Defaults to 1.
             The number of instance used for deployment.
         deployment_bandwidth_mbps: (int, optional). Defaults to 10.
@@ -2084,6 +2087,10 @@ def deploy(
                 self.properties.deployment_instance_shape
                 or existing_infrastructure.shape_name
             )
+            .with_subnet_id(
+                self.properties.deployment_instance_subnet_id
+                or existing_infrastructure.subnet_id
+            )
             .with_replica(
                 self.properties.deployment_instance_count
                 or existing_infrastructure.replica
@@ -2253,6 +2260,7 @@ def prepare_save_deploy(
         deployment_display_name: Optional[str] = None,
         deployment_description: Optional[str] = None,
         deployment_instance_shape: Optional[str] = None,
+        deployment_instance_subnet_id: Optional[str] = None,
         deployment_instance_count: Optional[int] = None,
         deployment_bandwidth_mbps: Optional[int] = None,
         deployment_log_group_id: Optional[str] = None,
@@ -2340,6 +2348,8 @@ def prepare_save_deploy(
             The description of the model.
         deployment_instance_shape: (str, optional). Default to `VM.Standard2.1`.
             The shape of the instance used for deployment.
+        deployment_instance_subnet_id: (str, optional). Default to None.
+            The subnet id of the instance used for deployment.
         deployment_instance_count: (int, optional). Defaults to 1.
             The number of instance used for deployment.
         deployment_bandwidth_mbps: (int, optional). Defaults to 10.
@@ -2481,6 +2491,7 @@ def prepare_save_deploy(
             display_name=deployment_display_name,
             description=deployment_description,
             deployment_instance_shape=self.properties.deployment_instance_shape,
+            deployment_instance_subnet_id=self.properties.deployment_instance_subnet_id,
             deployment_instance_count=self.properties.deployment_instance_count,
             deployment_bandwidth_mbps=self.properties.deployment_bandwidth_mbps,
             deployment_log_group_id=self.properties.deployment_log_group_id,

ads/model/model_properties.py

@@ -28,6 +28,7 @@ class ModelProperties(BaseProperties):
     remove_existing_artifact: bool = None
     overwrite_existing_artifact: bool = None
     deployment_instance_shape: str = None
+    deployment_instance_subnet_id: str = None
     deployment_instance_count: int = None
     deployment_bandwidth_mbps: int = None
     deployment_log_group_id: str = None