fix the dataflow bug (#154)

Author: Ziqun Ye

ads/jobs/builders/infrastructure/dataflow.py

@@ -436,7 +436,7 @@ def _load_default_properties() -> dict:
                 nb_session = dsc_client.get_notebook_session(
                     os.environ["NB_SESSION_OCID"]
                 ).data
-                nb_config = nb_session.notebook_session_configuration_details
+                nb_config = nb_session.notebook_session_config_details
                 defaults["driver_shape"] = nb_config.shape
                 logger.debug(f"Set driver shape to {nb_config.shape}")
                 defaults["executor_shape"] = nb_config.shape

ads/jobs/extension.py

@@ -1,26 +1,23 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2022 Oracle and/or its affiliates.
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
-import os
-import tempfile
 import json
+import os
 import shlex
+import tempfile
+import warnings
 from getopt import gnu_getopt
 
-from ads.opctl.constants import (
-    ADS_DATAFLOW_CONFIG_FILE_NAME,
-    DEFAULT_ADS_CONFIG_FOLDER,
-)
-from ads.jobs import Job, DataFlow, DataFlowRuntime, DataFlowRun
+from ads.common.decorator.runtime_dependency import (OptionalDependency,
+                                                     runtime_dependency)
+from ads.jobs import DataFlow, DataFlowRun, DataFlowRuntime, Job
 from ads.jobs.utils import get_dataflow_config
-from ads.common.decorator.runtime_dependency import (
-    runtime_dependency,
-    OptionalDependency,
-)
+from ads.opctl.constants import (ADS_DATAFLOW_CONFIG_FILE_NAME,
+                                 DEFAULT_ADS_CONFIG_FOLDER)
 
 
 def dataflow(line, cell=None):
@@ -91,6 +88,8 @@ def dataflow_run(options, args, cell):
         archive_name = options["-a"]
     elif "--archive" in options:
         archive_name = options["--archive"]
+    elif hasattr(dataflow_config, "archive_uri") and dataflow_config.archive_uri:
+        archive_name = dataflow_config.archive_uri
     else:
         archive_name = None
     with tempfile.TemporaryDirectory() as td:
@@ -105,7 +104,11 @@ def dataflow_run(options, args, cell):
             rt_spec["args"] = args[1:]
         if archive_name:
             rt_spec["archiveUri"] = archive_name
-            rt_spec["archiveBucket"] = dataflow_config.pop("archive_bucket")
+            rt_spec["archiveBucket"] = dataflow_config.pop("archive_bucket", None)
+            if not archive_name.startswith("oci://") and not rt_spec["archiveBucket"]:
+                raise ValueError(
+                    "`archiveBucket` has to be set in the config if `archive` is a local path."
+                )
         rt = DataFlowRuntime(rt_spec)
         infra = DataFlow(spec=dataflow_config)
         if "-o" in options or "--overwrite" in options:

ads/jobs/utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2022 Oracle and/or its affiliates.
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
@@ -43,7 +43,7 @@ def get_dataflow_config(path=None, oci_profile=None):
             )
         return config
     else:
-        logger.error(f"{dataflow_config_file_path} not found.")
+        logger.warning(f"{dataflow_config_file_path} not found. Follow this link https://accelerated-data-science.readthedocs.io/en/latest/user_guide/apachespark/dataflow.html to set up the config.")
         return {}