Handle the case where the destination folder does not exist in local mode (#458)

Author: derekhh

.pylintrc

@@ -84,3 +84,7 @@ dummy-variables-rgx=_|unused_
 # Apply logging string format checks to calls on these modules.
 logging-modules=
     logging
+
+[TYPECHECK]
+ignored-modules=
+    distutils

CHANGELOG.rst

@@ -2,8 +2,13 @@
 CHANGELOG
 =========
 
+1.14.3-dev
+=====
+
+* bug-fix: Local Mode: correctly handle the case where the model output folder doesn't exist yet
+
 1.14.2
-=======
+=====
 
 * bug-fix: support ``CustomAttributes`` argument in local mode ``invoke_endpoint`` requests
 * enhancement: add ``content_type`` parameter to ``sagemaker.tensorflow.serving.Predictor``

src/sagemaker/local/utils.py

@@ -15,6 +15,7 @@
 import os
 import shutil
 
+from distutils.dir_util import copy_tree
 from six.moves.urllib.parse import urlparse
 
 
@@ -45,6 +46,7 @@ def move_to_destination(source, destination, job_name, sagemaker_session):
     Args:
         source (str): root directory to move
         destination (str): file:// or s3:// URI that source will be moved to.
+        job_name (str): SageMaker job name.
         sagemaker_session (sagemaker.Session): a sagemaker_session to interact with S3 if needed
 
     Returns:
@@ -67,19 +69,12 @@ def move_to_destination(source, destination, job_name, sagemaker_session):
 
 
 def recursive_copy(source, destination):
-    """Similar to shutil.copy but the destination directory can exist. Existing files will be overriden.
+    """A wrapper around distutils.dir_util.copy_tree but won't throw any exception when the source
+    directory does not exist.
+
     Args:
         source (str): source path
         destination (str): destination path
     """
-    for root, dirs, files in os.walk(source):
-        root = os.path.relpath(root, source)
-        current_path = os.path.join(source, root)
-        target_path = os.path.join(destination, root)
-
-        for file in files:
-            shutil.copy(os.path.join(current_path, file), os.path.join(target_path, file))
-        for d in dirs:
-            new_dir = os.path.join(target_path, d)
-            if not os.path.exists(new_dir):
-                os.mkdir(os.path.join(target_path, d))
+    if os.path.isdir(source):
+        copy_tree(source, destination)

tests/unit/test_local_utils.py

@@ -17,15 +17,13 @@
 
 import sagemaker.local.utils
 
-BUCKET_NAME = 'some-nice-bucket'
-
 
 @patch('shutil.rmtree', Mock())
 @patch('sagemaker.local.utils.recursive_copy')
 def test_move_to_destination(recursive_copy):
-    # local files will just be recursive copied
+    # local files will just be recursively copied
     sagemaker.local.utils.move_to_destination('/tmp/data', 'file:///target/dir/', 'job', None)
-    recursive_copy.assert_called()
+    recursive_copy.assert_called_with('/tmp/data', '/target/dir/')
 
     # s3 destination will upload to S3
     sms = Mock()