fix: stop overwriting custom rules volume and type

Author: ajaykarpur

src/sagemaker/debugger.py

@@ -20,7 +20,9 @@
 and alerts when it detects errors during training.
 """
 from __future__ import absolute_import
-import smdebug_rulesconfig as rule_configs  # noqa: F401 # pylint: disable=unused-import
+
+# TODO-reinvent-2019 [knakad]: Uncomment this once PyPI integration is complete post-re:Invent-2019
+# import smdebug_rulesconfig as rule_configs  # noqa: F401 # pylint: disable=unused-import
 
 
 RULES_ECR_REPO_NAME = "sagemaker-debugger-rules"
@@ -191,11 +193,11 @@ def custom(
         name,
         image_uri,
         instance_type,
+        volume_size_in_gb,
         source=None,
         rule_to_invoke=None,
         container_local_output_path=None,
         s3_output_path=None,
-        volume_size_in_gb=None,
         other_trials_s3_input_paths=None,
         rule_parameters=None,
         collections_to_save=None,
@@ -210,14 +212,15 @@ def custom(
             image_uri (str): The URI of the image to be used by the debugger rule.
             instance_type (str): Type of EC2 instance to use, for example,
                 'ml.c4.xlarge'.
+            volume_size_in_gb (int): Size in GB of the EBS volume
+                to use for storing data.
             source (str): A source file containing a rule to invoke. If provided,
-                you must also provide rule_to_invoke.
+                you must also provide rule_to_invoke. This can either be an S3 uri or
+                a local path.
             rule_to_invoke (str): The name of the rule to invoke within the source.
                 If provided, you must also provide source.
             container_local_output_path (str): The path in the container.
             s3_output_path (str): The location in S3 to store the output.
-            volume_size_in_gb (int): Size in GB of the EBS volume
-                to use for storing data.
             other_trials_s3_input_paths ([str]): S3 input paths for other trials.
             rule_parameters (dict): A dictionary of parameters for the rule.
             collections_to_save ([sagemaker.debugger.CollectionConfig]): A list
@@ -293,7 +296,7 @@ class DebuggerHookConfig(object):
 
     def __init__(
         self,
-        s3_output_path,
+        s3_output_path=None,
         container_local_output_path=None,
         hook_parameters=None,
         collection_configs=None,

src/sagemaker/estimator.py

@@ -27,11 +27,9 @@
 import sagemaker
 from sagemaker import git_utils
 from sagemaker.analytics import TrainingJobAnalytics
-from sagemaker.debugger import (
-    DebuggerHookConfig,
-    TensorBoardOutputConfig,
-    get_rule_container_image_uri,
-)
+from sagemaker.debugger import DebuggerHookConfig
+from sagemaker.debugger import TensorBoardOutputConfig  # noqa: F401 # pylint: disable=unused-import
+from sagemaker.debugger import get_rule_container_image_uri
 from sagemaker.s3 import S3Uploader
 
 from sagemaker.fw_utils import (
@@ -331,6 +329,9 @@ def _prepare_for_training(self, job_name=None):
         # Prepare rules and debugger configs for training.
         if self.rules and not self.debugger_hook_config:
             self.debugger_hook_config = DebuggerHookConfig(s3_output_path=self.output_path)
+        # If an object was provided without an S3 URI is not provided, default it for the customer.
+        if self.debugger_hook_config and not self.debugger_hook_config.s3_output_path:
+            self.debugger_hook_config.s3_output_path = self.output_path
         self._prepare_rules()
         self._prepare_collection_configs()
 
@@ -340,17 +341,13 @@ def _prepare_rules(self):
         if self.rules is not None:
             # Iterate through each of the provided rules.
             for rule in self.rules:
-                # Set the instance type and volume size using the Estimator's defaults.
                 # Set the image URI using the default rule evaluator image and the region.
                 if rule.image_uri == "DEFAULT_RULE_EVALUATOR_IMAGE":
                     rule.image_uri = get_rule_container_image_uri(
                         self.sagemaker_session.boto_region_name
                     )
                     rule.instance_type = None
                     rule.volume_size_in_gb = None
-                else:
-                    rule.instance_type = self.train_instance_type
-                    rule.volume_size_in_gb = self.train_volume_size
                 # If source was provided as a rule parameter, upload to S3 and save the S3 uri.
                 if "source_s3_uri" in (rule.rule_parameters or {}):
                     parse_result = urlparse(rule.rule_parameters["source_s3_uri"])
@@ -384,6 +381,42 @@ def _prepare_collection_configs(self):
         if self.debugger_hook_config is not None:
             self.collection_configs.update(self.debugger_hook_config.collection_configs or [])
 
+    def get_debugger_artifacts_path(self):
+        """Gets the path to the DebuggerHookConfig output artifacts.
+
+        Returns:
+            str: An S3 path to the output artifacts.
+        """
+        self._ensure_latest_training_job(
+            error_message="""Cannot get the Debugger artifacts path.
+        The Estimator is not associated with a training job."""
+        )
+        if self.debugger_hook_config is not None:
+            return os.path.join(
+                self.debugger_hook_config.s3_output_path,
+                self.latest_training_job.name,
+                "debug-output",
+            )
+        return None
+
+    def get_tensorboard_artifacts_path(self):
+        """Gets the path to the TensorBoardOutputConfig output artifacts.
+
+        Returns:
+            str: An S3 path to the output artifacts.
+        """
+        self._ensure_latest_training_job(
+            error_message="""Cannot get the TensorBoard artifacts path.
+        The Estimator is not associated with a training job."""
+        )
+        if self.debugger_hook_config is not None:
+            return os.path.join(
+                self.tensorboard_output_config.s3_output_path,
+                self.latest_training_job.name,
+                "tensorboard-output",
+            )
+        return None
+
     def fit(self, inputs=None, wait=True, logs="All", job_name=None, experiment_config=None):
         """Train a model using the input training dataset.
 
@@ -1626,10 +1659,6 @@ def _prepare_for_training(self, job_name=None):
         # Set defaults for debugging.
         if self.debugger_hook_config is None:
             self.debugger_hook_config = DebuggerHookConfig(s3_output_path=self.output_path)
-        if self.tensorboard_output_config is None:
-            self.tensorboard_output_config = TensorBoardOutputConfig(
-                s3_output_path=self.output_path
-            )
 
     def _stage_user_code_in_s3(self):
         """Upload the user training script to s3 and return the location.

src/sagemaker/model_monitor/model_monitoring.py

@@ -1052,7 +1052,7 @@ def suggest_baseline(
                 script. This can be a local path or an S3 uri.
             output_s3_uri (str): Desired S3 destination Destination of the constraint_violations
                 and statistics json files.
-                Default: "s3://" + default_session_bucket + job_name + output
+                Default: "s3://<default_session_bucket>/<job_name>/output"
             wait (bool): Whether the call should wait until the job completes (default: True).
             logs (bool): Whether to show the logs produced by the job.
                 Only meaningful when wait is True (default: True).
@@ -1193,6 +1193,7 @@ def create_monitoring_schedule(
                 script. This can be a local path or an S3 uri.
             output_s3_uri (str): Desired S3 destination of the constraint_violations and
                 statistics json files.
+                Default: "s3://<default_session_bucket>/<job_name>/output"
             constraints (sagemaker.model_monitor.Constraints or str): If provided alongside
                 statistics, these will be used for monitoring the endpoint. This can be a
                 sagemaker.model_monitor.Constraints object or an s3_uri pointing to a constraints

src/sagemaker/sklearn/processing.py

@@ -66,8 +66,8 @@ def __init__(
                 Example: ["python3", "-v"]. If not provided, ["python3"] or ["python2"]
                 will be chosen based on the py_version parameter.
             py_version (str): The python version to use, for example, 'py3'.
-            volume_size_in_gb (int): Size in GB of the EBS volume
-                to use for storing data during processing (default: 30).
+            volume_size_in_gb (int): Size in GB of the EBS volume to
+                use for storing data during processing (default: 30).
             volume_kms_key (str): A KMS key for the processing
                 volume.
             output_kms_key (str): The KMS key id for all ProcessingOutputs.