Job-style resource testing improvements (#19)

* Add tests to completion for job style resources

* Address review comments

* Debugger Image TYPO fix

* Fixes

* Fix Transform Test and Black Formatting updates

* Minor fixes for issues found on final test run

Author: mbaijal

test/e2e/__init__.py

@@ -23,18 +23,17 @@
 CRD_GROUP = "sagemaker.services.k8s.aws"
 CRD_VERSION = "v1alpha1"
 
-ENDPOINT_CONFIG_RESOURCE_PLURAL = "endpointconfigs"
-MODEL_RESOURCE_PLURAL = "models"
-ENDPOINT_RESOURCE_PLURAL = "endpoints"
-DATA_QUALITY_JOB_DEFINITION_RESOURCE_PLURAL = "dataqualityjobdefinitions"
-
 # PyTest marker for the current service
 service_marker = pytest.mark.service(arg=SERVICE_NAME)
 
 bootstrap_directory = Path(__file__).parent
 resource_directory = Path(__file__).parent / "resources"
 
 
+def sagemaker_client():
+    return boto3.client("sagemaker")
+
+
 def create_sagemaker_resource(
     resource_plural, resource_name, spec_file, replacements, namespace="default"
 ):

test/e2e/common/config.py

@@ -0,0 +1,25 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+# 	 http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+"""String Constants
+"""
+
+ENDPOINT_CONFIG_RESOURCE_PLURAL = "endpointconfigs"
+MODEL_RESOURCE_PLURAL = "models"
+ENDPOINT_RESOURCE_PLURAL = "endpoints"
+DATA_QUALITY_JOB_DEFINITION_RESOURCE_PLURAL = "dataqualityjobdefinitions"
+
+# Job Type Resource Statuses
+LIST_JOB_STATUS_STOPPED = ("Stopped", "Stopping", "Completed")
+JOB_STATUS_INPROGRESS: str = "InProgress"
+JOB_STATUS_COMPLETED: str = "Completed"
+DEBUGGERJOB_STATUS_COMPLETED: str = "NoIssuesFound"

test/e2e/common/fixtures.py

@@ -16,17 +16,14 @@
 import pytest
 
 from e2e import (
-    ENDPOINT_CONFIG_RESOURCE_PLURAL,
-    MODEL_RESOURCE_PLURAL,
-    ENDPOINT_RESOURCE_PLURAL,
-    DATA_QUALITY_JOB_DEFINITION_RESOURCE_PLURAL,
     create_sagemaker_resource,
     wait_sagemaker_endpoint_status,
 )
 
 from e2e.replacement_values import REPLACEMENT_VALUES
 from acktest.resources import random_suffix_name
 from acktest.k8s import resource as k8s
+from e2e.common import config as cfg
 
 
 @pytest.fixture(scope="module")
@@ -48,7 +45,7 @@ def xgboost_churn_endpoint(sagemaker_client):
     ] = f"s3://{data_bucket}/sagemaker/model/xgb-churn-prediction-model.tar.gz"
 
     model_reference, model_spec, model_resource = create_sagemaker_resource(
-        resource_plural=MODEL_RESOURCE_PLURAL,
+        resource_plural=cfg.MODEL_RESOURCE_PLURAL,
         resource_name=model_resource_name,
         spec_file="xgboost_model_with_model_location",
         replacements=replacements,
@@ -61,7 +58,7 @@ def xgboost_churn_endpoint(sagemaker_client):
         endpoint_config_spec,
         endpoint_config_resource,
     ) = create_sagemaker_resource(
-        resource_plural=ENDPOINT_CONFIG_RESOURCE_PLURAL,
+        resource_plural=cfg.ENDPOINT_CONFIG_RESOURCE_PLURAL,
         resource_name=endpoint_config_resource_name,
         spec_file="endpoint_config_data_capture_single_variant",
         replacements=replacements,
@@ -70,7 +67,7 @@ def xgboost_churn_endpoint(sagemaker_client):
     assert k8s.get_resource_arn(endpoint_config_resource) is not None
 
     endpoint_reference, endpoint_spec, endpoint_resource = create_sagemaker_resource(
-        resource_plural=ENDPOINT_RESOURCE_PLURAL,
+        resource_plural=cfg.ENDPOINT_RESOURCE_PLURAL,
         resource_name=endpoint_resource_name,
         spec_file="endpoint_base",
         replacements=replacements,
@@ -100,7 +97,7 @@ def xgboost_churn_data_quality_job_definition(xgboost_churn_endpoint):
     replacements["ENDPOINT_NAME"] = endpoint_name
 
     job_definition_reference, _, resource = create_sagemaker_resource(
-        resource_plural=DATA_QUALITY_JOB_DEFINITION_RESOURCE_PLURAL,
+        resource_plural=cfg.DATA_QUALITY_JOB_DEFINITION_RESOURCE_PLURAL,
         resource_name=resource_name,
         spec_file="data_quality_job_definition_xgboost_churn",
         replacements=replacements,

test/e2e/replacement_values.py

@@ -42,6 +42,54 @@
     "sa-east-1": "737474898029.dkr.ecr.sa-east-1.amazonaws.com",
 }
 
+DEBUGGER_IMAGE_URIS = {
+    "us-west-1": "685455198987.dkr.ecr.us-west-1.amazonaws.com",
+    "us-west-2": "895741380848.dkr.ecr.us-west-2.amazonaws.com",
+    "us-east-1": "503895931360.dkr.ecr.us-east-1.amazonaws.com",
+    "us-east-2": "915447279597.dkr.ecr.us-east-2.amazonaws.com",
+    "ap-east-1": "199566480951.dkr.ecr.ap-east-1.amazonaws.com",
+    "ap-northeast-1": "430734990657.dkr.ecr.ap-northeast-1.amazonaws.com",
+    "ap-northeast-2": "578805364391.dkr.ecr.ap-northeast-2.amazonaws.com",
+    "ap-south-1": "904829902805.dkr.ecr.ap-south-1.amazonaws.com",
+    "ap-southeast-1": "972752614525.dkr.ecr.ap-southeast-1.amazonaws.com",
+    "ap-southeast-2": "184798709955.dkr.ecr.ap-southeast-2.amazonaws.com",
+    "ca-central-1": "519511493484.dkr.ecr.ca-central-1.amazonaws.com",
+    "cn-north-1": "618459771430.dkr.ecr.cn-north-1.amazonaws.com.cn",
+    "cn-northwest-1": "658757709296.dkr.ecr.cn-northwest-1.amazonaws.com.cn",
+    "eu-central-1": "482524230118.dkr.ecr.eu-central-1.amazonaws.com",
+    "eu-north-1": "314864569078.dkr.ecr.eu-north-1.amazonaws.com",
+    "eu-west-1": "929884845733.dkr.ecr.eu-west-1.amazonaws.com",
+    "eu-west-2": "250201462417.dkr.ecr.eu-west-2.amazonaws.com",
+    "eu-west-3": "447278800020.dkr.ecr.eu-west-3.amazonaws.com",
+    "me-south-1": "986000313247.dkr.ecr.me-south-1.amazonaws.com",
+    "sa-east-1": "818342061345.dkr.ecr.sa-east-1.amazonaws.com",
+}
+
+# https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-algo-docker-registry-paths.html
+XGBOOST_V1_IMAGE_URIS = {
+    "us-west-1": "632365934929.dkr.ecr.us-west-1.amazonaws.com",
+    "us-west-2": "433757028032.dkr.ecr.us-west-2.amazonaws.com",
+    "us-east-1": "811284229777.dkr.ecr.us-east-1.amazonaws.com",
+    "us-east-2": "825641698319.dkr.ecr.us-east-2.amazonaws.com",
+    "ap-east-1": "286214385809.dkr.ecr.ap-east-1.amazonaws.com",
+    "ap-northeast-1": "501404015308.dkr.ecr.ap-northeast-1.amazonaws.com",
+    "ap-northeast-2": "306986355934.dkr.ecr.ap-northeast-2.amazonaws.com",
+    "ap-south-1": "991648021394.dkr.ecr.ap-south-1.amazonaws.com",
+    "ap-southeast-1": "475088953585.dkr.ecr.ap-southeast-1.amazonaws.com",
+    "ap-southeast-2": "544295431143.dkr.ecr.ap-southeast-2.amazonaws.com",
+    "ca-central-1": "469771592824.dkr.ecr.ca-central-1.amazonaws.com",
+    "cn-north-1": "390948362332.dkr.ecr.cn-north-1.amazonaws.com",
+    "cn-northwest-1": "387376663083.dkr.ecr.cn-northwest-1.amazonaws.com",
+    "eu-central-1": "813361260812.dkr.ecr.eu-central-1.amazonaws.com",
+    "eu-north-1": "669576153137.dkr.ecr.eu-north-1.amazonaws.com",
+    "eu-west-1": "685385470294.dkr.ecr.eu-west-1.amazonaws.com",
+    "eu-west-2": "644912444149.dkr.ecr.eu-west-2.amazonaws.com",
+    "eu-west-3": "749696950732.dkr.ecr.eu-west-3.amazonaws.com",
+    "me-south-1": "249704162688.dkr.ecr.me-south-1.amazonaws.com",
+    "sa-east-1": "855470959533.dkr.ecr.sa-east-1.amazonaws.com",
+}
+
+
 PYTORCH_TRAIN_IMAGE_URIS = {
     "us-east-1": "763104351884.dkr.ecr.us-east-1.amazonaws.com",
     "us-east-2": "763104351884.dkr.ecr.us-east-2.amazonaws.com",
@@ -121,6 +169,8 @@
 REPLACEMENT_VALUES = {
     "SAGEMAKER_DATA_BUCKET": get_bootstrap_resources().DataBucketName,
     "XGBOOST_IMAGE_URI": f"{XGBOOST_IMAGE_URIS[get_region()]}/sagemaker-xgboost:1.0-1-cpu-py3",
+    "DEBUGGER_IMAGE_URI": f"{DEBUGGER_IMAGE_URIS[get_region()]}/sagemaker-debugger-rules:latest",
+    "XGBOOST_V1_IMAGE_URI": f"{XGBOOST_V1_IMAGE_URIS[get_region()]}/xgboost:latest",
     "PYTORCH_TRAIN_IMAGE_URI": f"{PYTORCH_TRAIN_IMAGE_URIS[get_region()]}/pytorch-training:1.5.0-cpu-py36-ubuntu16.04",
     "SAGEMAKER_EXECUTION_ROLE_ARN": get_bootstrap_resources().ExecutionRoleARN,
     "MODEL_MONITOR_ANALYZER_IMAGE_URI": f"{MODEL_MONITOR_IMAGE_URIS[get_region()]}/sagemaker-model-monitor-analyzer",

test/e2e/requirements.txt

@@ -1,2 +1,2 @@
 acktest @ git+https://github.com/aws-controllers-k8s/test-infra.git@955d7831ee374a212250179e95a5f3b75e555fd9
-black==20.*
+black==20.8b1

test/e2e/resources/kmeans_processingjob.yaml

@@ -47,6 +47,6 @@ spec:
           s3UploadMode: EndOfJob
       - outputName: valid_data
         s3Output:
-          s3URI: s3://{DATA_BUCKET}/sagemaker/processing/output/
+          s3URI: s3://$SAGEMAKER_DATA_BUCKET/sagemaker/processing/output/
           localPath: /opt/ml/processing/output_valid/
           s3UploadMode: EndOfJob

test/e2e/resources/xgboost_model.yaml

@@ -7,7 +7,7 @@ spec:
   primaryContainer:
     containerHostname: xgboost
     modelDataURL: s3://$SAGEMAKER_DATA_BUCKET/sagemaker/model/xgboost-mnist-model.tar.gz
-    image: $XGBOOST_IMAGE_URI
+    image: $XGBOOST_V1_IMAGE_URI
     environment:
       my_var: my_value
       my_var2: my_value2

test/e2e/resources/xgboost_trainingjob_debugger.yaml

@@ -0,0 +1,70 @@
+apiVersion: sagemaker.services.k8s.aws/v1alpha1
+kind: TrainingJob
+metadata:
+  name: $TRAINING_JOB_NAME
+spec:
+  trainingJobName: $TRAINING_JOB_NAME
+  roleARN: $SAGEMAKER_EXECUTION_ROLE_ARN
+  hyperParameters:
+    max_depth: "5"
+    gamma: "4"
+    eta: "0.2"
+    min_child_weight: "6"
+    silent: "0"
+    objective: "reg:squarederror"
+    subsample: "0.7"
+    num_round: "51"
+  algorithmSpecification:
+    trainingImage: $XGBOOST_IMAGE_URI
+    trainingInputMode: File
+  outputDataConfig:
+    s3OutputPath: s3://$SAGEMAKER_DATA_BUCKET/sagemaker/training/debugger/output
+  resourceConfig:
+    instanceCount: 1
+    instanceType: ml.m4.xlarge
+    volumeSizeInGB: 5
+  stoppingCondition:
+    maxRuntimeInSeconds: 86400
+  inputDataConfig:
+    - channelName: train
+      dataSource:
+        s3DataSource:
+          s3DataType: S3Prefix
+          s3URI: s3://$SAGEMAKER_DATA_BUCKET/sagemaker/training/train
+          s3DataDistributionType: FullyReplicated
+      contentType: text/csv
+      compressionType: None
+    - channelName: validation
+      dataSource:
+        s3DataSource:
+          s3DataType: S3Prefix
+          s3URI: s3://$SAGEMAKER_DATA_BUCKET/sagemaker/training/validation
+          s3DataDistributionType: FullyReplicated
+      contentType: text/csv
+      compressionType: None
+  debugHookConfig:
+    s3OutputPath: s3://$SAGEMAKER_DATA_BUCKET/sagemaker/training/debugger/hookconfig
+    collectionConfigurations:
+      - collectionName: feature_importance
+        collectionParameters:
+          name: save_interval
+          value: "5"
+      - collectionName: losses
+        collectionParameters:
+          name: save_interval
+          value: "500" 
+      - collectionName: average_shap
+        collectionParameters:
+          name: save_interval
+          value: "5" 
+      - collectionName: metrics
+        collectionParameters:
+          name: save_interval
+          value: "5" 
+  debugRuleConfigurations:
+    - ruleConfigurationName: LossNotDecreasing
+      ruleEvaluatorImage: $DEBUGGER_IMAGE_URI
+      ruleParameters:
+        collection_names: metrics
+        num_steps: "10"
+        rule_to_invoke: LossNotDecreasing

test/e2e/resources/xgboost_transformjob.yaml

@@ -4,7 +4,6 @@ metadata:
   name: $TRANSFORM_JOB_NAME
 spec:
   transformJobName: $TRANSFORM_JOB_NAME
-  roleARN: $SAGEMAKER_EXECUTION_ROLE_ARN
   modelName: $MODEL_NAME
   transformInput:
     contentType: text/csv

test/e2e/tests/test_endpoint.py

@@ -25,14 +25,12 @@
 
 from e2e import (
     service_marker,
-    ENDPOINT_CONFIG_RESOURCE_PLURAL,
-    MODEL_RESOURCE_PLURAL,
-    ENDPOINT_RESOURCE_PLURAL,
     create_sagemaker_resource,
     wait_sagemaker_endpoint_status,
     wait_resource_endpoint_status,
 )
 from e2e.replacement_values import REPLACEMENT_VALUES
+from e2e.common import config as cfg
 
 FAIL_UPDATE_ERROR_MESSAGE = "unable to update endpoint. check FailureReason"
 
@@ -49,7 +47,7 @@ def single_container_model(name_suffix):
     replacements["MODEL_NAME"] = model_resource_name
 
     model_reference, model_spec, model_resource = create_sagemaker_resource(
-        resource_plural=MODEL_RESOURCE_PLURAL,
+        resource_plural=cfg.MODEL_RESOURCE_PLURAL,
         resource_name=model_resource_name,
         spec_file="xgboost_model",
         replacements=replacements,
@@ -74,7 +72,7 @@ def multi_variant_config(name_suffix, single_container_model):
     replacements["MODEL_NAME"] = model_resource_name
 
     config_reference, config_spec, config_resource = create_sagemaker_resource(
-        resource_plural=ENDPOINT_CONFIG_RESOURCE_PLURAL,
+        resource_plural=cfg.ENDPOINT_CONFIG_RESOURCE_PLURAL,
         resource_name=config_resource_name,
         spec_file="endpoint_config_multi_variant",
         replacements=replacements,
@@ -99,7 +97,7 @@ def single_variant_config(name_suffix, single_container_model):
     replacements["MODEL_NAME"] = model_resource_name
 
     config_reference, config_spec, config_resource = create_sagemaker_resource(
-        resource_plural=ENDPOINT_CONFIG_RESOURCE_PLURAL,
+        resource_plural=cfg.ENDPOINT_CONFIG_RESOURCE_PLURAL,
         resource_name=config_resource_name,
         spec_file="endpoint_config_single_variant",
         replacements=replacements,
@@ -124,7 +122,7 @@ def xgboost_endpoint(name_suffix, single_variant_config):
     replacements["ENDPOINT_CONFIG_NAME"] = config_resource_name
 
     reference, spec, resource = create_sagemaker_resource(
-        resource_plural=ENDPOINT_RESOURCE_PLURAL,
+        resource_plural=cfg.ENDPOINT_RESOURCE_PLURAL,
         resource_name=endpoint_resource_name,
         spec_file="endpoint_base",
         replacements=replacements,
@@ -157,7 +155,7 @@ def faulty_config(name_suffix, single_container_model):
     replacements["MODEL_NAME"] = model_resource_name
     replacements["MODEL_LOCATION"] = f"s3://{model_bucket}/{model_destination_key}"
     model_reference, model_spec, model_resource = create_sagemaker_resource(
-        resource_plural=MODEL_RESOURCE_PLURAL,
+        resource_plural=cfg.MODEL_RESOURCE_PLURAL,
         resource_name=model_resource_name,
         spec_file="xgboost_model_with_model_location",
         replacements=replacements,
@@ -174,7 +172,7 @@ def faulty_config(name_suffix, single_container_model):
     replacements["ENDPOINT_CONFIG_NAME"] = config_resource_name
 
     config_reference, config_spec, config_resource = create_sagemaker_resource(
-        resource_plural=ENDPOINT_CONFIG_RESOURCE_PLURAL,
+        resource_plural=cfg.ENDPOINT_CONFIG_RESOURCE_PLURAL,
         resource_name=config_resource_name,
         spec_file="endpoint_config_multi_variant",
         replacements=replacements,