Add support for Chainer 5.0 (#491)

Author: iquintero

CHANGELOG.rst

@@ -12,6 +12,7 @@ CHANGELOG
 * doc-fix: Add estimator base classes to API docs
 * feature: HyperparameterTuner: add support for Automatic Model Tuning's Warm Start Jobs
 * feature: HyperparameterTuner: Make input channels optional
+* feature: Add support for Chainer 5.0
 
 1.14.2
 ======

README.rst

@@ -358,7 +358,7 @@ Chainer SageMaker Estimators
 
 By using Chainer SageMaker ``Estimators``, you can train and host Chainer models on Amazon SageMaker.
 
-Supported versions of Chainer: ``4.0.0``, ``4.1.0``.
+Supported versions of Chainer: ``4.0.0``, ``4.1.0``, ``5.0.0``.
 
 We recommend that you use the latest supported version, because that's where we focus most of our development efforts.
 

src/sagemaker/chainer/README.rst

@@ -4,7 +4,7 @@ Chainer SageMaker Estimators and Models
 
 With Chainer Estimators, you can train and host Chainer models on Amazon SageMaker.
 
-Supported versions of Chainer: ``4.0.0``, ``4.1.0``
+Supported versions of Chainer: ``4.0.0``, ``4.1.0``, ``5.0.0``
 
 You can visit the Chainer repository at https://github.com/chainer/chainer.
 
@@ -32,7 +32,7 @@ Suppose that you already have an Chainer training script called
                                 role='SageMakerRole',
                                 train_instance_type='ml.p3.2xlarge',
                                 train_instance_count=1,
-                                framework_version='4.1.0')
+                                framework_version='5.0.0')
     chainer_estimator.fit('s3://bucket/path/to/training/data')
 
 Where the S3 URL is a path to your training data, within Amazon S3. The constructor keyword arguments define how
@@ -111,7 +111,7 @@ directories ('train' and 'test').
     chainer_estimator = Chainer('chainer-train.py',
                                 train_instance_type='ml.p3.2xlarge',
                                 train_instance_count=1,
-                                framework_version='4.1.0',
+                                framework_version='5.0.0',
                                 hyperparameters = {'epochs': 20, 'batch-size': 64, 'learning-rate': 0.1})
     chainer_estimator.fit({'train': 's3://my-data-bucket/path/to/my/training/data',
                            'test': 's3://my-data-bucket/path/to/my/test/data'})
@@ -285,7 +285,7 @@ operation.
     chainer_estimator = Chainer(entry_point='train_and_deploy.py',
                                 train_instance_type='ml.p3.2xlarge',
                                 train_instance_count=1,
-                                framework_version='4.1.0')
+                                framework_version='5.0.0')
     chainer_estimator.fit('s3://my_bucket/my_training_data/')
 
     # Deploy my estimator to a SageMaker Endpoint and get a Predictor
@@ -631,38 +631,38 @@ This Python version applies to both the Training Job, created by fit, and the En
 
 The Chainer Docker images have the following dependencies installed:
 
-+-----------------------------+-------------+
-| Dependencies                | chainer 4.0 |
-+-----------------------------+-------------+
-| chainer                     | 4.0.0       |
-+-----------------------------+-------------+
-| chainercv                   | 0.9.0       |
-+-----------------------------+-------------+
-| chainermn                   | 1.2.0       |
-+-----------------------------+-------------+
-| CUDA (GPU image only)       | 9.0         |
-+-----------------------------+-------------+
-| cupy                        | 4.0.0       |
-+-----------------------------+-------------+
-| matplotlib                  | 2.2.0       |
-+-----------------------------+-------------+
-| mpi4py                      | 3.0.0       |
-+-----------------------------+-------------+
-| numpy                       | 1.14.3      |
-+-----------------------------+-------------+
-| opencv-python               | 3.4.0.12    |
-+-----------------------------+-------------+
-| Pillow                      | 5.1.0       |
-+-----------------------------+-------------+
-| Python                      | 2.7 or 3.5  |
-+-----------------------------+-------------+
++-----------------------------+-------------+-------------+-------------+
+| Dependencies                | chainer 4.0 | chainer 4.1 | chainer 5.0 |
++-----------------------------+-------------+-------------+-------------+
+| chainer                     | 4.0.0       | 4.1.0       | 5.0.0       |
++-----------------------------+-------------+-------------+-------------+
+| chainercv                   | 0.9.0       | 0.10.0      | 0.10.0      |
++-----------------------------+-------------+-------------+-------------+
+| chainermn                   | 1.2.0       | 1.3.0       | N/A         |
++-----------------------------+-------------+-------------+-------------+
+| CUDA (GPU image only)       | 9.0         | 9.0         | 9.0         |
++-----------------------------+-------------+-------------+-------------+
+| cupy                        | 4.0.0       | 4.1.0       | 5.0.0       |
++-----------------------------+-------------+-------------+-------------+
+| matplotlib                  | 2.2.0       | 2.2.0       | 2.2.0       |
++-----------------------------+-------------+-------------+-------------+
+| mpi4py                      | 3.0.0       | 3.0.0       | 3.0.0       |
++-----------------------------+-------------+-------------+-------------+
+| numpy                       | 1.14.3      | 1.15.3      | 1.15.4      |
++-----------------------------+-------------+-------------+-------------+
+| opencv-python               | 3.4.0.12    | 3.4.0.12    | 3.4.0.12    |
++-----------------------------+-------------+-------------+-------------+
+| Pillow                      | 5.1.0       | 5.3.0       | 5.3.0       |
++-----------------------------+-------------+-------------+-------------+
+| Python                      | 2.7 or 3.5  | 2.7 or 3.5  | 2.7 or 3.5  |
++-----------------------------+-------------+-------------+-------------+
 
 The Docker images extend Ubuntu 16.04.
 
-You can select version of Chainer by passing a framework_version keyword arg to the Chainer Estimator constructor.
-Currently supported versions are listed in the above table. You can also set framework_version to only specify major and
-minor version, which will cause your training script to be run on the latest supported patch version of that minor
-version.
+You must select a version of Chainer by passing a ``framework_version`` keyword arg to the Chainer Estimator
+constructor. Currently supported versions are listed in the above table. You can also set framework_version to only
+specify major and minor version, which will cause your training script to be run on the latest supported patch
+version of that minor version.
 
 Alternatively, you can build your own image by following the instructions in the SageMaker Chainer containers
 repository, and passing ``image_name`` to the Chainer Estimator constructor.

src/sagemaker/chainer/estimator.py

@@ -35,6 +35,8 @@ class Chainer(Framework):
     _process_slots_per_host = "sagemaker_process_slots_per_host"
     _additional_mpi_options = "sagemaker_additional_mpi_options"
 
+    LATEST_VERSION = '5.0.0'
+
     def __init__(self, entry_point, use_mpi=None, num_processes=None, process_slots_per_host=None,
                  additional_mpi_options=None, source_dir=None, hyperparameters=None, py_version='py3',
                  framework_version=None, image_name=None, **kwargs):
@@ -82,7 +84,7 @@ def __init__(self, entry_point, use_mpi=None, num_processes=None, process_slots_
             **kwargs: Additional kwargs passed to the :class:`~sagemaker.estimator.Framework` constructor.
         """
         if framework_version is None:
-            logger.warning(empty_framework_version_warning(CHAINER_VERSION, CHAINER_VERSION))
+            logger.warning(empty_framework_version_warning(CHAINER_VERSION, self.LATEST_VERSION))
         self.framework_version = framework_version or CHAINER_VERSION
 
         super(Chainer, self).__init__(entry_point, source_dir, hyperparameters,

tests/conftest.py

@@ -20,7 +20,8 @@
 
 from sagemaker import Session
 from sagemaker.local import LocalSession
-from sagemaker.chainer.defaults import CHAINER_VERSION
+from sagemaker.chainer import Chainer
+from sagemaker.mxnet import MXNet
 from sagemaker.pytorch.defaults import PYTORCH_VERSION
 from sagemaker.tensorflow.defaults import TF_VERSION
 
@@ -33,8 +34,8 @@ def pytest_addoption(parser):
     parser.addoption('--sagemaker-runtime-config', action='store', default=None)
     parser.addoption('--boto-config', action='store', default=None)
     parser.addoption('--tf-full-version', action='store', default=TF_VERSION)
-    parser.addoption('--mxnet-full-version', action='store', default='1.3.0')
-    parser.addoption('--chainer-full-version', action='store', default=CHAINER_VERSION)
+    parser.addoption('--mxnet-full-version', action='store', default=MXNet.LATEST_VERSION)
+    parser.addoption('--chainer-full-version', action='store', default=Chainer.LATEST_VERSION)
     parser.addoption('--pytorch-full-version', action='store', default=PYTORCH_VERSION)
 
 
@@ -96,7 +97,7 @@ def pytorch_version(request):
     return request.param
 
 
-@pytest.fixture(scope='module', params=['4.0', '4.0.0', '4.1', '4.1.0'])
+@pytest.fixture(scope='module', params=['4.0', '4.0.0', '4.1', '4.1.0', '5.0', '5.0.0'])
 def chainer_version(request):
     return request.param
 

tests/unit/test_chainer.py

@@ -468,4 +468,4 @@ def test_empty_framework_version(warning, sagemaker_session):
                         framework_version=None)
 
     assert estimator.framework_version == defaults.CHAINER_VERSION
-    warning.assert_called_with(defaults.CHAINER_VERSION, defaults.CHAINER_VERSION)
+    warning.assert_called_with(defaults.CHAINER_VERSION, Chainer.LATEST_VERSION)