Skip to content

Commit bea33dd

Browse files
akartskychuyang-deng
akartsky
authored and
chuyang-deng
committed
Util changes (#257)
1 parent 8e9b536 commit bea33dd

File tree

7 files changed

+187
-0
lines changed

7 files changed

+187
-0
lines changed

‎docker/2.0.0/py2/Dockerfile.cpu‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,4 +116,11 @@ RUN ${PIP} install --no-cache-dir -U \
116116
&& ${PIP} uninstall -y --no-cache-dir \
117117
markdown
118118

119+
COPY dockerd-entrypoint.py /usr/local/bin/dockerd-entrypoint.py
120+
COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py
121+
122+
RUN chmod +x /usr/local/bin/dockerd-entrypoint.py \
123+
&& chmod +x /usr/local/bin/deep_learning_container.py
124+
125+
ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"]
119126
CMD ["bin/bash"]

‎docker/2.0.0/py2/Dockerfile.gpu‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,4 +152,11 @@ RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_confi
152152
&& echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \
153153
&& mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
154154

155+
COPY dockerd-entrypoint.py /usr/local/bin/dockerd-entrypoint.py
156+
COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py
157+
158+
RUN chmod +x /usr/local/bin/dockerd-entrypoint.py \
159+
&& chmod +x /usr/local/bin/deep_learning_container.py
160+
161+
ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"]
155162
CMD ["bin/bash"]

‎docker/2.0.0/py2/dockerd-entrypoint.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
14+
from __future__ import absolute_import
15+
16+
import os.path
17+
import shlex
18+
import subprocess
19+
import sys
20+
21+
if not os.path.exists("/opt/ml/input/config"):
22+
subprocess.call(['python', '/usr/local/bin/deep_learning_container.py', '&>/dev/null', '&'])
23+
24+
subprocess.check_call(shlex.split(' '.join(sys.argv[1:])))

‎docker/2.0.0/py3/Dockerfile.cpu‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,4 +118,11 @@ RUN ${PIP} install --no-cache-dir -U \
118118
&& ${PIP} uninstall -y --no-cache-dir \
119119
markdown
120120

121+
COPY dockerd-entrypoint.py /usr/local/bin/dockerd-entrypoint.py
122+
COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py
123+
124+
RUN chmod +x /usr/local/bin/dockerd-entrypoint.py \
125+
&& chmod +x /usr/local/bin/deep_learning_container.py
126+
127+
ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"]
121128
CMD ["bin/bash"]

‎docker/2.0.0/py3/Dockerfile.gpu‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,4 +167,11 @@ RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_confi
167167
&& echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \
168168
&& mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
169169

170+
COPY dockerd-entrypoint.py /usr/local/bin/dockerd-entrypoint.py
171+
COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py
172+
173+
RUN chmod +x /usr/local/bin/dockerd-entrypoint.py \
174+
&& chmod +x /usr/local/bin/deep_learning_container.py
175+
176+
ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"]
170177
CMD ["bin/bash"]

‎docker/2.0.0/py3/dockerd-entrypoint.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
14+
from __future__ import absolute_import
15+
16+
import os.path
17+
import shlex
18+
import subprocess
19+
import sys
20+
21+
if not os.path.exists("/opt/ml/input/config"):
22+
subprocess.call(['python', '/usr/local/bin/deep_learning_container.py', '&>/dev/null', '&'])
23+
24+
subprocess.check_call(shlex.split(' '.join(sys.argv[1:])))

‎src/sagemaker_tensorflow_container/deep_learning_container.py‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
from __future__ import absolute_import
14+
15+
import json
16+
import logging
17+
import re
18+
19+
import requests
20+
21+
22+
def _validate_instance_id(instance_id):
23+
"""
24+
Validate instance ID
25+
"""
26+
compiled_regex = re.compile(r"^(i-\S{17})")
27+
match = compiled_regex.match(instance_id)
28+
29+
if not match:
30+
return None
31+
32+
return match.group(1)
33+
34+
35+
def _retrieve_instance_id():
36+
"""
37+
Retrieve instance ID from instance metadata service
38+
"""
39+
instance_id = None
40+
url = "http://169.254.169.254/latest/meta-data/instance-id"
41+
response = requests_helper(url, timeout=0.1)
42+
43+
if response is not None:
44+
instance_id = _validate_instance_id(response.text)
45+
46+
return instance_id
47+
48+
49+
def _retrieve_instance_region():
50+
"""
51+
Retrieve instance region from instance metadata service
52+
"""
53+
region = None
54+
valid_regions = ['ap-northeast-1', 'ap-northeast-2', 'ap-southeast-1', 'ap-southeast-2',
55+
'ap-south-1', 'ca-central-1', 'eu-central-1', 'eu-north-1',
56+
'eu-west-1', 'eu-west-2', 'eu-west-3', 'sa-east-1',
57+
'us-east-1', 'us-east-2', 'us-west-1', 'us-west-2']
58+
59+
url = "http://169.254.169.254/latest/dynamic/instance-identity/document"
60+
response = requests_helper(url, timeout=0.1)
61+
62+
if response is not None:
63+
response_json = json.loads(response.text)
64+
65+
if response_json['region'] in valid_regions:
66+
region = response_json['region']
67+
68+
return region
69+
70+
71+
def query_bucket():
72+
"""
73+
GET request on an empty object from an Amazon S3 bucket
74+
"""
75+
response = None
76+
instance_id = _retrieve_instance_id()
77+
region = _retrieve_instance_region()
78+
79+
if instance_id is not None and region is not None:
80+
url = "https://aws-deep-learning-containers-{0}.s3.{0}.amazonaws.com/dlc-containers.txt?x-instance-id={1}"\
81+
.format(region, instance_id)
82+
response = requests_helper(url, timeout=0.2)
83+
84+
logging.debug("Query bucket finished: {}".format(response))
85+
86+
return response
87+
88+
89+
def requests_helper(url, timeout):
90+
response = None
91+
try:
92+
response = requests.get(url, timeout=timeout)
93+
except requests.exceptions.RequestException as e:
94+
logging.error("Request exception: {}".format(e))
95+
96+
return response
97+
98+
99+
def main():
100+
"""
101+
Invoke bucket query
102+
"""
103+
# Logs are not necessary for normal run. Remove this line while debugging.
104+
logging.getLogger().disabled = True
105+
106+
logging.basicConfig(level=logging.ERROR)
107+
query_bucket()
108+
109+
110+
if __name__ == '__main__':
111+
main()

0 commit comments

Comments
 (0)