Use aws credentials if available in examples (#1070)

Author: vishalbollu

examples/keras/document-denoiser/predictor.py

@@ -38,7 +38,11 @@ class PythonPredictor:
     def __init__(self, config):
         # download the model
         bucket, key = re.match("s3://(.+?)/(.+)", config["model"]).groups()
-        s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+
+        if os.environ.get("AWS_ACCESS_KEY_ID"):
+            s3 = boto3.client("s3")  # client will use your credentials if available
+        else:
+            s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))  # anonymous client
 
         model_path = os.path.join("/tmp/model.h5")
         s3.download_file(bucket, key, model_path)

examples/pytorch/iris-classifier/predictor.py

@@ -2,6 +2,7 @@
 
 import re
 import torch
+import os
 import boto3
 from botocore import UNSIGNED
 from botocore.client import Config
@@ -14,7 +15,12 @@ class PythonPredictor:
     def __init__(self, config):
         # download the model
         bucket, key = re.match("s3://(.+?)/(.+)", config["model"]).groups()
-        s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+
+        if os.environ.get("AWS_ACCESS_KEY_ID"):
+            s3 = boto3.client("s3")  # client will use your credentials if available
+        else:
+            s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))  # anonymous client
+
         s3.download_file(bucket, key, "/tmp/model.pth")
 
         # initialize the model

examples/sklearn/iris-classifier/README.md

@@ -59,6 +59,7 @@ $ python3 trainer.py
 ```python
 # predictor.py
 
+import os
 import boto3
 from botocore import UNSIGNED
 from botocore.client import Config
@@ -69,7 +70,11 @@ labels = ["setosa", "versicolor", "virginica"]
 
 class PythonPredictor:
     def __init__(self, config):
-        s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+        if os.environ.get("AWS_ACCESS_KEY_ID"):
+            s3 = boto3.client("s3")  # client will use your credentials if available
+        else:
+            s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))  # anonymous client
+
         s3.download_file(config["bucket"], config["key"], "/tmp/model.pkl")
         self.model = pickle.load(open("/tmp/model.pkl", "rb"))
 
@@ -343,6 +348,7 @@ First, implement `batch-predictor.py` with a `predict` function that can process
 ```python
 # batch-predictor.py
 
+import os
 import boto3
 from botocore import UNSIGNED
 from botocore.client import Config
@@ -353,7 +359,11 @@ labels = ["setosa", "versicolor", "virginica"]
 
 class PythonPredictor:
     def __init__(self, config):
-        s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+        if os.environ.get("AWS_ACCESS_KEY_ID"):
+            s3 = boto3.client("s3")  # client will use your credentials if available
+        else:
+            s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))  # anonymous client
+
         s3.download_file(config["bucket"], config["key"], "/tmp/model.pkl")
         self.model = pickle.load(open("/tmp/model.pkl", "rb"))
 

examples/sklearn/iris-classifier/batch-predictor.py

@@ -1,5 +1,6 @@
 # WARNING: you are on the master branch, please refer to the examples on the branch that matches your `cortex version`
 
+import os
 import boto3
 from botocore import UNSIGNED
 from botocore.client import Config
@@ -10,7 +11,11 @@
 
 class PythonPredictor:
     def __init__(self, config):
-        s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+        if os.environ.get("AWS_ACCESS_KEY_ID"):
+            s3 = boto3.client("s3")  # client will use your credentials if available
+        else:
+            s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))  # anonymous client
+
         s3.download_file(config["bucket"], config["key"], "/tmp/model.pkl")
         self.model = pickle.load(open("/tmp/model.pkl", "rb"))
 

examples/sklearn/iris-classifier/predictor.py

@@ -1,5 +1,6 @@
 # WARNING: you are on the master branch, please refer to the examples on the branch that matches your `cortex version`
 
+import os
 import boto3
 from botocore import UNSIGNED
 from botocore.client import Config
@@ -10,7 +11,11 @@
 
 class PythonPredictor:
     def __init__(self, config):
-        s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+        if os.environ.get("AWS_ACCESS_KEY_ID"):
+            s3 = boto3.client("s3")  # client will use your credentials if available
+        else:
+            s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))  # anonymous client
+
         s3.download_file(config["bucket"], config["key"], "/tmp/model.pkl")
         self.model = pickle.load(open("/tmp/model.pkl", "rb"))
 

examples/sklearn/mpg-estimator/predictor.py

@@ -14,8 +14,12 @@ def __init__(self, config):
         model_path = "/tmp/model"
         os.makedirs(model_path, exist_ok=True)
 
-        # download mlflow model folder from S3 using unsigned (anonymous) aws credentials
-        s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+        if os.environ.get("AWS_ACCESS_KEY_ID"):
+            s3 = boto3.client("s3")  # client will use your credentials if available
+        else:
+            s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))  # anonymous client
+
+        # download mlflow model folder from S3
         bucket, prefix = re.match("s3://(.+?)/(.+)", config["model"]).groups()
         response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix)
         for s3_obj in response["Contents"]:

examples/tensorflow/license-plate-reader/predictor_lite.py

@@ -15,7 +15,12 @@ class PythonPredictor:
     def __init__(self, config):
         # download yolov3 model
         bucket, key = re.match("s3://(.+?)/(.+)", config["yolov3"]).groups()
-        s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+
+        if os.environ.get("AWS_ACCESS_KEY_ID"):
+            s3 = boto3.client("s3")  # client will use your credentials if available
+        else:
+            s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))  # anonymous client
+
         model_path = "/tmp/model.h5"
         s3.download_file(bucket, key, model_path)
 

examples/tensorflow/text-generator/predictor.py

@@ -1,5 +1,6 @@
 # WARNING: you are on the master branch, please refer to the examples on the branch that matches your `cortex version`
 
+import os
 import boto3
 from botocore import UNSIGNED
 from botocore.client import Config
@@ -9,8 +10,13 @@
 class TensorFlowPredictor:
     def __init__(self, tensorflow_client, config):
         self.client = tensorflow_client
-        s3_client = boto3.client("s3", config=Config(signature_version=UNSIGNED))
-        self.encoder = get_encoder(s3_client)
+
+        if os.environ.get("AWS_ACCESS_KEY_ID"):
+            s3 = boto3.client("s3")  # client will use your credentials if available
+        else:
+            s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))  # anonymous client
+
+        self.encoder = get_encoder(s3)
 
     def predict(self, payload):
         model_input = {"context": [self.encoder.encode(payload["text"])]}