Merge pull request #600 from johanneswuerbach/retry-s3

Add read retries to avoid problems related to s3 eventual consistency

Author: dmp42

docker_registry/drivers/s3.py

@@ -11,8 +11,8 @@
 gevent.monkey.patch_all()
 
 import docker_registry.core.boto as coreboto
-# from docker_registry.core import exceptions
 from docker_registry.core import compat
+from docker_registry.core import exceptions
 from docker_registry.core import lru
 
 import logging
@@ -146,3 +146,13 @@ def content_redirect_url(self, path):
 
         # Have cloudfront? Sign it
         return self.signer(path, expire_time=60)
+
+    def get_content(self, path, tries=0):
+        try:
+            return super(Storage, self).get_content(path)
+        except exceptions.FileNotFoundError as e:
+            if tries <= 3:
+                time.sleep(.1)
+                return self.get_content(path, tries + 1)
+            else:
+                raise e

tests/test_s3.py

@@ -2,6 +2,7 @@
 
 import StringIO
 import sys
+import time
 
 from nose import tools
 
@@ -137,3 +138,38 @@ def test_get_tags(self):
 
         tag_content = store.get_content(tag_path)
         assert tag_content == 'randomdata'
+
+    def test_consistency_latency(self):
+        self.testCount = -1
+        mockKey = mock_boto.Key()
+
+        def mockExists():
+            self.testCount += 1
+            return self.testCount == 1
+        mockKey.exists = mockExists
+        mockKey.get_contents_as_string = lambda: "Foo bar"
+        self._storage.makeKey = lambda x: mockKey
+        startTime = time.time()
+
+        content = self._storage.get_content("/FOO")
+
+        waitTime = time.time() - startTime
+        assert waitTime >= 0.1, ("Waiting time was less than %sms "
+                                 "(actual : %sms)" %
+                                 (0.1 * 1000, waitTime * 1000))
+        assert content == "Foo bar", ("expected : %s; actual: %s" %
+                                      ("Foo bar", content))
+
+    @tools.raises(exceptions.FileNotFoundError)
+    def test_too_many_read_retries(self):
+        self.testCount = -1
+        mockKey = mock_boto.Key()
+
+        def mockExists():
+            self.testCount += 1
+            return self.testCount == 5
+        mockKey.exists = mockExists
+        mockKey.get_contents_as_string = lambda: "Foo bar"
+        self._storage.makeKey = lambda x: mockKey
+
+        self._storage.get_content("/FOO")