Merge pull request #889 from docker/legacy-work

Bump to 0.9.1

Author: dmp42

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Docker-registry
 
+## 0.9.1
+
+ * fixed database initialization issues
+ * move away from rsa to m2crypto
+ * s3 enhancements and bug fixes
+ * enforce additional checks on image ids
+
 ## 0.9.0
 
  * "loose" dependencies mechanism (DEPS=loose environment var will let you install without strictly versioned dependencies)

README.md

@@ -13,7 +13,7 @@ As the documentation evolves with different registry versions, be sure that befo
  * check which version of the registry you are running
  * switch to the corresponding tag to access the README that matches your product version
 
-The stable, released version is the [0.9.0 tag](https://github.com/docker/docker-registry/tree/0.9.0).
+The stable, released version is the [0.9.1 tag](https://github.com/docker/docker-registry/tree/0.9.1).
 
 Please also have a quick look at the [FAQ](FAQ.md) before reporting bugs.
 
@@ -224,11 +224,7 @@ common:
 On initialization, the `SQLAlchemyIndex` class checks the database
 version.  If the database doesn't exist yet (or does exist, but lacks
 a `version` table), the `SQLAlchemyIndex` creates the database and
-required tables.  To avoid several Gunicorn workers racing to create
-the database, you should launch your registry with
-[--preload][gunicorn-preload].  For example:
-
-    $ docker run -e GUNICORN_OPTS=[--preload] -p 5000:5000 registry
+required tables.
 
 ## Mirroring Options
 
@@ -369,5 +365,4 @@ Read [contributing](CONTRIBUTING.md)
 [search-endpoint]: http://docs.docker.com/reference/api/docker-io_api/#search
 [SQLAlchemy]: http://docs.sqlalchemy.org/
 [create_engine]: http://docs.sqlalchemy.org/en/latest/core/engines.html#sqlalchemy.create_engine
-[gunicorn-preload]: http://gunicorn-docs.readthedocs.org/en/latest/settings.html#preload-app
 

docker_registry/images.py

@@ -172,6 +172,7 @@ def _valid_bytes_range(bytes_range):
 @app.route('/v1/images/<image_id>/layer', methods=['GET'])
 @toolkit.requires_auth
 @require_completion
+@toolkit.valid_image_id
 @set_cache_headers
 @mirroring.source_lookup(cache=True, stream=True)
 def get_image_layer(image_id, headers):
@@ -193,6 +194,7 @@ def get_image_layer(image_id, headers):
 
 @app.route('/v1/images/<image_id>/layer', methods=['PUT'])
 @toolkit.requires_auth
+@toolkit.valid_image_id
 def put_image_layer(image_id):
     client_version = toolkit.docker_client_version()
     if client_version and client_version < (0, 10):
@@ -227,6 +229,7 @@ def put_image_layer(image_id):
 
 @app.route('/v1/images/<image_id>/checksum', methods=['PUT'])
 @toolkit.requires_auth
+@toolkit.valid_image_id
 def put_image_checksum(image_id):
     checksum = flask.request.headers.get('X-Docker-Checksum-Payload')
     if checksum is None:
@@ -256,6 +259,7 @@ def put_image_checksum(image_id):
 
 @app.route('/v1/images/<image_id>/json', methods=['GET'])
 @toolkit.requires_auth
+@toolkit.valid_image_id
 @require_completion
 @set_cache_headers
 @mirroring.source_lookup(cache=True, stream=False)
@@ -274,6 +278,7 @@ def get_image_json(image_id, headers):
 
 @app.route('/v1/images/<image_id>/ancestry', methods=['GET'])
 @toolkit.requires_auth
+@toolkit.valid_image_id
 @require_completion
 @set_cache_headers
 @mirroring.source_lookup(cache=True, stream=False)
@@ -325,6 +330,7 @@ def load_checksums(image_id):
 
 @app.route('/v1/images/<image_id>/json', methods=['PUT'])
 @toolkit.requires_auth
+@toolkit.valid_image_id
 def put_image_json(image_id):
     data = None
     try:
@@ -372,6 +378,7 @@ def put_image_json(image_id):
 
 @app.route('/v1/images/<image_id>/files', methods=['GET'])
 @toolkit.requires_auth
+@toolkit.valid_image_id
 @require_completion
 @set_cache_headers
 def get_image_files(image_id, headers):
@@ -392,6 +399,7 @@ def get_image_files(image_id, headers):
 
 @app.route('/v1/images/<image_id>/diff', methods=['GET'])
 @toolkit.requires_auth
+@toolkit.valid_image_id
 @require_completion
 @set_cache_headers
 def get_image_diff(image_id, headers):

docker_registry/server/__init__.py

@@ -5,7 +5,7 @@
 __credits__ = []
 
 __license__ = 'Apache 2.0'
-__version__ = '0.9.0'
+__version__ = '0.9.1'
 __maintainer__ = __author__
 __email__ = 'docker-dev@googlegroups.com'
 __status__ = 'Production'

docker_registry/toolkit.py

@@ -8,6 +8,7 @@
 import random
 import re
 import string
+import time
 import urllib
 
 import flask
@@ -25,6 +26,17 @@
 logger = logging.getLogger(__name__)
 _re_docker_version = re.compile('docker/([^\s]+)')
 _re_authorization = re.compile(r'(\w+)[:=][\s"]?([^",]+)"?')
+_re_hex_image_id = re.compile(r'^([a-f0-9]{16}|[a-f0-9]{64})$')
+
+
+def valid_image_id(f):
+    @functools.wraps(f)
+    def wrapper(*args, **kwargs):
+        image_id = kwargs.get('image_id', '')
+        if _re_hex_image_id.match(image_id):
+            return f(*args, **kwargs)
+        return api_error("Invalid image ID", 404)
+    return wrapper
 
 
 def docker_client_version():
@@ -299,14 +311,23 @@ def exclusive_lock(f):
     @functools.wraps(f)
     def wrapper(*args, **kwargs):
         lock_path = os.path.join(
-            '/var/lock', 'registry.{0}.lock'.format(f.func_name)
+            './', 'registry.{0}.lock'.format(f.func_name)
         )
         if os.path.exists(lock_path):
+            x = 0
+            while os.path.exists(lock_path) and x < 100:
+                logger.warn('Another process is creating the search database')
+                x += 1
+                time.sleep(1)
+            if x == 100:
+                raise Exception('Timedout waiting for db init')
             return
         lock_file = open(lock_path, 'w')
         lock_file.close()
-        result = f(*args, **kwargs)
-        os.remove(lock_path)
+        try:
+            result = f(*args, **kwargs)
+        finally:
+            os.remove(lock_path)
         return result
     return wrapper
 

requirements/main.txt

@@ -1,7 +1,7 @@
 blinker==1.3
 Flask==0.10.1
 gevent==1.0.1
-gunicorn==19.1
+gunicorn==19.1.1
 PyYAML==3.11
 requests==2.3.0
 M2Crypto==0.22.3

tests/base.py

@@ -35,6 +35,10 @@ def gen_random_string(self, length=16):
         return ''.join([random.choice(string.ascii_uppercase + string.digits)
                         for x in range(length)]).lower()
 
+    def gen_hex_string(self, length=16):
+        return ''.join([random.choice(string.hexdigits)
+                        for x in range(length)]).lower()
+
     def set_image_checksum(self, image_id, checksum):
         headers = {'X-Docker-Checksum-Payload': checksum}
         url = '/v1/images/{0}/checksum'.format(image_id)

tests/test_images.py

@@ -14,14 +14,14 @@
 class TestImages(base.TestCase):
 
     def test_unset_nginx_accel_redirect_layer(self):
-        image_id = self.gen_random_string()
+        image_id = self.gen_hex_string()
         layer_data = self.gen_random_string(1024)
         self.upload_image(image_id, parent_id=None, layer=layer_data)
         resp = self.http_client.get('/v1/images/{0}/layer'.format(image_id))
         self.assertEqual(layer_data, resp.data)
 
     def test_nginx_accel_redirect_layer(self):
-        image_id = self.gen_random_string()
+        image_id = self.gen_hex_string()
         layer_data = self.gen_random_string(1024)
         self.upload_image(image_id, parent_id=None, layer=layer_data)
 
@@ -48,8 +48,8 @@ def test_nginx_accel_redirect_layer(self):
             images.cfg._config.pop('nginx_x_accel_redirect')
 
     def test_simple(self):
-        image_id = self.gen_random_string()
-        parent_id = self.gen_random_string()
+        image_id = self.gen_hex_string()
+        parent_id = self.gen_hex_string()
         layer_data = self.gen_random_string(1024)
         self.upload_image(parent_id, parent_id=None, layer=layer_data)
         self.upload_image(image_id, parent_id=parent_id, layer=layer_data)
@@ -67,7 +67,7 @@ def test_notfound(self):
         self.assertEqual(resp.status_code, 404, resp.data)
 
     def test_bytes_range(self):
-        image_id = self.gen_random_string()
+        image_id = self.gen_hex_string()
         layer_data = self.gen_random_string(1024)
         b = random.randint(0, len(layer_data) / 2)
         bytes_range = (b, random.randint(b + 1, len(layer_data) - 1))
@@ -88,7 +88,7 @@ def before_put_image_json_handler_not_ok(self, sender, image_json):
         return "Not ok"
 
     def test_before_put_image_json_ok(self):
-        image_id = self.gen_random_string()
+        image_id = self.gen_hex_string()
         json_obj = {
             'id': image_id
         }
@@ -100,7 +100,7 @@ def test_before_put_image_json_ok(self):
             self.assertEqual(resp.status_code, 200, resp.data)
 
     def test_before_put_image_json_not_ok(self):
-        image_id = self.gen_random_string()
+        image_id = self.gen_hex_string()
         json_obj = {
             'id': image_id
         }

tests/test_mirrors.py

@@ -17,15 +17,16 @@ def mock_lookup_source(path, stream=False, source=None):
     resp.status_code = 200
     resp._content_consumed = True
     # resp.headers['X-Fake-Source-Header'] = 'foobar'
-    if path.endswith('0145/layer'):
+    if path.endswith('01451234/layer'):
         resp._content = "abcdef0123456789xxxxxx=-//"
-    elif path.endswith('0145/json'):
-        resp._content = ('{"id": "cafebabe0145","created":"2014-02-03T16:47:06'
-                         '.615279788Z"}')
-    elif path.endswith('0145/ancestry'):
-        resp._content = '["cafebabe0145"]'
+    elif path.endswith('01451234/json'):
+        resp._content = ('{"id": "cafebabe01451234",'
+                         '"created":"2014-02-03T16:47:06.615279788Z"}')
+    elif path.endswith('01451234/ancestry'):
+        resp._content = '["cafebabe01451234"]'
     elif path.endswith('test/tags'):
-        resp._content = '{"latest": "cafebabe0145", "0.1.2": "cafebabe0145"}'
+        resp._content = ('{"latest": "cafebabe01451234", "0.1.2": '
+                         '"cafebabe01451234"}')
     else:
         resp.status_code = 404
 
@@ -52,24 +53,24 @@ def test_is_mirror(self):
     @mock.patch('docker_registry.lib.mirroring.lookup_source',
                 mock_lookup_source)
     def test_source_lookup(self):
-        resp = self.http_client.get('/v1/images/cafebabe0145/layer')
+        resp = self.http_client.get('/v1/images/cafebabe01451234/layer')
         self.assertEqual(resp.status_code, 200)
         self.assertEqual(resp.data, "abcdef0123456789xxxxxx=-//")
 
-        resp_2 = self.http_client.get('/v1/images/cafebabe0145/json')
+        resp_2 = self.http_client.get('/v1/images/cafebabe01451234/json')
         self.assertEqual(resp_2.status_code, 200)
         # Note(dmp): unicode patch XXX not applied assume requests does the job
         json_data = json.loads(resp_2.data)
         assert 'id' in json_data
         assert 'created' in json_data
-        self.assertEqual(json_data['id'], 'cafebabe0145')
+        self.assertEqual(json_data['id'], 'cafebabe01451234')
 
-        resp_3 = self.http_client.get('/v1/images/cafebabe0145/ancestry')
+        resp_3 = self.http_client.get('/v1/images/cafebabe01451234/ancestry')
         self.assertEqual(resp_3.status_code, 200)
         # Note(dmp): unicode patch XXX not applied assume requests does the job
         json_data_2 = json.loads(resp_3.data)
         self.assertEqual(len(json_data_2), 1)
-        self.assertEqual(json_data_2[0], 'cafebabe0145')
+        self.assertEqual(json_data_2[0], 'cafebabe01451234')
 
         resp_4 = self.http_client.get('/v1/images/doe587e8157/json')
         self.assertEqual(resp_4.status_code, 404)
@@ -81,7 +82,7 @@ def test_source_lookup_tag(self):
         self.assertEqual(resp.status_code, 200)
         self.assertEqual(
             resp.data,
-            '{"latest": "cafebabe0145", "0.1.2": "cafebabe0145"}'
+            '{"latest": "cafebabe01451234", "0.1.2": "cafebabe01451234"}'
         )
 
         resp_2 = self.http_client.get('/v1/repositories/testing/bogus/tags')

tests/test_tags.py

@@ -11,7 +11,7 @@ class TestTags(base.TestCase):
     def test_simple(self, repos_name=None):
         if repos_name is None:
             repos_name = self.gen_random_string()
-        image_id = self.gen_random_string()
+        image_id = self.gen_hex_string()
         layer_data = self.gen_random_string(1024)
         self.upload_image(image_id, parent_id=None, layer=layer_data)