Exclude registry from manifest cache

ENABLE_MANIFEST_CACHING make it impossible to do roolups more freq than
once per manifest cache interval. Add exclude list, so that manifest
caching isn't applied for registries in this list.

Author: alexdev-wb

README.md

@@ -30,6 +30,8 @@ This proxy can be configured with the env var `ENABLE_MANIFEST_CACHE=true` which
 configurable caching of the manifest requests that DockerHub throttles. You can then fine-tune other parameters to your needs.
 Together with the possibility to centrally inject authentication (since 0.3x), this is probably one of the best ways to bring relief to your distressed cluster, while at the same time saving lots of bandwidth and time. 
 
+It is possible to disable manifest caching for your own private registry, see this [example](#exclude-registry-from-manifest-caching)
+
 Note: enabling manifest caching, in its default config, effectively makes some tags **immutable**. Use with care. The configuration ENVs are explained in the [Dockerfile](./Dockerfile), relevant parts included below.
 
 ```dockerfile
@@ -266,6 +268,19 @@ EOF
 k3d cluster create --config /etc/k3d-proxy-config.yaml
 ```
 
+### Exclude registry from manifest caching
+
+In some cases you may want to disable manifest caching for some registries (most preferably, for your private registry):
+
+```bash
+docker run --rm --name docker_registry_proxy -it \
+       -p 0.0.0.0:3128:3128 -e ENABLE_MANIFEST_CACHE=true \
+       -e MANIFEST_CACHE_EXCLUDE_HOSTS="private-0.registry.tld private-1.registry.tld" \
+       -v $(pwd)/docker_mirror_cache:/docker_mirror_cache \
+       -v $(pwd)/docker_mirror_certs:/ca \
+       rpardini/docker-registry-proxy:0.6.2
+```
+
 ## Configuring the Docker clients using Docker Desktop for Mac
 
 Separate instructions for Mac clients available in [this dedicated Doc Desktop for Mac document](Docker-for-Mac.md).

entrypoint.sh

@@ -115,6 +115,8 @@ echo -n "" >/etc/nginx/nginx.manifest.caching.config.conf
     # First tier caching of manifests; configure via MANIFEST_CACHE_PRIMARY_REGEX and MANIFEST_CACHE_PRIMARY_TIME
     location ~ ^/v2/(.*)/manifests/${MANIFEST_CACHE_PRIMARY_REGEX} {
         set \$docker_proxy_request_type "manifest-primary";
+        proxy_no_cache \$manifestcacheExclude;
+        proxy_cache_bypass \$manifestcacheExclude;
         proxy_cache_valid ${MANIFEST_CACHE_PRIMARY_TIME};
         include "/etc/nginx/nginx.manifest.stale.conf";
     }
@@ -124,6 +126,8 @@ EOD
     # Secondary tier caching of manifests; configure via MANIFEST_CACHE_SECONDARY_REGEX and MANIFEST_CACHE_SECONDARY_TIME
     location ~ ^/v2/(.*)/manifests/${MANIFEST_CACHE_SECONDARY_REGEX} {
         set \$docker_proxy_request_type "manifest-secondary";
+        proxy_no_cache \$manifestcacheExclude;
+        proxy_cache_bypass \$manifestcacheExclude;
         proxy_cache_valid ${MANIFEST_CACHE_SECONDARY_TIME};
         include "/etc/nginx/nginx.manifest.stale.conf";
     }
@@ -133,6 +137,8 @@ EOD
     # Default tier caching for manifests. Caches for ${MANIFEST_CACHE_DEFAULT_TIME} (from MANIFEST_CACHE_DEFAULT_TIME)
     location ~ ^/v2/(.*)/manifests/ {
         set \$docker_proxy_request_type "manifest-default";
+        proxy_no_cache \$manifestcacheExclude;
+        proxy_cache_bypass \$manifestcacheExclude;
         proxy_cache_valid ${MANIFEST_CACHE_DEFAULT_TIME};
         include "/etc/nginx/nginx.manifest.stale.conf";
     }
@@ -174,6 +180,17 @@ else
 EOF
 fi
 
+# Manifest cache exclude per host basis:
+## default 0 should always be here:
+echo "default 0;" > /etc/nginx/nginx.manifest.cache.exclude.map;
+if [[ "x$MANIFEST_CACHE_EXCLUDE_HOSTS" != "x" ]]; then
+    MANIFEST_CACHE_EXCLUDE_LIST=( $MANIFEST_CACHE_EXCLUDE_HOSTS )
+    for index in "${!MANIFEST_CACHE_EXCLUDE_LIST[@]}"; do
+        echo "\"${MANIFEST_CACHE_EXCLUDE_LIST[$index]}\" 1;";
+    done >> /etc/nginx/nginx.manifest.cache.exclude.map;
+fi
+
+
 # normally use non-debug version of nginx
 NGINX_BIN="/usr/sbin/nginx"
 

nginx.conf

@@ -129,6 +129,11 @@ http {
         default "DID_NOT_MATCH_PATH";
     }
 
+    # Do not use manifest caching for hosts in MANIFEST_CACHE_EXCLUDE_HOSTS
+    map $host $manifestcacheExclude {
+        include /etc/nginx/nginx.manifest.cache.exclude.map;
+    }
+
 
     # The proxy director layer, listens on 3128
     server {