feat: Add option to specify a prefix to be prepended to object paths (#190)

Adds an optional configuration environment variable `PREFIX_LEADING_DIRECTORY_PATH`. If defined, this prefix is prepended to all S3 object paths. This is intended as a simple way to allow only some subset of a bucket to be served by the gateway. When used in combination with the existing STRIP_LEADING_DIRECTORY_PATH option, this effectively allows the leading prefix to be replaced, rather than just removed.

Special thanks to @hoffmanr-cshs for this thoughtful contribution.
---------

Co-authored-by: Javier Evans <j.evans@f5.com>

Author: hoffmanr-cshs

Dockerfile.buildkit.plus

@@ -12,6 +12,8 @@ ENV PROXY_CACHE_VALID_NOTFOUND "1m"
 ENV PROXY_CACHE_VALID_FORBIDDEN "30s"
 ENV CORS_ENABLED 0
 ENV DIRECTORY_LISTING_PATH_PREFIX ""
+ENV STRIP_LEADING_DIRECTORY_PATH ""
+ENV PREFIX_LEADING_DIRECTORY_PATH ""
 
 COPY plus/usr /usr
 

Dockerfile.oss

@@ -11,6 +11,7 @@ ENV PROXY_CACHE_VALID_FORBIDDEN "30s"
 ENV CORS_ENABLED 0
 ENV DIRECTORY_LISTING_PATH_PREFIX ""
 ENV STRIP_LEADING_DIRECTORY_PATH ""
+ENV PREFIX_LEADING_DIRECTORY_PATH ""
 
 # We modify the nginx base image by:
 # 1. Adding configuration files needed for proxying private S3 buckets

Dockerfile.plus

@@ -13,6 +13,7 @@ ENV PROXY_CACHE_VALID_FORBIDDEN "30s"
 ENV CORS_ENABLED 0
 ENV DIRECTORY_LISTING_PATH_PREFIX ""
 ENV STRIP_LEADING_DIRECTORY_PATH ""
+ENV PREFIX_LEADING_DIRECTORY_PATH ""
 
 COPY plus/etc/ssl /etc/ssl
 COPY plus/usr /usr

common/etc/nginx/nginx.conf

@@ -32,6 +32,7 @@ env PROXY_CACHE_VALID_FORBIDDEN;
 env HEADER_PREFIXES_TO_STRIP;
 env FOUR_O_FOUR_ON_EMPTY_BUCKET;
 env STRIP_LEADING_DIRECTORY_PATH;
+env PREFIX_LEADING_DIRECTORY_PATH;
 
 events {
     worker_connections  1024;

common/etc/nginx/templates/default.conf.template

@@ -13,10 +13,10 @@ map $request_uri $uri_full_path {
     "~^(?P<path>.*?)(\?.*)*$"  $path;
 }
 
-# Remove a portion of request URL (if configured)
+# Remove/replace a portion of request URL (if configured)
 map $uri_full_path $uri_path {
-	"~^$STRIP_LEADING_DIRECTORY_PATH(.*)"  $1;
-    default $uri_full_path;
+	"~^$STRIP_LEADING_DIRECTORY_PATH(.*)"  $PREFIX_LEADING_DIRECTORY_PATH$1;
+    default $PREFIX_LEADING_DIRECTORY_PATH$uri_full_path;
 }
 
 map $S3_STYLE $s3_host_hdr {

docs/getting_started.md

@@ -39,8 +39,9 @@ running as a Container or as a Systemd service.
 | `JS_TRUSTED_CERT_PATH`                | No        |                              |           | Enables the `js_fetch_trusted_certificate` directive when retrieving AWS credentials and sets the path (on the container) to the specified path                                                                                                                                                                                                                                                                                                              |
 | `HEADER_PREFIXES_TO_STRIP`            | No        |                              |           | A list of HTTP header prefixes that exclude headers client responses. List should be specified in lower-case and a semicolon (;) should be used to as a deliminator between values. For example: `x-goog-;x-something-`                                                                                                                                                                                                                                      |
 | `CORS_ENABLED`                        | No        | `true`, `false`              | `false`   | Flag that enables CORS headers on GET requests and enables pre-flight OPTIONS requests. If enabled, this will add CORS headers for "fully open" cross domain requests by default, meaning all domains are allowed, similar to the settings show in [this example](https://enable-cors.org/server_nginx.html). CORS settings can be fine-tuned by overwriting the [`cors.conf.template`](/common/etc/nginx/templates/gateway/cors.conf.template) file.        |
-| `CORS_ALLOWED_ORIGIN`                 | No        |                              |           | value to set to be returned from the CORS `Access-Control-Allow-Origin` header. This value is only used if CORS is enabled. (default: \*)                                                                                                                                                                                                                                                 
-| `STRIP_LEADING_DIRECTORY_PATH`        | No        |                              |           | Removes a portion of the path in the requested URL (if configured). Useful when deploying to an ALB under a folder (eg. www.mysite.com/somepath).
+| `CORS_ALLOWED_ORIGIN`                 | No        |                              |           | value to set to be returned from the CORS `Access-Control-Allow-Origin` header. This value is only used if CORS is enabled. (default: \*)                                                                                                                                                                                                                                                                                                                    |
+| `STRIP_LEADING_DIRECTORY_PATH`        | No        |                              |           | Removes a portion of the path in the requested URL (if configured). Useful when deploying to an ALB under a folder (eg. www.mysite.com/somepath).                                                                                                                                                                                                                                                                                                            |
+| `PREFIX_LEADING_DIRECTORY_PATH`       | No        |                              |           | Prefix to prepend to all S3 object paths. Useful to serve only a subset of an S3 bucket. When used in combination with `STRIP_LEADING_DIRECTORY_PATH`, this allows the leading path to be replaced, rather than just removed.                                                                                                                                                                                                                                |
 
 If you are using [AWS instance profile credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html),
 you will need to omit the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` variables from

standalone_ubuntu_oss_install.sh

@@ -173,6 +173,8 @@ PROXY_CACHE_VALID_FORBIDDEN=${PROXY_CACHE_VALID_FORBIDDEN:-'30s'}
 CORS_ENABLED=${CORS_ENABLED:-'false'}
 # Configure portion of URL to be removed (optional)
 STRIP_LEADING_DIRECTORY_PATH=${STRIP_LEADING_DIRECTORY_PATH:-''}
+# Configure portion of URL to be added to the beginning of the requested path (optional)
+PREFIX_LEADING_DIRECTORY_PATH=${PREFIX_LEADING_DIRECTORY_PATH:-''}
 EOF
 
 # By enabling CORS, we also need to enable the OPTIONS method which

test.sh

@@ -249,10 +249,13 @@ integration_test() {
   printf "\e[1m Integration test suite with APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=%s\e[22m\n" "$4"
   printf "\033[34;1m▶\033[0m"
   printf "\e[1m Integration test suite with STRIP_LEADING_DIRECTORY_PATH=%s\e[22m\n" "$5"
+  printf "\033[34;1m▶\033[0m"
+  printf "\e[1m Integration test suite with PREFIX_LEADING_DIRECTORY_PATH=%s\e[22m\n" "$6"
+  
 
   p "Starting Docker Compose Environment"
   # COMPOSE_COMPATIBILITY=true Supports older style compose filenames with _ vs -
-  COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 STRIP_LEADING_DIRECTORY_PATH=$5 compose up -d
+  COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 STRIP_LEADING_DIRECTORY_PATH=$5 PREFIX_LEADING_DIRECTORY_PATH=$6 compose up -d
 
   if [ "${wait_for_it_installed}" ]; then
     if [ -x "${wait_for_it_cmd}" ]; then
@@ -261,8 +264,8 @@ integration_test() {
   fi
 
   p "Starting HTTP API tests (v$1 signatures)"
-  echo "  test/integration/test_api.sh \"$test_server\" \"$test_dir\" $1 $2 $3 $4 $5"
-  bash "${test_dir}/integration/test_api.sh" "${test_server}" "${test_dir}" "$1" "$2" "$3" "$4" "$5";
+  echo "  test/integration/test_api.sh \"$test_server\" \"$test_dir\" $1 $2 $3 $4 $5 $6"
+  bash "${test_dir}/integration/test_api.sh" "${test_server}" "${test_dir}" "$1" "$2" "$3" "$4" "$5" "$6";
 
   # We check to see if NGINX is in fact using the correct version of AWS
   # signatures as it was configured to do.
@@ -404,41 +407,52 @@ runUnitTestWithSessionToken "s3gateway_test.js"
 integration_test_data
 
 p "Testing API with AWS Signature V2 and allow directory listing off"
-integration_test 2 0 0 0 ""
+integration_test 2 0 0 0 "" ""
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Testing API with AWS Signature V2 and allow directory listing on"
-integration_test 2 1 0 0 ""
+integration_test 2 1 0 0 "" ""
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Testing API with AWS Signature V2 and static site on"
-integration_test 2 0 1 0 ""
+integration_test 2 0 1 0 "" ""
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Testing API with AWS Signature V2 and allow directory listing on and append slash and allow index"
-integration_test 2 1 1 1 ""
+integration_test 2 1 1 1 "" ""
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Test API with AWS Signature V4 and allow directory listing off"
-integration_test 4 0 0 0 ""
+integration_test 4 0 0 0 "" ""
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Test API with AWS Signature V4 and allow directory listing on and appending /"
-integration_test 4 1 0 1 ""
+integration_test 4 1 0 1 "" ""
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Test API with AWS Signature V4 and static site on appending /"
-integration_test 4 0 1 1 ""
+integration_test 4 0 1 1 "" ""
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Testing API with AWS Signature V2 and allow directory listing off and prefix stripping on"
-integration_test 2 0 0 0 /my-bucket
+integration_test 2 0 0 0 /my-bucket ""
+
+compose stop nginx-s3-gateway # Restart with new config
+
+p "Test API with AWS Signature V4 and prefix leading directory path on"
+integration_test 4 0 0 0 "" "/b"
+
+p "Test API with AWS Signature V4 and prefix leading directory path on and prefix stripping on"
+integration_test 4 0 0 0 "/tostrip" "/b"
+
+p "Testing API with AWS Signature V2 and prefix leading directory path"
+integration_test 2 0 0 0 "" "/b"
 
 p "All integration tests complete"

test/docker-compose.yaml

@@ -27,6 +27,7 @@ services:
       PROVIDE_INDEX_PAGE:
       APPEND_SLASH_FOR_POSSIBLE_DIRECTORY:
       STRIP_LEADING_DIRECTORY_PATH:
+      PREFIX_LEADING_DIRECTORY_PATH:
       AWS_SIGS_VERSION:
       STATIC_SITE_HOSTING:
       PROXY_CACHE_MAX_SIZE: "10g"

test/integration/test_api.sh

@@ -26,6 +26,8 @@ allow_directory_list=$4
 index_page=$5
 append_slash=$6
 strip_leading_directory=$7
+prefix_leading_directory_path=$8
+
 test_fail_exit_code=2
 no_dep_exit_code=3
 checksum_length=32
@@ -159,6 +161,19 @@ for (( i=1; i<=3; i++ )); do
 done
 set -o errexit
 
+if [ -n "${prefix_leading_directory_path}" ]; then
+  assertHttpRequestEquals "GET" "/c/d.txt" "data/bucket-1/b/c/d.txt"
+
+  if [ -n "${strip_leading_directory}" ]; then
+    # When these two flags are used together, stripped value is basically
+    # replaced with the specified prefix
+    assertHttpRequestEquals "GET" "/tostrip/c/d.txt" "data/bucket-1/b/c/d.txt"
+  fi
+
+  # Exit early for this case since all tests following will fail because of the added prefix
+  exit 0
+fi
+
 # Ordinary filenames
 
 assertHttpRequestEquals "HEAD" "a.txt" "200"