feat: Add new configuration option STRIP_LEADING_DIRECTORY_PATH. (#185)

Many thanks to https://github.com/elJosho for this contribution.

This will strip the leading directory from a request if configured.  It's useful when you're trying to expose a bucket at a specific path on an ALB.  For example, if I have an ALB rule that forwards all traffic from "www.mysite.com/mybucket" to the S3 gateway, and I want to request a file in the root of my bucket, "myfile.txt".  Making a request to "www.mysite.com/mybucket/myfile.txt" will fail as the S3 gateway will look for the file inside a folder named "mybucket", rather than at the root of the bucket.  

After this change, setting the new env var to
`STRIP_LEADING_DIRECTORY_PATH = /mybucket`
will send all requests to the root of the bucket.

---------

Co-authored-by: Josh Watt <josh.watt@moodys.com>

Author: 4141done

Dockerfile.oss

@@ -10,6 +10,7 @@ ENV PROXY_CACHE_VALID_NOTFOUND "1m"
 ENV PROXY_CACHE_VALID_FORBIDDEN "30s"
 ENV CORS_ENABLED 0
 ENV DIRECTORY_LISTING_PATH_PREFIX ""
+ENV STRIP_LEADING_DIRECTORY_PATH ""
 
 # We modify the nginx base image by:
 # 1. Adding configuration files needed for proxying private S3 buckets

Dockerfile.plus

@@ -12,6 +12,7 @@ ENV PROXY_CACHE_VALID_NOTFOUND "1m"
 ENV PROXY_CACHE_VALID_FORBIDDEN "30s"
 ENV CORS_ENABLED 0
 ENV DIRECTORY_LISTING_PATH_PREFIX ""
+ENV STRIP_LEADING_DIRECTORY_PATH ""
 
 COPY plus/etc/ssl /etc/ssl
 COPY plus/usr /usr

common/docker-entrypoint.d/00-check-for-required-env.sh

@@ -35,7 +35,7 @@ if [[ -v AWS_CONTAINER_CREDENTIALS_RELATIVE_URI ]]; then
   echo "Running inside an ECS task, using container credentials"
 
 elif [[ -v S3_SESSION_TOKEN ]]; then
-  echo "Depreciated the S3_SESSION_TOKEN! Use the environment variable of AWS_SESSION_TOKEN instead"
+  echo "Deprecated the S3_SESSION_TOKEN! Use the environment variable of AWS_SESSION_TOKEN instead"
   failed=1
 
 elif [[ -v AWS_SESSION_TOKEN ]]; then
@@ -58,11 +58,11 @@ elif [[ -v AWS_WEB_IDENTITY_TOKEN_FILE ]]; then
   fi
 
 elif [[ -v S3_ACCESS_KEY_ID ]]; then
-  echo "Depreciated the S3_ACCESS_KEY_ID! Use the environment variable of AWS_ACCESS_KEY_ID instead"
+  echo "Deprecated the S3_ACCESS_KEY_ID! Use the environment variable of AWS_ACCESS_KEY_ID instead"
   failed=1
 
 elif [[ -v S3_SECRET_KEY ]]; then
-  echo "Depreciated the S3_SECRET_KEY! Use the environment variable of AWS_SECRET_ACCESS_KEY instead"
+  echo "Deprecated the S3_SECRET_KEY! Use the environment variable of AWS_SECRET_ACCESS_KEY instead"
   failed=1
 
 elif [[ -v AWS_SECRET_KEY ]]; then
@@ -77,7 +77,7 @@ else
 fi
 
 if [[ -v S3_DEBUG ]]; then
-  echo "Depreciated the S3_DEBUG! Use the environment variable of DEBUG instead"
+  echo "Deprecated the S3_DEBUG! Use the environment variable of DEBUG instead"
   failed=1
 fi
 

common/etc/nginx/nginx.conf

@@ -31,6 +31,7 @@ env PROXY_CACHE_VALID_NOTFOUND;
 env PROXY_CACHE_VALID_FORBIDDEN;
 env HEADER_PREFIXES_TO_STRIP;
 env FOUR_O_FOUR_ON_EMPTY_BUCKET;
+env STRIP_LEADING_DIRECTORY_PATH;
 
 events {
     worker_connections  1024;

common/etc/nginx/templates/default.conf.template

@@ -9,10 +9,16 @@ include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_js_vars.conf;
 # Extracts only the path from the requested URI. This strips out all query
 # parameters and anchors in order to prevent extraneous data from being sent
 # to S3.
-map $request_uri $uri_path {
+map $request_uri $uri_full_path {
     "~^(?P<path>.*?)(\?.*)*$"  $path;
 }
 
+# Remove a portion of request URL (if configured)
+map $uri_full_path $uri_path {
+	"~^$STRIP_LEADING_DIRECTORY_PATH(.*)"  $1;
+    default $uri_full_path;
+}
+
 map $S3_STYLE $s3_host_hdr {
     virtual "${S3_BUCKET_NAME}.${S3_SERVER}";
     path    "${S3_SERVER}:${S3_SERVER_PORT}";

docs/getting_started.md

@@ -39,8 +39,8 @@ running as a Container or as a Systemd service.
 | `JS_TRUSTED_CERT_PATH`                | No        |                              |           | Enables the `js_fetch_trusted_certificate` directive when retrieving AWS credentials and sets the path (on the container) to the specified path                                                                                                                                                                                                                                                                                                              |
 | `HEADER_PREFIXES_TO_STRIP`            | No        |                              |           | A list of HTTP header prefixes that exclude headers client responses. List should be specified in lower-case and a semicolon (;) should be used to as a deliminator between values. For example: `x-goog-;x-something-`                                                                                                                                                                                                                                      |
 | `CORS_ENABLED`                        | No        | `true`, `false`              | `false`   | Flag that enables CORS headers on GET requests and enables pre-flight OPTIONS requests. If enabled, this will add CORS headers for "fully open" cross domain requests by default, meaning all domains are allowed, similar to the settings show in [this example](https://enable-cors.org/server_nginx.html). CORS settings can be fine-tuned by overwriting the [`cors.conf.template`](/common/etc/nginx/templates/gateway/cors.conf.template) file.        |
-| `CORS_ALLOWED_ORIGIN`                 | No        |                              |           | value to set to be returned from the CORS `Access-Control-Allow-Origin` header. This value is only used if CORS is enabled. (default: \*)                                                                                                                                                                                                                                                                                                                    |
-
+| `CORS_ALLOWED_ORIGIN`                 | No        |                              |           | value to set to be returned from the CORS `Access-Control-Allow-Origin` header. This value is only used if CORS is enabled. (default: \*)                                                                                                                                                                                                                                                 
+| `STRIP_LEADING_DIRECTORY_PATH`        | No        |                              |           | Removes a portion of the path in the requested URL (if configured). Useful when deploying to an ALB under a folder (eg. www.mysite.com/somepath).
 
 If you are using [AWS instance profile credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html),
 you will need to omit the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` variables from
@@ -104,6 +104,13 @@ It will also redirect `/some/path` to `/some/path/` when S3 returns 404 on
 look like a possible directory, it must not start with a `.` and not have an 
 extension.  
 
+### Hosting a Bucket as a Subfolder on an ALB
+
+The `STRIP_LEADING_DIRECTORY_PATH` environment variable allows one to host an
+S3 bucket in a subfolder on an ALB.  For example, if you wanted to expose the
+root of a bucket under the path "www.mysite.com/somepath", you would set this
+variable to "/somepath".
+
 ## Running as a Systemd Service
 
 An [install script](/standalone_ubuntu_oss_install.sh) for the gateway shows

settings.example

@@ -2,7 +2,7 @@ S3_BUCKET_NAME=my-bucket
 AWS_ACCESS_KEY_ID=ZZZZZZZZZZZZZZZZZZZZ
 AWS_SECRET_ACCESS_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
 AWS_SESSION_TOKEN=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
-S3_SERVER=s3-us-east-1.amazonaws.com
+S3_SERVER=s3.us-east-1.amazonaws.com
 S3_SERVER_PORT=443
 S3_SERVER_PROTO=https
 S3_REGION=us-east-1
@@ -18,3 +18,4 @@ PROXY_CACHE_INACTIVE=60m
 PROXY_CACHE_VALID_OK=1h
 PROXY_CACHE_VALID_NOTFOUND=1m
 PROXY_CACHE_VALID_FORBIDDEN=30s
+STRIP_LEADING_DIRECTORY_PATH=/somepath

standalone_ubuntu_oss_install.sh

@@ -171,6 +171,8 @@ PROXY_CACHE_VALID_NOTFOUND=${PROXY_CACHE_VALID_NOTFOUND:-'1m'}
 PROXY_CACHE_VALID_FORBIDDEN=${PROXY_CACHE_VALID_FORBIDDEN:-'30s'}
 # Enables or disables CORS for the S3 Gateway (true=enabled, false=disabled)
 CORS_ENABLED=${CORS_ENABLED:-'false'}
+# Configure portion of URL to be removed (optional)
+STRIP_LEADING_DIRECTORY_PATH=${STRIP_LEADING_DIRECTORY_PATH:-''}
 EOF
 
 # By enabling CORS, we also need to enable the OPTIONS method which

test.sh

@@ -247,10 +247,12 @@ integration_test() {
   printf "\e[1m Integration test suite with PROVIDE_INDEX_PAGE=%s\e[22m\n" "$3"
   printf "\033[34;1m▶\033[0m"
   printf "\e[1m Integration test suite with APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=%s\e[22m\n" "$4"
+  printf "\033[34;1m▶\033[0m"
+  printf "\e[1m Integration test suite with STRIP_LEADING_DIRECTORY_PATH=%s\e[22m\n" "$5"
 
   p "Starting Docker Compose Environment"
   # COMPOSE_COMPATIBILITY=true Supports older style compose filenames with _ vs -
-  COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 compose up -d
+  COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 STRIP_LEADING_DIRECTORY_PATH=$5 compose up -d
 
   if [ "${wait_for_it_installed}" ]; then
     if [ -x "${wait_for_it_cmd}" ]; then
@@ -259,8 +261,8 @@ integration_test() {
   fi
 
   p "Starting HTTP API tests (v$1 signatures)"
-  echo "  test/integration/test_api.sh \"$test_server\" \"$test_dir\" $1 $2 $3 $4"
-  bash "${test_dir}/integration/test_api.sh" "${test_server}" "${test_dir}" "$1" "$2" "$3" "$4";
+  echo "  test/integration/test_api.sh \"$test_server\" \"$test_dir\" $1 $2 $3 $4 $5"
+  bash "${test_dir}/integration/test_api.sh" "${test_server}" "${test_dir}" "$1" "$2" "$3" "$4" "$5";
 
   # We check to see if NGINX is in fact using the correct version of AWS
   # signatures as it was configured to do.
@@ -402,36 +404,41 @@ runUnitTestWithSessionToken "s3gateway_test.js"
 integration_test_data
 
 p "Testing API with AWS Signature V2 and allow directory listing off"
-integration_test 2 0 0 0
+integration_test 2 0 0 0 ""
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Testing API with AWS Signature V2 and allow directory listing on"
-integration_test 2 1 0 0
+integration_test 2 1 0 0 ""
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Testing API with AWS Signature V2 and static site on"
-integration_test 2 0 1 0
+integration_test 2 0 1 0 ""
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Testing API with AWS Signature V2 and allow directory listing on and append slash and allow index"
-integration_test 2 1 1 1
+integration_test 2 1 1 1 ""
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Test API with AWS Signature V4 and allow directory listing off"
-integration_test 4 0 0 0
+integration_test 4 0 0 0 ""
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Test API with AWS Signature V4 and allow directory listing on and appending /"
-integration_test 4 1 0 1
+integration_test 4 1 0 1 ""
 
 compose stop nginx-s3-gateway # Restart with new config
 
 p "Test API with AWS Signature V4 and static site on appending /"
-integration_test 4 0 1 1
+integration_test 4 0 1 1 ""
+
+compose stop nginx-s3-gateway # Restart with new config
+
+p "Testing API with AWS Signature V2 and allow directory listing off and prefix stripping on"
+integration_test 2 0 0 0 /my-bucket
 
 p "All integration tests complete"

test/docker-compose.yaml

@@ -26,6 +26,7 @@ services:
       ALLOW_DIRECTORY_LIST:
       PROVIDE_INDEX_PAGE:
       APPEND_SLASH_FOR_POSSIBLE_DIRECTORY:
+      STRIP_LEADING_DIRECTORY_PATH:
       AWS_SIGS_VERSION:
       STATIC_SITE_HOSTING:
       PROXY_CACHE_MAX_SIZE: "10g"