feat: allow the AWS SDK to get auth from the environment (#88)

Co-authored-by: Julian Meinking <12785972+DrJume@users.noreply.github.com>

Author: Makeshift

.env

@@ -8,11 +8,9 @@ STORAGE_FILESYSTEM_PATH=.data/storage/filesystem
 # s3
 # STORAGE_DRIVER=s3
 # STORAGE_S3_BUCKET=test
-# STORAGE_S3_ENDPOINT=localhost
-# STORAGE_S3_PORT=9000
-# STORAGE_S3_USE_SSL=false
-# STORAGE_S3_ACCESS_KEY=minioadmin
-# STORAGE_S3_SECRET_KEY=minioadmin
+# AWS_ENDPOINT_URL=http://minio:9000
+# AWS_ACCESS_KEY_ID=minioadmin
+# AWS_SECRET_ACCESS_KEY=minioadmin
 
 # sqlite
 DB_DRIVER=sqlite
@@ -32,4 +30,4 @@ DB_SQLITE_PATH=.data/sqlite.db
 # DB_MYSQL_HOST=localhost
 # DB_MYSQL_USER=root
 # DB_MYSQL_PASSWORD=root
-# DB_MYSQL_PORT=3306
+# DB_MYSQL_PORT=3306

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.9'
-
 services:
   postgres:
     image: postgres:15
@@ -24,3 +22,25 @@ services:
     command: -c 'mkdir -p /data/test && /usr/bin/minio server /data'
     ports:
       - 9000:9000
+    environment:
+      MINIO_ROOT_USER: access_key
+      MINIO_ROOT_PASSWORD: secret_key
+
+  cache-server:
+    build:
+      dockerfile: Dockerfile
+      context: .
+    ports:
+      - '3000:3000'
+    depends_on:
+      - minio
+
+    environment:
+      API_BASE_URL: http://localhost:3000
+
+      STORAGE_DRIVER: s3
+      STORAGE_S3_BUCKET: test
+
+      AWS_ACCESS_KEY_ID: access_key
+      AWS_SECRET_ACCESS_KEY: secret_key
+      AWS_ENDPOINT_URL: http://minio:9000

docs/content/2.storage-drivers/s3.md

@@ -24,12 +24,10 @@ services:
 
       STORAGE_DRIVER: s3
       STORAGE_S3_BUCKET: gh-actions-cache
-      STORAGE_S3_ACCESS_KEY: access_key
-      STORAGE_S3_SECRET_KEY: secret_key
 
-      STORAGE_S3_ENDPOINT: minio
-      STORAGE_S3_PORT: '9000'
-      STORAGE_S3_USE_SSL: 'false'
+      AWS_ACCESS_KEY_ID: access_key
+      AWS_SECRET_ACCESS_KEY: secret_key
+      AWS_ENDPOINT_URL: http://minio:9000
     volumes:
       - cache-data:/app/.data
 
@@ -47,6 +45,8 @@ volumes:
 
 ### `docker-compose` AWS S3 example
 
+This example assumes that credentials are being provided by the environment, e.g. via an [instance profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) or [EKS IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html).
+
 ```yaml [docker-compose.yml]
 version: '3.9'
 
@@ -60,12 +60,7 @@ services:
 
       STORAGE_DRIVER: s3
       STORAGE_S3_BUCKET: gh-actions-cache
-      STORAGE_S3_ACCESS_KEY: access_key
-      STORAGE_S3_SECRET_KEY: secret_key
 
-      STORAGE_S3_ENDPOINT: s3.amazonaws.com
-      STORAGE_S3_PORT: '443'
-      STORAGE_S3_USE_SSL: 'true'
     volumes:
       - cache-data:/app/.data
 
@@ -75,46 +70,65 @@ volumes:
 
 ### Environment Variables
 
-Don't forget to set the `STORAGE_DRIVER` environment variable to `s3` to use the S3 storage driver.
+The only required S3-related environment variables are `STORAGE_DRIVER: s3` and `STORAGE_S3_BUCKET`. The rest of the environment variables are optional and depend on your S3-compatible storage provider.
+
+The AWS SDK will automatically use any AWS credentials available in the environment, e.g. `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_REGION`. Outside of AWS, these environment variables can still be used to authenticate with S3-compatible storage, as seen in the Minio example above.
+
+Common environment variables are listed below. For a full list of configuration options, see the [AWS SDK documentation](https://docs.aws.amazon.com/sdkref/latest/guide/settings-reference.html#EVarSettings).
 
 #### `STORAGE_S3_BUCKET`
 
 Example: `gh-actions-cache`
 
-The name of the S3 bucket used for storage.
+The name of the S3 bucket used for storage. This environment variable is always required.
 
-#### `STORAGE_S3_ACCESS_KEY`
+#### `AWS_REGION`
 
-Example: `access_key`
+Example: `us-east-1`
 
-The access key for S3 storage.
+The AWS SDK relies on this variable being set. In the cache server, it defaults to `us-east-1` if not provided. This has no effect if you are using a non-AWS S3-compatible storage provider, such as MinIO.
 
-#### `STORAGE_S3_SECRET_KEY`
+#### `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
 
-Example: `secret_key`
+Example:
+  `AWS_ACCESS_KEY_ID: access_key`
+  `AWS_SECRET_ACCESS_KEY: secret_key`
 
-The secret key for S3 storage.
+This is the access key/secret key used to authenticate with S3-compatible storage. If required to authenticate with your provider, these should be provided by the provider. Alternatively, you can use the `AWS_PROFILE` environment variable to specify a profile from your AWS credentials file.
 
-#### `STORAGE_S3_ENDPOINT`
+#### `AWS_PROFILE`
 
-Example: `s3.amazonaws.com`, `minio`
+Example: `my-profile`
 
-The endpoint hostname for S3 storage.
+If you wish to run the cache server locally and utilize a profile from your AWS credentials file or local AWS CLI configuration, you can set the `AWS_PROFILE` environment variable to the name of the profile. Note that this will also require mounting the AWS credentials file into the container in order for the SDK to be able to find it.
 
-#### `STORAGE_S3_REGION`
+```yaml [docker-compose.yml]
+version: '3.9'
 
-Example: `us-west-1`
+services:
+  cache-server:
+    image: ghcr.io/falcondev-oss/github-actions-cache-server:latest
+    ports:
+      - '3000:3000'
+    environment:
+      API_BASE_URL: http://localhost:3000
 
-The region for AWS S3. Not needed with MinIO.
+      STORAGE_DRIVER: s3
+      STORAGE_S3_BUCKET: gh-actions-cache
 
-#### `STORAGE_S3_PORT`
+      AWS_PROFILE: my-profile
 
-Example: `443`, `9000`
+    volumes:
+      - cache-data:/app/.data
+      # Mount the AWS CLI credentials and config into the container
+      - ~/.aws:/root/.aws:ro
 
-The port S3 storage is running on.
+volumes:
+  cache-data:
+```
 
-#### `STORAGE_S3_USE_SSL`
+#### `AWS_ENDPOINT_URL`
 
-Example: `false`
+Example: `http://minio:9000`
 
-Whether to use SSL for S3 storage connections.
+This is the endpoint URL for the S3-compatible storage. This is only required if you are using a non-AWS S3-compatible storage provider, such as MinIO.

lib/storage/drivers/s3.ts

@@ -18,25 +18,13 @@ import { streamToBuffer } from '~/lib/utils'
 export const s3Driver = defineStorageDriver({
   envSchema: z.object({
     STORAGE_S3_BUCKET: z.string().min(1),
-    STORAGE_S3_ENDPOINT: z.string().min(1),
-    STORAGE_S3_REGION: z.string().min(1).default('us-east-1'),
-    STORAGE_S3_PORT: z.coerce.number().positive(),
-    STORAGE_S3_USE_SSL: z.string().transform((v) => v === 'true'),
-    STORAGE_S3_ACCESS_KEY: z.string().min(1),
-    STORAGE_S3_SECRET_KEY: z.string().min(1),
+    // AWS SDK requires an AWS_REGION to be set, even if you're using a custom endpoint
+    AWS_REGION: z.string().default('us-east-1'),
   }),
   async setup(options) {
-    const protocol = options.STORAGE_S3_USE_SSL ? 'https' : 'http'
-    const port = options.STORAGE_S3_PORT ? `:${options.STORAGE_S3_PORT}` : ''
-
     const s3 = new S3Client({
-      credentials: {
-        secretAccessKey: options.STORAGE_S3_SECRET_KEY,
-        accessKeyId: options.STORAGE_S3_ACCESS_KEY,
-      },
-      endpoint: `${protocol}://${options.STORAGE_S3_ENDPOINT}${port}`,
-      region: options.STORAGE_S3_REGION,
       forcePathStyle: true,
+      region: options.AWS_REGION,
     })
 
     try {

tests/.env.s3.storage

@@ -1,7 +1,5 @@
 STORAGE_DRIVER=s3
 STORAGE_S3_BUCKET=test
-STORAGE_S3_ENDPOINT=localhost
-STORAGE_S3_PORT=9000
-STORAGE_S3_USE_SSL=false
-STORAGE_S3_ACCESS_KEY=minioadmin
-STORAGE_S3_SECRET_KEY=minioadmin
+AWS_ENDPOINT_URL=http://localhost:9000
+AWS_ACCESS_KEY_ID=minioadmin
+AWS_SECRET_ACCESS_KEY=minioadmin