convex-backend PR 187: Refactor S3 credential handling in aws_utils (#40594)

Refactor S3 credential handling in aws_utils to allow S3 credentials from sources other than environment variables by using the AWS default credential chain.

Preserved credential checking before first use to prevent logs from filling up.

Also added `aws-credential-types` and `tokio` to dependencies in Cargo.toml and aws_utils/Cargo.toml

This PR preserves the same S3 client environment variable credential checks while allowing credential sources from IAM roles which is useful for local debugging (`aws sso`) and self-hosting on AWS (EC2 IAM role attachment).

The AWS S3 Rust SDK uses this default credential chain:  `env vars -> shared config/credentials (incl. SSO) -> web identity -> container creds -> EC2 IMDSv2`.  The new code checks the credential sources and bails with an error if none are found.

Closes #185

----

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.

Co-authored-by: Heath Hopkins <heath.hopkins@emory.edu>
GitOrigin-RevId: 2aa20649ba78df338cfae387a99f3e581f1d8e72

Author: 2 people

Cargo.lock

@@ -76,11 +76,12 @@ async-trait = "0.1"
 async_zip = { version = "0.0.17", default-features = false, features = [ "deflate", "tokio", "zstd" ] }
 async_zip_0_0_9 = { package = "async_zip", version = "0.0.9", default-features = false, features = [ "zstd", "deflate" ] }
 atomic_refcell = "0.1.13"
-aws-config = { version = "1.6", default-features = false, features = [ "client-hyper", "default-https-client", "rustls", "rt-tokio" ] }
+aws-config = { version = "1.6", default-features = false, features = [ "client-hyper", "default-https-client", "rustls", "rt-tokio", "sso" ] }
 aws-lc-rs = { version = "1.13", default-features = false, features = [ "aws-lc-sys", "prebuilt-nasm" ] }
 aws-sdk-s3 = { version = "1.83", default-features = false, features = [ "default-https-client", "rt-tokio", "sigv4a" ] }
 aws-smithy-http = "0.62.0"
 aws-smithy-types-convert = { version = "0.60", features = [ "convert-streams" ] }
+aws-credential-types = "1"
 aws-types = "1"
 axum = { version = "0.8", features = [ "ws", "original-uri", "macros", "multipart" ] }
 axum-extra = { version = "0.10", features = [ "typed-header", "cookie" ] }

Cargo.toml

@@ -7,10 +7,12 @@ license = "LicenseRef-FSL-1.1-Apache-2.0"
 [dependencies]
 anyhow = { workspace = true }
 aws-config = { workspace = true }
+aws-credential-types = { workspace = true }
 aws-sdk-s3 = { workspace = true }
 aws-smithy-types-convert = { workspace = true }
 aws-types = { workspace = true }
 futures = { workspace = true }
+tokio = { workspace = true }
 tracing = { workspace = true }
 
 [lints]

crates/aws_utils/Cargo.toml

@@ -0,0 +1,52 @@
+use std::env;
+
+use anyhow::{
+    Context,
+    Result,
+};
+use aws_config::BehaviorVersion;
+use aws_sdk_s3 as s3;
+use aws_utils::preflight_credentials;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // 1) Preflight: try to resolve credentials using the standard chain.
+    let _creds = preflight_credentials().await?;
+
+    println!(
+        "✅ Credentials resolved{}",
+        match env::var("AWS_PROFILE") {
+            Ok(p) => format!(" (AWS_PROFILE={p})"),
+            Err(_) => String::new(),
+        }
+    );
+
+    // 2) Load full config explicitly setting profile if available
+    let mut config_loader = aws_config::defaults(BehaviorVersion::latest());
+    if let Ok(profile) = env::var("AWS_PROFILE") {
+        config_loader = config_loader.profile_name(&profile);
+    }
+    let conf = config_loader.load().await;
+
+    // 3) Use S3 client safely now that we know creds exist.
+    let client = s3::Client::new(&conf);
+
+    // Example: list buckets
+    println!("Testing S3 access by listing buckets...");
+    let resp = client.list_buckets().send().await.context(
+        "S3 call failed (credentials may be invalid/expired or region/network misconfigured)",
+    )?;
+
+    println!("✅ S3 access successful!");
+    println!("Buckets:");
+    let buckets = resp.buckets();
+    if buckets.is_empty() {
+        println!(" (no buckets found)");
+    } else {
+        for b in buckets {
+            println!(" - {}", b.name().unwrap_or("<unnamed>"));
+        }
+    }
+
+    Ok(())
+}

crates/aws_utils/src/bin/demo_test_credentials.rs

@@ -1,15 +1,16 @@
-#![feature(coroutines)]
-#![feature(exit_status_error)]
+// #![feature(coroutines)]
+// #![feature(exit_status_error)]
 use std::{
     env,
     sync::LazyLock,
 };
 
 use aws_config::{
-    environment::credentials::EnvironmentVariableCredentialsProvider,
+    default_provider::credentials::DefaultCredentialsChain,
     BehaviorVersion,
     ConfigLoader,
 };
+use aws_credential_types::provider::ProvideCredentials;
 use aws_sdk_s3::config::Builder as S3ConfigBuilder;
 use aws_types::region::Region;
 
@@ -18,12 +19,6 @@ pub mod s3;
 static S3_ENDPOINT_URL: LazyLock<Option<String>> =
     LazyLock::new(|| env::var("S3_ENDPOINT_URL").ok());
 
-static AWS_ACCESS_KEY_ID: LazyLock<Option<String>> =
-    LazyLock::new(|| env::var("AWS_ACCESS_KEY_ID").ok());
-
-static AWS_SECRET_ACCESS_KEY: LazyLock<Option<String>> =
-    LazyLock::new(|| env::var("AWS_SECRET_ACCESS_KEY").ok());
-
 static AWS_REGION: LazyLock<Option<String>> = LazyLock::new(|| env::var("AWS_REGION").ok());
 
 static AWS_S3_FORCE_PATH_STYLE: LazyLock<bool> = LazyLock::new(|| {
@@ -50,25 +45,20 @@ static AWS_S3_DISABLE_CHECKSUMS: LazyLock<bool> = LazyLock::new(|| {
 /// Similar aws_config::from_env but returns an error if credentials or
 /// region is are not. It also doesn't spew out log lines every time
 /// credentials are accessed.
-pub fn must_config_from_env() -> anyhow::Result<ConfigLoader> {
+pub async fn must_config_from_env() -> anyhow::Result<ConfigLoader> {
     let Some(region) = AWS_REGION.clone() else {
         anyhow::bail!("AWS_REGION env variable must be set");
     };
     let region = Region::new(region);
-    let Some(_) = AWS_ACCESS_KEY_ID.clone() else {
-        anyhow::bail!("AWS_ACCESS_KEY_ID env variable must be set");
-    };
-    let Some(_) = AWS_SECRET_ACCESS_KEY.clone() else {
-        anyhow::bail!("AWS_SECRET_ACCESS_KEY env variable must be set");
-    };
-    let credentials = EnvironmentVariableCredentialsProvider::new();
-    Ok(aws_config::defaults(BehaviorVersion::v2025_01_17())
-        .region(region)
-        .credentials_provider(credentials))
+
+    // Check for credentials using the default provider chain
+    let _creds = preflight_credentials().await?;
+
+    Ok(aws_config::defaults(BehaviorVersion::v2025_01_17()).region(region))
 }
 
 pub async fn must_s3_config_from_env() -> anyhow::Result<S3ConfigBuilder> {
-    let base_config = must_config_from_env()?.load().await;
+    let base_config = must_config_from_env().await?.load().await;
     let mut s3_config_builder = S3ConfigBuilder::from(&base_config);
     if let Some(s3_endpoint_url) = S3_ENDPOINT_URL.clone() {
         s3_config_builder = s3_config_builder.endpoint_url(s3_endpoint_url);
@@ -77,6 +67,64 @@ pub async fn must_s3_config_from_env() -> anyhow::Result<S3ConfigBuilder> {
     Ok(s3_config_builder)
 }
 
+/// Attempts to resolve credentials using the default chain:
+/// env vars -> shared config/credentials (incl. SSO) -> web identity ->
+/// container creds -> EC2 IMDSv2. Returns early with a helpful error if nothing
+/// is available.
+pub async fn preflight_credentials() -> anyhow::Result<aws_credential_types::Credentials> {
+    let chain = DefaultCredentialsChain::builder().build().await;
+
+    match chain.provide_credentials().await {
+        Ok(creds) => Ok(creds),
+        Err(err) => {
+            // Give actionable hints based on common setups.
+            let profile = env::var("AWS_PROFILE").unwrap_or_else(|_| "default".to_string());
+            let mut help = String::new();
+            help.push_str("No AWS credentials were found by the default provider chain.\n\n");
+            help.push_str("Tried in this order:\n");
+            help.push_str(
+                "  1) Environment: AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY [/ \
+                 AWS_SESSION_TOKEN]\n",
+            );
+            help.push_str(
+                "  2) Shared config/credentials files (~/.aws/config, ~/.aws/credentials) ",
+            );
+            help.push_str(&format!("(profile: {profile})\n"));
+            help.push_str("     - If you use IAM Identity Center (SSO), run: aws sso login");
+            if profile != "default" {
+                help.push_str(&format!(" --profile {profile}"));
+            }
+            help.push('\n');
+            help.push_str(
+                "  3) Web identity (AssumeRoleWithWebIdentity; env/profiles with role_arn & \
+                 web_identity_token_file)\n",
+            );
+            help.push_str(
+                "  4) Container credentials (ECS/EKS env: AWS_CONTAINER_CREDENTIALS_* or Pod \
+                 Identity)\n",
+            );
+            help.push_str("  5) EC2 Instance Metadata (IMDSv2; instance role)\n\n");
+
+            help.push_str("Fixes:\n");
+            help.push_str(
+                "  • For access keys: set AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and \
+                 AWS_SESSION_TOKEN (optional)\n",
+            );
+            help.push_str(
+                "  • For profiles: set AWS_PROFILE or add a [profile] with credentials in \
+                 ~/.aws/credentials\n",
+            );
+            help.push_str("  • For SSO: aws configure sso && aws sso login\n");
+            help.push_str(
+                "  • For web identity: ensure web_identity_token_file and role_arn are set\n",
+            );
+            help.push_str("  • For containers/EC2: attach the proper task/IRSA/instance role\n");
+
+            anyhow::bail!("{}Underlying error: {}", help, err)
+        },
+    }
+}
+
 /// Returns true if server-side encryption headers should be disabled
 pub fn is_sse_disabled() -> bool {
     *AWS_S3_DISABLE_SSE

crates/aws_utils/src/lib.rs

@@ -33,7 +33,9 @@ impl S3Client {
         };
         let config = must_s3_config_from_env()
             .await
-            .context("AWS env variables are required when using AWS Lambda")?
+            .context(
+                "Failed to create S3 configuration. Check AWS env variables or IAM permissions.",
+            )?
             .retry_config(retry_config)
             .build();