RUST-2008 Remove almost all uses of add_expansions_to_env (#1186)

Author: abr-egn

.evergreen/config.yml

@@ -565,7 +565,14 @@ task_groups:
         params:
           working_dir: src
           binary: bash
-          add_expansions_to_env: true
+          include_expansions_in_env:
+            - DRIVERS_ATLAS_PUBLIC_API_KEY
+            - DRIVERS_ATLAS_PRIVATE_API_KEY
+            - DRIVERS_ATLAS_GROUP_ID
+            - DRIVERS_ATLAS_LAMBDA_USER
+            - DRIVERS_ATLAS_LAMBDA_PASSWORD
+            - LAMBDA_STACK_NAME
+            - MONGODB_VERSION
           env:
             MONGODB_VERSION: "7.0"
           args:
@@ -578,7 +585,11 @@ task_groups:
         params:
           working_dir: src
           binary: bash
-          add_expansions_to_env: true
+          include_expansions_in_env:
+            - DRIVERS_ATLAS_PUBLIC_API_KEY
+            - DRIVERS_ATLAS_PRIVATE_API_KEY
+            - DRIVERS_ATLAS_GROUP_ID
+            - CLUSTER_NAME
           args:
             - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh
       - func: "upload test results"
@@ -599,7 +610,14 @@ task_groups:
         params:
           working_dir: src
           binary: bash
-          add_expansions_to_env: true
+          include_expansions_in_env:
+            - DRIVERS_ATLAS_PUBLIC_API_KEY
+            - DRIVERS_ATLAS_PRIVATE_API_KEY
+            - DRIVERS_ATLAS_GROUP_ID
+            - DRIVERS_ATLAS_LAMBDA_USER
+            - DRIVERS_ATLAS_LAMBDA_PASSWORD
+            - LAMBDA_STACK_NAME
+            - MONGODB_VERSION
           args:
             - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh
       - command: expansions.update
@@ -610,7 +628,11 @@ task_groups:
         params:
           working_dir: src
           binary: bash
-          add_expansions_to_env: true
+          include_expansions_in_env:
+            - DRIVERS_ATLAS_PUBLIC_API_KEY
+            - DRIVERS_ATLAS_PRIVATE_API_KEY
+            - DRIVERS_ATLAS_GROUP_ID
+            - CLUSTER_NAME
           args:
             - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh
     setup_group_can_fail_task: true
@@ -891,7 +913,10 @@ tasks:
         params:
           working_dir: src
           binary: bash
-          add_expansions_to_env: true
+          include_expansions_in_env:
+            - PROJECT_DIRECTORY
+            - MONGODB_URI
+            - PATH
           args:
             - .evergreen/run-search-index-test.sh
 
@@ -1088,9 +1113,22 @@ tasks:
         params:
           working_dir: src
           binary: bash
-          add_expansions_to_env: true
           args:
             - .evergreen/run-aws-lambda-test.sh
+          include_expansions_in_env:
+            - AWS_ACCESS_KEY_ID
+            - AWS_SECRET_ACCESS_KEY
+            - AWS_SESSION_TOKEN
+            - DRIVERS_ATLAS_PUBLIC_API_KEY
+            - DRIVERS_ATLAS_PRIVATE_API_KEY
+            - DRIVERS_ATLAS_LAMBDA_USER
+            - DRIVERS_ATLAS_LAMBDA_PASSWORD
+            - DRIVERS_ATLAS_GROUP_ID
+            - LAMBDA_STACK_NAME
+            - PROJECT_DIRECTORY
+            - DRIVERS_TOOLS
+            - CLUSTER_NAME
+            - MONGODB_URI
           env:
             TEST_LAMBDA_DIRECTORY: ${PROJECT_DIRECTORY}/.evergreen/aws-lambda-test
             AWS_REGION: us-east-1
@@ -1173,7 +1211,11 @@ functions:
         binary: bash
         args:
           - .evergreen/create-expansions.sh
-        add_expansions_to_env: true
+        include_expansions_in_env:
+          - is_patch
+          - version_id
+          - project
+          - LIBMONGOCRYPT_OS
     - command: expansions.update
       params:
         file: src/expansion.yml
@@ -1391,7 +1433,15 @@ functions:
         binary: sh
         args:
           - ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
-        add_expansions_to_env: true
+        include_expansions_in_env:
+          - AUTH
+          - SSL
+          - TOPOLOGY
+          - LOAD_BALANCER
+          - REQUIRE_API_VERSION
+          - MONGODB_VERSION
+          - ORCHESTRATION_FILE
+          - MONGODB_BINARIES
     - command: expansions.update
       params:
         file: mo-expansion.yml
@@ -1401,19 +1451,12 @@ functions:
         binary: bash
         args:
           - .evergreen/generate-uri.sh
-        add_expansions_to_env: true
-    - command: expansions.update
-      params:
-        file: src/uri-expansions.yml
-
-  "generate uris":
-    - command: subprocess.exec
-      params:
-        working_dir: src
-        binary: bash
-        args:
-          - .evergreen/generate-uri.sh
-        add_expansions_to_env: true
+        include_expansions_in_env:
+          - DRIVERS_TOOLS
+          - MONGODB_URI
+          - SINGLE_MONGOS_LB_URI
+          - MULTI_MONGOS_LB_URI
+          - SSL
     - command: expansions.update
       params:
         file: src/uri-expansions.yml
@@ -1426,9 +1469,6 @@ functions:
         binary: bash
         args:
           - .evergreen/run-tests.sh
-        # The driver uses the absence/presence of variables defined in the project's settings to
-        # determine how to create a test client, so the relevant expansions here MUST be listed
-        # explicitly rather than setting add_expansions_to_env: true.
         include_expansions_in_env:
           - PROJECT_DIRECTORY
           - OPENSSL
@@ -1447,7 +1487,6 @@ functions:
         binary: bash
         args:
           - .evergreen/run-sync-tests.sh
-        # Ditto comment above regarding explicitly listing expansions.
         include_expansions_in_env:
           - PROJECT_DIRECTORY
           - MONGODB_URI
@@ -1529,7 +1568,20 @@ functions:
         binary: bash
         args:
           - .evergreen/run-csfle-tests.sh
-        add_expansions_to_env: true
+        include_expansions_in_env:
+          - DRIVERS_TOOLS
+          - PROJECT_DIRECTORY
+          - MONGODB_URI
+          - MONGOCRYPT_LIB_DIR
+          - OPENSSL
+          - OS
+          - LD_LIBRARY_PATH
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+          - CSFLE_LOCAL_KEY
+          - CRYPT_SHARED_LIB_PATH
+          - DISABLE_CRYPT_SHARED
+          - AZURE_IMDS_MOCK_PORT
 
   "run csfle serverless tests":
     - command: shell.exec
@@ -1669,7 +1721,10 @@ functions:
       type: test
       params:
         working_dir: src
-        add_expansions_to_env: true
+        include_expansions_in_env:
+          - DRIVERS_TOOLS
+          - PROJECT_DIRECTORY
+          - MONGOCRYPT_LIB_DIR
         binary: bash
         args:
           - .evergreen/check-semgrep.sh

.evergreen/run-csfle-tests.sh

@@ -3,7 +3,7 @@
 set -o errexit
 set -o pipefail
 
-source ./.evergreen/env.sh
+source .evergreen/env.sh
 source .evergreen/cargo-test.sh
 
 set -o xtrace

.evergreen/run-sync-tests.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-. ${PREPARE_SHELL}
-
 set -o errexit
 set -o pipefail
 

src/error.rs

@@ -50,7 +50,8 @@ pub type Result<T> = std::result::Result<T, Error>;
 /// [`ErrorKind`](enum.ErrorKind.html) is wrapped in an `Arc` to allow the errors to be
 /// cloned.
 #[derive(Clone, Debug, Error)]
-#[error("Kind: {kind}, labels: {labels:?}")]
+#[cfg_attr(test, error("Kind: {kind}, labels: {labels:?}, backtrace: {bt}"))]
+#[cfg_attr(not(test), error("Kind: {kind}, labels: {labels:?}"))]
 #[non_exhaustive]
 pub struct Error {
     /// The type of error that occurred.
@@ -59,6 +60,8 @@ pub struct Error {
     pub(crate) wire_version: Option<i32>,
     #[source]
     pub(crate) source: Option<Box<Error>>,
+    #[cfg(test)]
+    bt: Arc<std::backtrace::Backtrace>,
 }
 
 impl Error {
@@ -90,6 +93,8 @@ impl Error {
             labels,
             wire_version: None,
             source: None,
+            #[cfg(test)]
+            bt: Arc::new(std::backtrace::Backtrace::capture()),
         }
     }
 

src/test/csfle.rs

@@ -1,6 +1,6 @@
 use std::{
     collections::BTreeMap,
-    env::var,
+    env,
     path::PathBuf,
     sync::{
         atomic::{AtomicBool, Ordering},
@@ -103,8 +103,8 @@ pub(crate) static AWS_KMS: Lazy<KmsInfo> = Lazy::new(|| {
     (
         KmsProvider::aws(),
         doc! {
-        "accessKeyId": var("FLE_AWS_KEY").unwrap(),
-        "secretAccessKey": var("FLE_AWS_SECRET").unwrap()},
+        "accessKeyId": env::var("FLE_AWS_KEY").unwrap(),
+        "secretAccessKey": env::var("FLE_AWS_SECRET").unwrap()},
         None,
     )
 });
@@ -116,8 +116,8 @@ pub(crate) static AWS_KMS_NAME2: Lazy<KmsInfo> = Lazy::new(|| {
     (
         KmsProvider::aws().with_name("name2"),
         doc! {
-            "accessKeyId": var("FLE_AWS_KEY").unwrap(),
-            "secretAccessKey": var("FLE_AWS_SECRET").unwrap()
+            "accessKeyId": env::var("FLE_AWS_KEY").unwrap(),
+            "secretAccessKey": env::var("FLE_AWS_SECRET").unwrap()
         },
         None,
     )
@@ -126,9 +126,9 @@ pub(crate) static AZURE_KMS: Lazy<KmsInfo> = Lazy::new(|| {
     (
         KmsProvider::azure(),
         doc! {
-            "tenantId": var("FLE_AZURE_TENANTID").unwrap(),
-            "clientId": var("FLE_AZURE_CLIENTID").unwrap(),
-            "clientSecret": var("FLE_AZURE_CLIENTSECRET").unwrap(),
+            "tenantId": env::var("FLE_AZURE_TENANTID").unwrap(),
+            "clientId": env::var("FLE_AZURE_CLIENTID").unwrap(),
+            "clientSecret": env::var("FLE_AZURE_CLIENTSECRET").unwrap(),
         },
         None,
     )
@@ -141,8 +141,8 @@ pub(crate) static GCP_KMS: Lazy<KmsInfo> = Lazy::new(|| {
     (
         KmsProvider::gcp(),
         doc! {
-            "email": var("FLE_GCP_EMAIL").unwrap(),
-            "privateKey": var("FLE_GCP_PRIVATEKEY").unwrap(),
+            "email": env::var("FLE_GCP_EMAIL").unwrap(),
+            "privateKey": env::var("FLE_GCP_PRIVATEKEY").unwrap(),
         },
         None,
     )
@@ -157,7 +157,7 @@ pub(crate) static LOCAL_KMS: Lazy<KmsInfo> = Lazy::new(|| {
         doc! {
             "key": bson::Binary {
                 subtype: bson::spec::BinarySubtype::Generic,
-                bytes: base64::decode(var("CSFLE_LOCAL_KEY").unwrap()).unwrap(),
+                bytes: base64::decode(env::var("CSFLE_LOCAL_KEY").unwrap()).unwrap(),
             },
         },
         None,
@@ -168,7 +168,7 @@ pub(crate) static LOCAL_KMS_NAME1: Lazy<KmsInfo> = Lazy::new(|| {
     (local_info.0.with_name("name1"), local_info.1, local_info.2)
 });
 pub(crate) static KMIP_KMS: Lazy<KmsInfo> = Lazy::new(|| {
-    let cert_dir = PathBuf::from(var("CSFLE_TLS_CERT_DIR").unwrap());
+    let cert_dir = PathBuf::from(env::var("CSFLE_TLS_CERT_DIR").unwrap());
     let tls_options = TlsOptions::builder()
         .ca_file_path(cert_dir.join("ca.pem"))
         .cert_key_file_path(cert_dir.join("client.pem"))
@@ -212,14 +212,14 @@ pub(crate) static ALL_KMS_PROVIDERS: Lazy<KmsProviderList> = Lazy::new(|| {
 });
 
 static EXTRA_OPTIONS: Lazy<Document> =
-    Lazy::new(|| doc! { "cryptSharedLibPath": std::env::var("CRYPT_SHARED_LIB_PATH").unwrap() });
+    Lazy::new(|| doc! { "cryptSharedLibPath": env::var("CRYPT_SHARED_LIB_PATH").unwrap() });
 static KV_NAMESPACE: Lazy<Namespace> =
     Lazy::new(|| Namespace::from_str("keyvault.datakeys").unwrap());
 static DISABLE_CRYPT_SHARED: Lazy<bool> =
-    Lazy::new(|| std::env::var("DISABLE_CRYPT_SHARED").map_or(false, |s| s == "true"));
+    Lazy::new(|| env::var("DISABLE_CRYPT_SHARED").map_or(false, |s| s == "true"));
 
 fn check_env(name: &str, kmip: bool) -> bool {
-    if std::env::var("CSFLE_LOCAL_KEY").is_err() {
+    if env::var("CSFLE_LOCAL_KEY").is_err() {
         log_uncaptured(format!(
             "skipping csfle test {}: no kms providers configured",
             name
@@ -1757,7 +1757,7 @@ async fn kms_tls_options() -> Result<()> {
         base_providers
     }
 
-    let cert_dir = PathBuf::from(std::env::var("CSFLE_TLS_CERT_DIR").unwrap());
+    let cert_dir = PathBuf::from(env::var("CSFLE_TLS_CERT_DIR").unwrap());
     let ca_path = cert_dir.join("ca.pem");
     let key_path = cert_dir.join("client.pem");
 
@@ -2731,8 +2731,7 @@ async fn on_demand_aws_failure() -> Result<()> {
     if !check_env("on_demand_aws_failure", false) {
         return Ok(());
     }
-    if std::env::var("AWS_ACCESS_KEY_ID").is_ok() && std::env::var("AWS_SECRET_ACCESS_KEY").is_ok()
-    {
+    if env::var("AWS_ACCESS_KEY_ID").is_ok() && env::var("AWS_SECRET_ACCESS_KEY").is_ok() {
         log_uncaptured("Skipping on_demand_aws_failure: credentials set");
         return Ok(());
     }
@@ -2803,7 +2802,7 @@ async fn on_demand_gcp_credentials() -> Result<()> {
         )
         .await;
 
-    if std::env::var("ON_DEMAND_GCP_CREDS_SHOULD_SUCCEED").is_ok() {
+    if env::var("ON_DEMAND_GCP_CREDS_SHOULD_SUCCEED").is_ok() {
         result.unwrap();
     } else {
         let error = result.unwrap_err();
@@ -2830,10 +2829,7 @@ async fn azure_imds() -> Result<()> {
     let mut azure_exec = crate::client::csfle::state_machine::azure::ExecutorState::new()?;
     azure_exec.test_host = Some((
         "localhost",
-        std::env::var("AZURE_IMDS_MOCK_PORT")
-            .unwrap()
-            .parse()
-            .unwrap(),
+        env::var("AZURE_IMDS_MOCK_PORT").unwrap().parse().unwrap(),
     ));
 
     // Case 1: Success
@@ -3220,6 +3216,7 @@ async fn range_explicit_encryption_test(
         KV_NAMESPACE.clone(),
         vec![LOCAL_KMS.clone()],
     )?
+    .extra_options(EXTRA_OPTIONS.clone())
     .bypass_query_analysis(true)
     .build()
     .await?;
@@ -3564,6 +3561,7 @@ async fn fle2_example() -> Result<()> {
         KV_NAMESPACE.clone(),
         vec![LOCAL_KMS.clone()],
     )?
+    .extra_options(EXTRA_OPTIONS.clone())
     .encrypted_fields_map(encrypted_fields_map)
     .build()
     .await?;