PYTHON-3983 Clean up handling of PYTHON_BINARY (#1382)

blink1073 · web-flow · commit ed3ae4140775 · 2023-10-06T19:11:52.000-05:00
diff --git a/.evergreen/config.yml b/.evergreen/config.yml
@@ -79,17 +79,6 @@ functions:
               export PATH="$MONGODB_BINARIES:$PATH"
               export PROJECT="${project}"
               export PIP_QUIET=1
-           ENSURE_TOOLCHAIN_PYTHON_BINARY: |
-              # Make sure PYTHON_BINARY is set to a suitable toolchain python.
-              if [ -z "$PYTHON_BINARY" ]; then
-                  if [ "$(uname -s)" = "Darwin" ]; then
-                      export PYTHON_BINARY=/Library/Frameworks/Python.Framework/Versions/3.9/bin/python3
-                  elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
-                      export PYTHON_BINARY=/cygdrive/c/python/Python39/python
-                  else
-                      export PYTHON_BINARY=/opt/python/3.9/bin/python3
-                  fi
-              fi
            EOT
 
     # Load the expansion file to make an evergreen variable with the current unique version
@@ -502,12 +491,8 @@ functions:
             export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}"
           fi
 
-          export PYTHON_BINARY=${PYTHON_BINARY}
-          if [ -z "$PYTHON_BINARY" ]; then
-            export PYTHON_BINARY=${python3_binary}
-          fi
-
           GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \
+            PYTHON_BINARY=${PYTHON_BINARY} \
             C_EXTENSIONS=${C_EXTENSIONS} \
             COVERAGE=${COVERAGE} \
             COMPRESSORS=${COMPRESSORS} \
@@ -678,7 +663,6 @@ functions:
         include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
         script: |
           ${PREPARE_SHELL}
-          ${ENSURE_TOOLCHAIN_PYTHON_BINARY}
           bash .evergreen/run-mongodb-oidc-test.sh
 
   "run aws auth test with aws credentials as environment variables":
@@ -1630,8 +1614,6 @@ tasks:
       tags: ["no-server"]
       commands:
         - func: "run tests"
-          vars:
-            PYTHON_BINARY: /opt/python/3.7/bin/python3
 
     - name: "atlas-connect"
       tags: ["atlas-connect"]
@@ -2108,7 +2090,6 @@ tasks:
           vars:
             VERSION: "latest"
             TOPOLOGY: "server"
-            PYTHON_BINARY: "/opt/mongodbtoolchain/v3/bin/python3"
         - func: "run tests"
 # }}}
     - name: "coverage-report"
@@ -2260,7 +2241,6 @@ axes:
           skip_EC2_auth_test: true
           skip_ECS_auth_test: true
           skip_web_identity_auth_test: true
-          python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3
           libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz
       - id: macos-1100
         display_name: "macOS 11.00"
@@ -2269,7 +2249,6 @@ axes:
           skip_EC2_auth_test: true
           skip_ECS_auth_test: true
           skip_web_identity_auth_test: true
-          python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3
           libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz
       - id: macos-1100-arm64
         display_name: "macOS 11.00 Arm64"
@@ -2278,21 +2257,18 @@ axes:
           skip_EC2_auth_test: true
           skip_ECS_auth_test: true
           skip_web_identity_auth_test: true
-          python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3
           libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz
       - id: rhel7
         display_name: "RHEL 7.x"
         run_on: rhel79-small
         batchtime: 10080  # 7 days
         variables:
-          python3_binary: "/opt/python/3.8/bin/python3"
           libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-70-64-bit/master/latest/libmongocrypt.tar.gz
       - id: rhel8
         display_name: "RHEL 8.x"
         run_on: rhel87-small
         batchtime: 10080  # 7 days
         variables:
-          python3_binary: "/opt/python/3.8/bin/python3"
           libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-80-64-bit/master/latest/libmongocrypt.tar.gz
       - id: rhel80-fips
         display_name: "RHEL 8.0 FIPS"
@@ -2304,14 +2280,10 @@ axes:
         display_name: "Ubuntu 22.04"
         run_on: ubuntu2204-small
         batchtime: 10080  # 7 days
-        variables:
-          python3_binary: python3
       - id: ubuntu-18.04
         display_name: "Ubuntu 18.04"
         run_on: ubuntu1804-small
         batchtime: 10080  # 7 days
-        variables:
-          python3_binary: python3
       - id: rhel83-zseries
         display_name: "RHEL 8.3 (zSeries)"
         run_on: rhel83-zseries-small
@@ -2334,7 +2306,6 @@ axes:
           skip_ECS_auth_test: true
           skip_EC2_auth_test: true
           skip_web_identity_auth_test: true
-          python3_binary: "C:/python/Python38/python.exe"
           venv_bin_dir: "Scripts"
           libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/windows-test/master/latest/libmongocrypt.tar.gz
 
@@ -2476,14 +2447,6 @@ axes:
         variables:
            PYTHON_BINARY: "/opt/python/pypy3.10/bin/pypy3"
 
-  - id: python-version-mac
-    display_name: "Python"
-    values:
-      - id: "system-python3"
-        display_name: "Python3"
-        variables:
-          PYTHON_BINARY: "python3"
-
   - id: python-version-windows
     display_name: "Python"
     values:
@@ -3166,7 +3129,6 @@ buildvariants:
 - matrix_name: "aws-auth-test-mac"
   matrix_spec:
     platform: [macos-1014]
-    python-version-mac: ["system-python3"]
   display_name: "MONGODB-AWS Auth ${platform} ${python-version-mac}"
   tasks:
     - name: "aws-auth-test-4.4"
diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh
@@ -18,6 +18,9 @@ echo "Running MONGODB-AWS authentication tests"
 # ensure no secrets are printed in log files
 set +x
 
+. ./.evergreen/utils.sh
+PYTHON_BINARY=$(find_python3)
+
 # Try to source exported AWS Secrets
 if [ -f ./secrets-export.sh ]; then
   source ./secrets-export.sh
diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh
@@ -13,13 +13,13 @@ fi
 
 # Get the drivers secrets.  Use an existing secrets file first.
 if [ ! -f "./secrets-export.sh" ]; then
-    bash .evergreen/tox.sh -m aws-secrets -- drivers/oidc
+    bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/oidc
 fi
 source ./secrets-export.sh
 
 # # If the file did not have our creds, get them from the vault.
 if [ -z "$OIDC_ATLAS_URI_SINGLE" ]; then
-    bash .evergreen/tox.sh -m aws-secrets -- drivers/oidc
+    bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/oidc
     source ./secrets-export.sh
 fi
 
diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh
@@ -1,18 +1,11 @@
 #!/bin/bash
 set -o errexit  # Exit the script with error if any of the commands fail
+set -x
 
 . .evergreen/utils.sh
 
 if [ -z "$PYTHON_BINARY" ]; then
-    # Use Python 3 from the server toolchain to test on ARM, POWER or zSeries if a
-    # system python3 doesn't exist or exists but is older than 3.7.
-    if is_python_37 "$(command -v python3)"; then
-        PYTHON_BINARY=$(command -v python3)
-    elif is_python_37 "$(command -v /opt/mongodbtoolchain/v3/bin/python3)"; then
-        PYTHON_BINARY=$(command -v /opt/mongodbtoolchain/v3/bin/python3)
-    else
-        echo "Cannot test without python3.7+ installed!"
-    fi
+    PYTHON_BINARY=$(find_python3)
 fi
 
 if $PYTHON_BINARY -m tox --version; then
diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh
@@ -2,24 +2,59 @@
 
 set -o xtrace
 
+find_python3() {
+    PYTHON=""
+    # Add a fallback system python3 if it is available and Python 3.7+.
+    if is_python_37 "$(command -v python3)"; then
+        PYTHON="$(command -v python3)"
+    fi
+    # Find a suitable toolchain version, if available.
+    if [ "$(uname -s)" = "Darwin" ]; then
+        # macos 11.00
+        if [ -d "/Library/Frameworks/Python.Framework/Versions/3.10" ]; then
+            PYTHON="/Library/Frameworks/Python.Framework/Versions/3.10/bin/python3"
+        # macos 10.14
+        elif [ -d "/Library/Frameworks/Python.Framework/Versions/3.7" ]; then
+            PYTHON="/Library/Frameworks/Python.Framework/Versions/3.7/bin/python3"
+        fi
+    elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
+        PYTHON="C:/python/Python37/python.exe"
+    else
+        # Prefer our own toolchain, fall back to mongodb toolchain if it has Python 3.7+.
+        if [ -f "/opt/python/3.7/bin/python3" ]; then
+            PYTHON="/opt/python/3.7/bin/python3"
+        elif is_python_37 "$(command -v /opt/mongodbtoolchain/v4/bin/python3)"; then
+            PYTHON="/opt/mongodbtoolchain/v4/bin/python3"
+        elif is_python_37 "$(command -v /opt/mongodbtoolchain/v3/bin/python3)"; then
+            PYTHON="/opt/mongodbtoolchain/v3/bin/python3"
+        fi
+    fi
+    if [ -z "$PYTHON" ]; then
+        echo "Cannot test without python3.7+ installed!"
+        exit 1
+    fi
+    echo "$PYTHON"
+}
+
 # Usage:
 # createvirtualenv /path/to/python /output/path/for/venv
 # * param1: Python binary to use for the virtualenv
 # * param2: Path to the virtualenv to create
 createvirtualenv () {
     PYTHON=$1
     VENVPATH=$2
-    if $PYTHON -m virtualenv --version; then
+    # Prefer venv
+    VENV="$PYTHON -m venv"
+    if [ "$(uname -s)" = "Darwin" ]; then
         VIRTUALENV="$PYTHON -m virtualenv"
-    elif $PYTHON -m venv -h > /dev/null; then
-        # System virtualenv might not be compatible with the python3 on our path
-        VIRTUALENV="$PYTHON -m venv"
     else
-        echo "Cannot test without virtualenv"
-        exit 1
+        VIRTUALENV=$(command -v virtualenv 2>/dev/null || echo "$PYTHON -m virtualenv")
+        VIRTUALENV="$VIRTUALENV -p $PYTHON"
+    fi
+    if ! $VENV $VENVPATH 2>/dev/null; then
+        # Workaround for bug in older versions of virtualenv.
+        $VIRTUALENV $VENVPATH 2>/dev/null || $VIRTUALENV $VENVPATH
     fi
-    # Workaround for bug in older versions of virtualenv.
-    $VIRTUALENV $VENVPATH || $PYTHON -m venv $VENVPATH
     if [ "Windows_NT" = "$OS" ]; then
         # Workaround https://bugs.python.org/issue32451:
         # mongovenv/Scripts/activate: line 3: $'\r': command not found
@@ -31,7 +66,7 @@ createvirtualenv () {
 
     export PIP_QUIET=1
     python -m pip install --upgrade pip
-    python -m pip install --upgrade setuptools wheel tox
+    python -m pip install --upgrade setuptools tox
 }
 
 # Usage:
diff --git a/test/test_index_management.py b/test/test_index_management.py
@@ -67,9 +67,13 @@ def setUpClass(cls) -> None:
         password = os.environ["DB_PASSWORD"]
         cls.client = MongoClient(url, username=username, password=password)
         cls.client.drop_database(_NAME)
-        cls.addClassCleanup(cls.client.close)
         cls.db = cls.client.test_search_index_prose
 
+    @classmethod
+    def tearDownClass(cls):
+        cls.client.drop_database(_NAME)
+        cls.client.close()
+
     def wait_for_ready(self, coll, name=_NAME, predicate=None):
         """Wait for a search index to be ready."""
         indices: list[Mapping[str, Any]] = []
