PYTHON-3004 Support kmip FLE KMS provider (#786)

Resync CSFLE spec tests.

Author: ShaneHarvey

.evergreen/config.yml

@@ -359,6 +359,49 @@ functions:
            PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/run-doctests.sh
 
   "run tests":
+    # If testing FLE, start the KMS mock servers, first create the virtualenv.
+    - command: shell.exec
+      params:
+        script: |
+          if [ -n "${test_encryption}" ]; then
+            ${PREPARE_SHELL}
+            cd ${DRIVERS_TOOLS}/.evergreen/csfle
+            . ./activate_venv.sh
+          fi
+    # Run in the background so the mock servers don't block the EVG task.
+    - command: shell.exec
+      params:
+        background: true
+        script: |
+          if [ -n "${test_encryption}" ]; then
+            ${PREPARE_SHELL}
+            cd ${DRIVERS_TOOLS}/.evergreen/csfle
+            . ./activate_venv.sh
+            # The -u options forces the stdout and stderr streams to be unbuffered.
+            # TMPDIR is required to avoid "AF_UNIX path too long" errors.
+            TMPDIR="$(dirname $DRIVERS_TOOLS)" python -u kms_kmip_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem  --port 5698 &
+            python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 &
+            python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 &
+            python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert &
+          fi
+    # Wait up to 10 seconds for the KMIP server to start.
+    - command: shell.exec
+      params:
+        script: |
+          if [ -n "${test_encryption}" ]; then
+            ${PREPARE_SHELL}
+            cd ${DRIVERS_TOOLS}/.evergreen/csfle
+            . ./activate_venv.sh
+            for i in $(seq 1 1 10); do
+               sleep 1
+               if python -u kms_kmip_client.py; then
+                  echo 'KMS KMIP server started!'
+                  exit 0
+               fi
+            done
+            echo 'Failed to start KMIP server!'
+            exit 1
+          fi
     - command: shell.exec
       type: test
       params:

.evergreen/run-tests.sh

@@ -146,13 +146,6 @@ if [ -n "$TEST_ENCRYPTION" ]; then
     # Get access to the AWS temporary credentials:
     # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN
     . $DRIVERS_TOOLS/.evergreen/csfle/set-temp-creds.sh
-
-    # Start the mock KMS servers.
-    pushd ${DRIVERS_TOOLS}/.evergreen/csfle
-    python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 &
-    python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 &
-    trap 'kill $(jobs -p)' EXIT HUP
-    popd
 fi
 
 if [ -z "$DATA_LAKE" ]; then

pymongo/encryption.py

@@ -109,7 +109,7 @@ def kms_request(self, kms_context):
         message = kms_context.message
         provider = kms_context.kms_provider
         ctx = self.opts._kms_ssl_contexts.get(provider)
-        if not ctx:
+        if ctx is None:
             # Enable strict certificate verification, OCSP, match hostname, and
             # SNI using the system default CA certificates.
             ctx = get_ssl_context(
@@ -378,9 +378,8 @@ def __init__(self, kms_providers, key_vault_namespace, key_vault_client,
         See :ref:`explicit-client-side-encryption` for an example.
 
         :Parameters:
-          - `kms_providers`: Map of KMS provider options. Two KMS providers
-            are supported: "aws" and "local". The kmsProviders map values
-            differ by provider:
+          - `kms_providers`: Map of KMS provider options. The `kms_providers`
+            map values differ by provider:
 
               - `aws`: Map with "accessKeyId" and "secretAccessKey" as strings.
                 These are the AWS access key ID and AWS secret access key used
@@ -396,6 +395,8 @@ def __init__(self, kms_providers, key_vault_namespace, key_vault_client,
                 Additionally, "endpoint" may also be specified as a string
                 (defaults to 'oauth2.googleapis.com'). These are the
                 credentials used to generate Google Cloud KMS messages.
+              - `kmip`: Map with "endpoint" as a host with required port.
+                For example: ``{"endpoint": "example.com:443"}``.
               - `local`: Map with "key" as `bytes` (96 bytes in length) or
                 a base64 encoded string which decodes
                 to 96 bytes. "key" is the master key used to encrypt/decrypt
@@ -424,7 +425,7 @@ def __init__(self, kms_providers, key_vault_namespace, key_vault_client,
               kms_tls_options={'kmip': {'tlsCAFile': certifi.where()}}
 
         .. versionchanged:: 4.0
-           Added the `kms_tls_options` parameter.
+           Added the `kms_tls_options` parameter and the "kmip" KMS provider.
 
         .. versionadded:: 3.9
         """
@@ -458,7 +459,7 @@ def create_data_key(self, kms_provider, master_key=None,
 
         :Parameters:
           - `kms_provider`: The KMS provider to use. Supported values are
-            "aws" and "local".
+            "aws", "azure", "gcp", "kmip", and "local".
           - `master_key`: Identifies a KMS-specific key used to encrypt the
             new data key. If the kmsProvider is "local" the `master_key` is
             not applicable and may be omitted.
@@ -493,6 +494,16 @@ def create_data_key(self, kms_provider, master_key=None,
               - `endpoint` (string): Optional. Host with optional port.
                 Defaults to "cloudkms.googleapis.com".
 
+            If the `kms_provider` is "kmip" it is optional and has the
+            following fields::
+
+              - `keyId` (string): Optional. `keyId` is the KMIP Unique
+                Identifier to a 96 byte KMIP Secret Data managed object. If
+                keyId is omitted, the driver creates a random 96 byte KMIP
+                Secret Data managed object.
+              - `endpoint` (string): Optional. Host with optional
+                 port, e.g. "example.vault.azure.net:".
+
           - `key_alt_names` (optional): An optional list of string alternate
             names used to reference a key. If a key is created with alternate
             names, then encryption may refer to the key by the unique alternate

pymongo/encryption_options.py

@@ -55,9 +55,8 @@ def __init__(self, kms_providers, key_vault_namespace,
         See :ref:`automatic-client-side-encryption` for an example.
 
         :Parameters:
-          - `kms_providers`: Map of KMS provider options. Two KMS providers
-            are supported: "aws" and "local". The kmsProviders map values
-            differ by provider:
+          - `kms_providers`: Map of KMS provider options. The `kms_providers`
+            map values differ by provider:
 
               - `aws`: Map with "accessKeyId" and "secretAccessKey" as strings.
                 These are the AWS access key ID and AWS secret access key used
@@ -73,6 +72,8 @@ def __init__(self, kms_providers, key_vault_namespace,
                 Additionally, "endpoint" may also be specified as a string
                 (defaults to 'oauth2.googleapis.com'). These are the
                 credentials used to generate Google Cloud KMS messages.
+              - `kmip`: Map with "endpoint" as a host with required port.
+                For example: ``{"endpoint": "example.com:443"}``.
               - `local`: Map with "key" as `bytes` (96 bytes in length) or
                 a base64 encoded string which decodes
                 to 96 bytes. "key" is the master key used to encrypt/decrypt
@@ -129,7 +130,7 @@ def __init__(self, kms_providers, key_vault_namespace,
               kms_tls_options={'kmip': {'tlsCAFile': certifi.where()}}
 
         .. versionchanged:: 4.0
-           Added the `kms_tls_options` parameter.
+           Added the `kms_tls_options` parameter and the "kmip" KMS provider.
 
         .. versionadded:: 3.9
         """