PYTHON-2453 Add MongoDB Versioned API (#536)

Add pymongo.server_api.ServerApi and the MongoClient server_api option.
Support Unified Test Format version 1.1 (serverParameters in
runOnRequirements)
Skip dropRole tests due to SERVER-53499.

(cherry picked from commit ac2f506)

Author: ShaneHarvey

.evergreen/config.yml

@@ -290,6 +290,7 @@ functions:
             STORAGE_ENGINE=${STORAGE_ENGINE} \
             DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS} \
             ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \
+            REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \
             sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
     # run-orchestration generates expansion file with the MONGODB_URI for the cluster
     - command: expansions.update
@@ -425,6 +426,7 @@ functions:
             AUTH=${AUTH} \
             SSL=${SSL} \
             DATA_LAKE=${DATA_LAKE} \
+            MONGODB_API_VERSION=${MONGODB_API_VERSION} \
             sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
 
   "run enterprise auth tests":
@@ -2023,6 +2025,19 @@ axes:
         variables:
           SETDEFAULTENCODING: "cp1251"
 
+  - id: requireApiVersion
+    display_name: "requireApiVersion"
+    values:
+      - id: "requireApiVersion1"
+        display_name: "requireApiVersion1"
+        tags: [ "requireApiVersion_tag" ]
+        variables:
+          # REQUIRE_API_VERSION is set to make drivers-evergreen-tools
+          # start a cluster with the requireApiVersion parameter.
+          REQUIRE_API_VERSION: "1"
+          # MONGODB_API_VERSION is the apiVersion to use in the test suite.
+          MONGODB_API_VERSION: "1"
+
 buildvariants:
 - matrix_name: "tests-all"
   matrix_spec:
@@ -2605,6 +2620,17 @@ buildvariants:
   tasks:
     - name: atlas-data-lake-tests
 
+- matrix_name: "versioned-api-tests"
+  matrix_spec:
+    platform: ubuntu-16.04
+    python-version: ["2.7", "3.9"]
+    auth: "auth"
+    requireApiVersion: "*"
+  display_name: "requireApiVersion ${python-version}"
+  tasks:
+    # Versioned API was introduced in MongoDB 4.7
+    - "test-latest-standalone"
+
 - matrix_name: "ocsp-test"
   matrix_spec:
     platform: ubuntu-16.04

.evergreen/run-tests.sh

@@ -26,6 +26,7 @@ GREEN_FRAMEWORK=${GREEN_FRAMEWORK:-}
 C_EXTENSIONS=${C_EXTENSIONS:-}
 COVERAGE=${COVERAGE:-}
 COMPRESSORS=${COMPRESSORS:-}
+MONGODB_API_VERSION=${MONGODB_API_VERSION:-}
 TEST_ENCRYPTION=${TEST_ENCRYPTION:-}
 LIBMONGOCRYPT_URL=${LIBMONGOCRYPT_URL:-}
 SETDEFAULTENCODING=${SETDEFAULTENCODING:-}
@@ -35,6 +36,11 @@ if [ -n "$COMPRESSORS" ]; then
     export COMPRESSORS=$COMPRESSORS
 fi
 
+if [ -n "$MONGODB_API_VERSION" ]; then
+    export MONGODB_API_VERSION=$MONGODB_API_VERSION
+fi
+
+
 export JAVA_HOME=/opt/java/jdk8
 
 if [ "$AUTH" != "noauth" ]; then

doc/api/pymongo/index.rst

@@ -54,6 +54,7 @@ Sub-modules:
    read_preferences
    results
    son_manipulator
+   server_api
    uri_parser
    write_concern
    event_loggers

doc/api/pymongo/server_api.rst

@@ -0,0 +1,11 @@
+:mod:`server_api` -- Support for MongoDB Versioned API
+======================================================
+
+.. automodule:: pymongo.server_api
+   :synopsis: Support for MongoDB Versioned API
+
+   .. autoclass:: pymongo.server_api.ServerApi
+      :members:
+
+   .. autoclass:: pymongo.server_api.ServerApiVersion
+      :members:

doc/changelog.rst

@@ -4,6 +4,19 @@ Changelog
 Changes in Version 3.12.0
 -------------------------
 
+Notable improvements
+....................
+
+- Support for MongoDB Versioned API, see :class:`~pymongo.server_api.ServerApi`.
+
+Issues Resolved
+...............
+
+See the `PyMongo 3.12 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 3.12 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=29594
+
 
 Changes in Version 3.11.1
 -------------------------

pymongo/client_options.py

@@ -125,6 +125,7 @@ def _parse_pool_options(options):
     event_listeners = options.get('event_listeners')
     appname = options.get('appname')
     driver = options.get('driver')
+    server_api = options.get('server_api')
     compression_settings = CompressionSettings(
         options.get('compressors', []),
         options.get('zlibcompressionlevel', -1))
@@ -138,7 +139,8 @@ def _parse_pool_options(options):
                        _EventListeners(event_listeners),
                        appname,
                        driver,
-                       compression_settings)
+                       compression_settings,
+                       server_api=server_api)
 
 
 class ClientOptions(object):

pymongo/client_session.py

@@ -297,6 +297,9 @@ def __init__(self, opts):
     def active(self):
         return self.state in (_TxnState.STARTING, _TxnState.IN_PROGRESS)
 
+    def starting(self):
+        return self.state == _TxnState.STARTING
+
     def reset(self):
         self.state = _TxnState.NONE
         self.sharded = False
@@ -762,6 +765,12 @@ def in_transaction(self):
         """
         return self._transaction.active()
 
+    @property
+    def _starting_transaction(self):
+        """True if this session is starting a multi-statement transaction.
+        """
+        return self._transaction.starting()
+
     @property
     def _pinned_address(self):
         """The mongos address this transaction was created on."""

pymongo/common.py

@@ -27,6 +27,7 @@
 from pymongo.compression_support import (validate_compressors,
                                          validate_zlib_compression_level)
 from pymongo.driver_info import DriverInfo
+from pymongo.server_api import ServerApi
 from pymongo.encryption_options import validate_auto_encryption_opts_or_none
 from pymongo.errors import ConfigurationError
 from pymongo.monitoring import _validate_event_listeners
@@ -525,6 +526,15 @@ def validate_driver_or_none(option, value):
     return value
 
 
+def validate_server_api_or_none(option, value):
+    """Validate the server_api keyword arg."""
+    if value is None:
+        return value
+    if not isinstance(value, ServerApi):
+        raise TypeError("%s must be an instance of ServerApi" % (option,))
+    return value
+
+
 def validate_is_callable_or_none(option, value):
     """Validates that 'value' is a callable."""
     if value is None:
@@ -640,6 +650,7 @@ def validate_tzinfo(dummy, value):
 NONSPEC_OPTIONS_VALIDATOR_MAP = {
     'connect': validate_boolean_or_string,
     'driver': validate_driver_or_none,
+    'server_api': validate_server_api_or_none,
     'fsync': validate_boolean_or_string,
     'minpoolsize': validate_non_negative_integer,
     'socketkeepalive': validate_boolean_or_string,

pymongo/database.py

@@ -704,6 +704,12 @@ def command(self, command, value=1, check=True,
         .. note:: :meth:`command` does **not** apply any custom TypeDecoders
            when decoding the command response.
 
+        .. note:: If this client has been configured to use MongoDB Versioned
+           API (see :ref:`versioned-api-ref`), then :meth:`command` will
+           automactically add API versioning options to the given command.
+           Explicitly adding API versioning options in the command and
+           declaring an API version on the client is not supported.
+
         .. versionchanged:: 3.6
            Added ``session`` parameter.
 

pymongo/message.py

@@ -307,6 +307,7 @@ def as_command(self, sock_info):
             self.name = 'explain'
             cmd = SON([('explain', cmd)])
         session = self.session
+        sock_info.add_server_api(cmd, session)
         if session:
             session._apply_to(cmd, False, self.read_preference)
             # Explain does not support readConcern.
@@ -892,6 +893,7 @@ def __init__(self, database_name, command, sock_info, operation_id,
         self.compress = True if sock_info.compression_context else False
         self.op_type = op_type
         self.codec = codec
+        sock_info.add_server_api(command, session)
 
     def _batch_command(self, docs):
         namespace = self.db_name + '.$cmd'