Add support for Python 3.12 (#972)

* Add support for Python 3.12

* Use pyproject.toml to make setuptools pick up the driver version

* Add python 3.12 tag to package metadata

* Comment why wait_for shim not needed in Python 3.12

* Silence deprecation warnings for Python 3.12 for now

---------

Signed-off-by: Rouven Bauer <rouven.bauer@neo4j.com>
Signed-off-by: Grant Lodge <6323995+thelonelyvulpes@users.noreply.github.com>
Co-authored-by: Grant Lodge <6323995+thelonelyvulpes@users.noreply.github.com>

Author: robsdedude

docs/source/index.rst

@@ -12,6 +12,7 @@ Neo4j versions supported:
 
 Python versions supported:
 
+* Python 3.12 (added in driver version 5.14.0)
 * Python 3.11 (added in driver version 5.3.0)
 * Python 3.10
 * Python 3.9

pyproject.toml

@@ -38,6 +38,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 dynamic = ["version", "readme"]
 
@@ -50,17 +51,19 @@ pandas = [
     "pandas >= 1.1.0, < 3.0.0",
     "numpy >= 1.7.0, < 2.0.0",
 ]
+pyarrow = ["pyarrow >= 1.0.0"]
+
 
 [build-system]
 requires = [
-    "setuptools~=65.6",
-    "tomlkit~=0.11.6",
+    "setuptools >= 66.1.0",
+    # TODO: 6.0 - can be removed once `setup.py` is simplified
+    "tomlkit ~= 0.11.6",
 ]
 build-backend = "setuptools.build_meta"
 
-# still in beta
-#[tool.setuptools.dynamic]
-#version = {attr = "neo4j._meta.version"}
+[tool.setuptools.dynamic]
+version = {attr = "neo4j._meta.version"}
 
 
 [tool.coverage]

requirements-dev.txt

@@ -1,5 +1,5 @@
 # the driver itself
--e .[pandas,numpy]
+-e .[pandas,numpy,pyarrow]
 
 # auto-generate sync driver from async code
 unasync>=0.5.0
@@ -10,16 +10,10 @@ mypy>=0.971
 typing-extensions>=4.3.0
 types-pytz>=2022.1.2
 
-# for packaging
-setuptools~=65.6
-# TODO: 6.0 - can be removed once `setup.py` is simplified
-tomlkit~=0.11.6
-
 # needed for running tests
 coverage[toml]>=5.5
 freezegun >= 1.2.2
 mock>=4.0.3
-pyarrow>=1.0.0
 pytest>=6.2.5
 pytest-asyncio>=0.16.0
 pytest-benchmark>=3.4.1

setup.py

@@ -35,7 +35,6 @@
 from neo4j._meta import (
     deprecated_package as deprecated,
     package,
-    version,
 )
 
 
@@ -84,8 +83,4 @@ def changed_package_name(new_name):
 
 
 with changed_package_name(package):
-    setup(
-        # until `[tool.setuptools.dynamic]` in pyproject.toml is out of beta
-        version=version,
-        long_description=readme,
-    )
+    setup(long_description=readme)

src/neo4j/_async_compat/concurrency.py

@@ -136,6 +136,8 @@ async def acquire(self, blocking=True, timeout=-1):
             try:
                 await wait_for(fut, timeout)
             except asyncio.CancelledError:
+                if fut.cancelled():
+                    raise
                 already_finished = not fut.cancel()
                 if already_finished:
                     # Too late to cancel the acquisition.

src/neo4j/_async_compat/shims/__init__.py

@@ -25,12 +25,11 @@
 # The shipped wait_for can swallow cancellation errors (starting with 3.8).
 # See: https://github.com/python/cpython/pull/26097
 # and https://github.com/python/cpython/pull/28149
-# Since 3.8 and 3.9 already received their final maintenance release, there
-# will be now fix for this. So this patch needs to stick around at least until
-# we remove support for Python 3.9.
+# Ultimately, this got fixed in https://github.com/python/cpython/pull/98518
+# (released with Python 3.12) by re-doing how wait_for works.
 
 
-if sys.version_info >= (3, 8):
+if (3, 12) > sys.version_info >= (3, 8):
     # copied from Python 3.10's asyncio package with applied patch
 
     # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,

testkit/Dockerfile

@@ -42,19 +42,29 @@ ENV PYENV_ROOT /.pyenv
 ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH
 
 # Setup python version
-ENV PYTHON_VERSIONS 3.7 3.8 3.9 3.10 3.11
+ENV PYTHON_VERSIONS 3.12 3.11 3.10 3.9 3.8 3.7
 
 RUN for version in $PYTHON_VERSIONS; do \
         pyenv install $version; \
     done
 RUN pyenv rehash
-RUN pyenv global $(pyenv versions --bare --skip-aliases)
+RUN pyenv global $(pyenv versions --bare --skip-aliases | sort --version-sort --reverse)
 
 # Install Latest pip and setuptools for each environment
 # + tox and tools for starting the tests
 # https://pip.pypa.io/en/stable/news/
 RUN for version in $PYTHON_VERSIONS; do \
         python$version -m pip install -U pip && \
-        python$version -m pip install -U setuptools && \
         python$version -m pip install -U coverage tox; \
     done
+
+# Installing pyarrow lib until pre-built wheel for Python 3.12 exists
+# https://github.com/apache/arrow/issues/37880
+RUN apt update && \
+    apt install -y -V lsb-release cmake gcc && \
+    distro_name=$(lsb_release --id --short | tr 'A-Z' 'a-z') && \
+    code_name=$(lsb_release --codename --short) && \
+    wget https://apache.jfrog.io/artifactory/arrow/${distro_name}/apache-arrow-apt-source-latest-${code_name}.deb && \
+    apt install -y -V ./apache-arrow-apt-source-latest-${code_name}.deb && \
+    apt update && \
+    apt install -y -V libarrow-dev # For C++ \

testkit/_common.py

@@ -1,4 +1,5 @@
 import os
+import re
 import subprocess
 import sys
 
@@ -7,14 +8,28 @@
 
 
 def run(args, env=None):
+    print(args)
     return subprocess.run(
         args, universal_newlines=True, stdout=sys.stdout, stderr=sys.stderr,
         check=True, env=env
     )
 
 
+def get_python_version():
+    cmd = [TEST_BACKEND_VERSION, "-V"]
+    res = subprocess.check_output(cmd, universal_newlines=True,
+                                  stderr=sys.stderr)
+    raw_version = re.match(r"(?:.*?)((?:\d+\.)+(?:\d+))", res).group(1)
+    return tuple(int(e) for e in raw_version.split("."))
+
+
 def run_python(args, env=None, warning_as_error=True):
     cmd = [TEST_BACKEND_VERSION, "-u"]
+    if get_python_version() >= (3, 12):
+        # Ignore warnings for Python 3.12 for now
+        # https://github.com/dateutil/dateutil/issues/1284 needs to be released
+        # and propagate through our dependency graph
+        warning_as_error = False
     if warning_as_error:
         cmd += ["-W", "error"]
     cmd += list(args)

testkitbackend/__main__.py

@@ -49,7 +49,11 @@ async def main():
 
 
 if __name__ == "__main__":
-    warnings.simplefilter("error")
+    if sys.version_info < (3, 12):
+        # Ignore warnings for Python 3.12 for now
+        # https://github.com/dateutil/dateutil/issues/1284 needs to be released
+        # and propagate through our dependency graph
+        warnings.simplefilter("error")
     if len(sys.argv) == 2 and sys.argv[1].lower().strip() == "async":
         async_main()
     else:

tests/unit/async_/test_driver.py

@@ -716,6 +716,7 @@ async def test_execute_query_keyword_parameters(
 @pytest.mark.parametrize("parameters", (
     {"_": "a"}, {"foo_": None}, {"foo_": 1, "bar_": 2}
 ))
+@mark_async_test
 async def test_reserved_query_keyword_parameters(
     mocker, parameters: t.Dict[str, t.Any],
 ) -> None: