Backport type stubs in python<=3.9 to support PEP 585 and 604 syntax.

Author: mauvilsa

CHANGELOG.rst

@@ -21,6 +21,9 @@ Added
 - Support for pydantic models and attr defines similar to dataclasses.
 - Support for `pydantic types
   <https://docs.pydantic.dev/usage/types/#pydantic-types>`__.
+- Backport type stubs in python<=3.9 to support PEP `585
+  <https://peps.python.org/pep-0585/>`__ and `604
+  <https://peps.python.org/pep-0604/>`__ syntax.
 
 Fixed
 ^^^^^
@@ -39,6 +42,8 @@ Changed
 - Include enum members in error when invalid value is given
   (`pytorch-lightning#17247
   <https://github.com/Lightning-AI/lightning/issues/17247>`__).
+- The ``signatures`` extras now installs the ``typing-extensions`` package on
+  python<=3.9.
 
 Deprecated
 ^^^^^^^^^^

README.rst

@@ -1689,10 +1689,9 @@ Many of the types defined in stub files use the latest syntax for type hints,
 that is, bitwise or operator ``|`` for unions and generics, e.g.
 ``list[<type>]`` instead of ``typing.List[<type>]``, see PEPs `604
 <https://peps.python.org/pep-0604>`__ and `585
-<https://peps.python.org/pep-0585>`__. The types with this new syntax can't be
-evaluated at runtime in Python versions older than ``3.10``. Since jsonargparse
-needs to interpret the types at runtime, these will only be resolved in newer
-versions of Python.
+<https://peps.python.org/pep-0585>`__. On python>=3.10 these are fully
+supported. On python<=3.9 backporting these types is attempted and in some cases
+it can fail. On failure the type annotation is set to ``Any``.
 
 Most of the types in the Python standard library have their types in stubs. An
 example from the standard library would be:

jsonargparse/_backports.py

@@ -0,0 +1,74 @@
+import ast
+from collections import namedtuple
+from copy import deepcopy
+from typing import Dict, FrozenSet, List, Set, Tuple, Type, Union
+
+var_map = namedtuple('var_map', 'name value')
+none_map = var_map(name='NoneType', value=type(None))
+union_map = var_map(name='Union', value=Union)
+pep585_map = {
+    'dict': var_map(name='Dict', value=Dict),
+    'frozenset': var_map(name='FrozenSet', value=FrozenSet),
+    'list': var_map(name='List', value=List),
+    'set': var_map(name='Set', value=Set),
+    'tuple': var_map(name='Tuple', value=Tuple),
+    'type': var_map(name='Type', value=Type),
+}
+
+
+class BackportTypeHints(ast.NodeTransformer):
+
+    _typing = __import__('typing')
+
+    def visit_Subscript(self, node: ast.Subscript) -> ast.Subscript:
+        if isinstance(node.value, ast.Name) and node.value.id in pep585_map:
+            value = self.new_name_load(pep585_map[node.value.id])
+        else:
+            value = node.value  # type: ignore
+        return ast.Subscript(
+            value=value,
+            slice=self.visit(node.slice),
+            ctx=ast.Load(),
+        )
+
+    def visit_Constant(self, node: ast.Constant) -> Union[ast.Constant, ast.Name]:
+        if node.value is None:
+            return self.new_name_load(none_map)
+        return node
+
+    def visit_BinOp(self, node: ast.BinOp) -> Union[ast.BinOp, ast.Subscript]:
+        out_node: Union[ast.BinOp, ast.Subscript] = node
+        if isinstance(node.op, ast.BitOr):
+            elts: list = []
+            self.append_union_elts(node.left, elts)
+            self.append_union_elts(node.right, elts)
+            out_node = ast.Subscript(
+                value=self.new_name_load(union_map),
+                slice=ast.Index(
+                    value=ast.Tuple(elts=elts, ctx=ast.Load()),
+                    ctx=ast.Load(),
+                ),
+                ctx=ast.Load(),
+            )
+        return out_node
+
+    def append_union_elts(self, node: ast.AST, elts: list) -> None:
+        if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr):
+            self.append_union_elts(node.left, elts)
+            self.append_union_elts(node.right, elts)
+        else:
+            elts.append(self.visit(node))
+
+    def new_name_load(self, var: var_map) -> ast.Name:
+        name = f'_{self.__class__.__name__}_{var.name}'
+        self.exec_vars[name] = var.value
+        return ast.Name(id=name, ctx=ast.Load())
+
+    def backport(self, input_ast: ast.AST, exec_vars: dict) -> ast.AST:
+        for key, value in exec_vars.items():
+            if getattr(value, '__module__', '') == 'collections.abc':
+                if hasattr(self._typing, key):
+                    exec_vars[key] = getattr(self._typing, key)
+        self.exec_vars = exec_vars
+        backport_ast = self.visit(deepcopy(input_ast))
+        return ast.fix_missing_locations(backport_ast)

jsonargparse/_stubs_resolver.py

@@ -293,6 +293,10 @@ def get_arg_type(arg_ast, aliases):
             type_alias = typing_extensions_import('TypeAlias')
             if type_alias:
                 exec_vars['TypeAlias'] = type_alias
+    if sys.version_info < (3, 10):
+        from ._backports import BackportTypeHints
+        backporter = BackportTypeHints()
+        type_ast = backporter.backport(type_ast, exec_vars)
     try:
         exec(compile(type_ast, filename="<ast>", mode="exec"), exec_vars, exec_vars)
     except NameError as ex:

jsonargparse_tests/test_stubs_resolver.py

@@ -95,9 +95,10 @@ def test_get_params_conditional_python_version(self):
         self.assertEqual(['a', 'version'], get_param_names(params))
         if sys.version_info >= (3, 10):
             self.assertEqual('int | float | str | bytes | bytearray | None', str(params[0].annotation))
+        elif sys.version_info[:2] == (3, 9):
+            self.assertEqual('typing.Union[int, float, str, bytes, bytearray, NoneType]', str(params[0].annotation))
         else:
-            expected = Any if sys.version_info < (3, 9) else inspect._empty
-            self.assertEqual(expected, params[0].annotation)
+            self.assertEqual(Any, params[0].annotation)
         self.assertEqual(int, params[1].annotation)
         with mock_typeshed_client_unavailable():
             params = get_params(Random, 'seed')
@@ -140,7 +141,7 @@ def test_get_params_function(self):
     def test_get_param_relative_import_from_init(self):
         params = get_params(yaml.safe_load)
         self.assertEqual(['stream'], get_param_names(params))
-        if sys.version_info >= (3, 10):
+        if sys.version_info >= (3, 8):
             self.assertNotEqual(params[0].annotation, inspect._empty)
         else:
             self.assertEqual(params[0].annotation, inspect._empty)

pyproject.toml

@@ -49,6 +49,7 @@ all = [
     "jsonargparse[reconplogger]",
 ]
 signatures = [
+    "jsonargparse[typing-extensions]",
     "docstring-parser>=0.15",
     "typeshed-client>=2.1.0",
 ]
@@ -75,7 +76,7 @@ omegaconf = [
     "omegaconf>=2.1.1",
 ]
 typing-extensions = [
-    "typing-extensions>=3.10.0.0; python_version < '3.8'",
+    "typing-extensions>=3.10.0.0; python_version < '3.10'",
 ]
 reconplogger = [
     "reconplogger>=4.4.0",