Treat NewTypes like normal subclasses

NewTypes are assumed not to inherit any members from their base classes.
This results in incorrect inference results. Avoid this by changing the
transformation for NewTypes to treat them like any other subclass.

pylint-dev/pylint#3162
pylint-dev/pylint#2296

Author: colatkinson

ChangeLog

@@ -55,6 +55,11 @@ Release date: TBA
 * Fix test for Python ``3.11``. In some instances ``err.__traceback__`` will
   be uninferable now.
 
+* Treat ``typing.NewType()`` values as normal subclasses.
+
+  Closes PyCQA/pylint#2296
+  Closes PyCQA/pylint#3162
+
 What's New in astroid 2.11.6?
 =============================
 Release date: TBA

astroid/brain/brain_typing.py

@@ -10,7 +10,7 @@
 from collections.abc import Iterator
 from functools import partial
 
-from astroid import context, extract_node, inference_tip
+from astroid import context, extract_node, inference_tip, nodes
 from astroid.builder import _extract_single_node
 from astroid.const import PY38_PLUS, PY39_PLUS
 from astroid.exceptions import (
@@ -35,8 +35,6 @@
 from astroid.util import Uninferable
 
 TYPING_NAMEDTUPLE_BASENAMES = {"NamedTuple", "typing.NamedTuple"}
-TYPING_TYPEVARS = {"TypeVar", "NewType"}
-TYPING_TYPEVARS_QUALIFIED = {"typing.TypeVar", "typing.NewType"}
 TYPING_TYPE_TEMPLATE = """
 class Meta(type):
     def __getitem__(self, item):
@@ -49,6 +47,13 @@ def __args__(self):
 class {0}(metaclass=Meta):
     pass
 """
+# PEP484 suggests NewType is equivalent to this for typing purposes
+# https://www.python.org/dev/peps/pep-0484/#newtype-helper-function
+TYPING_NEWTYPE_TEMPLATE = """
+class {derived}({base}):
+    def __init__(self, val: {base}) -> None:
+        ...
+"""
 TYPING_MEMBERS = set(getattr(typing, "__all__", []))
 
 TYPING_ALIAS = frozenset(
@@ -103,24 +108,33 @@ def __class_getitem__(cls, item):
 """
 
 
-def looks_like_typing_typevar_or_newtype(node):
+def looks_like_typing_typevar(node: nodes.Call) -> bool:
+    func = node.func
+    if isinstance(func, Attribute):
+        return func.attrname == "TypeVar"
+    if isinstance(func, Name):
+        return func.name == "TypeVar"
+    return False
+
+
+def looks_like_typing_newtype(node: nodes.Call) -> bool:
     func = node.func
     if isinstance(func, Attribute):
-        return func.attrname in TYPING_TYPEVARS
+        return func.attrname == "NewType"
     if isinstance(func, Name):
-        return func.name in TYPING_TYPEVARS
+        return func.name == "NewType"
     return False
 
 
-def infer_typing_typevar_or_newtype(node, context_itton=None):
-    """Infer a typing.TypeVar(...) or typing.NewType(...) call"""
+def infer_typing_typevar(
+    node: nodes.Call, ctx: context.InferenceContext | None = None
+) -> Iterator[nodes.ClassDef]:
+    """Infer a typing.TypeVar(...) call"""
     try:
-        func = next(node.func.infer(context=context_itton))
+        next(node.func.infer(context=ctx))
     except (InferenceError, StopIteration) as exc:
         raise UseInferenceDefault from exc
 
-    if func.qname() not in TYPING_TYPEVARS_QUALIFIED:
-        raise UseInferenceDefault
     if not node.args:
         raise UseInferenceDefault
     # Cannot infer from a dynamic class name (f-string)
@@ -129,7 +143,53 @@ def infer_typing_typevar_or_newtype(node, context_itton=None):
 
     typename = node.args[0].as_string().strip("'")
     node = extract_node(TYPING_TYPE_TEMPLATE.format(typename))
-    return node.infer(context=context_itton)
+    return node.infer(context=ctx)
+
+
+def infer_typing_newtype(
+    node: nodes.Call, ctx: context.InferenceContext | None = None
+) -> Iterator[nodes.ClassDef]:
+    """Infer a typing.NewType(...) call"""
+    try:
+        next(node.func.infer(context=ctx))
+    except (InferenceError, StopIteration) as exc:
+        raise UseInferenceDefault from exc
+
+    if len(node.args) != 2:
+        raise UseInferenceDefault
+
+    # Cannot infer from a dynamic class name (f-string)
+    if isinstance(node.args[0], JoinedStr) or isinstance(node.args[1], JoinedStr):
+        raise UseInferenceDefault
+
+    derived, base = node.args
+    derived_name = derived.as_string().strip("'")
+    base_name = base.as_string().strip("'")
+
+    new_node: ClassDef = extract_node(
+        TYPING_NEWTYPE_TEMPLATE.format(derived=derived_name, base=base_name)
+    )
+    new_node.parent = node.parent
+
+    # Base type arg is a normal reference, so no need to do special lookups
+    if not isinstance(base, nodes.Const):
+        new_node.postinit(
+            bases=[base], body=new_node.body, decorators=new_node.decorators
+        )
+
+    # If the base type is given as a string (e.g. for a forward reference),
+    # make a naive attempt to find the corresponding node.
+    # Note that this will not work with imported types.
+    if isinstance(base, nodes.Const) and isinstance(base.value, str):
+        _, resolved_base = node.frame().lookup(base_name)
+        if resolved_base:
+            new_node.postinit(
+                bases=[resolved_base[0]],
+                body=new_node.body,
+                decorators=new_node.decorators,
+            )
+
+    return new_node.infer(context=ctx)
 
 
 def _looks_like_typing_subscript(node):
@@ -403,8 +463,13 @@ def infer_typing_cast(
 
 AstroidManager().register_transform(
     Call,
-    inference_tip(infer_typing_typevar_or_newtype),
-    looks_like_typing_typevar_or_newtype,
+    inference_tip(infer_typing_typevar),
+    looks_like_typing_typevar,
+)
+AstroidManager().register_transform(
+    Call,
+    inference_tip(infer_typing_newtype),
+    looks_like_typing_newtype,
 )
 AstroidManager().register_transform(
     Subscript, inference_tip(infer_typing_attr), _looks_like_typing_subscript

tests/unittest_brain.py

@@ -1640,6 +1640,26 @@ def test_typing_types(self) -> None:
             inferred = next(node.infer())
             self.assertIsInstance(inferred, nodes.ClassDef, node.as_string())
 
+    def test_typing_typevar_bad_args(self) -> None:
+        ast_nodes = builder.extract_node(
+            """
+        from typing import TypeVar
+
+        T = TypeVar()
+        T #@
+
+        U = TypeVar(f"U")
+        U #@
+        """
+        )
+        assert isinstance(ast_nodes, list)
+
+        no_args_node = ast_nodes[0]
+        assert list(no_args_node.infer()) == [util.Uninferable]
+
+        fstr_node = ast_nodes[1]
+        assert list(fstr_node.infer()) == [util.Uninferable]
+
     def test_typing_type_without_tip(self):
         """Regression test for https://github.com/PyCQA/pylint/issues/5770"""
         node = builder.extract_node(
@@ -1651,7 +1671,171 @@ def make_new_type(t):
         """
         )
         with self.assertRaises(UseInferenceDefault):
-            astroid.brain.brain_typing.infer_typing_typevar_or_newtype(node.value)
+            astroid.brain.brain_typing.infer_typing_newtype(node.value)
+
+    def test_typing_newtype_attrs(self) -> None:
+        ast_nodes = builder.extract_node(
+            """
+        from typing import NewType
+        import decimal
+        from decimal import Decimal
+
+        NewType("Foo", str) #@
+        NewType("Bar", "int") #@
+        NewType("Baz", Decimal) #@
+        NewType("Qux", decimal.Decimal) #@
+        """
+        )
+        assert isinstance(ast_nodes, list)
+
+        # Base type given by reference
+        foo_node = ast_nodes[0]
+
+        # Should be unambiguous
+        foo_inferred_all = list(foo_node.infer())
+        assert len(foo_inferred_all) == 1
+
+        foo_inferred = foo_inferred_all[0]
+        assert isinstance(foo_inferred, astroid.ClassDef)
+
+        # Check base type method is inferred by accessing one of its methods
+        foo_base_class_method = foo_inferred.getattr("endswith")[0]
+        assert isinstance(foo_base_class_method, astroid.FunctionDef)
+        assert foo_base_class_method.qname() == "builtins.str.endswith"
+
+        # Base type given by string (i.e. "int")
+        bar_node = ast_nodes[1]
+        bar_inferred_all = list(bar_node.infer())
+        assert len(bar_inferred_all) == 1
+        bar_inferred = bar_inferred_all[0]
+        assert isinstance(bar_inferred, astroid.ClassDef)
+
+        bar_base_class_method = bar_inferred.getattr("bit_length")[0]
+        assert isinstance(bar_base_class_method, astroid.FunctionDef)
+        assert bar_base_class_method.qname() == "builtins.int.bit_length"
+
+        # Decimal may be reexported from an implementation-defined module. For
+        # example, in CPython 3.10 this is _decimal, but in PyPy 7.3 it's
+        # _pydecimal. So the expected qname needs to be grabbed dynamically.
+        decimal_quant_node = builder.extract_node(
+            """
+        from decimal import Decimal
+        Decimal.quantize #@
+        """
+        )
+        assert isinstance(decimal_quant_node, nodes.NodeNG)
+
+        # Just grab the first result, since infer() may return values for both
+        # _decimal and _pydecimal
+        decimal_quant_qname = next(decimal_quant_node.infer()).qname()
+
+        # Base type is from an "import from"
+        baz_node = ast_nodes[2]
+        baz_inferred_all = list(baz_node.infer())
+        assert len(baz_inferred_all) == 1
+        baz_inferred = baz_inferred_all[0]
+        assert isinstance(baz_inferred, astroid.ClassDef)
+
+        baz_base_class_method = baz_inferred.getattr("quantize")[0]
+        assert isinstance(baz_base_class_method, astroid.FunctionDef)
+        assert decimal_quant_qname == baz_base_class_method.qname()
+
+        # Base type is from an import
+        qux_node = ast_nodes[3]
+        qux_inferred_all = list(qux_node.infer())
+        qux_inferred = qux_inferred_all[0]
+        assert isinstance(qux_inferred, astroid.ClassDef)
+
+        qux_base_class_method = qux_inferred.getattr("quantize")[0]
+        assert isinstance(qux_base_class_method, astroid.FunctionDef)
+        assert decimal_quant_qname == qux_base_class_method.qname()
+
+    def test_typing_newtype_bad_args(self) -> None:
+        ast_nodes = builder.extract_node(
+            """
+        from typing import NewType
+
+        NoArgs = NewType()
+        NoArgs #@
+
+        OneArg = NewType("OneArg")
+        OneArg #@
+
+        ThreeArgs = NewType("ThreeArgs", int, str)
+        ThreeArgs #@
+
+        DynamicArg = NewType(f"DynamicArg", int)
+        DynamicArg #@
+
+        DynamicBase = NewType("DynamicBase", f"int")
+        DynamicBase #@
+        """
+        )
+        assert isinstance(ast_nodes, list)
+
+        node: nodes.NodeNG
+        for node in ast_nodes:
+            assert list(node.infer()) == [util.Uninferable]
+
+    def test_typing_newtype_user_defined(self) -> None:
+        ast_nodes = builder.extract_node(
+            """
+        from typing import NewType
+
+        class A:
+            def __init__(self, value: int):
+                self.value = value
+
+        a = A(5)
+        a #@
+
+        B = NewType("B", A)
+        b = B(5)
+        b #@
+        """
+        )
+        assert isinstance(ast_nodes, list)
+
+        for node in ast_nodes:
+            self._verify_node_has_expected_attr(node)
+
+    def test_typing_newtype_forward_reference(self) -> None:
+        # Similar to the test above, but using a forward reference for "A"
+        ast_nodes = builder.extract_node(
+            """
+        from typing import NewType
+
+        B = NewType("B", "A")
+
+        class A:
+            def __init__(self, value: int):
+                self.value = value
+
+        a = A(5)
+        a #@
+
+        b = B(5)
+        b #@
+        """
+        )
+        assert isinstance(ast_nodes, list)
+
+        for node in ast_nodes:
+            self._verify_node_has_expected_attr(node)
+
+    def _verify_node_has_expected_attr(self, node: nodes.NodeNG) -> None:
+        inferred_all = list(node.infer())
+        assert len(inferred_all) == 1
+        inferred = inferred_all[0]
+        assert isinstance(inferred, astroid.Instance)
+
+        # Should be able to infer that the "value" attr is present on both types
+        val = inferred.getattr("value")[0]
+        assert isinstance(val, astroid.AssignAttr)
+
+        # Sanity check: nonexistent attr is not inferred
+        with self.assertRaises(AttributeInferenceError):
+            inferred.getattr("bad_attr")
 
     def test_namedtuple_nested_class(self):
         result = builder.extract_node(