Revamp of cpychecker warnings

This commit combines several changes, all of which affect the "gold"
output for the selftests.

  * RefcountValue:

       * eliminate int "min_external" attribute in favor of
         a WithinRange "external" attribute, capturing the range of
         possible external references, rather than just a lower bound.

       * revamp __str__ to eliminate the "0 + N where N >= " since this
         was confusing to everyone (including me).  Instead, display
         references as owned vs borrowed references e.g.

           * "refs: 1 owned"

           * "refs: 0 owned 1 borrowed"

         resorting to:

           * "refs: 0 owned + B borrowed where 1 <= B <= 0x80000000"

         and the like where necessary (though this does not yet occur in
         the test suite).

  * error messages:

      * emit_refcount_warning

        * combine the various notes into one; tweak some wording;
          eliminate the "for some N" gobblegook in favor of str() of the
          refcount value.

        * use "desc" rather than "v_obj.name" since is more likely to
          be in terms that the user can understand (e.g. "*dictA" rather
          than "PyDictObject").

        * mention " since nothing references it" when dealing with an
          expected refcount of 0.

      * add "memory leak: " and 'future use-after-free: " prefixes to
        reports about refcounting errors, to better indicate the issue.

Also, regenerate all the expected testcase results (using 4.8.2-7)

This also strips out exact column numbering from the results, in favor
of "nn", due to this in run-test-suite.py:
            # GCC 4.7 tracks macro expansions, and this can change the column
            # numbers in error reports:

Author: davidmalcolm

libcpychecker/refcounts.py

@@ -276,52 +276,72 @@ class RefcountValue(AbstractValue):
 
     'relvalue' is all of the references owned within this function.
 
-    'min_external' is a lower bound on all references owned outside the
+    'external' is a WithinRange, a bound on all references owned outside the
     scope of this function.
 
-    The actual value of ob_refcnt >= (relvalue + min_external)
+    The actual value of ob_refcnt = (relvalue + external)
+    hence:
+
+        (relvalue + external.minvalue) <= ob_refcnt <= (relvalue + external.minvalue)
 
     Examples:
 
-      - an argument passed in a a borrowed ref starts with (0, 1), in that
+      - an argument passed in a borrowed ref starts with (0, [1, 1]), in that
       the function doesn't own any refs on it, but it has a refcount of at
       least 1, due to refs we know nothing about.
 
-      - a newly constructed object gets (1, 0): we own a reference on it,
+      - a newly constructed object gets (1, [0, 0]): we own a reference on it,
       and we don't know if there are any external refs on it.
     """
-    __slots__ = ('r_obj', 'relvalue', 'min_external')
+    __slots__ = ('r_obj', 'relvalue', 'external')
 
-    def __init__(self, loc, r_obj, relvalue, min_external):
+    def __init__(self, loc, r_obj, relvalue, external):
         if loc:
             check_isinstance(loc, gcc.Location)
         if r_obj:
             check_isinstance(r_obj, Region)
+        check_isinstance(external, WithinRange)
         AbstractValue.__init__(self, get_Py_ssize_t().type, loc)
         self.r_obj = r_obj
         self.relvalue = relvalue
-        self.min_external = min_external
+        self.external = external
 
     @classmethod
     def new_ref(cls, loc, r_obj):
         return RefcountValue(loc, r_obj,
                              relvalue=1,
-                             min_external=0)
+                             external=WithinRange(get_Py_ssize_t().type, loc,
+                                                  0, 0))
 
     @classmethod
     def borrowed_ref(cls, loc, r_obj):
         return RefcountValue(loc, r_obj,
                              relvalue=0,
-                             min_external=1)
+                             external=WithinRange(get_Py_ssize_t().type, loc,
+                                                  1, 1))
 
     def get_min_value(self):
-        return self.relvalue + self.min_external
+        return self.relvalue + self.external.minvalue
 
     def __str__(self):
-        return 'refs: %i + N where N >= %i' % (self.relvalue, self.min_external)
+        # Try to present the simplest possible string representation of
+        # a refcount:
+        if self.external.minvalue == self.external.maxvalue:
+            # Known value for external refcount
+            if self.external.minvalue == 0:
+                # We know there are no external refs; don't display it
+                # at all, rather than "0":
+                return 'refs: %i owned' % self.relvalue
+            else:
+                # We know of an exact number of external refs; display
+                # it as borrowed:
+                return ('refs: %i owned, %i borrowed'
+                        % (self.relvalue, self.external.minvalue))
+        return ('refs: %i owned + B borrowed where %s'
+                % (self.relvalue, str(self.external).replace('val', 'B')))
 
     def __repr__(self):
-        return 'RefcountValue(%i, %i)' % (self.relvalue, self.min_external)
+        return 'RefcountValue(%i, %r)' % (self.relvalue, self.external)
 
     def get_referrers_as_json(self, state):
         # FIXME:
@@ -346,7 +366,7 @@ def get_referrers_as_json(self, state):
 
     def json_fields(self, state):
         actual = OrderedDict(refs_we_own=self.relvalue,
-                             lower_bound_of_other_refs=self.min_external)
+                             lower_bound_of_other_refs=self.external.minvalue)
         exp_refs = self.get_referrers_as_json(state)
         expected = dict(pointers_to_this=exp_refs)
         return dict(actual_ob_refcnt=actual,
@@ -356,10 +376,10 @@ def eval_binop(self, exprcode, rhs, rhsdesc, gcctype, loc):
         if isinstance(rhs, ConcreteValue):
             if exprcode == gcc.PlusExpr:
                 return RefcountValue(loc, self.r_obj,
-                                     self.relvalue + rhs.value, self.min_external)
+                                     self.relvalue + rhs.value, self.external)
             elif exprcode == gcc.MinusExpr:
                 return RefcountValue(loc, self.r_obj,
-                                     self.relvalue - rhs.value, self.min_external)
+                                     self.relvalue - rhs.value, self.external)
         return UnknownValue.make(gcctype, loc)
 
     def eval_comparison(self, opname, rhs, rhsdesc):
@@ -566,7 +586,7 @@ def _incref_internal(oldvalue):
             return RefcountValue(loc,
                                  pyobjectptr.region,
                                  oldvalue.relvalue + 1,
-                                 oldvalue.min_external)
+                                 oldvalue.external)
         self.change_refcount(pyobjectptr,
                              loc,
                              _incref_internal)
@@ -580,7 +600,10 @@ def _incref_external(oldvalue):
             return RefcountValue(loc,
                                  pyobjectptr.region,
                                  oldvalue.relvalue,
-                                 oldvalue.min_external + 1)
+                                 WithinRange(oldvalue.external.gcctype,
+                                             loc,
+                                             oldvalue.external.minvalue + 1,
+                                             oldvalue.external.maxvalue + 1))
         self.change_refcount(pyobjectptr,
                              loc,
                              _incref_external)
@@ -594,7 +617,7 @@ def _decref_internal(oldvalue):
             return RefcountValue(loc,
                                  pyobjectptr.region,
                                  oldvalue.relvalue - 1,
-                                 oldvalue.min_external)
+                                 oldvalue.external)
         check_isinstance(pyobjectptr, PointerToRegion)
         v_ob_refcnt = self.change_refcount(pyobjectptr,
                                            loc,
@@ -737,7 +760,10 @@ def _steal_ref(v_old):
             return RefcountValue(loc,
                                  pyobjectptr.region,
                                  v_old.relvalue - 1,
-                                 v_old.min_external + 1)
+                                 WithinRange(v_old.external.gcctype,
+                                             loc,
+                                             v_old.external.minvalue + 1,
+                                             v_old.external.maxvalue + 1))
         check_isinstance(pyobjectptr, PointerToRegion)
         self.change_refcount(pyobjectptr,
                              loc,
@@ -4019,26 +4045,28 @@ def emit_refcount_warning(msg,
                           exp_refcnt, exp_refs, v_ob_refcnt, r_obj, desc,
                           trace, endstate, fun, rep):
     w = rep.make_warning(fun, endstate.get_gcc_loc(fun), msg)
-    w.add_note(endstate.get_gcc_loc(fun),
-               ('was expecting final ob_refcnt to be N + %i (for some unknown N)'
-                % exp_refcnt))
-    if exp_refcnt > 0:
-        w.add_note(endstate.get_gcc_loc(fun),
-                   ('due to object being referenced by: %s'
-                    % ', '.join(exp_refs)))
-    w.add_note(endstate.get_gcc_loc(fun),
-               ('but final ob_refcnt is N + %i'
-                % v_ob_refcnt.relvalue))
+
     # For dynamically-allocated objects, indicate where they
     # were allocated:
     if isinstance(r_obj, RegionOnHeap):
         alloc_loc = r_obj.alloc_stmt.loc
         if alloc_loc:
             w.add_note(r_obj.alloc_stmt.loc,
-                         ('%s allocated at: %s'
-                          % (r_obj.name,
+                         ('%s was allocated at: %s'
+                          % (desc,
                              get_src_for_loc(alloc_loc))))
 
+    details = ('was expecting final owned ob_refcnt of %s to be %i'
+               % (desc, exp_refcnt))
+    if exp_refcnt > 0:
+        details += (' due to object being referenced by: %s'
+                    % ', '.join(exp_refs))
+    else:
+        details += (' since nothing references it')
+    details += ' but final ob_refcnt is %s' % v_ob_refcnt
+    w.add_note(endstate.get_gcc_loc(fun),
+               details)
+
     # Summarize the control flow we followed through the function:
     if 1:
         annotator = RefcountAnnotator(r_obj, desc)
@@ -4102,14 +4130,14 @@ def check_refcount_for_one_object(r_obj, v_ob_refcnt, v_return,
     if isinstance(v_ob_refcnt, RefcountValue):
         if v_ob_refcnt.relvalue > exp_refcnt:
             # Refcount is too high:
-            w = emit_refcount_warning('ob_refcnt of %s is %i too high'
+            w = emit_refcount_warning('memory leak: ob_refcnt of %s is %i too high'
                                       % (desc,
                                          v_ob_refcnt.relvalue - exp_refcnt),
                                       exp_refcnt, exp_refs, v_ob_refcnt, r_obj, desc,
                                       trace, endstate, fun, rep)
         elif v_ob_refcnt.relvalue < exp_refcnt:
             # Refcount is too low:
-            w = emit_refcount_warning('ob_refcnt of %s is %i too low'
+            w = emit_refcount_warning('future use-after-free: ob_refcnt of %s is %i too low'
                                       % (desc,
                                          exp_refcnt - v_ob_refcnt.relvalue),
                                       exp_refcnt, exp_refs, v_ob_refcnt, r_obj, desc,

tests/cpychecker/PyArg_Parse/incorrect_codes_S_and_U/stderr.txt

@@ -1,11 +1,11 @@
-tests/cpychecker/PyArg_Parse/incorrect_codes_S_and_U/input.c: In function 'incorrect_usage_of_S_and_U':
-tests/cpychecker/PyArg_Parse/incorrect_codes_S_and_U/input.c:33:21: warning: Mismatching type in call to PyArg_Parse with format code "SU" [enabled by default]
+In function 'incorrect_usage_of_S_and_U':
+tests/cpychecker/PyArg_Parse/incorrect_codes_S_and_U/input.c:33:nn: warning: Mismatching type in call to PyArg_Parse with format code "SU" [enabled by default]
   argument 3 ("&val1") had type
     "int *" (pointing to 32 bits)
   but was expecting
     one of "struct PyStringObject * *" or "struct PyObject * *"
   for format code "S"
-tests/cpychecker/PyArg_Parse/incorrect_codes_S_and_U/input.c:33:21: warning: Mismatching type in call to PyArg_Parse with format code "SU" [enabled by default]
+tests/cpychecker/PyArg_Parse/incorrect_codes_S_and_U/input.c:33:nn: warning: Mismatching type in call to PyArg_Parse with format code "SU" [enabled by default]
   argument 4 ("&val2") had type
     "int *" (pointing to 32 bits)
   but was expecting

tests/cpychecker/PyArg_ParseTuple/code_O_bang/stderr.txt

@@ -1,17 +1,17 @@
-tests/cpychecker/PyArg_ParseTuple/code_O_bang/input.c: In function 'handle_subclasses':
-tests/cpychecker/PyArg_ParseTuple/code_O_bang/input.c:85:19: warning: Mismatching type in call to PyArg_ParseTuple with format code "O!O!O!O!O!O!O!" [enabled by default]
+In function 'handle_subclasses':
+tests/cpychecker/PyArg_ParseTuple/code_O_bang/input.c:85:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "O!O!O!O!O!O!O!" [enabled by default]
   argument 10 ("long_obj") had type
     "struct PyLongObject *"
   but was expecting
     "struct PyUnicodeObject * *" (based on PyTypeObject: 'PyUnicode_Type') or "struct PyObject * *"
   for format code "O!"
-tests/cpychecker/PyArg_ParseTuple/code_O_bang/input.c:90:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "O!O!O!O!O!O!O!" [enabled by default]
+tests/cpychecker/PyArg_ParseTuple/code_O_bang/input.c:90:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "O!O!O!O!O!O!O!" [enabled by default]
   argument 12 ("&unknown_obj") had type
     "struct UnknownObject * *"
   but was expecting
     ""struct PyObject * *"" (unable to determine relevant PyTypeObject)
   for format code "O!"
-tests/cpychecker/PyArg_ParseTuple/code_O_bang/input.c:90:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "O!O!O!O!O!O!O!" [enabled by default]
+tests/cpychecker/PyArg_ParseTuple/code_O_bang/input.c:90:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "O!O!O!O!O!O!O!" [enabled by default]
   argument 14 ("&extension_obj") had type
     "struct ExtensionObject * *"
   but was expecting

tests/cpychecker/PyArg_ParseTuple/code_s/incorrect-constness/stderr.txt

@@ -1,5 +1,5 @@
-tests/cpychecker/PyArg_ParseTuple/code_s/incorrect-constness/input.c: In function 'test':
-tests/cpychecker/PyArg_ParseTuple/code_s/incorrect-constness/input.c:34:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "s" [enabled by default]
+In function 'test':
+tests/cpychecker/PyArg_ParseTuple/code_s/incorrect-constness/input.c:34:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "s" [enabled by default]
   argument 3 ("&str") had type
     "char * *"
   but was expecting

tests/cpychecker/PyArg_ParseTuple/incorrect_code_z_hash/stderr.txt

@@ -1,5 +1,5 @@
-tests/cpychecker/PyArg_ParseTuple/incorrect_code_z_hash/input.c: In function 'incorrect_code_z_hash':
-tests/cpychecker/PyArg_ParseTuple/incorrect_code_z_hash/input.c:34:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "z#" [enabled by default]
+In function 'incorrect_code_z_hash':
+tests/cpychecker/PyArg_ParseTuple/incorrect_code_z_hash/input.c:34:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "z#" [enabled by default]
   argument 4 ("&len") had type
     "float *" (pointing to 32 bits)
   but was expecting

tests/cpychecker/PyArg_ParseTuple/incorrect_codes_S_and_U/stderr.txt

@@ -1,11 +1,11 @@
-tests/cpychecker/PyArg_ParseTuple/incorrect_codes_S_and_U/input.c: In function 'incorrect_usage_of_S_and_U':
-tests/cpychecker/PyArg_ParseTuple/incorrect_codes_S_and_U/input.c:29:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "SU" [enabled by default]
+In function 'incorrect_usage_of_S_and_U':
+tests/cpychecker/PyArg_ParseTuple/incorrect_codes_S_and_U/input.c:29:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "SU" [enabled by default]
   argument 3 ("&val1") had type
     "int *" (pointing to 32 bits)
   but was expecting
     one of "struct PyStringObject * *" or "struct PyObject * *"
   for format code "S"
-tests/cpychecker/PyArg_ParseTuple/incorrect_codes_S_and_U/input.c:29:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "SU" [enabled by default]
+tests/cpychecker/PyArg_ParseTuple/incorrect_codes_S_and_U/input.c:29:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "SU" [enabled by default]
   argument 4 ("&val2") had type
     "int *" (pointing to 32 bits)
   but was expecting

tests/cpychecker/PyArg_ParseTuple/incorrect_converters/stderr.txt

@@ -1,60 +1,60 @@
-tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c: In function 'incorrect_usages_of_converter':
-tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:36:26: warning: Not enough arguments in call to PyArg_ParseTuple with format string "O&"
+In function 'incorrect_usages_of_converter':
+tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:36:nn: warning: Not enough arguments in call to PyArg_ParseTuple with format string "O&"
   expected 2 extra arguments:
     "int (converter)(PyObject *, T*)" for some type T
     "T*" for some type T
   but got none
  [enabled by default]
-tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:39:26: warning: Not enough arguments in call to PyArg_ParseTuple with format string "O&"
+tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:39:nn: warning: Not enough arguments in call to PyArg_ParseTuple with format string "O&"
   expected 2 extra arguments:
     "int (converter)(PyObject *, T*)" for some type T
     "T*" for some type T
   but got 1:
     "int"
  [enabled by default]
-tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:44:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
+tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:44:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
   argument 3 ("42") had type
     "int"
   but was expecting
     "int (converter)(PyObject *, T*)" for some type T
   for format code "O&"
-tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:49:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
+tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:49:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
   argument 3 ("&i0") had type
     "int *" (pointing to 32 bits)
   but was expecting
     "int (converter)(PyObject *, T*)" for some type T
   for format code "O&"
-tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:54:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
+tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:54:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
   argument 3 ("not_returning_int") had type
     "void (*fn) ()"
   but was expecting
     "int (converter)(PyObject *, T*)" for some type T
   for format code "O&"
-tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:57:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
+tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:57:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
   argument 3 ("not_enough_args") had type
     "int (*fn) (struct PyObject *)"
   but was expecting
     "int (converter)(PyObject *, T*)" for some type T
   for format code "O&"
-tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:60:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
+tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:60:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
   argument 3 ("too_many_args") had type
     "int (*fn) (struct PyObject *, int *, int)"
   but was expecting
     "int (converter)(PyObject *, T*)" for some type T
   for format code "O&"
-tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:63:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
+tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:63:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
   argument 3 ("not_taking_PyObjectPtr") had type
     "int (*fn) (int, int)"
   but was expecting
     "int (converter)(PyObject *, T*)" for some type T
   for format code "O&"
-tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:68:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
+tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:68:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
   argument 4 ("42") had type
     "int"
   but was expecting
     "Py_ssize_t *" (pointing to 64 bits) (from second argument of "int (*fn) (struct PyObject *, Py_ssize_t *)")
   for format code "O&"
-tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:73:26: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
+tests/cpychecker/PyArg_ParseTuple/incorrect_converters/input.c:73:nn: warning: Mismatching type in call to PyArg_ParseTuple with format code "O&" [enabled by default]
   argument 4 ("&i0") had type
     "int *" (pointing to 32 bits)
   but was expecting

tests/cpychecker/PyArg_ParseTuple/keywords/stderr.txt

@@ -1,2 +1,2 @@
-tests/cpychecker/PyArg_ParseTuple/keywords/input.c: In function 'parse_to_a_typedef':
-tests/cpychecker/PyArg_ParseTuple/keywords/input.c:32:37: warning: keywords to PyArg_ParseTupleAndKeywords are not NULL-terminated [enabled by default]
+In function 'parse_to_a_typedef':
+tests/cpychecker/PyArg_ParseTuple/keywords/input.c:32:nn: warning: keywords to PyArg_ParseTupleAndKeywords are not NULL-terminated [enabled by default]