Rewrite the MRTG framework, and fix some associated serialization issues.

Prior to this change, the MRTG (MutuallyRecursiveTypeGroup) framework was not threadsafe.
What was happening was that if two threads both attempted to hash a big group of types,
such as a large class with many functions in it, there was a race condition where one
thread would check in the entire group while the other was still traversing it. The
other group would then construct a second group consisting of the types it had visited,
but none of the ones that it had not seen but that had just been installed, and this
could lead to two copies of the type existing with different hashes (since this can
happen during deserialization of type groups).

This change fixes that by rewriting the entire hashing and MRTG framework to have
a disciplined model for how threads interact while they are hashing the same
objects and types, and for how our framework walks the python and Type* graph.

Prior to this change, we had two different ways of walking the graph for hashing
purposes - one for discovery (CompilerVisibleObjectVisitor) and one for hashing
(Type::computeIdentityHash).  Now, we have one way of walking objects, which is
owned by CompilerVisibleObjectVisitor, who invokes a Type's visitCompilerVisibleInternals
method to walk the internals.  This allows for a single unified framework where
we are guaranteed that the same object relations that are used to determine the
graph structure of the MRTG are also used in hashing. We also ensured that we
don't have any Instance objects floating around in this framework - everything
is either a PyObject* or a Type*, encapsulated by instances of TypeOrPyobj.

Finally, we correct a number of serious errors in the serialization/deserialization
layer that were sliding undetected under the prior framework. In particular, we're
careful to preserve the "Forward" structure (since Forward instances can sometimes
make their way into modules), and to be sure that we are consistent about how
we call 'representationFor' and 'nameForObject' when serializing, for both
Type* and PyObject*.  We also introduce SerializationContext.factoryFor to
handle TypeFunction classes, since these, like nameForObject, can be uniquely
identified by a much simpler-to-deserialize set of objects.

Author: braxtonmckee

typed_python/AlternativeMatcherType.hpp

@@ -32,8 +32,10 @@ class AlternativeMatcher : public Type {
         endOfConstructorInitialization(); // finish initializing the type object.
     }
 
-    ShaHash _computeIdentityHash(MutuallyRecursiveTypeGroup* groupHead = nullptr) {
-        return ShaHash(1, m_typeCategory) + m_alternative->identityHash(groupHead);
+    template<class visitor_type>
+    void _visitCompilerVisibleInternals(const visitor_type& v) {
+        v.visitHash(ShaHash(1, m_typeCategory));
+        v.visitTopo(m_alternative);
     }
 
     template<class visitor_type>

typed_python/AlternativeType.cpp

@@ -50,20 +50,6 @@ int64_t Alternative::refcount(instance_ptr i) const {
     return ((layout**)i)[0]->refcount;
 }
 
-ShaHash Alternative::_computeIdentityHash(MutuallyRecursiveTypeGroup* groupHead) {
-    ShaHash newHash(1, m_typeCategory);
-
-    newHash = newHash + ShaHash(m_name) + ShaHash(0);
-    for (auto& subtype_pair: m_subtypes) {
-        newHash = newHash + ShaHash(subtype_pair.first) + subtype_pair.second->identityHash(groupHead);
-    }
-    newHash = newHash + ShaHash(1);
-    for (auto nameAndMethod: m_methods) {
-        newHash = newHash + ShaHash(nameAndMethod.first) + nameAndMethod.second->identityHash(groupHead);
-    }
-    return newHash;
-}
-
 bool Alternative::_updateAfterForwardTypesChanged() {
     m_arg_positions.clear();
     m_default_construction_type = nullptr;

typed_python/AlternativeType.hpp

@@ -157,7 +157,23 @@ class Alternative : public Type {
 
     bool _updateAfterForwardTypesChanged();
 
-    ShaHash _computeIdentityHash(MutuallyRecursiveTypeGroup* groupHead = nullptr);
+    template<class visitor_type>
+    void _visitCompilerVisibleInternals(const visitor_type& v) {
+        v.visitHash(ShaHash(1, m_typeCategory));
+        v.visitName(m_name);
+        v.visitHash(ShaHash(m_subtypes.size()));
+
+        for (auto& subtype_pair: m_subtypes) {
+            v.visitNamedTopo(subtype_pair.first, subtype_pair.second);
+        }
+
+        v.visitHash(ShaHash(m_methods.size()));
+
+        for (auto nameAndMethod: m_methods) {
+            v.visitNamedTopo(nameAndMethod.first, nameAndMethod.second);
+        }
+    }
+
 
     bool cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions);
     static bool cmpStatic(Alternative* altT, instance_ptr left, instance_ptr right, int64_t pyComparisonOp);

typed_python/BoundMethodType.hpp

@@ -32,12 +32,11 @@ class BoundMethod : public Type {
         endOfConstructorInitialization(); // finish initializing the type object.
     }
 
-    ShaHash _computeIdentityHash(MutuallyRecursiveTypeGroup* groupHead = nullptr) {
-        return (
-            ShaHash(1, m_typeCategory) +
-            m_first_arg->identityHash(groupHead) +
-            ShaHash(m_funcName)
-        );
+    template<class visitor_type>
+    void _visitCompilerVisibleInternals(const visitor_type& v) {
+        v.visitHash(ShaHash(1, m_typeCategory));
+        v.visitTopo(m_first_arg);
+        v.visitName(m_funcName);
     }
 
     template<class visitor_type>

typed_python/BytesType.hpp

@@ -143,6 +143,11 @@ class BytesType : public Type {
     template<class visitor_type>
     void _visitContainedTypes(const visitor_type& v) {}
 
+    template<class visitor_type>
+    void _visitCompilerVisibleInternals(const visitor_type& v) {
+        v.visitHash(ShaHash(1, m_typeCategory));
+    }
+
     typed_python_hash_type hash(instance_ptr left);
 
     bool cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions);

typed_python/ClassType.hpp

@@ -187,8 +187,10 @@ class Class : public Type {
 
     bool checkInitializationFlag(instance_ptr self, int64_t ix) const;
 
-    ShaHash _computeIdentityHash(MutuallyRecursiveTypeGroup* groupHead = nullptr) {
-        return ShaHash(1, m_typeCategory) + m_heldClass->identityHash(groupHead);
+    template<class visitor_type>
+    void _visitCompilerVisibleInternals(const visitor_type& v) {
+        v.visitHash(ShaHash(1, m_typeCategory));
+        v.visitTopo(m_heldClass);
     }
 
     bool cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions);