Ensure that we don't serialize module names on FunctionType when we're suppressing line info.

Author: Braxton Mckee

typed_python/NullSerializationContext.hpp

@@ -40,4 +40,7 @@ class NullSerializationContext : public SerializationContext {
     virtual bool isCompressionEnabled() const {
         return false;
     }
+    virtual bool isLineInfoSuppressed() const {
+        return false;
+    }
 };

typed_python/PythonSerializationContext.cpp

@@ -46,6 +46,14 @@ void PythonSerializationContext::setFlags() {
     }
 
     mSerializeHashSequence = ((PyObject*)serializeHashSequence) == Py_True;
+
+    PyObjectStealer encodeLineInformationForCode(PyObject_GetAttrString(mContextObj, "encodeLineInformationForCode"));
+
+    if (!encodeLineInformationForCode) {
+        throw PythonExceptionSet();
+    }
+
+    mSuppressLineInfo = ((PyObject*)encodeLineInformationForCode) == Py_False;
 }
 
 std::string PythonSerializationContext::getNameForPyObj(PyObject* o) const {

typed_python/PythonSerializationContext.hpp

@@ -93,6 +93,10 @@ class PythonSerializationContext : public SerializationContext {
     bool isCompressionEnabled() const {
         return mCompressionEnabled;
     }
+    
+    bool isLineInfoSuppressed() const {
+        return mSuppressLineInfo;
+    }
 
     // should we serialize an integer in the order of the
     // hash sequence rather than the hash itself?
@@ -185,6 +189,8 @@ class PythonSerializationContext : public SerializationContext {
 
     bool mCompressionEnabled;
 
+    bool mSuppressLineInfo;
+
     bool mInternalizeTypeGroups;
 
     bool mSerializeHashSequence;

typed_python/PythonSerializationContext_serialization.cpp

@@ -824,7 +824,14 @@ void PythonSerializationContext::serializeNativeTypeInner(
         serializeNativeType(ftype->getClosureType(), b, 1);
         b.writeStringObject(2, ftype->name());
         b.writeStringObject(3, ftype->qualname());
-        b.writeStringObject(4, ftype->moduleName());
+        b.writeStringObject(
+            4, 
+            // if we're suppressing line info then we also don't want to write
+            // module names. Otherwise, it's impossible to get sha hashes of 
+            // code that gets relocated across file boundaries.
+            b.getContext().isLineInfoSuppressed() ? std::string("") : ftype->moduleName()
+        );
+
         b.writeUnsignedVarintObject(5, ftype->isEntrypoint() ? 1 : 0);
         b.writeUnsignedVarintObject(6, ftype->isNocompile() ? 1 : 0);
 

typed_python/SerializationContext.hpp

@@ -35,4 +35,5 @@ class SerializationContext {
     virtual Type* deserializeNativeType(DeserializationBuffer& b, size_t wireType) const = 0;
 
     virtual bool isCompressionEnabled() const = 0;
+    virtual bool isLineInfoSuppressed() const = 0;
 };

typed_python/types_serialization_test.py

@@ -45,12 +45,12 @@
     Dict, Set, SerializationContext, EmbeddedMessage,
     serializeStream, deserializeStream, decodeSerializedObject,
     Forward, Final, Function, Entrypoint, TypeFunction, PointerTo,
-    SubclassOf
+    SubclassOf, NotCompiled
 )
 
 from typed_python._types import (
     refcount, isRecursive, identityHash, buildPyFunctionObject,
-    setFunctionClosure, typesAreEquivalent, recursiveTypeGroupDeepRepr
+    setFunctionClosure, typesAreEquivalent, recursiveTypeGroupDeepRepr,
 )
 
 module_level_testfun = dummy_test_module.testfunction
@@ -2735,6 +2735,61 @@ def test_serialization_independent_of_whether_function_is_hashed(self):
 
         assert s1 == s2
 
+    def test_serialization_has_no_filename_reference(self):
+        def makeF(modulename):
+            with tempfile.TemporaryDirectory() as tempdir:
+                path = os.path.join(tempdir, modulename + ".py")
+
+                CONTENTS = (
+                    "def f(x):\n"
+                    "    return x\n"
+                )
+
+                with open(path, "w") as f:
+                    f.write(CONTENTS)
+
+                globals = {'__file__': path}
+
+                exec(
+                    compile(CONTENTS, path, "exec"),
+                    globals
+                )
+
+                s = SerializationContext()
+                return s.serialize(globals['f'])
+
+        f1 = SerializationContext().deserialize(callFunctionInFreshProcess(makeF, ('asdf',)))
+        f2 = SerializationContext().deserialize(callFunctionInFreshProcess(makeF, ('asdf2',)))
+
+        def checkSame(f1, f2):
+            s = SerializationContext().withoutCompression()
+            s2 = SerializationContext().withoutCompression().withoutLineInfoEncoded().withSerializeHashSequence()
+
+            assert s.serialize(f1) != s.serialize(f2)
+
+            if s2.serialize(f1) != s2.serialize(f2):
+                decoded1 = decodeSerializedObject(s2.serialize(f1))
+                decoded2 = decodeSerializedObject(s2.serialize(f2))
+
+                decoded1Print = pprint.PrettyPrinter(indent=2).pformat(decoded1).split("\n")
+                decoded2Print = pprint.PrettyPrinter(indent=2).pformat(decoded2).split("\n")
+
+                for i in range(len(decoded1Print)):
+                    if decoded1Print[i] != decoded2Print[i]:
+                        for j in range(max(0, i-5), i):
+                            print(decoded1Print[j])
+                        print("******************************** DIFFERENCE *******************")
+                        print(decoded1Print[i])
+                        print(decoded2Print[i])
+                        print("***************************************************************")
+                        break
+
+            assert s2.serialize(f1) == s2.serialize(f2)
+
+        checkSame(f1, f2)
+        checkSame(Entrypoint(f1), Entrypoint(f2))
+        checkSame(NotCompiled(f1), NotCompiled(f2))
+
     def test_serialize_anonymous_class_with_defaults_and_nonempty(self):
         class C1(Class):
             x1 = Member(int, default_value=10, nonempty=True)
@@ -2821,3 +2876,43 @@ def test_serialization_context_names_for_pmap_functions(self):
         from typed_python.lib.pmap import ensureThreads
         sc = SerializationContext()
         assert sc.nameForObject(type(ensureThreads)) is not None
+
+    def test_pmap_of_notcompiled_serialized_externally(self):
+        def makeC():
+            with tempfile.TemporaryDirectory() as tempdir:
+                path = os.path.join(tempdir, "asdf.py")
+
+                CONTENTS = (
+                    "from typed_python import NotCompiled\n"
+                    "@NotCompiled\n"
+                    "def f(x) -> str:\n"
+                    "    return str(x)\n"
+                )
+
+                with open(path, "w") as f:
+                    f.write(CONTENTS)
+
+                globals = {'__file__': path}
+
+                exec(
+                    compile(CONTENTS, path, "exec"),
+                    globals
+                )
+
+                s = SerializationContext()
+                return s.serialize(globals['f'])
+
+        serializedF = callFunctionInFreshProcess(makeC, ())
+
+        f = SerializationContext().deserialize(serializedF)
+
+        assert f(2) == "2"
+
+        from typed_python.lib.pmap import pmap
+
+        args = ListOf(int)(range(1000000))
+
+        t0 = time.time()
+        while time.time() - t0 < 10.0:
+            print("DO!")
+            pmap(args, f, str, minGranularity=10000)