Fix canSerialize logic

Author: elsh

lib/SILOptimizer/IPO/CrossModuleOptimization.cpp

@@ -203,7 +203,8 @@ class InstructionVisitor : public SILCloner<InstructionVisitor> {
     for (Type replType : Subs.getReplacementTypes()) {
       switch (mode) {
         case VisitMode::DetectSerializableInst:
-          CMS.canSerializeType(replType->getCanonicalType());
+          if (!CMS.canSerializeType(replType->getCanonicalType()))
+            isInstSerializable = false;
           break;
         case VisitMode::SerializeInst:
           /// Ensure that all replacement types of \p Subs are usable from serialized
@@ -667,16 +668,17 @@ bool CrossModuleOptimization::canSerializeType(CanType type) {
   if (iter != canTypesChecked.end())
     return iter->getSecond();
 
-  bool success = type.findIf(
+  bool success = !type.findIf(
      [this](Type rawSubType) {
        CanType subType = rawSubType->getCanonicalType();
        if (auto nominal = subType->getNominalOrBoundGenericNominal()) {
-         return canSerializeDecl(nominal);
+         return !canSerializeDecl(nominal);
        }
-       // If reached here, the type is a Builtin type or similar,
-       // e.g. Builtin.Int64, Builtin.Word, Builtin.NativeObject, etc.
-       return true;
-     });
+       // Types that might not have nominal include Builtin types (e.g. Builtin.Int64),
+       // generic parameter types (e.g. T as in Foo<T>), SIL function result types
+       // (e.g. @convention(method) (Int, @thin Hasher.Type) -> Hasher), etc.
+       return false;
+  });
 
   canTypesChecked[type] = success;
   return success;

test/SILOptimizer/default-cmo-with-opaque-types.swift

@@ -0,0 +1,65 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: %target-swift-frontend -emit-module -wmo -enable-default-cmo -parse-as-library %t/Lib.swift -emit-module-path=%t/Lib.swiftmodule -module-name=Lib -enable-testing -I%t
+
+/// TEST: Deserializing Lib module should not cause an assert fail.
+// RUN: %target-swift-frontend -emit-sil -sil-verify-all %t/Client.swift -I%t
+
+// REQUIRES: swift_in_compiler
+// REQUIRES: asserts
+
+//--- Client.swift
+
+@testable import Lib
+
+
+//--- Lib.swift
+
+public class PubKlass {
+  var field = InternalStruct()
+  /// This function should not be serialized as it returns an opaque type that isn't cast to a concrete nominal type.
+  func run(from arg: PubStruct) -> some Sequence<InternalStruct.Klass> {
+    field.run(from: arg)
+  }
+}
+
+public struct PubStruct: Hashable { }
+
+struct InternalStruct: Hashable {
+  class Klass: Hashable {
+    let data: PubStruct
+    init(_ arg: PubStruct) {
+      self.data = arg
+    }
+    func hash(into hasher: inout Hasher) {
+        hasher.combine(data)
+    }
+    static func == (lhs: Klass, rhs: Klass) -> Bool {
+        return lhs.data == rhs.data
+    }
+  }
+
+  var entities: [PubStruct: Klass] = [:]
+
+  func run(from arg: PubStruct) -> some Sequence<Klass> {
+    IteratorSequence(EntityIterator(from: arg))
+      .lazy.map { entities[$0]! }
+  }
+}
+
+
+private struct EntityIterator<T: Hashable>: IteratorProtocol {
+  private var list: [T]
+  init(from start: T) {
+    self.list = [start]
+  }
+  private mutating func pop() -> T? {
+    return nil
+  }
+  mutating func next() -> T? {
+    return nil
+  }
+}
+
+