[SILOpt] Reject existentials in AnyObject specialization application (#63477)

Also allow this feature to be enabled in non-assert compilers

Author: drexin

include/swift/Basic/Features.def

@@ -111,7 +111,7 @@ EXPERIMENTAL_FEATURE(MoveOnlyClasses, true)
 
 EXPERIMENTAL_FEATURE(OneWayClosureParameters, false)
 EXPERIMENTAL_FEATURE(TypeWitnessSystemInference, false)
-EXPERIMENTAL_FEATURE(LayoutPrespecialization, false)
+EXPERIMENTAL_FEATURE(LayoutPrespecialization, true)
 EXPERIMENTAL_FEATURE(ModuleInterfaceExportAs, true)
 
 /// Whether to enable experimental differentiable programming features:

lib/SILOptimizer/Utils/Generics.cpp

@@ -2884,8 +2884,9 @@ bool usePrespecialized(
 
         if (!erased || !layout || !layout->isClass()) {
           newSubs.push_back(entry.value());
-        } else if (!entry.value()->isAnyClassReferenceType()) {
-          // non-reference type can't be applied
+        } else if (!entry.value()->isAnyClassReferenceType() ||
+                   entry.value()->isAnyExistentialType()) {
+          // non-reference or existential type can't be applied
           break;
         } else if (!specializedSig->getRequiredProtocols(genericParam)
                         .empty()) {

test/SILOptimizer/pre_specialize_layouts.swift

@@ -16,8 +16,6 @@
 // RUN: %target-swift-frontend -enable-experimental-feature LayoutPrespecialization -O -swift-version 5 -enable-library-evolution -emit-module -o /dev/null -emit-module-interface-path %t/pre_specialized_module_layouts.swiftinterface %S/Inputs/pre_specialized_module_layouts.swift -module-name pre_specialized_module_layouts
 // RUN: %target-swift-frontend -enable-experimental-feature LayoutPrespecialization -I %t -O -Xllvm -sil-disable-pass=function-signature-opts -emit-sil %s | %FileCheck %s --check-prefix=OPT
 
-// REQUIRES: asserts
-
 import pre_specialized_module_layouts
 
 // Helper to prevent return values from getting optimized away
@@ -187,6 +185,17 @@ public func usePartialApply(y: SomeClass) -> (SomeClass) -> SomeClass {
   return usePartialApplyInner
 }
 
+// NOTE: AnyObject specializations MUST not be applied to existential references.
+
+// OPT: sil @$s22pre_specialize_layouts48useLayoutPrespecializedEntryPointWithExistentialyyAA21SomeReferenceProtocol_pF : $@convention(thin) (@guaranteed any SomeReferenceProtocol) -> () {
+// OPT:   {{bb.*}}([[A1:%.*]] : $any SomeReferenceProtocol):
+// OPT-NOT: {{%.*}} = function_ref @$s30pre_specialized_module_layouts20publicPrespecializedyyxlFyXl_Ts5 : $@convention(thin) (@guaranteed AnyObject) -> ()
+// OPT: } // end sil function '$s22pre_specialize_layouts48useLayoutPrespecializedEntryPointWithExistentialyyAA21SomeReferenceProtocol_pF'
+public protocol SomeReferenceProtocol: AnyObject {}
+public func useLayoutPrespecializedEntryPointWithExistential(_ p: any SomeReferenceProtocol) {
+  publicPrespecialized(p)
+}
+
 // OPT-macosx: sil [available 10.50] @$s22pre_specialize_layouts40usePrespecializedEntryPointsAvailabilityyyF : $@convention(thin) () -> () {
 // OPT-macosx:  [[F1:%.*]] = function_ref @$s30pre_specialized_module_layouts20publicPrespecializedyyxlFAA8SomeDataV_Ts5 : $@convention(thin) (SomeData) -> ()
 // OPT-macosx:  apply [[F1]](