[cxx-interop] Improve performance of deep template instantiations.

If there are more than 10,000 instantiations of the same class template
then we need to bail because it will take to long to mangle,
instantiate, etc.

Author: zoecarver

lib/ClangImporter/ImportDecl.cpp

@@ -3725,6 +3725,14 @@ namespace {
 
     Decl *VisitClassTemplateSpecializationDecl(
                  const clang::ClassTemplateSpecializationDecl *decl) {
+      // Before we go any further, check if we've already got tens of thousands
+      // of specializations. If so, it means we're likely instantiating a very
+      // deep/complex template, or we've run into an infinite loop. In either
+      // case, its not worth the compile time, so bail.
+      // TODO: this could be configurable at some point.
+      if (llvm::size(decl->getSpecializedTemplate()->specializations()) > 10000)
+        return nullptr;
+
       // `Sema::isCompleteType` will try to instantiate the class template as a
       // side-effect and we rely on this here. `decl->getDefinition()` can
       // return nullptr before the call to sema and return its definition

test/Interop/Cxx/templates/Inputs/large-class-templates.h

@@ -4,3 +4,47 @@ struct HasTypeWithSelfAsParam {
 };
 
 using WillBeInfinite = HasTypeWithSelfAsParam<int>;
+
+namespace RegressionTest {
+
+// This is a regression test to check that we don't instantiate exponentially
+// large templates. The below template will instantiate hundreds of thousands
+// (or more) of specializations before hitting the 8-template-param-deep limit.
+template<class E, int>
+struct SliceExpr {
+  using type = typename E::type;
+  E expr;
+  
+  typename E::type* test() { return nullptr; }
+};
+
+template<class T>
+struct Array {
+  using type = T;
+};
+
+template<class E>
+struct ValExpr {
+  using type = typename E::type;
+  E expr;
+  
+  ValExpr<SliceExpr<E, 1>> test1() { return {SliceExpr<E, 1>{expr}}; }
+  ValExpr<SliceExpr<E, 2>> test2() { return {SliceExpr<E, 2>{expr}}; }
+  ValExpr<SliceExpr<E, 3>> test3() { return {SliceExpr<E, 3>{expr}}; }
+  ValExpr<SliceExpr<E, 4>> test4() { return {SliceExpr<E, 4>{expr}}; }
+  ValExpr<SliceExpr<E, 5>> test5() { return {SliceExpr<E, 5>{expr}}; }
+  ValExpr<SliceExpr<E, 6>> test6() { return {SliceExpr<E, 6>{expr}}; }
+  ValExpr<SliceExpr<E, 7>> test7() { return {SliceExpr<E, 7>{expr}}; }
+  ValExpr<SliceExpr<E, 8>> test8() { return {SliceExpr<E, 8>{expr}}; }
+  ValExpr<SliceExpr<E, 9>> test9() { return {SliceExpr<E, 8>{expr}}; }
+  ValExpr<SliceExpr<E, 11>> test11() { return {SliceExpr<E, 11>{expr}}; }
+  ValExpr<SliceExpr<E, 12>> test12() { return {SliceExpr<E, 12>{expr}}; }
+};
+
+// This class template is exponentially slow to *fully* instantiate (and the
+// exponent is the number of testX methods). Make sure that we can still
+// partially import it without importing all of its child template
+// instantiations.
+void user(ValExpr<Array<int>> *) { }
+
+} // end namespace 'RegressionTest'

test/Interop/Cxx/templates/large-class-templates-module-interface.swift

@@ -9,3 +9,9 @@
 // CHECK: }
 
 // CHECK: typealias WillBeInfinite = __CxxTemplateInst22HasTypeWithSelfAsParamIiE
+
+// Make sure we fail to import super slow templates.
+// CHECK-NOT: __CxxTemplateInstN14RegressionTest7ValExprINS_9SliceExprINS_5ArrayIiEELi1EEEEE
+// TODO: we should not be importing functions that use this type in their
+// signature (such as the function below).
+// CHECK: mutating func test1() -> RegressionTest.__CxxTemplateInstN14RegressionTest7ValExprINS_9SliceExprINS_5ArrayIiEELi1EEEEE