PR47372: Fix Lambda invoker calling conventions

As mentioned in the defect, the lambda static invoker does not follow
the calling convention of the lambda itself, which seems wrong. This
patch ensures that the calling convention of operator() is passed onto
the invoker and conversion-operator type.

This is accomplished by extracting the calling-convention determination
code out into a separate function in order to better reflect the 'thiscall'
work, as well as somewhat better support the future implementation of
https://devblogs.microsoft.com/oldnewthing/20150220-00/?p=44623

For any target (basically just win32) that has a different free and
static function calling convention, this generates BOTH alternatives.
This required some work to get the Windows mangler to work correctly for
this, as well as some tie-breaking for the unary operators.

Differential Revision: https://reviews.llvm.org/D89559

Author: Erich Keane

clang/include/clang/AST/DeclCXX.h

@@ -1008,8 +1008,13 @@ class CXXRecordDecl : public RecordDecl {
 
   /// Retrieve the lambda static invoker, the address of which
   /// is returned by the conversion operator, and the body of which
-  /// is forwarded to the lambda call operator.
+  /// is forwarded to the lambda call operator. The version that does not
+  /// take a calling convention uses the 'default' calling convention for free
+  /// functions if the Lambda's calling convention was not modified via
+  /// attribute. Otherwise, it will return the calling convention specified for
+  /// the lambda.
   CXXMethodDecl *getLambdaStaticInvoker() const;
+  CXXMethodDecl *getLambdaStaticInvoker(CallingConv CC) const;
 
   /// Retrieve the generic lambda's template parameter list.
   /// Returns null if the class does not represent a lambda or a generic

clang/include/clang/Sema/Sema.h

@@ -6597,7 +6597,8 @@ class Sema final {
   /// Get the return type to use for a lambda's conversion function(s) to
   /// function pointer type, given the type of the call operator.
   QualType
-  getLambdaConversionFunctionResultType(const FunctionProtoType *CallOpType);
+  getLambdaConversionFunctionResultType(const FunctionProtoType *CallOpType,
+                                        CallingConv CC);
 
   /// Define the "body" of the conversion from a lambda object to a
   /// function pointer.

clang/lib/AST/DeclCXX.cpp

@@ -1507,18 +1507,38 @@ CXXMethodDecl *CXXRecordDecl::getLambdaCallOperator() const {
 }
 
 CXXMethodDecl* CXXRecordDecl::getLambdaStaticInvoker() const {
-  if (!isLambda()) return nullptr;
+  CXXMethodDecl *CallOp = getLambdaCallOperator();
+  CallingConv CC = CallOp->getType()->getAs<FunctionType>()->getCallConv();
+  return getLambdaStaticInvoker(CC);
+}
+
+static DeclContext::lookup_result
+getLambdaStaticInvokers(const CXXRecordDecl &RD) {
+  assert(RD.isLambda() && "Must be a lambda");
   DeclarationName Name =
-    &getASTContext().Idents.get(getLambdaStaticInvokerName());
-  DeclContext::lookup_result Invoker = lookup(Name);
-  if (Invoker.empty()) return nullptr;
-  assert(allLookupResultsAreTheSame(Invoker) &&
-         "More than one static invoker operator!");
-  NamedDecl *InvokerFun = Invoker.front();
-  if (const auto *InvokerTemplate = dyn_cast<FunctionTemplateDecl>(InvokerFun))
+      &RD.getASTContext().Idents.get(getLambdaStaticInvokerName());
+  return RD.lookup(Name);
+}
+
+static CXXMethodDecl *getInvokerAsMethod(NamedDecl *ND) {
+  if (const auto *InvokerTemplate = dyn_cast<FunctionTemplateDecl>(ND))
     return cast<CXXMethodDecl>(InvokerTemplate->getTemplatedDecl());
+  return cast<CXXMethodDecl>(ND);
+}
+
+CXXMethodDecl *CXXRecordDecl::getLambdaStaticInvoker(CallingConv CC) const {
+  if (!isLambda())
+    return nullptr;
+  DeclContext::lookup_result Invoker = getLambdaStaticInvokers(*this);
+
+  for (NamedDecl *ND : Invoker) {
+    const FunctionType *FTy =
+        cast<ValueDecl>(ND->getAsFunction())->getType()->getAs<FunctionType>();
+    if (FTy->getCallConv() == CC)
+      return getInvokerAsMethod(ND);
+  }
 
-  return cast<CXXMethodDecl>(InvokerFun);
+  return nullptr;
 }
 
 void CXXRecordDecl::getCaptureFields(
@@ -2459,14 +2479,8 @@ bool CXXMethodDecl::hasInlineBody() const {
 
 bool CXXMethodDecl::isLambdaStaticInvoker() const {
   const CXXRecordDecl *P = getParent();
-  if (P->isLambda()) {
-    if (const CXXMethodDecl *StaticInvoker = P->getLambdaStaticInvoker()) {
-      if (StaticInvoker == this) return true;
-      if (P->isGenericLambda() && this->isFunctionTemplateSpecialization())
-        return StaticInvoker == this->getPrimaryTemplate()->getTemplatedDecl();
-    }
-  }
-  return false;
+  return P->isLambda() && getDeclName().isIdentifier() &&
+         getName() == getLambdaStaticInvokerName();
 }
 
 CXXCtorInitializer::CXXCtorInitializer(ASTContext &Context,

clang/lib/AST/MicrosoftMangle.cpp

@@ -2260,10 +2260,20 @@ void MicrosoftCXXNameMangler::mangleFunctionType(const FunctionType *T,
       return;
     }
     Out << '@';
+  } else if (IsInLambda && D && isa<CXXConversionDecl>(D)) {
+    // The only lambda conversion operators are to function pointers, which
+    // can differ by their calling convention and are typically deduced.  So
+    // we make sure that this type gets mangled properly.
+    mangleType(T->getReturnType(), Range, QMM_Result);
   } else {
     QualType ResultType = T->getReturnType();
-    if (const auto *AT =
-            dyn_cast_or_null<AutoType>(ResultType->getContainedAutoType())) {
+    if (IsInLambda && isa<CXXConversionDecl>(D)) {
+      // The only lambda conversion operators are to function pointers, which
+      // can differ by their calling convention and are typically deduced.  So
+      // we make sure that this type gets mangled properly.
+      mangleType(ResultType, Range, QMM_Result);
+    } else if (const auto *AT = dyn_cast_or_null<AutoType>(
+                   ResultType->getContainedAutoType())) {
       Out << '?';
       mangleQualifiers(ResultType.getLocalQualifiers(), /*IsMember=*/false);
       Out << '?';

clang/lib/Sema/SemaDeclCXX.cpp

@@ -14811,9 +14811,13 @@ void Sema::DefineImplicitLambdaToFunctionPointerConversion(
   SynthesizedFunctionScope Scope(*this, Conv);
   assert(!Conv->getReturnType()->isUndeducedType());
 
+  QualType ConvRT = Conv->getType()->getAs<FunctionType>()->getReturnType();
+  CallingConv CC =
+      ConvRT->getPointeeType()->getAs<FunctionType>()->getCallConv();
+
   CXXRecordDecl *Lambda = Conv->getParent();
   FunctionDecl *CallOp = Lambda->getLambdaCallOperator();
-  FunctionDecl *Invoker = Lambda->getLambdaStaticInvoker();
+  FunctionDecl *Invoker = Lambda->getLambdaStaticInvoker(CC);
 
   if (auto *TemplateArgs = Conv->getTemplateSpecializationArgs()) {
     CallOp = InstantiateFunctionDeclaration(

clang/lib/Sema/SemaLambda.cpp

@@ -1263,30 +1263,61 @@ void Sema::ActOnLambdaError(SourceLocation StartLoc, Scope *CurScope,
   PopFunctionScopeInfo();
 }
 
+template <typename Func>
+static void repeatForLambdaConversionFunctionCallingConvs(
+    Sema &S, const FunctionProtoType &CallOpProto, Func F) {
+  CallingConv DefaultFree = S.Context.getDefaultCallingConvention(
+      CallOpProto.isVariadic(), /*IsCXXMethod=*/false);
+  CallingConv DefaultMember = S.Context.getDefaultCallingConvention(
+      CallOpProto.isVariadic(), /*IsCXXMethod=*/true);
+  CallingConv CallOpCC = CallOpProto.getCallConv();
+
+  if (CallOpCC == DefaultMember && DefaultMember != DefaultFree) {
+    F(DefaultFree);
+    F(DefaultMember);
+  } else {
+    F(CallOpCC);
+  }
+}
+
+// Returns the 'standard' calling convention to be used for the lambda
+// conversion function, that is, the 'free' function calling convention unless
+// it is overridden by a non-default calling convention attribute.
+static CallingConv
+getLambdaConversionFunctionCallConv(Sema &S,
+                                    const FunctionProtoType *CallOpProto) {
+  CallingConv DefaultFree = S.Context.getDefaultCallingConvention(
+      CallOpProto->isVariadic(), /*IsCXXMethod=*/false);
+  CallingConv DefaultMember = S.Context.getDefaultCallingConvention(
+      CallOpProto->isVariadic(), /*IsCXXMethod=*/true);
+  CallingConv CallOpCC = CallOpProto->getCallConv();
+
+  // If the call-operator hasn't been changed, return both the 'free' and
+  // 'member' function calling convention.
+  if (CallOpCC == DefaultMember && DefaultMember != DefaultFree)
+    return DefaultFree;
+  return CallOpCC;
+}
+
 QualType Sema::getLambdaConversionFunctionResultType(
-    const FunctionProtoType *CallOpProto) {
-  // The function type inside the pointer type is the same as the call
-  // operator with some tweaks. The calling convention is the default free
-  // function convention, and the type qualifications are lost.
+    const FunctionProtoType *CallOpProto, CallingConv CC) {
   const FunctionProtoType::ExtProtoInfo CallOpExtInfo =
       CallOpProto->getExtProtoInfo();
   FunctionProtoType::ExtProtoInfo InvokerExtInfo = CallOpExtInfo;
-  CallingConv CC = Context.getDefaultCallingConvention(
-      CallOpProto->isVariadic(), /*IsCXXMethod=*/false);
   InvokerExtInfo.ExtInfo = InvokerExtInfo.ExtInfo.withCallingConv(CC);
   InvokerExtInfo.TypeQuals = Qualifiers();
   assert(InvokerExtInfo.RefQualifier == RQ_None &&
-      "Lambda's call operator should not have a reference qualifier");
+         "Lambda's call operator should not have a reference qualifier");
   return Context.getFunctionType(CallOpProto->getReturnType(),
                                  CallOpProto->getParamTypes(), InvokerExtInfo);
 }
 
 /// Add a lambda's conversion to function pointer, as described in
 /// C++11 [expr.prim.lambda]p6.
-static void addFunctionPointerConversion(Sema &S,
-                                         SourceRange IntroducerRange,
+static void addFunctionPointerConversion(Sema &S, SourceRange IntroducerRange,
                                          CXXRecordDecl *Class,
-                                         CXXMethodDecl *CallOperator) {
+                                         CXXMethodDecl *CallOperator,
+                                         QualType InvokerFunctionTy) {
   // This conversion is explicitly disabled if the lambda's function has
   // pass_object_size attributes on any of its parameters.
   auto HasPassObjectSizeAttr = [](const ParmVarDecl *P) {
@@ -1296,8 +1327,6 @@ static void addFunctionPointerConversion(Sema &S,
     return;
 
   // Add the conversion to function pointer.
-  QualType InvokerFunctionTy = S.getLambdaConversionFunctionResultType(
-      CallOperator->getType()->castAs<FunctionProtoType>());
   QualType PtrToFunctionTy = S.Context.getPointerType(InvokerFunctionTy);
 
   // Create the type of the conversion function.
@@ -1442,13 +1471,37 @@ static void addFunctionPointerConversion(Sema &S,
     Class->addDecl(Invoke);
 }
 
+/// Add a lambda's conversion to function pointers, as described in
+/// C++11 [expr.prim.lambda]p6. Note that in most cases, this should emit only a
+/// single pointer conversion. In the event that the default calling convention
+/// for free and member functions is different, it will emit both conventions.
+/// FIXME: Implement emitting a version of the operator for EVERY calling
+/// convention for MSVC, as described here:
+/// https://devblogs.microsoft.com/oldnewthing/20150220-00/?p=44623.
+static void addFunctionPointerConversions(Sema &S, SourceRange IntroducerRange,
+                                          CXXRecordDecl *Class,
+                                          CXXMethodDecl *CallOperator) {
+  const FunctionProtoType *CallOpProto =
+      CallOperator->getType()->castAs<FunctionProtoType>();
+
+  repeatForLambdaConversionFunctionCallingConvs(
+      S, *CallOpProto, [&](CallingConv CC) {
+        QualType InvokerFunctionTy =
+            S.getLambdaConversionFunctionResultType(CallOpProto, CC);
+        addFunctionPointerConversion(S, IntroducerRange, Class, CallOperator,
+                                     InvokerFunctionTy);
+      });
+}
+
 /// Add a lambda's conversion to block pointer.
 static void addBlockPointerConversion(Sema &S,
                                       SourceRange IntroducerRange,
                                       CXXRecordDecl *Class,
                                       CXXMethodDecl *CallOperator) {
+  const FunctionProtoType *CallOpProto =
+      CallOperator->getType()->castAs<FunctionProtoType>();
   QualType FunctionTy = S.getLambdaConversionFunctionResultType(
-      CallOperator->getType()->castAs<FunctionProtoType>());
+      CallOpProto, getLambdaConversionFunctionCallConv(S, CallOpProto));
   QualType BlockPtrTy = S.Context.getBlockPointerType(FunctionTy);
 
   FunctionProtoType::ExtProtoInfo ConversionEPI(
@@ -1795,8 +1848,8 @@ ExprResult Sema::BuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc,
     //   to pointer to function having the same parameter and return
     //   types as the closure type's function call operator.
     if (Captures.empty() && CaptureDefault == LCD_None)
-      addFunctionPointerConversion(*this, IntroducerRange, Class,
-                                   CallOperator);
+      addFunctionPointerConversions(*this, IntroducerRange, Class,
+                                    CallOperator);
 
     // Objective-C++:
     //   The closure type for a lambda-expression has a public non-virtual

clang/lib/Sema/SemaOverload.cpp

@@ -3645,13 +3645,32 @@ Sema::DiagnoseMultipleUserDefinedConversion(Expr *From, QualType ToType) {
   return true;
 }
 
+// Helper for compareConversionFunctions that gets the FunctionType that the
+// conversion-operator return  value 'points' to, or nullptr.
+static const FunctionType *
+getConversionOpReturnTyAsFunction(CXXConversionDecl *Conv) {
+  const FunctionType *ConvFuncTy = Conv->getType()->castAs<FunctionType>();
+  const PointerType *RetPtrTy =
+      ConvFuncTy->getReturnType()->getAs<PointerType>();
+
+  if (!RetPtrTy)
+    return nullptr;
+
+  return RetPtrTy->getPointeeType()->getAs<FunctionType>();
+}
+
 /// Compare the user-defined conversion functions or constructors
 /// of two user-defined conversion sequences to determine whether any ordering
 /// is possible.
 static ImplicitConversionSequence::CompareKind
 compareConversionFunctions(Sema &S, FunctionDecl *Function1,
                            FunctionDecl *Function2) {
-  if (!S.getLangOpts().ObjC || !S.getLangOpts().CPlusPlus11)
+  CXXConversionDecl *Conv1 = dyn_cast_or_null<CXXConversionDecl>(Function1);
+  CXXConversionDecl *Conv2 = dyn_cast_or_null<CXXConversionDecl>(Function2);
+  if (!Conv1 || !Conv2)
+    return ImplicitConversionSequence::Indistinguishable;
+
+  if (!Conv1->getParent()->isLambda() || !Conv2->getParent()->isLambda())
     return ImplicitConversionSequence::Indistinguishable;
 
   // Objective-C++:
@@ -3660,22 +3679,47 @@ compareConversionFunctions(Sema &S, FunctionDecl *Function1,
   //   respectively, always prefer the conversion to a function pointer,
   //   because the function pointer is more lightweight and is more likely
   //   to keep code working.
-  CXXConversionDecl *Conv1 = dyn_cast_or_null<CXXConversionDecl>(Function1);
-  if (!Conv1)
-    return ImplicitConversionSequence::Indistinguishable;
-
-  CXXConversionDecl *Conv2 = dyn_cast<CXXConversionDecl>(Function2);
-  if (!Conv2)
-    return ImplicitConversionSequence::Indistinguishable;
-
-  if (Conv1->getParent()->isLambda() && Conv2->getParent()->isLambda()) {
+  if (S.getLangOpts().ObjC && S.getLangOpts().CPlusPlus11) {
     bool Block1 = Conv1->getConversionType()->isBlockPointerType();
     bool Block2 = Conv2->getConversionType()->isBlockPointerType();
     if (Block1 != Block2)
       return Block1 ? ImplicitConversionSequence::Worse
                     : ImplicitConversionSequence::Better;
   }
 
+  // In order to support multiple calling conventions for the lambda conversion
+  // operator (such as when the free and member function calling convention is
+  // different), prefer the 'free' mechanism, followed by the calling-convention
+  // of operator(). The latter is in place to support the MSVC-like solution of
+  // defining ALL of the possible conversions in regards to calling-convention.
+  const FunctionType *Conv1FuncRet = getConversionOpReturnTyAsFunction(Conv1);
+  const FunctionType *Conv2FuncRet = getConversionOpReturnTyAsFunction(Conv2);
+
+  if (Conv1FuncRet && Conv2FuncRet &&
+      Conv1FuncRet->getCallConv() != Conv2FuncRet->getCallConv()) {
+    CallingConv Conv1CC = Conv1FuncRet->getCallConv();
+    CallingConv Conv2CC = Conv2FuncRet->getCallConv();
+
+    CXXMethodDecl *CallOp = Conv2->getParent()->getLambdaCallOperator();
+    const FunctionProtoType *CallOpProto =
+        CallOp->getType()->getAs<FunctionProtoType>();
+
+    CallingConv CallOpCC =
+        CallOp->getType()->getAs<FunctionType>()->getCallConv();
+    CallingConv DefaultFree = S.Context.getDefaultCallingConvention(
+        CallOpProto->isVariadic(), /*IsCXXMethod=*/false);
+    CallingConv DefaultMember = S.Context.getDefaultCallingConvention(
+        CallOpProto->isVariadic(), /*IsCXXMethod=*/true);
+
+    CallingConv PrefOrder[] = {DefaultFree, DefaultMember, CallOpCC};
+    for (CallingConv CC : PrefOrder) {
+      if (Conv1CC == CC)
+        return ImplicitConversionSequence::Better;
+      if (Conv2CC == CC)
+        return ImplicitConversionSequence::Worse;
+    }
+  }
+
   return ImplicitConversionSequence::Indistinguishable;
 }
 
@@ -10180,6 +10224,22 @@ void Sema::NoteOverloadCandidate(NamedDecl *Found, FunctionDecl *Fn,
   if (Fn->isMultiVersion() && Fn->hasAttr<TargetAttr>() &&
       !Fn->getAttr<TargetAttr>()->isDefaultVersion())
     return;
+  if (isa<CXXConversionDecl>(Fn) &&
+      cast<CXXRecordDecl>(Fn->getParent())->isLambda()) {
+    // Don't print candidates other than the one that matches the calling
+    // convention of the call operator, since that is guaranteed to exist.
+    const auto *RD = cast<CXXRecordDecl>(Fn->getParent());
+    CXXMethodDecl *CallOp = RD->getLambdaCallOperator();
+    CallingConv CallOpCC =
+        CallOp->getType()->getAs<FunctionType>()->getCallConv();
+    CXXConversionDecl *ConvD = cast<CXXConversionDecl>(Fn);
+    QualType ConvRTy = ConvD->getType()->getAs<FunctionType>()->getReturnType();
+    CallingConv ConvToCC =
+        ConvRTy->getPointeeType()->getAs<FunctionType>()->getCallConv();
+
+    if (ConvToCC != CallOpCC)
+      return;
+  }
 
   std::string FnDesc;
   std::pair<OverloadCandidateKind, OverloadCandidateSelect> KSPair =

clang/lib/Sema/SemaTemplateDeduction.cpp

@@ -5022,8 +5022,12 @@ bool Sema::DeduceReturnType(FunctionDecl *FD, SourceLocation Loc,
            "failed to deduce lambda return type");
 
     // Build the new return type from scratch.
+    CallingConv RetTyCC = FD->getReturnType()
+                              ->getPointeeType()
+                              ->castAs<FunctionType>()
+                              ->getCallConv();
     QualType RetType = getLambdaConversionFunctionResultType(
-        CallOp->getType()->castAs<FunctionProtoType>());
+        CallOp->getType()->castAs<FunctionProtoType>(), RetTyCC);
     if (FD->getReturnType()->getAs<PointerType>())
       RetType = Context.getPointerType(RetType);
     else {