[cxx-interop] Lazily import members of Clang namespaces and records via requests.

Also adds a ClangImporter request zone and move some requests into it.

Author: zoecarver

include/swift/ClangImporter/ClangImporterRequests.h

@@ -34,18 +34,21 @@ class EnumDecl;
 /// The input type for a clang direct lookup request.
 struct ClangDirectLookupDescriptor final {
   Decl *decl;
+  const clang::Decl *clangDecl;
   DeclName name;
 
-  ClangDirectLookupDescriptor(Decl *decl, DeclName name)
-      : decl(decl), name(name) {}
+  ClangDirectLookupDescriptor(Decl *decl, const clang::Decl *clangDecl,
+                              DeclName name)
+      : decl(decl), clangDecl(clangDecl), name(name) {}
 
   friend llvm::hash_code hash_value(const ClangDirectLookupDescriptor &desc) {
-    return llvm::hash_combine(desc.name, desc.decl);
+    return llvm::hash_combine(desc.name, desc.decl, desc.clangDecl);
   }
 
   friend bool operator==(const ClangDirectLookupDescriptor &lhs,
                          const ClangDirectLookupDescriptor &rhs) {
-    return lhs.name == rhs.name && lhs.decl == rhs.decl;
+    return lhs.name == rhs.name && lhs.decl == rhs.decl &&
+           lhs.clangDecl == rhs.clangDecl;
   }
 
   friend bool operator!=(const ClangDirectLookupDescriptor &lhs,

lib/AST/ASTPrinter.cpp

@@ -30,6 +30,7 @@
 #include "swift/AST/GenericSignature.h"
 #include "swift/AST/Module.h"
 #include "swift/AST/NameLookup.h"
+#include "swift/AST/NameLookupRequests.h"
 #include "swift/AST/ParameterList.h"
 #include "swift/AST/PrintOptions.h"
 #include "swift/AST/ProtocolConformance.h"
@@ -44,6 +45,7 @@
 #include "swift/Basic/QuotedString.h"
 #include "swift/Basic/STLExtras.h"
 #include "swift/Basic/StringExtras.h"
+#include "swift/ClangImporter/ClangImporterRequests.h"
 #include "swift/Config.h"
 #include "swift/Parse/Lexer.h"
 #include "swift/Strings.h"
@@ -2157,10 +2159,54 @@ void PrintAST::printAccessors(const AbstractStorageDecl *ASD) {
   indent();
 }
 
+// This provides logic for looking up all members of a namespace. This is
+// intentionally implemented only in the printer and should *only* be used for
+// debugging, testing, generating module dumps, etc. (In other words, if you're
+// trying to get all the members of a namespace in another part of the compiler,
+// you're probably doing somethign wrong. This is a very expensive operation,
+// so we want to do it only when absolutely nessisary.)
+static void addNamespaceMembers(Decl *decl,
+                                llvm::SmallVector<Decl *, 16> &members) {
+  auto &ctx = decl->getASTContext();
+  auto namespaceDecl = cast<clang::NamespaceDecl>(decl->getClangDecl());
+
+  // This is only to keep track of the members we've already seen.
+  llvm::SmallPtrSet<Decl *, 16> addedMembers;
+  for (auto redecl : namespaceDecl->redecls()) {
+    for (auto member : redecl->decls()) {
+      if (auto classTemplate = dyn_cast<clang::ClassTemplateDecl>(member)) {
+        // Hack in the class template specializations that live here.
+        for (auto spec : classTemplate->specializations()) {
+          if (auto import =
+                  ctx.getClangModuleLoader()->importDeclDirectly(spec))
+            if (addedMembers.insert(import).second)
+              members.push_back(import);
+        }
+      }
+
+      auto namedDecl = dyn_cast<clang::NamedDecl>(member);
+      if (!namedDecl)
+        continue;
+
+      auto name = ctx.getClangModuleLoader()->importName(namedDecl);
+      if (!name)
+        continue;
+
+      // If we're building libSyntaxParser, #if out the clang importer request
+      // because libSyntaxParser doesn't know about the clang importer.
+      CXXNamespaceMemberLookup lookupRequest({cast<EnumDecl>(decl), name});
+      for (auto found : evaluateOrDefault(ctx.evaluator, lookupRequest, {})) {
+        if (addedMembers.insert(found).second)
+          members.push_back(found);
+      }
+    }
+  }
+}
+
 void PrintAST::printMembersOfDecl(Decl *D, bool needComma,
                                   bool openBracket,
                                   bool closeBracket) {
-  llvm::SmallVector<Decl *, 3> Members;
+  llvm::SmallVector<Decl *, 16> Members;
   auto AddMembers = [&](IterableDeclContext *idc) {
     if (Options.PrintCurrentMembersOnly) {
       for (auto RD : idc->getMembers())
@@ -2187,6 +2233,8 @@ void PrintAST::printMembersOfDecl(Decl *D, bool needComma,
         }
       }
     }
+    if (isa_and_nonnull<clang::NamespaceDecl>(D->getClangDecl()))
+      addNamespaceMembers(D, Members);
   }
   printMembers(Members, needComma, openBracket, closeBracket);
 }

lib/AST/NameLookup.cpp

@@ -34,8 +34,10 @@
 #include "swift/Basic/STLExtras.h"
 #include "swift/Basic/SourceManager.h"
 #include "swift/Basic/Statistic.h"
+#include "swift/ClangImporter/ClangImporterRequests.h"
 #include "swift/Parse/Lexer.h"
 #include "clang/AST/DeclObjC.h"
+#include "clang/Basic/Specifiers.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/Support/Debug.h"
@@ -1448,15 +1450,34 @@ DirectLookupRequest::evaluate(Evaluator &evaluator,
 
     Table.updateLookupTable(decl);
   } else if (!Table.isLazilyComplete(name.getBaseName())) {
-    // The lookup table believes it doesn't have a complete accounting of this
-    // name - either because we're never seen it before, or another extension
-    // was registered since the last time we searched. Ask the loaders to give
-    // us a hand.
     DeclBaseName baseName(name.getBaseName());
-    populateLookupTableEntryFromLazyIDCLoader(ctx, Table, baseName, decl);
-    populateLookupTableEntryFromExtensions(ctx, Table, baseName, decl);
 
-    Table.markLazilyComplete(baseName);
+    if (isa_and_nonnull<clang::NamespaceDecl>(decl->getClangDecl())) {
+      // Namespaces will never have any members so we can just return whatever
+      // the lookup finds.
+      return evaluateOrDefault(
+          ctx.evaluator, CXXNamespaceMemberLookup({cast<EnumDecl>(decl), name}),
+          {});
+    } else if (isa_and_nonnull<clang::RecordDecl>(decl->getClangDecl())) {
+      auto allFound = evaluateOrDefault(
+          ctx.evaluator,
+          ClangRecordMemberLookup({cast<StructDecl>(decl), name}), {});
+      // Add all the members we found, later we'll combine these with the
+      // existing members.
+      for (auto found : allFound)
+        Table.addMember(found);
+
+      populateLookupTableEntryFromExtensions(ctx, Table, baseName, decl);
+    } else {
+      // The lookup table believes it doesn't have a complete accounting of this
+      // name - either because we're never seen it before, or another extension
+      // was registered since the last time we searched. Ask the loaders to give
+      // us a hand.
+      populateLookupTableEntryFromLazyIDCLoader(ctx, Table, baseName, decl);
+      populateLookupTableEntryFromExtensions(ctx, Table, baseName, decl);
+    }
+
+    Table.markLazilyComplete(name.getBaseName());
   }
 
   // Look for a declaration with this name.

lib/ClangImporter/ClangImporter.cpp

@@ -68,8 +68,8 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Memory.h"
 #include "llvm/Support/Path.h"
-#include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/YAMLParser.h"
+#include "llvm/Support/YAMLTraits.h"
 #include <algorithm>
 #include <memory>
 
@@ -3863,11 +3863,6 @@ void ClangImporter::Implementation::lookupValue(
           importDeclReal(clangDecl->getMostRecentDecl(), CurrentVersion,
                          /*useCanonicalDecl*/ !isNamespace);
 
-      if (isNamespace) {
-        if (auto extension = cast_or_null<ExtensionDecl>(realDecl))
-          realDecl = extension->getExtendedNominal();
-      }
-
       if (!realDecl)
         continue;
       decl = cast<ValueDecl>(realDecl);
@@ -4049,7 +4044,7 @@ SmallVector<SwiftLookupTable::SingleEntry, 4>
 ClangDirectLookupRequest::evaluate(Evaluator &evaluator,
                                    ClangDirectLookupDescriptor desc) const {
   auto &ctx = desc.decl->getASTContext();
-  auto *clangDecl = desc.decl->getClangDecl();
+  auto *clangDecl = desc.clangDecl;
   // Class templates aren't in the lookup table.
   if (auto spec = dyn_cast<clang::ClassTemplateSpecializationDecl>(clangDecl))
     return lookupInClassTemplateSpecialization(ctx, spec, desc.name);
@@ -4065,49 +4060,20 @@ ClangDirectLookupRequest::evaluate(Evaluator &evaluator,
 
   auto foundDecls = lookupTable->lookup(
       SerializedSwiftName(desc.name.getBaseName()), effectiveClangContext);
-  // TODO: in `SwiftLookupTable::lookup` the `searchContext` is only a string.
-  // This is problematic because we might have two nested search contexts with
-  // the same name. The search context needs to be updated to be a decl (or
-  // decl context), but that will be a larger, seperate refactor. When that
-  // happens we can remove the below logic.
-  //
-  // We filter here by ensuring that each decl and all of its parents have the
-  // same name as `clangDecl` and its parents. This is prefered to checking the
-  // decl context pointer is equal to `clangDecl` because we may have a forward
-  // declared struct, multiple redecls of a namespace, or transparent contexts.
-  auto deepCompareDeclContexts = [](auto first, auto second) {
-    while (first->getParent() && second->getParent()) {
-      first = first->getParent();
-      second = second->getParent();
-
-      // Look through transparent contexts. If we have one (or more) extern C++
-      // contexts, that is the same as none.
-      while (first->isTransparentContext())
-        first = first->getParent();
-      while (second->isTransparentContext())
-        second = second->getParent();
-
-      auto firstName = dyn_cast<clang::NamedDecl>(first);
-      auto secondName = dyn_cast<clang::NamedDecl>(second);
-
-      if (!firstName || !secondName) {
-        if (firstName != secondName)
-          return false;
-        continue;
-      }
-
-      if (firstName->getName() != secondName->getName())
-        return false;
-    }
-    return !first->getParent() && !second->getParent();
-  };
+  // Make sure that `clangDecl` is the parent of all the members we found.
   SmallVector<SwiftLookupTable::SingleEntry, 4> filteredDecls;
   llvm::copy_if(
       foundDecls, std::back_inserter(filteredDecls),
-      [&deepCompareDeclContexts, dc = cast<clang::DeclContext>(clangDecl)](
-          SwiftLookupTable::SingleEntry decl) {
-        return deepCompareDeclContexts(
-            decl.get<clang::NamedDecl *>()->getDeclContext(), dc);
+      [clangDecl](SwiftLookupTable::SingleEntry decl) {
+            auto first = decl.get<clang::NamedDecl *>()->getDeclContext();
+            auto second = cast<clang::DeclContext>(clangDecl);
+            if (auto firstDecl = dyn_cast<clang::Decl>(first)) {
+              if (auto secondDecl = dyn_cast<clang::Decl>(second))
+                return firstDecl->getCanonicalDecl() == secondDecl->getCanonicalDecl();
+              else
+                return false;
+            }
+            return first == second;
       });
   return filteredDecls;
 }
@@ -4116,17 +4082,22 @@ TinyPtrVector<ValueDecl *> CXXNamespaceMemberLookup::evaluate(
     Evaluator &evaluator, CXXNamespaceMemberLookupDescriptor desc) const {
   EnumDecl *namespaceDecl = desc.namespaceDecl;
   DeclName name = desc.name;
-
+  auto *clangNamespaceDecl =
+      cast<clang::NamespaceDecl>(namespaceDecl->getClangDecl());
   auto &ctx = namespaceDecl->getASTContext();
-  auto allResults = evaluateOrDefault(
-      ctx.evaluator, ClangDirectLookupRequest({namespaceDecl, name}), {});
 
   TinyPtrVector<ValueDecl *> result;
-  for (auto found : allResults) {
-    auto clangMember = found.get<clang::NamedDecl *>();
-    if (auto import =
-            ctx.getClangModuleLoader()->importDeclDirectly(clangMember))
-      result.push_back(cast<ValueDecl>(import));
+  for (auto redecl : clangNamespaceDecl->redecls()) {
+    auto allResults = evaluateOrDefault(
+        ctx.evaluator, ClangDirectLookupRequest({namespaceDecl, redecl, name}),
+        {});
+
+    for (auto found : allResults) {
+      auto clangMember = found.get<clang::NamedDecl *>();
+      if (auto import =
+              ctx.getClangModuleLoader()->importDeclDirectly(clangMember))
+        result.push_back(cast<ValueDecl>(import));
+    }
   }
 
   return result;
@@ -4139,7 +4110,9 @@ TinyPtrVector<ValueDecl *> ClangRecordMemberLookup::evaluate(
 
   auto &ctx = recordDecl->getASTContext();
   auto allResults = evaluateOrDefault(
-      ctx.evaluator, ClangDirectLookupRequest({recordDecl, name}), {});
+      ctx.evaluator,
+      ClangDirectLookupRequest({recordDecl, recordDecl->getClangDecl(), name}),
+      {});
 
   // Find the results that are actually a member of "recordDecl".
   TinyPtrVector<ValueDecl *> result;
@@ -4188,8 +4161,9 @@ ClangImporter::Implementation::loadNamedMembers(
   auto table = findLookupTable(*CMO);
   assert(table && "clang module without lookup table");
 
-  assert(isa<clang::ObjCContainerDecl>(CD) || isa<clang::NamespaceDecl>(CD) ||
-         isa<clang::RecordDecl>(CD));
+  assert(!isa<clang::NamespaceDecl>(CD) && "Namespace members should be loaded"
+                                           "via a request.");
+  assert(isa<clang::ObjCContainerDecl>(CD));
 
   // Force the members of the entire inheritance hierarchy to be loaded and
   // deserialized before loading the named member of a class. This warms up
@@ -4202,6 +4176,7 @@ ClangImporter::Implementation::loadNamedMembers(
     if (auto *superclassDecl = classDecl->getSuperclassDecl())
       (void) const_cast<ClassDecl *>(superclassDecl)->lookupDirect(N);
 
+  // TODO: update this to use the requestified lookup.
   TinyPtrVector<ValueDecl *> Members;
   for (auto entry : table->lookup(SerializedSwiftName(N),
                                   effectiveClangContext)) {