Parse and record top-level "items" rather than always forcing declarations.

In the Swift grammar, the top-level of a source file is a mix of three
different kinds of "items": declarations, statements, and expressions.
However, the existing parser forces all of these into declarations at
parse time, wrapping statements and expressions in TopLevelCodeDecls,
so the primary API for getting the top-level entities in source files
is based on getting declarations.

Start generalizing the representation by storing ASTNode instances at
the top level, rather than declaration pointers, updating many (but
not all!) uses of this API. The walk over declarations is a (cached)
filter to pick out all of the declarations. Existing parsed files are
unaffected (the parser still creates top-level code declarations), but
the new "macro expansion" source file kind skips creating top-level
code declarations so we get the pure parse tree. Additionally, some
generalized clients (like ASTScope lookup) will now look at the list
of items, so they'll be able to walk into statements and expressions
without the intervening TopLevelCodeDecl.

Over time, I'd like to phase out `getTopLevelDecls()` entirely,
relying on the new `getTopLevelItems()` for parsed content. We can
introduce TopLevelCodeDecls more lazily for semantic walks.

Author: DougGregor

docs/Generics/generics.tex

@@ -1351,7 +1351,7 @@ \subsection*{Module System}
 \apiref{SourceFile}{class}
 Represents a parsed source file from disk. Inherits from \texttt{FileUnit}.
 \begin{itemize}
-\item \texttt{getTopLevelDecls()} returns an array of all top-level declarations in this source file.
+\item \texttt{getTopLevelItems()} returns an array of all top-level items in this source file.
 \item \texttt{isPrimary()} returns \texttt{true} if this is a primary file, \texttt{false} if this is a secondary file.
 \item \texttt{isScriptMode()} answers if this is the main file of a module.
 \item \texttt{getScope()} returns the root of the scope tree for unqualified lookup.

include/swift/AST/ParseRequests.h

@@ -25,6 +25,8 @@
 
 namespace swift {
 
+struct ASTNode;
+
 /// Report that a request of the given kind is being evaluated, so it
 /// can be recorded by the stats reporter.
 template<typename Request>
@@ -85,13 +87,13 @@ class ParseAbstractFunctionBodyRequest
 };
 
 struct SourceFileParsingResult {
-  ArrayRef<Decl *> TopLevelDecls;
+  ArrayRef<ASTNode> TopLevelItems;
   Optional<ArrayRef<Token>> CollectedTokens;
   Optional<StableHasher> InterfaceHasher;
   Optional<syntax::SourceFileSyntax> SyntaxRoot;
 };
 
-/// Parse the top-level decls of a SourceFile.
+/// Parse the top-level items of a SourceFile.
 class ParseSourceFileRequest
     : public SimpleRequest<
           ParseSourceFileRequest, SourceFileParsingResult(SourceFile *),
@@ -116,6 +118,25 @@ class ParseSourceFileRequest
   readDependencySource(const evaluator::DependencyRecorder &) const;
 };
 
+/// Parse the top-level items of a SourceFile.
+class ParseTopLevelDeclsRequest
+    : public SimpleRequest<
+          ParseTopLevelDeclsRequest, ArrayRef<Decl *>(SourceFile *),
+          RequestFlags::Cached> {
+public:
+  using SimpleRequest::SimpleRequest;
+
+private:
+  friend SimpleRequest;
+
+  // Evaluation.
+  ArrayRef<Decl *> evaluate(Evaluator &evaluator, SourceFile *SF) const;
+
+public:
+  // Caching.
+  bool isCached() const { return true; }
+};
+
 void simple_display(llvm::raw_ostream &out,
                     const CodeCompletionCallbacksFactory *factory);
 

include/swift/AST/ParseTypeIDZone.def

@@ -25,3 +25,6 @@ SWIFT_REQUEST(Parse, ParseAbstractFunctionBodyRequest,
 SWIFT_REQUEST(Parse, ParseSourceFileRequest,
               SourceFileParsingResult(SourceFile *), SeparatelyCached,
               NoLocationInfo)
+SWIFT_REQUEST(Parse, ParseTopLevelDeclsRequest,
+              ArrayRef<Decl *>(SourceFile *), Cached,
+              NoLocationInfo)

include/swift/AST/SourceFile.h

@@ -21,6 +21,7 @@
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/STLExtras.h"
 
 namespace swift {
 
@@ -152,11 +153,11 @@ class SourceFile final : public FileUnit {
   /// been validated.
   llvm::SetVector<ValueDecl *> UnvalidatedDeclsWithOpaqueReturnTypes;
 
-  /// The list of top-level declarations in the source file. This is \c None if
+  /// The list of top-level items in the source file. This is \c None if
   /// they have not yet been parsed.
   /// FIXME: Once addTopLevelDecl/prependTopLevelDecl
   /// have been removed, this can become an optional ArrayRef.
-  Optional<std::vector<Decl *>> Decls;
+  Optional<std::vector<ASTNode>> Items;
 
   /// The list of hoisted declarations. See Decl::isHoisted().
   /// This is only used by lldb.
@@ -207,40 +208,37 @@ class SourceFile final : public FileUnit {
 
   /// Appends the given declaration to the end of the top-level decls list. Do
   /// not add any additional uses of this function.
-  void addTopLevelDecl(Decl *d) {
-    // Force decl parsing if we haven't already.
-    (void)getTopLevelDecls();
-    Decls->push_back(d);
-  }
+  void addTopLevelDecl(Decl *d);
 
   /// Prepends a declaration to the top-level decls list.
   ///
   /// FIXME: This entrypoint exists to support LLDB. Calls to this function are
   /// always a mistake, and additional uses should not be added.
   ///
   /// See rdar://58355191
-  void prependTopLevelDecl(Decl *d) {
-    // Force decl parsing if we haven't already.
-    (void)getTopLevelDecls();
-    Decls->insert(Decls->begin(), d);
-  }
+  void prependTopLevelDecl(Decl *d);
 
   /// Add a hoisted declaration. See Decl::isHoisted().
   void addHoistedDecl(Decl *d);
 
+  /// Retrieves an immutable view of the list of top-level items in this file.
+  ArrayRef<ASTNode> getTopLevelItems() const;
+
   /// Retrieves an immutable view of the list of top-level decls in this file.
+  ///
+  /// NOTE: Please use getTopLevelItems() instead.
   ArrayRef<Decl *> getTopLevelDecls() const;
 
   /// Retrieves an immutable view of the list of hoisted decls in this file.
   /// See Decl::isHoisted().
   ArrayRef<Decl *> getHoistedDecls() const;
 
-  /// Retrieves an immutable view of the top-level decls if they have already
+  /// Retrieves an immutable view of the top-level items if they have already
   /// been parsed, or \c None if they haven't. Should only be used for dumping.
-  Optional<ArrayRef<Decl *>> getCachedTopLevelDecls() const {
-    if (!Decls)
+  Optional<ArrayRef<ASTNode>> getCachedTopLevelItems() const {
+    if (!Items)
       return None;
-    return llvm::makeArrayRef(*Decls);
+    return llvm::makeArrayRef(*Items);
   }
 
   /// Retrieve the parsing options for the file.
@@ -522,7 +520,7 @@ class SourceFile final : public FileUnit {
     // FIXME: Ideally the parser state should be an output of
     // ParseSourceFileRequest, but the evaluator doesn't currently support
     // move-only outputs for cached requests.
-    (void)getTopLevelDecls();
+    (void)getTopLevelItems();
 
     auto *state = DelayedParserState.get();
     assert(state && "Didn't set any delayed parser state!");

include/swift/Parse/Parser.h

@@ -79,6 +79,8 @@ namespace swift {
     InactiveConditionalBlock,
     /// The body of the active clause of an #if/#else/#endif block
     ActiveConditionalBlock,
+    /// The top-level of a macro expansion "file".
+    MacroExpansion,
   };
 
 /// The receiver will be fed with consumed tokens while parsing. The main purpose
@@ -969,8 +971,10 @@ class Parser {
   /// Returns true if the parser is at the start of a SIL decl.
   bool isStartOfSILDecl();
 
-  /// Parse the top-level Swift decls into the provided vector.
-  void parseTopLevel(SmallVectorImpl<Decl *> &decls);
+  /// Parse the top-level Swift items into the provided vector.
+  ///
+  /// Each item will be a declaration, statement, or expression.
+  void parseTopLevelItems(SmallVectorImpl<ASTNode> &items);
 
   /// Parse the top-level SIL decls into the SIL module.
   /// \returns \c true if there was a parsing error.

lib/AST/ASTDumper.cpp

@@ -804,13 +804,21 @@ namespace {
       PrintWithColorRAII(OS, ASTNodeColor) << "source_file ";
       PrintWithColorRAII(OS, LocationColor) << '\"' << SF.getFilename() << '\"';
 
-      if (auto decls = SF.getCachedTopLevelDecls()) {
-        for (Decl *D : *decls) {
-          if (D->isImplicit())
+      if (auto items = SF.getCachedTopLevelItems()) {
+        for (auto item : *items) {
+          if (item.isImplicit())
             continue;
 
           OS << '\n';
-          printRec(D);
+
+          if (auto decl = item.dyn_cast<Decl *>()) {
+            printRec(decl);
+          } else if (auto stmt = item.dyn_cast<Stmt *>()) {
+            stmt->dump(OS, &SF.getASTContext(), Indent + 2);
+          } else {
+            auto expr = item.get<Expr *>();
+            expr->dump(OS, Indent + 2);
+          }
         }
       }
       PrintWithColorRAII(OS, ParenthesisColor) << ')';
@@ -1470,7 +1478,7 @@ void SourceFile::dump(llvm::raw_ostream &OS, bool parseIfNeeded) const {
   // parsing request as by default the dumping logic tries not to kick any
   // requests.
   if (parseIfNeeded)
-    (void)getTopLevelDecls();
+    (void)getTopLevelItems();
 
   PrintDecl(OS).visitSourceFile(*this);
   llvm::errs() << '\n';

lib/AST/ASTScopeCreation.cpp

@@ -692,9 +692,9 @@ ASTSourceFileScope::expandAScopeThatCreatesANewInsertionPoint(
   SourceLoc endLoc = getSourceRangeOfThisASTNode().End;
 
   ASTScopeImpl *insertionPoint = this;
-  for (auto *d : SF->getTopLevelDecls()) {
+  for (auto node : SF->getTopLevelItems()) {
     insertionPoint = scopeCreator.addToScopeTreeAndReturnInsertionPoint(
-      ASTNode(d), insertionPoint, endLoc);
+      node, insertionPoint, endLoc);
   }
 
   return {insertionPoint, "Next time decls are added they go here."};

lib/AST/ASTScopeSourceRange.cpp

@@ -181,12 +181,12 @@ SourceRange ASTSourceFileScope::getSourceRangeOfThisASTNode(
     return SourceRange(charRange.getStart(), charRange.getEnd());
   }
 
-  if (SF->getTopLevelDecls().empty())
+  if (SF->getTopLevelItems().empty())
     return SourceRange();
 
   // Use the source ranges of the declarations in the file.
-  return SourceRange(SF->getTopLevelDecls().front()->getStartLoc(),
-                     SF->getTopLevelDecls().back()->getEndLoc());
+  return SourceRange(SF->getTopLevelItems().front().getStartLoc(),
+                     SF->getTopLevelItems().back().getEndLoc());
 }
 
 SourceRange GenericTypeOrExtensionScope::getSourceRangeOfThisASTNode(

lib/AST/Module.cpp

@@ -3274,10 +3274,39 @@ void SourceFile::addHoistedDecl(Decl *d) {
 }
 
 ArrayRef<Decl *> SourceFile::getTopLevelDecls() const {
+  auto &ctx = getASTContext();
+  auto *mutableThis = const_cast<SourceFile *>(this);
+  return evaluateOrDefault(
+      ctx.evaluator, ParseTopLevelDeclsRequest{mutableThis}, {});
+}
+
+void SourceFile::addTopLevelDecl(Decl *d) {
+  // Force decl parsing if we haven't already.
+  (void)getTopLevelItems();
+  Items->push_back(d);
+
+  // FIXME: This violates core properties of the evaluator.
+  auto &ctx = getASTContext();
+  auto *mutableThis = const_cast<SourceFile *>(this);
+  ctx.evaluator.clearCachedOutput(ParseTopLevelDeclsRequest{mutableThis});
+}
+
+void SourceFile::prependTopLevelDecl(Decl *d) {
+  // Force decl parsing if we haven't already.
+  (void)getTopLevelItems();
+  Items->insert(Items->begin(), d);
+
+  // FIXME: This violates core properties of the evaluator.
+  auto &ctx = getASTContext();
+  auto *mutableThis = const_cast<SourceFile *>(this);
+  ctx.evaluator.clearCachedOutput(ParseTopLevelDeclsRequest{mutableThis});
+}
+
+ArrayRef<ASTNode> SourceFile::getTopLevelItems() const {
   auto &ctx = getASTContext();
   auto *mutableThis = const_cast<SourceFile *>(this);
   return evaluateOrDefault(ctx.evaluator, ParseSourceFileRequest{mutableThis},
-                           {}).TopLevelDecls;
+                           {}).TopLevelItems;
 }
 
 ArrayRef<Decl *> SourceFile::getHoistedDecls() const {
@@ -3335,20 +3364,21 @@ bool FileUnit::walk(ASTWalker &walker) {
 bool SourceFile::walk(ASTWalker &walker) {
   llvm::SaveAndRestore<ASTWalker::ParentTy> SAR(walker.Parent,
                                                 getParentModule());
-  for (Decl *D : getTopLevelDecls()) {
-#ifndef NDEBUG
-    PrettyStackTraceDecl debugStack("walking into decl", D);
-#endif
-
-    if (D->walk(walker))
-      return true;
+  for (auto Item : getTopLevelItems()) {
+    if (auto D = Item.dyn_cast<Decl *>()) {
+      if (D->walk(walker))
+        return true;
+    } else {
+      Item.walk(walker);
+    }
 
     if (walker.shouldWalkAccessorsTheOldWay()) {
       // Pretend that accessors share a parent with the storage.
       //
       // FIXME: Update existing ASTWalkers to deal with accessors appearing as
       // children of the storage instead.
-      if (auto *ASD = dyn_cast<AbstractStorageDecl>(D)) {
+      if (auto *ASD = dyn_cast_or_null<AbstractStorageDecl>(
+              Item.dyn_cast<Decl *>())) {
         for (auto AD : ASD->getAllAccessors()) {
           if (AD->walk(walker))
             return true;

lib/Parse/ParseDecl.cpp

@@ -197,7 +197,7 @@ extern "C" void swift_ASTGen_buildTopLevelASTNodes(void *sourceFile,
 ///     decl-sil       [[only in SIL mode]
 ///     decl-sil-stage [[only in SIL mode]
 /// \endverbatim
-void Parser::parseTopLevel(SmallVectorImpl<Decl *> &decls) {
+void Parser::parseTopLevelItems(SmallVectorImpl<ASTNode> &items) {
 #if SWIFT_SWIFT_PARSER
   if ((Context.LangOpts.hasFeature(Feature::Macros) ||
        Context.LangOpts.hasFeature(Feature::BuiltinMacros) ||
@@ -220,7 +220,7 @@ void Parser::parseTopLevel(SmallVectorImpl<Decl *> &decls) {
     // If we want to do ASTGen, do so now.
     if (Context.LangOpts.hasFeature(Feature::ParserASTGen)) {
       swift_ASTGen_buildTopLevelASTNodes(
-          exportedSourceFile, CurDeclContext, &Context, &decls, appendToVector);
+          exportedSourceFile, CurDeclContext, &Context, &items, appendToVector);
 
       // Spin the C++ parser to the end; we won't be using it.
       while (!Tok.is(tok::eof)) {
@@ -237,7 +237,6 @@ void Parser::parseTopLevel(SmallVectorImpl<Decl *> &decls) {
     consumeTokenWithoutFeedingReceiver();
 
   // Parse the body of the file.
-  SmallVector<ASTNode, 128> items;
   while (!Tok.is(tok::eof)) {
     // If we run into a SIL decl, skip over until the next Swift decl. We need
     // to delay parsing these, as SIL parsing currently requires type checking
@@ -248,9 +247,25 @@ void Parser::parseTopLevel(SmallVectorImpl<Decl *> &decls) {
       continue;
     }
 
-    parseBraceItems(items, allowTopLevelCode()
-                               ? BraceItemListKind::TopLevelCode
-                               : BraceItemListKind::TopLevelLibrary);
+    // Figure out how to parse the items in this source file.
+    BraceItemListKind braceItemListKind;
+    switch (SF.Kind) {
+    case SourceFileKind::Main:
+      braceItemListKind = BraceItemListKind::TopLevelCode;
+      break;
+
+    case SourceFileKind::Library:
+    case SourceFileKind::Interface:
+    case SourceFileKind::SIL:
+      braceItemListKind = BraceItemListKind::TopLevelLibrary;
+      break;
+
+    case SourceFileKind::MacroExpansion:
+      braceItemListKind = BraceItemListKind::MacroExpansion;
+      break;
+    }
+
+    parseBraceItems(items, braceItemListKind);
 
     // In the case of a catastrophic parse error, consume any trailing
     // #else, #elseif, or #endif and move on to the next statement or
@@ -267,13 +282,6 @@ void Parser::parseTopLevel(SmallVectorImpl<Decl *> &decls) {
     }
   }
 
-  // Then append the top-level decls we parsed.
-  for (auto item : items) {
-    auto *decl = item.get<Decl *>();
-    assert(!isa<AccessorDecl>(decl) && "accessors should not be added here");
-    decls.push_back(decl);
-  }
-
   // Finalize the syntax context.
   SyntaxContext->addToken(Tok, LeadingTrivia, TrailingTrivia);
 }