Merge pull request #69761 from rintaro/astgen-incremental-develop

[ASTGen] Infrastructure for implementing ASTGen incrementally

Author: rintaro

include/module.modulemap

@@ -10,6 +10,12 @@ module ASTBridging {
   export *
 }
 
+module ParseBridging {
+  header "swift/Parse/ParseBridging.h"
+  requires cplusplus
+  export *
+}
+
 module SILBridging {
   header "swift/SIL/SILBridging.h"
   requires cplusplus

include/swift/AST/ASTBridgingWrappers.def

@@ -59,8 +59,10 @@
 // optional parameters.
 AST_BRIDGING_WRAPPER_NONNULL(Decl)
 AST_BRIDGING_WRAPPER_NONNULL(DeclContext)
+AST_BRIDGING_WRAPPER_NONNULL(SourceFile)
 AST_BRIDGING_WRAPPER_NULLABLE(Stmt)
 AST_BRIDGING_WRAPPER_NULLABLE(Expr)
+AST_BRIDGING_WRAPPER_NULLABLE(Pattern)
 AST_BRIDGING_WRAPPER_NULLABLE(TypeRepr)
 
 // Misc AST types to generate wrappers for.

include/swift/Basic/Features.def

@@ -193,10 +193,6 @@ EXPERIMENTAL_FEATURE(ImplicitSome, false)
 /// corresponding syntax tree.
 EXPERIMENTAL_FEATURE(ParserASTGen, false)
 
-/// Use the syntax tree produced by the Swift (swift-syntax) parser for type
-/// parsing, using ASTGen to translate them into AST nodes.
-EXPERIMENTAL_FEATURE(ASTGenTypes, false)
-
 /// Parse using the Swift (swift-syntax) parser and use ASTGen to generate the
 /// corresponding syntax tree.
 EXPERIMENTAL_FEATURE(BuiltinMacros, false)

include/swift/Bridging/ASTGen.h

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "swift/AST/ASTBridging.h"
+#include "swift/Parse/ParseBridging.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -40,6 +41,23 @@ void *_Nullable swift_ASTGen_parseSourceFile(const char *_Nonnull buffer,
                                              void *_Nullable ctx);
 void swift_ASTGen_destroySourceFile(void *_Nonnull sourceFile);
 
+/// Check whether the given source file round-trips correctly. Returns 0 if
+/// round-trip succeeded, non-zero otherwise.
+int swift_ASTGen_roundTripCheck(void *_Nonnull sourceFile);
+
+/// Emit parser diagnostics for given source file.. Returns non-zero if any
+/// diagnostics were emitted.
+int swift_ASTGen_emitParserDiagnostics(
+    void *_Nonnull diagEngine, void *_Nonnull sourceFile, int emitOnlyErrors,
+    int downgradePlaceholderErrorsToWarnings);
+
+// Build AST nodes for the top-level entities in the syntax.
+void swift_ASTGen_buildTopLevelASTNodes(
+    BridgedDiagnosticEngine diagEngine, void *_Nonnull sourceFile,
+    BridgedDeclContext declContext, BridgedASTContext astContext,
+    BridgedLegacyParser legacyParser, void *_Nonnull outputContext,
+    void (*_Nonnull)(void *_Nonnull, void *_Nonnull));
+
 void *_Nullable swift_ASTGen_resolveMacroType(const void *_Nonnull macroType);
 void swift_ASTGen_destroyMacro(void *_Nonnull macro);
 
@@ -84,6 +102,38 @@ bool swift_ASTGen_pluginServerLoadLibraryPlugin(
     void *_Nonnull handle, const char *_Nonnull libraryPath,
     const char *_Nonnull moduleName, BridgedStringRef *_Nullable errorOut);
 
+/// Build a TypeRepr for AST node for the type at the given source location in
+/// the specified file.
+swift::TypeRepr *_Nullable swift_ASTGen_buildTypeRepr(
+    BridgedDiagnosticEngine diagEngine, void *_Nonnull sourceFile,
+    BridgedSourceLoc sourceLoc, BridgedDeclContext declContext,
+    BridgedASTContext astContext, BridgedLegacyParser legacyParser,
+    BridgedSourceLoc *_Nonnull endSourceLoc);
+
+/// Build a Decl for AST node for the type at the given source location in the
+/// specified file.
+swift::Decl *_Nullable swift_ASTGen_buildDecl(
+    BridgedDiagnosticEngine diagEngine, void *_Nonnull sourceFile,
+    BridgedSourceLoc sourceLoc, BridgedDeclContext declContext,
+    BridgedASTContext astContext, BridgedLegacyParser legacyParser,
+    BridgedSourceLoc *_Nonnull endSourceLoc);
+
+/// Build a Expr for AST node for the type at the given source location in the
+/// specified file.
+swift::Expr *_Nullable swift_ASTGen_buildExpr(
+    BridgedDiagnosticEngine diagEngine, void *_Nonnull sourceFile,
+    BridgedSourceLoc sourceLoc, BridgedDeclContext declContext,
+    BridgedASTContext astContext, BridgedLegacyParser legacyParser,
+    BridgedSourceLoc *_Nonnull endSourceLoc);
+
+/// Build a Stmt for AST node for the type at the given source location in the
+/// specified file.
+swift::Stmt *_Nullable swift_ASTGen_buildStmt(
+    BridgedDiagnosticEngine diagEngine, void *_Nonnull sourceFile,
+    BridgedSourceLoc sourceLoc, BridgedDeclContext declContext,
+    BridgedASTContext astContext, BridgedLegacyParser legacyParser,
+    BridgedSourceLoc *_Nonnull endSourceLoc);
+
 #ifdef __cplusplus
 }
 #endif

include/swift/Parse/ParseBridging.h

@@ -0,0 +1,63 @@
+//===--- ParseBridging.h --------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_PARSE_PARSEBRIDGING_H
+#define SWIFT_PARSE_PARSEBRIDGING_H
+
+#include "swift/AST/ASTBridging.h"
+#include "swift/Basic/BasicBridging.h"
+
+#ifdef USED_IN_CPP_SOURC
+#include "swift/Parse/Parser.h"
+#else
+namespace swift {
+class Parser;
+}
+#endif
+
+SWIFT_BEGIN_NULLABILITY_ANNOTATIONS
+
+class BridgedLegacyParser {
+  swift::Parser *_Nonnull const handle;
+
+public:
+#ifdef USED_IN_CPP_SOURCE
+  BridgedLegacyParser(swift::Parser &P) : handle(&P) {}
+
+  swift::Parser &unbridged() const { return *handle; }
+#endif
+};
+
+SWIFT_NAME("BridgedLegacyParser.parseExpr(self:_:_:_:)")
+BridgedExpr BridgedLegacyParser_parseExpr(BridgedLegacyParser,
+                                          BridgedSourceLoc loc,
+                                          BridgedDeclContext DC,
+                                          bool isExprBasic);
+
+SWIFT_NAME("BridgedLegacyParser.parseDecl(self:_:_:)")
+BridgedDecl BridgedLegacyParser_parseDecl(BridgedLegacyParser,
+                                          BridgedSourceLoc loc,
+                                          BridgedDeclContext DC);
+
+SWIFT_NAME("BridgedLegacyParser.parseStmt(self:_:_:)")
+BridgedStmt BridgedLegacyParser_parseStmt(BridgedLegacyParser,
+                                          BridgedSourceLoc loc,
+                                          BridgedDeclContext DC);
+
+SWIFT_NAME("BridgedLegacyParser.parseType(self:_:_:)")
+BridgedTypeRepr BridgedLegacyParser_parseType(BridgedLegacyParser,
+                                              BridgedSourceLoc loc,
+                                              BridgedDeclContext DC);
+
+SWIFT_END_NULLABILITY_ANNOTATIONS
+
+#endif // SWIFT_PARSE_PARSEBRIDGING_H

include/swift/Parse/Parser.h

@@ -158,6 +158,11 @@ class Parser {
   bool InSwiftKeyPath = false;
   bool InFreestandingMacroArgument = false;
 
+#if SWIFT_BUILD_SWIFT_SYNTAX
+  // This Parser is a fallback parser for ASTGen.
+  bool IsForASTGen = false;
+#endif
+
   // A cached answer to
   //     Context.LangOpts.hasFeature(Feature::NoncopyableGenerics)
   // to ensure there's no parsing performance regression.
@@ -957,7 +962,8 @@ class Parser {
 
   ParserResult<Decl> parseDecl(bool IsAtStartOfLineOrPreviousHadSemi,
                                bool IfConfigsAreDeclAttrs,
-                               llvm::function_ref<void(Decl *)> Handler);
+                               llvm::function_ref<void(Decl *)> Handler,
+                               bool fromASTGen = false);
 
   std::pair<std::vector<Decl *>, llvm::Optional<Fingerprint>>
   parseDeclListDelayed(IterableDeclContext *IDC);
@@ -1349,50 +1355,6 @@ class Parser {
   /// Get the location for a type error.
   SourceLoc getTypeErrorLoc() const;
 
-  /// Callback function used for creating a C++ AST from the syntax node at the given source location.
-  ///
-  /// The arguments to this callback are the source file to pass into ASTGen (the exported source file)
-  /// and the source location pointer to pass into ASTGen (to find the syntax node).
-  ///
-  /// The callback returns the new AST node and the ending location of the syntax node. If the AST node
-  /// is NULL, something went wrong.
-  template<typename T>
-  using ASTFromSyntaxTreeCallback = std::pair<T*, const void *>(
-      void *sourceFile, const void *sourceLoc
-  );
-
-  /// Parse by constructing a C++ AST node from the Swift syntax tree via ASTGen.
-  template<typename T>
-  ParserResult<T> parseASTFromSyntaxTree(
-      llvm::function_ref<ASTFromSyntaxTreeCallback<T>> body
-  ) {
-    if (!Context.LangOpts.hasFeature(Feature::ASTGenTypes))
-      return nullptr;
-
-    auto exportedSourceFile = SF.getExportedSourceFile();
-    if (!exportedSourceFile)
-      return nullptr;
-
-    // Perform the translation.
-    auto sourceLoc = Tok.getLoc().getOpaquePointerValue();
-    T* astNode;
-    const void *endLocPtr;
-    std::tie(astNode, endLocPtr) = body(exportedSourceFile, sourceLoc);
-
-    if (!astNode) {
-      assert(false && "Could not build AST node from syntax tree");
-      return nullptr;
-    }
-
-    // Reset the lexer to the ending location.
-    StringRef contents =
-        SourceMgr.extractText(SourceMgr.getRangeForBuffer(L->getBufferID()));
-    L->resetToOffset((const char *)endLocPtr - contents.data());
-    L->lex(Tok);
-
-    return makeParserResult(astNode);
-  }
-
   //===--------------------------------------------------------------------===//
   // Type Parsing
 
@@ -1409,9 +1371,10 @@ class Parser {
       ParseTypeReason reason);
 
   ParserResult<TypeRepr> parseType();
-  ParserResult<TypeRepr> parseType(
-      Diag<> MessageID,
-      ParseTypeReason reason = ParseTypeReason::Unspecified);
+  ParserResult<TypeRepr>
+  parseType(Diag<> MessageID,
+            ParseTypeReason reason = ParseTypeReason::Unspecified,
+            bool fromASTGen = false);
 
   /// Parse a type optionally prefixed by a list of named opaque parameters. If
   /// no params present, return 'type'. Otherwise, return 'type-named-opaque'.
@@ -1735,7 +1698,8 @@ class Parser {
   ParserResult<Expr> parseExprBasic(Diag<> ID) {
     return parseExprImpl(ID, /*isExprBasic=*/true);
   }
-  ParserResult<Expr> parseExprImpl(Diag<> ID, bool isExprBasic);
+  ParserResult<Expr> parseExprImpl(Diag<> ID, bool isExprBasic,
+                                   bool fromASTGen = false);
   ParserResult<Expr> parseExprIs();
   ParserResult<Expr> parseExprAs();
   ParserResult<Expr> parseExprArrow();
@@ -1944,7 +1908,7 @@ class Parser {
 
   bool isTerminatorForBraceItemListKind(BraceItemListKind Kind,
                                         ArrayRef<ASTNode> ParsedDecls);
-  ParserResult<Stmt> parseStmt();
+  ParserResult<Stmt> parseStmt(bool fromASTGen = false);
   ParserStatus parseExprOrStmt(ASTNode &Result);
   ParserResult<Stmt> parseStmtBreak();
   ParserResult<Stmt> parseStmtContinue();
@@ -2075,6 +2039,18 @@ class Parser {
   bool parseLegacyTildeCopyable(SourceLoc *parseTildeCopyable,
                                 ParserStatus &Status,
                                 SourceLoc &TildeCopyableLoc);
+
+  //===--------------------------------------------------------------------===//
+  // ASTGen support.
+
+  /// Parse a TypeRepr from the syntax tree. i.e. SF->getExportedSourceFile()
+  ParserResult<TypeRepr> parseTypeReprFromSyntaxTree();
+  /// Parse a Stmt from the syntax tree. i.e. SF->getExportedSourceFile()
+  ParserResult<Stmt> parseStmtFromSyntaxTree();
+  /// Parse a Decl from the syntax tree. i.e. SF->getExportedSourceFile()
+  ParserResult<Decl> parseDeclFromSyntaxTree();
+  /// Parse an Expr from the syntax tree. i.e. SF->getExportedSourceFile()
+  ParserResult<Expr> parseExprFromSyntaxTree();
 };
 
 /// Describes a parsed declaration name.

lib/AST/ASTBridging.cpp

@@ -326,7 +326,42 @@ BridgedParamDecl BridgedParamDecl_createParsed(
   auto *paramDecl = new (cContext.unbridged())
       ParamDecl(cSpecifierLoc.unbridged(), firstNameLoc, firstName,
                 secondNameLoc, secondName, declContext);
-  paramDecl->setTypeRepr(opaqueType.unbridged());
+
+  if (auto type = opaqueType.unbridged()) {
+    paramDecl->setTypeRepr(type);
+
+    // FIXME: Copied from 'Parser::parsePattern()'. This should be in Sema.
+    // Dig through the type to find any attributes or modifiers that are
+    // associated with the type but should also be reflected on the
+    // declaration.
+    auto unwrappedType = type;
+    while (true) {
+      if (auto *ATR = dyn_cast<AttributedTypeRepr>(unwrappedType)) {
+        auto &attrs = ATR->getAttrs();
+        // At this point we actually don't know if that's valid to mark
+        // this parameter declaration as `autoclosure` because type has
+        // not been resolved yet - it should either be a function type
+        // or typealias with underlying function type.
+        paramDecl->setAutoClosure(attrs.has(TypeAttrKind::TAK_autoclosure));
+
+        unwrappedType = ATR->getTypeRepr();
+        continue;
+      }
+
+      if (auto *STR = dyn_cast<SpecifierTypeRepr>(unwrappedType)) {
+        if (isa<IsolatedTypeRepr>(STR))
+          paramDecl->setIsolated(true);
+        else if (isa<CompileTimeConstTypeRepr>(STR))
+          paramDecl->setCompileTimeConst(true);
+
+        unwrappedType = STR->getBase();
+        continue;
+      }
+
+      break;
+    }
+  }
+
   paramDecl->setDefaultExpr(defaultValue, /*isTypeChecked*/ false);
   paramDecl->setDefaultArgumentKind(defaultArgumentKind);
 

lib/AST/ASTPrinter.cpp

@@ -3426,10 +3426,6 @@ static bool usesFeatureParserASTGen(Decl *decl) {
   return false;
 }
 
-static bool usesFeatureASTGenTypes(Decl *decl) {
-  return false;
-}
-
 static bool usesFeatureBuiltinMacros(Decl *decl) {
   return false;
 }

lib/ASTGen/CMakeLists.txt

@@ -13,6 +13,7 @@ add_pure_swift_host_library(swiftASTGen STATIC
   Sources/ASTGen/DiagnosticsBridge.swift
   Sources/ASTGen/Exprs.swift
   Sources/ASTGen/Generics.swift
+  Sources/ASTGen/LegacyParse.swift
   Sources/ASTGen/Literals.swift
   Sources/ASTGen/Macros.swift
   Sources/ASTGen/ParameterClause.swift