Introduce and use #externalMacro for externally-defined macros.

Align the grammar of macro declarations with SE-0382, so that macro
definitions are parsed as an expression. External macro definitions
are referenced via a referenced to the macro `#externalMacro`. Define
that macro in the standard library, and recognize uses of it as the
definition of other macros to use externally-defined macros. For
example, this means that the "stringify" macro used in a lot of
examples is now defined as something like this:

    @expression macro stringify<T>(_ value: T) -> (T, String) =
        #externalMacro(module: "MyMacros", type: "StringifyMacro")

We still parse the old "A.B" syntax for two reasons. First, it's
helpful to anyone who has existing code using the prior syntax, so they
get a warning + Fix-It to rewrite to the new syntax. Second, we use it
to define builtin macros like `externalMacro` itself, which looks like this:

    @expression
    public macro externalMacro<T>(module: String, type: String) -> T =
        Builtin.ExternalMacro

This uses the same virtual `Builtin` module as other library builtins,
and we can expand it to handle other builtin macro implementations
(such as #line) over time.

Author: DougGregor

include/swift/AST/Decl.h

@@ -60,6 +60,7 @@ namespace swift {
   struct ASTNode;
   class ASTPrinter;
   class ASTWalker;
+  enum class BuiltinMacroKind: uint8_t;
   class ConstructorDecl;
   class DestructorDecl;
   class DiagnosticEngine;
@@ -79,6 +80,7 @@ namespace swift {
   class GenericSignature;
   class GenericTypeParamDecl;
   class GenericTypeParamType;
+  class MacroDefinition;
   class ModuleDecl;
   class NamedPattern;
   class EnumCaseDecl;
@@ -8344,27 +8346,16 @@ class MacroDecl : public GenericContext, public ValueDecl {
   /// The result type.
   TypeLoc resultType;
 
-  /// The module name for the external macro definition.
-  Identifier externalModuleName;
-
-  /// The location of the module name for the external macro definition.
-  SourceLoc externalModuleNameLoc;
-
-  /// The type name for the external macro definition.
-  Identifier externalMacroTypeName;
-
-  /// The location of the type name for the external macro definition.
-  SourceLoc externalMacroTypeNameLoc;
+  /// The macro definition, which should always be a
+  /// \c MacroExpansionExpr in well-formed code.
+  Expr *definition;
 
   MacroDecl(SourceLoc macroLoc, DeclName name, SourceLoc nameLoc,
             GenericParamList *genericParams,
             ParameterList *parameterList,
             SourceLoc arrowOrColonLoc,
             TypeRepr *resultType,
-            Identifier externalModuleName,
-            SourceLoc externalModuleNameLoc,
-            Identifier externalMacroTypeName,
-            SourceLoc externalMacroTypeNameLoc,
+            Expr *definition,
             DeclContext *parent);
 
   SourceRange getSourceRange() const;
@@ -8375,6 +8366,13 @@ class MacroDecl : public GenericContext, public ValueDecl {
   /// Determine the contexts in which this macro can be applied.
   MacroContexts getMacroContexts() const;
 
+  /// Retrieve the definition of this macro.
+  MacroDefinition getDefinition() const;
+
+  /// Retrieve the builtin macro kind for this macro, or \c None if it is a
+  /// user-defined macro with no special semantics.
+  Optional<BuiltinMacroKind> getBuiltinKind() const;
+
   static bool classof(const DeclContext *C) {
     if (auto D = C->getAsDecl())
       return classof(D);

include/swift/AST/DiagnosticsParse.def

@@ -1996,14 +1996,8 @@ ERROR(expected_lparen_macro,PointsToFirstBadToken,
       "expected '(' for macro parameters or ':' for a value-like macro", ())
 ERROR(expected_type_macro_result,PointsToFirstBadToken,
       "expected macro result type", ())
-ERROR(macro_decl_expected_equal,PointsToFirstBadToken,
-      "expected '=' to introduce external macro name", ())
-ERROR(macro_decl_expected_macro_module,PointsToFirstBadToken,
-      "expected external macro name", ())
-ERROR(macro_decl_expected_period,PointsToFirstBadToken,
-      "expected '.' between external macro module and type name", ())
-ERROR(macro_decl_expected_macro_type,PointsToFirstBadToken,
-      "expected external macro type name", ())
+ERROR(macro_decl_expected_macro_definition,PointsToFirstBadToken,
+      "expected macro definition following '=''", ())
 
 ERROR(macro_expansion_expr_expected_macro_identifier,PointsToFirstBadToken,
       "expected a macro identifier for a pound literal expression", ())

include/swift/AST/DiagnosticsSema.def

@@ -6801,6 +6801,18 @@ ERROR(macro_unsupported,none,
       "macros are not supported in this compiler", ())
 ERROR(macro_recursive,none,
       "recursive expansion of macro %0", (DeclName))
+WARNING(macro_definition_old_style,none,
+        "external macro definitions are now written using #externalMacro", ())
+WARNING(macro_definition_unknown_builtin,none,
+        "ignoring definition of unknown builtin macro %0", (Identifier))
+ERROR(macro_definition_not_expansion,none,
+      "macro must itself be defined by a macro expansion such as "
+      "'#externalMacro(...)'", ())
+ERROR(macro_definition_unsupported,none,
+      "macro definitions other than '#externalMacro(...)' are unsupported", ())
+ERROR(external_macro_arg_not_type_name,none,
+      "argument to `#externalMacro` must be a string literal naming "
+      "the external macro's %select{module|type}", (unsigned))
 
 //------------------------------------------------------------------------------
 // MARK: Move Only Errors

include/swift/AST/MacroDefinition.h

@@ -22,64 +22,97 @@
 
 namespace swift {
 
+/// A reference to an external macro definition that is understood by ASTGen.
+struct ExternalMacroDefinition {
+  /// ASTGen's notion of an macro definition, which is opaque to the C++ part
+  /// of the compiler.
+  void *opaqueHandle = nullptr;
+};
+
+/// A reference to an external macro.
+struct ExternalMacroReference {
+  Identifier moduleName;
+  Identifier macroTypeName;
+};
+
+/// Describes the known kinds of builtin macros.
+enum class BuiltinMacroKind: uint8_t {
+  /// #externalMacro, which references an external macro.
+  ExternalMacro,
+};
+
 /// Provides the definition of a macro.
 class MacroDefinition {
 public:
-  /// Describes a missing macro definition.
-  struct MissingDefinition {
-    Identifier externalModuleName;
-    Identifier externalMacroTypeName;
-  };
-
     /// Describes how the macro is implemented.
-  enum class ImplementationKind: uint8_t {
+  enum class Kind: uint8_t {
+    /// The macro has a definition, but it is invalid, so the macro cannot be
+    /// expanded.
+    Invalid,
+
     /// The macro has no definition.
     Undefined,
 
-    /// The macro has a definition, but it could not be found.
-    Missing,
+    /// An externally-provided macro definition.
+    External,
 
-    /// The macro is in the same process as the compiler, whether built-in or
-    /// loaded via a compiler plugin.
-    InProcess,
+    /// A builtin macro definition, which has a separate builtin kind.
+    Builtin,
   };
 
-  ImplementationKind implKind;
+  Kind kind;
 
 private:
-  void *opaqueHandle;
+  union Data {
+    ExternalMacroReference external;
+    BuiltinMacroKind builtin;
 
-  MacroDefinition(ImplementationKind implKind, void *opaqueHandle)
-    : implKind(implKind), opaqueHandle(opaqueHandle) { }
+    Data() : builtin(BuiltinMacroKind::ExternalMacro) { }
+  } data;
+
+  MacroDefinition(Kind kind) : kind(kind) { }
+
+  MacroDefinition(ExternalMacroReference external) : kind(Kind::External) {
+    data.external = external;
+  }
+
+  MacroDefinition(BuiltinMacroKind builtinKind) : kind(Kind::Builtin) {
+    data.builtin = builtinKind;
+  }
 
 public:
+  static MacroDefinition forInvalid() {
+    return MacroDefinition(Kind::Invalid);
+  }
+
   static MacroDefinition forUndefined() {
-    return MacroDefinition{
-      ImplementationKind::Undefined, nullptr
-    };
+    return MacroDefinition(Kind::Undefined);
   }
 
-  static MacroDefinition forMissing(
-      ASTContext &ctx,
-      Identifier externalModuleName,
-      Identifier externalMacroTypeName
-  );
+  static MacroDefinition forExternal(
+      Identifier moduleName,
+      Identifier macroTypeName
+   ) {
+    return MacroDefinition(ExternalMacroReference{moduleName, macroTypeName});
+  }
 
-  static MacroDefinition forInProcess(void *opaqueHandle) {
-    return MacroDefinition{ImplementationKind::InProcess, opaqueHandle};
+  static MacroDefinition forBuiltin(BuiltinMacroKind builtinKind) {
+    return MacroDefinition(builtinKind);
   }
 
-  /// Return the opaque handle for an in-process macro definition.
-  void *getInProcessOpaqueHandle() const {
-    assert(implKind == ImplementationKind::InProcess);
-    return opaqueHandle;
+  /// Retrieve the external macro being referenced.
+  ExternalMacroReference getExternalMacro() const {
+    assert(kind == Kind::External);
+    return data.external;
   }
 
-  /// Return more information about a missing macro definition.
-  MissingDefinition *getMissingDefinition() const {
-    assert(implKind == ImplementationKind::Missing);
-    return static_cast<MissingDefinition *>(opaqueHandle);
+  /// Retrieve the builtin kind.
+  BuiltinMacroKind getBuiltinKind() const {
+    assert(kind == Kind::Builtin);
+    return data.builtin;
   }
+
+  explicit operator bool() const { return kind != Kind::Invalid; }
 };
 
 }

include/swift/AST/TypeCheckRequests.h

@@ -45,6 +45,7 @@ class ContextualPattern;
 class ContinueStmt;
 class DefaultArgumentExpr;
 class DefaultArgumentType;
+struct ExternalMacroDefinition;
 class ClosureExpr;
 class GenericParamList;
 class LabeledStmt;
@@ -71,6 +72,8 @@ void simple_display(
     llvm::raw_ostream &out,
     const llvm::PointerUnion<const TypeDecl *, const ExtensionDecl *> &value);
 
+void simple_display(llvm::raw_ostream &out, ASTContext *ctx);
+
 /// Request the type from the ith entry in the inheritance clause for the
 /// given declaration.
 class InheritedTypeRequest
@@ -3796,6 +3799,30 @@ class MacroDefinitionRequest
   bool isCached() const { return true; }
 };
 
+/// Resolve an external macro given its module and type name.
+class ExternalMacroDefinitionRequest
+    : public SimpleRequest<ExternalMacroDefinitionRequest,
+                           ExternalMacroDefinition(
+                               ASTContext *, Identifier, Identifier),
+                           RequestFlags::Cached> {
+public:
+  using SimpleRequest::SimpleRequest;
+
+private:
+  friend SimpleRequest;
+
+  ExternalMacroDefinition evaluate(
+      Evaluator &evaluator, ASTContext *ctx, Identifier moduleName,
+      Identifier typeName
+  ) const;
+
+public:
+  // Source location
+  SourceLoc getNearestLoc() const { return SourceLoc(); }
+
+  bool isCached() const { return true; }
+};
+
 class GetRuntimeDiscoverableAttributes
     : public SimpleRequest<GetRuntimeDiscoverableAttributes,
                            ArrayRef<CustomAttr *>(ValueDecl *),

include/swift/AST/TypeCheckerTypeIDZone.def

@@ -449,6 +449,9 @@ SWIFT_REQUEST(TypeChecker, UsesTypeWrapperFeature,
 SWIFT_REQUEST(TypeChecker, MacroDefinitionRequest,
               MacroDefinition(MacroDecl *),
               Cached, NoLocationInfo)
+SWIFT_REQUEST(TypeChecker, ExternalMacroDefinitionRequest,
+              ExternalMacroDefinition(ASTContext *, Identifier, Identifier),
+              Cached, NoLocationInfo)
 SWIFT_REQUEST(TypeChecker, SynthesizeRuntimeMetadataAttrGenerator,
               Expr *(CustomAttr *, ValueDecl *),
               Cached, NoLocationInfo)

lib/AST/ASTPrinter.cpp

@@ -14,6 +14,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "InlinableText.h"
 #include "swift/AST/ASTPrinter.h"
 #include "swift/AST/ASTContext.h"
 #include "swift/AST/ASTMangler.h"
@@ -29,6 +30,7 @@
 #include "swift/AST/GenericEnvironment.h"
 #include "swift/AST/GenericParamList.h"
 #include "swift/AST/GenericSignature.h"
+#include "swift/AST/MacroDefinition.h"
 #include "swift/AST/Module.h"
 #include "swift/AST/NameLookup.h"
 #include "swift/AST/NameLookupRequests.h"
@@ -4541,8 +4543,36 @@ void PrintAST::visitMacroDecl(MacroDecl *decl) {
     Printer.printStructurePost(PrintStructureKind::FunctionReturnType);
   }
 
-  Printer << " = ";
-  Printer << decl->externalModuleName << "." << decl->externalMacroTypeName;
+  if (decl->definition) {
+    ASTContext &ctx = decl->getASTContext();
+    SmallString<64> scratch;
+    Printer << " = "
+            << extractInlinableText(ctx.SourceMgr, decl->definition, scratch);
+  } else {
+    auto def = decl->getDefinition();
+    switch (def.kind) {
+    case MacroDefinition::Kind::Invalid:
+    case MacroDefinition::Kind::Undefined:
+      // Nothing to do.
+      break;
+
+    case MacroDefinition::Kind::External: {
+      auto external = def.getExternalMacro();
+      Printer << " = #externalMacro(module: \"" << external.moduleName << "\", "
+              << "type: \"" << external.macroTypeName << "\")";
+      break;
+    }
+
+    case MacroDefinition::Kind::Builtin:
+      Printer << " = Builtin.";
+      switch (def.getBuiltinKind()) {
+      case BuiltinMacroKind::ExternalMacro:
+        Printer << "ExternalMacro";
+        break;
+      }
+      break;
+    }
+  }
 
   printDeclGenericRequirements(decl);
 }
@@ -7130,7 +7160,6 @@ swift::getInheritedForPrinting(
   }
 
   // Collect synthesized conformances.
-  auto &ctx = decl->getASTContext();
   llvm::SetVector<ProtocolDecl *> protocols;
   llvm::TinyPtrVector<ProtocolDecl *> uncheckedProtocols;
   for (auto attr : decl->getAttrs().getAttributes<SynthesizedProtocolAttr>()) {