Frontend: Introduce -emit-api-descriptor flag.

tshortli · tshortli · commit e1f2e25ed52e · 2023-10-05T11:40:53.000-07:00
An "API descriptor" file is JSON describing the externally accessible symbols
of a module and metadata associated with those symbols like availability and
SPI status. This output was previously only generated by the
`swift-api-extract` alias of `swift-frontend`, which is desgined to take an
already built module as input. Post-processing a built module to extract this
information is inefficient because the module and the module's dependencies
need to be deserialized in order to visit the entire AST. We can generate this
output more efficiently as a supplementary output of the -emit-module job that
originally produced the module (since the AST is already available in-memory).
The -emit-api-descriptor flag can be used to request this output.

This change lays the groundwork by introducing frontend flags. Follow up
changes are needed to make API descriptor emission during -emit-module
functional.

Part of rdar://110916764.
diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def
@@ -152,7 +152,9 @@ ERROR(error_mode_cannot_emit_module_summary,none,
 ERROR(error_mode_cannot_emit_symbol_graph,none,
       "this mode does not support emitting symbol graph files", ())
 ERROR(error_mode_cannot_emit_abi_descriptor,none,
-      "this mode does not support emitting ABI descriptor", ())
+      "this mode does not support emitting ABI descriptor files", ())
+ERROR(error_mode_cannot_emit_api_descriptor,none,
+      "this mode does not support emitting API descriptor files", ())
 ERROR(error_mode_cannot_emit_const_values,none,
       "this mode does not support emitting extracted const values", ())
 ERROR(error_mode_cannot_emit_module_semantic_info,none,
diff --git a/include/swift/Basic/FileTypes.def b/include/swift/Basic/FileTypes.def
@@ -98,6 +98,7 @@ TYPE("pch",                 PCH,                       "pch",             "")
 TYPE("none",                Nothing,                   "",                "")
 
 TYPE("abi-baseline-json",   SwiftABIDescriptor,        "abi.json",        "")
+TYPE("api-json",            SwiftAPIDescriptor,        "",                "")
 TYPE("fixit",               SwiftFixIt,                "",                "")
 TYPE("module-semantic-info", ModuleSemanticInfo,       "",                "")
 TYPE("cached-diagnostics",  CachedDiagnostics,         "",                "")
diff --git a/include/swift/Basic/SupplementaryOutputPaths.def b/include/swift/Basic/SupplementaryOutputPaths.def
@@ -143,6 +143,9 @@ OUTPUT(ModuleSummaryOutputPath, TY_SwiftModuleSummaryFile)
 /// The output path to generate ABI baseline.
 OUTPUT(ABIDescriptorOutputPath, TY_SwiftABIDescriptor)
 
+/// The output path to the module's API description.
+OUTPUT(APIDescriptorOutputPath, TY_SwiftAPIDescriptor)
+
 /// The output path for extracted compile-time-known value information
 OUTPUT(ConstValuesOutputPath, TY_ConstValues)
 
diff --git a/include/swift/Frontend/Frontend.h b/include/swift/Frontend/Frontend.h
@@ -428,6 +428,11 @@ class CompilerInvocation {
   std::string getModuleInterfaceOutputPathForWholeModule() const;
   std::string getPrivateModuleInterfaceOutputPathForWholeModule() const;
 
+  /// APIDescriptorPath only makes sense in whole module compilation mode,
+  /// so return the APIDescriptorPath when in that mode and fail an assert
+  /// if not in that mode.
+  std::string getAPIDescriptorPathForWholeModule() const;
+
 public:
   /// Given the current configuration of this frontend invocation, a set of
   /// supplementary output paths, and a module, compute the appropriate set of
diff --git a/include/swift/Frontend/FrontendInputsAndOutputs.h b/include/swift/Frontend/FrontendInputsAndOutputs.h
@@ -265,6 +265,7 @@ class FrontendInputsAndOutputs {
   bool hasModuleInterfaceOutputPath() const;
   bool hasPrivateModuleInterfaceOutputPath() const;
   bool hasABIDescriptorOutputPath() const;
+  bool hasAPIDescriptorOutputPath() const;
   bool hasConstValuesOutputPath() const;
   bool hasModuleSemanticInfoOutputPath() const;
   bool hasModuleSummaryOutputPath() const;
diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h
@@ -569,6 +569,7 @@ class FrontendOptions {
   static bool canActionEmitModuleSummary(ActionType);
   static bool canActionEmitInterface(ActionType);
   static bool canActionEmitABIDescriptor(ActionType);
+  static bool canActionEmitAPIDescriptor(ActionType);
   static bool canActionEmitConstValues(ActionType);
   static bool canActionEmitModuleSemanticInfo(ActionType);
 
diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td
@@ -626,7 +626,19 @@ def emit_const_values :
 def emit_const_values_path : Separate<["-"], "emit-const-values-path">,
   Flags<[FrontendOption, NoInteractiveOption, ArgumentIsPath,
          SupplementaryOutput, CacheInvariant]>,
-  MetaVarName<"<path>">, HelpText<"Emit the extracted compile-time known values to <path>">;
+  MetaVarName<"<path>">,
+  HelpText<"Emit the extracted compile-time known values to <path>">;
+
+def emit_api_descriptor :
+  Flag<["-"], "emit-api-descriptor">,
+  Flags<[NoInteractiveOption, SupplementaryOutput, CacheInvariant]>,
+  HelpText<"Output a JSON file describing the module's API">;
+def emit_api_descriptor_path :
+  Separate<["-"], "emit-api-descriptor-path">,
+  Flags<[FrontendOption, NoInteractiveOption, ArgumentIsPath,
+         SupplementaryOutput, CacheInvariant]>,
+  MetaVarName<"<path>">,
+  HelpText<"Output a JSON file describing the module's API to <path>">;
 
 def emit_objc_header : Flag<["-"], "emit-objc-header">,
   Flags<[FrontendOption, NoInteractiveOption, SupplementaryOutput, CacheInvariant]>,
diff --git a/lib/Basic/FileTypes.cpp b/lib/Basic/FileTypes.cpp
@@ -86,6 +86,7 @@ bool file_types::isTextual(ID Id) {
   case file_types::TY_JSONDependencies:
   case file_types::TY_JSONFeatures:
   case file_types::TY_SwiftABIDescriptor:
+  case file_types::TY_SwiftAPIDescriptor:
   case file_types::TY_ConstValues:
     return true;
   case file_types::TY_Image:
@@ -164,6 +165,7 @@ bool file_types::isAfterLLVM(ID Id) {
   case file_types::TY_JSONFeatures:
   case file_types::TY_IndexUnitOutputPath:
   case file_types::TY_SwiftABIDescriptor:
+  case file_types::TY_SwiftAPIDescriptor:
   case file_types::TY_ConstValues:
   case file_types::TY_SwiftFixIt:
   case file_types::TY_ModuleSemanticInfo:
@@ -220,6 +222,7 @@ bool file_types::isPartOfSwiftCompilation(ID Id) {
   case file_types::TY_JSONFeatures:
   case file_types::TY_IndexUnitOutputPath:
   case file_types::TY_SwiftABIDescriptor:
+  case file_types::TY_SwiftAPIDescriptor:
   case file_types::TY_ConstValues:
   case file_types::TY_SwiftFixIt:
   case file_types::TY_ModuleSemanticInfo:
diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp
@@ -2090,6 +2090,7 @@ void Driver::buildActions(SmallVectorImpl<const Action *> &TopLevelActions,
       case file_types::TY_JSONDependencies:
       case file_types::TY_JSONFeatures:
       case file_types::TY_SwiftABIDescriptor:
+      case file_types::TY_SwiftAPIDescriptor:
       case file_types::TY_ConstValues:
       case file_types::TY_SwiftFixIt:
       case file_types::TY_ModuleSemanticInfo:
diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp
@@ -733,6 +733,7 @@ const char *ToolChain::JobContext::computeFrontendModeForCompile() const {
   case file_types::TY_SwiftOverlayFile:
   case file_types::TY_IndexUnitOutputPath:
   case file_types::TY_SwiftABIDescriptor:
+  case file_types::TY_SwiftAPIDescriptor:
   case file_types::TY_ConstValues:
   case file_types::TY_SwiftFixIt:
   case file_types::TY_ModuleSemanticInfo:
@@ -997,6 +998,7 @@ ToolChain::constructInvocation(const BackendJobAction &job,
     case file_types::TY_SwiftOverlayFile:
     case file_types::TY_IndexUnitOutputPath:
     case file_types::TY_SwiftABIDescriptor:
+    case file_types::TY_SwiftAPIDescriptor:
     case file_types::TY_ConstValues:
     case file_types::TY_SwiftFixIt:
     case file_types::TY_ModuleSemanticInfo:
diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp
@@ -765,6 +765,11 @@ bool ArgsToFrontendOptionsConverter::checkUnusedSupplementaryOutputPaths()
     Diags.diagnose(SourceLoc(), diag::error_mode_cannot_emit_abi_descriptor);
     return true;
   }
+  if (!FrontendOptions::canActionEmitAPIDescriptor(Opts.RequestedAction) &&
+      Opts.InputsAndOutputs.hasAPIDescriptorOutputPath()) {
+    Diags.diagnose(SourceLoc(), diag::error_mode_cannot_emit_api_descriptor);
+    return true;
+  }
   if (!FrontendOptions::canActionEmitConstValues(Opts.RequestedAction) &&
       Opts.InputsAndOutputs.hasConstValuesOutputPath()) {
     Diags.diagnose(SourceLoc(), diag::error_mode_cannot_emit_const_values);
diff --git a/lib/Frontend/ArgsToFrontendOutputsConverter.cpp b/lib/Frontend/ArgsToFrontendOutputsConverter.cpp
@@ -331,6 +331,8 @@ SupplementaryOutputPathsComputer::getSupplementaryOutputPathsFromArguments()
       options::OPT_emit_module_summary_path);
   auto abiDescriptorOutput = getSupplementaryFilenamesFromArguments(
       options::OPT_emit_abi_descriptor_path);
+  auto apiDescriptorOutput = getSupplementaryFilenamesFromArguments(
+      options::OPT_emit_api_descriptor_path);
   auto constValuesOutput = getSupplementaryFilenamesFromArguments(
       options::OPT_emit_const_values_path);
   auto moduleSemanticInfoOutput = getSupplementaryFilenamesFromArguments(
@@ -365,6 +367,7 @@ SupplementaryOutputPathsComputer::getSupplementaryOutputPathsFromArguments()
     sop.ModuleSourceInfoOutputPath = (*moduleSourceInfoOutput)[i];
     sop.ModuleSummaryOutputPath = (*moduleSummaryOutput)[i];
     sop.ABIDescriptorOutputPath = (*abiDescriptorOutput)[i];
+    sop.APIDescriptorOutputPath = (*apiDescriptorOutput)[i];
     sop.ConstValuesOutputPath = (*constValuesOutput)[i];
     sop.ModuleSemanticInfoOutputPath = (*moduleSemanticInfoOutput)[i];
     sop.YAMLOptRecordPath = (*optRecordOutput)[i];
@@ -475,8 +478,13 @@ SupplementaryOutputPathsComputer::computeOutputPathsForOneInput(
 
   // There is no non-path form of -emit-abi-descriptor-path
   auto ABIDescriptorOutputPath = pathsFromArguments.ABIDescriptorOutputPath;
+
+  // There is no non-path form of -emit-api-descriptor-path
+  auto APIDescriptorOutputPath = pathsFromArguments.APIDescriptorOutputPath;
+
   // There is no non-path form of -emit-module-semantic-info-path
-  auto ModuleSemanticInfoOutputPath = pathsFromArguments.ModuleSemanticInfoOutputPath;
+  auto ModuleSemanticInfoOutputPath =
+      pathsFromArguments.ModuleSemanticInfoOutputPath;
 
   ID emitModuleOption;
   std::string moduleExtension;
@@ -513,6 +521,7 @@ SupplementaryOutputPathsComputer::computeOutputPathsForOneInput(
   sop.ModuleSourceInfoOutputPath = moduleSourceInfoOutputPath;
   sop.ModuleSummaryOutputPath = moduleSummaryOutputPath;
   sop.ABIDescriptorOutputPath = ABIDescriptorOutputPath;
+  sop.APIDescriptorOutputPath = APIDescriptorOutputPath;
   sop.ConstValuesOutputPath = constValuesOutputPath;
   sop.ModuleSemanticInfoOutputPath = ModuleSemanticInfoOutputPath;
   sop.YAMLOptRecordPath = YAMLOptRecordPath;
diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp
@@ -165,6 +165,14 @@ CompilerInvocation::getPrivateModuleInterfaceOutputPathForWholeModule() const {
       .SupplementaryOutputs.PrivateModuleInterfaceOutputPath;
 }
 
+std::string CompilerInvocation::getAPIDescriptorPathForWholeModule() const {
+  assert(
+      getFrontendOptions().InputsAndOutputs.isWholeModule() &&
+      "APIDescriptorPath only makes sense when the whole module can be seen");
+  return getPrimarySpecificPathsForAtMostOnePrimary()
+      .SupplementaryOutputs.APIDescriptorOutputPath;
+}
+
 SerializationOptions CompilerInvocation::computeSerializationOptions(
     const SupplementaryOutputPaths &outs, const ModuleDecl *module) const {
   const FrontendOptions &opts = getFrontendOptions();
diff --git a/lib/Frontend/FrontendInputsAndOutputs.cpp b/lib/Frontend/FrontendInputsAndOutputs.cpp
@@ -511,6 +511,12 @@ bool FrontendInputsAndOutputs::hasABIDescriptorOutputPath() const {
         return outs.ABIDescriptorOutputPath;
       });
 }
+bool FrontendInputsAndOutputs::hasAPIDescriptorOutputPath() const {
+  return hasSupplementaryOutputPath(
+      [](const SupplementaryOutputPaths &outs) -> const std::string & {
+        return outs.APIDescriptorOutputPath;
+      });
+}
 bool FrontendInputsAndOutputs::hasConstValuesOutputPath() const {
   return hasSupplementaryOutputPath(
       [](const SupplementaryOutputPaths &outs) -> const std::string & {
diff --git a/lib/Frontend/FrontendOptions.cpp b/lib/Frontend/FrontendOptions.cpp
@@ -743,6 +743,48 @@ bool FrontendOptions::canActionEmitInterface(ActionType action) {
   llvm_unreachable("unhandled action");
 }
 
+bool FrontendOptions::canActionEmitAPIDescriptor(ActionType action) {
+  switch (action) {
+  case ActionType::NoneAction:
+  case ActionType::Parse:
+  case ActionType::DumpParse:
+  case ActionType::DumpInterfaceHash:
+  case ActionType::DumpAST:
+  case ActionType::PrintAST:
+  case ActionType::PrintASTDecl:
+  case ActionType::EmitImportedModules:
+  case ActionType::EmitPCH:
+  case ActionType::DumpScopeMaps:
+  case ActionType::DumpTypeRefinementContexts:
+  case ActionType::DumpTypeInfo:
+  case ActionType::EmitSILGen:
+  case ActionType::EmitSIBGen:
+  case ActionType::CompileModuleFromInterface:
+  case ActionType::TypecheckModuleFromInterface:
+  case ActionType::Immediate:
+  case ActionType::REPL:
+  case ActionType::EmitPCM:
+  case ActionType::DumpPCM:
+  case ActionType::ScanDependencies:
+  case ActionType::PrintFeature:
+    return false;
+  case ActionType::ResolveImports:
+  case ActionType::Typecheck:
+  case ActionType::MergeModules:
+  case ActionType::EmitModuleOnly:
+  case ActionType::EmitSIL:
+  case ActionType::EmitSIB:
+  case ActionType::EmitIRGen:
+  case ActionType::EmitIR:
+  case ActionType::EmitBC:
+  case ActionType::EmitAssembly:
+  case ActionType::EmitObject:
+  case ActionType::PrintVersion:
+    return true;
+  }
+  llvm_unreachable("unhandled action");
+}
+
 bool FrontendOptions::doesActionProduceOutput(ActionType action) {
   switch (action) {
   case ActionType::Parse:
diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp
@@ -809,6 +809,36 @@ static bool writeTBDIfNeeded(CompilerInstance &Instance) {
                   Instance.getOutputBackend(), tbdOpts);
 }
 
+static bool writeAPIDescriptor(ModuleDecl *M, StringRef OutputPath,
+                               llvm::vfs::OutputBackend &Backend,
+                               bool PrettyPrinted) {
+  return withOutputPath(M->getDiags(), Backend, OutputPath,
+                        [&](raw_ostream &OS) -> bool {
+                          writeAPIJSONFile(M, OS, PrettyPrinted);
+                          return false;
+                        });
+}
+
+static bool writeAPIDescriptorIfNeeded(CompilerInstance &Instance) {
+  const auto &Invocation = Instance.getInvocation();
+  const auto &frontendOpts = Invocation.getFrontendOptions();
+  if (!frontendOpts.InputsAndOutputs.hasAPIDescriptorOutputPath())
+    return false;
+
+  if (!frontendOpts.InputsAndOutputs.isWholeModule()) {
+    Instance.getDiags().diagnose(
+        SourceLoc(), diag::api_descriptor_only_supported_in_whole_module);
+    return false;
+  }
+
+  const std::string &APIDescriptorPath =
+      Invocation.getAPIDescriptorPathForWholeModule();
+
+  // FIXME: Need a frontend flag for pretty printing
+  return writeAPIDescriptor(Instance.getMainModule(), APIDescriptorPath,
+                            Instance.getOutputBackend(), true);
+}
+
 static bool performCompileStepsPostSILGen(CompilerInstance &Instance,
                                           std::unique_ptr<SILModule> SM,
                                           ModuleOrSourceFile MSF,
@@ -975,6 +1005,10 @@ static bool emitAnyWholeModulePostTypeCheckSupplementaryOutputs(
     hadAnyError |= writeTBDIfNeeded(Instance);
   }
 
+  {
+    hadAnyError |= writeAPIDescriptorIfNeeded(Instance);
+  }
+
   {
     hadAnyError |= writeModuleSemanticInfoIfNeeded(Instance);
   }
diff --git a/test/Frontend/supplementary-output-support.swift b/test/Frontend/supplementary-output-support.swift
@@ -36,3 +36,6 @@
 
 // RUN: not %target-swift-frontend -parse -emit-const-values-path %t %s 2>&1 | %FileCheck -check-prefix=PARSE_NO_CONST_VALUES %s
 // PARSE_NO_CONST_VALUES: error: this mode does not support emitting extracted const values{{$}}
+
+// RUN: not %target-swift-frontend -parse -emit-api-descriptor-path %t %s 2>&1 | %FileCheck -check-prefix=PARSE_NO_API_DESCRIPTOR %s
+// PARSE_NO_API_DESCRIPTOR: error: this mode does not support emitting API descriptor files{{$}}
