[SourceKit] Update SyntaxInfo but with lazy parsing in "edit" request

Previously, if the client requests no information from "edit" request,
the syntax info wasn't updated. But "expand placeholder" request
requires the up-to-date syntax info.

rdar://77665805

Author: rintaro

test/SourceKit/CodeExpand/code-expand-rdar77665805.swift

@@ -0,0 +1,32 @@
+// BEGIN main.swift
+enum E { case foo, bar }
+func foo(x: (E) -> Void) {}
+func test() {
+  foo(x: <#T##(E) -> Void#>)
+}
+
+// BEGIN expand.json.in
+{
+    key.request: source.request.editor.expand_placeholder,
+    key.offset: 23,
+    key.length: 18,
+    key.name: "FILENAME"
+}
+
+// RUN: %empty-directory(%t)
+// RUN: %{python} %utils/split_file.py -o %t %s
+
+// RUN: sed "s#FILENAME#%t/main.swift#" %t/expand.json.in > %t/expand.json
+
+// RUN: %sourcekitd-test \
+// RUN:   -req=open %t/main.swift -- %t/main.swift == \
+// RUN:   -req=edit -offset=0 -length=53 -replace="" -req-opts=enablesyntaxmap=0,enablesubstructure=0,enablediagnostics=0 %t/main.swift -- %t/main.swift == \
+// RUN:   -json-request-path %t/expand.json \
+// RUN: | %FileCheck %s
+
+// CHECK: {
+// CHECK:   key.offset: 19,
+// CHECK:   key.length: 23,
+// CHECK:   key.sourcetext: " { <#E#> in\n<#code#>\n}"
+// CHECK: }
+

tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp

@@ -677,7 +677,8 @@ class SwiftDocumentSyntaxInfo {
   std::string PrimaryFile;
   /// Whether or not the AST stored in the source file is up-to-date or just an
   /// artifact of incremental syntax parsing
-  bool HasUpToDateAST;
+  bool IncrementalParsingEnabled;
+  bool IsParsed;
 
 public:
   SwiftDocumentSyntaxInfo(const CompilerInvocation &CompInv,
@@ -713,13 +714,16 @@ class SwiftDocumentSyntaxInfo {
         Parser->getParser().Context.evaluator);
     Parser->getDiagnosticEngine().addConsumer(DiagConsumer);
 
-    // If there is a syntax parsing cache, incremental syntax parsing is
-    // performed and thus the generated AST may not be up-to-date.
-    HasUpToDateAST = CompInv.getMainFileSyntaxParsingCache() == nullptr;
+    IncrementalParsingEnabled =
+        CompInv.getMainFileSyntaxParsingCache() != nullptr;
+    IsParsed = false;
   }
 
-  void parse() {
-    Parser->parse();
+  void parseIfNeeded() {
+    if (!IsParsed) {
+      Parser->parse();
+      IsParsed = true;
+    }
   }
 
   SourceFile &getSourceFile() {
@@ -738,7 +742,7 @@ class SwiftDocumentSyntaxInfo {
     return SM;
   }
 
-  bool hasUpToDateAST() { return HasUpToDateAST; }
+  bool isIncrementalParsingEnabled() { return IncrementalParsingEnabled; }
 
   ArrayRef<DiagnosticEntryInfo> getDiagnostics() {
     return DiagConsumer.getDiagnosticsForBuffer(BufferID);
@@ -1074,6 +1078,7 @@ struct SwiftEditorDocument::Implementation {
 
   std::shared_ptr<SwiftDocumentSyntaxInfo> getSyntaxInfo() {
     llvm::sys::ScopedLock L(AccessMtx);
+    SyntaxInfo->parseIfNeeded();
     return SyntaxInfo;
   }
 
@@ -1949,9 +1954,10 @@ void SwiftEditorDocument::updateSemaInfo() {
   ::updateSemaInfo(SemanticInfo, EditableBuffer);
 }
 
-void SwiftEditorDocument::parse(ImmutableTextSnapshotRef Snapshot,
-                                SwiftLangSupport &Lang, bool BuildSyntaxTree,
-                                SyntaxParsingCache *SyntaxCache) {
+void SwiftEditorDocument::resetSyntaxInfo(ImmutableTextSnapshotRef Snapshot,
+                                          SwiftLangSupport &Lang,
+                                          bool BuildSyntaxTree,
+                                          SyntaxParsingCache *SyntaxCache) {
   llvm::sys::ScopedLock L(Impl.AccessMtx);
 
   assert(Impl.SemanticInfo && "Impl.SemanticInfo must be set");
@@ -1982,15 +1988,14 @@ void SwiftEditorDocument::parse(ImmutableTextSnapshotRef Snapshot,
   // Access to Impl.SyntaxInfo is guarded by Impl.AccessMtx
   Impl.SyntaxInfo.reset(
     new SwiftDocumentSyntaxInfo(CompInv, Snapshot, Args, Impl.FilePath));
-
-  Impl.SyntaxInfo->parse();
 }
 
 static UIdent SemaDiagStage("source.diagnostic.stage.swift.sema");
 static UIdent ParseDiagStage("source.diagnostic.stage.swift.parse");
 
 void SwiftEditorDocument::readSyntaxInfo(EditorConsumer &Consumer, bool ReportDiags) {
   llvm::sys::ScopedLock L(Impl.AccessMtx);
+  Impl.SyntaxInfo->parseIfNeeded();
 
   Impl.ParserDiagnostics = Impl.SyntaxInfo->getDiagnostics();
   if (ReportDiags) {
@@ -2105,8 +2110,8 @@ SwiftEditorDocument::getSyntaxTree() const {
 
 std::string SwiftEditorDocument::getFilePath() const { return Impl.FilePath; }
 
-bool SwiftEditorDocument::hasUpToDateAST() const {
-  return Impl.SyntaxInfo->hasUpToDateAST();
+bool SwiftEditorDocument::isIncrementalParsingEnabled() const {
+  return Impl.SyntaxInfo->isIncrementalParsingEnabled();
 }
 
 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
@@ -2335,7 +2340,7 @@ void SwiftLangSupport::editorOpen(
     EditorDoc = new SwiftEditorDocument(Name, *this, fileSystem);
     Snapshot = EditorDoc->initializeText(
         Buf, Args, Consumer.needsSemanticInfo(), fileSystem);
-    EditorDoc->parse(Snapshot, *this, Consumer.syntaxTreeEnabled());
+    EditorDoc->resetSyntaxInfo(Snapshot, *this, Consumer.syntaxTreeEnabled());
     if (EditorDocuments->getOrUpdate(Name, *this, EditorDoc)) {
       // Document already exists, re-initialize it. This should only happen
       // if we get OPEN request while the previous document is not closed.
@@ -2349,7 +2354,7 @@ void SwiftLangSupport::editorOpen(
   if (!Snapshot) {
     Snapshot = EditorDoc->initializeText(
         Buf, Args, Consumer.needsSemanticInfo(), fileSystem);
-    EditorDoc->parse(Snapshot, *this, Consumer.syntaxTreeEnabled());
+    EditorDoc->resetSyntaxInfo(Snapshot, *this, Consumer.syntaxTreeEnabled());
   }
 
   if (Consumer.needsSemanticInfo()) {
@@ -2402,7 +2407,7 @@ void verifyIncrementalParse(SwiftEditorDocumentRef EditorDoc,
   SwiftDocumentSyntaxInfo ScratchSyntaxInfo(Invocation,
                                             EditorDoc->getLatestSnapshot(),
                                             Args, EditorDoc->getFilePath());
-  ScratchSyntaxInfo.parse();
+  ScratchSyntaxInfo.parseIfNeeded();
 
   // Dump the from-scratch syntax tree
   std::string FromScratchTreeString;
@@ -2507,19 +2512,22 @@ void SwiftLangSupport::editorReplaceText(StringRef Name,
     }
 
     // If client doesn't need any information, we doen't need to parse it.
+
+
+    SyntaxParsingCache *SyntaxCachePtr = nullptr;
+    if (SyntaxCache.hasValue()) {
+      SyntaxCachePtr = SyntaxCache.getPointer();
+    }
+    EditorDoc->resetSyntaxInfo(Snapshot, *this, Consumer.syntaxTreeEnabled(),
+                               SyntaxCachePtr);
+
     if (!Consumer.documentStructureEnabled() &&
         !Consumer.syntaxMapEnabled() &&
         !Consumer.diagnosticsEnabled() &&
         !Consumer.syntaxTreeEnabled()) {
       return;
     }
 
-    SyntaxParsingCache *SyntaxCachePtr = nullptr;
-    if (SyntaxCache.hasValue()) {
-      SyntaxCachePtr = SyntaxCache.getPointer();
-    }
-    EditorDoc->parse(Snapshot, *this, Consumer.syntaxTreeEnabled(),
-                     SyntaxCachePtr);
     // Do not report syntactic diagnostics; will be handled in readSemanticInfo.
     EditorDoc->readSyntaxInfo(Consumer, /*ReportDiags=*/false);
 
@@ -2578,11 +2586,12 @@ void SwiftLangSupport::editorFormatText(StringRef Name, unsigned Line,
     return;
   }
 
-  if (!EditorDoc->hasUpToDateAST()) {
-    // An up-to-date AST is needed for formatting. If it does not exist, fall
-    // back to a full reparse of the file
-    EditorDoc->parse(EditorDoc->getLatestSnapshot(), *this,
-                     /*BuildSyntaxTree=*/true);
+  if (EditorDoc->isIncrementalParsingEnabled()) {
+    // If incremental parsing is enabled, AST is not updated properly. Fall
+    // back to a full reparse of the file.
+    EditorDoc->resetSyntaxInfo(EditorDoc->getLatestSnapshot(), *this,
+                               /*BuildSyntaxTree=*/true,
+                               /*SyntaxCache=*/nullptr);
   }
 
   EditorDoc->formatText(Line, Length, Consumer);
@@ -2616,5 +2625,13 @@ void SwiftLangSupport::editorExpandPlaceholder(StringRef Name, unsigned Offset,
     return;
   }
 
+  if (EditorDoc->isIncrementalParsingEnabled()) {
+    // If incremental parsing is enabled, AST is not updated properly. Fall
+    // back to a full reparse of the file.
+    EditorDoc->resetSyntaxInfo(EditorDoc->getLatestSnapshot(), *this,
+                               /*BuildSyntaxTree=*/true,
+                               /*SyntaxCache=*/nullptr);
+  }
+
   EditorDoc->expandPlaceholder(Offset, Length, Consumer);
 }

tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h

@@ -107,9 +107,9 @@ class SwiftEditorDocument :
   ImmutableTextSnapshotRef getLatestSnapshot() const;
   std::pair<unsigned, unsigned> getLineAndColumnInBuffer(unsigned Offset);
 
-  void parse(ImmutableTextSnapshotRef Snapshot, SwiftLangSupport &Lang,
-             bool BuildSyntaxTree,
-             swift::SyntaxParsingCache *SyntaxCache = nullptr);
+  void resetSyntaxInfo(ImmutableTextSnapshotRef Snapshot,
+                       SwiftLangSupport &Lang, bool BuildSyntaxTree,
+                       swift::SyntaxParsingCache *SyntaxCache = nullptr);
   void readSyntaxInfo(EditorConsumer &consumer, bool ReportDiags);
   void readSemanticInfo(ImmutableTextSnapshotRef Snapshot,
                         EditorConsumer& Consumer);
@@ -129,7 +129,7 @@ class SwiftEditorDocument :
 
   /// Whether or not the AST stored for this document is up-to-date or just an
   /// artifact of incremental syntax parsing
-  bool hasUpToDateAST() const;
+  bool isIncrementalParsingEnabled() const;
 
   /// Returns the virtual filesystem associated with this document.
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getFileSystem() const;