Merge pull request #36352 from ahoppen/pr/dont-cache-tokens

[libSyntax] Don't cache token nodes

Author: ahoppen

include/swift/SyntaxParse/SyntaxTreeCreator.h

@@ -18,7 +18,6 @@
 #include "llvm/ADT/StringRef.h"
 
 namespace swift {
-  class RawSyntaxTokenCache;
   class SourceManager;
   class SyntaxParsingCache;
   class SourceFile;
@@ -49,10 +48,6 @@ class SyntaxTreeCreator: public SyntaxParseActions {
   /// tree.
   SyntaxParsingCache *SyntaxCache;
 
-  /// Tokens nodes that have already been created and may be reused in other
-  /// parts of the syntax tree.
-  std::unique_ptr<RawSyntaxTokenCache> TokenCache;
-
 public:
   SyntaxTreeCreator(SourceManager &SM, unsigned bufferID,
                     SyntaxParsingCache *syntaxCache,

lib/SyntaxParse/CMakeLists.txt

@@ -1,6 +1,5 @@
 set_swift_llvm_is_available()
 add_swift_host_library(swiftSyntaxParse STATIC
-  RawSyntaxTokenCache.cpp
   SyntaxTreeCreator.cpp)
 target_link_libraries(swiftSyntaxParse PRIVATE
   swiftParse

lib/SyntaxParse/RawSyntaxTokenCache.cpp

@@ -22,18 +22,15 @@
 #include "swift/AST/Module.h"
 #include "swift/AST/SourceFile.h"
 #include "swift/Basic/OwnedString.h"
-#include "RawSyntaxTokenCache.h"
 
 using namespace swift;
 using namespace swift::syntax;
 
 SyntaxTreeCreator::SyntaxTreeCreator(SourceManager &SM, unsigned bufferID,
                                      SyntaxParsingCache *syntaxCache,
                                      RC<syntax::SyntaxArena> arena)
-    : SM(SM), BufferID(bufferID),
-      Arena(std::move(arena)),
-      SyntaxCache(syntaxCache),
-      TokenCache(new RawSyntaxTokenCache()) {
+    : SM(SM), BufferID(bufferID), Arena(std::move(arena)),
+      SyntaxCache(syntaxCache) {
   StringRef BufferContent = SM.getEntireTextForBuffer(BufferID);
   const char *Data = BufferContent.data();
   Arena->copyStringToArenaIfNecessary(Data, BufferContent.size());
@@ -126,9 +123,9 @@ OpaqueSyntaxNode SyntaxTreeCreator::recordToken(tok tokenKind,
   StringRef trailingTriviaText = ArenaSourceBuffer.substr(
       trailingTriviaStartOffset, trailingTrivia.size());
 
-  auto raw =
-      TokenCache->getToken(Arena, tokenKind, range.getByteLength(), tokenText,
-                           leadingTriviaText, trailingTriviaText);
+  auto raw = RawSyntax::make(tokenKind, tokenText, range.getByteLength(),
+                             leadingTriviaText, trailingTriviaText,
+                             SourcePresence::Present, Arena);
   return static_cast<OpaqueSyntaxNode>(raw);
 }
 

lib/SyntaxParse/RawSyntaxTokenCache.h

@@ -1,9 +1,9 @@
 {
-  "id": 17,
+  "id": 19,
   "kind": "SourceFile",
   "layout": [
     {
-      "id": 16,
+      "id": 18,
       "kind": "CodeBlockItemList",
       "layout": [
         {
@@ -78,11 +78,11 @@
           "presence": "Present"
         },
         {
-          "id": 14,
+          "id": 16,
           "kind": "CodeBlockItem",
           "layout": [
             {
-              "id": 13,
+              "id": 15,
               "kind": "StructDecl",
               "layout": [
                 null,
@@ -110,11 +110,11 @@
                 null,
                 null,
                 {
-                  "id": 12,
+                  "id": 14,
                   "kind": "MemberDeclBlock",
                   "layout": [
                     {
-                      "id": 3,
+                      "id": 11,
                       "tokenKind": {
                         "kind": "l_brace"
                       },
@@ -123,13 +123,13 @@
                       "presence": "Present"
                     },
                     {
-                      "id": 11,
+                      "id": 12,
                       "kind": "MemberDeclList",
                       "layout": [],
                       "presence": "Present"
                     },
                     {
-                      "id": 5,
+                      "id": 13,
                       "tokenKind": {
                         "kind": "r_brace"
                       },
@@ -152,7 +152,7 @@
       "presence": "Present"
     },
     {
-      "id": 15,
+      "id": 17,
       "tokenKind": {
         "kind": "eof",
         "text": ""