Remove Explicit Trivia from Lexer

Author: CodaFi

include/swift/Parse/Lexer.h

@@ -22,7 +22,6 @@
 #include "swift/Basic/SourceManager.h"
 #include "swift/Parse/LexerState.h"
 #include "swift/Parse/Token.h"
-#include "swift/Parse/ParsedTrivia.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/SaveAndRestore.h"
 
@@ -45,11 +44,6 @@ enum class CommentRetentionMode {
   ReturnAsTokens,
 };
 
-enum class TriviaRetentionMode {
-  WithoutTrivia,
-  WithTrivia,
-};
-
 enum class HashbangMode : bool {
   Disallowed,
   Allowed,
@@ -131,21 +125,10 @@ class Lexer {
 
   const CommentRetentionMode RetainComments;
 
-  const TriviaRetentionMode TriviaRetention;
-
   /// InSILBody - This is true when we're lexing the body of a SIL declaration
   /// in a SIL file.  This enables some context-sensitive lexing.
   bool InSILBody = false;
 
-  /// The current leading trivia for the next token.
-  ///
-  /// The StringRef points into the source buffer that is currently being lexed.
-  StringRef LeadingTrivia;
-
-  /// The current trailing trivia for the next token.
-  /// The StringRef points into the source buffer that is currently being lexed.
-  StringRef TrailingTrivia;
-
   /// The location at which the comment of the next token starts. \c nullptr if
   /// the next token doesn't have a comment.
   const char *CommentStart;
@@ -166,8 +149,8 @@ class Lexer {
   Lexer(const PrincipalTag &, const LangOptions &LangOpts,
         const SourceManager &SourceMgr, unsigned BufferID,
         DiagnosticEngine *Diags, LexerMode LexMode,
-        HashbangMode HashbangAllowed, CommentRetentionMode RetainComments,
-        TriviaRetentionMode TriviaRetention);
+        HashbangMode HashbangAllowed,
+        CommentRetentionMode RetainComments);
 
   void initialize(unsigned Offset, unsigned EndOffset);
 
@@ -202,14 +185,13 @@ class Lexer {
       const LangOptions &Options, const SourceManager &SourceMgr,
       unsigned BufferID, DiagnosticEngine *Diags, LexerMode LexMode,
       HashbangMode HashbangAllowed = HashbangMode::Disallowed,
-      CommentRetentionMode RetainComments = CommentRetentionMode::None,
-      TriviaRetentionMode TriviaRetention = TriviaRetentionMode::WithoutTrivia);
+      CommentRetentionMode RetainComments = CommentRetentionMode::None);
 
   /// Create a lexer that scans a subrange of the source buffer.
   Lexer(const LangOptions &Options, const SourceManager &SourceMgr,
         unsigned BufferID, DiagnosticEngine *Diags, LexerMode LexMode,
         HashbangMode HashbangAllowed, CommentRetentionMode RetainComments,
-        TriviaRetentionMode TriviaRetention, unsigned Offset,
+        unsigned Offset,
         unsigned EndOffset);
 
   /// Create a sub-lexer that lexes from the same buffer, but scans
@@ -233,15 +215,9 @@ class Lexer {
     return LexMode == LexerMode::SwiftInterface;
   }
 
-  /// Lex a token. If \c TriviaRetentionMode is \c WithTrivia, passed pointers
-  /// to trivia are populated.
-  void lex(Token &Result, StringRef &LeadingTriviaResult,
-           StringRef &TrailingTriviaResult) {
+  /// Lex a token.
+  void lex(Token &Result) {
     Result = NextToken;
-    if (TriviaRetention == TriviaRetentionMode::WithTrivia) {
-      LeadingTriviaResult = LeadingTrivia;
-      TrailingTriviaResult = TrailingTrivia;
-    }
     // Emit any diagnostics recorded for this token.
     if (DiagQueue)
       DiagQueue->emit();
@@ -250,11 +226,6 @@ class Lexer {
       lexImpl();
   }
 
-  void lex(Token &Result) {
-    StringRef LeadingTrivia, TrailingTrivia;
-    lex(Result, LeadingTrivia, TrailingTrivia);
-  }
-
   /// Reset the lexer's buffer pointer to \p Offset bytes after the buffer
   /// start.
   void resetToOffset(size_t Offset) {
@@ -304,8 +275,7 @@ class Lexer {
   /// Returns the lexer state for the beginning of the given token.
   /// After restoring the state, lexer will return this token and continue from
   /// there.
-  State getStateForBeginningOfToken(const Token &Tok,
-                                    const StringRef &LeadingTrivia = {}) const {
+  State getStateForBeginningOfToken(const Token &Tok) const {
 
     // If the token has a comment attached to it, rewind to before the comment,
     // not just the start of the token.  This ensures that we will re-lex and
@@ -314,11 +284,6 @@ class Lexer {
     if (TokStart.isInvalid())
       TokStart = Tok.getLoc();
     auto S = getStateForBeginningOfTokenLoc(TokStart);
-    if (TriviaRetention == TriviaRetentionMode::WithTrivia) {
-      S.LeadingTrivia = LeadingTrivia;
-    } else {
-      S.LeadingTrivia = StringRef();
-    }
     return S;
   }
 
@@ -340,10 +305,6 @@ class Lexer {
     // Don't re-emit diagnostics from readvancing the lexer.
     if (DiagQueue && !enableDiagnostics)
       DiagQueue->clear();
-
-    // Restore Trivia.
-    if (TriviaRetention == TriviaRetentionMode::WithTrivia)
-      LeadingTrivia = S.LeadingTrivia;
   }
 
   /// Restore the lexer state to a given state that is located before
@@ -681,13 +642,6 @@ class Lexer {
 
 };
 
-/// A lexer that can lex trivia into its pieces
-class TriviaLexer {
-public:
-  /// Decompose the trivia in \p TriviaStr into their pieces.
-  static ParsedTrivia lexTrivia(StringRef TriviaStr);
-};
-
 /// Given an ordered token \param Array , get the iterator pointing to the first
 /// token that is not before \param Loc .
 template<typename ArrayTy, typename Iterator = typename ArrayTy::iterator>

include/swift/Parse/LexerState.h

@@ -19,7 +19,6 @@
 
 #include "llvm/ADT/Optional.h"
 #include "swift/Basic/SourceLoc.h"
-#include "swift/Parse/ParsedTrivia.h"
 
 namespace swift {
 class Lexer;
@@ -39,7 +38,6 @@ class LexerState {
 private:
   explicit LexerState(SourceLoc Loc) : Loc(Loc) {}
   SourceLoc Loc;
-  StringRef LeadingTrivia;
   friend class Lexer;
 };
 

lib/Sema/TypeCheckStmt.cpp

@@ -43,7 +43,6 @@
 #include "swift/Parse/LocalContext.h"
 #include "swift/Parse/Parser.h"
 #include "swift/Sema/IDETypeChecking.h"
-#include "swift/Syntax/TokenKinds.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/SmallString.h"

unittests/Parse/CMakeLists.txt

@@ -1,7 +1,6 @@
 add_swift_unittest(SwiftParseTests
   BuildConfigTests.cpp
   LexerTests.cpp
-  LexerTriviaTests.cpp
   TokenizerTests.cpp
 )