[Parse] Introduce /.../ regex literals

Start parsing regex literals with `/.../`
delimiters.

rdar://83253726

Author: hamishknight

include/swift/AST/DiagnosticsParse.def

@@ -143,7 +143,10 @@ ERROR(lex_invalid_escape_delimiter,none,
 ERROR(lex_invalid_closing_delimiter,none,
       "too many '#' characters in closing delimiter", ())
 
-ERROR(lex_unterminated_regex,none,
+ERROR(lex_regex_literal_invalid_starting_char,none,
+      "regex literal may not start with %0; add backslash to escape",
+      (StringRef))
+ERROR(lex_regex_literal_unterminated,none,
       "unterminated regex literal", ())
 
 ERROR(lex_invalid_unicode_scalar,none,

include/swift/Parse/Lexer.h

@@ -61,6 +61,16 @@ enum class LexerMode {
   SIL
 };
 
+/// Whether or not the lexer should attempt to lex a `/.../` regex literal.
+enum class LexerForwardSlashRegexMode {
+  /// No `/.../` regex literals will be lexed.
+  None,
+  /// A `/.../` regex literal will be lexed, but only if successful.
+  Tentative,
+  /// A `/.../` regex literal will always be lexed for a '/' character.
+  Always
+};
+
 /// Kinds of conflict marker which the lexer might encounter.
 enum class ConflictMarkerKind {
   /// A normal or diff3 conflict marker, initiated by at least 7 "<"s,
@@ -112,6 +122,10 @@ class Lexer {
   /// a .sil file.
   const LexerMode LexMode;
 
+  /// Whether or not a `/.../` literal will be lexed.
+  LexerForwardSlashRegexMode ForwardSlashRegexMode =
+      LexerForwardSlashRegexMode::None;
+
   /// True if we should skip past a `#!` line at the start of the file.
   const bool IsHashbangAllowed;
 
@@ -551,6 +565,11 @@ class Lexer {
     void operator=(const SILBodyRAII&) = delete;
   };
 
+  /// Attempt to re-lex a regex literal with forward slashes `/.../` from a
+  /// given lexing state. If \p mustBeRegex is set to true, a regex literal will
+  /// always be lexed. Otherwise, it will not be lexed if it may be ambiguous.
+  void tryLexForwardSlashRegexLiteralFrom(State S, bool mustBeRegex);
+
 private:
   /// Nul character meaning kind.
   enum class NulCharacterKind {
@@ -615,8 +634,8 @@ class Lexer {
   void lexStringLiteral(unsigned CustomDelimiterLen = 0);
   void lexEscapedIdentifier();
 
-  /// Attempt to lex a regex literal, returning true if a regex literal was
-  /// lexed, false if this is not a regex literal.
+  /// Attempt to lex a regex literal, returning true if lexing should continue,
+  /// false if this is not a regex literal.
   bool tryLexRegexLiteral(const char *TokStart);
 
   void tryLexEditorPlaceholder();

include/swift/Parse/Parser.h

@@ -559,6 +559,11 @@ class Parser {
     return f(backtrackScope);
   }
 
+  /// Discard the current token. This will avoid interface hashing or updating
+  /// the previous loc. Only should be used if you've completely re-lexed
+  /// a different token at that position.
+  SourceLoc discardToken();
+
   /// Consume a token that we created on the fly to correct the original token
   /// stream from lexer.
   void consumeExtraToken(Token K);
@@ -1752,8 +1757,17 @@ class Parser {
   ParserResult<Expr>
   parseExprPoundCodeCompletion(Optional<StmtKind> ParentKind);
 
+  UnresolvedDeclRefExpr *makeExprOperator(const Token &opToken);
   UnresolvedDeclRefExpr *parseExprOperator();
 
+  /// Try re-lex a '/' operator character as a regex literal. This should be
+  /// called when parsing in an expression position to ensure a regex literal is
+  /// correctly parsed.
+  ///
+  /// If \p mustBeRegex is set to true, a regex literal will always be lexed if
+  /// enabled. Otherwise, it will not be lexed if it may be ambiguous.
+  void tryLexRegexLiteral(bool mustBeRegex);
+
   void validateCollectionElement(ParserResult<Expr> element);
 
   //===--------------------------------------------------------------------===//

lib/Parse/Lexer.cpp

@@ -1969,22 +1969,109 @@ bool Lexer::tryLexRegexLiteral(const char *TokStart) {
   if (!LangOpts.EnableExperimentalStringProcessing || !regexLiteralLexingFn)
     return false;
 
-  // Ask libswift to try and lex a regex literal.
+  bool MustBeRegex = true;
+  bool IsForwardSlash = (*TokStart == '/');
+
+  // Check if we're able to lex a `/.../` regex.
+  if (IsForwardSlash) {
+    switch (ForwardSlashRegexMode) {
+    case LexerForwardSlashRegexMode::None:
+      return false;
+    case LexerForwardSlashRegexMode::Tentative:
+      MustBeRegex = false;
+      break;
+    case LexerForwardSlashRegexMode::Always:
+      break;
+    }
+
+    // For `/.../` regex literals, we need to ban space and tab at the start of
+    // a regex to avoid ambiguity with operator chains, e.g:
+    //
+    // Builder {
+    //   0
+    //   / 1 /
+    //   2
+    // }
+    //
+    // This takes advantage of the consistent operator spacing rule. We also
+    // need to ban ')' to avoid ambiguity with unapplied operator references e.g
+    // `reduce(1, /)`. This would be invalid regex syntax anyways. Note this
+    // doesn't totally save us from e.g `foo(/, 0)`, but it should at least
+    // help, and it ensures users can always surround their operator ref in
+    // parens `(/)` to fix the issue.
+    // TODO: This heuristic should be sunk into the Swift library once we have a
+    // way of doing fix-its from there.
+    auto *RegexContentStart = TokStart + 1;
+    switch (*RegexContentStart) {
+    case ')': {
+      if (!MustBeRegex)
+        return false;
+
+      // ')' is invalid anyway, so we can let the parser diagnose it.
+      break;
+    }
+    case ' ':
+    case '\t': {
+      if (!MustBeRegex)
+        return false;
+
+      // We must have a regex, so emit an error for space and tab.
+      StringRef DiagChar;
+      switch (*RegexContentStart) {
+      case ' ':
+        DiagChar = "space";
+        break;
+      case '\t':
+        DiagChar = "tab";
+        break;
+      default:
+        llvm_unreachable("Unhandled case");
+      }
+      diagnose(RegexContentStart, diag::lex_regex_literal_invalid_starting_char,
+               DiagChar)
+          .fixItInsert(getSourceLoc(RegexContentStart), "\\");
+      break;
+    }
+    default:
+      break;
+    }
+  }
+
+  // Ask the Swift library to try and lex a regex literal.
   // - Ptr will not be advanced if this is not for a regex literal.
   // - ErrStr will be set if there is any error to emit.
   // - CompletelyErroneous will be set if there was an error that cannot be
   //   recovered from.
   auto *Ptr = TokStart;
   const char *ErrStr = nullptr;
   bool CompletelyErroneous = regexLiteralLexingFn(&Ptr, BufferEnd, &ErrStr);
-  if (ErrStr)
-    diagnose(TokStart, diag::regex_literal_parsing_error, ErrStr);
 
   // If we didn't make any lexing progress, this isn't a regex literal and we
   // should fallback to lexing as something else.
   if (Ptr == TokStart)
     return false;
 
+  if (ErrStr) {
+    if (!MustBeRegex)
+      return false;
+
+    diagnose(TokStart, diag::regex_literal_parsing_error, ErrStr);
+  }
+
+  // If we're lexing `/.../`, error if we ended on the opening of a comment.
+  // We prefer to lex the comment as it's more likely than not that is what
+  // the user is expecting.
+  // TODO: This should be sunk into the Swift library.
+  if (IsForwardSlash && Ptr[-1] == '/' && (*Ptr == '*' || *Ptr == '/')) {
+    if (!MustBeRegex)
+      return false;
+
+    diagnose(TokStart, diag::lex_regex_literal_unterminated);
+
+    // Move the pointer back to the '/' of the comment.
+    Ptr--;
+  }
+
   // Update to point to where we ended regex lexing.
   assert(Ptr > TokStart && Ptr <= BufferEnd);
   CurPtr = Ptr;
@@ -1996,12 +2083,23 @@ bool Lexer::tryLexRegexLiteral(const char *TokStart) {
     return true;
   }
 
-  // Otherwise, we either had a successful lex, or something that was
-  // recoverable.
+  // We either had a successful lex, or something that was recoverable.
   formToken(tok::regex_literal, TokStart);
   return true;
 }
 
+void Lexer::tryLexForwardSlashRegexLiteralFrom(State S, bool mustBeRegex) {
+  if (!LangOpts.EnableBareSlashRegexLiterals)
+    return;
+
+  // Try re-lex with forward slash enabled.
+  llvm::SaveAndRestore<LexerForwardSlashRegexMode> RegexLexingScope(
+      ForwardSlashRegexMode, mustBeRegex
+                                 ? LexerForwardSlashRegexMode::Always
+                                 : LexerForwardSlashRegexMode::Tentative);
+  restoreState(S, /*enableDiagnostics*/ true);
+}
+
 /// lexEscapedIdentifier:
 ///   identifier ::= '`' identifier '`'
 ///
@@ -2483,8 +2581,7 @@ void Lexer::lexImpl() {
     if (unsigned CustomDelimiterLen = advanceIfCustomDelimiter(CurPtr, Diags))
       return lexStringLiteral(CustomDelimiterLen);
 
-    // If we have experimental string processing enabled, try lex a regex
-    // literal.
+    // Try lex a regex literal.
     if (tryLexRegexLiteral(TokStart))
       return;
 
@@ -2505,6 +2602,10 @@ void Lexer::lexImpl() {
              "Non token comment should be eaten by lexTrivia as LeadingTrivia");
       return formToken(tok::comment, TokStart);
     }
+    // Try lex a regex literal.
+    if (tryLexRegexLiteral(TokStart))
+      return;
+
     return lexOperatorIdentifier();
   case '%':
     // Lex %[0-9a-zA-Z_]+ as a local SIL value

lib/Parse/ParseExpr.cpp

@@ -511,6 +511,10 @@ ParserResult<Expr> Parser::parseExprSequenceElement(Diag<> message,
 ParserResult<Expr> Parser::parseExprUnary(Diag<> Message, bool isExprBasic) {
   SyntaxParsingContext UnaryContext(SyntaxContext, SyntaxContextKind::Expr);
   UnresolvedDeclRefExpr *Operator;
+
+  // First check to see if we have the start of a regex literal `/.../`.
+  tryLexRegexLiteral(/*mustBeRegex*/ true);
+
   switch (Tok.getKind()) {
   default:
     // If the next token is not an operator, just parse this as expr-postfix.
@@ -532,16 +536,32 @@ ParserResult<Expr> Parser::parseExprUnary(Diag<> Message, bool isExprBasic) {
   case tok::backslash:
     return parseExprKeyPath();
 
-  case tok::oper_postfix:
+  case tok::oper_postfix: {
     // Postfix operators cannot start a subexpression, but can happen
     // syntactically because the operator may just follow whatever precedes this
     // expression (and that may not always be an expression).
     diagnose(Tok, diag::invalid_postfix_operator);
     Tok.setKind(tok::oper_prefix);
-    LLVM_FALLTHROUGH;
-  case tok::oper_prefix:
     Operator = parseExprOperator();
     break;
+  }
+  case tok::oper_prefix: {
+    // Check to see if we can split a prefix operator containing `/`, e.g `!/`,
+    // which might be a prefix operator on a regex literal.
+    if (Context.LangOpts.EnableBareSlashRegexLiterals) {
+      auto slashIdx = Tok.getText().find("/");
+      if (slashIdx != StringRef::npos) {
+        auto prefix = Tok.getText().take_front(slashIdx);
+        if (!prefix.empty()) {
+          Operator = makeExprOperator({Tok.getKind(), prefix});
+          consumeStartingCharacterOfCurrentToken(Tok.getKind(), prefix.size());
+          break;
+        }
+      }
+    }
+    Operator = parseExprOperator();
+    break;
+  }
   case tok::oper_binary_spaced:
   case tok::oper_binary_unspaced: {
     // For recovery purposes, accept an oper_binary here.
@@ -860,19 +880,52 @@ static DeclRefKind getDeclRefKindForOperator(tok kind) {
   }
 }
 
-/// parseExprOperator - Parse an operator reference expression.  These
-/// are not "proper" expressions; they can only appear in binary/unary
-/// operators.
-UnresolvedDeclRefExpr *Parser::parseExprOperator() {
+UnresolvedDeclRefExpr *Parser::makeExprOperator(const Token &Tok) {
   assert(Tok.isAnyOperator());
   DeclRefKind refKind = getDeclRefKindForOperator(Tok.getKind());
   SourceLoc loc = Tok.getLoc();
   DeclNameRef name(Context.getIdentifier(Tok.getText()));
-  consumeToken();
   // Bypass local lookup.
   return new (Context) UnresolvedDeclRefExpr(name, refKind, DeclNameLoc(loc));
 }
 
+/// parseExprOperator - Parse an operator reference expression.  These
+/// are not "proper" expressions; they can only appear in binary/unary
+/// operators.
+UnresolvedDeclRefExpr *Parser::parseExprOperator() {
+  auto *op = makeExprOperator(Tok);
+  consumeToken();
+  return op;
+}
+
+void Parser::tryLexRegexLiteral(bool mustBeRegex) {
+  if (!Context.LangOpts.EnableBareSlashRegexLiterals)
+    return;
+
+  // Check to see if we have the start of a regex literal `/.../`.
+  switch (Tok.getKind()) {
+  case tok::oper_prefix:
+  case tok::oper_binary_spaced:
+  case tok::oper_binary_unspaced: {
+    if (!Tok.getText().startswith("/"))
+      break;
+
+    // Try re-lex as a `/.../` regex literal.
+    auto state = getParserPosition().LS;
+    L->tryLexForwardSlashRegexLiteralFrom(state, mustBeRegex);
+
+    // Discard the current token, which will be replaced by the re-lexed token,
+    // which may or may not be a regex literal token.
+    discardToken();
+
+    assert(Tok.getText().startswith("/"));
+    break;
+  }
+  default:
+    break;
+  }
+}
+
 /// parseExprSuper
 ///
 ///   expr-super:
@@ -3160,6 +3213,11 @@ ParserStatus Parser::parseExprList(tok leftTok, tok rightTok,
     SourceLoc FieldNameLoc;
     parseOptionalArgumentLabel(FieldName, FieldNameLoc);
 
+    // First check to see if we have the start of a regex literal `/.../`. We
+    // need to do this before handling unapplied operator references, as e.g
+    // `(/, /)` might be a regex literal.
+    tryLexRegexLiteral(/*mustBeRegex*/ false);
+
     // See if we have an operator decl ref '(<op>)'. The operator token in
     // this case lexes as a binary operator because it neither leads nor
     // follows a proper subexpression.

lib/Parse/Parser.cpp

@@ -579,13 +579,16 @@ const Token &Parser::peekToken() {
   return L->peekNextToken();
 }
 
-SourceLoc Parser::consumeTokenWithoutFeedingReceiver() {
-  SourceLoc Loc = Tok.getLoc();
+SourceLoc Parser::discardToken() {
   assert(Tok.isNot(tok::eof) && "Lexing past eof!");
+  SourceLoc Loc = Tok.getLoc();
+  L->lex(Tok, LeadingTrivia, TrailingTrivia);
+  return Loc;
+}
 
+SourceLoc Parser::consumeTokenWithoutFeedingReceiver() {
   recordTokenHash(Tok);
-
-  L->lex(Tok, LeadingTrivia, TrailingTrivia);
+  auto Loc = discardToken();
   PreviousLoc = Loc;
   return Loc;
 }

test/IDE/complete_regex.swift

@@ -4,7 +4,7 @@
 // RUN: %target-swift-ide-test -enable-bare-slash-regex -batch-code-completion -source-filename %s -filecheck %raw-FileCheck -completion-output-dir %t
 
 func testLiteral() {
-  #/foo/#.#^RE_LITERAL_MEMBER^#
+  /foo/.#^RE_LITERAL_MEMBER^#
 // RE_LITERAL_MEMBER: Begin completions
 // RE_LITERAL_MEMBER-DAG: Keyword[self]/CurrNominal:          self[#Regex<Substring>#];
 // RE_LITERAL_MEMBER: End completions