[Parse] Lenient prefix / parsing

Treat a prefix operator containing `/` the same as
the unapplied infix operator case, where we
tentatively lex. This means that we bail if there
is no closing `/` or the starting character is
invalid. This leaves binary operator containing
`/` in expression position as the last place where
we know that we definitely have a regex literal.

Author: hamishknight

include/swift/Parse/Parser.h

@@ -1763,10 +1763,7 @@ class Parser {
   /// Try re-lex a '/' operator character as a regex literal. This should be
   /// called when parsing in an expression position to ensure a regex literal is
   /// correctly parsed.
-  ///
-  /// If \p mustBeRegex is set to true, a regex literal will always be lexed if
-  /// enabled. Otherwise, it will not be lexed if it may be ambiguous.
-  void tryLexRegexLiteral(bool mustBeRegex);
+  void tryLexRegexLiteral(bool forUnappliedOperator);
 
   void validateCollectionElement(ParserResult<Expr> element);
 

lib/Parse/ParseExpr.cpp

@@ -513,7 +513,7 @@ ParserResult<Expr> Parser::parseExprUnary(Diag<> Message, bool isExprBasic) {
   UnresolvedDeclRefExpr *Operator;
 
   // First check to see if we have the start of a regex literal `/.../`.
-  tryLexRegexLiteral(/*mustBeRegex*/ true);
+  tryLexRegexLiteral(/*forUnappliedOperator*/ false);
 
   switch (Tok.getKind()) {
   default:
@@ -880,56 +880,64 @@ UnresolvedDeclRefExpr *Parser::parseExprOperator() {
   return new (Context) UnresolvedDeclRefExpr(name, refKind, DeclNameLoc(loc));
 }
 
-void Parser::tryLexRegexLiteral(bool mustBeRegex) {
+void Parser::tryLexRegexLiteral(bool forUnappliedOperator) {
   if (!Context.LangOpts.EnableBareSlashRegexLiterals)
     return;
 
   // Check to see if we have a regex literal `/.../`, optionally with a prefix
   // operator e.g `!/.../`.
+  bool mustBeRegex = false;
   switch (Tok.getKind()) {
   case tok::oper_prefix:
+    // Prefix operators may contain `/` characters, so this may not be a regex,
+    // and as such need to make sure we have a closing `/`.
+    break;
   case tok::oper_binary_spaced:
-  case tok::oper_binary_unspaced: {
-    // Check to see if we have an operator containing '/'.
-    auto slashIdx = Tok.getText().find("/");
-    if (slashIdx == StringRef::npos)
-      break;
+  case tok::oper_binary_unspaced:
+    // When re-lexing for a unary expression, binary operators are always
+    // invalid, so we can be confident in always lexing a regex literal.
+    mustBeRegex = !forUnappliedOperator;
+    break;
+  default:
+    // We only re-lex regex literals for operator tokens.
+    return;
+  }
 
-    CancellableBacktrackingScope backtrack(*this);
-    {
-      Optional<Lexer::ForwardSlashRegexRAII> regexScope;
-      regexScope.emplace(*L, mustBeRegex);
-
-      // Try re-lex as a `/.../` regex literal, this will split an operator if
-      // necessary.
-      L->restoreState(getParserPosition().LS, /*enableDiagnostics*/ true);
-
-      // If we didn't split a prefix operator, reset the regex lexing scope.
-      // Otherwise, we want to keep it in place for the next token.
-      auto didSplit = L->peekNextToken().getLength() == slashIdx;
-      if (!didSplit)
-        regexScope.reset();
-
-      // Discard the current token, which will be replaced by the re-lexed
-      // token, which will either be a regex literal token, a prefix operator,
-      // or the original unchanged token.
-      discardToken();
-
-      // If we split a prefix operator from the regex literal, and are not sure
-      // whether this should be a regex, backtrack if we didn't end up lexing a
-      // regex literal.
-      if (didSplit && !mustBeRegex &&
-          !L->peekNextToken().is(tok::regex_literal)) {
-        return;
-      }
+  // Check to see if we have an operator containing '/'.
+  auto slashIdx = Tok.getText().find("/");
+  if (slashIdx == StringRef::npos)
+    return;
+
+  CancellableBacktrackingScope backtrack(*this);
+  {
+    Optional<Lexer::ForwardSlashRegexRAII> regexScope;
+    regexScope.emplace(*L, mustBeRegex);
 
-      // Otherwise, accept the result.
-      backtrack.cancelBacktrack();
+    // Try re-lex as a `/.../` regex literal, this will split an operator if
+    // necessary.
+    L->restoreState(getParserPosition().LS, /*enableDiagnostics*/ true);
+
+    // If we didn't split a prefix operator, reset the regex lexing scope.
+    // Otherwise, we want to keep it in place for the next token.
+    auto didSplit = L->peekNextToken().getLength() == slashIdx;
+    if (!didSplit)
+      regexScope.reset();
+
+    // Discard the current token, which will be replaced by the re-lexed
+    // token, which will either be a regex literal token, a prefix operator,
+    // or the original unchanged token.
+    discardToken();
+
+    // If we split a prefix operator from the regex literal, and are not sure
+    // whether this should be a regex, backtrack if we didn't end up lexing a
+    // regex literal.
+    if (didSplit && !mustBeRegex &&
+        !L->peekNextToken().is(tok::regex_literal)) {
+      return;
     }
-    break;
-  }
-  default:
-    break;
+
+    // Otherwise, accept the result.
+    backtrack.cancelBacktrack();
   }
 }
 
@@ -3223,17 +3231,23 @@ ParserStatus Parser::parseExprList(tok leftTok, tok rightTok,
     SourceLoc FieldNameLoc;
     parseOptionalArgumentLabel(FieldName, FieldNameLoc);
 
-    // First check to see if we have the start of a regex literal `/.../`. We
-    // need to do this before handling unapplied operator references, as e.g
-    // `(/, /)` might be a regex literal.
-    tryLexRegexLiteral(/*mustBeRegex*/ false);
-
     // See if we have an operator decl ref '(<op>)'. The operator token in
     // this case lexes as a binary operator because it neither leads nor
     // follows a proper subexpression.
+    auto isUnappliedOperator = [&]() {
+      return Tok.isBinaryOperator() && peekToken().isAny(rightTok, tok::comma);
+    };
+
+    if (isUnappliedOperator()) {
+      // Check to see if we have the start of a regex literal `/.../`. We need
+      // to do this for an unapplied operator reference, as e.g `(/, /)` might
+      // be a regex literal.
+      tryLexRegexLiteral(/*forUnappliedOperator*/ true);
+    }
+
     ParserStatus Status;
     Expr *SubExpr = nullptr;
-    if (Tok.isBinaryOperator() && peekToken().isAny(rightTok, tok::comma)) {
+    if (isUnappliedOperator()) {
       SyntaxParsingContext operatorContext(SyntaxContext,
                                            SyntaxKind::IdentifierExpr);
       DeclNameLoc Loc;

test/StringProcessing/Parse/forward-slash-regex.swift

@@ -53,8 +53,9 @@ do {
   // expected-error@-3 {{'/' is not a postfix unary operator}}
 }
 
+// No closing '/' so a prefix operator.
 _ = /x
-// expected-error@-1 {{unterminated regex literal}}
+// expected-error@-1 {{'/' is not a prefix unary operator}}
 
 _ = !/x/
 // expected-error@-1 {{cannot convert value of type 'Regex<Substring>' to expected argument type 'Bool'}}
@@ -250,13 +251,13 @@ _ = await /x / // expected-warning {{no 'async' operations occur within 'await'
 // written a comment and is still in the middle of writing the characters before
 // it.
 _ = /x// comment
-// expected-error@-1 {{unterminated regex literal}}
+// expected-error@-1 {{'/' is not a prefix unary operator}}
 
 _ = /x // comment
-// expected-error@-1 {{unterminated regex literal}}
+// expected-error@-1 {{'/' is not a prefix unary operator}}
 
 _ = /x/*comment*/
-// expected-error@-1 {{unterminated regex literal}}
+// expected-error@-1 {{'/' is not a prefix unary operator}}
 
 // These become regex literals, unless surrounded in parens.
 func baz(_ x: (Int, Int) -> Int, _ y: (Int, Int) -> Int) {} // expected-note 4{{'baz' declared here}}

test/StringProcessing/Parse/regex_parse_error.swift

@@ -30,17 +30,15 @@ _ = #/\(?'abc/#
 do {
   _ = /\
   /
-  // expected-error@-2:7 {{unterminated regex literal}}
-  // expected-error@-3:9 {{expected escape sequence}}
-} // expected-error@:1 {{expected expression after operator}}
+  // expected-error@-1:3 {{expected expression path in Swift key path}}
+}
 
 do {
   _ = #/\
   /#
   // expected-error@-2:7 {{unterminated regex literal}}
   // expected-error@-3:10 {{expected escape sequence}}
-  // expected-error@-3:3 {{unterminated regex literal}}
-  // expected-warning@-4:3 {{regular expression literal is unused}}
+  // expected-error@-3:4 {{expected expression}}
 }
 
 func foo<T>(_ x: T, _ y: T) {}