Introduce a multi-line literal mode

When an extended delimiter `#/` is followed by a
newline, enter a multi-line mode where the literal
may span multiple lines, and extended syntax is
enabled by default.

Author: hamishknight

Sources/_RegexParser/Regex/Parse/DelimiterLexing.swift

@@ -9,8 +9,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-// TODO: mock up multi-line soon
-
 struct Delimiter: Hashable {
   let kind: Kind
   let poundCount: Int
@@ -28,13 +26,13 @@ struct Delimiter: Hashable {
     kind.closing + String(repeating: "#", count: poundCount)
   }
 
-  /// The default set of syntax options that the delimiter indicates.
-  var defaultSyntaxOptions: SyntaxOptions {
+  /// Whether or not multi-line mode is permitted.
+  var allowsMultiline: Bool {
     switch kind {
-    case .forwardSlash, .reSingleQuote:
-      return .traditional
-    case .experimental, .rxSingleQuote:
-      return .experimental
+    case .forwardSlash:
+      return poundCount > 0
+    case .experimental, .reSingleQuote, .rxSingleQuote:
+      return false
     }
   }
 }
@@ -76,6 +74,7 @@ struct DelimiterLexError: Error, CustomStringConvertible {
     case invalidUTF8 // TODO: better range reporting
     case unknownDelimiter
     case unprintableASCII
+    case multilineClosingNotOnNewline
   }
 
   var kind: Kind
@@ -94,6 +93,7 @@ struct DelimiterLexError: Error, CustomStringConvertible {
     case .invalidUTF8: return "invalid UTF-8 found in source file"
     case .unknownDelimiter: return "unknown regex literal delimiter"
     case .unprintableASCII: return "unprintable ASCII character found in source file"
+    case .multilineClosingNotOnNewline: return "closing delimiter must appear on new line"
     }
   }
 }
@@ -103,6 +103,9 @@ fileprivate struct DelimiterLexer {
   var cursor: UnsafeRawPointer
   let end: UnsafeRawPointer
 
+  var firstNewline: UnsafeRawPointer?
+  var isMultiline: Bool { firstNewline != nil }
+
   init(start: UnsafeRawPointer, end: UnsafeRawPointer) {
     precondition(start <= end)
     self.start = start
@@ -262,12 +265,23 @@ fileprivate struct DelimiterLexer {
     let contentsEnd = cursor
     guard tryEat(delimiter.closing.utf8) else { return nil }
 
-    // Form a string from the contents and make sure it's valid UTF-8.
     let count = contentsEnd - contentsStart
     let contents = UnsafeRawBufferPointer(
       start: contentsStart, count: count)
-    let s = String(decoding: contents, as: UTF8.self)
 
+    // In multi-line mode, we must be on a new line. So scan backwards and make
+    // sure we only have whitespace until the newline.
+    if isMultiline {
+      let idx = contents.lastIndex(
+        where: { $0 == ascii("\n") || $0 == ascii("\r") })! + 1
+      guard contents[idx...].all({ $0 == ascii(" ") || $0 == ascii("\t") })
+      else {
+        throw DelimiterLexError(.multilineClosingNotOnNewline, resumeAt: cursor)
+      }
+    }
+
+    // Form a string from the contents and make sure it's valid UTF-8.
+    let s = String(decoding: contents, as: UTF8.self)
     guard s.utf8.elementsEqual(contents) else {
       throw DelimiterLexError(.invalidUTF8, resumeAt: cursor)
     }
@@ -278,7 +292,10 @@ fileprivate struct DelimiterLexer {
   /// the end of the buffer is reached.
   mutating func advance(escaped: Bool = false) throws {
     guard let next = load() else {
-      throw DelimiterLexError(.unterminated, resumeAt: cursor)
+      // We've hit the end of the buffer. In multi-line mode, we don't want to
+      // skip over what is likely otherwise valid Swift code, so resume from the
+      // first newline.
+      throw DelimiterLexError(.unterminated, resumeAt: firstNewline ?? cursor)
     }
     switch UnicodeScalar(next) {
     case let next where !next.isASCII:
@@ -289,7 +306,10 @@ fileprivate struct DelimiterLexer {
       advanceCursor()
 
     case "\n", "\r":
-      throw DelimiterLexError(.unterminated, resumeAt: cursor)
+      guard isMultiline else {
+        throw DelimiterLexError(.unterminated, resumeAt: cursor)
+      }
+      advanceCursor()
 
     case "\0":
       // TODO: Warn to match the behavior of String literal lexer? Or should
@@ -301,8 +321,12 @@ fileprivate struct DelimiterLexer {
       advanceCursor()
       try advance(escaped: true)
 
-    case let next where !next.isPrintableASCII:
+    case let next
+      where !next.isPrintableASCII && !(isMultiline && next == "\t"):
       // Diagnose unprintable ASCII.
+      // Note that tabs are allowed in multi-line literals.
+      // TODO: This matches the string literal behavior, but should we allow
+      // tabs for single-line regex literals too?
       // TODO: Ideally we would recover and continue to lex until the ending
       // delimiter.
       throw DelimiterLexError(.unprintableASCII, resumeAt: cursor.successor())
@@ -349,6 +373,23 @@ fileprivate struct DelimiterLexer {
       throw DelimiterLexError(.unknownDelimiter, resumeAt: cursor.successor())
     }
     let contentsStart = cursor
+
+    // If the delimiter allows multi-line, try skipping over any whitespace to a
+    // newline character. If we can do that, we enter multi-line mode.
+    if delimiter.allowsMultiline {
+      while let next = load() {
+        switch next {
+        case ascii(" "), ascii("\t"):
+          advanceCursor()
+          continue
+        case ascii("\n"), ascii("\r"):
+          firstNewline = cursor
+        default:
+          break
+        }
+        break
+      }
+    }
     while true {
       // Check to see if we're at a character that looks like a delimiter, but
       // likely isn't. In such a case, we can attempt to skip over it.

Sources/_RegexParser/Regex/Parse/Diagnostics.swift

@@ -70,6 +70,8 @@ enum ParseError: Error, Hashable {
 
   case cannotRemoveTextSegmentOptions
   case cannotRemoveSemanticsOptions
+  case cannotRemoveExtendedSyntaxInMultilineMode
+
   case expectedCalloutArgument
 }
 
@@ -158,6 +160,8 @@ extension ParseError: CustomStringConvertible {
       return "text segment mode cannot be unset, only changed"
     case .cannotRemoveSemanticsOptions:
       return "semantic level cannot be unset, only changed"
+    case .cannotRemoveExtendedSyntaxInMultilineMode:
+      return "extended syntax may not be disabled in multi-line mode"
     case .expectedCalloutArgument:
       return "expected argument to callout"
     }

Sources/_RegexParser/Regex/Parse/LexicalAnalysis.swift

@@ -657,6 +657,7 @@ extension Source {
   ///                        | MatchingOption* '-' MatchingOption*
   ///
   mutating func lexMatchingOptionSequence(
+    context: ParsingContext
   ) throws -> AST.MatchingOptionSequence? {
     let ateCaret = recordLoc { $0.tryEat("^") }
 
@@ -691,6 +692,11 @@ extension Source {
         if opt.isSemanticMatchingLevel {
           throw ParseError.cannotRemoveSemanticsOptions
         }
+        // Extended syntax may not be removed if in multi-line mode.
+        if context.syntax.contains(.multilineExtendedSyntax) &&
+            opt.isAnyExtended {
+          throw ParseError.cannotRemoveExtendedSyntaxInMultilineMode
+        }
         removing.append(opt)
       }
       return .init(caretLoc: nil, adding: adding, minusLoc: ateMinus.location,
@@ -864,7 +870,7 @@ extension Source {
           }
 
           // Matching option changing group (?iJmnsUxxxDPSWy{..}-iJmnsUxxxDPSW:).
-          if let seq = try src.lexMatchingOptionSequence() {
+          if let seq = try src.lexMatchingOptionSequence(context: context) {
             if src.tryEat(":") {
               return .changeMatchingOptions(seq, isIsolated: false)
             }

Sources/_RegexParser/Regex/Parse/Mocking.swift

@@ -62,8 +62,8 @@ func libswiftLexRegexLiteral(
     curPtrPtr.pointee = error.resumePtr.assumingMemoryBound(to: CChar.self)
 
     switch error.kind {
-    case .unterminated:
-      // Missing closing delimiter can be recovered from.
+    case .unterminated, .multilineClosingNotOnNewline:
+      // These can be recovered from.
       return false
     case .unprintableASCII, .invalidUTF8:
       // We don't currently have good recovery behavior for these.

Sources/_RegexParser/Regex/Parse/Parse.swift

@@ -288,22 +288,25 @@ extension Parser {
   ) throws -> AST.Group {
     context.recordGroup(kind.value)
 
-    // Check if we're introducing or removing extended syntax.
+    // Check if we're introducing or removing extended syntax. We skip this for
+    // multi-line, as extended syntax is always enabled there.
     // TODO: PCRE differentiates between (?x) and (?xx) where only the latter
     // handles non-semantic whitespace in a custom character class. Other
     // engines such as Oniguruma, Java, and ICU do this under (?x). Therefore,
     // treat (?x) and (?xx) as the same option here. If we ever get a strict
     // PCRE mode, we will need to change this to handle that.
     let currentSyntax = context.syntax
-    if case .changeMatchingOptions(let c, isIsolated: _) = kind.value {
-      if c.resetsCurrentOptions {
-        context.syntax.remove(.extendedSyntax)
-      }
-      if c.adding.contains(where: \.isAnyExtended) {
-        context.syntax.insert(.extendedSyntax)
-      }
-      if c.removing.contains(where: \.isAnyExtended) {
-        context.syntax.remove(.extendedSyntax)
+    if !context.syntax.contains(.multilineExtendedSyntax) {
+      if case .changeMatchingOptions(let c, isIsolated: _) = kind.value {
+        if c.resetsCurrentOptions {
+          context.syntax.remove(.extendedSyntax)
+        }
+        if c.adding.contains(where: \.isAnyExtended) {
+          context.syntax.insert(.extendedSyntax)
+        }
+        if c.removing.contains(where: \.isAnyExtended) {
+          context.syntax.remove(.extendedSyntax)
+        }
       }
     }
     defer {
@@ -532,11 +535,32 @@ public func parse<S: StringProtocol>(
   return try parser.parse()
 }
 
+/// Retrieve the default set of syntax options that a delimiter and literal
+/// contents indicates.
+fileprivate func defaultSyntaxOptions(
+  _ delim: Delimiter, contents: String
+) -> SyntaxOptions {
+  switch delim.kind {
+  case .forwardSlash:
+    // For an extended syntax forward slash e.g #/.../#, extended syntax is
+    // permitted if it spans multiple lines.
+    if delim.poundCount > 0 &&
+        contents.unicodeScalars.contains(where: { $0 == "\n" || $0 == "\r" }) {
+      return .multilineExtendedSyntax
+    }
+    return .traditional
+  case .reSingleQuote:
+    return .traditional
+  case .experimental, .rxSingleQuote:
+    return .experimental
+  }
+}
+
 /// Parse a given regex string with delimiters, inferring the syntax options
 /// from the delimiter used.
 public func parseWithDelimiters<S: StringProtocol>(
   _ regex: S
 ) throws -> AST where S.SubSequence == Substring {
   let (contents, delim) = droppingRegexDelimiters(String(regex))
-  return try parse(contents, delim.defaultSyntaxOptions)
+  return try parse(contents, defaultSyntaxOptions(delim, contents: contents))
 }

Sources/_RegexParser/Regex/Parse/SyntaxOptions.swift

@@ -58,6 +58,11 @@ public struct SyntaxOptions: OptionSet {
   ///  `(_: .*)` == `(?:.*)`
   public static var experimentalCaptures: Self { Self(1 << 5) }
 
+  /// The default syntax for a multi-line regex literal.
+  public static var multilineExtendedSyntax: Self {
+    return [Self(1 << 6), .extendedSyntax]
+  }
+
   /*
 
     /// `<digit>*` == `[[:digit:]]*` == `\d*`

Tests/RegexTests/LexTests.swift

@@ -110,6 +110,11 @@ extension RegexTests {
       ("###/abc/###", ("abc", delim(.forwardSlash, poundCount: 3))),
       ("#|abc|#", ("abc", delim(.experimental))),
 
+      // Multiline
+      ("#/\na\nb\n/#", ("\na\nb\n", delim(.forwardSlash, poundCount: 1))),
+      ("#/ \na\nb\n  /#", (" \na\nb\n  ", delim(.forwardSlash, poundCount: 1))),
+      ("##/ \na\nb\n  /##", (" \na\nb\n  ", delim(.forwardSlash, poundCount: 2))),
+
       // TODO: Null characters are lexically valid, similar to string literals,
       // but we ought to warn the user about them.
       ("#|ab\0c|#", ("ab\0c", delim(.experimental))),