Allow lexer recovery for missing closing delimiter

Allow the C++ lexer to form a tok::regex_literal.
This avoids generic fallback behavior, and better
allows for things like code completion. The test
case for this will be in the C++ repo.

Author: hamishknight

Sources/_MatchingEngine/Regex/Parse/DelimiterLexing.swift

@@ -208,14 +208,17 @@ fileprivate struct DelimiterLexer {
 /// Drop a set of regex delimiters from the input string, returning the contents
 /// and the delimiter used. The input string must have valid delimiters.
 func droppingRegexDelimiters(_ str: String) -> (String, Delimiter) {
-  let utf8 = str.utf8
   func stripDelimiter(_ delim: Delimiter) -> String? {
-    let prefix = delim.opening.utf8
-    let suffix = delim.closing.utf8
-    guard utf8.prefix(prefix.count).elementsEqual(prefix),
-          utf8.suffix(suffix.count).elementsEqual(suffix) else { return nil }
-
-    return String(utf8.dropFirst(prefix.count).dropLast(suffix.count))
+    // The opening delimiter must match.
+    guard var slice = str.utf8.tryDropPrefix(delim.opening.utf8)
+    else { return nil }
+
+    // The closing delimiter may optionally match, as it may not be present in
+    // invalid code.
+    if let newSlice = slice.tryDropSuffix(delim.closing.utf8) {
+      slice = newSlice
+    }
+    return String(slice)
   }
   for d in Delimiter.allCases {
     if let contents = stripDelimiter(d) {

Sources/_MatchingEngine/Regex/Parse/Mocking.swift

@@ -61,10 +61,16 @@ func libswiftLexRegexLiteral(
     errOut.pointee = copyCString("\(error)")
     curPtrPtr.pointee = error.resumePtr.assumingMemoryBound(to: CChar.self)
 
-    // For now, treat every error as unrecoverable.
-    // TODO: We should ideally be able to recover from a regex with missing
-    // closing delimiters, which would help with code completion.
-    return true
+    switch error.kind {
+    case .endOfString:
+      // Missing closing delimiter can be recovered from.
+      return false
+    case .unprintableASCII, .invalidUTF8:
+      // We don't currently have good recovery behavior for these.
+      return true
+    case .unknownDelimiter:
+      fatalError("Already handled")
+    }
   } catch {
     fatalError("Should be a DelimiterLexError")
   }

Sources/_MatchingEngine/Utility/Misc.swift

@@ -108,7 +108,28 @@ extension Collection {
   >(_ idx: Index, in c: C) -> C.Index {
     c.index(atOffset: offset(of: idx))
   }
+}
 
+extension Collection where Element: Equatable {
+  /// Attempt to drop a given prefix from the collection, returning the
+  /// resulting subsequence, or `nil` if the prefix does not match.
+  public func tryDropPrefix<C : Collection>(
+    _ other: C
+  ) -> SubSequence? where C.Element == Element {
+    let prefixCount = other.count
+    guard prefix(prefixCount).elementsEqual(other) else { return nil }
+    return dropFirst(prefixCount)
+  }
+
+  /// Attempt to drop a given suffix from the collection, returning the
+  /// resulting subsequence, or `nil` if the suffix does not match.
+  public func tryDropSuffix<C : Collection>(
+    _ other: C
+  ) -> SubSequence? where C.Element == Element {
+    let suffixCount = other.count
+    guard suffix(suffixCount).elementsEqual(other) else { return nil }
+    return dropLast(suffixCount)
+  }
 }
 
 extension UnsafeMutableRawPointer {