Diagnose unprintable ASCII characters

This matches the behavior of the C++ lexer for
string literals.

Author: hamishknight

Sources/_MatchingEngine/Regex/Parse/DelimiterLexing.swift

@@ -42,6 +42,7 @@ struct DelimiterLexError: Error, CustomStringConvertible {
     case endOfString
     case invalidUTF8 // TODO: better range reporting
     case unknownDelimiter
+    case unprintableASCII
   }
 
   var kind: Kind
@@ -59,6 +60,7 @@ struct DelimiterLexError: Error, CustomStringConvertible {
     case .endOfString: return "unterminated regex literal"
     case .invalidUTF8: return "invalid UTF-8 found in source file"
     case .unknownDelimiter: return "unknown regex literal delimiter"
+    case .unprintableASCII: return "unprintable ASCII character found in source file"
     }
   }
 }
@@ -169,6 +171,11 @@ fileprivate struct DelimiterLexer {
       advanceCursor()
       try advance(escaped: true)
 
+    case let next where !next.isPrintableASCII:
+      // Diagnose unprintable ASCII.
+      // TODO: Ideally we would recover and continue to lex until the ending
+      // delimiter.
+      throw DelimiterLexError(.unprintableASCII, resumeAt: cursor.successor())
 
     default:
       advanceCursor()

Sources/_MatchingEngine/Utility/MissingUnicode.swift

@@ -661,3 +661,11 @@ extension Character {
 
   public var isWordCharacter: Bool { isLetter || isNumber || self == "_" }
 }
+
+extension UnicodeScalar {
+  public var isPrintableASCII: Bool {
+    // Exclude non-printables before the space character U+20, and anything
+    // including and above the DEL character U+7F.
+    value >= 0x20 && value < 0x7F
+  }
+}

Tests/RegexTests/ParseTests.swift

@@ -107,20 +107,26 @@ func parseTest(
   serializedCaptures.deallocate()
 }
 
-func parseWithDelimitersTest(
-  _ input: String, _ expecting: AST.Node,
-  file: StaticString = #file, line: UInt = #line
+func delimiterLexingTest(
+  _ input: String, file: StaticString = #file, line: UInt = #line
 ) {
-  // First try lexing.
-  input.withCString { ptr in
-    let (contents, delim, end) = try! lexRegex(start: ptr,
-                                               end: ptr + input.count)
-    XCTAssertEqual(end, ptr + input.count, file: file, line: line)
+  input.withCString(encodedAs: UTF8.self) { ptr in
+    let endPtr = ptr + input.utf8.count
+    let (contents, delim, end) = try! lexRegex(start: ptr, end: endPtr)
+    XCTAssertEqual(end, endPtr, file: file, line: line)
 
     let (parseContents, parseDelim) = droppingRegexDelimiters(input)
     XCTAssertEqual(contents, parseContents, file: file, line: line)
     XCTAssertEqual(delim, parseDelim, file: file, line: line)
   }
+}
+
+func parseWithDelimitersTest(
+  _ input: String, _ expecting: AST.Node,
+  file: StaticString = #file, line: UInt = #line
+) {
+  // First try lexing.
+  delimiterLexingTest(input, file: file, line: line)
 
   let orig = try! parseWithDelimiters(input)
   let ast = orig.root
@@ -199,6 +205,32 @@ func diagnosticTest(
   }
 }
 
+func delimiterLexingDiagnosticTest(
+  _ input: String, _ expected: DelimiterLexError.Kind,
+  syntax: SyntaxOptions = .traditional,
+  file: StaticString = #file, line: UInt = #line
+) {
+  do {
+    _ = try input.withCString { ptr in
+      try lexRegex(start: ptr, end: ptr + input.count)
+    }
+    XCTFail("""
+      Passed, but expected error: \(expected)
+    """, file: file, line: line)
+  } catch let e as DelimiterLexError {
+    guard e.kind == expected else {
+      XCTFail("""
+
+        Expected: \(expected)
+        Actual: \(e.kind)
+      """, file: file, line: line)
+      return
+    }
+  } catch let e {
+    XCTFail("Unexpected error type: \(e)", file: file, line: line)
+  }
+}
+
 func libswiftDiagnosticMessageTest(
   _ input: String, _ expectedErr: String, file: StaticString = #file,
   line: UInt = #line
@@ -1472,6 +1504,11 @@ extension RegexTests {
 
     parseWithDelimitersTest("re'x*'", zeroOrMore(of: "x"))
 
+    parseWithDelimitersTest(#"re'🔥🇩🇰'"#, concat("🔥", "🇩🇰"))
+    parseWithDelimitersTest(#"re'\🔥✅'"#, concat("🔥", "✅"))
+
+    // Printable ASCII characters.
+    delimiterLexingTest(##"re' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~'"##)
     // MARK: Parse not-equal
 
     // Make sure dumping output correctly reflects differences in AST.
@@ -1890,6 +1927,16 @@ extension RegexTests {
     diagnosticTest("(*LIMIT_DEPTH=-1", .expectedNumber("", kind: .decimal))
   }
 
+  func testDelimiterLexingErrors() {
+    delimiterLexingDiagnosticTest(#"re'\\#n'"#, .endOfString)
+    for i: UInt8 in 0x1 ..< 0x20 where i != 0xA && i != 0xD { // U+A & U+D are \n and \r.
+      delimiterLexingDiagnosticTest("re'\(UnicodeScalar(i))'", .unprintableASCII)
+    }
+    delimiterLexingDiagnosticTest("re'\n'", .endOfString)
+    delimiterLexingDiagnosticTest("re'\r'", .endOfString)
+    delimiterLexingDiagnosticTest("re'\u{7F}'", .unprintableASCII)
+  }
+
   func testlibswiftDiagnostics() {
     libswiftDiagnosticMessageTest(
       "#/[x*/#", "cannot parse regular expression: expected ']'")