Error on unknown escape sequences

Throw an error for unknown a-z escape sequences
as well as non-ASCII non-whitespace escape
sequences.

Author: hamishknight

Sources/_RegexParser/Regex/Parse/Diagnostics.swift

@@ -39,6 +39,7 @@ enum ParseError: Error, Hashable {
 
   case expectedNonEmptyContents
   case expectedEscape
+  case invalidEscape(Character)
 
   case cannotReferToWholePattern
 
@@ -107,6 +108,8 @@ extension ParseError: CustomStringConvertible {
       return "expected non-empty contents"
     case .expectedEscape:
       return "expected escape sequence"
+    case .invalidEscape(let c):
+      return "invalid escape sequence '\\\(c)'"
     case .cannotReferToWholePattern:
       return "cannot refer to whole pattern here"
     case .notQuantifiable:

Sources/_RegexParser/Regex/Parse/LexicalAnalysis.swift

@@ -1489,8 +1489,17 @@ extension Source {
         return try .scalar(
           src.expectUnicodeScalar(escapedCharacter: char).value)
       default:
-        return .char(char)
+        break
       }
+
+      // We only allow unknown escape sequences for non-letter ASCII, and
+      // non-ASCII whitespace.
+      guard (char.isASCII && !char.isLetter) ||
+              (!char.isASCII && char.isWhitespace)
+      else {
+        throw ParseError.invalidEscape(char)
+      }
+      return .char(char)
     }
   }
 

Tests/RegexTests/ParseTests.swift

@@ -544,9 +544,8 @@ extension RegexTests {
       #"a\Q \Q \\.\Eb"#,
       concat("a", quote(#" \Q \\."#), "b"))
 
-    // These follow the PCRE behavior.
+    // This follows the PCRE behavior.
     parseTest(#"\Q\\E"#, quote("\\"))
-    parseTest(#"\E"#, "E")
 
     parseTest(#"a" ."b"#, concat("a", quote(" ."), "b"),
               syntax: .experimental)
@@ -566,6 +565,16 @@ extension RegexTests {
 
     parseTest(#"["-"]"#, charClass(range_m("\"", "\"")))
 
+    // MARK: Escapes
+
+    // Not metachars, but we allow their escape as ASCII.
+    parseTest(#"\<"#, "<")
+    parseTest(#"\ "#, " ")
+    parseTest(#"\\"#, "\\")
+
+    // Escaped U+3000 IDEOGRAPHIC SPACE.
+    parseTest(#"\\#u{3000}"#, "\u{3000}")
+
     // MARK: Comments
 
     parseTest(
@@ -989,13 +998,6 @@ extension RegexTests {
     // Backreferences are not valid in custom character classes.
     parseTest(#"[\8]"#, charClass("8"))
     parseTest(#"[\9]"#, charClass("9"))
-    parseTest(#"[\g]"#, charClass("g"))
-    parseTest(#"[\g+30]"#, charClass("g", "+", "3", "0"))
-    parseTest(#"[\g{1}]"#, charClass("g", "{", "1", "}"))
-    parseTest(#"[\k'a']"#, charClass("k", "'", "a", "'"))
-
-    parseTest(#"\g"#, atom(.char("g")))
-    parseTest(#"\k"#, atom(.char("k")))
 
     // MARK: Character names.
 
@@ -1526,7 +1528,7 @@ extension RegexTests {
     parseWithDelimitersTest("re'x*'", zeroOrMore(of: "x"))
 
     parseWithDelimitersTest(#"re'🔥🇩🇰'"#, concat("🔥", "🇩🇰"))
-    parseWithDelimitersTest(#"re'\🔥✅'"#, concat("🔥", "✅"))
+    parseWithDelimitersTest(#"re'🔥✅'"#, concat("🔥", "✅"))
 
     // Printable ASCII characters.
     delimiterLexingTest(##"re' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~'"##)
@@ -1875,6 +1877,26 @@ extension RegexTests {
 
     diagnosticTest("\\", .expectedEscape)
 
+    // TODO: Custom diagnostic for expected backref
+    diagnosticTest(#"\g"#, .invalidEscape("g"))
+    diagnosticTest(#"\k"#, .invalidEscape("k"))
+
+    // TODO: Custom diagnostic for backref in custom char class
+    diagnosticTest(#"[\g]"#, .invalidEscape("g"))
+    diagnosticTest(#"[\g+30]"#, .invalidEscape("g"))
+    diagnosticTest(#"[\g{1}]"#, .invalidEscape("g"))
+    diagnosticTest(#"[\k'a']"#, .invalidEscape("k"))
+
+    // TODO: Custom diagnostic for missing '\Q'
+    diagnosticTest(#"\E"#, .invalidEscape("E"))
+
+    // Non-ASCII non-whitespace cases.
+    diagnosticTest(#"\🔥"#, .invalidEscape("🔥"))
+    diagnosticTest(#"\🇩🇰"#, .invalidEscape("🇩🇰"))
+    diagnosticTest(#"\e\#u{301}"#, .invalidEscape("e\u{301}"))
+    diagnosticTest(#"\\#u{E9}"#, .invalidEscape("é"))
+    diagnosticTest(#"\˂"#, .invalidEscape("˂"))
+
     // MARK: Text Segment options
 
     diagnosticTest("(?-y{g})", .cannotRemoveTextSegmentOptions)