Parse PCRE callout syntax

Parse the `(?C)` syntax with an integer or string
argument. This doesn't yet handle the Oniguruma
specific callout syntax, which is a little more
involved.

Author: hamishknight

Sources/_MatchingEngine/Regex/AST/Atom.swift

@@ -66,6 +66,9 @@ extension AST {
       // References
       case backreference(Reference)
       case subpattern(Reference)
+
+      // (?C)
+      case callout(Callout)
     }
   }
 }
@@ -443,6 +446,19 @@ extension AST.Atom {
   }
 }
 
+extension AST.Atom {
+  public struct Callout: Hashable {
+    public enum Argument: Hashable {
+      case number(Int)
+      case string(String)
+    }
+    public var arg: AST.Located<Argument>
+    public init(_ arg: AST.Located<Argument>) {
+      self.arg = arg
+    }
+  }
+}
+
 extension AST.Atom {
   /// Retrieve the character value of the atom if it represents a literal
   /// character or unicode scalar, nil otherwise.
@@ -458,7 +474,7 @@ extension AST.Atom {
       fallthrough
 
     case .property, .escaped, .any, .startOfLine, .endOfLine,
-        .backreference, .subpattern, .namedCharacter:
+        .backreference, .subpattern, .namedCharacter, .callout:
       return nil
     }
   }
@@ -483,7 +499,7 @@ extension AST.Atom {
       return "\\M-\\C-\(x)"
 
     case .property, .escaped, .any, .startOfLine, .endOfLine,
-        .backreference, .subpattern, .namedCharacter:
+        .backreference, .subpattern, .namedCharacter, .callout:
       return nil
     }
   }

Sources/_MatchingEngine/Regex/Parse/Diagnostics.swift

@@ -39,6 +39,7 @@ enum ParseError: Error, Hashable {
   case cannotReferToWholePattern
 
   case unknownGroupKind(String)
+  case unknownCalloutKind(String)
 
   case invalidMatchingOption(Character)
   case cannotRemoveMatchingOptionsAfterCaret
@@ -86,6 +87,8 @@ extension ParseError: CustomStringConvertible {
       return "\(str) cannot be used as condition"
     case let .unknownGroupKind(str):
       return "unknown group kind '(\(str)'"
+    case let .unknownCalloutKind(str):
+      return "unknown callout kind '\(str)'"
     case let .invalidMatchingOption(c):
       return "invalid matching option '\(c)'"
     case .cannotRemoveMatchingOptionsAfterCaret:

Sources/_MatchingEngine/Regex/Parse/LexicalAnalysis.swift

@@ -631,8 +631,15 @@ extension Source {
   ) throws -> Located<AST.Group.Kind>? {
     try recordLoc { src in
       try src.tryEating { src in
-        guard src.tryEat("(") else { return nil }
+        // There are some atoms that syntactically look like groups, bail here
+        // if we see any. Care needs to be taken here as e.g a group starting
+        // with '(?-' is a subpattern if the next character is a digit,
+        // otherwise a matching option specifier. Conversely, '(?P' can be the
+        // start of a matching option sequence, or a reference if it is followed
+        // by '=' or '<'.
+        guard !src.shouldLexGroupLikeAtom() else { return nil }
 
+        guard src.tryEat("(") else { return nil }
         if src.tryEat("?") {
           if src.tryEat(":") { return .nonCapture }
           if src.tryEat("|") { return .nonCaptureReset }
@@ -658,15 +665,6 @@ extension Source {
             return .namedCapture(name)
           }
 
-          // Check if we can lex a group-like reference. Do this before matching
-          // options to avoid ambiguity with a group starting with (?-, which
-          // is a subpattern if the next character is a digit, otherwise a
-          // matching option specifier. In addition, we need to be careful with
-          // (?P, which can also be the start of a matching option sequence.
-          if src.canLexGroupLikeReference() {
-            return nil
-          }
-
           // Matching option changing group (?iJmnsUxxxDPSWy{..}-iJmnsUxxxDPSW:).
           if let seq = try src.lexMatchingOptionSequence() {
             if src.tryEat(":") {
@@ -1059,11 +1057,11 @@ extension Source {
     for openChar: Character
   ) -> Character {
     switch openChar {
+      // Identically-balanced delimiters.
+      case "'", "\"", "`", "^", "%", "#", "$": return openChar
       case "<": return ">"
-      case "'": return "'"
       case "{": return "}"
-      default:
-        fatalError("Not implemented")
+      default: fatalError("Not implemented")
     }
   }
 
@@ -1204,6 +1202,24 @@ extension Source {
     return src.canLexNumberedReference()
   }
 
+  /// Whether a group specifier should be lexed as an atom instead of a group.
+  private func shouldLexGroupLikeAtom() -> Bool {
+    var src = self
+    guard src.tryEat("(") else { return false }
+
+    if src.tryEat("?") {
+      // The start of a reference '(?P=', '(?R', ...
+      if src.canLexGroupLikeReference() { return true }
+
+      // The start of a callout.
+      if src.tryEat("C") { return true }
+
+      return false
+    }
+
+    return false
+  }
+
   /// Consume an escaped atom, starting from after the backslash
   ///
   ///     Escaped          -> KeyboardModified | Builtin
@@ -1265,6 +1281,78 @@ extension Source {
     }
   }
 
+  /// Try to consume a callout.
+  ///
+  ///     Callout     -> '(?C' CalloutBody ')'
+  ///     CalloutBody -> '' | <Number>
+  ///                  | '`' <String> '`'
+  ///                  | "'" <String> "'"
+  ///                  | '"' <String> '"'
+  ///                  | '^' <String> '^'
+  ///                  | '%' <String> '%'
+  ///                  | '#' <String> '#'
+  ///                  | '$' <String> '$'
+  ///                  | '{' <String> '}'
+  ///
+  mutating func lexCallout() throws -> AST.Atom.Callout? {
+    guard tryEat(sequence: "(?C") else { return nil }
+    let arg = try recordLoc { src -> AST.Atom.Callout.Argument in
+      // Parse '(?C' followed by a number.
+      if let num = try src.lexNumber() {
+        return .number(num.value)
+      }
+      // '(?C)' is implicitly '(?C0)'.
+      if src.peek() == ")" {
+        return .number(0)
+      }
+      // Parse '(C?' followed by a set of balanced delimiters as defined by
+      // http://pcre.org/current/doc/html/pcre2pattern.html#SEC28
+      if let open = src.tryEat(anyOf: "`", "'", "\"", "^", "%", "#", "$", "{") {
+        let closing = String(Source.getClosingDelimiter(for: open))
+        return .string(try src.expectQuoted(endingWith: closing).value)
+      }
+      // If we don't know what this syntax is, consume up to the ending ')' and
+      // emit an error.
+      let remaining = src.lexUntil { $0.isEmpty || $0.tryEat(")") }.value
+      if remaining.isEmpty {
+        throw ParseError.expected(")")
+      }
+      throw ParseError.unknownCalloutKind("(?C\(remaining))")
+    }
+    try expect(")")
+    return .init(arg)
+  }
+
+  /// Consume a group-like atom, throwing an error if an atom could not be
+  /// produced.
+  ///
+  ///     GroupLikeAtom -> GroupLikeReference | Callout | BacktrackingDirective
+  ///
+  mutating func expectGroupLikeAtom() throws -> AST.Atom.Kind {
+    try recordLoc { src in
+      // References that look like groups, e.g (?R), (?1), ...
+      if let ref = try src.lexGroupLikeReference() {
+        return ref.value
+      }
+
+      // (?C)
+      if let callout = try src.lexCallout() {
+        return .callout(callout)
+      }
+
+      // If we didn't produce an atom, consume up until a reasonable end-point
+      // and throw an error.
+      try src.expect("(")
+      let remaining = src.lexUntil {
+        $0.isEmpty || $0.tryEat(anyOf: ":", ")") != nil
+      }.value
+      if remaining.isEmpty {
+        throw ParseError.expected(")")
+      }
+      throw ParseError.unknownGroupKind(remaining)
+    }.value
+  }
+
 
   /// Try to consume an Atom.
   ///
@@ -1293,9 +1381,10 @@ extension Source {
         return .property(prop)
       }
 
-      // References that look like groups, e.g (?R), (?1), ...
-      if !customCC, let ref = try src.lexGroupLikeReference() {
-        return ref.value
+      // If we have group syntax that was skipped over in lexGroupStart, we
+      // need to handle it as an atom, or throw an error.
+      if !customCC && src.shouldLexGroupLikeAtom() {
+        return try src.expectGroupLikeAtom()
       }
 
       let char = src.eat()

Sources/_MatchingEngine/Regex/Parse/Source.swift

@@ -60,7 +60,7 @@ extension Source {
 
   var isEmpty: Bool { _slice.isEmpty }
 
-  mutating func peek() -> Char? { _slice.first }
+  func peek() -> Char? { _slice.first }
 
   mutating func advance() {
     assert(!isEmpty)

Sources/_MatchingEngine/Regex/Printing/DumpAST.swift

@@ -133,12 +133,18 @@ extension AST.Atom {
     case .backreference(let r), .subpattern(let r):
       return "\(r._dumpBase)"
 
+    case .callout(let c): return "\(c)"
+
     case .char, .scalar:
       fatalError("Unreachable")
     }
   }
 }
 
+extension AST.Atom.Callout: _ASTPrintable {
+  public var _dumpBase: String { "callout <\(arg.value)>" }
+}
+
 extension AST.Reference: _ASTPrintable {
   public var _dumpBase: String {
     "\(kind)"

Sources/_MatchingEngine/Regex/Printing/PrintAsPattern.swift

@@ -266,6 +266,9 @@ extension AST.Atom {
 
     case .subpattern:
       return " /* TODO: subpattern */"
+
+    case .callout:
+      return " /* TODO: callout */"
     }
   }
 }

Sources/_StringProcessing/ASTBuilder.swift

@@ -165,6 +165,10 @@ func groupCondition(
   .group(.init(.init(faking: kind), child, .fake))
 }
 
+func callout(_ arg: AST.Atom.Callout.Argument) -> AST {
+  atom(.callout(.init(.init(faking: arg))))
+}
+
 func quant(
   _ amount: AST.Quantification.Amount,
   _ kind: AST.Quantification.Kind = .eager,

Sources/_StringProcessing/ConsumerInterface.swift

@@ -112,7 +112,7 @@ extension AST.Atom {
 
     case .escaped, .keyboardControl, .keyboardMeta, .keyboardMetaControl,
         .any, .startOfLine, .endOfLine,
-        .backreference, .subpattern:
+        .backreference, .subpattern, .callout:
       // FIXME: implement
       return nil
     }

Tests/RegexTests/ParseTests.swift

@@ -1087,6 +1087,17 @@ extension RegexTests {
       trueBranch: empty(), falseBranch: empty())
     )
 
+    // MARK: Callouts
+
+    parseTest(#"(?C)"#, callout(.number(0)))
+    parseTest(#"(?C0)"#, callout(.number(0)))
+    parseTest(#"(?C20)"#, callout(.number(20)))
+    parseTest("(?C{abc})", callout(.string("abc")))
+
+    for delim in ["`", "'", "\"", "^", "%", "#", "$"] {
+      parseTest("(?C\(delim)hello\(delim))", callout(.string("hello")))
+    }
+
     // MARK: Parse with delimiters
 
     parseWithDelimitersTest("'/a b/'", concat("a", " ", "b"))
@@ -1150,6 +1161,9 @@ extension RegexTests {
     parseNotEqualTest(#"(?(VERSION=0.1))"#, #"(?(VERSION=0.2))"#)
     parseNotEqualTest(#"(?(VERSION=0.1))"#, #"(?(VERSION>=0.1))"#)
 
+    parseNotEqualTest("(?C0)", "(?C1)")
+    parseNotEqualTest("(?C0)", "(?C'hello')")
+
     // TODO: failure tests
   }
 
@@ -1242,6 +1256,8 @@ extension RegexTests {
     diagnosticTest(#""ab\""#, .expected("\""), syntax: .experimental)
     diagnosticTest("\"ab\\", .expectedEscape, syntax: .experimental)
 
+    diagnosticTest("(?C", .expected(")"))
+
     // MARK: Text Segment options
 
     diagnosticTest("(?-y{g})", .cannotRemoveTextSegmentOptions)
@@ -1277,5 +1293,10 @@ extension RegexTests {
     diagnosticTest(#"(?(1)a|b|c)"#, .tooManyBranchesInConditional(3))
     diagnosticTest(#"(?(1)||)"#, .tooManyBranchesInConditional(3))
     diagnosticTest(#"(?(?i))"#, .unsupportedCondition("implicitly scoped group"))
+
+    // MARK: Callouts
+
+    diagnosticTest("(?C-1)", .unknownCalloutKind("(?C-1)"))
+    diagnosticTest("(?C-1", .unknownCalloutKind("(?C-1)"))
   }
 }