Parse .NET balanced captures

This requires imposing some restrictions on what
can be used as a group name to allow for the
syntax `(?<a-b>)`. For now, restrict the characters
to letters, numbers and `_`, and forbid the first
character from being a number. This should be no
stricter than the rules imposed by PCRE, Oniguruma,
ICU, Java and .NET.

Author: hamishknight

Sources/_MatchingEngine/Regex/AST/Group.swift

@@ -31,6 +31,9 @@ extension AST {
       // (?<name>...) (?'name'...) (?P<name>...)
       case namedCapture(Located<String>)
 
+      // (?<name-priorName>) (?'name-priorName')
+      case balancedCapture(BalancedCapture)
+
       // (?:...)
       case nonCapture
 
@@ -79,7 +82,7 @@ extension AST {
 extension AST.Group.Kind {
   public var isCapturing: Bool {
     switch self {
-    case .capture, .namedCapture: return true
+    case .capture, .namedCapture, .balancedCapture: return true
     default: return false
     }
   }
@@ -103,6 +106,7 @@ extension AST.Group.Kind {
   public var name: String? {
     switch self {
     case .namedCapture(let name): return name.value
+    case .balancedCapture(let b): return b.name?.value
     default: return nil
     }
   }
@@ -121,5 +125,26 @@ extension AST.Group {
     default: return nil
     }
   }
+}
+
+extension AST.Group {
+  public struct BalancedCapture: Hashable {
+    /// The name of the group, or nil if the group has no name.
+    public var name: AST.Located<String>?
+
+    /// The location of the `-` in the group.
+    public var dash: SourceLocation
 
+    /// The name of the prior group that the balancing group references.
+    public var priorName: AST.Located<String>
+
+    public init(
+      name: AST.Located<String>?, dash: SourceLocation,
+      priorName: AST.Located<String>
+    ) {
+      self.name = name
+      self.dash = dash
+      self.priorName = priorName
+    }
+  }
 }

Sources/_MatchingEngine/Regex/Parse/CaptureStructure.swift

@@ -44,7 +44,10 @@ extension AST {
         return .atom() + innerCaptures
       case .namedCapture(let name):
         return .atom(name: name.value) + innerCaptures
+      case .balancedCapture(let b):
+        return .atom(name: b.name?.value) + innerCaptures
       default:
+        precondition(!group.kind.value.isCapturing)
         return innerCaptures
       }
     case .conditional(let c):

Sources/_MatchingEngine/Regex/Parse/Diagnostics.swift

@@ -55,6 +55,9 @@ enum ParseError: Error, Hashable {
   case emptyProperty
 
   case expectedGroupSpecifier
+  case expectedGroupName
+  case groupNameMustBeAlphaNumeric
+  case groupNameCannotStartWithNumber
   case cannotRemoveTextSegmentOptions
 }
 
@@ -113,6 +116,12 @@ extension ParseError: CustomStringConvertible {
       return "empty property"
     case .expectedGroupSpecifier:
       return "expected group specifier"
+    case .expectedGroupName:
+      return "expected group name"
+    case .groupNameMustBeAlphaNumeric:
+      return "group name must only contain alphanumeric characters"
+    case .groupNameCannotStartWithNumber:
+      return "group name must not start with number"
     case .cannotRemoveTextSegmentOptions:
       return "text segment mode cannot be unset, only changed"
     }

Sources/_MatchingEngine/Regex/Parse/LexicalAnalysis.swift

@@ -659,13 +659,61 @@ extension Source {
     }
   }
 
+  /// Consume a group name.
+  private mutating func expectGroupName(
+    endingWith ending: String, eatEnding: Bool = true
+  ) throws -> Located<String> {
+    let str = try recordLoc { src -> String in
+      if src.isEmpty || src.tryEat(sequence: ending) {
+        throw ParseError.expectedGroupName
+      }
+      if src.peek()!.isNumber {
+        throw ParseError.groupNameCannotStartWithNumber
+      }
+      guard let str = src.tryEatPrefix(\.isWordCharacter)?.string else {
+        throw ParseError.groupNameMustBeAlphaNumeric
+      }
+      return str
+    }
+    if eatEnding {
+      try expect(sequence: ending)
+    }
+    return str
+  }
+
+  /// Consume a named group field, producing either a named capture or balanced
+  /// capture.
+  ///
+  ///     NamedGroup    -> 'P<' GroupNameBody '>'
+  ///                    | '<' GroupNameBody '>'
+  ///                    | "'" GroupNameBody "'"
+  ///     GroupNameBody -> \w+ | \w* '-' \w+
+  ///
+  private mutating func expectNamedGroup(
+    endingWith ending: String
+  ) throws -> AST.Group.Kind {
+    func lexBalanced(_ lhs: Located<String>? = nil) throws -> AST.Group.Kind? {
+      // If we have a '-', this is a .NET-style 'balanced group'.
+      guard let dash = tryEatWithLoc("-") else { return nil }
+      let rhs = try expectGroupName(endingWith: ending)
+      return .balancedCapture(.init(name: lhs, dash: dash, priorName: rhs))
+    }
+
+    // Lex a group name, trying to lex a '-rhs' for a balanced capture group
+    // both before and after.
+    if let b = try lexBalanced() { return b }
+    let name = try expectGroupName(endingWith: ending, eatEnding: false)
+    if let b = try lexBalanced(name) { return b }
+
+    try expect(sequence: ending)
+    return .namedCapture(name)
+  }
+
   /// Try to consume the start of a group
   ///
   ///     GroupStart -> '(?' GroupKind | '('
-  ///     GroupKind  -> Named | ':' | '|' | '>' | '=' | '!' | '*' | '<=' | '<!'
-  ///                 | '<*' | MatchingOptionSeq (':' | ')')
-  ///     Named      -> '<' [^'>']+ '>' | 'P<' [^'>']+ '>'
-  ///                 | '\'' [^'\'']+ '\''
+  ///     GroupKind  -> ':' | '|' | '>' | '=' | '!' | '*' | '<=' | '<!' | '<*'
+  ///                 | NamedGroup | MatchingOptionSeq (':' | ')')
   ///
   /// If `SyntaxOptions.experimentalGroups` is enabled, also accepts:
   ///
@@ -709,16 +757,11 @@ extension Source {
           if src.tryEat(sequence: "<*") { return .nonAtomicLookbehind }
 
           // Named
-          // TODO: Group name validation, PCRE (and ICU + Oniguruma as far as I
-          // can tell), enforce word characters only, with the first character
-          // being a non-digit.
           if src.tryEat("<") || src.tryEat(sequence: "P<") {
-            let name = try src.expectQuoted(endingWith: ">")
-            return .namedCapture(name)
+            return try src.expectNamedGroup(endingWith: ">")
           }
           if src.tryEat("'") {
-            let name = try src.expectQuoted(endingWith: "'")
-            return .namedCapture(name)
+            return try src.expectNamedGroup(endingWith: "'")
           }
 
           // Matching option changing group (?iJmnsUxxxDPSWy{..}-iJmnsUxxxDPSW:).
@@ -853,9 +896,9 @@ extension Source {
         // FIXME: This should apply to future groups too.
         // TODO: We should probably advise users to use the more explicit
         // syntax.
-        let nameRef = try src.expectNamedReference(
-          endingWith: ")", eatEnding: false)
-        if context.isPriorGroupRef(nameRef.kind) {
+        if let nameRef = src.lexNamedReference(endingWith: ")",
+                                               eatEnding: false),
+           context.isPriorGroupRef(nameRef.kind) {
           return .groupMatched(nameRef)
         }
         return nil
@@ -1046,11 +1089,20 @@ extension Source {
   private mutating func expectNamedReference(
     endingWith end: String, eatEnding: Bool = true
   ) throws -> AST.Reference {
-    // TODO: Group name validation, see comment in lexGroupStart.
-    let str = try expectQuoted(endingWith: end, eatEnding: eatEnding)
+    let str = try expectGroupName(endingWith: end, eatEnding: eatEnding)
     return .init(.named(str.value), innerLoc: str.location)
   }
 
+  /// Try to consume a named reference up to a closing delimiter, returning
+  /// `nil` if the characters aren't valid for a named reference.
+  private mutating func lexNamedReference(
+    endingWith end: String, eatEnding: Bool = true
+  ) -> AST.Reference? {
+    tryEating { src in
+      try? src.expectNamedReference(endingWith: end, eatEnding: eatEnding)
+    }
+  }
+
   /// Try to lex a numbered reference, or otherwise a named reference.
   ///
   ///     NameOrNumberRef -> NumberRef | <String>

Sources/_MatchingEngine/Regex/Printing/DumpAST.swift

@@ -156,19 +156,20 @@ extension AST.Reference: _ASTPrintable {
 extension AST.Group.Kind: _ASTPrintable {
   public var _dumpBase: String {
     switch self {
-    case .capture:               return "capture"
-    case .namedCapture(let s):   return "capture<\(s.value)>"
-    case .nonCapture:            return "nonCapture"
-    case .nonCaptureReset:       return "nonCaptureReset"
-    case .atomicNonCapturing:    return "atomicNonCapturing"
-    case .lookahead:             return "lookahead"
-    case .negativeLookahead:     return "negativeLookahead"
-    case .nonAtomicLookahead:    return "nonAtomicLookahead"
-    case .lookbehind:            return "lookbehind"
-    case .negativeLookbehind:    return "negativeLookbehind"
-    case .nonAtomicLookbehind:   return "nonAtomicLookbehind"
-    case .scriptRun:             return "scriptRun"
-    case .atomicScriptRun:       return "atomicScriptRun"
+    case .capture:                return "capture"
+    case .namedCapture(let s):    return "capture<\(s.value)>"
+    case .balancedCapture(let b): return "balanced capture \(b)"
+    case .nonCapture:             return "nonCapture"
+    case .nonCaptureReset:        return "nonCaptureReset"
+    case .atomicNonCapturing:     return "atomicNonCapturing"
+    case .lookahead:              return "lookahead"
+    case .negativeLookahead:      return "negativeLookahead"
+    case .nonAtomicLookahead:     return "nonAtomicLookahead"
+    case .lookbehind:             return "lookbehind"
+    case .negativeLookbehind:     return "negativeLookbehind"
+    case .nonAtomicLookbehind:    return "nonAtomicLookbehind"
+    case .scriptRun:              return "scriptRun"
+    case .atomicScriptRun:        return "atomicScriptRun"
     case .changeMatchingOptions(let seq, let isIsolated):
       return "changeMatchingOptions<\(seq), \(isIsolated)>"
     }
@@ -257,3 +258,9 @@ extension AST.Atom.BacktrackingDirective: _ASTPrintable {
     return result
   }
 }
+
+extension AST.Group.BalancedCapture: _ASTPrintable {
+  public var _dumpBase: String {
+   "\(name?.value ?? "")-\(priorName.value)"
+  }
+}

Sources/_MatchingEngine/Regex/Printing/PrintAsCanonical.swift

@@ -138,19 +138,20 @@ extension AST.Quote {
 extension AST.Group.Kind {
   var _canonicalBase: String {
     switch self {
-    case .capture:              return "("
-    case .namedCapture(let n):  return "(?<\(n.value)>"
-    case .nonCapture:           return "(?:"
-    case .nonCaptureReset:      return "(?|"
-    case .atomicNonCapturing:   return "(?>"
-    case .lookahead:            return "(?="
-    case .negativeLookahead:    return "(?!"
-    case .nonAtomicLookahead:   return "(?*"
-    case .lookbehind:           return "(?<="
-    case .negativeLookbehind:   return "(?<!"
-    case .nonAtomicLookbehind:  return "(?<*"
-    case .scriptRun:            return "(*sr:"
-    case .atomicScriptRun:      return "(*asr:"
+    case .capture:                return "("
+    case .namedCapture(let n):    return "(?<\(n.value)>"
+    case .balancedCapture(let b): return "(?<\(b._canonicalBase)>"
+    case .nonCapture:             return "(?:"
+    case .nonCaptureReset:        return "(?|"
+    case .atomicNonCapturing:     return "(?>"
+    case .lookahead:              return "(?="
+    case .negativeLookahead:      return "(?!"
+    case .nonAtomicLookahead:     return "(?*"
+    case .lookbehind:             return "(?<="
+    case .negativeLookbehind:     return "(?<!"
+    case .nonAtomicLookbehind:    return "(?<*"
+    case .scriptRun:              return "(*sr:"
+    case .atomicScriptRun:        return "(*asr:"
 
     case .changeMatchingOptions:
       return "(/* TODO: matchign options in canonical form */"
@@ -220,3 +221,9 @@ extension AST.Reference {
 extension AST.CustomCharacterClass.Start {
   var _canonicalBase: String { self.rawValue }
 }
+
+extension AST.Group.BalancedCapture {
+  var _canonicalBase: String {
+    "\(name?.value ?? "")-\(priorName.value)"
+  }
+}

Sources/_MatchingEngine/Regex/Printing/PrintAsPattern.swift

@@ -286,6 +286,9 @@ extension AST.Group.Kind {
     case .namedCapture(let n):
       return "name: \"\(n)\""
 
+    case .balancedCapture:
+      return "/* TODO: balanced captures */"
+
     case .nonCapture: return ""
 
     case .nonCaptureReset:

Sources/_MatchingEngine/Utility/MissingUnicode.swift

@@ -657,5 +657,7 @@ public enum OnigurumaSpecialProperty: String, Hashable {
 }
 
 extension Character {
-  var isOctalDigit: Bool { ("0"..."7").contains(self) }
+  public var isOctalDigit: Bool { ("0"..."7").contains(self) }
+
+  public var isWordCharacter: Bool { isLetter || isNumber || self == "_" }
 }

Sources/_StringProcessing/ASTBuilder.swift

@@ -68,6 +68,12 @@ func namedCapture(
 ) -> AST {
   group(.namedCapture(.init(faking: name)), child)
 }
+func balancedCapture(name: String?, priorName: String, _ child: AST) -> AST {
+  group(.balancedCapture(
+    .init(name: name.map { .init(faking: $0) }, dash: .fake,
+          priorName: .init(faking: priorName))
+  ), child)
+}
 func nonCaptureReset(
   _ child: AST
 ) -> AST {

Sources/_StringProcessing/CharacterClass.swift

@@ -148,7 +148,7 @@ public struct CharacterClass: Hashable {
       case .newlineSequence: matched = c.isNewline
       case .verticalWhitespace: fatalError("Not implemented")
       case .whitespace: matched = c.isWhitespace
-      case .word: matched = c.isLetter || c.isNumber || c == "_"
+      case .word: matched = c.isWordCharacter
       case .custom(let set): matched = set.any { $0.matches(c) }
       }
       if isInverted {