Merge pull request #237 from hamishknight/mix-n-match

Author: hamishknight

Sources/_RegexParser/Regex/AST/Atom.swift

@@ -72,6 +72,9 @@ extension AST {
 
       // (*ACCEPT), (*FAIL), ...
       case backtrackingDirective(BacktrackingDirective)
+
+      // (?i), (?i-m), ...
+      case changeMatchingOptions(MatchingOptionSequence)
     }
   }
 }
@@ -91,6 +94,7 @@ extension AST.Atom {
     case .subpattern(let v):            return v
     case .callout(let v):               return v
     case .backtrackingDirective(let v): return v
+    case .changeMatchingOptions(let v): return v
     case .any:                          return nil
     case .startOfLine:                  return nil
     case .endOfLine:                    return nil
@@ -691,7 +695,7 @@ extension AST.Atom {
       return nil
 
     case .property, .any, .startOfLine, .endOfLine, .backreference, .subpattern,
-        .callout, .backtrackingDirective:
+        .callout, .backtrackingDirective, .changeMatchingOptions:
       return nil
     }
   }
@@ -731,7 +735,7 @@ extension AST.Atom {
 
     case .property, .escaped, .any, .startOfLine, .endOfLine,
         .backreference, .subpattern, .namedCharacter, .callout,
-        .backtrackingDirective:
+        .backtrackingDirective, .changeMatchingOptions:
       return nil
     }
   }
@@ -740,6 +744,8 @@ extension AST.Atom {
     switch kind {
     case .backtrackingDirective(let b):
       return b.isQuantifiable
+    case .changeMatchingOptions:
+      return false
     // TODO: Are callouts quantifiable?
     default:
       return true

Sources/_RegexParser/Regex/AST/Group.swift

@@ -68,9 +68,7 @@ extension AST {
       case atomicScriptRun
 
       // (?iJmnsUxxxDPSWy{..}-iJmnsUxxxDPSW:)
-      // Isolated options are written as e.g (?i), and implicitly form a group
-      // containing all the following elements of the current group.
-      case changeMatchingOptions(MatchingOptionSequence, isIsolated: Bool)
+      case changeMatchingOptions(MatchingOptionSequence)
 
       // NOTE: Comments appear to be groups, but are not parsed
       // the same. They parse more like quotes, so are not
@@ -87,21 +85,6 @@ extension AST.Group.Kind {
     }
   }
 
-  /// Whether this is a group with an implicit scope, e.g isolated matching
-  /// options implicitly become parent groups for the rest of the elements in
-  /// the current group:
-  ///
-  ///      (a(?i)bc)de -> (a(?i:bc))de
-  ///
-  public var hasImplicitScope: Bool {
-    switch self {
-    case .changeMatchingOptions(_, let isIsolated):
-      return isIsolated
-    default:
-      return false
-    }
-  }
-
   /// If this is a named group, its name, `nil` otherwise.
   public var name: String? {
     switch self {

Sources/_RegexParser/Regex/Parse/LexicalAnalysis.swift

@@ -21,6 +21,16 @@ API convention:
 - eat() and tryEat() is still used by the parser as a character-by-character interface
 */
 
+extension Error {
+  func addingLocation(_ loc: Range<Source.Position>) -> Error {
+    // If we're already a LocatedError, don't change the location.
+    if self is _LocatedErrorProtocol {
+      return self
+    }
+    return Source.LocatedError<Self>(self, loc)
+  }
+}
+
 extension Source {
   // MARK: - recordLoc
 
@@ -51,12 +61,8 @@ extension Source {
     do {
       guard let result = try f(&self) else { return nil }
       return Located(result, start..<currentPosition)
-    } catch let e as Source.LocatedError<ParseError> {
-      throw e
-    } catch let e as ParseError {
-      throw LocatedError(e, start..<currentPosition)
-    } catch {
-      fatalError("FIXME: Let's not keep the boxed existential...")
+    } catch let e {
+      throw e.addingLocation(start..<currentPosition)
     }
   }
 
@@ -706,6 +712,22 @@ extension Source {
     return .init(caretLoc: nil, adding: adding, minusLoc: nil, removing: [])
   }
 
+  /// A matching option changing atom.
+  ///
+  ///     '(?' MatchingOptionSeq ')'
+  ///
+  mutating func lexChangeMatchingOptionAtom(
+    context: ParsingContext
+  ) throws -> AST.MatchingOptionSequence? {
+    try tryEating { src in
+      guard src.tryEat(sequence: "(?"),
+            let seq = try src.lexMatchingOptionSequence(context: context)
+      else { return nil }
+      try src.expect(")")
+      return seq
+    }
+  }
+
   /// Try to consume explicitly spelled-out PCRE2 group syntax.
   mutating func lexExplicitPCRE2GroupStart() -> AST.Group.Kind? {
     tryEating { src in
@@ -846,7 +868,7 @@ extension Source {
         // otherwise a matching option specifier. Conversely, '(?P' can be the
         // start of a matching option sequence, or a reference if it is followed
         // by '=' or '<'.
-        guard !src.shouldLexGroupLikeAtom() else { return nil }
+        guard !src.shouldLexGroupLikeAtom(context: context) else { return nil }
 
         guard src.tryEat("(") else { return nil }
         if src.tryEat("?") {
@@ -871,22 +893,13 @@ extension Source {
 
           // Matching option changing group (?iJmnsUxxxDPSWy{..}-iJmnsUxxxDPSW:).
           if let seq = try src.lexMatchingOptionSequence(context: context) {
-            if src.tryEat(":") {
-              return .changeMatchingOptions(seq, isIsolated: false)
-            }
-            // If this isn't start of an explicit group, we should have an
-            // implicit group that covers the remaining elements of the current
-            // group.
-            // TODO: This implicit scoping behavior matches Oniguruma, but PCRE
-            // also does it across alternations, which will require additional
-            // handling.
-            guard src.tryEat(")") else {
+            guard src.tryEat(":") else {
               if let next = src.peek() {
                 throw ParseError.invalidMatchingOption(next)
               }
               throw ParseError.expected(")")
             }
-            return .changeMatchingOptions(seq, isIsolated: true)
+            return .changeMatchingOptions(seq)
           }
 
           guard let next = src.peek() else {
@@ -1035,18 +1048,8 @@ extension Source {
     context: ParsingContext
   ) throws -> Located<AST.Group.Kind>? {
     try tryEating { src in
-      guard src.tryEat(sequence: "(?"),
-            let group = try src.lexGroupStart(context: context)
-      else { return nil }
-
-      // Implicitly scoped groups are not supported here.
-      guard !group.value.hasImplicitScope else {
-        throw LocatedError(
-          ParseError.unsupportedCondition("implicitly scoped group"),
-          group.location
-        )
-      }
-      return group
+      guard src.tryEat(sequence: "(?") else { return nil }
+      return try src.lexGroupStart(context: context)
     }
   }
 
@@ -1233,17 +1236,19 @@ extension Source {
     allowWholePatternRef: Bool = false, allowRecursionLevel: Bool = false
   ) throws -> AST.Reference? {
     let kind = try recordLoc { src -> AST.Reference.Kind? in
-      // Note this logic should match canLexNumberedReference.
-      if src.tryEat("+") {
-        return .relative(try src.expectNumber().value)
-      }
-      if src.tryEat("-") {
-        return .relative(try -src.expectNumber().value)
-      }
-      if let num = try src.lexNumber() {
-        return .absolute(num.value)
+      try src.tryEating { src in
+        // Note this logic should match canLexNumberedReference.
+        if src.tryEat("+"), let num = try src.lexNumber() {
+          return .relative(num.value)
+        }
+        if src.tryEat("-"), let num = try src.lexNumber() {
+          return .relative(-num.value)
+        }
+        if let num = try src.lexNumber() {
+          return .absolute(num.value)
+        }
+        return nil
       }
-      return nil
     }
     guard let kind = kind else { return nil }
     guard allowWholePatternRef || kind.value != .recurseWholePattern else {
@@ -1472,8 +1477,21 @@ extension Source {
     return src.canLexNumberedReference()
   }
 
+  private func canLexMatchingOptionsAsAtom(context: ParsingContext) -> Bool {
+    var src = self
+
+    // See if we can lex a matching option sequence that terminates in ')'. Such
+    // a sequence is an atom. If an error is thrown, there are invalid elements
+    // of the matching option sequence. In such a case, we can lex as a group
+    // and diagnose the invalid group kind.
+    guard (try? src.lexMatchingOptionSequence(context: context)) != nil else {
+      return false
+    }
+    return src.tryEat(")")
+  }
+
   /// Whether a group specifier should be lexed as an atom instead of a group.
-  private func shouldLexGroupLikeAtom() -> Bool {
+  private func shouldLexGroupLikeAtom(context: ParsingContext) -> Bool {
     var src = self
     guard src.tryEat("(") else { return false }
 
@@ -1487,6 +1505,9 @@ extension Source {
       // The start of an Oniguruma 'of-contents' callout.
       if src.tryEat("{") { return true }
 
+      // A matching option atom (?x), (?i), ...
+      if src.canLexMatchingOptionsAsAtom(context: context) { return true }
+
       return false
     }
     // The start of a backreference directive or Oniguruma named callout.
@@ -1747,13 +1768,20 @@ extension Source {
   ///
   ///     GroupLikeAtom -> GroupLikeReference | Callout | BacktrackingDirective
   ///
-  mutating func expectGroupLikeAtom() throws -> AST.Atom.Kind {
+  mutating func expectGroupLikeAtom(
+    context: ParsingContext
+  ) throws -> AST.Atom.Kind {
     try recordLoc { src in
       // References that look like groups, e.g (?R), (?1), ...
       if let ref = try src.lexGroupLikeReference() {
         return ref.value
       }
 
+      // Change matching options atom (?i), (?x-i), ...
+      if let seq = try src.lexChangeMatchingOptionAtom(context: context) {
+        return .changeMatchingOptions(seq)
+      }
+
       // (*ACCEPT), (*FAIL), (*MARK), ...
       if let b = try src.lexBacktrackingDirective() {
         return .backtrackingDirective(b)
@@ -1822,8 +1850,8 @@ extension Source {
 
       // If we have group syntax that was skipped over in lexGroupStart, we
       // need to handle it as an atom, or throw an error.
-      if !customCC && src.shouldLexGroupLikeAtom() {
-        return try src.expectGroupLikeAtom()
+      if !customCC && src.shouldLexGroupLikeAtom(context: context) {
+        return try src.expectGroupLikeAtom(context: context)
       }
 
       // A quantifier here is invalid.
@@ -1841,6 +1869,9 @@ extension Source {
         }
         throw Unreachable("TODO: reason")
 
+      case "(" where !customCC:
+        throw Unreachable("Should have lexed a group or group-like atom")
+
       // (sometimes) special metacharacters
       case ".": return customCC ? .char(".") : .any
       case "^": return customCC ? .char("^") : .startOfLine

Sources/_RegexParser/Regex/Parse/Parse.swift

@@ -282,42 +282,53 @@ extension Parser {
       loc(start)))
   }
 
+  /// Apply the syntax options of a given matching option sequence to the
+  /// current set of options.
+  private mutating func applySyntaxOptions(
+    of opts: AST.MatchingOptionSequence
+  ) {
+    // We skip this for multi-line, as extended syntax is always enabled there.
+    if context.syntax.contains(.multilineExtendedSyntax) { return }
+
+    // Check if we're introducing or removing extended syntax.
+    // TODO: PCRE differentiates between (?x) and (?xx) where only the latter
+    // handles non-semantic whitespace in a custom character class. Other
+    // engines such as Oniguruma, Java, and ICU do this under (?x). Therefore,
+    // treat (?x) and (?xx) as the same option here. If we ever get a strict
+    // PCRE mode, we will need to change this to handle that.
+    if opts.resetsCurrentOptions {
+      context.syntax.remove(.extendedSyntax)
+    }
+    if opts.adding.contains(where: \.isAnyExtended) {
+      context.syntax.insert(.extendedSyntax)
+    }
+    if opts.removing.contains(where: \.isAnyExtended) {
+      context.syntax.remove(.extendedSyntax)
+    }
+  }
+
+  /// Apply the syntax options of a matching option changing group to the
+  /// current set of options.
+  private mutating func applySyntaxOptions(of group: AST.Group.Kind) {
+    if case .changeMatchingOptions(let seq) = group {
+      applySyntaxOptions(of: seq)
+    }
+  }
+
   /// Perform a recursive parse for the body of a group.
   mutating func parseGroupBody(
     start: Source.Position, _ kind: AST.Located<AST.Group.Kind>
   ) throws -> AST.Group {
     context.recordGroup(kind.value)
 
-    // Check if we're introducing or removing extended syntax. We skip this for
-    // multi-line, as extended syntax is always enabled there.
-    // TODO: PCRE differentiates between (?x) and (?xx) where only the latter
-    // handles non-semantic whitespace in a custom character class. Other
-    // engines such as Oniguruma, Java, and ICU do this under (?x). Therefore,
-    // treat (?x) and (?xx) as the same option here. If we ever get a strict
-    // PCRE mode, we will need to change this to handle that.
     let currentSyntax = context.syntax
-    if !context.syntax.contains(.multilineExtendedSyntax) {
-      if case .changeMatchingOptions(let c, isIsolated: _) = kind.value {
-        if c.resetsCurrentOptions {
-          context.syntax.remove(.extendedSyntax)
-        }
-        if c.adding.contains(where: \.isAnyExtended) {
-          context.syntax.insert(.extendedSyntax)
-        }
-        if c.removing.contains(where: \.isAnyExtended) {
-          context.syntax.remove(.extendedSyntax)
-        }
-      }
-    }
+    applySyntaxOptions(of: kind.value)
     defer {
       context.syntax = currentSyntax
     }
 
     let child = try parseNode()
-    // An implicit scoped group has already consumed its closing paren.
-    if !kind.value.hasImplicitScope {
-      try source.expect(")")
-    }
+    try source.expect(")")
     return .init(kind, child, loc(start))
   }
 
@@ -409,6 +420,11 @@ extension Parser {
     }
 
     if let atom = try source.lexAtom(context: context) {
+      // If we have a change matching options atom, apply the syntax options. We
+      // already take care of scoping syntax options within a group.
+      if case .changeMatchingOptions(let opts) = atom.kind {
+        applySyntaxOptions(of: opts)
+      }
       // TODO: track source locations
       return .atom(atom)
     }