Introduce scalar sequences \u{AA BB CC}

hamishknight · hamishknight · commit f436ccad016d · 2022-05-10T11:31:32.000+01:00
Allow a whitespace-separated list of scalars within
the `\u{...}` syntax. This is syntactic sugar that
gets implicitly splatted out, for example `\u{A B C}`
becomes `\u{A}\u{B}\u{C}`.
diff --git a/Sources/_RegexParser/Regex/AST/Atom.swift b/Sources/_RegexParser/Regex/AST/Atom.swift
@@ -31,6 +31,12 @@ extension AST {
       /// \u{...}, \0dd, \x{...}, ...
       case scalar(Scalar)
 
+      /// A whitespace-separated sequence of Unicode scalar values which are
+      /// implicitly splatted out.
+      ///
+      /// `\u{A B C}` -> `\u{A}\u{B}\u{C}`
+      case scalarSequence(ScalarSequence)
+
       /// A Unicode property, category, or script, including those written using
       /// POSIX syntax.
       ///
@@ -84,6 +90,7 @@ extension AST.Atom {
     switch kind {
     case .char(let v):                  return v
     case .scalar(let v):                return v
+    case .scalarSequence(let v):        return v
     case .property(let v):              return v
     case .escaped(let v):               return v
     case .keyboardControl(let v):       return v
@@ -116,6 +123,18 @@ extension AST.Atom {
       self.location = location
     }
   }
+
+  public struct ScalarSequence: Hashable {
+    public var scalars: [Scalar]
+    public var trivia: [AST.Trivia]
+
+    public init(_ scalars: [Scalar], trivia: [AST.Trivia]) {
+      precondition(scalars.count > 1, "Expected multiple scalars")
+      self.scalars = scalars
+      self.trivia = trivia
+    }
+    public var scalarValues: [Unicode.Scalar] { scalars.map(\.value) }
+  }
 }
 
 extension AST.Atom {
@@ -725,8 +744,9 @@ extension AST.Atom {
       // the AST? Or defer for the matching engine?
       return nil
 
-    case .property, .any, .startOfLine, .endOfLine, .backreference, .subpattern,
-        .callout, .backtrackingDirective, .changeMatchingOptions:
+    case .scalarSequence, .property, .any, .startOfLine, .endOfLine,
+        .backreference, .subpattern, .callout, .backtrackingDirective,
+        .changeMatchingOptions:
       return nil
     }
   }
@@ -748,13 +768,21 @@ extension AST.Atom {
   /// A string literal representation of the atom, if possible.
   ///
   /// Individual characters are returned as-is, and Unicode scalars are
-  /// presented using "\u{nnnn}" syntax.
+  /// presented using "\u{nn nn ...}" syntax.
   public var literalStringValue: String? {
+    func scalarLiteral(_ u: [UnicodeScalar]) -> String {
+      let digits = u.map { String($0.value, radix: 16, uppercase: true) }
+        .joined(separator: " ")
+      return "\\u{\(digits)}"
+    }
     switch kind {
     case .char(let c):
       return String(c)
     case .scalar(let s):
-      return "\\u{\(String(s.value.value, radix: 16, uppercase: true))}"
+      return scalarLiteral([s.value])
+
+    case .scalarSequence(let s):
+      return scalarLiteral(s.scalarValues)
 
     case .keyboardControl(let x):
       return "\\C-\(x)"
diff --git a/Sources/_RegexParser/Regex/Parse/LexicalAnalysis.swift b/Sources/_RegexParser/Regex/Parse/LexicalAnalysis.swift
@@ -290,10 +290,54 @@ extension Source {
     return try Source.validateUnicodeScalar(str, .hex)
   }
 
+  /// Try to lex a seqence of hex digit unicode scalars.
+  ///
+  ///     UniScalarSequence   -> Whitespace? UniScalarSequencElt+
+  ///     UniScalarSequencElt -> HexDigit{1...} Whitespace?
+  ///
+  mutating func expectUnicodeScalarSequence(
+    eating ending: Character
+  ) throws -> AST.Atom.Kind {
+    try recordLoc { src in
+      var scalars = [AST.Atom.Scalar]()
+      var trivia = [AST.Trivia]()
+
+      // Eat up any leading whitespace.
+      if let t = src.lexWhitespace() { trivia.append(t) }
+
+      while true {
+        let str = src.lexUntil { src in
+          // Hit the ending, stop lexing.
+          if src.isEmpty || src.peek() == ending {
+            return true
+          }
+          // Eat up trailing whitespace, and stop lexing to record the scalar.
+          if let t = src.lexWhitespace() {
+            trivia.append(t)
+            return true
+          }
+          // Not the ending or trivia, must be a digit of the scalar.
+          return false
+        }
+        guard !str.value.isEmpty else { break }
+        scalars.append(try Source.validateUnicodeScalar(str, .hex))
+      }
+      guard !scalars.isEmpty else {
+        throw ParseError.expectedNumber("", kind: .hex)
+      }
+      try src.expect(ending)
+
+      if scalars.count == 1 {
+        return .scalar(scalars[0])
+      }
+      return .scalarSequence(.init(scalars, trivia: trivia))
+    }.value
+  }
+
   /// Eat a scalar off the front, starting from after the
   /// backslash and base character (e.g. `\u` or `\x`).
   ///
-  ///     UniScalar -> 'u{' HexDigit{1...} '}'
+  ///     UniScalar -> 'u{' UniScalarSequence '}'
   ///                | 'u'  HexDigit{4}
   ///                | 'x{' HexDigit{1...} '}'
   ///                | 'x'  HexDigit{0...2}
@@ -314,7 +358,10 @@ extension Source {
       // TODO: PCRE offers a different behavior if PCRE2_ALT_BSUX is set.
       switch base {
       // Hex numbers.
-      case "u" where src.tryEat("{"), "x" where src.tryEat("{"):
+      case "u" where src.tryEat("{"):
+        return try src.expectUnicodeScalarSequence(eating: "}")
+
+      case "x" where src.tryEat("{"):
         let str = try src.lexUntil(eating: "}")
         return .scalar(try Source.validateUnicodeScalar(str, .hex))
 
@@ -598,6 +645,16 @@ extension Source {
     // inside a custom character class (and only treats whitespace as
     // non-semantic there for the extra-extended `(?xx)` mode). If we get a
     // strict-PCRE mode, we'll need to add a case for that.
+    return lexWhitespace()
+  }
+
+  /// Try to consume whitespace as trivia
+  ///
+  ///     Whitespace -> WhitespaceChar+
+  ///
+  /// Unlike `lexNonSemanticWhitespace`, this will always attempt to lex
+  /// whitespace.
+  mutating func lexWhitespace() -> AST.Trivia? {
     let trivia: Located<String>? = recordLoc { src in
       src.tryEatPrefix(\.isPatternWhitespace)?.string
     }
diff --git a/Sources/_RegexParser/Regex/Parse/Sema.swift b/Sources/_RegexParser/Regex/Parse/Sema.swift
@@ -210,7 +210,7 @@ extension RegexValidator {
     }
   }
 
-  func validateAtom(_ atom: AST.Atom) throws {
+  func validateAtom(_ atom: AST.Atom, inCustomCharacterClass: Bool) throws {
     switch atom.kind {
     case .escaped(let esc):
       try validateEscaped(esc, at: atom.location)
@@ -243,6 +243,13 @@ extension RegexValidator {
       // TODO: We should error on unknown Unicode scalar names.
       break
 
+    case .scalarSequence:
+      // Not currently supported in a custom character class.
+      if inCustomCharacterClass {
+        throw error(.unsupported("scalar sequence in custom character class"),
+                    at: atom.location)
+      }
+
     case .char, .scalar, .startOfLine, .endOfLine, .any:
       break
     }
@@ -260,8 +267,8 @@ extension RegexValidator {
     let lhs = range.lhs
     let rhs = range.rhs
 
-    try validateAtom(lhs)
-    try validateAtom(rhs)
+    try validateAtom(lhs, inCustomCharacterClass: true)
+    try validateAtom(rhs, inCustomCharacterClass: true)
 
     guard lhs.isValidCharacterClassRangeBound else {
       throw error(.invalidCharacterClassRangeOperand, at: lhs.location)
@@ -297,7 +304,7 @@ extension RegexValidator {
       try validateCharacterClassRange(r)
 
     case .atom(let a):
-      try validateAtom(a)
+      try validateAtom(a, inCustomCharacterClass: true)
 
     case .setOperation(let lhs, _, let rhs):
       for lh in lhs { try validateCharacterClassMember(lh) }
@@ -379,7 +386,7 @@ extension RegexValidator {
       try validateQuantification(q)
 
     case .atom(let a):
-      try validateAtom(a)
+      try validateAtom(a, inCustomCharacterClass: false)
 
     case .customCharacterClass(let c):
       try validateCustomCharacterClass(c)
diff --git a/Sources/_RegexParser/Regex/Printing/DumpAST.swift b/Sources/_RegexParser/Regex/Printing/DumpAST.swift
@@ -138,6 +138,9 @@ extension AST.Atom {
     switch kind {
     case .escaped(let c): return "\\\(c.character)"
 
+    case .scalarSequence(let s):
+      return s.scalars.map(\.value.halfWidthCornerQuoted).joined()
+
     case .namedCharacter(let charName):
       return "\\N{\(charName)}"
 
diff --git a/Sources/_StringProcessing/ConsumerInterface.swift b/Sources/_StringProcessing/ConsumerInterface.swift
@@ -230,9 +230,9 @@ extension AST.Atom {
       // handled in emitAssertion
       return nil
 
-    case .escaped, .keyboardControl, .keyboardMeta, .keyboardMetaControl,
-        .backreference, .subpattern, .callout, .backtrackingDirective,
-        .changeMatchingOptions:
+    case .scalarSequence, .escaped, .keyboardControl, .keyboardMeta,
+        .keyboardMetaControl, .backreference, .subpattern, .callout,
+        .backtrackingDirective, .changeMatchingOptions:
       // FIXME: implement
       return nil
     }
diff --git a/Sources/_StringProcessing/PrintAsPattern.swift b/Sources/_StringProcessing/PrintAsPattern.swift
@@ -671,13 +671,19 @@ extension AST.Atom {
   }
   
   var _dslBase: String {
+    func scalarLiteral(_ s: UnicodeScalar) -> String {
+      let hex = String(s.value, radix: 16, uppercase: true)
+      return "\\u{\(hex)}"
+    }
     switch kind {
     case let .char(c):
       return String(c)
 
     case let .scalar(s):
-      let hex = String(s.value.value, radix: 16, uppercase: true)
-      return "\\u{\(hex)}"
+      return scalarLiteral(s.value)
+
+    case let .scalarSequence(seq):
+      return seq.scalarValues.map(scalarLiteral).joined()
 
     case let .property(p):
       return p._dslBase
@@ -769,13 +775,9 @@ extension AST.Atom {
   
   var _regexBase: String {
     switch kind {
-    case let .char(c):
-      return String(c)
-      
-    case let .scalar(s):
-      let hex = String(s.value.value, radix: 16, uppercase: true)
-      return "\\u{\(hex)}"
-      
+    case .char, .scalar, .scalarSequence:
+      return literalStringValue!
+
     case let .property(p):
       return p._regexBase
       
diff --git a/Sources/_StringProcessing/Regex/ASTConversion.swift b/Sources/_StringProcessing/Regex/ASTConversion.swift
@@ -60,15 +60,17 @@ extension AST.Node {
           var result = ""
           var idx = idx
           while idx < astChildren.endIndex {
-            let atom: AST.Atom? = astChildren[idx].as()
+            guard let atom: AST.Atom = astChildren[idx].as() else { break }
 
             // TODO: For printing, nice to coalesce
             // scalars literals too. We likely need a different
             // approach even before we have a better IR.
-            if let char = atom?.singleCharacter  {
+            if let char = atom.singleCharacter  {
               result.append(char)
-            } else if let scalar = atom?.singleScalar {
+            } else if let scalar = atom.singleScalar {
               result.append(Character(scalar))
+            } else if case .scalarSequence(let seq) = atom.kind {
+              result += seq.scalarValues.map(Character.init)
             } else {
               break
             }
@@ -136,7 +138,15 @@ extension AST.Node {
         return .trivia(v.contents)
 
       case let .atom(v):
-        return .atom(v.dslTreeAtom)
+        switch v.kind {
+        case .scalarSequence(let seq):
+          // Scalar sequences are splatted into concatenated scalars, which
+          // becomes a quoted literal. Sequences nested in concatenations have
+          // already been coalesced, this just handles the lone atom case.
+          return .quotedLiteral(String(seq.scalarValues.map(Character.init)))
+        default:
+          return .atom(v.dslTreeAtom)
+        }
 
       case let .customCharacterClass(ccc):
         return .customCharacterClass(ccc.dslTreeClass)
diff --git a/Sources/_StringProcessing/Utility/ASTBuilder.swift b/Sources/_StringProcessing/Utility/ASTBuilder.swift
@@ -338,10 +338,26 @@ func escaped(
   atom(.escaped(e))
 }
 func scalar(_ s: Unicode.Scalar) -> AST.Node {
-  atom(.scalar(.init(s, .fake)))
+  .atom(scalar_a(s))
+}
+func scalar_a(_ s: Unicode.Scalar) -> AST.Atom {
+  atom_a(.scalar(.init(s, .fake)))
 }
 func scalar_m(_ s: Unicode.Scalar) -> AST.CustomCharacterClass.Member {
-  atom_m(.scalar(.init(s, .fake)))
+  .atom(scalar_a(s))
+}
+
+func scalarSeq(_ s: Unicode.Scalar...) -> AST.Node {
+  .atom(scalarSeq_a(s))
+}
+func scalarSeq_a(_ s: Unicode.Scalar...) -> AST.Atom {
+  scalarSeq_a(s)
+}
+func scalarSeq_a(_ s: [Unicode.Scalar]) -> AST.Atom {
+  atom_a(.scalarSequence(.init(s.map { .init($0, .fake) }, trivia: [])))
+}
+func scalarSeq_m(_ s: Unicode.Scalar...) -> AST.CustomCharacterClass.Member {
+  .atom(scalarSeq_a(s))
 }
 
 func backreference(_ r: AST.Reference.Kind, recursionLevel: Int? = nil) -> AST.Node {
diff --git a/Tests/RegexTests/MatchTests.swift b/Tests/RegexTests/MatchTests.swift
@@ -285,7 +285,20 @@ extension RegexTests {
     firstMatchTest(#"\0707"#, input: "12387\u{1C7}xyz", match: "\u{1C7}")
 
     // code point sequence
-    firstMatchTest(#"\u{61 62 63}"#, input: "123abcxyz", match: "abc", xfail: true)
+    firstMatchTest(#"\u{61 62 63}"#, input: "123abcxyz", match: "abc")
+    firstMatchTest(#"3\u{  61  62 63 }"#, input: "123abcxyz", match: "3abc")
+    firstMatchTest(#"\u{61 62}\u{63}"#, input: "123abcxyz", match: "abc")
+    firstMatchTest(#"\u{61}\u{62 63}"#, input: "123abcxyz", match: "abc")
+    firstMatchTest(#"9|\u{61 62 63}"#, input: "123abcxyz", match: "abc")
+    firstMatchTest(#"(?:\u{61 62 63})"#, input: "123abcxyz", match: "abc")
+    firstMatchTest(#"23\u{61 62 63}xy"#, input: "123abcxyz", match: "23abcxy")
+
+    // o + horn + dot_below
+    firstMatchTest(
+      #"\u{006f 031b 0323}"#,
+      input: "\u{006f}\u{031b}\u{0323}",
+      match: "\u{006f}\u{031b}\u{0323}"
+    )
 
     // Escape sequences that represent scalar values.
     firstMatchTest(#"\a[\b]\e\f\n\r\t"#,
@@ -1405,6 +1418,9 @@ extension RegexTests {
     firstMatchTest(#"\u{65}\u{301}$"#, input: eDecomposed, match: eDecomposed)
     firstMatchTest(#"\u{65}\u{301}$"#, input: eComposed, match: eComposed)
 
+    firstMatchTest(#"\u{65 301}$"#, input: eDecomposed, match: eDecomposed)
+    firstMatchTest(#"\u{65 301}$"#, input: eComposed, match: eComposed)
+
     // FIXME: Implicit \y at end of match
     firstMatchTest(#"\u{65}"#, input: eDecomposed, match: nil,
       xfail: true)
@@ -1516,7 +1532,8 @@ extension RegexTests {
     firstMatchTest(#"🇰🇷"#, input: flag, match: flag)
     firstMatchTest(#"[🇰🇷]"#, input: flag, match: flag)
     firstMatchTest(#"\u{1F1F0}\u{1F1F7}"#, input: flag, match: flag)
-    
+    firstMatchTest(#"\u{1F1F0 1F1F7}"#, input: flag, match: flag)
+
     // First Unicode scalar followed by CCC of regional indicators
     firstMatchTest(#"\u{1F1F0}[\u{1F1E6}-\u{1F1FF}]"#, input: flag, match: flag,
               xfail: true)
diff --git a/Tests/RegexTests/ParseTests.swift b/Tests/RegexTests/ParseTests.swift
