Add list-based compilation

This implements bytecode generation from a DSLList instead of a
DSLTree. The change includes tests that all regex patterns in the
`MatchTests` file produce the exact same bytecode from a list as from
a tree.

Author: natecook1000

Sources/_StringProcessing/ByteCodeGen+DSLList.swift

@@ -23,7 +23,7 @@ extension Compiler {
     var hasEmittedFirstMatchableAtom = false
 
     private let compileOptions: _CompileOptions
-    fileprivate var optimizationsEnabled: Bool {
+    internal var optimizationsEnabled: Bool {
       !compileOptions.contains(.disableOptimizations)
     }
 
@@ -61,7 +61,7 @@ extension Compiler.ByteCodeGen {
   }
 }
 
-fileprivate extension Compiler.ByteCodeGen {
+extension Compiler.ByteCodeGen {
   mutating func emitAtom(_ a: DSLTree.Atom) throws {
     defer {
       if a.isMatchable {

Sources/_StringProcessing/ByteCodeGen.swift

@@ -40,6 +40,20 @@ class Compiler {
       captureList: tree.captureList)
     return try codegen.emitRoot(tree.root)
   }
+  
+  __consuming func emitViaList() throws -> MEProgram {
+    // TODO: Handle global options
+    let dslList = DSLList(tree: tree)
+    for (n, el) in dslList.nodes.enumerated() {
+      print("\(n): \(el)")
+    }
+    var codegen = ByteCodeGen(
+      options: options,
+      compileOptions:
+        compileOptions,
+      captureList: tree.captureList)
+    return try codegen.emitRoot(dslList)
+  }
 }
 
 /// Hashable wrapper for `Any.Type`.

Sources/_StringProcessing/Compiler.swift

@@ -20,21 +20,6 @@ extension AST {
 extension AST.Node {
   /// Converts an AST node to a `convertedRegexLiteral` node.
   var dslTreeNode: DSLTree.Node {
-//    func wrap(_ node: DSLTree.Node) -> DSLTree.Node {
-//      switch node {
-//      case .convertedRegexLiteral(let child, _):
-//        // FIXME: DSL can have one item concats
-////        assertionFailure("Double wrapping?")
-//        return child
-//      default:
-//        break
-//      }
-//      // TODO: Should we do this for the
-//      // single-concatenation child too, or should?
-//      // we wrap _that_?
-//      return node
-//    }
-
     // Convert the top-level node without wrapping
     func convert() throws -> DSLTree.Node {
       switch self {
@@ -105,9 +90,7 @@ extension AST.Node {
       }
     }
 
-    // FIXME: make total function again
     let converted = try! convert()
-//    return wrap(converted)
     return converted
   }
 }

Sources/_StringProcessing/Regex/ASTConversion.swift

@@ -1,9 +1,13 @@
+//===----------------------------------------------------------------------===//
 //
-//  DSLList.swift
-//  swift-experimental-string-processing
+// This source file is part of the Swift.org open source project
 //
-//  Created by Nate Cook on 9/25/25.
+// Copyright (c) 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
 //
+// See https://swift.org/LICENSE.txt for license information
+//
+//===----------------------------------------------------------------------===//
 
 struct DSLList {
   var nodes: [DSLTree.Node]
@@ -16,8 +20,8 @@ struct DSLList {
     self.nodes = nodes
   }
 
-  init(root: DSLTree.Node) {
-    self.nodes = Array(root)
+  init(tree: DSLTree) {
+    self.nodes = Array(tree.depthFirst)
   }
 }
 
@@ -78,8 +82,8 @@ extension DSLTree.Node {
   }
 }
 
-extension DSLTree.Node: Sequence {
-  struct Iterator: Sequence, IteratorProtocol {
+extension DSLTree {
+  struct DepthFirst: Sequence, IteratorProtocol {
     typealias Element = DSLTree.Node
     private var stack: [Frame]
     private let getChildren: (Element) -> [Element]
@@ -104,11 +108,24 @@ extension DSLTree.Node: Sequence {
       for child in top.children.reversed() {
         stack.append(Frame(node: child, children: getChildren(child)))
       }
-      return top.node
+      
+      // Since we coalesce the children before adding them to the stack,
+      // we need an exact matching number of children in the list's
+      // concatenation node, so that it can provide the correct component
+      // count. This will go away/change when .concatenation only stores
+      // a count.
+      return switch top.node {
+      case .concatenation:
+        .concatenation(top.node.coalescedChildren)
+      default:
+        top.node
+      }
     }
   }
 
-  func makeIterator() -> Iterator {
-    Iterator(root: self, getChildren: { $0.children })
+  var depthFirst: DepthFirst {
+    DepthFirst(root: root, getChildren: {
+      $0.coalescedChildren
+    })
   }
 }

Sources/_StringProcessing/Regex/DSLList.swift

@@ -393,16 +393,72 @@ extension DSLTree.Node {
     switch self {
 
     case let .orderedChoice(v):   return v
-    case let .concatenation(v): return v
-
+    case let .concatenation(v):   return v
+      
     case let .capture(_, _, n, _):        return [n]
     case let .nonCapturingGroup(_, n):    return [n]
     case let .quantification(_, _, n):    return [n]
     case let .ignoreCapturesInTypedOutput(n): return [n]
+    case let .limitCaptureNesting(n):     return [n]
+      
+    case let .conditional(_, t, f): return [t,f]
+      
+    case .trivia, .empty, .quotedLiteral,
+        .consumer, .matcher, .characterPredicate,
+        .customCharacterClass, .atom:
+      return []
+      
+    case let .absentFunction(abs):
+      return abs.ast.children.map(\.dslTreeNode)
+    }
+  }
+  
+  public var coalescedChildren: [DSLTree.Node] {
+    // Before converting a concatenation in a tree to list form, we need to
+    // flatten out any nested concatenations, and coalesce any adjacent
+    // characters and scalars, forming quoted literals of their contents,
+    // over which we can perform grapheme breaking.
+
+    func flatten(_ node: DSLTree.Node) -> [DSLTree.Node] {
+      switch node {
+      case .concatenation(let ch):
+        return ch.flatMap(flatten)
+      case .ignoreCapturesInTypedOutput(let n), .limitCaptureNesting(let n):
+        return flatten(n)
+      default:
+        return [node]
+      }
+    }
 
-    case let .limitCaptureNesting(n):
-      // This is a transparent wrapper
-      return n.children
+    switch self {
+    case let .orderedChoice(v):   return v
+    case let .concatenation(v):
+      let children = v
+        .flatMap(flatten)
+        .coalescing(with: "", into: DSLTree.Node.quotedLiteral) { str, node in
+          switch node {
+          case .atom(let a):
+            guard let c = a.literalCharacterValue else { return false }
+            str.append(c)
+            return true
+          case .quotedLiteral(let q):
+            str += q
+            return true
+          case .trivia:
+            // Trivia can be completely ignored if we've already coalesced
+            // something.
+            return !str.isEmpty
+          default:
+            return false
+          }
+        }
+      return children
+
+    case let .capture(_, _, n, _):        return [n]
+    case let .nonCapturingGroup(_, n):    return [n]
+    case let .quantification(_, _, n):    return [n]
+    case let .ignoreCapturesInTypedOutput(n): return [n]
+    case let .limitCaptureNesting(n):     return [n]
 
     case let .conditional(_, t, f): return [t,f]
 

Sources/_StringProcessing/Regex/DSLTree.swift

@@ -1,21 +1,37 @@
+//===----------------------------------------------------------------------===//
 //
-//  DSLListTests.swift
-//  swift-experimental-string-processing
+// This source file is part of the Swift.org open source project
 //
-//  Created by Nate Cook on 9/25/25.
+// Copyright (c) 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
 //
+// See https://swift.org/LICENSE.txt for license information
+//
+//===----------------------------------------------------------------------===//
 
 import Testing
 @testable import _StringProcessing
 
 @Suite
 struct DSLListTests {
-  @Test(arguments: [(#/abc/#, 4), (#/a(?:b+)c*/#, 7)])
-  func simple(regex: Regex<Substring>, nodeCount: Int) {
-    let dslList = DSLList(root: regex.root)
+  @available(macOS 9999, *)
+  @Test(arguments: [
+    (#/a/#, 2),             // literal, a
+    (#/abcd+/#, 5),         // literal, concat, abc, quant, d
+    (#/a(?:b+)c*/#, 8),     // literal, concat, a, noncap grp, quant, b, quant, c
+  ])
+  func convertedNodeCount(regex: Regex<Substring>, nodeCount: Int) {
+    let dslList = DSLList(tree: regex.program.tree)
     #expect(dslList.nodes.count == nodeCount)
-    for (i, node) in dslList.nodes.enumerated() {
-      print(i, node)
-    }
+  }
+  
+  @Test(arguments: [#/a|b/#, #/a+b?c/#, #/abc/#, #/a(?:b+)c*/#, #/;[\r\n]/#, #/(?=(?:[1-9]|(?:a|b)))/#])
+  func compilationComparison(regex: Regex<Substring>) throws {
+    let listCompiler = Compiler(tree: regex.program.tree)
+    let listProgram = try listCompiler.emitViaList()
+    let treeCompiler = Compiler(tree: regex.program.tree)
+    let treeProgram = try treeCompiler.emit()
+
+    #expect(treeProgram.instructions == listProgram.instructions)
   }
 }

Tests/RegexTests/DSLListTests.swift

@@ -37,16 +37,34 @@ func _roundTripLiteral(
   return remadeRegex
 }
 
+func _validateListCompilation<T>(
+  _ regex: Regex<T>
+) throws -> Bool {
+  let treeCompiler = Compiler(tree: regex.program.tree)
+  let treeProgram = try treeCompiler.emit()
+  let listCompiler = Compiler(tree: regex.program.tree)
+  let listProgram = try listCompiler.emitViaList()
+  return treeProgram.instructions == listProgram.instructions
+}
+
 func _firstMatch(
   _ regexStr: String,
   input: String,
   validateOptimizations: Bool,
   semanticLevel: RegexSemanticLevel = .graphemeCluster,
-  syntax: SyntaxOptions = .traditional
+  syntax: SyntaxOptions = .traditional,
+  file: StaticString = #file,
+  line: UInt = #line
 ) throws -> (String, [String?])? {
   var regex = try Regex(regexStr, syntax: syntax).matchingSemantics(semanticLevel)
   let result = try regex.firstMatch(in: input)
-
+  
+  if try !_validateListCompilation(regex) {
+    XCTFail(
+      "List compilation failed for '\(regexStr)'",
+      file: file, line: line)
+  }
+  
   func validateSubstring(_ substringInput: Substring) throws {
     // Sometimes the characters we add to a substring merge with existing
     // string members. This messes up cross-validation, so skip the test.
@@ -105,14 +123,18 @@ func _firstMatch(
         For input '\(input)'
         Original: '\(regexStr)'
         _literalPattern: '\(roundTripRegex?._literalPattern ?? "<no pattern>")'
-        """)
+        """,
+        file: file,
+        line: line)
     case let (_, rtMatch?):
       XCTFail("""
         Incorrectly matched as '\(rtMatch)'
         For input '\(input)'
         Original: '\(regexStr)'
         _literalPattern: '\(roundTripRegex!._literalPattern!)'
-        """)
+        """,
+        file: file,
+        line: line)
     }
   }
 
@@ -184,7 +206,8 @@ func flatCaptureTest(
         input: test,
         validateOptimizations: validateOptimizations,
         semanticLevel: semanticLevel,
-        syntax: syntax
+        syntax: syntax,
+        file: file, line: line
       ) else {
         if expect == nil {
           continue
@@ -303,7 +326,8 @@ func firstMatchTest(
       input: input,
       validateOptimizations: validateOptimizations,
       semanticLevel: semanticLevel,
-      syntax: syntax)?.0
+      syntax: syntax,
+      file: file, line: line)?.0
 
     if xfail {
       XCTAssertNotEqual(found, match, file: file, line: line)