v1 benchmarker

Author: rctcwyvrn

Package.swift

@@ -41,7 +41,10 @@ let package = Package(
             targets: ["_RegexParser"]),
         .executable(
             name: "VariadicsGenerator",
-            targets: ["VariadicsGenerator"])
+            targets: ["VariadicsGenerator"]),
+        .executable(
+            name: "RegexBenchmark",
+            targets: ["RegexBenchmark"])
     ],
     dependencies: [
         .package(url: "https://github.com/apple/swift-argument-parser", from: "1.0.0"),
@@ -112,6 +115,17 @@ let package = Package(
                 "_RegexParser",
                 "_StringProcessing"
             ]),
+        .executableTarget(
+            name: "RegexBenchmark",
+            dependencies: [
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+                "_RegexParser",
+                "_StringProcessing",
+                "RegexBuilder"
+            ],
+            swiftSettings: [
+                .unsafeFlags(["-Xfrontend", "-disable-availability-checking"]),
+            ]),
 
         // MARK: Exercises
         .target(

Sources/RegexBenchmark/Benchmark.swift

@@ -0,0 +1,106 @@
+import _StringProcessing
+import Foundation
+
+public protocol RegexBenchmark {
+  var name: String { get }
+  func run()
+}
+
+public struct Benchmark: RegexBenchmark {
+  public let name: String
+  let regex: Regex<Substring> // need to figure out this type to allow for other regexes
+  let ty: MatchType
+  let target: String
+
+  public enum MatchType {
+    case whole
+    case first
+    case enumerate
+  }
+  
+  public func run() {
+    switch ty {
+    case .whole: blackHole(target.wholeMatch(of: regex))
+    case .enumerate: blackHole(target.matches(of: regex))
+    case .first: blackHole(target.firstMatch(of: regex))
+    }
+  }
+}
+
+public struct NSBenchmark: RegexBenchmark {
+  public let name: String
+  let regex: NSRegularExpression
+  let ty: NSMatchType
+  let target: String
+  
+  var range: NSRange {
+    NSRange(target.startIndex..<target.endIndex, in: target)
+  }
+
+  public enum NSMatchType {
+    case all
+    case first
+  }
+  
+  public func run() {
+    switch ty {
+    case .all: blackHole(regex.matches(in: target, range: range))
+    case .first: blackHole(regex.firstMatch(in: target, range: range))
+    }
+  }
+}
+
+public struct BenchmarkRunner {
+  // Register instances of Benchmark and run them
+  let suiteName: String
+  var suite: [any RegexBenchmark]
+  let samples: Int = 20
+  
+  public init(suiteName: String) {
+    self.suiteName = suiteName
+    self.suite = []
+  }
+
+  public mutating func register(new: some RegexBenchmark) {
+    suite.append(new)
+  }
+  
+  // requires the macos13 beta
+  //  public func measure() -> Duration {
+  //    let clock = SuspendingClock()
+  //    var times = (0..<samples).map { _ in clock.measure(run) }
+  //    assert(times.count == samples)
+  //
+  //    times.sort()
+  //    return times[samples/2]
+  //  }
+  
+  func measure(benchmark: some RegexBenchmark) -> Time {
+    var times: [Time] = []
+    
+    // initial run to make sure the regex has been compiled
+    benchmark.run()
+    
+    for _ in 0..<samples {
+      let start = Tick.now
+      benchmark.run()
+      let end = Tick.now
+      let time = end.elapsedTime(since: start)
+      times.append(time)
+    }
+    times.sort()
+    return times[samples/2]
+  }
+  
+  public func run() {
+    print("Running")
+    for b in suite {
+      print("- \(b.name) \(measure(benchmark: b))")
+    }
+  }
+}
+
+// nom nom nom, consume the argument
+@inline(never)
+public func blackHole<T>(_ x: T) {
+}

Sources/RegexBenchmark/Suite/Backtracking.swift

@@ -0,0 +1,30 @@
+import _StringProcessing
+import RegexBuilder
+import Foundation
+
+// Tests that involve heavy backtracking
+
+extension BenchmarkRunner {
+  mutating func addBacktracking() {
+    register(new: basicBacktrack)
+    register(new: basicBacktrackNS)
+  }
+}
+
+
+private let r = " +A"
+private let s = String(repeating: " ", count: 100)
+
+private let basicBacktrack = Benchmark(
+  name: "BasicBacktrack",
+  regex: try! Regex(r),
+  ty: .enumerate,
+  target: s
+)
+
+private let basicBacktrackNS = NSBenchmark(
+  name: "BasicBacktrackNS",
+  regex: try! NSRegularExpression(pattern: r),
+  ty: .all,
+  target: s
+)