Add parse time measurement + cleanup benchmarker

Author: rctcwyvrn

Sources/RegexBenchmark/Benchmark.swift

@@ -9,7 +9,9 @@ protocol RegexBenchmark {
 
 protocol SwiftRegexBenchmark: RegexBenchmark {
   var regex: Regex<AnyRegexOutput> { get set }
+  var pattern: String? { get }
   mutating func compile()
+  mutating func parse() -> Bool
   mutating func enableTracing()
   mutating func enableMetrics()
 }
@@ -18,6 +20,14 @@ extension SwiftRegexBenchmark {
   mutating func compile() {
     let _ = regex._forceAction(.recompile)
   }
+  mutating func parse() -> Bool {
+    if let s = pattern {
+      let _ = regex._forceAction(.parse(s))
+      return true
+    } else {
+      return false
+    }
+  }
   mutating func enableTracing() {
     let _ = regex._forceAction(.addOptions(.enableTracing))
   }
@@ -29,6 +39,7 @@ extension SwiftRegexBenchmark {
 struct Benchmark: SwiftRegexBenchmark {
   let name: String
   var regex: Regex<AnyRegexOutput>
+  let pattern: String?
   let type: MatchType
   let target: String
 
@@ -74,6 +85,7 @@ struct NSBenchmark: RegexBenchmark {
 struct InputListBenchmark: SwiftRegexBenchmark {
   let name: String
   var regex: Regex<AnyRegexOutput>
+  let pattern: String?
   let targets: [String]
 
   func run() {
@@ -145,6 +157,7 @@ struct CrossBenchmark {
         Benchmark(
           name: baseName + "Whole",
           regex: swiftRegex,
+          pattern: regex,
           type: .whole,
           target: input))
       runner.register(
@@ -158,6 +171,7 @@ struct CrossBenchmark {
         Benchmark(
           name: baseName + "All",
           regex: swiftRegex,
+          pattern: regex,
           type: .allMatches,
           target: input))
       runner.register(
@@ -171,6 +185,7 @@ struct CrossBenchmark {
           Benchmark(
             name: baseName + "First",
             regex: swiftRegex,
+            pattern: regex,
             type: .first,
             target: input))
         runner.register(
@@ -200,6 +215,7 @@ struct CrossInputListBenchmark {
     runner.register(InputListBenchmark(
       name: baseName,
       regex: swiftRegex,
+      pattern: regex,
       targets: inputs
     ))
     runner.register(InputListNSBenchmark(

Sources/RegexBenchmark/BenchmarkChart.swift

@@ -17,23 +17,24 @@ import SwiftUI
 struct BenchmarkChart: View {
   var comparisons: [BenchmarkResult.Comparison]
 
+  // Sort by normalized difference
   var sortedComparisons: [BenchmarkResult.Comparison] {
     comparisons.sorted { a, b in
-      a.latest.median.seconds/a.baseline.median.seconds <
-        b.latest.median.seconds/b.baseline.median.seconds
+      a.normalizedDiff < b.normalizedDiff
     }
   }
   var body: some View {
     VStack(alignment: .leading) {
       Chart {
         ForEach(sortedComparisons) { comparison in
-          let new = comparison.latest.median.seconds
-          let old = comparison.baseline.median.seconds
-          chartBody(
-            name: comparison.name,
-            new: new,
-            old: old,
-            sampleCount: comparison.latest.samples)
+          // Normalized runtime
+          BarMark(
+            x: .value("Name", comparison.name),
+            y: .value("Normalized runtime", comparison.normalizedDiff))
+          .foregroundStyle(LinearGradient(
+            colors: [.accentColor, comparison.diff?.seconds ?? 0 <= 0 ? .green : .yellow],
+            startPoint: .bottom,
+            endPoint: .top))
         }
         // Baseline
         RuleMark(y: .value("Time", 1.0))
@@ -43,23 +44,6 @@ struct BenchmarkChart: View {
       }.frame(idealHeight: 400)
     }
   }
-
-  @ChartContentBuilder
-  func chartBody(
-    name: String,
-    new: TimeInterval,
-    old: TimeInterval,
-    sampleCount: Int
-  ) -> some ChartContent {
-    // Normalized runtime
-    BarMark(
-      x: .value("Name", name),
-      y: .value("Normalized runtime", new / old))
-    .foregroundStyle(LinearGradient(
-      colors: [.accentColor, new - old <= 0 ? .green : .yellow],
-      startPoint: .bottom,
-      endPoint: .top))
-  }
 }
 
 struct BenchmarkResultApp: App {

Sources/RegexBenchmark/BenchmarkResults.swift

@@ -106,15 +106,7 @@ extension BenchmarkRunner {
         === Comparison chart =================================================================
         Press Control-C to close...
         """)
-      BenchmarkResultApp.comparisons = {
-        return comparisons.sorted {
-          let delta0 = Float($0.latest.median.seconds - $0.baseline.median.seconds)
-            / Float($0.baseline.median.seconds)
-          let delta1 = Float($1.latest.median.seconds - $1.baseline.median.seconds)
-            / Float($1.baseline.median.seconds)
-          return delta0 > delta1
-        }
-      }()
+      BenchmarkResultApp.comparisons = comparisons
       BenchmarkResultApp.main()
     }
     #endif
@@ -141,11 +133,43 @@ extension BenchmarkRunner {
   }
 }
 
-struct BenchmarkResult: Codable {
-  let compileTime: Time
+struct Measurement: Codable, CustomStringConvertible {
   let median: Time
   let stdev: Double
   let samples: Int
+  
+  init(results: [Time]) {
+    let sorted = results.sorted()
+    self.samples = sorted.count
+    self.median = sorted[samples/2]
+    let sum = results.reduce(0.0) {acc, next in acc + next.seconds}
+    let mean = sum / Double(samples)
+    let squareDiffs = results.reduce(0.0) { acc, next in
+      acc + pow(next.seconds - mean, 2)
+    }
+    self.stdev = (squareDiffs / Double(samples)).squareRoot()
+  }
+  
+  var description: String {
+    return "\(median) (stdev: \(Time(stdev)), N = \(samples))"
+  }
+}
+
+struct BenchmarkResult: Codable, CustomStringConvertible {
+  let runtime: Measurement
+  let compileTime: Measurement?
+  let parseTime: Measurement?
+  
+  var description: String {
+    var base = "> run time: \(runtime.description)"
+    if let compileTime = compileTime {
+      base += "\n> compile time: \(compileTime)"
+    }
+    if let parseTime = parseTime {
+      base += "\n> parse time: \(parseTime)"
+    }
+    return base
+  }
 }
 
 extension BenchmarkResult {
@@ -160,56 +184,57 @@ extension BenchmarkResult {
       case runtime
       case compileTime
     }
+    
+    var latestTime: Time {
+      switch type {
+      case .compileTime:
+        return latest.compileTime?.median ?? .zero
+      case .runtime:
+        return latest.runtime.median
+      }
+    }
+    
+    var baselineTime: Time {
+      switch type {
+      case .compileTime:
+        return baseline.compileTime?.median ?? .zero
+      case .runtime:
+        return baseline.runtime.median
+      }
+    }
 
     var diff: Time? {
       switch type {
       case .compileTime:
-        return latest.compileTime - baseline.compileTime
+        return latestTime - baselineTime
       case .runtime:
-        if Stats.tTest(baseline, latest) {
-          return latest.median - baseline.median
+        if Stats.tTest(baseline.runtime, latest.runtime) {
+          return latestTime - baselineTime
         }
         return nil
       }
-
+    }
+    
+    var normalizedDiff: Double {
+      latestTime.seconds/baselineTime.seconds
     }
 
     var description: String {
       guard let diff = diff else {
         return "- \(name) N/A"
       }
-      let oldVal: Time
-      let newVal: Time
-      switch type {
-      case .compileTime:
-        oldVal = baseline.compileTime
-        newVal = latest.compileTime
-      case .runtime:
-        oldVal = baseline.median
-        newVal = latest.median
-      }
-      let percentage = (1000 * diff.seconds / oldVal.seconds).rounded()/10
+      let percentage = (1000 * diff.seconds / baselineTime.seconds).rounded()/10
       let len = max(40 - name.count, 1)
       let nameSpacing = String(repeating: " ", count: len)
-      return "- \(name)\(nameSpacing)\(newVal)\t\(oldVal)\t\(diff)\t\t\(percentage)%"
+      return "- \(name)\(nameSpacing)\(latestTime)\t\(baselineTime)\t\(diff)\t\t\(percentage)%"
     }
 
     var asCsv: String {
       guard let diff = diff else {
         return "\(name),N/A"
       }
-      let oldVal: Time
-      let newVal: Time
-      switch type {
-      case .compileTime:
-        oldVal = baseline.compileTime
-        newVal = latest.compileTime
-      case .runtime:
-        oldVal = baseline.median
-        newVal = latest.median
-      }
-      let percentage = (1000 * diff.seconds / oldVal.seconds).rounded()/10
-      return "\"\(name)\",\(newVal.seconds),\(oldVal.seconds),\(diff.seconds),\(percentage)%"
+      let percentage = (1000 * diff.seconds / baselineTime.seconds).rounded()/10
+      return "\"\(name)\",\(latestTime.seconds),\(baselineTime.seconds),\(diff.seconds),\(percentage)%"
     }
   }
 }