Make comparison generic + Adjust stdev requirement to be a %

Author: rctcwyvrn

Sources/RegexBenchmark/Benchmark.swift

@@ -11,10 +11,6 @@ protocol RegexBenchmark {
 protocol SwiftRegexBenchmark: RegexBenchmark {
   var regex: Regex<AnyRegexOutput> { get set }
   var pattern: String? { get }
-  mutating func compile()
-  mutating func parse() -> Bool
-  mutating func enableTracing()
-  mutating func enableMetrics()
 }
 
 extension SwiftRegexBenchmark {

Sources/RegexBenchmark/BenchmarkResults.swift

@@ -161,12 +161,12 @@ struct BenchmarkResult: Codable, CustomStringConvertible {
   let parseTime: Measurement?
 
   var description: String {
-    var base = "> run time: \(runtime.description)"
+    var base = "  > run time: \(runtime.description)"
     if let compileTime = compileTime {
-      base += "\n> compile time: \(compileTime)"
+      base += "\n  > compile time: \(compileTime)"
     }
     if let parseTime = parseTime {
-      base += "\n> parse time: \(parseTime)"
+      base += "\n  > parse time: \(parseTime)"
     }
     return base
   }
@@ -176,45 +176,17 @@ extension BenchmarkResult {
   struct Comparison: Identifiable, CustomStringConvertible {
     var id = UUID()
     var name: String
-    var baseline: BenchmarkResult
-    var latest: BenchmarkResult
-    var type: ComparisonType = .runtime
+    var baseline: Measurement
+    var latest: Measurement
 
-    enum ComparisonType {
-      case runtime
-      case compileTime
-    }
-    
-    var latestTime: Time {
-      switch type {
-      case .compileTime:
-        return latest.compileTime?.median ?? .zero
-      case .runtime:
-        return latest.runtime.median
-      }
-    }
-    
-    var baselineTime: Time {
-      switch type {
-      case .compileTime:
-        return baseline.compileTime?.median ?? .zero
-      case .runtime:
-        return baseline.runtime.median
-      }
-    }
-
+    var latestTime: Time { latest.median }
+    var baselineTime: Time { baseline.median }
     var diff: Time? {
-      switch type {
-      case .compileTime:
+      if Stats.tTest(baseline, latest) {
         return latestTime - baselineTime
-      case .runtime:
-        if Stats.tTest(baseline.runtime, latest.runtime) {
-          return latestTime - baselineTime
-        }
-        return nil
       }
+      return nil
     }
-    
     var normalizedDiff: Double {
       latestTime.seconds/baselineTime.seconds
     }
@@ -248,10 +220,11 @@ struct SuiteResult {
 
   func compare(with other: SuiteResult) -> [BenchmarkResult.Comparison] {
     var comparisons: [BenchmarkResult.Comparison] = []
-    for item in results {
-      if let otherVal = other.results[item.key] {
+    for latest in results {
+      if let otherVal = other.results[latest.key] {
         comparisons.append(
-          .init(name: item.key, baseline: otherVal, latest: item.value))
+          .init(name: latest.key,
+                baseline: otherVal.runtime, latest: latest.value.runtime))
       }
     }
     return comparisons
@@ -260,11 +233,12 @@ struct SuiteResult {
   /// Compares with the NSRegularExpression benchmarks generated by CrossBenchmark
   func compareWithNS() -> [BenchmarkResult.Comparison] {
     var comparisons: [BenchmarkResult.Comparison] = []
-    for item in results {
-      let key = item.key + CrossBenchmark.nsSuffix
+    for latest in results {
+      let key = latest.key + CrossBenchmark.nsSuffix
       if let nsResult = results[key] {
         comparisons.append(
-          .init(name: item.key, baseline: nsResult, latest: item.value))
+          .init(name: latest.key,
+                baseline: nsResult.runtime, latest: latest.value.runtime))
       }
     }
     return comparisons
@@ -274,13 +248,14 @@ struct SuiteResult {
     with other: SuiteResult
   ) -> [BenchmarkResult.Comparison] {
     var comparisons: [BenchmarkResult.Comparison] = []
-    for item in results {
-      if let otherVal = other.results[item.key] {
+    for latest in results {
+      if let baseline = other.results[latest.key],
+         let baselineTime = baseline.compileTime,
+         let latestTime = latest.value.compileTime {
         comparisons.append(
-          .init(name: item.key,
-                baseline: otherVal,
-                latest: item.value,
-                type: .compileTime))
+          .init(name: latest.key,
+                baseline: baselineTime,
+                latest: latestTime))
       }
     }
     return comparisons

Sources/RegexBenchmark/BenchmarkRunner.swift

@@ -81,8 +81,8 @@ struct BenchmarkRunner {
     print("Running")
     for b in suite {
       var result = measure(benchmark: b, samples: samples)
-      if result.runtime.stdev > Stats.maxAllowedStdev {
-        print("Warning: Standard deviation > \(Time(Stats.maxAllowedStdev)) for \(b.name)")
+      if result.runtime.stdev > Stats.maxAllowedStdev * result.runtime.median.seconds {
+        print("Warning: Standard deviation > \(Stats.maxAllowedStdev*100)% for \(b.name)")
         print(result.runtime)
         print("Rerunning \(b.name)")
         result = measure(benchmark: b, samples: result.runtime.samples*2)

Sources/RegexBenchmark/Utils/Stats.swift

@@ -3,8 +3,8 @@ import Foundation
 enum Stats {}
 
 extension Stats {
-  // 500µs, maybe this should be a % of the runtime for each benchmark?
-  static let maxAllowedStdev = 500e-6
+  // Maximum allowed standard deviation is 5% of the median runtime
+  static let maxAllowedStdev = 0.05
 
   static func tTest(_ a: Measurement, _ b: Measurement) -> Bool {
     // Student's t-test