Merge pull request #595 from rctcwyvrn/charts-and-compares

More benchmarker features

- Loading results from a file
- Measuring compile times (via a spi function in Regex)
- Comparing compile times and against NSRegularExpression
- Saving comparison results as csv
- Charts (based on #585)
- Added the ability to set compiler options to the SPI interface 
- Added metrics and tracing enabled as options for the compiler
- Added CLI options for turning on metrics/tracing for benchmarker regexes
- Added parse time measurements to the benchmarker

Author: rctcwyvrn

Sources/RegexBenchmark/Benchmark.swift

@@ -1,15 +1,45 @@
-import _StringProcessing
+@_spi(RegexBenchmark) import _StringProcessing
+@_implementationOnly import _RegexParser
 import Foundation
 
-protocol RegexBenchmark {
+protocol RegexBenchmark: Debug {
   var name: String { get }
   func run()
-  func debug()
 }
 
-struct Benchmark: RegexBenchmark {
+protocol SwiftRegexBenchmark: RegexBenchmark {
+  var regex: Regex<AnyRegexOutput> { get set }
+  var pattern: String? { get }
+}
+
+extension SwiftRegexBenchmark {
+  mutating func compile() {
+    let _ = regex._forceAction(.recompile)
+  }
+  mutating func parse() -> Bool {
+    guard let s = pattern else {
+      return false
+    }
+    
+    do {
+      let _ = try _RegexParser.parse(s, .traditional)
+      return true
+    } catch {
+      return false
+    }
+  }
+  mutating func enableTracing() {
+    let _ = regex._forceAction(.addOptions(.enableTracing))
+  }
+  mutating func enableMetrics() {
+    let _ = regex._forceAction(.addOptions([.enableMetrics]))
+  }
+}
+
+struct Benchmark: SwiftRegexBenchmark {
   let name: String
-  let regex: Regex<AnyRegexOutput>
+  var regex: Regex<AnyRegexOutput>
+  let pattern: String?
   let type: MatchType
   let target: String
 
@@ -52,11 +82,12 @@ struct NSBenchmark: RegexBenchmark {
 }
 
 /// A benchmark running a regex on strings in input set
-struct InputListBenchmark: RegexBenchmark {
+struct InputListBenchmark: SwiftRegexBenchmark {
   let name: String
-  let regex: Regex<AnyRegexOutput>
+  var regex: Regex<AnyRegexOutput>
+  let pattern: String?
   let targets: [String]
-  
+
   func run() {
     for target in targets {
       blackHole(target.wholeMatch(of: regex))
@@ -78,7 +109,7 @@ struct InputListNSBenchmark: RegexBenchmark {
   func range(in target: String) -> NSRange {
     NSRange(target.startIndex..<target.endIndex, in: target)
   }
-  
+
   func run() {
     for target in targets {
       let range = range(in: target)
@@ -89,6 +120,9 @@ struct InputListNSBenchmark: RegexBenchmark {
 
 /// A benchmark meant to be ran across multiple engines
 struct CrossBenchmark {
+  /// Suffix added onto NSRegularExpression benchmarks
+  static var nsSuffix: String { "_NS" }
+  
   /// The base name of the benchmark
   var baseName: String
 
@@ -123,11 +157,12 @@ struct CrossBenchmark {
         Benchmark(
           name: baseName + "Whole",
           regex: swiftRegex,
+          pattern: regex,
           type: .whole,
           target: input))
       runner.register(
         NSBenchmark(
-          name: baseName + "Whole_NS",
+          name: baseName + "Whole" + CrossBenchmark.nsSuffix,
           regex: nsRegex,
           type: .first,
           target: input))
@@ -136,24 +171,26 @@ struct CrossBenchmark {
         Benchmark(
           name: baseName + "All",
           regex: swiftRegex,
+          pattern: regex,
           type: .allMatches,
           target: input))
       runner.register(
         NSBenchmark(
-          name: baseName + "All_NS",
+          name: baseName + "All" + CrossBenchmark.nsSuffix,
           regex: nsRegex,
           type: .allMatches,
           target: input))
-      if includeFirst {
+      if includeFirst || runner.includeFirstOverride {
         runner.register(
           Benchmark(
             name: baseName + "First",
             regex: swiftRegex,
+            pattern: regex,
             type: .first,
             target: input))
         runner.register(
           NSBenchmark(
-            name: baseName + "First_NS",
+            name: baseName + "First" + CrossBenchmark.nsSuffix,
             regex: nsRegex,
             type: .first,
             target: input))
@@ -178,10 +215,11 @@ struct CrossInputListBenchmark {
     runner.register(InputListBenchmark(
       name: baseName,
       regex: swiftRegex,
+      pattern: regex,
       targets: inputs
     ))
     runner.register(InputListNSBenchmark(
-      name: baseName + "NS",
+      name: baseName + CrossBenchmark.nsSuffix,
       regex: regex,
       targets: inputs
     ))

Sources/RegexBenchmark/BenchmarkChart.swift

@@ -0,0 +1,80 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+//
+//===----------------------------------------------------------------------===//
+
+#if os(macOS) && canImport(Charts)
+
+import Charts
+import SwiftUI
+
+struct BenchmarkChart: View {
+  var comparisons: [BenchmarkResult.Comparison]
+
+  // Sort by normalized difference
+  var sortedComparisons: [BenchmarkResult.Comparison] {
+    comparisons.sorted { a, b in
+      a.normalizedDiff < b.normalizedDiff
+    }
+  }
+  var body: some View {
+    VStack(alignment: .leading) {
+      Chart {
+        ForEach(sortedComparisons) { comparison in
+          // Normalized runtime
+          BarMark(
+            x: .value("Name", comparison.name),
+            y: .value("Normalized runtime", comparison.normalizedDiff))
+          .foregroundStyle(LinearGradient(
+            colors: [.accentColor, comparison.diff?.seconds ?? 0 <= 0 ? .green : .yellow],
+            startPoint: .bottom,
+            endPoint: .top))
+        }
+        // Baseline
+        RuleMark(y: .value("Time", 1.0))
+        .foregroundStyle(.red)
+        .lineStyle(.init(lineWidth: 1, dash: [2]))
+        .annotation(position: .top, alignment: .leading) {
+          Text("Baseline").foregroundStyle(.red)
+        }
+        
+      }
+      .frame(idealWidth: 800, idealHeight: 800)
+      .chartYScale(domain: 0...2.0)
+      .chartYAxis {
+        AxisMarks(values: .stride(by: 0.1))
+      }
+      .chartXAxis {
+        AxisMarks { value in
+          AxisGridLine()
+          AxisTick()
+          AxisValueLabel(value.as(String.self)!, orientation: .vertical)
+        }
+      }
+    }
+  }
+}
+
+struct BenchmarkResultApp: App {
+  static var comparisons: [BenchmarkResult.Comparison]?
+
+  var body: some Scene {
+    WindowGroup {
+      if let comparisons = Self.comparisons {
+        ScrollView {
+          BenchmarkChart(comparisons: comparisons)
+        }
+      } else {
+        Text("No data")
+      }
+    }
+  }
+}
+
+#endif

Sources/RegexBenchmark/BenchmarkRegistration.swift

@@ -2,27 +2,22 @@
 // Do not remove the start of registration or end of registration markers
 
 extension BenchmarkRunner {
-  static func makeRunner(
-    _ samples: Int,
-    _ quiet: Bool
-  ) -> BenchmarkRunner {
-    var benchmark = BenchmarkRunner("RegexBench", samples, quiet)
+  mutating func registerDefault() {
     // -- start of registrations --
-    benchmark.addReluctantQuant()
-    benchmark.addCSS()
-    benchmark.addNotFound()
-    benchmark.addGraphemeBreak()
-    benchmark.addHangulSyllable()
-    // benchmark.addHTML() // Disabled due to \b being unusably slow
-    benchmark.addEmail()
-    benchmark.addCustomCharacterClasses()
-    benchmark.addBuiltinCC()
-    benchmark.addUnicode()
-    benchmark.addLiteralSearch()
-    benchmark.addDiceNotation()
-    benchmark.addErrorMessages()
-    benchmark.addIpAddress()
+    self.addReluctantQuant()
+    self.addCSS()
+    self.addNotFound()
+    self.addGraphemeBreak()
+    self.addHangulSyllable()
+    // self.addHTML() // Disabled due to \b being unusably slow
+    self.addEmail()
+    self.addCustomCharacterClasses()
+    self.addBuiltinCC()
+    self.addUnicode()
+    self.addLiteralSearch()
+    self.addDiceNotation()
+    self.addErrorMessages()
+    self.addIpAddress()
     // -- end of registrations --
-    return benchmark
   }
 }