Benchmark: option to output perf as csv

Author: milseman

Sources/RegexBenchmark/BenchmarkResults.swift

@@ -21,7 +21,21 @@ extension BenchmarkRunner {
     self.results = result
     print("Loaded results from \(url.path)")
   }
-  
+
+  /// Attempts to save results in a CSV format to the given path
+  func saveCSV(to savePath: String) throws {
+    let url = URL(fileURLWithPath: savePath, isDirectory: false)
+    let parent = url.deletingLastPathComponent()
+    if !FileManager.default.fileExists(atPath: parent.path) {
+      try! FileManager.default.createDirectory(
+        atPath: parent.path,
+        withIntermediateDirectories: true)
+    }
+    print("Saving result as CSV to \(url.path)")
+    try results.saveCSV(to: url)
+
+  }
+
   /// Compare this runner's results against the results stored in the given file path
   func compare(
     against compareFilePath: String,
@@ -153,6 +167,12 @@ struct Measurement: Codable, CustomStringConvertible {
   var description: String {
     return "\(median) (stdev: \(Time(stdev)), N = \(samples))"
   }
+
+  var asCSV: String {
+    """
+    \(median.asCSVSeconds), \(stdev), \(samples)
+    """
+  }
 }
 
 struct BenchmarkResult: Codable, CustomStringConvertible {
@@ -170,6 +190,13 @@ struct BenchmarkResult: Codable, CustomStringConvertible {
     }
     return base
   }
+
+  var asCSV: String {
+    let na = "N/A, N/A, N/A"
+    return """
+    \(runtime.asCSV), \(compileTime?.asCSV ?? na), \(parseTime?.asCSV ?? na)
+    """
+  }
 }
 
 extension BenchmarkResult {
@@ -263,6 +290,27 @@ struct SuiteResult {
 }
 
 extension SuiteResult: Codable {
+  func saveCSV(to url: URL) throws {
+    var output: [(name: String, result: BenchmarkResult)] = []
+    for key in results.keys {
+      output.append((key, results[key]!))
+    }
+    output.sort {
+      $0.name < $1.name
+    }
+    var contents = """
+    name,\
+    runtime_median, runTime_stddev, runTime_samples,\
+    compileTime_median, compileTime_stddev, compileTime_samples,\
+    parseTime_median, parseTime_stddev, parseTime_samples\n
+    """
+    for (name, result) in output {
+      contents.append("\(name), \(result.asCSV))\n")
+    }
+    print("Saving result as .csv to \(url.path())")
+    try contents.write(to: url, atomically: true, encoding: String.Encoding.utf8)
+  }
+
   func save(to url: URL) throws {
     let encoder = JSONEncoder()
     let data = try encoder.encode(self)

Sources/RegexBenchmark/CLI.swift

@@ -32,6 +32,9 @@ struct Runner: ParsableCommand {
   @Option(help: "Save comparison results as csv")
   var saveComparison: String?
 
+  @Option(help: "Save benchmark results as csv")
+  var saveCSV: String?
+
   @Flag(help: "Quiet mode")
   var quiet = false
 
@@ -91,9 +94,14 @@ swift build -c release -Xswiftc -DPROCESSOR_MEASUREMENTS_ENABLED
 
     if let loadFile = load {
       try runner.load(from: loadFile)
+      if excludeNs {
+        runner.results.results = runner.results.results.filter {
+          !$0.key.contains("_NS")
+        }
+      }
     } else {
       if excludeNs {
-        runner.suite = runner.suite.filter { b in !b.name.contains("NS") }
+        runner.suite = runner.suite.filter { b in !b.name.contains("_NS") }
       }
       runner.run()
     }
@@ -116,5 +124,8 @@ swift build -c release -Xswiftc -DPROCESSOR_MEASUREMENTS_ENABLED
     if let compareFile = compareCompileTime {
       try runner.compareCompileTimes(against: compareFile, showChart: showChart)
     }
+    if let csvPath = saveCSV {
+      try runner.saveCSV(to: csvPath)
+    }
   }
 }

Sources/RegexBenchmark/Utils/Time.swift

@@ -66,6 +66,11 @@ extension Time {
 }
 
 extension Time: CustomStringConvertible {
+  /// Normalize our time to fractions of a second for CSV output
+  public var asCSVSeconds: String {
+    return String(format: "%.3g", seconds)
+  }
+
   public var description: String {
     if self.seconds == 0 { return "0" }
     if self.abs() < .attosecond { return String(format: "%.3gas", seconds * 1e18) }