Replace performance tests with Benchmarks (#35)

* Replace perfomance test with Benchmarks

* Don't compare Foundation benchmarks

* Fix GH action, and fixup benchmark names

* Run benchmarks on linux and macOS, remove old performance CI

Author: adam-fowler

.github/workflows/benchmarks.yaml

@@ -18,7 +18,7 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-latest, macOS-14]
 
     steps:
       - uses: actions/checkout@v4
@@ -46,7 +46,7 @@ jobs:
         if: ${{ env.hasBenchmark == '1' }}
         run: |
           cd Benchmarks
-          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update pull_request
+          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update pull_request --filter "^(?!Foundation).*"
       - name: Switch to branch 'main'
         if: ${{ env.hasBenchmark == '1' }}
         run: |
@@ -56,7 +56,7 @@ jobs:
         if: ${{ env.hasBenchmark == '1' }}
         run: |
           cd Benchmarks
-          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update main
+          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update main --filter "^(?!Foundation).*"
       - name: Compare PR and main
         if: ${{ env.hasBenchmark == '1' }}
         id: benchmark

.github/workflows/ci.yaml

@@ -43,21 +43,6 @@ jobs:
       with:
         file: info.lcov
 
-  "tuxOS-Performance-Tests":
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        images:
-        - swift:5.10
-    container:
-      image: ${{ matrix.images }}
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-    - name: Build & run
-      run: swift run -c release
-
   "macOS-Tests":
     runs-on: macOS-14
     strategy:
@@ -73,13 +58,3 @@ jobs:
       uses: codecov/codecov-action@v4
       with:
         file: info.lcov
-
-  "macOS-Performance-Tests":
-    runs-on: macOS-13
-    strategy:
-      fail-fast: false
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-    - name: Build & run
-      run: swift run -c release

Benchmarks/Benchmarks/BaseN/BaseN.swift

@@ -1,5 +1,6 @@
 import Benchmark
 import ExtrasBase64
+import Foundation
 
 let benchmarks = {
     Benchmark.defaultConfiguration = .init(
@@ -53,4 +54,48 @@ let benchmarks = {
             try blackHole(Base64.decode(string: base64))
         }
     }
+
+    Benchmark("Foundation.encodeToData") { benchmark in
+        let bytes = Array(UInt8(0) ... UInt8(255))
+        let data = Data(bytes)
+
+        benchmark.startMeasurement()
+
+        for _ in benchmark.scaledIterations {
+            blackHole(data.base64EncodedData())
+        }
+    }
+
+    Benchmark("Foundation.encodeToString") { benchmark in
+        let bytes = Array(UInt8(0) ... UInt8(255))
+        let data = Data(bytes)
+
+        benchmark.startMeasurement()
+
+        for _ in benchmark.scaledIterations {
+            blackHole(data.base64EncodedString())
+        }
+    }
+
+    Benchmark("Foundation.decodeString") { benchmark in
+        let bytes = Array(UInt8(0) ... UInt8(255))
+        let base64 = Base64.encodeToString(bytes: bytes)
+
+        benchmark.startMeasurement()
+
+        for _ in benchmark.scaledIterations {
+            blackHole(Data(base64Encoded: base64))
+        }
+    }
+
+    Benchmark("Foundation.decodeStringIgnoreUnknownCharacters") { benchmark in
+        let bytes = Array(UInt8(0) ... UInt8(255))
+        let base64 = Base64.encodeToString(bytes: bytes)
+
+        benchmark.startMeasurement()
+
+        for _ in benchmark.scaledIterations {
+            blackHole(Data(base64Encoded: base64, options: .ignoreUnknownCharacters))
+        }
+    }
 }

Package.swift

@@ -11,7 +11,6 @@ let package = Package(
     dependencies: [],
     targets: [
         .target(name: "ExtrasBase64", dependencies: []),
-        .target(name: "PerformanceTest", dependencies: ["ExtrasBase64"]),
         .testTarget(name: "ExtrasBase64Tests", dependencies: ["ExtrasBase64"]),
     ]
 )