Add benchmark job in CI

Author: natecook1000

.github/workflows/pull_request.yml

@@ -19,3 +19,61 @@ jobs:
       license_header_check_project_name: "Swift.org"
       unacceptable_language_check_enabled: false
       format_check_enabled: false
+  bench:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    env:
+      BUILD_CMD: swift build -c release
+      BENCH_CMD: .build/release/RegexBenchmark
+      BASELINE_FILE: benchmark-baseline
+      COMPARE_FILE: benchmark-pr
+      COMPARE_OUT_FILE: benchmark-results.txt
+    steps:
+      - name: Check out baseline branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+          fetch-depth: 0
+      - name: Build baseline branch
+        working-directory: base
+        run: |
+          set -euo pipefail
+          eval "$BUILD_CMD"
+      - name: Run baseline benchmark
+        working-directory: base
+        run: |
+          set -euo pipefail
+          eval "$BENCH_CMD --save $RUNNER_TEMP/$BASELINE_FILE"
+          test -s "$RUNNER_TEMP/$BASELINE_FILE" || { echo "Baseline not created at $BASELINE_FILE"; exit 1; }
+      - name: Check out PR branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: pr
+          fetch-depth: 0
+      - name: Build PR branch
+        working-directory: pr
+        run: |
+          set -euo pipefail
+          eval "$BUILD_CMD"
+      - name: Run PR benchmark
+        working-directory: pr
+        run: |
+          set -euo pipefail
+          eval "$BENCH_CMD --save $RUNNER_TEMP/$COMPARE_FILE"
+          test -s "$RUNNER_TEMP/$COMPARE_FILE" || { echo "Comparison not created at $COMPARE_FILE"; exit 1; }
+          eval "$BENCH_CMD --compare $RUNNER_TEMP/$BASELINE_FILE" | tee "$RUNNER_TEMP/$COMPARE_OUT_FILE"
+      - name: 📊 Compare benchmarks
+        working-directory: pr
+        run: |
+          set -euo pipefail
+          eval "$BENCH_CMD --load $RUNNER_TEMP/$COMPARE_FILE --compare $RUNNER_TEMP/$BASELINE_FILE --compare-compile-time $RUNNER_TEMP/$BASELINE_FILE" | tee "$RUNNER_TEMP/$COMPARE_OUT_FILE"
+      - name: Upload benchmark artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: |
+            ${{ runner.temp }}/${{ env.BASELINE_FILE }}
+            ${{ runner.temp }}/${{ env.COMPARE_FILE }}
+            ${{ runner.temp }}/${{ env.COMPARE_OUT_FILE }}

Sources/RegexBenchmark/BenchmarkResults.swift

@@ -115,20 +115,20 @@ extension BenchmarkRunner {
       .sorted(by: {(a,b) in a.diff!.seconds < b.diff!.seconds})
 
     print("Comparing against \(against)")
-    print("=== Regressions ======================================================================")
+    print("=== Regressions ===============================================================")
     for item in regressions {
       print(item)
     }
 
-    print("=== Improvements =====================================================================")
+    print("=== Improvements ==============================================================")
     for item in improvements {
       print(item)
     }
 
     #if os(macOS) && canImport(Charts)
     if showChart {
       print("""
-        === Comparison chart =================================================================
+        === Comparison chart ==========================================================
         Press Control-C to close...
         """)
       BenchmarkResultApp.comparisons = comparisons
@@ -234,9 +234,17 @@ extension BenchmarkResult {
         return "- \(name) N/A"
       }
       let percentage = (1000 * diff.seconds / baselineTime.seconds).rounded()/10
-      let len = max(40 - name.count, 1)
-      let nameSpacing = String(repeating: " ", count: len)
-      return "- \(name)\(nameSpacing)\(latestTime)\t\(baselineTime)\t\(diff)\t\t\(percentage)%"
+      let start = if name.count > 40 {
+        "- \(name)\n" + String(repeating: " ", count: 43)
+      } else {
+        "- \(name, paddingTo: 40) "
+      }
+      return start + """
+        \(latestTime, paddingTo: 8, alignRight: true)  \
+        \(baselineTime, paddingTo: 8, alignRight: true)  \
+        \(diff, paddingTo: 8, alignRight: true)  \
+        \(percentage, paddingTo: 5, alignRight: true)%
+        """
     }
 
     var asCsv: String {
@@ -334,3 +342,16 @@ extension SuiteResult: Codable {
     return try decoder.decode(SuiteResult.self, from: data)
   }
 }
+
+extension DefaultStringInterpolation {
+  mutating func appendInterpolation<T>(_ value: T, paddingTo length: Int, alignRight: Bool = false) {
+    let s = String(describing: value)
+    let paddingCount = max(0, length - s.count)
+    let padding = String(repeating: " ", count: paddingCount)
+    if alignRight {
+      appendLiteral(padding + s)
+    } else {
+      appendLiteral(s + padding)
+    }
+  }
+}

Sources/RegexBenchmark/Utils/Stats.swift

@@ -15,7 +15,7 @@ enum Stats {}
 
 extension Stats {
   // Maximum allowed standard deviation is 7.5% of the median runtime
-  static let maxAllowedStdev = 0.075
+  static let maxAllowedStdev = 0.15
 
   static func tTest(_ a: Measurement, _ b: Measurement) -> Bool {
     // Student's t-test