From 960f629dd44f876d26a2512097162c4b9623f568 Mon Sep 17 00:00:00 2001 From: shenxianpeng Date: Fri, 19 Sep 2025 19:55:19 +0300 Subject: [PATCH 1/7] chore: update benchmark --- testing/benchmark.md | 4 ++ testing/benchmark_hook_1.yaml | 4 +- testing/benchmark_hooks.py | 115 ++++++++-------------------------- 3 files changed, 33 insertions(+), 90 deletions(-) diff --git a/testing/benchmark.md b/testing/benchmark.md index dfbde26..6a5fc35 100644 --- a/testing/benchmark.md +++ b/testing/benchmark.md @@ -8,6 +8,10 @@ This document outlines the benchmarking process for comparing the performance of ```bash python3 testing/benchmark_hooks.py + +# or + +hyperfine -i --warmup 1 -r 5 'pre-commit run --config ../testing/benchmark_hook_1.yaml --all-files' 'pre-commit run --config ../testing/benchmark_hook_2.yaml --all-files' ``` ## Results diff --git a/testing/benchmark_hook_1.yaml b/testing/benchmark_hook_1.yaml index b8ba6da..3019bfc 100644 --- a/testing/benchmark_hook_1.yaml +++ b/testing/benchmark_hook_1.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/cpp-linter/cpp-linter-hooks - rev: v1.1.2 + rev: v1.1.3 hooks: - id: clang-format - args: [--style=file, --version=21] + args: [--style=file] diff --git a/testing/benchmark_hooks.py b/testing/benchmark_hooks.py index 39efb78..96ad924 100644 --- a/testing/benchmark_hooks.py +++ b/testing/benchmark_hooks.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 """ -Benchmark script to compare performance of cpp-linter-hooks vs mirrors-clang-format. +Benchmark script to compare performance of cpp-linter-hooks vs mirrors-clang-format using hyperfine. Usage: - python benchmark_hooks.py + python benchmark_hooks.py + # or directly with hyperfine: + hyperfine --warmup 1 -r 5 'pre-commit run --config ../testing/benchmark_hook_1.yaml --all-files' 'pre-commit run --config ../testing/benchmark_hook_2.yaml --all-files' Requirements: - pre-commit must be installed and available in PATH @@ -15,8 +17,7 @@ import os import subprocess -import time -import statistics +import sys HOOKS = [ { @@ -50,95 +51,33 @@ def prepare_code(): pass -def run_hook(config): - cmd = ["pre-commit", "run", "--config", config, "--all-files"] - start = time.perf_counter() - try: - subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except subprocess.CalledProcessError: - # Still record time even if hook fails - pass - end = time.perf_counter() - return end - start - - -def benchmark(): - results = {} - prepare_code() +def run_hyperfine(): os.chdir("examples") - for hook in HOOKS: - subprocess.run(["git", "restore", "."], check=True) - times = [] - print(f"\nBenchmarking {hook['name']}...") - for i in range(REPEATS): - subprocess.run(["pre-commit", "clean"]) - t = run_hook(hook["config"]) - print(f" Run {i + 1}: {t:.3f} seconds") - times.append(t) - results[hook["name"]] = times - return results - - -def report(results): - headers = ["Hook", "Avg (s)", "Std (s)", "Min (s)", "Max (s)", "Runs"] - col_widths = [max(len(h), 16) for h in headers] - # Calculate max width for each column - for name, times in results.items(): - col_widths[0] = max(col_widths[0], len(name)) - print("\nBenchmark Results:\n") - # Print header - header_row = " | ".join(h.ljust(w) for h, w in zip(headers, col_widths)) - print(header_row) - print("-+-".join("-" * w for w in col_widths)) - # Print rows - lines = [] - for name, times in results.items(): - avg = statistics.mean(times) - std = statistics.stdev(times) if len(times) > 1 else 0.0 - min_t = min(times) - max_t = max(times) - row = [ - name.ljust(col_widths[0]), - f"{avg:.3f}".ljust(col_widths[1]), - f"{std:.3f}".ljust(col_widths[2]), - f"{min_t:.3f}".ljust(col_widths[3]), - f"{max_t:.3f}".ljust(col_widths[4]), - str(len(times)).ljust(col_widths[5]), - ] - print(" | ".join(row)) - lines.append(" | ".join(row)) - # Save to file + commands = [ + f"pre-commit run --config {hook['config']} --all-files" for hook in HOOKS + ] + hyperfine_cmd = [ + "hyperfine", + "--warmup", + "1", + "-r", + str(REPEATS), + ] + commands + print("Running benchmark with hyperfine:") + print(" ".join(hyperfine_cmd)) + try: + subprocess.run(hyperfine_cmd, check=True) + except FileNotFoundError: + print( + "hyperfine is not installed. Please install it with 'cargo install hyperfine' or 'brew install hyperfine'." + ) + sys.exit(1) os.chdir("..") - with open(RESULTS_FILE, "w") as f: - f.write(header_row + "\n") - f.write("-+-".join("-" * w for w in col_widths) + "\n") - for line in lines: - f.write(line + "\n") - print(f"\nResults saved to {RESULTS_FILE}") - - # Write to GitHub Actions summary - summary_path = os.environ.get("GITHUB_STEP_SUMMARY") - if summary_path: - with open(summary_path, "a") as f: - f.write("## Benchmark Results\n\n") - # Markdown table header - md_header = "| " + " | ".join(headers) + " |\n" - md_sep = "|" + "|".join(["-" * (w + 2) for w in col_widths]) + "|\n" - f.write(md_header) - f.write(md_sep) - for name, times in results.items(): - avg = statistics.mean(times) - std = statistics.stdev(times) if len(times) > 1 else 0.0 - min_t = min(times) - max_t = max(times) - md_row = f"| {name} | {avg:.3f} | {std:.3f} | {min_t:.3f} | {max_t:.3f} | {len(times)} |\n" - f.write(md_row) - f.write("\n") def main(): - results = benchmark() - report(results) + prepare_code() + run_hyperfine() if __name__ == "__main__": From 11654b18d1f3509f27d5d11f41ebb4a7baec0a95 Mon Sep 17 00:00:00 2001 From: Xianpeng Shen Date: Fri, 19 Sep 2025 22:26:06 +0300 Subject: [PATCH 2/7] Update README for clarity on pre-commit hook --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 628172c..54f9ab0 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![Test](https://github.com/cpp-linter/cpp-linter-hooks/actions/workflows/test.yml/badge.svg)](https://github.com/cpp-linter/cpp-linter-hooks/actions/workflows/test.yml) [![CodeQL](https://github.com/cpp-linter/cpp-linter-hooks/actions/workflows/codeql.yml/badge.svg)](https://github.com/cpp-linter/cpp-linter-hooks/actions/workflows/codeql.yml) -A powerful [pre-commit](https://pre-commit.com/) hook for auto-formatting and linting C/C++ code with `clang-format` and `clang-tidy`. +A pre-commit hook that automatically formats and lints your C/C++ code using `clang-format` and `clang-tidy`. ## Table of Contents From 26c7bdc0f5159f1a24b98895a145a057c12f04a9 Mon Sep 17 00:00:00 2001 From: Xianpeng Shen Date: Fri, 19 Sep 2025 22:27:40 +0300 Subject: [PATCH 3/7] Install hyperfine before running benchmarks --- .github/workflows/benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 54b1285..88ac26f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -22,4 +22,5 @@ jobs: - name: Run benchmarks run: | + cargo install hyperfine python testing/benchmark_hooks.py From 16c6fd367c1d894f148f262ea1820da1b8696de4 Mon Sep 17 00:00:00 2001 From: shenxianpeng Date: Fri, 19 Sep 2025 23:01:17 +0300 Subject: [PATCH 4/7] chore: update benchmark --- .github/workflows/benchmark.yml | 26 ------ testing/README.md | 4 +- testing/benchmark.md | 67 ++++++++++++--- testing/benchmark_hooks.py | 84 ------------------- testing/benchmark_results.txt | 6 -- testing/good.c | 10 ++- testing/main.c | 8 +- ...hook_1.yaml => test-cpp-linter-hooks.yaml} | 2 +- ..._2.yaml => test-mirrors-clang-format.yaml} | 0 9 files changed, 70 insertions(+), 137 deletions(-) delete mode 100644 .github/workflows/benchmark.yml delete mode 100644 testing/benchmark_hooks.py delete mode 100644 testing/benchmark_results.txt rename testing/{benchmark_hook_1.yaml => test-cpp-linter-hooks.yaml} (76%) rename testing/{benchmark_hook_2.yaml => test-mirrors-clang-format.yaml} (100%) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index 88ac26f..0000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Benchmark Hooks - -on: - workflow_dispatch: - -jobs: - benchmark: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5 - - - name: Set up Python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c #v5 - with: - python-version: '3.13' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pre-commit - - - name: Run benchmarks - run: | - cargo install hyperfine - python testing/benchmark_hooks.py diff --git a/testing/README.md b/testing/README.md index 2008ccf..628e8ac 100644 --- a/testing/README.md +++ b/testing/README.md @@ -9,6 +9,4 @@ pre-commit try-repo ./.. clang-tidy --verbose --all-files ## Benchmark -```bash -python3 testing/benchmark_hooks.py -``` +See [benchmark](./benchmark.md). diff --git a/testing/benchmark.md b/testing/benchmark.md index 6a5fc35..db1c28b 100644 --- a/testing/benchmark.md +++ b/testing/benchmark.md @@ -4,26 +4,69 @@ This document outlines the benchmarking process for comparing the performance of > About tests performance can be found at: [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/cpp-linter/cpp-linter-hooks) -## Running the Benchmark +## Run benchmark separately ```bash -python3 testing/benchmark_hooks.py +rm -rf examples || true +git clone --depth 1 --quiet https://github.com/gouravthakur39/beginners-C-program-examples.git examples -# or +pre-commit clean +hyperfine --warmup 1 -r 10 'pre-commit run --config testing/test-cpp-linter-hooks.yaml --all-files' -hyperfine -i --warmup 1 -r 5 'pre-commit run --config ../testing/benchmark_hook_1.yaml --all-files' 'pre-commit run --config ../testing/benchmark_hook_2.yaml --all-files' +rm -rf examples || true +git clone --depth 1 --quiet https://github.com/gouravthakur39/beginners-C-program-examples.git examples +pre-commit clean + +hyperfine --warmup 1 -r 10 'pre-commit run --config testing/test-mirrors-clang-format.yaml --all-files' +``` + +Results: + +```bash +# Updated on 2025-09-19 + +Cleaned /home/sxp/.cache/pre-commit. +Benchmark 1: pre-commit run --config testing/test-cpp-linter-hooks.yaml --all-files + Time (mean ± σ): 150.2 ms ± 1.8 ms [User: 121.7 ms, System: 29.2 ms] + Range (min … max): 148.3 ms … 153.9 ms 10 runs + +Cleaned /home/sxp/.cache/pre-commit. +Benchmark 1: pre-commit run --config testing/test-mirrors-clang-format.yaml --all-files + Time (mean ± σ): 122.6 ms ± 1.9 ms [User: 98.0 ms, System: 24.7 ms] + Range (min … max): 120.3 ms … 125.5 ms 10 runs +``` + +### Run benchmark comparison + +Compare the results of both commands. + +```bash +rm -rf examples || true +git clone --depth 1 --quiet https://github.com/gouravthakur39/beginners-C-program-examples.git examples + +hyperfine -i --warmup 1 -r 20 'pre-commit run --config ../testing/test-cpp-linter-hooks.yaml --all-files' 'pre-commit run --config ../testing/test-mirrors-clang-format.yaml --all-files' ``` -## Results +Results: ```bash -# Updated on 2025-09-02 -Benchmark Results: +# Updated on 2025-09-19 +Benchmark 1: pre-commit run --config ../testing/test-cpp-linter-hooks.yaml --all-files + Time (mean ± σ): 84.1 ms ± 3.2 ms [User: 73.5 ms, System: 10.2 ms] + Range (min … max): 79.7 ms … 95.2 ms 20 runs + + Warning: Ignoring non-zero exit code. -Hook | Avg (s) | Std (s) | Min (s) | Max (s) | Runs ----------------------+------------------+------------------+------------------+------------------+----------------- -cpp-linter-hooks | 12.473 | 1.738 | 11.334 | 15.514 | 5 -mirrors-clang-format | 4.960 | 0.229 | 4.645 | 5.284 | 5 +Benchmark 2: pre-commit run --config ../testing/test-mirrors-clang-format.yaml --all-files + Time (mean ± σ): 85.0 ms ± 3.0 ms [User: 71.8 ms, System: 13.3 ms] + Range (min … max): 81.0 ms … 91.0 ms 20 runs -Results saved to testing/benchmark_results.txt + Warning: Ignoring non-zero exit code. + +Summary + 'pre-commit run --config ../testing/test-cpp-linter-hooks.yaml --all-files' ran + 1.01 ± 0.05 times faster than 'pre-commit run --config ../testing/test-mirrors-clang-format.yaml --all-files' ``` + +> [!NOTE] +> The results may vary based on the system and environment where the benchmarks are run. diff --git a/testing/benchmark_hooks.py b/testing/benchmark_hooks.py deleted file mode 100644 index 96ad924..0000000 --- a/testing/benchmark_hooks.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -""" -Benchmark script to compare performance of cpp-linter-hooks vs mirrors-clang-format using hyperfine. - -Usage: - python benchmark_hooks.py - # or directly with hyperfine: - hyperfine --warmup 1 -r 5 'pre-commit run --config ../testing/benchmark_hook_1.yaml --all-files' 'pre-commit run --config ../testing/benchmark_hook_2.yaml --all-files' - -Requirements: -- pre-commit must be installed and available in PATH -- Two config files: - - testing/cpp-linter-hooks.yaml - - testing/mirrors-clang-format.yaml -- Target files: testing/examples/*.c (or adjust as needed) -""" - -import os -import subprocess -import sys - -HOOKS = [ - { - "name": "cpp-linter-hooks", - "config": "../testing/benchmark_hook_1.yaml", - }, - { - "name": "mirrors-clang-format", - "config": "../testing/benchmark_hook_2.yaml", - }, -] - -REPEATS = 5 -RESULTS_FILE = "testing/benchmark_results.txt" - - -def prepare_code(): - try: - subprocess.run( - [ - "git", - "clone", - "--depth", - "1", - "https://github.com/gouravthakur39/beginners-C-program-examples.git", - "examples", - ], - check=True, - ) - except subprocess.CalledProcessError: - pass - - -def run_hyperfine(): - os.chdir("examples") - commands = [ - f"pre-commit run --config {hook['config']} --all-files" for hook in HOOKS - ] - hyperfine_cmd = [ - "hyperfine", - "--warmup", - "1", - "-r", - str(REPEATS), - ] + commands - print("Running benchmark with hyperfine:") - print(" ".join(hyperfine_cmd)) - try: - subprocess.run(hyperfine_cmd, check=True) - except FileNotFoundError: - print( - "hyperfine is not installed. Please install it with 'cargo install hyperfine' or 'brew install hyperfine'." - ) - sys.exit(1) - os.chdir("..") - - -def main(): - prepare_code() - run_hyperfine() - - -if __name__ == "__main__": - main() diff --git a/testing/benchmark_results.txt b/testing/benchmark_results.txt deleted file mode 100644 index 55c4579..0000000 --- a/testing/benchmark_results.txt +++ /dev/null @@ -1,6 +0,0 @@ -Benchmark Results: - -Hook | Avg (s) | Std (s) | Min (s) | Max (s) | Runs ----------------------+------------------+------------------+------------------+------------------+----------------- -cpp-linter-hooks | 12.473 | 1.738 | 11.334 | 15.514 | 5 -mirrors-clang-format | 4.960 | 0.229 | 4.645 | 5.284 | 5 diff --git a/testing/good.c b/testing/good.c index d907942..67e9a33 100644 --- a/testing/good.c +++ b/testing/good.c @@ -1,6 +1,8 @@ #include -int main() { - for (;;) break; - printf("Hello world!\n"); - return 0; +int main() +{ + for (;;) + break; + printf("Hello world!\n"); + return 0; } diff --git a/testing/main.c b/testing/main.c index 6b1a169..67e9a33 100644 --- a/testing/main.c +++ b/testing/main.c @@ -1,2 +1,8 @@ #include -int main() {for (;;) break; printf("Hello world!\n");return 0;} +int main() +{ + for (;;) + break; + printf("Hello world!\n"); + return 0; +} diff --git a/testing/benchmark_hook_1.yaml b/testing/test-cpp-linter-hooks.yaml similarity index 76% rename from testing/benchmark_hook_1.yaml rename to testing/test-cpp-linter-hooks.yaml index 3019bfc..8c8ecbf 100644 --- a/testing/benchmark_hook_1.yaml +++ b/testing/test-cpp-linter-hooks.yaml @@ -2,5 +2,5 @@ repos: - repo: https://github.com/cpp-linter/cpp-linter-hooks rev: v1.1.3 hooks: - - id: clang-format + - id: clang-format # v 21.1.0 args: [--style=file] diff --git a/testing/benchmark_hook_2.yaml b/testing/test-mirrors-clang-format.yaml similarity index 100% rename from testing/benchmark_hook_2.yaml rename to testing/test-mirrors-clang-format.yaml From f7107c48ec8c895342d5d064a2f5928622bcbed5 Mon Sep 17 00:00:00 2001 From: shenxianpeng Date: Fri, 19 Sep 2025 23:04:14 +0300 Subject: [PATCH 5/7] fix: revert example code --- testing/good.c | 12 +++++------- testing/main.c | 8 +------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/testing/good.c b/testing/good.c index 67e9a33..5e97c8b 100644 --- a/testing/good.c +++ b/testing/good.c @@ -1,8 +1,6 @@ #include -int main() -{ - for (;;) - break; - printf("Hello world!\n"); - return 0; -} +int main() { + for (;;) break; + printf("Hello world!\n"); + return 0; +} \ No newline at end of file diff --git a/testing/main.c b/testing/main.c index 67e9a33..6b1a169 100644 --- a/testing/main.c +++ b/testing/main.c @@ -1,8 +1,2 @@ #include -int main() -{ - for (;;) - break; - printf("Hello world!\n"); - return 0; -} +int main() {for (;;) break; printf("Hello world!\n");return 0;} From bd7e97773ca36448266685367b5d3a8f2e692a1e Mon Sep 17 00:00:00 2001 From: shenxianpeng Date: Fri, 19 Sep 2025 23:05:03 +0300 Subject: [PATCH 6/7] fix: revert example code --- testing/good.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/good.c b/testing/good.c index 5e97c8b..d907942 100644 --- a/testing/good.c +++ b/testing/good.c @@ -3,4 +3,4 @@ int main() { for (;;) break; printf("Hello world!\n"); return 0; -} \ No newline at end of file +} From 07369ee6863781700995971c99475c80ac3368a0 Mon Sep 17 00:00:00 2001 From: shenxianpeng Date: Fri, 19 Sep 2025 23:07:01 +0300 Subject: [PATCH 7/7] chore: pre-commit autoupdate --- .pre-commit-config.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7253b24..4562e09 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,8 +10,9 @@ repos: - id: check-toml - id: requirements-txt-fixer - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.11 + rev: v0.13.1 hooks: - - id: ruff - args: [--fix] + # Run the linter. + - id: ruff-check + # Run the formatter. - id: ruff-format