Merge pull request #424 from TypedDevs/chore/max-ms-on-bench

Add max_ms option on bench

Author: Chemaclass

bashunit

@@ -39,6 +39,7 @@ _BENCH_MODE=false
 # Determine bench mode early so path arguments use correct pattern
 for arg in "$@"; do
   if [[ $arg == "-b" || $arg == "--bench" ]]; then
+    export BASHUNIT_BENCH_MODE=true
     _BENCH_MODE=true
     break
   fi

docs/benchmarks.md

@@ -4,7 +4,8 @@ bashunit allows defining benchmark functions to measure execution time of your s
 
 Functions prefixed with `bench` are treated as benchmarks. You can annotate them with
 `@revs` (revolutions) and `@its` (iterations) comments to control how many times the code
-is executed.
+is executed. Use `@max_ms` to mark the benchmark as failed when the average
+execution time exceeds the given value in milliseconds.
 
 Each iteration is executed in a separate process, and the average time is reported.
 
@@ -15,6 +16,15 @@ function bench_my_function() {
 }
 ```
 
+You can optionally fail a benchmark when it exceeds a max_ms:
+
+```bash
+# @revs=10 @its=1 @max_ms=5
+function bench_slow() {
+  slow_operation
+}
+```
+
 Run benchmarks using the `--bench` option. Each iteration prints its duration
 when using detailed output, or a dot when simple output is enabled:
 
@@ -26,13 +36,57 @@ If no file is provided, **bashunit** uses `BASHUNIT_DEFAULT_PATH` to locate all
 
 Example output:
 
-```-vue
-bashunit - {{ pkg.version }}
+::: code-group
+```bash [Simple]
+./bashunit --bench --simple
+
+.........
+
+Benchmark Results (avg ms)
+======================================================================
+
+Name                                 Revs    Its    Avg(ms)     Status
+bench_bashunit_runs_benchmarks          3      2        727
+bench_bashunit_functional_run           1      1        243
+bench_bashunit_default_path             1      1        731      > 600
+bench_run_bashunit_functional           2      1        371      > 300
+bench_sleep                             5      2         21       ≤ 25
+bench_sleep_synonym                     3      2         32
+```
+```bash [Detailed]
+./bashunit --bench
+
+Running tests/benchmark/bashunit_bench.sh
+Bench bashunit runs benchmarks [1/2] 843 ms
+Bench bashunit runs benchmarks [2/2] 834 ms
+Bench bashunit functional run [1/1] 281 ms
+Bench bashunit default path [1/1] 736 ms
+
+Running tests/benchmark/fixtures/bashunit_functional_bench.sh
+Bench run bashunit functional [1/1] 430 ms
+
+Running tests/benchmark/fixtures/bashunit_sleep_bench.sh
+Bench sleep [1/2] 48 ms
+Bench sleep [2/2] 26 ms
+Bench sleep synonym [1/2] 32 ms
+Bench sleep synonym [2/2] 34 ms
+
 
 Benchmark Results (avg ms)
-Name                                         Revs      Its  Avg(ms)
-bench_my_function                             1000        5    12.34
+=====================================================================
+
+Name                                 Revs    Its    Avg(ms)    Status
+bench_bashunit_runs_benchmarks          3      2        838
+bench_bashunit_functional_run           1      1        281
+bench_bashunit_default_path             1      1        736     > 600
+bench_run_bashunit_functional           2      1        430     > 300
+bench_sleep                             5      2         37      > 25
+bench_sleep_synonym                     3      2         33
+
 ```
+:::
+
+
 
 <script setup>
 import pkg from '../package.json'

src/benchmark.sh

@@ -4,12 +4,14 @@ _BENCH_NAMES=()
 _BENCH_REVS=()
 _BENCH_ITS=()
 _BENCH_AVERAGES=()
+_BENCH_MAX_MILLIS=()
 
 function benchmark::parse_annotations() {
   local fn_name=$1
   local script=$2
   local revs=1
   local its=1
+  local max_ms=""
 
   local annotation
   annotation=$(awk "/function[[:space:]]+${fn_name}[[:space:]]*\(/ {print prev; exit} {prev=\$0}" "$script")
@@ -26,21 +28,33 @@ function benchmark::parse_annotations() {
     its="${BASH_REMATCH[1]}"
   fi
 
-  echo "$revs" "$its"
+  if [[ $annotation =~ @max_ms=([0-9.]+) ]]; then
+    max_ms="${BASH_REMATCH[1]}"
+  elif [[ $annotation =~ @max_ms=([0-9.]+) ]]; then
+    max_ms="${BASH_REMATCH[1]}"
+  fi
+
+  if [[ -n "$max_ms" ]]; then
+    echo "$revs" "$its" "$max_ms"
+  else
+    echo "$revs" "$its"
+  fi
 }
 
 function benchmark::add_result() {
   _BENCH_NAMES+=("$1")
   _BENCH_REVS+=("$2")
   _BENCH_ITS+=("$3")
   _BENCH_AVERAGES+=("$4")
+  _BENCH_MAX_MILLIS+=("$5")
 }
 
 # shellcheck disable=SC2155
 function benchmark::run_function() {
   local fn_name=$1
   local revs=$2
   local its=$3
+  local max_ms=$4
   local durations=()
 
   for ((i=1; i<=its; i++)); do
@@ -55,28 +69,75 @@ function benchmark::run_function() {
     local dur_ms=$(math::calculate "$dur_ns / 1000000")
     durations+=("$dur_ms")
 
-    local line="bench $fn_name [$i/$its] ${dur_ms} ms"
-    state::print_line "successful" "$line"
+    if env::is_bench_mode_enabled; then
+      local label="$(helper::normalize_test_function_name "$fn_name")"
+      local line="$label [$i/$its] ${dur_ms} ms"
+      state::print_line "successful" "$line"
+    fi
   done
 
   local sum=0
   for d in "${durations[@]}"; do
     sum=$(math::calculate "$sum + $d")
   done
   local avg=$(math::calculate "$sum / ${#durations[@]}")
-  benchmark::add_result "$fn_name" "$revs" "$its" "$avg"
+  benchmark::add_result "$fn_name" "$revs" "$its" "$avg" "$max_ms"
 }
 
 function benchmark::print_results() {
+  if ! env::is_bench_mode_enabled; then
+    return
+  fi
+
   if (( ${#_BENCH_NAMES[@]} == 0 )); then
     return
   fi
+
   if env::is_simple_output_enabled; then
     printf "\n"
   fi
+
   printf "\nBenchmark Results (avg ms)\n"
-  printf '%-40s %8s %8s %8s\n' "Name" "Revs" "Its" "Avg(ms)"
+  print_line 80 "="
+  printf "\n"
+
+  local has_threshold=false
+  for val in "${_BENCH_MAX_MILLIS[@]}"; do
+    if [[ -n "$val" ]]; then
+      has_threshold=true
+      break
+    fi
+  done
+
+  if $has_threshold; then
+    printf '%-40s %6s %6s %10s %12s\n' "Name" "Revs" "Its" "Avg(ms)" "Status"
+  else
+    printf '%-40s %6s %6s %10s\n' "Name" "Revs" "Its" "Avg(ms)"
+  fi
+
   for i in "${!_BENCH_NAMES[@]}"; do
-    printf '%-40s %8s %8s %8s\n' "${_BENCH_NAMES[$i]}" "${_BENCH_REVS[$i]}" "${_BENCH_ITS[$i]}" "${_BENCH_AVERAGES[$i]}"
+    local name="${_BENCH_NAMES[$i]}"
+    local revs="${_BENCH_REVS[$i]}"
+    local its="${_BENCH_ITS[$i]}"
+    local avg="${_BENCH_AVERAGES[$i]}"
+    local max_ms="${_BENCH_MAX_MILLIS[$i]}"
+
+    if [[ -z "$max_ms" ]]; then
+      printf '%-40s %6s %6s %10s\n' "$name" "$revs" "$its" "$avg"
+      continue
+    fi
+
+    if (( $(echo "$avg <= $max_ms" | bc -l) )); then
+      local raw="≤ ${max_ms}"
+      printf -v padded "%14s" "$raw"
+      printf '%-40s %6s %6s %10s %12s\n' "$name" "$revs" "$its" "$avg" "$padded"
+      continue
+    fi
+
+    local raw="> ${max_ms}"
+    printf -v padded "%12s" "$raw"
+    printf '%-40s %6s %6s %10s %s%s%s\n' \
+      "$name" "$revs" "$its" "$avg" \
+      "$_COLOR_FAILED" "$padded" "${_COLOR_DEFAULT}"
   done
 }

src/env.sh

@@ -27,6 +27,7 @@ _DEFAULT_SIMPLE_OUTPUT="false"
 _DEFAULT_STOP_ON_FAILURE="false"
 _DEFAULT_SHOW_EXECUTION_TIME="true"
 _DEFAULT_VERBOSE="false"
+_DEFAULT_BENCH_MODE="false"
 
 : "${BASHUNIT_PARALLEL_RUN:=${PARALLEL_RUN:=$_DEFAULT_PARALLEL_RUN}}"
 : "${BASHUNIT_SHOW_HEADER:=${SHOW_HEADER:=$_DEFAULT_SHOW_HEADER}}"
@@ -35,6 +36,7 @@ _DEFAULT_VERBOSE="false"
 : "${BASHUNIT_STOP_ON_FAILURE:=${STOP_ON_FAILURE:=$_DEFAULT_STOP_ON_FAILURE}}"
 : "${BASHUNIT_SHOW_EXECUTION_TIME:=${SHOW_EXECUTION_TIME:=$_DEFAULT_SHOW_EXECUTION_TIME}}"
 : "${BASHUNIT_VERBOSE:=${VERBOSE:=$_DEFAULT_VERBOSE}}"
+: "${BASHUNIT_BENCH_MODE:=${BENCH_MODE:=$_DEFAULT_BENCH_MODE}}"
 
 function env::is_parallel_run_enabled() {
   [[ "$BASHUNIT_PARALLEL_RUN" == "true" ]]
@@ -68,6 +70,10 @@ function env::is_verbose_enabled() {
   [[ "$BASHUNIT_VERBOSE" == "true" ]]
 }
 
+function env::is_bench_mode_enabled() {
+  [[ "$BASHUNIT_BENCH_MODE" == "true" ]]
+}
+
 function env::active_internet_connection() {
   if ping -c 1 -W 3 google.com &> /dev/null; then
     return 0

src/globals.sh

@@ -87,3 +87,9 @@ function log() {
   local RESET='\033[0m'
   echo -e "$(current_timestamp) [$level]: $@ ${GRAY}#${BASH_SOURCE[1]}:${BASH_LINENO[0]}${RESET}" >> "$BASHUNIT_DEV_LOG"
 }
+
+function print_line() {
+  local length="${1:-70}"   # Default to 70 if not passed
+  local char="${2:--}"      # Default to '-' if not passed
+  printf '%*s\n' "$length" '' | tr ' ' "$char"
+}

src/runner.sh

@@ -143,9 +143,13 @@ function runner::call_bench_functions() {
     return
   fi
 
+  if env::is_bench_mode_enabled; then
+    runner::render_running_file_header "$script"
+  fi
+
   for fn_name in "${functions_to_run[@]}"; do
-    read -r revs its <<< "$(benchmark::parse_annotations "$fn_name" "$script")"
-    benchmark::run_function "$fn_name" "$revs" "$its"
+    read -r revs its max_ms <<< "$(benchmark::parse_annotations "$fn_name" "$script")"
+    benchmark::run_function "$fn_name" "$revs" "$its" "$max_ms"
     unset fn_name
   done
 

tests/benchmark/bashunit_bench.sh

@@ -21,7 +21,7 @@ function bench_bashunit_functional_run() {
   assert_successful_code "$output"
 }
 
-# @revs=1 @its=1
+# @revs=1 @its=1 @max_ms=600
 function bench_bashunit_default_path() {
   local env_file=./tests/benchmark/fixtures/.env.with_path
 

tests/benchmark/fixtures/bashunit_functional_bench.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# @revs=2 @its=1
+# @revs=2 @its=1 @max_ms=300
 function bench_run_bashunit_functional() {
   ./bashunit tests/functional/for_bench_test.sh -s -p >/dev/null
 }

tests/benchmark/fixtures/bashunit_sleep_bench.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-# @revs=5 @its=2
+# @revs=5 @its=2 @max_ms=25
 function bench_sleep() {
-  sleep 0.01
+  sleep 0.001
 }
 
 # @revolutions=3 @iterations=2

tests/unit/benchmark_test.sh

@@ -5,7 +5,7 @@ function set_up() {
 }
 
 function test_parse_annotations() {
-  assert_same "5 2" "$(benchmark::parse_annotations bench_sleep "$SCRIPT")"
+  assert_same "5 2 25" "$(benchmark::parse_annotations bench_sleep "$SCRIPT")"
 }
 
 function test_parse_annotations_with_synonyms() {
@@ -21,7 +21,7 @@ function test_run_function_collects_results() {
   _BENCH_ITS=()
   _BENCH_AVERAGES=()
 
-  benchmark::run_function bench_sleep 2 1
+  benchmark::run_function bench_sleep 2 1 ""
 
   assert_same "bench_sleep" "${_BENCH_NAMES[0]}"
   assert_same "2" "${_BENCH_REVS[0]}"