From af065079ed8db0e3cc530d5f6954430b8638c710 Mon Sep 17 00:00:00 2001 From: Edwin Chiu Date: Tue, 4 Nov 2025 23:16:54 -0500 Subject: [PATCH 1/2] Set iodepth=1 to suppress warning from fio fio warning is "note: both iodepth >= 1 and synchronous I/O engine are selected, queue depth will be capped at 1" --- internal/script/script_defs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/script/script_defs.go b/internal/script/script_defs.go index c506d3a..1257855 100644 --- a/internal/script/script_defs.go +++ b/internal/script/script_defs.go @@ -1174,7 +1174,7 @@ sync # single-threaded read & write bandwidth test fio --name=bandwidth --directory=$test_dir --numjobs=$numjobs \ --size="$file_size_g"G --time_based --runtime=$runtime --ramp_time=$ramp_time --ioengine=$ioengine \ ---direct=1 --verify=0 --bs=1M --iodepth=64 --rw=rw \ +--direct=1 --verify=0 --bs=1M --iodepth=1 --rw=rw \ --group_reporting=1 --iodepth_batch_submit=64 \ --iodepth_batch_complete_max=64 rm -rf $test_dir From 17c88ba206a7b106cd9a20aa8de2b45a4729e0be Mon Sep 17 00:00:00 2001 From: Edwin Chiu Date: Wed, 5 Nov 2025 00:16:02 -0500 Subject: [PATCH 2/2] Update fio benchmark to output as JSON and include latency and IOPs in report --- internal/report/table_defs.go | 10 +- internal/report/table_helpers_benchmarking.go | 139 +++++++++++++++--- internal/script/script_defs.go | 3 +- 3 files changed, 127 insertions(+), 25 deletions(-) diff --git a/internal/report/table_defs.go b/internal/report/table_defs.go index f42d675..1052685 100644 --- a/internal/report/table_defs.go +++ b/internal/report/table_defs.go @@ -2363,13 +2363,15 @@ func numaBenchmarkTableValues(outputs map[string]script.ScriptOutput) []Field { } func storageBenchmarkTableValues(outputs map[string]script.ScriptOutput) []Field { - readBW, writeBW := storagePerfFromOutput(outputs) - if readBW == "" && writeBW == "" { + readLat, readBw, writeLat, writeBw := storagePerfFromOutput(outputs) + if readLat == "" && readBw == "" && writeLat == "" && writeBw == "" { return []Field{} } return []Field{ - {Name: "Single-Thread Read Bandwidth", Values: []string{readBW}}, - {Name: "Single-Thread Write Bandwidth", Values: []string{writeBW}}, + {Name: "Single-Thread Read Latency (ns)", Values: []string{readLat}}, + {Name: "Single-Thread Read Bandwidth (MiB/s)", Values: []string{readBw}}, + {Name: "Single-Thread Write Latency (ns)", Values: []string{writeLat}}, + {Name: "Single-Thread Write Bandwidth (MiB/s)", Values: []string{writeBw}}, } } diff --git a/internal/report/table_helpers_benchmarking.go b/internal/report/table_helpers_benchmarking.go index 01fe9c2..8f62ca4 100644 --- a/internal/report/table_helpers_benchmarking.go +++ b/internal/report/table_helpers_benchmarking.go @@ -4,15 +4,100 @@ package report // SPDX-License-Identifier: BSD-3-Clause import ( + "encoding/json" "fmt" "log/slog" "perfspect/internal/script" "perfspect/internal/util" - "regexp" "strconv" "strings" ) +// fioOutput is the top-level struct for the FIO JSON report. +// ref: https://fio.readthedocs.io/en/latest/fio_doc.html#json-output +type fioOutput struct { + FioVersion string `json:"fio version"` + Timestamp int64 `json:"timestamp"` + TimestampMs int64 `json:"timestamp_ms"` + Time string `json:"time"` + Jobs []fioJob `json:"jobs"` +} + +// Job represents a single job's results within the FIO report. +type fioJob struct { + Jobname string `json:"jobname"` + Groupid int `json:"groupid"` + JobStart int64 `json:"job_start"` + Error int `json:"error"` + Eta int `json:"eta"` + Elapsed int `json:"elapsed"` + Read fioIOStats `json:"read"` + Write fioIOStats `json:"write"` + Trim fioIOStats `json:"trim"` + JobRuntime int `json:"job_runtime"` + UsrCPU float64 `json:"usr_cpu"` + SysCPU float64 `json:"sys_cpu"` + Ctx int `json:"ctx"` + Majf int `json:"majf"` + Minf int `json:"minf"` + IodepthLevel map[string]float64 `json:"iodepth_level"` + IodepthSubmit map[string]float64 `json:"iodepth_submit"` + IodepthComplete map[string]float64 `json:"iodepth_complete"` + LatencyNs map[string]float64 `json:"latency_ns"` + LatencyUs map[string]float64 `json:"latency_us"` + LatencyMs map[string]float64 `json:"latency_ms"` + LatencyDepth int `json:"latency_depth"` + LatencyTarget int `json:"latency_target"` + LatencyPercentile float64 `json:"latency_percentile"` + LatencyWindow int `json:"latency_window"` +} + +// IOStats holds the detailed I/O statistics for read, write, or trim operations. +type fioIOStats struct { + IoBytes int64 `json:"io_bytes"` + IoKbytes int64 `json:"io_kbytes"` + BwBytes int64 `json:"bw_bytes"` + Bw int64 `json:"bw"` + Iops float64 `json:"iops"` + Runtime int `json:"runtime"` + TotalIos int `json:"total_ios"` + ShortIos int `json:"short_ios"` + DropIos int `json:"drop_ios"` + SlatNs fioLatencyStats `json:"slat_ns"` + ClatNs fioLatencyStatsPercentiles `json:"clat_ns"` + LatNs fioLatencyStats `json:"lat_ns"` + BwMin int `json:"bw_min"` + BwMax int `json:"bw_max"` + BwAgg float64 `json:"bw_agg"` + BwMean float64 `json:"bw_mean"` + BwDev float64 `json:"bw_dev"` + BwSamples int `json:"bw_samples"` + IopsMin int `json:"iops_min"` + IopsMax int `json:"iops_max"` + IopsMean float64 `json:"iops_mean"` + IopsStddev float64 `json:"iops_stddev"` + IopsSamples int `json:"iops_samples"` +} + +// fioLatencyStats holds basic latency metrics. +type fioLatencyStats struct { + Min int64 `json:"min"` + Max int64 `json:"max"` + Mean float64 `json:"mean"` + Stddev float64 `json:"stddev"` + N int `json:"N"` +} + +// LatencyStatsPercentiles holds latency metrics including percentiles. +type fioLatencyStatsPercentiles struct { + Min int64 `json:"min"` + Max int64 `json:"max"` + Mean float64 `json:"mean"` + Stddev float64 `json:"stddev"` + N int `json:"N"` + Percentile map[string]int64 `json:"percentile"` +} + func cpuSpeedFromOutput(outputs map[string]script.ScriptOutput) string { var vals []float64 for line := range strings.SplitSeq(strings.TrimSpace(outputs[script.SpeedBenchmarkScriptName].Stdout), "\n") { @@ -35,26 +120,40 @@ func cpuSpeedFromOutput(outputs map[string]script.ScriptOutput) string { return fmt.Sprintf("%.0f", util.GeoMean(vals)) } -func storagePerfFromOutput(outputs map[string]script.ScriptOutput) (readBW, writeBW string) { - // fio output format: - // READ: bw=140MiB/s (146MB/s), 140MiB/s-140MiB/s (146MB/s-146MB/s), io=16.4GiB (17.6GB), run=120004-120004msec - // WRITE: bw=139MiB/s (146MB/s), 139MiB/s-139MiB/s (146MB/s-146MB/s), io=16.3GiB (17.5GB), run=120004-120004msec - re := regexp.MustCompile(` bw=(\d+[.]?[\d]*\w+\/s)`) - for line := range strings.SplitSeq(strings.TrimSpace(outputs[script.StorageBenchmarkScriptName].Stdout), "\n") { - if strings.Contains(line, "READ: bw=") { - matches := re.FindStringSubmatch(line) - if len(matches) != 0 { - readBW = matches[1] - } - } else if strings.Contains(line, "WRITE: bw=") { - matches := re.FindStringSubmatch(line) - if len(matches) != 0 { - writeBW = matches[1] - } - } else if strings.Contains(line, "ERROR: ") { - slog.Error("failed to run storage benchmark", slog.String("line", line)) - } +func storagePerfFromOutput(outputs map[string]script.ScriptOutput) (readLat, readBw, writeLat, writeBw string) { + output := outputs[script.StorageBenchmarkScriptName].Stdout + slog.Debug("storage benchmark output", slog.String("output", output)) + + i := strings.Index(output, "{\n \"fio version\"") + if i >= 0 { + output = output[i:] + } else { + slog.Error("Unable to find fio output", slog.String("output", output)) + return + } + if strings.Contains(output, "ERROR:") { + slog.Error("failed to run storage benchmark", slog.String("output", output)) + return } + + slog.Debug("parsing storage benchmark output") + var fioData fioOutput + if err := json.Unmarshal([]byte(output), &fioData); err != nil { + slog.Error("Error unmarshalling JSON", slog.String("error", err.Error())) + return + } + if len(fioData.Jobs) > 0 { + slog.Debug("jobs found in storage benchmark output") + job := fioData.Jobs[0] + readBw = fmt.Sprintf("%d", job.Read.Bw/1024) + readLat = fmt.Sprintf("%.0f", job.Read.LatNs.Mean) + writeBw = fmt.Sprintf("%d", job.Write.Bw/1024) + writeLat = fmt.Sprintf("%.0f", job.Write.LatNs.Mean) + } else { + slog.Error("No jobs found in storage benchmark output", slog.String("output", output)) + } + + slog.Debug("storage benchmark output", slog.String("readLat", readLat), slog.String("readBw", readBw), slog.String("writeLat", writeLat), slog.String("writeBw", writeBw)) return } diff --git a/internal/script/script_defs.go b/internal/script/script_defs.go index 1257855..70561c8 100644 --- a/internal/script/script_defs.go +++ b/internal/script/script_defs.go @@ -1176,7 +1176,8 @@ fio --name=bandwidth --directory=$test_dir --numjobs=$numjobs \ --size="$file_size_g"G --time_based --runtime=$runtime --ramp_time=$ramp_time --ioengine=$ioengine \ --direct=1 --verify=0 --bs=1M --iodepth=1 --rw=rw \ --group_reporting=1 --iodepth_batch_submit=64 \ ---iodepth_batch_complete_max=64 +--iodepth_batch_complete_max=64 \ +--output-format=json rm -rf $test_dir `, Superuser: true,