lockhammer.c: separate file, improve reproducibility and quality-of-life

Move the measurement routines out of the test harness for separate
compilation

- The test harness is compiled using a generic compiler target, while
  the lock routine (inlined in measure.c) uses a specific build target
  selected by the Makefile's USE_* variables.

Enhancements to improve run-to-run reproducibility

- duration-based instead of acquires-based measurement
- multiple measurements per run, permuting iterations, critical/parallel
  duration, num_threads/pinorders
- lock variable allocated from persistent hugetlb page to reuse the same
  physical address run-to-run
- any-cpu arbitrary pinorder assignment
- add a runtime flag --disable-outline-atomics-lse to use exclusives
  on a system that provides LSE support, when compiled with the outline
  atomics library (aarch64, USE_BUILTIN=1)

Quality-of-life enhancements

- run results summary deliminted for easy copy/paste to spreadsheet
- output measurements to JSON

Change-Id: I1fa7918827d98ebcf3c17059c14c683fa2bf8079

Author: jty2

benchmarks/lockhammer/include/lockhammer.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2017-2025, The Linux Foundation. All rights reserved.
  *
  * SPDX-License-Identifier:    BSD-3-Clause
  *
@@ -33,49 +33,187 @@
 #define __LOCKHAMMER_H__
 
 
-#ifndef initialize_lock
-    #define initialize_lock(lock, thread)
+// PROGRESS_TICK_PROFILE - prints each thread's timer value at lock_acquires milestones to show thread concurrency
+#define PROGRESS_TICK_PROFILE
+
+enum units { NS,
+             INSTS, NOT_SET };
+typedef enum units Units;
+
+#define _stringify(x) #x
+#define stringify(x) _stringify(x)
+
+// per_thread_results_t - each thread returns its results in this struct (inside thread_args_t)
+typedef struct {
+    unsigned long cpu_affined;  // which CPU this was pinned on.
+
+    unsigned long lock_acquires;// number of locks acquired-and-released per thread
+    unsigned long cputime_ns;   // this thread's CPU time in nanoseconds
+    unsigned long walltime_ns;  // this thread's wall clock time in nanoseconds
+    unsigned long hmrdepth;     // depth=lock-specific notion of contention
+
+    unsigned long hwtimer_start; // timer value at start of measurement loop
+    unsigned long hwtimer_end;   //         "'  at end
+
+    unsigned long hwtimer_10p;   // timer value at 10% of work completion
+    unsigned long hwtimer_25p;   //         ""  at 25%
+    unsigned long hwtimer_50p;   //         ""  at 50%
+    unsigned long hwtimer_75p;   //         ""  at 75%
+    unsigned long hwtimer_90p;   //         ""  at 90%
+
+    // hold/post durations from calibrate_timer()
+    double hold_ns, post_ns;
+
+    // metrics only for osq_lock
+    unsigned long osq_lock_wait_next_spins;
+    unsigned long osq_unlock_wait_next_spins;
+    unsigned long osq_lock_locked_spins;
+    unsigned long osq_lock_unqueue_spins;
+    unsigned long osq_lock_acquire_backoffs;
+
+} per_thread_results_t;
+
+
+// thread_args_t -- pointer to an instance of this is passed to each thread
+typedef struct {
+    unsigned long thread_num;       // thread number, ordinal 0
+    unsigned long num_threads;      // number of worker threads in total for experiment
+    unsigned long num_acquires;     // -a flag, aka nacqrs, aka number of acquires per thread to do
+    unsigned long *lock;            // pointer to the lock variable
+
+    unsigned long *p_start_ns;      // marshal thread's monotonic start time, in ns, for computing wall_elapsed_ns; only marshall thread sets this
+    unsigned long hold, post;       // ncrit, nparallel
+    Units hold_unit, post_unit;     // NS or INSTS, hold_unit = ncrit_units, post_unit = nparallel_units
+    unsigned long hold_count;
+    unsigned long post_count;
+
+    double tickspns;                // number of ticks_per_ns
+
+    unsigned long run_on_this_cpu;  // logical CPU on which a worker thread is to run
+
+    unsigned long run_limit_ticks;  // if non-zero, the number of timer ticks to run for when using --run-limit-ticks or --run-limit-seconds
+    unsigned long run_limit_inner_loop_iters;  // the number of lock acquire/release sequences to run before checking the hwtimer when using --run-limit-ticks or --run-limit-seconds
+    unsigned long hwtimer_frequency;
+
+    int verbose;
+    unsigned long blackhole_numtries;
+
+    per_thread_results_t results;   // output data structure
+
+} thread_args_t;
+
+// pinorder_t - describes a set of CPUs on which to run worker threads
+typedef struct {
+    int * cpu_list;     // pointer to an array of int.  index into this array is the thread number, each element is the logical CPU on which that thread is to run.
+    size_t num_threads; // number of threads defined for this pinorder (i.e. length of the number of valid entries in the pinorder array).
+} pinorder_t;
+
+
+typedef struct {
+    unsigned long t;   // duration time, either in nanoseconds or iterations
+    Units unit;        // duration unit, either NS or INSTS
+} duration_t;
+
+// test_args_t - mostly command line parameters
+typedef struct {
+    unsigned long num_acquires;  // -a    number of acquires (not documented?)
+    duration_t * crits;     // -c, --cn=, --ci=    critical duration
+    duration_t * pars;      // -p, --pn=, --pi=    parallel duration
+    size_t num_crits;
+    size_t num_pars;
+    unsigned long ileave;    // -i    interleave value for SMT pinning
+    int scheduling_policy;   // -S    use explicit scheduling policy
+    size_t num_pinorders;
+    pinorder_t * pinorders;  // -o    CPU pinning order
+    unsigned long  timeout_usec;   // -A  timeout_usec
+
+    int hugepagesz;
+    int use_mmap;
+    int mmap_hugepage_offset_exists;
+    int print_hugepage_physaddr;
+    size_t mmap_hugepage_offset;
+    size_t mmap_hugepage_physaddr;
+    unsigned long hwtimer_frequency;
+    unsigned long probed_hwtimer_frequency;
+    long estimate_hwtimer_freq_cpu;
+
+    double run_limit_seconds;
+    unsigned long run_limit_ticks;
+    unsigned long run_limit_inner_loop_iters;
+    int ignore_unknown_scaling_governor;
+    int suppress_cpu_frequency_warnings;
+    const char * cpuorder_filename;
+#ifdef JSON_OUTPUT
+    const char * json_output_filename;
 #endif
-#ifndef parse_test_args
-    #define parse_test_args(args, argc, argv)
+#ifdef __aarch64__
+    char disable_outline_atomics_lse;
 #endif
-#ifndef thread_local_init
-    #define thread_local_init(smtid)
+    int verbose;
+    size_t iterations;
+    size_t blackhole_numtries;
+} test_args_t;
+
+// system_info_t - system configuration data
+typedef struct {
+    unsigned long num_cores;    // number of processors configured by the operating system
+    size_t page_size_bytes;     // page size in bytes
+    size_t erg_bytes;           // number of bytes per exclusive reservation granule (e.g. cache line/block)
+
+    cpu_set_t avail_cores;      // cores that the CPU affinity mask allows us to run on
+    size_t num_avail_cores;     // number of cores that the CPU affinity mask allows us to run on
+    size_t num_online_cores;    // the number of cores that getconf _NPROCESSORS_ONLN returns
+
+    // num_online_cores can be less than num_cores because some may be offline or not permitted by affinity mask
+    // num_avail_cores may be less than num_online_cores because some online cores may be isolated
+} system_info_t;
+
+// locks_t -- pointers to the actual locks to be used
+typedef struct {
+    unsigned long * p_test_lock;        // address of main lock
+    unsigned long * p_ready_lock;       // lock to synchronize all threads' entry into hmr()
+    unsigned long * p_sync_lock;        // lock to synchronize before blackhole cabliration
+    unsigned long * p_calibrate_lock;   // lock to synchronize after blackhole calibration
+} locks_t;
+
+// calibrate_blackhole -- (used in osq_lock)
+unsigned long calibrate_blackhole(unsigned long target, unsigned long tokens_low, unsigned long tokens_high, unsigned long core_id, unsigned long NUMTRIES);
+
+// evaluate_blackhole -- returns average duration of NUMTRIES
+int64_t evaluate_blackhole( const unsigned long tokens_mid, const unsigned long NUMTRIES);
+
+// blackhole() -- runs a small loop to consume time (also used in osq_lock)
+void blackhole(unsigned long iters);
+
+// measure_setup_initialize_lock() -- calls lock-specific setup routine if it exists
+void measure_setup_initialize_lock(locks_t * p_locks, pinorder_t * pinorder);
+
+// measure_setup_parse_test_args() -- calls lock-specific parsing routine if it exists
+void measure_setup_parse_test_args(test_args_t * p_test_args, int argc, char ** argv);
+
+// convert the struct timespec to only nanoseconds
+unsigned long timespec_to_ns (struct timespec * ts);
+
+// selectively disable LSE instructions in outline atomics/libgcc; in measure.c
+void handle_disable_outline_atomics_lse(void);
+
+#if __GNUC__==1
+#define NOINLINE __attribute__((noinline))
+#elif __clang__==1
+#define NOINLINE __attribute__((noinline))
+#else
+#define NOINLINE
 #endif
 
-enum units { NS,
-             INSTS };
-typedef enum units Units;
+#if __GNUC__==1
+#define NO_UNROLL_LOOP _Pragma("GCC unroll 0")
+#elif __clang__==1
+#define NO_UNROLL_LOOP _Pragma("clang loop unroll(disable)")
+#else
+#define NO_UNROLL_LOOP
+#endif
 
-struct thread_args {
-    unsigned long ncores;
-    unsigned long nthrds;
-    unsigned long ileave;
-    unsigned long iter;
-    unsigned long *lock;
-    unsigned long *rst;
-    unsigned long *nsec;
-    unsigned long *real_nsec;
-    unsigned long *depth;
-    unsigned long *nstart;
-    unsigned long hold, post;
-    Units hold_unit, post_unit;
-    double tickspns;
-    int *pinorder;
-};
-typedef struct thread_args thread_args;
-
-struct test_args {
-    unsigned long nthrds;
-    unsigned long nacqrs;
-    unsigned long ncrit;
-    Units ncrit_units;
-    unsigned long nparallel;
-    Units nparallel_units;
-    unsigned long ileave;
-    unsigned char safemode;
-    int *pinorder;
-};
-typedef struct test_args test_args;
 
 #endif
+
+/* vim: set tabstop=4 shiftwidth=4 softtabstop=4 expandtab: */