CLH (Craig Landin Hagersten) spinlocks initial checkin

CLH is a starvation-free queue spinlock which has FCFS
(first come, first served) order like MCS spinlock.
CLH spinlock is simpler than MCS spinlock, and each thread
is spinning on its previous node's wait flag. Our CLH
spinlock has an optional parameter to disable SEVL and WFE.

Author: Yangzheng Bai

benchmarks/lockhammer/Makefile

@@ -1,4 +1,5 @@
-override CFLAGS += -g -O3 -I. -I./include -I../../ext/mysql/include -I../../ext/linux/include -I../../ext/tbb/include
+# override keyword overwrites make command-line option LSE_ENABLE=y, therefore it has been removed
+CFLAGS += -g -O3 -I. -I./include -I../../ext/mysql/include -I../../ext/linux/include -I../../ext/tbb/include -I../../ext/sms/base
 
 ifneq ($(DEBUG_LEVEL),)
 ifeq ($(shell test $(DEBUG_LEVEL) -gt 0; echo $$?),0)
@@ -32,7 +33,8 @@ TEST_TARGETS=lh_swap_mutex \
 	lh_empty \
 	lh_jvm_objectmonitor \
 	lh_tbb_spin_rw_mutex \
-	lh_osq_lock
+	lh_osq_lock \
+    lh_clh_spinlock
 
 ifeq ($(TARGET_ARCH),aarch64)
 	TEST_TARGETS+=lh_hybrid_spinlock \
@@ -59,6 +61,9 @@ lh_hybrid_spinlock_fastdequeue: ../../ext/linux/hybrid_spinlock_fastdequeue.h in
 lh_osq_lock: ../../ext/linux/osq_lock.h ../../ext/linux/include/lk_atomics.h ../../ext/linux/include/lk_barrier.h ../../ext/linux/include/lk_cmpxchg.h include/atomics.h src/lockhammer.c
 	${CC} ${CFLAGS} -DATOMIC_TEST=\"$<\" src/lockhammer.c -o build/$@ ${LDFLAGS}
 
+lh_clh_spinlock: ../../ext/sms/clh_spinlock.h ../../ext/sms/base/build_config.h ../../ext/sms/base/cpu.h ../../ext/sms/base/llsc.h src/lockhammer.c
+	${CC} ${CFLAGS} -DATOMIC_TEST=\"$<\" src/lockhammer.c -o build/$@ ${LDFLAGS}
+
 lh_queued_spinlock: ../../ext/linux/queued_spinlock.h include/atomics.h ../../ext/linux/include/lk_atomics.h src/lockhammer.c
 	${CC} ${CFLAGS} -DATOMIC_TEST=\"$<\" src/lockhammer.c -o build/$@ ${LDFLAGS}
 

benchmarks/lockhammer/include/atomics.h

@@ -34,12 +34,6 @@
 #ifndef __LH_ATOMICS_H_
 #define __LH_ATOMICS_H_
 
-#ifndef initialize_lock
-	#define initialize_lock(lock, thread)
-#endif
-#ifndef parse_test_args
-	#define parse_test_args(args, argc, argv)
-#endif
 
 static inline void spin_wait (unsigned long wait_iter) {
 #if defined(__aarch64__)

benchmarks/lockhammer/include/lockhammer.h

@@ -32,6 +32,17 @@
 #ifndef __LOCKHAMMER_H__
 #define __LOCKHAMMER_H__
 
+
+#ifndef initialize_lock
+    #define initialize_lock(lock, thread)
+#endif
+#ifndef parse_test_args
+    #define parse_test_args(args, argc, argv)
+#endif
+#ifndef thread_local_init
+    #define thread_local_init(smtid)
+#endif
+
 enum units { NS,
              INSTS };
 typedef enum units Units;

benchmarks/lockhammer/scripts/lh_sweeptest_cfg.yaml

@@ -66,6 +66,7 @@ sweeptest:
         - lh_swap_mutex
         - lh_tbb_spin_rw_mutex
         - lh_ticket_spinlock
+        - lh_clh_spinlock
     cmd_aarch64: [lh_hybrid_spinlock, lh_hybrid_spinlock_fastdequeue]
     cmd_x86_64:
     repeat: 9

benchmarks/lockhammer/scripts/runall_obsolete.sh

@@ -448,6 +448,8 @@ void* hmr(void *ptr)
         synchronize_threads(&calibrate_lock, nthrds);
     }
 
+    thread_local_init(mycore);
+
 #ifdef DDEBUG
     printf("%ld %ld\n", hold_count, post_count);
 #endif

benchmarks/lockhammer/src/lockhammer.c

@@ -0,0 +1,75 @@
+// Copyright (c) 2017 ARM Limited. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+// Architecture detection is inferred from the toolchain. This relies on
+// the C compiler's system-specific macros.
+#if defined(__aarch64__)
+#define CONFIG_ARCH_ARM_V8
+#define CONFIG_ARCH_64BIT
+#elif defined(__arm__)
+#define CONFIG_ARCH_ARM_V7
+#define CONFIG_ARCH_32BIT
+#elif defined(__x86_64__)
+#define CONFIG_ARCH_X86_64
+#define CONFIG_ARCH_64BIT
+#elif defined(__i386__)
+#define CONFIG_ARCH_X86
+#define CONFIG_ARCH_32BIT
+#endif
+
+#if !defined(CONFIG_ARCH_64BIT) && !defined(CONFIG_ARCH_32BIT)
+#error Please add support for N-bit computing to build_config.h
+// If you experience this C pre-processor error, take a look at the place
+// in this file where CONFIG_ARCH_64/32BIT are defined. If there are no issues
+// there and you are needing to add support for a new N-bit processor, please
+// search the source code for all occurances of CONFIG_ARCH_64BIT and
+// CONFIG_ARCH_32BIT to check whether further modification is necessary.
+// These places will not necessarily #error for unsupported N-bit computing.
+#endif
+
+// OS detection is also inferred from the toolchain.
+#if defined(__APPLE__)
+#define OS_MACOSX 1
+#elif defined(__linux__)
+#define OS_LINUX 1
+#elif defined(__FreeBSD__)
+#define OS_FREEBSD 1
+#endif
+
+#if defined(OS_MACOSX) || defined(OS_LINUX) || defined(OS_FREEBSD)
+#define OS_POSIX 1
+#endif
+
+#define MAX_THREADS 32
+
+//Use LL/SC atomic primitives instead of __atomic_compare_exchange built-ins
+//This seems to be the most performant option on ARM but may violate
+//recommendations by the ARM architecture (e.g. no memory accesses between
+//LL and SC)
+//USE_LLSC overrides the use of __atomic_compare_exchange
+#ifdef __ARM_ARCH
+#define USE_LLSC
+#endif
+
+//Use barrier + relaxed store (DMB;STR) instead of store-release (STRL)
+//This is more performant on Cortex-A57 and possibly also on Cortex-A53
+#if defined(__aarch64__)
+#define USE_DMB
+#endif
+
+#if defined(USE_DMB) && defined(__arm__)
+#error USE_DMB optimization only applies to select ARMv8 processors
+#endif
+
+//Use ARM wait-for-event mechanism when busy polling
+//This will minimise interconnect transactions and often increase system-wide
+//performance
+#if defined __ARM_ARCH
+#define USE_WFE
+#if defined(__arm__)
+//TODO: WFE on ARMv7
+#undef USE_WFE
+#endif
+#endif

ext/sms/base/build_config.h

@@ -0,0 +1,28 @@
+// Copyright (c) 2017 ARM Limited. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+#ifndef CACHE_LINE
+// Default CPU cache line size
+#define CACHE_LINE 128
+#endif
+
+static inline void doze(void)
+{
+#if defined(__ARM_ARCH)
+    // YIELD hints the CPU to switch to another thread if available
+    // but otherwise executes as a NOP
+    // ISB flushes the pipeline, then restarts. This is guaranteed to stall
+    // the CPU a number of cycles
+    __asm__ volatile("isb" : : : "memory");
+#elif defined(__x86_64__)
+    __asm__ volatile("pause" : : : "memory");
+#else
+#error Please add support for your CPU in cpu.h
+#endif
+}
+
+int num_cpus(void);
+
+unsigned long cpu_hz(void);