Merge pull request #6 from ARM-software/master

Sync with upstream CLH spinlock

Author: zoybai

benchmarks/lockhammer/Makefile

@@ -1,4 +1,5 @@
-override CFLAGS += -g -O3 -I. -I./include -I../../ext/mysql/include -I../../ext/linux/include -I../../ext/tbb/include
+# override keyword overwrites make command-line option LSE_ENABLE=y, therefore it has been removed
+CFLAGS += -g -O3 -I. -I./include -I../../ext/mysql/include -I../../ext/linux/include -I../../ext/tbb/include -I../../ext/sms/base
 
 ifneq ($(DEBUG_LEVEL),)
 ifeq ($(shell test $(DEBUG_LEVEL) -gt 0; echo $$?),0)
@@ -32,7 +33,8 @@ TEST_TARGETS=lh_swap_mutex \
 	lh_empty \
 	lh_jvm_objectmonitor \
 	lh_tbb_spin_rw_mutex \
-	lh_osq_lock
+	lh_osq_lock \
+	lh_clh_spinlock
 
 ifeq ($(TARGET_ARCH),aarch64)
 	TEST_TARGETS+=lh_hybrid_spinlock \
@@ -59,6 +61,9 @@ lh_hybrid_spinlock_fastdequeue: ../../ext/linux/hybrid_spinlock_fastdequeue.h in
 lh_osq_lock: ../../ext/linux/osq_lock.h ../../ext/linux/include/lk_atomics.h ../../ext/linux/include/lk_barrier.h ../../ext/linux/include/lk_cmpxchg.h include/atomics.h src/lockhammer.c
 	${CC} ${CFLAGS} -DATOMIC_TEST=\"$<\" src/lockhammer.c -o build/$@ ${LDFLAGS}
 
+lh_clh_spinlock: ../../ext/sms/clh_spinlock.h ../../ext/sms/base/build_config.h ../../ext/sms/base/cpu.h ../../ext/sms/base/llsc.h src/lockhammer.c
+	${CC} ${CFLAGS} -DATOMIC_TEST=\"$<\" src/lockhammer.c -o build/$@ ${LDFLAGS}
+
 lh_queued_spinlock: ../../ext/linux/queued_spinlock.h include/atomics.h ../../ext/linux/include/lk_atomics.h src/lockhammer.c
 	${CC} ${CFLAGS} -DATOMIC_TEST=\"$<\" src/lockhammer.c -o build/$@ ${LDFLAGS}
 

benchmarks/lockhammer/include/atomics.h

@@ -34,12 +34,6 @@
 #ifndef __LH_ATOMICS_H_
 #define __LH_ATOMICS_H_
 
-#ifndef initialize_lock
-	#define initialize_lock(lock, thread)
-#endif
-#ifndef parse_test_args
-	#define parse_test_args(args, argc, argv)
-#endif
 
 static inline void spin_wait (unsigned long wait_iter) {
 #if defined(__aarch64__)

benchmarks/lockhammer/include/lockhammer.h

@@ -32,6 +32,17 @@
 #ifndef __LOCKHAMMER_H__
 #define __LOCKHAMMER_H__
 
+
+#ifndef initialize_lock
+    #define initialize_lock(lock, thread)
+#endif
+#ifndef parse_test_args
+    #define parse_test_args(args, argc, argv)
+#endif
+#ifndef thread_local_init
+    #define thread_local_init(smtid)
+#endif
+
 enum units { NS,
              INSTS };
 typedef enum units Units;

benchmarks/lockhammer/scripts/lh_sweeptest_cfg.yaml

@@ -66,6 +66,7 @@ sweeptest:
         - lh_swap_mutex
         - lh_tbb_spin_rw_mutex
         - lh_ticket_spinlock
+        - lh_clh_spinlock
     cmd_aarch64: [lh_hybrid_spinlock, lh_hybrid_spinlock_fastdequeue]
     cmd_x86_64:
     repeat: 9
@@ -78,6 +79,10 @@ sweeptest:
           c: 0ns
           p: 0ns
           o: lstopo
+        - a: 5000
+          c: 200ns
+          p: 0ns
+          o: lstopo
         - a: 5000
           c: 1000ns
           p: 0ns

benchmarks/lockhammer/scripts/runall_obsolete.sh

@@ -448,6 +448,8 @@ void* hmr(void *ptr)
         synchronize_threads(&calibrate_lock, nthrds);
     }
 
+    thread_local_init(mycore);
+
 #ifdef DDEBUG
     printf("%ld %ld\n", hold_count, post_count);
 #endif

benchmarks/lockhammer/src/lockhammer.c

@@ -0,0 +1,75 @@
+// Copyright (c) 2017 ARM Limited. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+// Architecture detection is inferred from the toolchain. This relies on
+// the C compiler's system-specific macros.
+#if defined(__aarch64__)
+#define CONFIG_ARCH_ARM_V8
+#define CONFIG_ARCH_64BIT
+#elif defined(__arm__)
+#define CONFIG_ARCH_ARM_V7
+#define CONFIG_ARCH_32BIT
+#elif defined(__x86_64__)
+#define CONFIG_ARCH_X86_64
+#define CONFIG_ARCH_64BIT
+#elif defined(__i386__)
+#define CONFIG_ARCH_X86
+#define CONFIG_ARCH_32BIT
+#endif
+
+#if !defined(CONFIG_ARCH_64BIT) && !defined(CONFIG_ARCH_32BIT)
+#error Please add support for N-bit computing to build_config.h
+// If you experience this C pre-processor error, take a look at the place
+// in this file where CONFIG_ARCH_64/32BIT are defined. If there are no issues
+// there and you are needing to add support for a new N-bit processor, please
+// search the source code for all occurances of CONFIG_ARCH_64BIT and
+// CONFIG_ARCH_32BIT to check whether further modification is necessary.
+// These places will not necessarily #error for unsupported N-bit computing.
+#endif
+
+// OS detection is also inferred from the toolchain.
+#if defined(__APPLE__)
+#define OS_MACOSX 1
+#elif defined(__linux__)
+#define OS_LINUX 1
+#elif defined(__FreeBSD__)
+#define OS_FREEBSD 1
+#endif
+
+#if defined(OS_MACOSX) || defined(OS_LINUX) || defined(OS_FREEBSD)
+#define OS_POSIX 1
+#endif
+
+#define MAX_THREADS 32
+
+//Use LL/SC atomic primitives instead of __atomic_compare_exchange built-ins
+//This seems to be the most performant option on ARM but may violate
+//recommendations by the ARM architecture (e.g. no memory accesses between
+//LL and SC)
+//USE_LLSC overrides the use of __atomic_compare_exchange
+#ifdef __ARM_ARCH
+#define USE_LLSC
+#endif
+
+//Use barrier + relaxed store (DMB;STR) instead of store-release (STRL)
+//This is more performant on Cortex-A57 and possibly also on Cortex-A53
+#if defined(__aarch64__)
+#define USE_DMB
+#endif
+
+#if defined(USE_DMB) && defined(__arm__)
+#error USE_DMB optimization only applies to select ARMv8 processors
+#endif
+
+//Use ARM wait-for-event mechanism when busy polling
+//This will minimise interconnect transactions and often increase system-wide
+//performance
+#if defined __ARM_ARCH
+#define USE_WFE
+#if defined(__arm__)
+//TODO: WFE on ARMv7
+#undef USE_WFE
+#endif
+#endif

ext/sms/base/build_config.h

@@ -0,0 +1,28 @@
+// Copyright (c) 2017 ARM Limited. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+#ifndef CACHE_LINE
+// Default CPU cache line size
+#define CACHE_LINE 128
+#endif
+
+static inline void doze(void)
+{
+#if defined(__ARM_ARCH)
+    // YIELD hints the CPU to switch to another thread if available
+    // but otherwise executes as a NOP
+    // ISB flushes the pipeline, then restarts. This is guaranteed to stall
+    // the CPU a number of cycles
+    __asm__ volatile("isb" : : : "memory");
+#elif defined(__x86_64__)
+    __asm__ volatile("pause" : : : "memory");
+#else
+#error Please add support for your CPU in cpu.h
+#endif
+}
+
+int num_cpus(void);
+
+unsigned long cpu_hz(void);