optimized sbgemm kernel for neoverse-v1 (sve-256)

Signed-off-by: Ye Tao <ye.tao@arm.com>

Author: taoye9

CONTRIBUTORS.md

@@ -240,3 +240,5 @@ In chronological order:
 * Marek Michalowski <https://github.com/michalowski-arm>
   * [2025-01-21] Add thread throttling profile for SGEMV on `NEOVERSEV1`
 
+* Ye Tao <ye.tao@arm.com>
+  * [2025-02-03] Optimize SBGEMM kernel on NEOVERSEV1

Makefile.arm64

@@ -101,7 +101,7 @@ ifeq ($(CORE), NEOVERSEV1)
 ifeq (1, $(filter 1,$(GCCVERSIONGTEQ7) $(ISCLANG)))
 ifeq (1, $(filter 1,$(GCCVERSIONGTEQ10) $(ISCLANG)))
 ifeq (1, $(filter 1,$(GCCMINORVERSIONGTEQ4) $(GCCVERSIONGTEQ11) $(ISCLANG)))
-CCOMMON_OPT += -march=armv8.4-a+sve
+CCOMMON_OPT += -march=armv8.4-a+sve+bf16
 ifeq (1, $(ISCLANG))
 CCOMMON_OPT += -mtune=cortex-x1
 else
@@ -111,7 +111,7 @@ ifneq ($(F_COMPILER), NAG)
 FCOMMON_OPT += -march=armv8.4-a -mtune=neoverse-v1
 endif
 else
-CCOMMON_OPT += -march=armv8.4-a+sve 
+CCOMMON_OPT += -march=armv8.4-a+sve+bf16 
 ifneq ($(CROSS), 1)
 CCOMMON_OPT += -mtune=native
 endif

cmake/system.cmake

@@ -291,10 +291,10 @@ if (DEFINED TARGET)
 
 if (${TARGET} STREQUAL NEOVERSEV1)
     if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE)
-	set (KERNEL_DEFINITIONS  "${KERNEL_DEFINITIONS} -Msve_intrinsics -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1")
+	set (KERNEL_DEFINITIONS  "${KERNEL_DEFINITIONS} -Msve_intrinsics -march=armv8.4-a+sve -mtune=neoverse-v1")
     else ()
     if (CMAKE_C_COMPILER_VERSION VERSION_GREATER 10.4 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 10.4)
-      set (KERNEL_DEFINITIONS  "${KERNEL_DEFINITIONS} -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1")
+      set (KERNEL_DEFINITIONS  "${KERNEL_DEFINITIONS} -march=armv8.4-a+sve -mtune=neoverse-v1")
     else ()
 	    message(FATAL_ERROR "Compiler ${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_VERSION} does not support Neoverse V1.")
     endif()

kernel/arm64/KERNEL.NEOVERSEV1

@@ -1,14 +1,18 @@
 include $(KERNELDIR)/KERNEL.ARMV8SVE
 
-SGEMVTKERNEL = gemv_t_sve.c
-DGEMVTKERNEL = gemv_t_sve.c
+SGEMVTKERNEL = gemv_t_sve_v1x3.c
+DGEMVTKERNEL = gemv_t_sve_v1x3.c
+ifeq ($(BUILD_BFLOAT16), 1)
 SBGEMM_BETA    =  sbgemm_beta_neoversev1.c
 SBGEMMKERNEL    = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_neoversev1.c
+ifneq ($(SBGEMM_UNROLL_M), $(SBGEMM_UNROLL_N))
 SBGEMMINCOPY    = sbgemm_ncopy_$(SBGEMM_UNROLL_M)_neoversev1.c
 SBGEMMITCOPY    = sbgemm_tcopy_$(SBGEMM_UNROLL_M)_neoversev1.c
-SBGEMMONCOPY    = sbgemm_ncopy_$(SBGEMM_UNROLL_N)_neoversev1.c
-SBGEMMOTCOPY    = sbgemm_tcopy_$(SBGEMM_UNROLL_N)_neoversev1.c
 SBGEMMINCOPYOBJ =  sbgemm_incopy$(TSUFFIX).$(SUFFIX)
 SBGEMMITCOPYOBJ =  sbgemm_itcopy$(TSUFFIX).$(SUFFIX)
+endif
+SBGEMMONCOPY    = sbgemm_ncopy_$(SBGEMM_UNROLL_N)_neoversev1.c
+SBGEMMOTCOPY    = sbgemm_tcopy_$(SBGEMM_UNROLL_N)_neoversev1.c
 SBGEMMONCOPYOBJ =  sbgemm_oncopy$(TSUFFIX).$(SUFFIX)
 SBGEMMOTCOPYOBJ =  sbgemm_otcopy$(TSUFFIX).$(SUFFIX)
+endif

kernel/arm64/sbgemm_kernel_8x4_neoversev1.c renamed to kernel/arm64/sbgemm_kernel_4x4_neoversev1.c

@@ -1,5 +1,5 @@
 /***************************************************************************
- * Copyright (c) 2024, The OpenBLAS Project
+ * Copyright (c) 2024-2025, The OpenBLAS Project
  * All rights reserved.
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -31,9 +31,9 @@
 #include "common.h"
 
 #define ALPHA_ONE
-#include "sbgemm_kernel_8x4_neoversev1_impl.c"
+#include "sbgemm_kernel_4x4_neoversev1_impl.c"
 #undef ALPHA_ONE
-#include "sbgemm_kernel_8x4_neoversev1_impl.c"
+#include "sbgemm_kernel_4x4_neoversev1_impl.c"
 
 int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT *A, IFLOAT *B,
           FLOAT *C, BLASLONG ldc) {