Merge pull request #5419 from Mousius/bgemm-optimisation

Optimize SBGEMM / BGEMM for NEOVERSEV1 further

Author: martin-frbg

kernel/arm64/KERNEL.NEOVERSEV1

@@ -34,31 +34,31 @@ SGEMVTKERNEL = gemv_t_sve_v1x3.c
 DGEMVTKERNEL = gemv_t_sve_v1x3.c
 ifeq ($(BUILD_BFLOAT16), 1)
 BGEMM_BETA    =  bgemm_beta_neon.c
-BGEMMKERNEL    = bgemm_kernel_$(BGEMM_UNROLL_M)x$(BGEMM_UNROLL_N)_neoversev1.c
+BGEMMKERNEL    = bgemm_kernel_2vlx4_neoversev1.c
 ifneq ($(BGEMM_UNROLL_M), $(BGEMM_UNROLL_N))
-BGEMMINCOPY    = sbgemm_ncopy_$(SBGEMM_UNROLL_M)_neoversev1.c
-BGEMMITCOPY    = sbgemm_tcopy_$(SBGEMM_UNROLL_M)_neoversev1.c
+BGEMMINCOPY    = bgemm_ncopy_2vl_neoversev1.c
+BGEMMITCOPY    = bgemm_tcopy_2vl_neoversev1.c
 BGEMMINCOPYOBJ =  bgemm_incopy$(TSUFFIX).$(SUFFIX)
 BGEMMITCOPYOBJ =  bgemm_itcopy$(TSUFFIX).$(SUFFIX)
 endif
-BGEMMONCOPY    = sbgemm_ncopy_$(BGEMM_UNROLL_N)_neoversev1.c
-BGEMMOTCOPY    = sbgemm_tcopy_$(BGEMM_UNROLL_N)_neoversev1.c
+BGEMMONCOPY    = bgemm_ncopy_4_neoversev1.c
+BGEMMOTCOPY    = bgemm_tcopy_4_neoversev1.c
 BGEMMONCOPYOBJ =  bgemm_oncopy$(TSUFFIX).$(SUFFIX)
 BGEMMOTCOPYOBJ =  bgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
 BGEMVTKERNEL = sbgemv_t_bfdot.c
 BGEMVNKERNEL = bgemv_n_sve_v3x4.c
 
 SBGEMM_BETA    =  sbgemm_beta_neoversev1.c
-SBGEMMKERNEL    = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_neoversev1.c
+SBGEMMKERNEL    = bgemm_kernel_2vlx4_neoversev1.c
 ifneq ($(SBGEMM_UNROLL_M), $(SBGEMM_UNROLL_N))
-SBGEMMINCOPY    = sbgemm_ncopy_$(SBGEMM_UNROLL_M)_neoversev1.c
-SBGEMMITCOPY    = sbgemm_tcopy_$(SBGEMM_UNROLL_M)_neoversev1.c
+SBGEMMINCOPY    = bgemm_ncopy_2vl_neoversev1.c
+SBGEMMITCOPY    = bgemm_tcopy_2vl_neoversev1.c
 SBGEMMINCOPYOBJ =  sbgemm_incopy$(TSUFFIX).$(SUFFIX)
 SBGEMMITCOPYOBJ =  sbgemm_itcopy$(TSUFFIX).$(SUFFIX)
 endif
-SBGEMMONCOPY    = sbgemm_ncopy_$(SBGEMM_UNROLL_N)_neoversev1.c
-SBGEMMOTCOPY    = sbgemm_tcopy_$(SBGEMM_UNROLL_N)_neoversev1.c
+SBGEMMONCOPY    = bgemm_ncopy_4_neoversev1.c
+SBGEMMOTCOPY    = bgemm_tcopy_4_neoversev1.c
 SBGEMMONCOPYOBJ =  sbgemm_oncopy$(TSUFFIX).$(SUFFIX)
 SBGEMMOTCOPYOBJ =  sbgemm_otcopy$(TSUFFIX).$(SUFFIX)
 

kernel/arm64/bgemm_kernel_4x4_neoversev1.c renamed to kernel/arm64/bgemm_kernel_2vlx4_neoversev1.c

@@ -32,20 +32,26 @@
 #include "common.h"
 
 #define ALPHA_ONE
-#include "bgemm_kernel_4x4_neoversev1_impl.c"
+#include "bgemm_kernel_2vlx4_neoversev1_impl.c"
 #undef ALPHA_ONE
 #undef UPDATE_C
-#include "bgemm_kernel_4x4_neoversev1_impl.c"
+#undef UPDATE_C2
+#undef UPDATE_C1
+#include "bgemm_kernel_2vlx4_neoversev1_impl.c"
 
 int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT *A, IFLOAT *B,
           FLOAT *C, BLASLONG ldc) {
+#ifdef BGEMM
   bfloat16_t alpha_bf16;
   memcpy(&alpha_bf16, &alpha, sizeof(bfloat16_t));
   float alpha_f32 = vcvtah_f32_bf16(alpha_bf16);
+#else
+  float alpha_f32 = alpha;
+#endif
 
   if (alpha_f32 == 1.0f)
-    return bgemm_kernel_neoversev1_alpha_one(m, n, k, alpha, A, B, C, ldc);
+    return bgemm_kernel_neoversev1_alpha_one(m, n, k, alpha_f32, A, B, C, ldc);
   else
-    return bgemm_kernel_neoversev1_alpha(m, n, k, alpha, A, B, C, ldc);
+    return bgemm_kernel_neoversev1_alpha(m, n, k, alpha_f32, A, B, C, ldc);
   return 0;
 }