Merge pull request #5399 from Mousius/bgemm-8x4

Add optimized BGEMM for NEOVERSEN2 target

Author: martin-frbg

kernel/arm64/KERNEL.NEOVERSEN2

@@ -188,6 +188,20 @@ ZGEMMOTCOPY    =  ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_N).c
 ZGEMMONCOPYOBJ =  zgemm_oncopy$(TSUFFIX).$(SUFFIX)
 ZGEMMOTCOPYOBJ =  zgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
+ifeq ($(BUILD_BFLOAT16), 1)
+BGEMM_BETA    =  sbgemm_beta_neoversen2.c
+BGEMMKERNEL    = sbgemm_kernel_$(BGEMM_UNROLL_M)x$(BGEMM_UNROLL_N)_neoversen2.c
+BGEMMINCOPY    = sbgemm_ncopy_$(BGEMM_UNROLL_M)_neoversen2.c
+BGEMMITCOPY    = sbgemm_tcopy_$(BGEMM_UNROLL_M)_neoversen2.c
+BGEMMONCOPY    = sbgemm_ncopy_$(BGEMM_UNROLL_N)_neoversen2.c
+BGEMMOTCOPY    = sbgemm_tcopy_$(BGEMM_UNROLL_N)_neoversen2.c
+BGEMMINCOPYOBJ =  bgemm_incopy$(TSUFFIX).$(SUFFIX)
+BGEMMITCOPYOBJ =  bgemm_itcopy$(TSUFFIX).$(SUFFIX)
+BGEMMONCOPYOBJ =  bgemm_oncopy$(TSUFFIX).$(SUFFIX)
+BGEMMOTCOPYOBJ =  bgemm_otcopy$(TSUFFIX).$(SUFFIX)
+BGEMVTKERNEL = sbgemv_t_bfdot.c
+BGEMVNKERNEL = bgemv_n_sve_v3x4.c
+
 SBGEMM_BETA    =  sbgemm_beta_neoversen2.c
 SBGEMMKERNEL    = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_neoversen2.c
 SBGEMMINCOPY    = sbgemm_ncopy_$(SBGEMM_UNROLL_M)_neoversen2.c
@@ -199,4 +213,5 @@ SBGEMMITCOPYOBJ =  sbgemm_itcopy$(TSUFFIX).$(SUFFIX)
 SBGEMMONCOPYOBJ =  sbgemm_oncopy$(TSUFFIX).$(SUFFIX)
 SBGEMMOTCOPYOBJ =  sbgemm_otcopy$(TSUFFIX).$(SUFFIX)
 SBGEMVTKERNEL = sbgemv_t_bfdot.c
-SBGEMVNKERNEL = sbgemv_n_neon.c
+SBGEMVNKERNEL = sbgemv_n_neon.c
+endif

kernel/arm64/KERNEL.NEOVERSEV2

@@ -1,6 +1 @@
-include $(KERNELDIR)/KERNEL.ARMV8SVE
-
-ifeq ($(BUILD_BFLOAT16), 1)
-SBGEMVTKERNEL = sbgemv_t_bfdot.c
-SBGEMVNKERNEL = sbgemv_n_neon.c
-endif
+include $(KERNELDIR)/KERNEL.NEOVERSEN2

kernel/arm64/sbgemm_kernel_8x4_neoversen2.c

@@ -1,5 +1,5 @@
 /***************************************************************************
- * Copyright (c) 2022, The OpenBLAS Project
+ * Copyright (c) 2022,2025 The OpenBLAS Project
  * All rights reserved.
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -33,13 +33,23 @@
 #define ALPHA_ONE
 #include "sbgemm_kernel_8x4_neoversen2_impl.c"
 #undef ALPHA_ONE
+#undef UPDATE_C
 #include "sbgemm_kernel_8x4_neoversen2_impl.c"
 
 int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT *A, IFLOAT *B,
           FLOAT *C, BLASLONG ldc) {
-  if (alpha == 1.0f)
-    return sbgemm_kernel_neoversen2_alpha_one(m, n, k, alpha, A, B, C, ldc);
+#ifdef BGEMM
+  bfloat16_t alpha_bf16;
+  memcpy(&alpha_bf16, &alpha, sizeof(bfloat16_t));
+  float alpha_f32 = vcvtah_f32_bf16(alpha_bf16);
+#else
+  float alpha_f32 = alpha;
+#endif
+
+  if (alpha_f32 == 1.0f)
+    return gemm_kernel_neoversen2_alpha_one(m, n, k, alpha, A, B, C, ldc);
   else
-    return sbgemm_kernel_neoversen2_alpha(m, n, k, alpha, A, B, C, ldc);
+    return gemm_kernel_neoversen2_alpha(m, n, k, alpha, A, B, C, ldc);
+
   return 0;
 }