|
| 1 | +/*************************************************************************** |
| 2 | + * Copyright (c) 2025, The OpenBLAS Project |
| 3 | + * All rights reserved. |
| 4 | + * Redistribution and use in source and binary forms, with or without |
| 5 | + * modification, are permitted provided that the following conditions are |
| 6 | + * met: |
| 7 | + * 1. Redistributions of source code must retain the above copyright |
| 8 | + * notice, this list of conditions and the following disclaimer. |
| 9 | + * 2. Redistributions in binary form must reproduce the above copyright |
| 10 | + * notice, this list of conditions and the following disclaimer in |
| 11 | + * the documentation and/or other materials provided with the |
| 12 | + * distribution. |
| 13 | + * 3. Neither the name of the OpenBLAS project nor the names of |
| 14 | + * its contributors may be used to endorse or promote products |
| 15 | + * derived from this software without specific prior written permission. |
| 16 | + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| 17 | + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 18 | + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| 19 | + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE |
| 20 | + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 21 | + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 22 | + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 23 | + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| 24 | + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 25 | + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| 26 | + * POSSIBILITY OF SUCH DAMAGE. |
| 27 | + * *****************************************************************************/ |
| 28 | + |
| 29 | +#include "common.h" |
| 30 | + |
| 31 | +#include <arm_neon.h> |
| 32 | + |
| 33 | +int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta_in, IFLOAT *dummy2, |
| 34 | + BLASLONG dummy3, IFLOAT *dummy4, BLASLONG dummy5, FLOAT *c, |
| 35 | + BLASLONG ldc) { |
| 36 | + BLASLONG i, j; |
| 37 | + BLASLONG chunk, remain; |
| 38 | + |
| 39 | + bfloat16_t *ptr_c, *ptr_c0; |
| 40 | + |
| 41 | + bfloat16x8_t x0, z0; |
| 42 | + float32x4_t y0, y1; |
| 43 | + |
| 44 | + float x; |
| 45 | + bfloat16_t z; |
| 46 | + |
| 47 | + bfloat16_t zero_bf16 = vcvth_bf16_f32(0.0f); |
| 48 | + bfloat16x8_t zeros = vdupq_n_bf16(zero_bf16); |
| 49 | + |
| 50 | + bfloat16_t beta_bf16; |
| 51 | + memcpy(&beta_bf16, &beta_in, sizeof(bfloat16_t)); |
| 52 | + float beta = vcvtah_f32_bf16(beta_bf16); |
| 53 | + float32x4_t beta_neon = vdupq_n_f32(beta); |
| 54 | + |
| 55 | + ptr_c = (bfloat16_t *)c; |
| 56 | + |
| 57 | + chunk = m >> 3; |
| 58 | + remain = m & 7; |
| 59 | + |
| 60 | + if (beta == 0.0f){ |
| 61 | + for (j = 0; j < n; j ++){ |
| 62 | + ptr_c0 = ptr_c; |
| 63 | + ptr_c += ldc; |
| 64 | + |
| 65 | + for (i = 0; i < chunk; i ++){ |
| 66 | + vst1q_bf16(ptr_c0, zeros); |
| 67 | + ptr_c0 += 8; |
| 68 | + } |
| 69 | + |
| 70 | + for (i = 0; i < remain; i ++){ |
| 71 | + ptr_c0[0] = zero_bf16; |
| 72 | + ptr_c0 ++; |
| 73 | + } |
| 74 | + } |
| 75 | + } else { |
| 76 | + for (j = 0; j < n; j ++){ |
| 77 | + ptr_c0 = ptr_c; |
| 78 | + ptr_c += ldc; |
| 79 | + |
| 80 | + for (i = 0; i < chunk; i ++){ |
| 81 | + x0 = vld1q_bf16(ptr_c0); |
| 82 | + |
| 83 | + y0 = vcvtq_low_f32_bf16(x0); |
| 84 | + y1 = vcvtq_high_f32_bf16(x0); |
| 85 | + |
| 86 | + y0 = vmulq_f32(y0, beta_neon); |
| 87 | + y1 = vmulq_f32(y1, beta_neon); |
| 88 | + |
| 89 | + z0 = vcvtq_low_bf16_f32(y0); |
| 90 | + z0 = vcvtq_high_bf16_f32(z0, y1); |
| 91 | + |
| 92 | + vst1q_bf16(ptr_c0, z0); |
| 93 | + |
| 94 | + ptr_c0 += 8; |
| 95 | + } |
| 96 | + |
| 97 | + for (i = 0; i < remain; i ++){ |
| 98 | + x = vcvtah_f32_bf16(ptr_c0[0]); |
| 99 | + z = vcvth_bf16_f32(x * beta); |
| 100 | + |
| 101 | + ptr_c0[0] = z; |
| 102 | + ptr_c0 ++; |
| 103 | + } |
| 104 | + } |
| 105 | + } |
| 106 | + return 0; |
| 107 | +}; |
0 commit comments