Skip to content

Commit 2e8543c

Browse files
Sp00phSp00ph
authored
[X86] Improve variable 8-bit shifts on AVX512BW (#164136)
Previously, `clang -march=znver5 -O3` would emit the following for `shl`, `lshr` and `ashr <64 x i8>`: ```asm .LCPI0_2: .byte 8 .byte 4 .byte 2 .byte 1 .byte 0 .byte 0 .byte 0 .byte 0 .LCPI0_3: .byte 32 .byte 16 .byte 8 .byte 4 .byte 2 .byte 1 .byte 0 .byte 0 shl: vpsllw zmm1, zmm1, 5 vpmovb2m k1, zmm1 vpaddb zmm1, zmm1, zmm1 vgf2p8affineqb zmm0 {k1}, zmm0, qword ptr [rip + .LCPI0_2]{1to8}, 0 vpmovb2m k1, zmm1 vpaddb zmm1, zmm1, zmm1 vgf2p8affineqb zmm0 {k1}, zmm0, qword ptr [rip + .LCPI0_3]{1to8}, 0 vpmovb2m k1, zmm1 vpaddb zmm0 {k1}, zmm0, zmm0 ret .LCPI1_3: .byte 0 .byte 0 .byte 0 .byte 0 .byte 128 .byte 64 .byte 32 .byte 16 .LCPI1_4: .byte 0 .byte 0 .byte 128 .byte 64 .byte 32 .byte 16 .byte 8 .byte 4 .LCPI1_5: .byte 0 .byte 128 .byte 64 .byte 32 .byte 16 .byte 8 .byte 4 .byte 2 lshr: vpsllw zmm1, zmm1, 5 vpmovb2m k1, zmm1 vpaddb zmm1, zmm1, zmm1 vgf2p8affineqb zmm0 {k1}, zmm0, qword ptr [rip + .LCPI1_3]{1to8}, 0 vpmovb2m k1, zmm1 vpaddb zmm1, zmm1, zmm1 vgf2p8affineqb zmm0 {k1}, zmm0, qword ptr [rip + .LCPI1_4]{1to8}, 0 vpmovb2m k1, zmm1 vgf2p8affineqb zmm0 {k1}, zmm0, qword ptr [rip + .LCPI1_5]{1to8}, 0 ret ashr: vpsllw zmm1, zmm1, 5 vpunpckhbw zmm2, zmm0, zmm0 vpunpckhbw zmm4, zmm1, zmm1 vpsraw zmm3, zmm2, 4 vpunpcklbw zmm0, zmm0, zmm0 vpmovb2m k1, zmm4 vpaddw zmm4, zmm4, zmm4 vpunpcklbw zmm1, zmm1, zmm1 vmovdqu8 zmm2 {k1}, zmm3 vpmovb2m k1, zmm4 vpsraw zmm3, zmm2, 2 vpaddw zmm4, zmm4, zmm4 vmovdqu8 zmm2 {k1}, zmm3 vpsraw zmm3, zmm2, 1 vpmovb2m k1, zmm4 vmovdqu8 zmm2 {k1}, zmm3 vpmovb2m k1, zmm1 vpsraw zmm3, zmm0, 4 vpaddw zmm1, zmm1, zmm1 vpsrlw zmm2, zmm2, 8 vmovdqu8 zmm0 {k1}, zmm3 vpmovb2m k1, zmm1 vpsraw zmm3, zmm0, 2 vpaddw zmm1, zmm1, zmm1 vmovdqu8 zmm0 {k1}, zmm3 vpsraw zmm3, zmm0, 1 vpmovb2m k1, zmm1 vmovdqu8 zmm0 {k1}, zmm3 vpsrlw zmm0, zmm0, 8 vpackuswb zmm0, zmm0, zmm2 ret ``` With this commit, the generated assembly becomes this: ```asm .LCPI0_2: .byte 0 .byte 255 .byte 0 .byte 255 .LCPI0_3: .byte 255 .byte 0 .byte 255 .byte 0 shl: vpsrlw zmm2, zmm1, 8 vpandd zmm3, zmm0, dword ptr [rip + .LCPI0_2]{1to16} vpandd zmm1, zmm1, dword ptr [rip + .LCPI0_3]{1to16} movabs rax, -6148914691236517206 kmovq k1, rax vpsllvw zmm2, zmm3, zmm2 vpsllvw zmm0, zmm0, zmm1 vmovdqu8 zmm0 {k1}, zmm2 ret .LCPI1_0: .byte 255 .byte 0 lshr: vpbroadcastw zmm2, word ptr [rip + .LCPI1_0] movabs rax, -6148914691236517206 kmovq k1, rax vpandq zmm3, zmm1, zmm2 vpandq zmm2, zmm0, zmm2 vpsrlw zmm1, zmm1, 8 vpsrlvw zmm2, zmm2, zmm3 vpsrlvw zmm0, zmm0, zmm1 vmovdqu8 zmm2 {k1}, zmm0 vmovdqa64 zmm0, zmm2 ret .LCPI2_1: .byte 255 .byte 0 .byte 255 .byte 0 ashr: vpsrlw zmm2, zmm1, 8 vpandd zmm1, zmm1, dword ptr [rip + .LCPI2_1]{1to16} movabs rax, -6148914691236517206 vpsravw zmm2, zmm0, zmm2 vpsllw zmm0, zmm0, 8 kmovq k1, rax vpsraw zmm0, zmm0, 8 vpsravw zmm0, zmm0, zmm1 vmovdqu8 zmm0 {k1}, zmm2 ret ``` While I don't have AVX512 hardware, llvm-mca suggests significant speedups, and I've done some simple correctness tests on random inputs using the Intel Software Development Emulator.
1 parent 831e79a commit 2e8543c

File tree

5 files changed

+109
-104
lines changed

5 files changed

+109
-104
lines changed

‎llvm/lib/Target/X86/X86ISelLowering.cpp‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30908,6 +30908,63 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
3090830908
return DAG.getNode(X86ISD::PACKUS, dl, VT, LoR, HiR);
3090930909
}
3091030910

30911+
if (VT == MVT::v64i8 && Subtarget.canExtendTo512BW()) {
30912+
// On AVX512BW, we can use variable 16-bit shifts to implement variable
30913+
// 8-bit shifts. For this, we split the input into two vectors, RLo and RHi.
30914+
// The i-th lane of RLo contains the (2*i)-th lane of R, and the i-th lane
30915+
// of RHi contains the (2*i+1)-th lane of R. After shifting, these vectors
30916+
// can efficiently be merged together using a masked move.
30917+
MVT ExtVT = MVT::v32i16;
30918+
30919+
SDValue RLo, RHi;
30920+
// Isolate lower and upper lanes of Amt by masking odd lanes in AmtLo and
30921+
// right shifting AmtHi.
30922+
SDValue AmtLo = DAG.getNode(ISD::AND, dl, ExtVT, DAG.getBitcast(ExtVT, Amt),
30923+
DAG.getConstant(0x00ff, dl, ExtVT));
30924+
SDValue AmtHi = getTargetVShiftByConstNode(
30925+
X86ISD::VSRLI, dl, ExtVT, DAG.getBitcast(ExtVT, Amt), 8, DAG);
30926+
switch (Opc) {
30927+
case ISD::SHL:
30928+
// Because we shift left, no bits from the high half can influence the low
30929+
// half, so we don't need to mask RLo. We do however need to mask RHi, to
30930+
// prevent high bits of an even lane overflowing into low bits of an odd
30931+
// lane.
30932+
RLo = DAG.getBitcast(ExtVT, R);
30933+
RHi = DAG.getNode(ISD::AND, dl, ExtVT, RLo,
30934+
DAG.getConstant(0xff00, dl, ExtVT));
30935+
break;
30936+
case ISD::SRL:
30937+
// Same idea as above, but this time we need to make sure no low bits of
30938+
// an odd lane can overflow into high bits of an even lane.
30939+
RHi = DAG.getBitcast(ExtVT, R);
30940+
RLo = DAG.getNode(ISD::AND, dl, ExtVT, RHi,
30941+
DAG.getConstant(0x00ff, dl, ExtVT));
30942+
break;
30943+
case ISD::SRA:
30944+
// For arithmetic right shifts, we want to sign extend each even lane of R
30945+
// such that the upper half of the corresponding lane of RLo is 0 or -1
30946+
// depending on the sign bit of the original lane. We do this using 2
30947+
// immediate shifts.
30948+
RHi = DAG.getBitcast(ExtVT, R);
30949+
RLo = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ExtVT, RHi, 8, DAG);
30950+
RLo = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExtVT, RLo, 8, DAG);
30951+
break;
30952+
default:
30953+
llvm_unreachable("Unexpected Shift Op");
30954+
}
30955+
30956+
SDValue ShiftedLo =
30957+
DAG.getBitcast(VT, DAG.getNode(Opc, dl, ExtVT, RLo, AmtLo));
30958+
SDValue ShiftedHi =
30959+
DAG.getBitcast(VT, DAG.getNode(Opc, dl, ExtVT, RHi, AmtHi));
30960+
30961+
// To merge the shifted vectors back together, we select even lanes
30962+
// from ShiftedLo and odd lanes from ShiftedHi.
30963+
SDValue SelectMask = DAG.getBitcast(
30964+
MVT::v64i1, DAG.getConstant(0x5555555555555555, dl, MVT::i64));
30965+
return DAG.getSelect(dl, VT, SelectMask, ShiftedLo, ShiftedHi);
30966+
}
30967+
3091130968
if (VT == MVT::v16i8 ||
3091230969
(VT == MVT::v32i8 && Subtarget.hasInt256() && !Subtarget.hasXOP()) ||
3091330970
(VT == MVT::v64i8 && Subtarget.hasBWI())) {

‎llvm/test/CodeGen/X86/gfni-shifts.ll‎

Lines changed: 27 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1684,15 +1684,14 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
16841684
;
16851685
; GFNIAVX512BW-LABEL: var_shl_v64i8:
16861686
; GFNIAVX512BW: # %bb.0:
1687-
; GFNIAVX512BW-NEXT: vpsllw $5, %zmm1, %zmm1
1688-
; GFNIAVX512BW-NEXT: vpmovb2m %zmm1, %k1
1689-
; GFNIAVX512BW-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 {%k1}
1690-
; GFNIAVX512BW-NEXT: vpaddb %zmm1, %zmm1, %zmm1
1691-
; GFNIAVX512BW-NEXT: vpmovb2m %zmm1, %k1
1692-
; GFNIAVX512BW-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 {%k1}
1693-
; GFNIAVX512BW-NEXT: vpaddb %zmm1, %zmm1, %zmm1
1694-
; GFNIAVX512BW-NEXT: vpmovb2m %zmm1, %k1
1695-
; GFNIAVX512BW-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1}
1687+
; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm2
1688+
; GFNIAVX512BW-NEXT: vpsllvw %zmm2, %zmm0, %zmm2
1689+
; GFNIAVX512BW-NEXT: vpsrlw $8, %zmm1, %zmm1
1690+
; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
1691+
; GFNIAVX512BW-NEXT: vpsllvw %zmm1, %zmm0, %zmm0
1692+
; GFNIAVX512BW-NEXT: movabsq $6148914691236517205, %rax # imm = 0x5555555555555555
1693+
; GFNIAVX512BW-NEXT: kmovq %rax, %k1
1694+
; GFNIAVX512BW-NEXT: vmovdqu8 %zmm2, %zmm0 {%k1}
16961695
; GFNIAVX512BW-NEXT: retq
16971696
%shift = shl <64 x i8> %a, %b
16981697
ret <64 x i8> %shift
@@ -1876,15 +1875,15 @@ define <64 x i8> @var_lshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
18761875
;
18771876
; GFNIAVX512BW-LABEL: var_lshr_v64i8:
18781877
; GFNIAVX512BW: # %bb.0:
1879-
; GFNIAVX512BW-NEXT: vpsllw $5, %zmm1, %zmm1
1880-
; GFNIAVX512BW-NEXT: vpmovb2m %zmm1, %k1
1881-
; GFNIAVX512BW-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 {%k1}
1882-
; GFNIAVX512BW-NEXT: vpaddb %zmm1, %zmm1, %zmm1
1883-
; GFNIAVX512BW-NEXT: vpmovb2m %zmm1, %k1
1884-
; GFNIAVX512BW-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 {%k1}
1885-
; GFNIAVX512BW-NEXT: vpaddb %zmm1, %zmm1, %zmm1
1886-
; GFNIAVX512BW-NEXT: vpmovb2m %zmm1, %k1
1887-
; GFNIAVX512BW-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 {%k1}
1878+
; GFNIAVX512BW-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
1879+
; GFNIAVX512BW-NEXT: vpandq %zmm2, %zmm1, %zmm3
1880+
; GFNIAVX512BW-NEXT: vpandq %zmm2, %zmm0, %zmm2
1881+
; GFNIAVX512BW-NEXT: vpsrlvw %zmm3, %zmm2, %zmm2
1882+
; GFNIAVX512BW-NEXT: vpsrlw $8, %zmm1, %zmm1
1883+
; GFNIAVX512BW-NEXT: vpsrlvw %zmm1, %zmm0, %zmm0
1884+
; GFNIAVX512BW-NEXT: movabsq $6148914691236517205, %rax # imm = 0x5555555555555555
1885+
; GFNIAVX512BW-NEXT: kmovq %rax, %k1
1886+
; GFNIAVX512BW-NEXT: vmovdqu8 %zmm2, %zmm0 {%k1}
18881887
; GFNIAVX512BW-NEXT: retq
18891888
%shift = lshr <64 x i8> %a, %b
18901889
ret <64 x i8> %shift
@@ -2232,36 +2231,16 @@ define <64 x i8> @var_ashr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
22322231
;
22332232
; GFNIAVX512BW-LABEL: var_ashr_v64i8:
22342233
; GFNIAVX512BW: # %bb.0:
2235-
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2236-
; GFNIAVX512BW-NEXT: vpsraw $4, %zmm2, %zmm3
2237-
; GFNIAVX512BW-NEXT: vpsllw $5, %zmm1, %zmm1
2238-
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
2239-
; GFNIAVX512BW-NEXT: vpmovb2m %zmm4, %k1
2240-
; GFNIAVX512BW-NEXT: vmovdqu8 %zmm3, %zmm2 {%k1}
2241-
; GFNIAVX512BW-NEXT: vpsraw $2, %zmm2, %zmm3
2242-
; GFNIAVX512BW-NEXT: vpaddw %zmm4, %zmm4, %zmm5
2243-
; GFNIAVX512BW-NEXT: vpmovb2m %zmm5, %k1
2244-
; GFNIAVX512BW-NEXT: vmovdqu8 %zmm3, %zmm2 {%k1}
2245-
; GFNIAVX512BW-NEXT: vpsraw $1, %zmm2, %zmm3
2246-
; GFNIAVX512BW-NEXT: vpsllw $2, %zmm4, %zmm4
2247-
; GFNIAVX512BW-NEXT: vpmovb2m %zmm4, %k1
2248-
; GFNIAVX512BW-NEXT: vmovdqu8 %zmm3, %zmm2 {%k1}
2249-
; GFNIAVX512BW-NEXT: vpsrlw $8, %zmm2, %zmm2
2250-
; GFNIAVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2251-
; GFNIAVX512BW-NEXT: vpsraw $4, %zmm0, %zmm3
2252-
; GFNIAVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
2253-
; GFNIAVX512BW-NEXT: vpmovb2m %zmm1, %k1
2254-
; GFNIAVX512BW-NEXT: vmovdqu8 %zmm3, %zmm0 {%k1}
2255-
; GFNIAVX512BW-NEXT: vpsraw $2, %zmm0, %zmm3
2256-
; GFNIAVX512BW-NEXT: vpaddw %zmm1, %zmm1, %zmm4
2257-
; GFNIAVX512BW-NEXT: vpmovb2m %zmm4, %k1
2258-
; GFNIAVX512BW-NEXT: vmovdqu8 %zmm3, %zmm0 {%k1}
2259-
; GFNIAVX512BW-NEXT: vpsraw $1, %zmm0, %zmm3
2260-
; GFNIAVX512BW-NEXT: vpsllw $2, %zmm1, %zmm1
2261-
; GFNIAVX512BW-NEXT: vpmovb2m %zmm1, %k1
2262-
; GFNIAVX512BW-NEXT: vmovdqu8 %zmm3, %zmm0 {%k1}
2263-
; GFNIAVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0
2264-
; GFNIAVX512BW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
2234+
; GFNIAVX512BW-NEXT: vpsrlw $8, %zmm1, %zmm2
2235+
; GFNIAVX512BW-NEXT: vpsravw %zmm2, %zmm0, %zmm2
2236+
; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
2237+
; GFNIAVX512BW-NEXT: vpsllw $8, %zmm0, %zmm0
2238+
; GFNIAVX512BW-NEXT: vpsraw $8, %zmm0, %zmm0
2239+
; GFNIAVX512BW-NEXT: vpsravw %zmm1, %zmm0, %zmm0
2240+
; GFNIAVX512BW-NEXT: movabsq $6148914691236517205, %rax # imm = 0x5555555555555555
2241+
; GFNIAVX512BW-NEXT: kmovq %rax, %k1
2242+
; GFNIAVX512BW-NEXT: vmovdqu8 %zmm0, %zmm2 {%k1}
2243+
; GFNIAVX512BW-NEXT: vmovdqa64 %zmm2, %zmm0
22652244
; GFNIAVX512BW-NEXT: retq
22662245
%shift = ashr <64 x i8> %a, %b
22672246
ret <64 x i8> %shift

‎llvm/test/CodeGen/X86/vector-shift-ashr-512.ll‎

Lines changed: 10 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -106,36 +106,16 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
106106
;
107107
; AVX512BW-LABEL: var_shift_v64i8:
108108
; AVX512BW: # %bb.0:
109-
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
110-
; AVX512BW-NEXT: vpsraw $4, %zmm2, %zmm3
111-
; AVX512BW-NEXT: vpsllw $5, %zmm1, %zmm1
112-
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
113-
; AVX512BW-NEXT: vpmovb2m %zmm4, %k1
114-
; AVX512BW-NEXT: vmovdqu8 %zmm3, %zmm2 {%k1}
115-
; AVX512BW-NEXT: vpsraw $2, %zmm2, %zmm3
116-
; AVX512BW-NEXT: vpaddw %zmm4, %zmm4, %zmm5
117-
; AVX512BW-NEXT: vpmovb2m %zmm5, %k1
118-
; AVX512BW-NEXT: vmovdqu8 %zmm3, %zmm2 {%k1}
119-
; AVX512BW-NEXT: vpsraw $1, %zmm2, %zmm3
120-
; AVX512BW-NEXT: vpsllw $2, %zmm4, %zmm4
121-
; AVX512BW-NEXT: vpmovb2m %zmm4, %k1
122-
; AVX512BW-NEXT: vmovdqu8 %zmm3, %zmm2 {%k1}
123-
; AVX512BW-NEXT: vpsrlw $8, %zmm2, %zmm2
124-
; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
125-
; AVX512BW-NEXT: vpsraw $4, %zmm0, %zmm3
126-
; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
127-
; AVX512BW-NEXT: vpmovb2m %zmm1, %k1
128-
; AVX512BW-NEXT: vmovdqu8 %zmm3, %zmm0 {%k1}
129-
; AVX512BW-NEXT: vpsraw $2, %zmm0, %zmm3
130-
; AVX512BW-NEXT: vpaddw %zmm1, %zmm1, %zmm4
131-
; AVX512BW-NEXT: vpmovb2m %zmm4, %k1
132-
; AVX512BW-NEXT: vmovdqu8 %zmm3, %zmm0 {%k1}
133-
; AVX512BW-NEXT: vpsraw $1, %zmm0, %zmm3
134-
; AVX512BW-NEXT: vpsllw $2, %zmm1, %zmm1
135-
; AVX512BW-NEXT: vpmovb2m %zmm1, %k1
136-
; AVX512BW-NEXT: vmovdqu8 %zmm3, %zmm0 {%k1}
137-
; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0
138-
; AVX512BW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
109+
; AVX512BW-NEXT: vpsrlw $8, %zmm1, %zmm2
110+
; AVX512BW-NEXT: vpsravw %zmm2, %zmm0, %zmm2
111+
; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
112+
; AVX512BW-NEXT: vpsllw $8, %zmm0, %zmm0
113+
; AVX512BW-NEXT: vpsraw $8, %zmm0, %zmm0
114+
; AVX512BW-NEXT: vpsravw %zmm1, %zmm0, %zmm0
115+
; AVX512BW-NEXT: movabsq $6148914691236517205, %rax # imm = 0x5555555555555555
116+
; AVX512BW-NEXT: kmovq %rax, %k1
117+
; AVX512BW-NEXT: vmovdqu8 %zmm0, %zmm2 {%k1}
118+
; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm0
139119
; AVX512BW-NEXT: retq
140120
%shift = ashr <64 x i8> %a, %b
141121
ret <64 x i8> %shift

‎llvm/test/CodeGen/X86/vector-shift-lshr-512.ll‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -85,20 +85,14 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
8585
;
8686
; AVX512BW-LABEL: var_shift_v64i8:
8787
; AVX512BW: # %bb.0:
88-
; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm2
89-
; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
90-
; AVX512BW-NEXT: vpsllw $5, %zmm1, %zmm1
91-
; AVX512BW-NEXT: vpmovb2m %zmm1, %k1
92-
; AVX512BW-NEXT: vmovdqu8 %zmm2, %zmm0 {%k1}
93-
; AVX512BW-NEXT: vpsrlw $2, %zmm0, %zmm2
94-
; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
95-
; AVX512BW-NEXT: vpaddb %zmm1, %zmm1, %zmm1
96-
; AVX512BW-NEXT: vpmovb2m %zmm1, %k1
97-
; AVX512BW-NEXT: vmovdqu8 %zmm2, %zmm0 {%k1}
98-
; AVX512BW-NEXT: vpsrlw $1, %zmm0, %zmm2
99-
; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
100-
; AVX512BW-NEXT: vpaddb %zmm1, %zmm1, %zmm1
101-
; AVX512BW-NEXT: vpmovb2m %zmm1, %k1
88+
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
89+
; AVX512BW-NEXT: vpandq %zmm2, %zmm1, %zmm3
90+
; AVX512BW-NEXT: vpandq %zmm2, %zmm0, %zmm2
91+
; AVX512BW-NEXT: vpsrlvw %zmm3, %zmm2, %zmm2
92+
; AVX512BW-NEXT: vpsrlw $8, %zmm1, %zmm1
93+
; AVX512BW-NEXT: vpsrlvw %zmm1, %zmm0, %zmm0
94+
; AVX512BW-NEXT: movabsq $6148914691236517205, %rax # imm = 0x5555555555555555
95+
; AVX512BW-NEXT: kmovq %rax, %k1
10296
; AVX512BW-NEXT: vmovdqu8 %zmm2, %zmm0 {%k1}
10397
; AVX512BW-NEXT: retq
10498
%shift = lshr <64 x i8> %a, %b

‎llvm/test/CodeGen/X86/vector-shift-shl-512.ll‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -82,19 +82,14 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
8282
;
8383
; AVX512BW-LABEL: var_shift_v64i8:
8484
; AVX512BW: # %bb.0:
85-
; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm2
86-
; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
87-
; AVX512BW-NEXT: vpsllw $5, %zmm1, %zmm1
88-
; AVX512BW-NEXT: vpmovb2m %zmm1, %k1
89-
; AVX512BW-NEXT: vmovdqu8 %zmm2, %zmm0 {%k1}
90-
; AVX512BW-NEXT: vpsllw $2, %zmm0, %zmm2
91-
; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
92-
; AVX512BW-NEXT: vpaddb %zmm1, %zmm1, %zmm1
93-
; AVX512BW-NEXT: vpmovb2m %zmm1, %k1
85+
; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm2
86+
; AVX512BW-NEXT: vpsllvw %zmm2, %zmm0, %zmm2
87+
; AVX512BW-NEXT: vpsrlw $8, %zmm1, %zmm1
88+
; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
89+
; AVX512BW-NEXT: vpsllvw %zmm1, %zmm0, %zmm0
90+
; AVX512BW-NEXT: movabsq $6148914691236517205, %rax # imm = 0x5555555555555555
91+
; AVX512BW-NEXT: kmovq %rax, %k1
9492
; AVX512BW-NEXT: vmovdqu8 %zmm2, %zmm0 {%k1}
95-
; AVX512BW-NEXT: vpaddb %zmm1, %zmm1, %zmm1
96-
; AVX512BW-NEXT: vpmovb2m %zmm1, %k1
97-
; AVX512BW-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1}
9893
; AVX512BW-NEXT: retq
9994
%shift = shl <64 x i8> %a, %b
10095
ret <64 x i8> %shift

0 commit comments

Comments
 (0)