Skip to content

Commit 3c89b75

Browse files
gchateletgchatelet
committed
[NFC] Introduce a type to model memory operation
Summary: This is a first step before changing the types to llvm::Align and introduce functions to ease client code. Reviewers: courbet Subscribers: arsenm, sdardis, nemanjai, jvesely, nhaehnle, hiraditya, kbarton, jrtc27, atanasyan, jsji, kerbowa, llvm-commits Tags: #llvm Differential Revision: https://reviews.llvm.org/D73785
1 parent edc3f4f commit 3c89b75

20 files changed

+164
-223
lines changed

‎llvm/include/llvm/CodeGen/TargetLowering.h‎

Lines changed: 49 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,49 @@ namespace Sched {
106106

107107
} // end namespace Sched
108108

109+
// MemOp models a memory operation, either memset or memcpy/memmove.
110+
struct MemOp {
111+
// Shared
112+
uint64_t Size;
113+
unsigned DstAlign; // Specified alignment of the memory operation or zero if
114+
// destination alignment can satisfy any constraint.
115+
bool AllowOverlap;
116+
// memset only
117+
bool IsMemset; // If setthis memory operation is a memset.
118+
bool ZeroMemset; // If set clears out memory with zeros.
119+
// memcpy only
120+
bool MemcpyStrSrc; // Indicates whether the memcpy source is an in-register
121+
// constant so it does not need to be loaded.
122+
unsigned SrcAlign; // Inferred alignment of the source or zero if the memory
123+
// operation does not need to load the value.
124+
125+
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, unsigned DstAlign,
126+
unsigned SrcAlign, bool IsVolatile,
127+
bool MemcpyStrSrc = false) {
128+
return {
129+
/*.Size =*/Size,
130+
/*.DstAlign =*/DstAlignCanChange ? 0 : DstAlign,
131+
/*.AllowOverlap =*/!IsVolatile,
132+
/*.IsMemset =*/false,
133+
/*.ZeroMemset =*/false,
134+
/*.MemcpyStrSrc =*/MemcpyStrSrc,
135+
/*.SrcAlign =*/SrcAlign,
136+
};
137+
}
138+
static MemOp Set(uint64_t Size, bool DstAlignCanChange, unsigned DstAlign,
139+
bool IsZeroMemset, bool IsVolatile) {
140+
return {
141+
/*.Size =*/Size,
142+
/*.DstAlign =*/DstAlignCanChange ? 0 : DstAlign,
143+
/*.AllowOverlap =*/!IsVolatile,
144+
/*.IsMemset =*/true,
145+
/*.ZeroMemset =*/IsZeroMemset,
146+
/*.MemcpyStrSrc =*/false,
147+
/*.SrcAlign =*/0,
148+
};
149+
}
150+
};
151+
109152
/// This base class for TargetLowering contains the SelectionDAG-independent
110153
/// parts that can be used from the rest of CodeGen.
111154
class TargetLoweringBase {
@@ -1518,29 +1561,17 @@ class TargetLoweringBase {
15181561

15191562
/// Returns the target specific optimal type for load and store operations as
15201563
/// a result of memset, memcpy, and memmove lowering.
1521-
///
1522-
/// If DstAlign is zero that means it's safe to destination alignment can
1523-
/// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
1524-
/// a need to check it against alignment requirement, probably because the
1525-
/// source does not need to be loaded. If 'IsMemset' is true, that means it's
1526-
/// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
1527-
/// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
1528-
/// does not need to be loaded. It returns EVT::Other if the type should be
1529-
/// determined using generic target-independent logic.
1564+
/// It returns EVT::Other if the type should be determined using generic
1565+
/// target-independent logic.
15301566
virtual EVT
1531-
getOptimalMemOpType(uint64_t /*Size*/, unsigned /*DstAlign*/,
1532-
unsigned /*SrcAlign*/, bool /*IsMemset*/,
1533-
bool /*ZeroMemset*/, bool /*MemcpyStrSrc*/,
1567+
getOptimalMemOpType(const MemOp &Op,
15341568
const AttributeList & /*FuncAttributes*/) const {
15351569
return MVT::Other;
15361570
}
15371571

1538-
15391572
/// LLT returning variant.
15401573
virtual LLT
1541-
getOptimalMemOpLLT(uint64_t /*Size*/, unsigned /*DstAlign*/,
1542-
unsigned /*SrcAlign*/, bool /*IsMemset*/,
1543-
bool /*ZeroMemset*/, bool /*MemcpyStrSrc*/,
1574+
getOptimalMemOpLLT(const MemOp &Op,
15441575
const AttributeList & /*FuncAttributes*/) const {
15451576
return LLT();
15461577
}
@@ -3102,14 +3133,8 @@ class TargetLowering : public TargetLoweringBase {
31023133
/// Return true if the number of memory ops is below the threshold (Limit).
31033134
/// It returns the types of the sequence of memory ops to perform
31043135
/// memset / memcpy by reference.
3105-
bool findOptimalMemOpLowering(std::vector<EVT> &MemOps,
3106-
unsigned Limit, uint64_t Size,
3107-
unsigned DstAlign, unsigned SrcAlign,
3108-
bool IsMemset,
3109-
bool ZeroMemset,
3110-
bool MemcpyStrSrc,
3111-
bool AllowOverlap,
3112-
unsigned DstAS, unsigned SrcAS,
3136+
bool findOptimalMemOpLowering(std::vector<EVT> &MemOps, unsigned Limit,
3137+
const MemOp &Op, unsigned DstAS, unsigned SrcAS,
31133138
const AttributeList &FuncAttributes) const;
31143139

31153140
/// Check to see if the specified operand of the specified instruction is a

‎llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp‎

Lines changed: 29 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -855,37 +855,30 @@ static bool shouldLowerMemFuncForSize(const MachineFunction &MF) {
855855

856856
// Returns a list of types to use for memory op lowering in MemOps. A partial
857857
// port of findOptimalMemOpLowering in TargetLowering.
858-
static bool findGISelOptimalMemOpLowering(
859-
std::vector<LLT> &MemOps, unsigned Limit, uint64_t Size, unsigned DstAlign,
860-
unsigned SrcAlign, bool IsMemset, bool ZeroMemset, bool MemcpyStrSrc,
861-
bool AllowOverlap, unsigned DstAS, unsigned SrcAS,
862-
const AttributeList &FuncAttributes, const TargetLowering &TLI) {
863-
// If 'SrcAlign' is zero, that means the memory operation does not need to
864-
// load the value, i.e. memset or memcpy from constant string. Otherwise,
865-
// it's the inferred alignment of the source. 'DstAlign', on the other hand,
866-
// is the specified alignment of the memory operation. If it is zero, that
867-
// means it's possible to change the alignment of the destination.
868-
// 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
869-
// not need to be loaded.
870-
if (SrcAlign != 0 && SrcAlign < DstAlign)
858+
static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
859+
unsigned Limit, const MemOp &Op,
860+
unsigned DstAS, unsigned SrcAS,
861+
const AttributeList &FuncAttributes,
862+
const TargetLowering &TLI) {
863+
if (Op.SrcAlign != 0 && Op.SrcAlign < Op.DstAlign)
871864
return false;
872865

873-
LLT Ty = TLI.getOptimalMemOpLLT(Size, DstAlign, SrcAlign, IsMemset,
874-
ZeroMemset, MemcpyStrSrc, FuncAttributes);
866+
LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
875867

876868
if (Ty == LLT()) {
877869
// Use the largest scalar type whose alignment constraints are satisfied.
878870
// We only need to check DstAlign here as SrcAlign is always greater or
879871
// equal to DstAlign (or zero).
880872
Ty = LLT::scalar(64);
881-
while (DstAlign && DstAlign < Ty.getSizeInBytes() &&
882-
!TLI.allowsMisalignedMemoryAccesses(Ty, DstAS, DstAlign))
873+
while (Op.DstAlign && Op.DstAlign < Ty.getSizeInBytes() &&
874+
!TLI.allowsMisalignedMemoryAccesses(Ty, DstAS, Op.DstAlign))
883875
Ty = LLT::scalar(Ty.getSizeInBytes());
884876
assert(Ty.getSizeInBits() > 0 && "Could not find valid type");
885877
// FIXME: check for the largest legal type we can load/store to.
886878
}
887879

888880
unsigned NumMemOps = 0;
881+
auto Size = Op.Size;
889882
while (Size != 0) {
890883
unsigned TySize = Ty.getSizeInBytes();
891884
while (TySize > Size) {
@@ -904,9 +897,9 @@ static bool findGISelOptimalMemOpLowering(
904897
bool Fast;
905898
// Need to get a VT equivalent for allowMisalignedMemoryAccesses().
906899
MVT VT = getMVTForLLT(Ty);
907-
if (NumMemOps && AllowOverlap && NewTySize < Size &&
900+
if (NumMemOps && Op.AllowOverlap && NewTySize < Size &&
908901
TLI.allowsMisalignedMemoryAccesses(
909-
VT, DstAS, DstAlign, MachineMemOperand::MONone, &Fast) &&
902+
VT, DstAS, Op.DstAlign, MachineMemOperand::MONone, &Fast) &&
910903
Fast)
911904
TySize = Size;
912905
else {
@@ -988,12 +981,13 @@ bool CombinerHelper::optimizeMemset(MachineInstr &MI, Register Dst, Register Val
988981
auto ValVRegAndVal = getConstantVRegValWithLookThrough(Val, MRI);
989982
bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
990983

991-
if (!findGISelOptimalMemOpLowering(
992-
MemOps, Limit, KnownLen, (DstAlignCanChange ? 0 : Align), 0,
993-
/*IsMemset=*/true,
994-
/*ZeroMemset=*/IsZeroVal, /*MemcpyStrSrc=*/false,
995-
/*AllowOverlap=*/!IsVolatile, DstPtrInfo.getAddrSpace(), ~0u,
996-
MF.getFunction().getAttributes(), TLI))
984+
if (!findGISelOptimalMemOpLowering(MemOps, Limit,
985+
MemOp::Set(KnownLen, DstAlignCanChange,
986+
Align,
987+
/*IsZeroMemset=*/IsZeroVal,
988+
/*IsVolatile=*/IsVolatile),
989+
DstPtrInfo.getAddrSpace(), ~0u,
990+
MF.getFunction().getAttributes(), TLI))
997991
return false;
998992

999993
if (DstAlignCanChange) {
@@ -1107,12 +1101,11 @@ bool CombinerHelper::optimizeMemcpy(MachineInstr &MI, Register Dst,
11071101
MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
11081102

11091103
if (!findGISelOptimalMemOpLowering(
1110-
MemOps, Limit, KnownLen, (DstAlignCanChange ? 0 : Alignment),
1111-
SrcAlign,
1112-
/*IsMemset=*/false,
1113-
/*ZeroMemset=*/false, /*MemcpyStrSrc=*/false,
1114-
/*AllowOverlap=*/!IsVolatile, DstPtrInfo.getAddrSpace(),
1115-
SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes(), TLI))
1104+
MemOps, Limit,
1105+
MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
1106+
IsVolatile),
1107+
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
1108+
MF.getFunction().getAttributes(), TLI))
11161109
return false;
11171110

11181111
if (DstAlignCanChange) {
@@ -1214,12 +1207,11 @@ bool CombinerHelper::optimizeMemmove(MachineInstr &MI, Register Dst,
12141207
// to a bug in it's findOptimalMemOpLowering implementation. For now do the
12151208
// same thing here.
12161209
if (!findGISelOptimalMemOpLowering(
1217-
MemOps, Limit, KnownLen, (DstAlignCanChange ? 0 : Alignment),
1218-
SrcAlign,
1219-
/*IsMemset=*/false,
1220-
/*ZeroMemset=*/false, /*MemcpyStrSrc=*/false,
1221-
/*AllowOverlap=*/false, DstPtrInfo.getAddrSpace(),
1222-
SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes(), TLI))
1210+
MemOps, Limit,
1211+
MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
1212+
/*IsVolatile*/ true),
1213+
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
1214+
MF.getFunction().getAttributes(), TLI))
12231215
return false;
12241216

12251217
if (DstAlignCanChange) {

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp‎

Lines changed: 12 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5908,13 +5908,12 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
59085908
bool CopyFromConstant = isMemSrcFromConstant(Src, Slice);
59095909
bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
59105910
unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
5911-
59125911
if (!TLI.findOptimalMemOpLowering(
5913-
MemOps, Limit, Size, (DstAlignCanChange ? 0 : Alignment),
5914-
(isZeroConstant ? 0 : SrcAlign), /*IsMemset=*/false,
5915-
/*ZeroMemset=*/false, /*MemcpyStrSrc=*/CopyFromConstant,
5916-
/*AllowOverlap=*/!isVol, DstPtrInfo.getAddrSpace(),
5917-
SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes()))
5912+
MemOps, Limit,
5913+
MemOp::Copy(Size, DstAlignCanChange, Alignment,
5914+
isZeroConstant ? 0 : SrcAlign, isVol, CopyFromConstant),
5915+
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
5916+
MF.getFunction().getAttributes()))
59185917
return SDValue();
59195918

59205919
if (DstAlignCanChange) {
@@ -6088,14 +6087,11 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
60886087
if (Align > SrcAlign)
60896088
SrcAlign = Align;
60906089
unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
6091-
// FIXME: `AllowOverlap` should really be `!isVol` but there is a bug in
6092-
// findOptimalMemOpLowering. Meanwhile, setting it to `false` produces the
6093-
// correct code.
6094-
bool AllowOverlap = false;
60956090
if (!TLI.findOptimalMemOpLowering(
6096-
MemOps, Limit, Size, (DstAlignCanChange ? 0 : Align), SrcAlign,
6097-
/*IsMemset=*/false, /*ZeroMemset=*/false, /*MemcpyStrSrc=*/false,
6098-
AllowOverlap, DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
6091+
MemOps, Limit,
6092+
MemOp::Copy(Size, DstAlignCanChange, Align, SrcAlign,
6093+
/*IsVolatile*/ true),
6094+
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
60996095
MF.getFunction().getAttributes()))
61006096
return SDValue();
61016097

@@ -6193,11 +6189,9 @@ static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
61936189
bool IsZeroVal =
61946190
isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
61956191
if (!TLI.findOptimalMemOpLowering(
6196-
MemOps, TLI.getMaxStoresPerMemset(OptSize), Size,
6197-
(DstAlignCanChange ? 0 : Align), 0, /*IsMemset=*/true,
6198-
/*ZeroMemset=*/IsZeroVal, /*MemcpyStrSrc=*/false,
6199-
/*AllowOverlap=*/!isVol, DstPtrInfo.getAddrSpace(), ~0u,
6200-
MF.getFunction().getAttributes()))
6192+
MemOps, TLI.getMaxStoresPerMemset(OptSize),
6193+
MemOp::Set(Size, DstAlignCanChange, Align, IsZeroVal, isVol),
6194+
DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes()))
62016195
return SDValue();
62026196

62036197
if (DstAlignCanChange) {

‎llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp‎

Lines changed: 10 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -176,37 +176,28 @@ TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
176176
return LowerCallTo(CLI);
177177
}
178178

179-
bool
180-
TargetLowering::findOptimalMemOpLowering(std::vector<EVT> &MemOps,
181-
unsigned Limit, uint64_t Size,
182-
unsigned DstAlign, unsigned SrcAlign,
183-
bool IsMemset,
184-
bool ZeroMemset,
185-
bool MemcpyStrSrc,
186-
bool AllowOverlap,
187-
unsigned DstAS, unsigned SrcAS,
188-
const AttributeList &FuncAttributes) const {
179+
bool TargetLowering::findOptimalMemOpLowering(
180+
std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
181+
unsigned SrcAS, const AttributeList &FuncAttributes) const {
189182
// If 'SrcAlign' is zero, that means the memory operation does not need to
190183
// load the value, i.e. memset or memcpy from constant string. Otherwise,
191184
// it's the inferred alignment of the source. 'DstAlign', on the other hand,
192185
// is the specified alignment of the memory operation. If it is zero, that
193186
// means it's possible to change the alignment of the destination.
194187
// 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
195188
// not need to be loaded.
196-
if (!(SrcAlign == 0 || SrcAlign >= DstAlign))
189+
if (!(Op.SrcAlign == 0 || Op.SrcAlign >= Op.DstAlign))
197190
return false;
198191

199-
EVT VT = getOptimalMemOpType(Size, DstAlign, SrcAlign,
200-
IsMemset, ZeroMemset, MemcpyStrSrc,
201-
FuncAttributes);
192+
EVT VT = getOptimalMemOpType(Op, FuncAttributes);
202193

203194
if (VT == MVT::Other) {
204195
// Use the largest integer type whose alignment constraints are satisfied.
205196
// We only need to check DstAlign here as SrcAlign is always greater or
206197
// equal to DstAlign (or zero).
207198
VT = MVT::i64;
208-
while (DstAlign && DstAlign < VT.getSizeInBits() / 8 &&
209-
!allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign))
199+
while (Op.DstAlign && Op.DstAlign < VT.getSizeInBits() / 8 &&
200+
!allowsMisalignedMemoryAccesses(VT, DstAS, Op.DstAlign))
210201
VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
211202
assert(VT.isInteger());
212203

@@ -223,6 +214,7 @@ TargetLowering::findOptimalMemOpLowering(std::vector<EVT> &MemOps,
223214
}
224215

225216
unsigned NumMemOps = 0;
217+
auto Size = Op.Size;
226218
while (Size != 0) {
227219
unsigned VTSize = VT.getSizeInBits() / 8;
228220
while (VTSize > Size) {
@@ -257,8 +249,8 @@ TargetLowering::findOptimalMemOpLowering(std::vector<EVT> &MemOps,
257249
// If the new VT cannot cover all of the remaining bits, then consider
258250
// issuing a (or a pair of) unaligned and overlapping load / store.
259251
bool Fast;
260-
if (NumMemOps && AllowOverlap && NewVTSize < Size &&
261-
allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign,
252+
if (NumMemOps && Op.AllowOverlap && NewVTSize < Size &&
253+
allowsMisalignedMemoryAccesses(VT, DstAS, Op.DstAlign,
262254
MachineMemOperand::MONone, &Fast) &&
263255
Fast)
264256
VTSize = Size;

0 commit comments

Comments
 (0)