@@ -1155,6 +1155,22 @@ Instruction *InstCombinerImpl::FoldOpIntoSelect(Instruction &Op, SelectInst *SI,
11551155 return SelectInst::Create (SI->getCondition (), NewTV, NewFV, " " nullptr , SI);
11561156}
11571157
1158+ static Value *foldOperationIntoPhiValue (BinaryOperator *I, Value *InV,
1159+ InstCombiner::BuilderTy &Builder) {
1160+ bool ConstIsRHS = isa<Constant>(I->getOperand (1 ));
1161+ Constant *C = cast<Constant>(I->getOperand (ConstIsRHS));
1162+
1163+ Value *Op0 = InV, *Op1 = C;
1164+ if (!ConstIsRHS)
1165+ std::swap (Op0, Op1);
1166+
1167+ Value *RI = Builder.CreateBinOp (I->getOpcode (), Op0, Op1, " phi.bo"
1168+ auto *FPInst = dyn_cast<Instruction>(RI);
1169+ if (FPInst && isa<FPMathOperator>(FPInst))
1170+ FPInst->copyFastMathFlags (I);
1171+ return RI;
1172+ }
1173+
11581174Instruction *InstCombinerImpl::foldOpIntoPhi (Instruction &I, PHINode *PN) {
11591175 unsigned NumPHIValues = PN->getNumIncomingValues ();
11601176 if (NumPHIValues == 0 )
@@ -1173,68 +1189,48 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) {
11731189 // Otherwise, we can replace *all* users with the new PHI we form.
11741190 }
11751191
1176- // Check to see whether the instruction can be folded into each phi operand.
1177- // If there is one operand that does not fold, remember the BB it is in.
1178- // If there is more than one or if *it* is a PHI, bail out.
1179- SmallVector<Value *> NewPhiValues;
1180- BasicBlock *NonSimplifiedBB = nullptr ;
1181- Value *NonSimplifiedInVal = nullptr ;
1192+ // Check to see if all of the operands of the PHI are simple constants
1193+ // (constantint/constantfp/undef). If there is one non-constant value,
1194+ // remember the BB it is in. If there is more than one or if *it* is a PHI,
1195+ // bail out. We don't do arbitrary constant expressions here because moving
1196+ // their computation can be expensive without a cost model.
1197+ BasicBlock *NonConstBB = nullptr ;
11821198 for (unsigned i = 0 ; i != NumPHIValues; ++i) {
11831199 Value *InVal = PN->getIncomingValue (i);
1184- BasicBlock *InBB = PN->getIncomingBlock (i);
1185-
1186- // NB: It is a precondition of this transform that the operands be
1187- // phi translatable! This is usually trivially satisfied by limiting it
1188- // to constant ops, and for selects we do a more sophisticated check.
1189- SmallVector<Value *> Ops;
1190- for (Value *Op : I.operands ()) {
1191- if (Op == PN)
1192- Ops.push_back (InVal);
1193- else
1194- Ops.push_back (Op->DoPHITranslation (PN->getParent (), InBB));
1195- }
1196-
1197- // Don't consider the simplification successful if we get back a constant
1198- // expression. That's just an instruction in hiding.
1199- // Also reject the case where we simplify back to the phi node. We wouldn't
1200- // be able to remove it in that case.
1201- Value *NewVal = simplifyInstructionWithOperands (&I, Ops, SQ);
1202- if (NewVal && NewVal != PN && !match (NewVal, m_ConstantExpr ())) {
1203- NewPhiValues.push_back (NewVal);
1200+ // For non-freeze, require constant operand
1201+ // For freeze, require non-undef, non-poison operand
1202+ if (!isa<FreezeInst>(I) && match (InVal, m_ImmConstant ()))
1203+ continue ;
1204+ if (isa<FreezeInst>(I) && isGuaranteedNotToBeUndefOrPoison (InVal))
12041205 continue ;
1205- }
12061206
12071207 if (isa<PHINode>(InVal))
return nullptr ;
// Itself a phi. 1208- if (NonSimplifiedBB ) return nullptr ; // More than one non-simplified value.
1208+ if (NonConstBB ) return nullptr ; // More than one non-const value.
12091209
1210- NonSimplifiedBB = InBB;
1211- NonSimplifiedInVal = InVal;
1212- NewPhiValues.push_back (nullptr );
1210+ NonConstBB = PN->getIncomingBlock (i);
12131211
12141212 // If the InVal is an invoke at the end of the pred block, then we can't
12151213 // insert a computation after it without breaking the edge.
12161214 if (isa<InvokeInst>(InVal))
1217- if (cast<Instruction>(InVal)->getParent () == NonSimplifiedBB )
1215+ if (cast<Instruction>(InVal)->getParent () == NonConstBB )
12181216 return nullptr ;
12191217
12201218 // If the incoming non-constant value is reachable from the phis block,
12211219 // we'll push the operation across a loop backedge. This could result in
12221220 // an infinite combine loop, and is generally non-profitable (especially
12231221 // if the operation was originally outside the loop).
1224- if (isPotentiallyReachable (PN->getParent (), NonSimplifiedBB, nullptr , &DT,
1225- LI))
1222+ if (isPotentiallyReachable (PN->getParent (), NonConstBB, nullptr , &DT, LI))
12261223 return nullptr ;
12271224 }
12281225
1229- // If there is exactly one non-simplified value, we can insert a copy of the
1226+ // If there is exactly one non-constant value, we can insert a copy of the
12301227 // operation in that block. However, if this is a critical edge, we would be
12311228 // inserting the computation on some other paths (e.g. inside a loop). Only
12321229 // do this if the pred block is unconditionally branching into the phi block.
12331230 // Also, make sure that the pred block is not dead code.
1234- if (NonSimplifiedBB != nullptr ) {
1235- BranchInst *BI = dyn_cast<BranchInst>(NonSimplifiedBB->getTerminator ());
1236- if (!BI || !BI->isUnconditional () ||
1237- !DT.isReachableFromEntry (NonSimplifiedBB))
1231+ if (NonConstBB != nullptr ) {
1232+ BranchInst *BI = dyn_cast<BranchInst>(NonConstBB->getTerminator ());
1233+ if (!BI || !BI->isUnconditional () || !DT.isReachableFromEntry (NonConstBB))
12381234 return nullptr ;
12391235 }
12401236
@@ -1245,23 +1241,88 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) {
12451241
12461242 // If we are going to have to insert a new computation, do so right before the
12471243 // predecessor's terminator.
1248- Instruction *Clone = nullptr ;
1249- if (NonSimplifiedBB) {
1250- Clone = I.clone ();
1251- for (Use &U : Clone->operands ()) {
1252- if (U == PN)
1253- U = NonSimplifiedInVal;
1244+ if (NonConstBB)
1245+ Builder.SetInsertPoint (NonConstBB->getTerminator ());
1246+
1247+ // Next, add all of the operands to the PHI.
1248+ if (SelectInst *SI = dyn_cast<SelectInst>(&I)) {
1249+ // We only currently try to fold the condition of a select when it is a phi,
1250+ // not the true/false values.
1251+ Value *TrueV = SI->getTrueValue ();
1252+ Value *FalseV = SI->getFalseValue ();
1253+ BasicBlock *PhiTransBB = PN->getParent ();
1254+ for (unsigned i = 0 ; i != NumPHIValues; ++i) {
1255+ BasicBlock *ThisBB = PN->getIncomingBlock (i);
1256+ Value *TrueVInPred = TrueV->DoPHITranslation (PhiTransBB, ThisBB);
1257+ Value *FalseVInPred = FalseV->DoPHITranslation (PhiTransBB, ThisBB);
1258+ Value *InV = nullptr ;
1259+ // Beware of ConstantExpr: it may eventually evaluate to getNullValue,
1260+ // even if currently isNullValue gives false.
1261+ Constant *InC = dyn_cast<Constant>(PN->getIncomingValue (i));
1262+ // For vector constants, we cannot use isNullValue to fold into
1263+ // FalseVInPred versus TrueVInPred. When we have individual nonzero
1264+ // elements in the vector, we will incorrectly fold InC to
1265+ // `TrueVInPred`.
1266+ if (InC && isa<ConstantInt>(InC))
1267+ InV = InC->isNullValue () ? FalseVInPred : TrueVInPred;
1268+ else {
1269+ // Generate the select in the same block as PN's current incoming block.
1270+ // Note: ThisBB need not be the NonConstBB because vector constants
1271+ // which are constants by definition are handled here.
1272+ // FIXME: This can lead to an increase in IR generation because we might
1273+ // generate selects for vector constant phi operand, that could not be
1274+ // folded to TrueVInPred or FalseVInPred as done for ConstantInt. For
1275+ // non-vector phis, this transformation was always profitable because
1276+ // the select would be generated exactly once in the NonConstBB.
1277+ Builder.SetInsertPoint (ThisBB->getTerminator ());
1278+ InV = Builder.CreateSelect (PN->getIncomingValue (i), TrueVInPred,
1279+ FalseVInPred, " phi.sel"
1280+ }
1281+ NewPN->addIncoming (InV, ThisBB);
1282+ }
1283+ } else if (CmpInst *CI = dyn_cast<CmpInst>(&I)) {
1284+ Constant *C = cast<Constant>(I.getOperand (1 ));
1285+ for (unsigned i = 0 ; i != NumPHIValues; ++i) {
1286+ Value *InV = nullptr ;
1287+ if (auto *InC = dyn_cast<Constant>(PN->getIncomingValue (i)))
1288+ InV = ConstantExpr::getCompare (CI->getPredicate (), InC, C);
12541289 else
1255- U = U->DoPHITranslation (PN->getParent (), NonSimplifiedBB);
1290+ InV = Builder.CreateCmp (CI->getPredicate (), PN->getIncomingValue (i),
1291+ C, " phi.cmp"
1292+ NewPN->addIncoming (InV, PN->getIncomingBlock (i));
1293+ }
1294+ } else if (auto *BO = dyn_cast<BinaryOperator>(&I)) {
1295+ for (unsigned i = 0 ; i != NumPHIValues; ++i) {
1296+ Value *InV = foldOperationIntoPhiValue (BO, PN->getIncomingValue (i),
1297+ Builder);
1298+ NewPN->addIncoming (InV, PN->getIncomingBlock (i));
1299+ }
1300+ } else if (isa<FreezeInst>(&I)) {
1301+ for (unsigned i = 0 ; i != NumPHIValues; ++i) {
1302+ Value *InV;
1303+ if (NonConstBB == PN->getIncomingBlock (i))
1304+ InV = Builder.CreateFreeze (PN->getIncomingValue (i), " phi.fr"
1305+ else
1306+ InV = PN->getIncomingValue (i);
1307+ NewPN->addIncoming (InV, PN->getIncomingBlock (i));
1308+ }
1309+ } else if (auto *EV = dyn_cast<ExtractValueInst>(&I)) {
1310+ for (unsigned i = 0 ; i != NumPHIValues; ++i)
1311+ NewPN->addIncoming (Builder.CreateExtractValue (PN->getIncomingValue (i),
1312+ EV->getIndices (), " phi.ev"
1313+ PN->getIncomingBlock (i));
1314+ } else {
1315+ CastInst *CI = cast<CastInst>(&I);
1316+ Type *RetTy = CI->getType ();
1317+ for (unsigned i = 0 ; i != NumPHIValues; ++i) {
1318+ Value *InV;
1319+ if (Constant *InC = dyn_cast<Constant>(PN->getIncomingValue (i)))
1320+ InV = ConstantExpr::getCast (CI->getOpcode (), InC, RetTy);
1321+ else
1322+ InV = Builder.CreateCast (CI->getOpcode (), PN->getIncomingValue (i),
1323+ I.getType (), " phi.cast"
1324+ NewPN->addIncoming (InV, PN->getIncomingBlock (i));
12561325 }
1257- InsertNewInstBefore (Clone, *NonSimplifiedBB->getTerminator ());
1258- }
1259-
1260- for (unsigned i = 0 ; i != NumPHIValues; ++i) {
1261- if (NewPhiValues[i])
1262- NewPN->addIncoming (NewPhiValues[i], PN->getIncomingBlock (i));
1263- else
1264- NewPN->addIncoming (Clone, PN->getIncomingBlock (i));
12651326 }
12661327
12671328 for (User *U : make_early_inc_range (PN->users ())) {
0 commit comments