Skip to content

Commit 04d4c90

Browse files
Krzysztof Parzyszeknikic
Krzysztof Parzyszek
authored and
nikic
committed
Handle part-word LL/SC in atomic expansion pass
Differential Revision: https://reviews.llvm.org/D77213
1 parent 23bf20e commit 04d4c90

File tree

8 files changed

+322
-145
lines changed

8 files changed

+322
-145
lines changed

‎llvm/lib/CodeGen/AtomicExpandPass.cpp‎

Lines changed: 162 additions & 98 deletions
Large diffs are not rendered by default.

‎llvm/lib/Target/Hexagon/HexagonISelLowering.cpp‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3357,9 +3357,5 @@ bool HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
33573357
TargetLowering::AtomicExpansionKind
33583358
HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR(
33593359
AtomicCmpXchgInst *AI) const {
3360-
const DataLayout &DL = AI->getModule()->getDataLayout();
3361-
unsigned Size = DL.getTypeStoreSize(AI->getCompareOperand()->getType());
3362-
if (Size >= 4 && Size <= 8)
3363-
return AtomicExpansionKind::LLSC;
3364-
return AtomicExpansionKind::None;
3360+
return AtomicExpansionKind::LLSC;
33653361
}

‎llvm/test/CodeGen/Hexagon/atomic-rmw-add.ll‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
; RUN: llc -march=hexagon < %s | FileCheck %s
2+
3+
; CHECK: .LBB0_1:
4+
; CHECK: [[R1:r[0-9]+]] = memw_locked(r0)
5+
; CHECK-DAG: [[R2:r[0-9]+]] = and([[R1]],
6+
; CHECK-DAG: [[R3:r[0-9]+]] = add([[R1]],
7+
; CHECK: [[R2]] |= and([[R3]],
8+
; CHECK: memw_locked(r0,[[P0:p[0-3]]]) = [[R2]]
9+
; CHECK: if (![[P0]]) jump:nt .LBB0_1
10+
11+
12+
%struct.a = type { i8 }
13+
14+
define void @b() #0 {
15+
%d = alloca %struct.a
16+
%c = getelementptr %struct.a, %struct.a* %d, i32 0, i32 0
17+
atomicrmw add i8* %c, i8 2 monotonic
18+
ret void
19+
}
20+
21+
attributes #0 = { "target-cpu"="hexagonv66" }
22+

‎llvm/test/CodeGen/Hexagon/partword-cmpxchg.ll‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
; RUN: llc -march=hexagon < %s | FileCheck %s
2+
3+
; CHECK-LABEL: danny
4+
; CHECK: memw_locked
5+
define i8 @danny(i8* %a0) unnamed_addr #0 {
6+
start:
7+
%v0 = cmpxchg i8* %a0, i8 0, i8 1 seq_cst seq_cst
8+
%v1 = extractvalue { i8, i1 } %v0, 0
9+
ret i8 %v1
10+
}
11+
12+
; CHECK-LABEL: sammy
13+
; CHECK: memw_locked
14+
define i16 @sammy(i16* %a0) unnamed_addr #0 {
15+
start:
16+
%v0 = cmpxchg i16* %a0, i16 0, i16 1 seq_cst seq_cst
17+
%v1 = extractvalue { i16, i1 } %v0, 0
18+
ret i16 %v1
19+
}
20+
21+
; CHECK-LABEL: kirby
22+
; CHECK: memw_locked
23+
define i32 @kirby(i32* %a0) unnamed_addr #0 {
24+
start:
25+
%v0 = cmpxchg i32* %a0, i32 0, i32 1 seq_cst seq_cst
26+
%v1 = extractvalue { i32, i1 } %v0, 0
27+
ret i32 %v1
28+
}

‎llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -257,12 +257,13 @@ define i8 @test_cmpxchg_i8_seqcst_seqcst(i8* %ptr, i8 %desired, i8 %newval) {
257257
; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
258258

259259
; CHECK: [[FAILURE_BB]]:
260+
; CHECK: [[LOADED_FAILURE:%.*]] = phi i8 [ [[LOADED_NO_STORE]], %[[NO_STORE_BB]] ]
260261
; CHECK: call void @llvm.arm.dmb(i32 11)
261262
; CHECK: br label %[[DONE]]
262263

263264
; CHECK: [[DONE]]:
265+
; CHECK: [[LOADED:%.*]] = phi i8 [ [[LOADED_LOOP]], %[[SUCCESS_BB]] ], [ [[LOADED_FAILURE]], %[[FAILURE_BB]] ]
264266
; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
265-
; CHECK: [[LOADED:%.*]] = phi i8 [ [[LOADED_LOOP]], %[[SUCCESS_BB]] ], [ [[LOADED_NO_STORE]], %[[FAILURE_BB]] ]
266267
; CHECK: ret i8 [[LOADED]]
267268

268269
%pairold = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst
@@ -307,12 +308,13 @@ define i16 @test_cmpxchg_i16_seqcst_monotonic(i16* %ptr, i16 %desired, i16 %newv
307308
; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
308309

309310
; CHECK: [[FAILURE_BB]]:
311+
; CHECK-NEXT: [[LOADED_FAILURE:%.*]] = phi i16 [ [[LOADED_NO_STORE]], %[[NO_STORE_BB]] ]
310312
; CHECK-NOT: dmb
311313
; CHECK: br label %[[DONE]]
312314

313315
; CHECK: [[DONE]]:
316+
; CHECK: [[LOADED:%.*]] = phi i16 [ [[LOADED_LOOP]], %[[SUCCESS_BB]] ], [ [[LOADED_FAILURE]], %[[FAILURE_BB]] ]
314317
; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
315-
; CHECK: [[LOADED:%.*]] = phi i16 [ [[LOADED_LOOP]], %[[SUCCESS_BB]] ], [ [[LOADED_NO_STORE]], %[[FAILURE_BB]] ]
316318
; CHECK: ret i16 [[LOADED]]
317319

318320
%pairold = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic
@@ -328,9 +330,13 @@ define i32 @test_cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %desired, i32 %newva
328330
; CHECK: [[LOOP]]:
329331
; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %ptr)
330332
; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[OLDVAL]], %desired
331-
; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
333+
; CHECK: br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]]
334+
335+
; CHECK: [[FENCED_STORE]]:
336+
; CHECK-NEXT: br label %[[TRY_STORE:.*]]
332337

333338
; CHECK: [[TRY_STORE]]:
339+
; CHECK: [[LOADED_TRYSTORE:%.*]] = phi i32 [ [[OLDVAL]], %[[FENCED_STORE]] ]
334340
; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %newval, i32* %ptr)
335341
; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0
336342
; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[LOOP]]
@@ -340,16 +346,19 @@ define i32 @test_cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %desired, i32 %newva
340346
; CHECK: br label %[[DONE:.*]]
341347

342348
; CHECK: [[NO_STORE_BB]]:
349+
; CHECK-NEXT: [[LOADED_NO_STORE:%.*]] = phi i32 [ [[OLDVAL]], %[[LOOP]] ]
343350
; CHECK-NEXT: call void @llvm.arm.clrex()
344351
; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
345352

346353
; CHECK: [[FAILURE_BB]]:
354+
; CHECK: [[LOADED_FAILURE:%.*]] = phi i32 [ [[LOADED_NO_STORE]], %[[NO_STORE_BB]] ]
347355
; CHECK: call void @llvm.arm.dmb(i32 11)
348356
; CHECK: br label %[[DONE]]
349357

350358
; CHECK: [[DONE]]:
359+
; CHECK: [[LOADED_EXIT:%.*]] = phi i32 [ [[LOADED_TRYSTORE]], %[[SUCCESS_BB]] ], [ [[LOADED_FAILURE]], %[[FAILURE_BB]] ]
351360
; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
352-
; CHECK: ret i32 [[OLDVAL]]
361+
; CHECK: ret i32 [[LOADED_EXIT]]
353362

354363
%pairold = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire
355364
%old = extractvalue { i32, i1 } %pairold, 0
@@ -371,9 +380,13 @@ define i64 @test_cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %desired, i64 %n
371380
; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32
372381
; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]]
373382
; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i64 [[OLDVAL]], %desired
374-
; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
383+
; CHECK: br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]]
384+
385+
; CHECK: [[FENCED_STORE]]:
386+
; CHECK-NEXT: br label %[[TRY_STORE:.*]]
375387

376388
; CHECK: [[TRY_STORE]]:
389+
; CHECK: [[LOADED_TRYSTORE:%.*]] = phi i64 [ [[OLDVAL]], %[[FENCED_STORE]] ]
377390
; CHECK: [[NEWLO:%.*]] = trunc i64 %newval to i32
378391
; CHECK: [[NEWHI_TMP:%.*]] = lshr i64 %newval, 32
379392
; CHECK: [[NEWHI:%.*]] = trunc i64 [[NEWHI_TMP]] to i32
@@ -387,16 +400,19 @@ define i64 @test_cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %desired, i64 %n
387400
; CHECK: br label %[[DONE:.*]]
388401

389402
; CHECK: [[NO_STORE_BB]]:
403+
; CHECK-NEXT: [[LOADED_NO_STORE:%.*]] = phi i64 [ [[OLDVAL]], %[[LOOP]] ]
390404
; CHECK-NEXT: call void @llvm.arm.clrex()
391405
; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
392406

393407
; CHECK: [[FAILURE_BB]]:
408+
; CHECK-NEXT: [[LOADED_FAILURE:%.*]] = phi i64 [ [[LOADED_NO_STORE]], %[[NO_STORE_BB]] ]
394409
; CHECK-NOT: dmb
395410
; CHECK: br label %[[DONE]]
396411

397412
; CHECK: [[DONE]]:
413+
; CHECK: [[LOADED_EXIT:%.*]] = phi i64 [ [[LOADED_TRYSTORE]], %[[SUCCESS_BB]] ], [ [[LOADED_FAILURE]], %[[FAILURE_BB]] ]
398414
; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
399-
; CHECK: ret i64 [[OLDVAL]]
415+
; CHECK: ret i64 [[LOADED_EXIT]]
400416

401417
%pairold = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic
402418
%old = extractvalue { i64, i1 } %pairold, 0
@@ -411,9 +427,13 @@ define i32 @test_cmpxchg_minsize(i32* %addr, i32 %desired, i32 %new) minsize {
411427
; CHECK: [[START]]:
412428
; CHECK: [[LOADED:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %addr)
413429
; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[LOADED]], %desired
414-
; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
430+
; CHECK: br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]]
431+
432+
; CHECK: [[FENCED_STORE]]:
433+
; CHECK-NEXT: br label %[[TRY_STORE:.*]]
415434

416435
; CHECK: [[TRY_STORE]]:
436+
; CHECK: [[LOADED_TRYSTORE:%.*]] = phi i32 [ [[LOADED]], %[[FENCED_STORE]] ]
417437
; CHECK: [[STREX:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %new, i32* %addr)
418438
; CHECK: [[SUCCESS:%.*]] = icmp eq i32 [[STREX]], 0
419439
; CHECK: br i1 [[SUCCESS]], label %[[SUCCESS_BB:.*]], label %[[START]]
@@ -423,16 +443,19 @@ define i32 @test_cmpxchg_minsize(i32* %addr, i32 %desired, i32 %new) minsize {
423443
; CHECK: br label %[[END:.*]]
424444

425445
; CHECK: [[NO_STORE_BB]]:
446+
; CHECK: [[LOADED_NO_STORE:%.*]] = phi i32 [ [[LOADED]], %[[START]] ]
426447
; CHECK: call void @llvm.arm.clrex()
427448
; CHECK: br label %[[FAILURE_BB]]
428449

429450
; CHECK: [[FAILURE_BB]]:
451+
; CHECK: [[LOADED_FAILURE:%.*]] = phi i32 [ [[LOADED_NO_STORE]], %[[NO_STORE_BB]] ]
430452
; CHECK: call void @llvm.arm.dmb(i32 11)
431453
; CHECK: br label %[[END]]
432454

433455
; CHECK: [[END]]:
456+
; CHECK: [[LOADED_EXIT:%.*]] = phi i32 [ [[LOADED_TRYSTORE]], %[[SUCCESS_BB]] ], [ [[LOADED_FAILURE]], %[[FAILURE_BB]] ]
434457
; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
435-
; CHECK: ret i32 [[LOADED]]
458+
; CHECK: ret i32 [[LOADED_EXIT]]
436459

437460
%pair = cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst seq_cst
438461
%oldval = extractvalue { i32, i1 } %pair, 0

‎llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -91,9 +91,13 @@ define i8 @test_cmpxchg_i8_seqcst_seqcst(i8* %ptr, i8 %desired, i8 %newval) {
9191
; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0i8(i8* %ptr)
9292
; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i8
9393
; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i8 [[OLDVAL]], %desired
94-
; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
94+
; CHECK: br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]]
95+
96+
; CHECK: [[FENCED_STORE]]:
97+
; CHECK-NEXT: br label %[[TRY_STORE:.*]]
9598

9699
; CHECK: [[TRY_STORE]]:
100+
; CHECK: [[LOADED_TRYSTORE:%.*]] = phi i8 [ [[OLDVAL]], %[[FENCED_STORE]] ]
97101
; CHECK: [[NEWVAL32:%.*]] = zext i8 %newval to i32
98102
; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlex.p0i8(i32 [[NEWVAL32]], i8* %ptr)
99103
; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0
@@ -104,16 +108,19 @@ define i8 @test_cmpxchg_i8_seqcst_seqcst(i8* %ptr, i8 %desired, i8 %newval) {
104108
; CHECK: br label %[[DONE:.*]]
105109

106110
; CHECK: [[NO_STORE_BB]]:
111+
; CHECK-NEXT: [[LOADED_NOSTORE:%.*]] = phi i8 [ [[OLDVAL]], %[[LOOP]] ]
107112
; CHECK-NEXT: call void @llvm.arm.clrex()
108113
; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
109114

110115
; CHECK: [[FAILURE_BB]]:
116+
; CHECK-NEXT: [[LOADED_FAILURE:%.*]] = phi i8 [ [[LOADED_NOSTORE]], %[[NO_STORE_BB]] ]
111117
; CHECK-NOT: fence_cst
112118
; CHECK: br label %[[DONE]]
113119

114120
; CHECK: [[DONE]]:
121+
; CHECK: [[LOADED_EXIT:%.*]] = phi i8 [ [[LOADED_TRYSTORE]], %[[SUCCESS_BB]] ], [ [[LOADED_FAILURE]], %[[FAILURE_BB]] ]
115122
; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
116-
; CHECK: ret i8 [[OLDVAL]]
123+
; CHECK: ret i8 [[LOADED_EXIT]]
117124

118125
%pairold = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst
119126
%old = extractvalue { i8, i1 } %pairold, 0
@@ -129,9 +136,13 @@ define i16 @test_cmpxchg_i16_seqcst_monotonic(i16* %ptr, i16 %desired, i16 %newv
129136
; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0i16(i16* %ptr)
130137
; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i16
131138
; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i16 [[OLDVAL]], %desired
132-
; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
139+
; CHECK: br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]]
140+
141+
; CHECK: [[FENCED_STORE]]:
142+
; CHECK-NEXT: br label %[[TRY_STORE:.*]]
133143

134144
; CHECK: [[TRY_STORE]]:
145+
; CHECK: [[LOADED_TRYSTORE:%.*]] = phi i16 [ [[OLDVAL]], %[[FENCED_STORE]] ]
135146
; CHECK: [[NEWVAL32:%.*]] = zext i16 %newval to i32
136147
; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlex.p0i16(i32 [[NEWVAL32]], i16* %ptr)
137148
; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0
@@ -142,16 +153,20 @@ define i16 @test_cmpxchg_i16_seqcst_monotonic(i16* %ptr, i16 %desired, i16 %newv
142153
; CHECK: br label %[[DONE:.*]]
143154

144155
; CHECK: [[NO_STORE_BB]]:
156+
; The PHI is not required.
157+
; CHECK-NEXT: [[LOADED_NOSTORE:%.*]] = phi i16 [ [[OLDVAL]], %[[LOOP]] ]
145158
; CHECK-NEXT: call void @llvm.arm.clrex()
146159
; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
147160

148161
; CHECK: [[FAILURE_BB]]:
162+
; CHECK-NEXT: [[LOADED_FAILURE:%.*]] = phi i16 [ [[LOADED_NOSTORE]], %[[NO_STORE_BB]] ]
149163
; CHECK-NOT: fence
150164
; CHECK: br label %[[DONE]]
151165

152166
; CHECK: [[DONE]]:
167+
; CHECK: [[LOADED_EXIT:%.*]] = phi i16 [ [[LOADED_TRYSTORE]], %[[SUCCESS_BB]] ], [ [[LOADED_FAILURE]], %[[FAILURE_BB]] ]
153168
; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
154-
; CHECK: ret i16 [[OLDVAL]]
169+
; CHECK: ret i16 [[LOADED_EXIT]]
155170

156171
%pairold = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic
157172
%old = extractvalue { i16, i1 } %pairold, 0
@@ -166,9 +181,13 @@ define i32 @test_cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %desired, i32 %newva
166181
; CHECK: [[LOOP]]:
167182
; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldaex.p0i32(i32* %ptr)
168183
; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[OLDVAL]], %desired
169-
; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
184+
; CHECK: br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]]
185+
186+
; CHECK: [[FENCED_STORE]]:
187+
; CHECK-NEXT: br label %[[TRY_STORE:.*]]
170188

171189
; CHECK: [[TRY_STORE]]:
190+
; CHECK: [[LOADED_TRYSTORE:%.*]] = phi i32 [ [[OLDVAL]], %[[FENCED_STORE]] ]
172191
; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %newval, i32* %ptr)
173192
; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0
174193
; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[LOOP]]
@@ -178,16 +197,19 @@ define i32 @test_cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %desired, i32 %newva
178197
; CHECK: br label %[[DONE:.*]]
179198

180199
; CHECK: [[NO_STORE_BB]]:
200+
; CHECK-NEXT: [[LOADED_NOSTORE:%.*]] = phi i32 [ [[OLDVAL]], %[[LOOP]] ]
181201
; CHECK-NEXT: call void @llvm.arm.clrex()
182202
; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
183203

184204
; CHECK: [[FAILURE_BB]]:
205+
; CHECK-NEXT: [[LOADED_FAILURE:%.*]] = phi i32 [ [[LOADED_NOSTORE]], %[[NO_STORE_BB]] ]
185206
; CHECK-NOT: fence_cst
186207
; CHECK: br label %[[DONE]]
187208

188209
; CHECK: [[DONE]]:
210+
; CHECK: [[LOADED_EXIT:%.*]] = phi i32 [ [[LOADED_TRYSTORE]], %[[SUCCESS_BB]] ], [ [[LOADED_FAILURE]], %[[FAILURE_BB]] ]
189211
; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
190-
; CHECK: ret i32 [[OLDVAL]]
212+
; CHECK: ret i32 [[LOADED_EXIT]]
191213

192214
%pairold = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire
193215
%old = extractvalue { i32, i1 } %pairold, 0
@@ -209,9 +231,13 @@ define i64 @test_cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %desired, i64 %n
209231
; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32
210232
; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]]
211233
; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i64 [[OLDVAL]], %desired
212-
; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
234+
; CHECK: br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]]
235+
236+
; CHECK: [[FENCED_STORE]]:
237+
; CHECK-NEXT: br label %[[TRY_STORE:.*]]
213238

214239
; CHECK: [[TRY_STORE]]:
240+
; CHECK: [[LOADED_TRYSTORE:%.*]] = phi i64 [ [[OLDVAL]], %[[FENCED_STORE]] ]
215241
; CHECK: [[NEWLO:%.*]] = trunc i64 %newval to i32
216242
; CHECK: [[NEWHI_TMP:%.*]] = lshr i64 %newval, 32
217243
; CHECK: [[NEWHI:%.*]] = trunc i64 [[NEWHI_TMP]] to i32
@@ -225,16 +251,19 @@ define i64 @test_cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %desired, i64 %n
225251
; CHECK: br label %[[DONE:.*]]
226252

227253
; CHECK: [[NO_STORE_BB]]:
254+
; CHECK-NEXT: [[LOADED_NOSTORE:%.*]] = phi i64 [ [[OLDVAL]], %[[LOOP]] ]
228255
; CHECK-NEXT: call void @llvm.arm.clrex()
229256
; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
230257

231258
; CHECK: [[FAILURE_BB]]:
259+
; CHECK-NEXT: [[LOADED_FAILURE:%.*]] = phi i64 [ [[LOADED_NOSTORE]], %[[NO_STORE_BB]] ]
232260
; CHECK-NOT: fence_cst
233261
; CHECK: br label %[[DONE]]
234262

235263
; CHECK: [[DONE]]:
264+
; CHECK: [[LOADED_EXIT:%.*]] = phi i64 [ [[LOADED_TRYSTORE]], %[[SUCCESS_BB]] ], [ [[LOADED_FAILURE]], %[[FAILURE_BB]] ]
236265
; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
237-
; CHECK: ret i64 [[OLDVAL]]
266+
; CHECK: ret i64 [[LOADED_EXIT]]
238267

239268
%pairold = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic
240269
%old = extractvalue { i64, i1 } %pairold, 0

0 commit comments

Comments
 (0)