Skip to content

Commit 44c1a24

Browse files
rmacnak-googleCommit Bot
rmacnak-google
authored and
Commit Bot
committed
[vm, compiler] Fix ARM64 TSAN.
TEST=local Change-Id: If7501a69c89763d9e62a5d0eab865e47c38f88cf Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/240322 Reviewed-by: Martin Kustermann <kustermann@google.com> Reviewed-by: Alexander Aprelev <aam@google.com> Commit-Queue: Ryan Macnak <rmacnak@google.com>
1 parent 34c2959 commit 44c1a24

File tree

3 files changed

+222
-119
lines changed

3 files changed

+222
-119
lines changed

‎runtime/vm/compiler/stub_code_compiler_arm64.cc‎

Lines changed: 215 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,101 @@ void StubCodeCompiler::EnsureIsNewOrRemembered(Assembler* assembler,
5252
__ Bind(&done);
5353
}
5454

55+
// In TSAN mode the runtime will throw an exception using an intermediary
56+
// longjmp() call to unwind the C frames in a way that TSAN can understand.
57+
//
58+
// This wrapper will setup a [jmp_buf] on the stack and initialize it to be a
59+
// target for a possible longjmp(). In the exceptional case we'll forward
60+
// control of execution to the usual JumpToFrame stub.
61+
//
62+
// In non-TSAN mode this will do nothing and the runtime will call the
63+
// JumpToFrame stub directly.
64+
//
65+
// The callback [fun] may be invoked with a modified [RSP] due to allocating
66+
// a [jmp_buf] allocating structure on the stack (as well as the saved old
67+
// [Thread::tsan_utils_->setjmp_buffer_]).
68+
static void WithExceptionCatchingTrampoline(Assembler* assembler,
69+
std::function<void()> fun) {
70+
#if defined(USING_THREAD_SANITIZER) && !defined(USING_SIMULATOR)
71+
const Register kTsanUtilsReg = R3;
72+
73+
// Reserve space for arguments and align frame before entering C++ world.
74+
const intptr_t kJumpBufferSize = sizeof(jmp_buf);
75+
// Save & Restore the volatile CPU registers across the setjmp() call.
76+
const RegisterSet volatile_registers(
77+
kAbiVolatileCpuRegs & ~(1 << R0) & ~(1 << SP),
78+
/*fpu_registers=*/0);
79+
80+
const Register kSavedRspReg = R20;
81+
COMPILE_ASSERT(IsCalleeSavedRegister(kSavedRspReg));
82+
// We rely on THR being preserved across the setjmp() call.
83+
COMPILE_ASSERT(IsCalleeSavedRegister(THR));
84+
85+
Label do_native_call;
86+
87+
// Save old jmp_buf.
88+
__ ldr(kTsanUtilsReg, Address(THR, target::Thread::tsan_utils_offset()));
89+
__ ldr(TMP,
90+
Address(kTsanUtilsReg, target::TsanUtils::setjmp_buffer_offset()));
91+
__ Push(TMP);
92+
93+
// Allocate jmp_buf struct on stack & remember pointer to it on the
94+
// [Thread::tsan_utils_->setjmp_buffer] (which exceptions.cc will longjmp()
95+
// to)
96+
__ AddImmediate(SP, -kJumpBufferSize);
97+
__ str(SP, Address(kTsanUtilsReg, target::TsanUtils::setjmp_buffer_offset()));
98+
99+
// Call setjmp() with a pointer to the allocated jmp_buf struct.
100+
__ MoveRegister(R0, SP);
101+
__ PushRegisters(volatile_registers);
102+
__ EnterCFrame(0);
103+
__ mov(R25, CSP);
104+
__ mov(CSP, SP);
105+
__ ldr(kTsanUtilsReg, Address(THR, target::Thread::tsan_utils_offset()));
106+
__ CallCFunction(
107+
Address(kTsanUtilsReg, target::TsanUtils::setjmp_function_offset()));
108+
__ mov(SP, CSP);
109+
__ mov(CSP, R25);
110+
__ LeaveCFrame();
111+
__ PopRegisters(volatile_registers);
112+
113+
// We are the target of a longjmp() iff setjmp() returns non-0.
114+
__ cbz(&do_native_call, R0);
115+
116+
// We are the target of a longjmp: Cleanup the stack and tail-call the
117+
// JumpToFrame stub which will take care of unwinding the stack and hand
118+
// execution to the catch entry.
119+
__ AddImmediate(SP, kJumpBufferSize);
120+
__ ldr(kTsanUtilsReg, Address(THR, target::Thread::tsan_utils_offset()));
121+
__ Pop(TMP);
122+
__ str(TMP,
123+
Address(kTsanUtilsReg, target::TsanUtils::setjmp_buffer_offset()));
124+
125+
__ ldr(R0, Address(kTsanUtilsReg, target::TsanUtils::exception_pc_offset()));
126+
__ ldr(R1, Address(kTsanUtilsReg, target::TsanUtils::exception_sp_offset()));
127+
__ ldr(R2, Address(kTsanUtilsReg, target::TsanUtils::exception_fp_offset()));
128+
__ MoveRegister(R3, THR);
129+
__ Jump(Address(THR, target::Thread::jump_to_frame_entry_point_offset()));
130+
131+
// We leave the created [jump_buf] structure on the stack as well as the
132+
// pushed old [Thread::tsan_utils_->setjmp_buffer_].
133+
__ Bind(&do_native_call);
134+
__ MoveRegister(kSavedRspReg, SP);
135+
#endif // defined(USING_THREAD_SANITIZER) && !defined(USING_SIMULATOR)
136+
137+
fun();
138+
139+
#if defined(USING_THREAD_SANITIZER) && !defined(USING_SIMULATOR)
140+
__ MoveRegister(SP, kSavedRspReg);
141+
__ AddImmediate(SP, kJumpBufferSize);
142+
const Register kTsanUtilsReg2 = kSavedRspReg;
143+
__ ldr(kTsanUtilsReg2, Address(THR, target::Thread::tsan_utils_offset()));
144+
__ Pop(TMP);
145+
__ str(TMP,
146+
Address(kTsanUtilsReg2, target::TsanUtils::setjmp_buffer_offset()));
147+
#endif // defined(USING_THREAD_SANITIZER) && !defined(USING_SIMULATOR)
148+
}
149+
55150
// Input parameters:
56151
// LR : return address.
57152
// SP : address of last argument in argument array.
@@ -93,73 +188,75 @@ void StubCodeCompiler::GenerateCallToRuntimeStub(Assembler* assembler) {
93188
// Mark that the thread is executing VM code.
94189
__ StoreToOffset(R5, THR, target::Thread::vm_tag_offset());
95190

96-
// Reserve space for arguments and align frame before entering C++ world.
97-
// target::NativeArguments are passed in registers.
98-
__ Comment("align stack");
99-
// Reserve space for arguments.
100-
ASSERT(target::NativeArguments::StructSize() == 4 * target::kWordSize);
101-
__ ReserveAlignedFrameSpace(target::NativeArguments::StructSize());
102-
103-
// Pass target::NativeArguments structure by value and call runtime.
104-
// Registers R0, R1, R2, and R3 are used.
105-
106-
ASSERT(thread_offset == 0 * target::kWordSize);
107-
// Set thread in NativeArgs.
108-
__ mov(R0, THR);
109-
110-
// There are no runtime calls to closures, so we do not need to set the tag
111-
// bits kClosureFunctionBit and kInstanceFunctionBit in argc_tag_.
112-
ASSERT(argc_tag_offset == 1 * target::kWordSize);
113-
__ mov(R1, R4); // Set argc in target::NativeArguments.
114-
115-
ASSERT(argv_offset == 2 * target::kWordSize);
116-
__ add(R2, ZR, Operand(R4, LSL, 3));
117-
__ add(R2, FP, Operand(R2)); // Compute argv.
118-
// Set argv in target::NativeArguments.
119-
__ AddImmediate(R2,
120-
target::frame_layout.param_end_from_fp * target::kWordSize);
191+
WithExceptionCatchingTrampoline(assembler, [&]() {
192+
// Reserve space for arguments and align frame before entering C++ world.
193+
// target::NativeArguments are passed in registers.
194+
__ Comment("align stack");
195+
// Reserve space for arguments.
196+
ASSERT(target::NativeArguments::StructSize() == 4 * target::kWordSize);
197+
__ ReserveAlignedFrameSpace(target::NativeArguments::StructSize());
121198

122-
ASSERT(retval_offset == 3 * target::kWordSize);
123-
__ AddImmediate(R3, R2, target::kWordSize);
199+
// Pass target::NativeArguments structure by value and call runtime.
200+
// Registers R0, R1, R2, and R3 are used.
124201

125-
__ StoreToOffset(R0, SP, thread_offset);
126-
__ StoreToOffset(R1, SP, argc_tag_offset);
127-
__ StoreToOffset(R2, SP, argv_offset);
128-
__ StoreToOffset(R3, SP, retval_offset);
129-
__ mov(R0, SP); // Pass the pointer to the target::NativeArguments.
202+
ASSERT(thread_offset == 0 * target::kWordSize);
203+
// Set thread in NativeArgs.
204+
__ mov(R0, THR);
130205

131-
// We are entering runtime code, so the C stack pointer must be restored from
132-
// the stack limit to the top of the stack. We cache the stack limit address
133-
// in a callee-saved register.
134-
__ mov(R25, CSP);
135-
__ mov(CSP, SP);
206+
// There are no runtime calls to closures, so we do not need to set the tag
207+
// bits kClosureFunctionBit and kInstanceFunctionBit in argc_tag_.
208+
ASSERT(argc_tag_offset == 1 * target::kWordSize);
209+
__ mov(R1, R4); // Set argc in target::NativeArguments.
136210

137-
__ blr(R5);
138-
__ Comment("CallToRuntimeStub return");
211+
ASSERT(argv_offset == 2 * target::kWordSize);
212+
__ add(R2, ZR, Operand(R4, LSL, 3));
213+
__ add(R2, FP, Operand(R2)); // Compute argv.
214+
// Set argv in target::NativeArguments.
215+
__ AddImmediate(R2,
216+
target::frame_layout.param_end_from_fp * target::kWordSize);
139217

140-
// Restore SP and CSP.
141-
__ mov(SP, CSP);
142-
__ mov(CSP, R25);
218+
ASSERT(retval_offset == 3 * target::kWordSize);
219+
__ AddImmediate(R3, R2, target::kWordSize);
143220

144-
// Refresh pinned registers values (inc. write barrier mask and null object).
145-
__ RestorePinnedRegisters();
221+
__ StoreToOffset(R0, SP, thread_offset);
222+
__ StoreToOffset(R1, SP, argc_tag_offset);
223+
__ StoreToOffset(R2, SP, argv_offset);
224+
__ StoreToOffset(R3, SP, retval_offset);
225+
__ mov(R0, SP); // Pass the pointer to the target::NativeArguments.
146226

147-
// Retval is next to 1st argument.
148-
// Mark that the thread is executing Dart code.
149-
__ LoadImmediate(R2, VMTag::kDartTagId);
150-
__ StoreToOffset(R2, THR, target::Thread::vm_tag_offset());
227+
// We are entering runtime code, so the C stack pointer must be restored
228+
// from the stack limit to the top of the stack. We cache the stack limit
229+
// address in a callee-saved register.
230+
__ mov(R25, CSP);
231+
__ mov(CSP, SP);
151232

152-
// Mark that the thread has not exited generated Dart code.
153-
__ StoreToOffset(ZR, THR, target::Thread::exit_through_ffi_offset());
233+
__ blr(R5);
234+
__ Comment("CallToRuntimeStub return");
154235

155-
// Reset exit frame information in Isolate's mutator thread structure.
156-
__ StoreToOffset(ZR, THR, target::Thread::top_exit_frame_info_offset());
236+
// Restore SP and CSP.
237+
__ mov(SP, CSP);
238+
__ mov(CSP, R25);
157239

158-
// Restore the global object pool after returning from runtime (old space is
159-
// moving, so the GOP could have been relocated).
160-
if (FLAG_precompiled_mode) {
161-
__ SetupGlobalPoolAndDispatchTable();
162-
}
240+
// Refresh pinned registers (write barrier mask, null, dispatch table, etc).
241+
__ RestorePinnedRegisters();
242+
243+
// Retval is next to 1st argument.
244+
// Mark that the thread is executing Dart code.
245+
__ LoadImmediate(R2, VMTag::kDartTagId);
246+
__ StoreToOffset(R2, THR, target::Thread::vm_tag_offset());
247+
248+
// Mark that the thread has not exited generated Dart code.
249+
__ StoreToOffset(ZR, THR, target::Thread::exit_through_ffi_offset());
250+
251+
// Reset exit frame information in Isolate's mutator thread structure.
252+
__ StoreToOffset(ZR, THR, target::Thread::top_exit_frame_info_offset());
253+
254+
// Restore the global object pool after returning from runtime (old space is
255+
// moving, so the GOP could have been relocated).
256+
if (FLAG_precompiled_mode) {
257+
__ SetupGlobalPoolAndDispatchTable();
258+
}
259+
});
163260

164261
__ LeaveStubFrame();
165262

@@ -678,73 +775,76 @@ static void GenerateCallNativeWithWrapperStub(Assembler* assembler,
678775
// Mark that the thread is executing native code.
679776
__ StoreToOffset(R5, THR, target::Thread::vm_tag_offset());
680777

681-
// Reserve space for the native arguments structure passed on the stack (the
682-
// outgoing pointer parameter to the native arguments structure is passed in
683-
// R0) and align frame before entering the C++ world.
684-
__ ReserveAlignedFrameSpace(target::NativeArguments::StructSize());
685-
686-
// Initialize target::NativeArguments structure and call native function.
687-
// Registers R0, R1, R2, and R3 are used.
688-
689-
ASSERT(thread_offset == 0 * target::kWordSize);
690-
// Set thread in NativeArgs.
691-
__ mov(R0, THR);
778+
WithExceptionCatchingTrampoline(assembler, [&]() {
779+
// Reserve space for the native arguments structure passed on the stack (the
780+
// outgoing pointer parameter to the native arguments structure is passed in
781+
// R0) and align frame before entering the C++ world.
782+
__ ReserveAlignedFrameSpace(target::NativeArguments::StructSize());
692783

693-
// There are no native calls to closures, so we do not need to set the tag
694-
// bits kClosureFunctionBit and kInstanceFunctionBit in argc_tag_.
695-
ASSERT(argc_tag_offset == 1 * target::kWordSize);
696-
// Set argc in target::NativeArguments: R1 already contains argc.
784+
// Initialize target::NativeArguments structure and call native function.
785+
// Registers R0, R1, R2, and R3 are used.
697786

698-
ASSERT(argv_offset == 2 * target::kWordSize);
699-
// Set argv in target::NativeArguments: R2 already contains argv.
787+
ASSERT(thread_offset == 0 * target::kWordSize);
788+
// Set thread in NativeArgs.
789+
__ mov(R0, THR);
700790

701-
// Set retval in NativeArgs.
702-
ASSERT(retval_offset == 3 * target::kWordSize);
703-
__ AddImmediate(
704-
R3, FP, (target::frame_layout.param_end_from_fp + 1) * target::kWordSize);
705-
706-
// Passing the structure by value as in runtime calls would require changing
707-
// Dart API for native functions.
708-
// For now, space is reserved on the stack and we pass a pointer to it.
709-
__ StoreToOffset(R0, SP, thread_offset);
710-
__ StoreToOffset(R1, SP, argc_tag_offset);
711-
__ StoreToOffset(R2, SP, argv_offset);
712-
__ StoreToOffset(R3, SP, retval_offset);
713-
__ mov(R0, SP); // Pass the pointer to the target::NativeArguments.
714-
715-
// We are entering runtime code, so the C stack pointer must be restored from
716-
// the stack limit to the top of the stack. We cache the stack limit address
717-
// in the Dart SP register, which is callee-saved in the C ABI.
718-
__ mov(R25, CSP);
719-
__ mov(CSP, SP);
791+
// There are no native calls to closures, so we do not need to set the tag
792+
// bits kClosureFunctionBit and kInstanceFunctionBit in argc_tag_.
793+
ASSERT(argc_tag_offset == 1 * target::kWordSize);
794+
// Set argc in target::NativeArguments: R1 already contains argc.
795+
796+
ASSERT(argv_offset == 2 * target::kWordSize);
797+
// Set argv in target::NativeArguments: R2 already contains argv.
798+
799+
// Set retval in NativeArgs.
800+
ASSERT(retval_offset == 3 * target::kWordSize);
801+
__ AddImmediate(
802+
R3, FP,
803+
(target::frame_layout.param_end_from_fp + 1) * target::kWordSize);
804+
805+
// Passing the structure by value as in runtime calls would require changing
806+
// Dart API for native functions.
807+
// For now, space is reserved on the stack and we pass a pointer to it.
808+
__ StoreToOffset(R0, SP, thread_offset);
809+
__ StoreToOffset(R1, SP, argc_tag_offset);
810+
__ StoreToOffset(R2, SP, argv_offset);
811+
__ StoreToOffset(R3, SP, retval_offset);
812+
__ mov(R0, SP); // Pass the pointer to the target::NativeArguments.
813+
814+
// We are entering runtime code, so the C stack pointer must be restored
815+
// from the stack limit to the top of the stack. We cache the stack limit
816+
// address in the Dart SP register, which is callee-saved in the C ABI.
817+
__ mov(R25, CSP);
818+
__ mov(CSP, SP);
720819

721-
__ mov(R1, R5); // Pass the function entrypoint to call.
820+
__ mov(R1, R5); // Pass the function entrypoint to call.
722821

723-
// Call native function invocation wrapper or redirection via simulator.
724-
__ Call(wrapper);
822+
// Call native function invocation wrapper or redirection via simulator.
823+
__ Call(wrapper);
725824

726-
// Restore SP and CSP.
727-
__ mov(SP, CSP);
728-
__ mov(CSP, R25);
825+
// Restore SP and CSP.
826+
__ mov(SP, CSP);
827+
__ mov(CSP, R25);
729828

730-
// Refresh pinned registers values (inc. write barrier mask and null object).
731-
__ RestorePinnedRegisters();
829+
// Refresh pinned registers (write barrier mask, null, dispatch table, etc).
830+
__ RestorePinnedRegisters();
732831

733-
// Mark that the thread is executing Dart code.
734-
__ LoadImmediate(R2, VMTag::kDartTagId);
735-
__ StoreToOffset(R2, THR, target::Thread::vm_tag_offset());
832+
// Mark that the thread is executing Dart code.
833+
__ LoadImmediate(R2, VMTag::kDartTagId);
834+
__ StoreToOffset(R2, THR, target::Thread::vm_tag_offset());
736835

737-
// Mark that the thread has not exited generated Dart code.
738-
__ StoreToOffset(ZR, THR, target::Thread::exit_through_ffi_offset());
836+
// Mark that the thread has not exited generated Dart code.
837+
__ StoreToOffset(ZR, THR, target::Thread::exit_through_ffi_offset());
739838

740-
// Reset exit frame information in Isolate's mutator thread structure.
741-
__ StoreToOffset(ZR, THR, target::Thread::top_exit_frame_info_offset());
839+
// Reset exit frame information in Isolate's mutator thread structure.
840+
__ StoreToOffset(ZR, THR, target::Thread::top_exit_frame_info_offset());
742841

743-
// Restore the global object pool after returning from runtime (old space is
744-
// moving, so the GOP could have been relocated).
745-
if (FLAG_precompiled_mode) {
746-
__ SetupGlobalPoolAndDispatchTable();
747-
}
842+
// Restore the global object pool after returning from runtime (old space is
843+
// moving, so the GOP could have been relocated).
844+
if (FLAG_precompiled_mode) {
845+
__ SetupGlobalPoolAndDispatchTable();
846+
}
847+
});
748848

749849
__ LeaveStubFrame();
750850
__ ret();
@@ -1393,7 +1493,7 @@ void StubCodeCompiler::GenerateInvokeDartCodeStub(Assembler* assembler) {
13931493
__ mov(THR, R3);
13941494
}
13951495

1396-
// Refresh pinned registers values (inc. write barrier mask and null object).
1496+
// Refresh pinned registers (write barrier mask, null, dispatch table, etc).
13971497
__ RestorePinnedRegisters();
13981498

13991499
// Save the current VMTag on the stack.
@@ -3125,7 +3225,7 @@ void StubCodeCompiler::GenerateJumpToFrameStub(Assembler* assembler) {
31253225
/*ignore_unwind_in_progress=*/true);
31263226
__ Bind(&exit_through_non_ffi);
31273227

3128-
// Refresh pinned registers values (inc. write barrier mask and null object).
3228+
// Refresh pinned registers (write barrier mask, null, dispatch table, etc).
31293229
__ RestorePinnedRegisters();
31303230
// Set the tag.
31313231
__ LoadImmediate(R2, VMTag::kDartTagId);

0 commit comments

Comments
 (0)