From 441008a86c2268f7baa9b10e6af78d5228f7f19e Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Wed, 14 May 2025 20:49:43 +0100 Subject: [PATCH 01/16] 8365047: Remove exception handler stub code in C2 The C2 exception handler stub code is only a trampoline to the generated exception handler blob. This change removes the extra step on the way to the generated blob. According to some comments in the source code, the exception handler stub code used to be patched upon deoptimization, however presumably these comments are outdated as the patching upon deoptimization happens for post-call NOPs only. --- src/hotspot/cpu/aarch64/aarch64.ad | 24 ------------ src/hotspot/cpu/aarch64/runtime_aarch64.cpp | 2 - src/hotspot/cpu/arm/arm.ad | 26 ------------- src/hotspot/cpu/arm/runtime_arm.cpp | 2 - src/hotspot/cpu/ppc/ppc.ad | 22 ----------- src/hotspot/cpu/ppc/runtime_ppc.cpp | 1 - src/hotspot/cpu/riscv/riscv.ad | 24 ------------ src/hotspot/cpu/riscv/runtime_riscv.cpp | 2 - src/hotspot/cpu/s390/runtime_s390.cpp | 2 - src/hotspot/cpu/s390/s390.ad | 41 --------------------- src/hotspot/cpu/x86/runtime_x86_64.cpp | 2 - src/hotspot/cpu/x86/x86.ad | 28 -------------- src/hotspot/share/ci/ciEnv.cpp | 5 ++- src/hotspot/share/code/nmethod.cpp | 13 +++++-- src/hotspot/share/opto/output.cpp | 6 +-- src/hotspot/share/runtime/sharedRuntime.cpp | 9 +++++ 16 files changed, 24 insertions(+), 185 deletions(-) diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 517da8066dec6..5a40332a4f3af 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -1194,13 +1194,8 @@ class HandlerImpl { public: - static int emit_exception_handler(C2_MacroAssembler *masm); static int emit_deopt_handler(C2_MacroAssembler* masm); - static uint size_exception_handler() { - return MacroAssembler::far_codestub_branch_size(); - } - static uint size_deopt_handler() { // count one adr and one far branch instruction return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size(); @@ -2261,25 +2256,6 @@ uint MachUEPNode::size(PhaseRegAlloc* ra_) const //============================================================================= -// Emit exception handler code. -int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm) -{ - // mov rscratch1 #exception_blob_entry_point - // br rscratch1 - // Note that the code buffer's insts_mark is always relative to insts. - // That's why we must use the macroassembler to generate a handler. - address base = __ start_a_stub(size_exception_handler()); - if (base == nullptr) { - ciEnv::current()->record_failure("CodeCache is full"); - return 0; // CodeBuffer::expand failed - } - int offset = __ offset(); - __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point())); - assert(__ offset() - offset <= (int) size_exception_handler(), "overflow"); - __ end_a_stub(); - return offset; -} - // Emit deopt handler code. int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { diff --git a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp index d45f9865bd26a..e36aa21b567af 100644 --- a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp @@ -260,8 +260,6 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() { //------------------------------generate_exception_blob--------------------------- // creates exception blob at the end -// Using exception blob, this code is jumped from a compiled method. -// (see emit_exception_handler in aarch64.ad file) // // Given an exception pc at a call we call into the runtime for the // handler in this method. This handler might merely restore state diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad index 3b6faa6c81a7a..e151267037268 100644 --- a/src/hotspot/cpu/arm/arm.ad +++ b/src/hotspot/cpu/arm/arm.ad @@ -105,14 +105,8 @@ class HandlerImpl { public: - static int emit_exception_handler(C2_MacroAssembler *masm); static int emit_deopt_handler(C2_MacroAssembler* masm); - static uint size_exception_handler() { - return ( 3 * 4 ); - } - - static uint size_deopt_handler() { return ( 9 * 4 ); } @@ -876,26 +870,6 @@ uint MachUEPNode::size(PhaseRegAlloc *ra_) const { //============================================================================= -// Emit exception handler code. -int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm) { - address base = __ start_a_stub(size_exception_handler()); - if (base == nullptr) { - ciEnv::current()->record_failure("CodeCache is full"); - return 0; // CodeBuffer::expand failed - } - - int offset = __ offset(); - - // OK to trash LR, because exception blob will kill it - __ jump(OptoRuntime::exception_blob()->entry_point(), relocInfo::runtime_call_type, LR_tmp); - - assert(__ offset() - offset <= (int) size_exception_handler(), "overflow"); - - __ end_a_stub(); - - return offset; -} - int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { // Can't use any of the current frame's registers as we may have deopted // at a poll and everything can be live. diff --git a/src/hotspot/cpu/arm/runtime_arm.cpp b/src/hotspot/cpu/arm/runtime_arm.cpp index ea4c9a76381bc..f5c343f58d8c8 100644 --- a/src/hotspot/cpu/arm/runtime_arm.cpp +++ b/src/hotspot/cpu/arm/runtime_arm.cpp @@ -182,8 +182,6 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() { //------------------------------ generate_exception_blob --------------------------- // creates exception blob at the end -// Using exception blob, this code is jumped from a compiled method. -// (see emit_exception_handler in sparc.ad file) // // Given an exception pc at a call we call into the runtime for the // handler in this method. This handler might merely restore state diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index bf43ecaba79f2..e2af81a240874 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -2088,14 +2088,8 @@ class HandlerImpl { public: - static int emit_exception_handler(C2_MacroAssembler *masm); static int emit_deopt_handler(C2_MacroAssembler* masm); - static uint size_exception_handler() { - // The exception_handler is a b64_patchable. - return MacroAssembler::b64_patchable_size; - } - static uint size_deopt_handler() { // The deopt_handler is a bl64_patchable. return MacroAssembler::bl64_patchable_size; @@ -2114,22 +2108,6 @@ public: source %{ -int HandlerImpl::emit_exception_handler(C2_MacroAssembler *masm) { - address base = __ start_a_stub(size_exception_handler()); - if (base == nullptr) { - ciEnv::current()->record_failure("CodeCache is full"); - return 0; // CodeBuffer::expand failed - } - - int offset = __ offset(); - __ b64_patchable((address)OptoRuntime::exception_blob()->content_begin(), - relocInfo::runtime_call_type); - assert(__ offset() - offset == (int)size_exception_handler(), "must be fixed size"); - __ end_a_stub(); - - return offset; -} - // The deopt_handler is like the exception handler, but it calls to // the deoptimization blob instead of jumping to the exception blob. int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { diff --git a/src/hotspot/cpu/ppc/runtime_ppc.cpp b/src/hotspot/cpu/ppc/runtime_ppc.cpp index 2654075f702c6..ab658e9de58bc 100644 --- a/src/hotspot/cpu/ppc/runtime_ppc.cpp +++ b/src/hotspot/cpu/ppc/runtime_ppc.cpp @@ -46,7 +46,6 @@ //------------------------------generate_exception_blob--------------------------- // Creates exception blob at the end. -// Using exception blob, this code is jumped from a compiled method. // // Given an exception pc at a call we call into the runtime for the // handler in this method. This handler might merely restore state diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index 8e7b772414a70..0427521499486 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -1049,13 +1049,8 @@ class HandlerImpl { public: - static int emit_exception_handler(C2_MacroAssembler *masm); static int emit_deopt_handler(C2_MacroAssembler* masm); - static uint size_exception_handler() { - return MacroAssembler::far_branch_size(); - } - static uint size_deopt_handler() { // count auipc + far branch return NativeInstruction::instruction_size + MacroAssembler::far_branch_size(); @@ -1810,25 +1805,6 @@ uint MachUEPNode::size(PhaseRegAlloc* ra_) const //============================================================================= -// Emit exception handler code. -int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm) -{ - // auipc t1, #exception_blob_entry_point - // jr (offset)t1 - // Note that the code buffer's insts_mark is always relative to insts. - // That's why we must use the macroassembler to generate a handler. - address base = __ start_a_stub(size_exception_handler()); - if (base == nullptr) { - ciEnv::current()->record_failure("CodeCache is full"); - return 0; // CodeBuffer::expand failed - } - int offset = __ offset(); - __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point())); - assert(__ offset() - offset <= (int) size_exception_handler(), "overflow"); - __ end_a_stub(); - return offset; -} - // Emit deopt handler code. int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { diff --git a/src/hotspot/cpu/riscv/runtime_riscv.cpp b/src/hotspot/cpu/riscv/runtime_riscv.cpp index e1add8dbb82b2..c52d5a31066a5 100644 --- a/src/hotspot/cpu/riscv/runtime_riscv.cpp +++ b/src/hotspot/cpu/riscv/runtime_riscv.cpp @@ -249,8 +249,6 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() { //------------------------------generate_exception_blob--------------------------- // creates exception blob at the end -// Using exception blob, this code is jumped from a compiled method. -// (see emit_exception_handler in riscv.ad file) // // Given an exception pc at a call we call into the runtime for the // handler in this method. This handler might merely restore state diff --git a/src/hotspot/cpu/s390/runtime_s390.cpp b/src/hotspot/cpu/s390/runtime_s390.cpp index 314c407af9134..658fba069b4c1 100644 --- a/src/hotspot/cpu/s390/runtime_s390.cpp +++ b/src/hotspot/cpu/s390/runtime_s390.cpp @@ -43,8 +43,6 @@ //------------------------------generate_exception_blob--------------------------- // creates exception blob at the end -// Using exception blob, this code is jumped from a compiled method. -// (see emit_exception_handler in s390.ad file) // // Given an exception pc at a call we call into the runtime for the // handler in this method. This handler might merely restore state diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index c32064be86d87..f6a83e5fec107 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -1652,10 +1652,6 @@ class HandlerImpl { static int emit_exception_handler(C2_MacroAssembler *masm); static int emit_deopt_handler(C2_MacroAssembler* masm); - static uint size_exception_handler() { - return NativeJump::max_instruction_size(); - } - static uint size_deopt_handler() { return NativeCall::max_instruction_size(); } @@ -1672,43 +1668,6 @@ public: source %{ -// This exception handler code snippet is placed after the method's -// code. It is the return point if an exception occurred. it jumps to -// the exception blob. -// -// If the method gets deoptimized, the method and this code snippet -// get patched. -// -// 1) Trampoline code gets patched into the end of this exception -// handler. the trampoline code jumps to the deoptimization blob. -// -// 2) The return address in the method's code will get patched such -// that it jumps to the trampoline. -// -// 3) The handler will get patched such that it does not jump to the -// exception blob, but to an entry in the deoptimization blob being -// aware of the exception. -int HandlerImpl::emit_exception_handler(C2_MacroAssembler *masm) { - Register temp_reg = Z_R1; - - address base = __ start_a_stub(size_exception_handler()); - if (base == nullptr) { - ciEnv::current()->record_failure("CodeCache is full"); - return 0; // CodeBuffer::expand failed - } - - int offset = __ offset(); - // Use unconditional pc-relative jump with 32-bit range here. - __ load_const_optimized(temp_reg, (address)OptoRuntime::exception_blob()->content_begin()); - __ z_br(temp_reg); - - assert(__ offset() - offset <= (int) size_exception_handler(), "overflow"); - - __ end_a_stub(); - - return offset; -} - // Emit deopt handler code. int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { address base = __ start_a_stub(size_deopt_handler()); diff --git a/src/hotspot/cpu/x86/runtime_x86_64.cpp b/src/hotspot/cpu/x86/runtime_x86_64.cpp index 7b98cf4fad76a..5bf65299a0c51 100644 --- a/src/hotspot/cpu/x86/runtime_x86_64.cpp +++ b/src/hotspot/cpu/x86/runtime_x86_64.cpp @@ -242,8 +242,6 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() { //------------------------------generate_exception_blob--------------------------- // creates exception blob at the end -// Using exception blob, this code is jumped from a compiled method. -// (see emit_exception_handler in x86_64.ad file) // // Given an exception pc at a call we call into the runtime for the // handler in this method. This handler might merely restore state diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index 2eb748e350c05..c5fee5ec7197f 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -1134,18 +1134,8 @@ class HandlerImpl { public: - static int emit_exception_handler(C2_MacroAssembler *masm); static int emit_deopt_handler(C2_MacroAssembler* masm); - static uint size_exception_handler() { - // NativeCall instruction size is the same as NativeJump. - // exception handler starts out as jump and can be patched to - // a call be deoptimization. (4932387) - // Note that this value is also credited (in output.cpp) to - // the size of the code section. - return NativeJump::instruction_size; - } - static uint size_deopt_handler() { // three 5 byte instructions plus one move for unreachable address. return 15+3; @@ -1240,24 +1230,6 @@ int MachNode::compute_padding(int current_offset) const { } } -// Emit exception handler code. -// Stuff framesize into a register and call a VM stub routine. -int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm) { - - // Note that the code buffer's insts_mark is always relative to insts. - // That's why we must use the macroassembler to generate a handler. - address base = __ start_a_stub(size_exception_handler()); - if (base == nullptr) { - ciEnv::current()->record_failure("CodeCache is full"); - return 0; // CodeBuffer::expand failed - } - int offset = __ offset(); - __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point())); - assert(__ offset() - offset <= (int) size_exception_handler(), "overflow"); - __ end_a_stub(); - return offset; -} - // Emit deopt handler code. int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { diff --git a/src/hotspot/share/ci/ciEnv.cpp b/src/hotspot/share/ci/ciEnv.cpp index 2de77f2828dfd..e30b58f4d0bb8 100644 --- a/src/hotspot/share/ci/ciEnv.cpp +++ b/src/hotspot/share/ci/ciEnv.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright 2025 Arm Limited and/or its affiliates. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1057,7 +1058,9 @@ void ciEnv::register_method(ciMethod* target, } assert(offsets->value(CodeOffsets::Deopt) != -1, "must have deopt entry"); - assert(offsets->value(CodeOffsets::Exceptions) != -1, "must have exception entry"); + + assert(compiler->type() == compiler_c2 || + offsets->value(CodeOffsets::Exceptions) != -1, "must have exception entry"); nm = nmethod::new_nmethod(method, compile_id(), diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index d8cbf6b9e6630..a80e4d62e370b 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright 2025 Arm Limited and/or its affiliates. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1473,10 +1474,16 @@ nmethod::nmethod( #endif { // Exception handler and deopt handler are in the stub section - assert(offsets->value(CodeOffsets::Exceptions) != -1, "must be set"); - assert(offsets->value(CodeOffsets::Deopt ) != -1, "must be set"); + bool has_exception_handler = (offsets->value(CodeOffsets::Exceptions) != -1); + assert(has_exception_handler == (compiler->type() != compiler_c2), + "C2 compiler doesn't provide exception handler stub code."); + if (has_exception_handler) { + _exception_offset = _stub_offset + offsets->value(CodeOffsets::Exceptions); + } else { + _exception_offset = -1; + } - _exception_offset = _stub_offset + offsets->value(CodeOffsets::Exceptions); + assert(offsets->value(CodeOffsets::Deopt ) != -1, "must be set"); _deopt_handler_offset = _stub_offset + offsets->value(CodeOffsets::Deopt); if (offsets->value(CodeOffsets::DeoptMH) != -1) { _deopt_mh_handler_offset = _stub_offset + offsets->value(CodeOffsets::DeoptMH); diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp index 124b00a654959..3cfeb80b48235 100644 --- a/src/hotspot/share/opto/output.cpp +++ b/src/hotspot/share/opto/output.cpp @@ -1366,20 +1366,18 @@ CodeBuffer* PhaseOutput::init_buffer() { // nmethod and CodeBuffer count stubs & constants as part of method's code. // class HandlerImpl is platform-specific and defined in the *.ad files. - int exception_handler_req = HandlerImpl::size_exception_handler() + MAX_stubs_size; // add marginal slop for handler int deopt_handler_req = HandlerImpl::size_deopt_handler() + MAX_stubs_size; // add marginal slop for handler stub_req += MAX_stubs_size; // ensure per-stub margin code_req += MAX_inst_size; // ensure per-instruction margin if (StressCodeBuffers) - code_req = const_req = stub_req = exception_handler_req = deopt_handler_req = 0x10; // force expansion + code_req = const_req = stub_req = deopt_handler_req = 0x10; // force expansion int total_req = const_req + code_req + pad_req + stub_req + - exception_handler_req + deopt_handler_req; // deopt handler if (C->has_method_handle_invokes()) @@ -1869,8 +1867,6 @@ void PhaseOutput::fill_buffer(C2_MacroAssembler* masm, uint* blk_starts) { // Only java methods have exception handlers and deopt handlers // class HandlerImpl is platform-specific and defined in the *.ad files. if (C->method()) { - // Emit the exception handler code. - _code_offsets.set_value(CodeOffsets::Exceptions, HandlerImpl::emit_exception_handler(masm)); if (C->failing()) { return; // CodeBuffer::expand failed } diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 554b94b56c6d1..7de7dc471e30d 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright 2025 Arm Limited and/or its affiliates. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -86,6 +87,9 @@ #ifdef COMPILER1 #include "c1/c1_Runtime1.hpp" #endif +#ifdef COMPILER2 +#include "opto/runtime.hpp" +#endif #if INCLUDE_JFR #include "jfr/jfr.inline.hpp" #endif @@ -565,6 +569,11 @@ address SharedRuntime::raw_exception_handler_for_return_address(JavaThread* curr // The deferred StackWatermarkSet::after_unwind check will be performed in // * OptoRuntime::handle_exception_C_helper for C2 code // * exception_handler_for_pc_helper via Runtime1::handle_exception_from_callee_id for C1 code +#ifdef COMPILER2 + if (nm->compiler_type() == compiler_c2) { + return OptoRuntime::exception_blob()->entry_point(); + } +#endif // COMPILER2 return nm->exception_begin(); } } From 2d8012a392265781a90b2c3e1ea0bce14efbd41b Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Thu, 14 Aug 2025 09:50:15 +0100 Subject: [PATCH 02/16] Address review comments --- src/hotspot/cpu/s390/s390.ad | 1 - src/hotspot/share/code/nmethod.cpp | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index f6a83e5fec107..16b0e373156f8 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -1649,7 +1649,6 @@ source_hpp %{ // Header information of the source block. class HandlerImpl { public: - static int emit_exception_handler(C2_MacroAssembler *masm); static int emit_deopt_handler(C2_MacroAssembler* masm); static uint size_deopt_handler() { diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index a80e4d62e370b..7eb6b04adfc8c 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -1474,6 +1474,8 @@ nmethod::nmethod( #endif { // Exception handler and deopt handler are in the stub section + assert(offsets->value(CodeOffsets::Deopt ) != -1, "must be set"); + bool has_exception_handler = (offsets->value(CodeOffsets::Exceptions) != -1); assert(has_exception_handler == (compiler->type() != compiler_c2), "C2 compiler doesn't provide exception handler stub code."); @@ -1483,7 +1485,6 @@ nmethod::nmethod( _exception_offset = -1; } - assert(offsets->value(CodeOffsets::Deopt ) != -1, "must be set"); _deopt_handler_offset = _stub_offset + offsets->value(CodeOffsets::Deopt); if (offsets->value(CodeOffsets::DeoptMH) != -1) { _deopt_mh_handler_offset = _stub_offset + offsets->value(CodeOffsets::DeoptMH); From 1f6546e99a8b5c40ec72ddc16a7f24bf39873596 Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Wed, 14 May 2025 20:49:43 +0100 Subject: [PATCH 03/16] Ensure stub code is not adjacent to a call --- src/hotspot/share/opto/output.cpp | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp index 3cfeb80b48235..2047ff2b0e663 100644 --- a/src/hotspot/share/opto/output.cpp +++ b/src/hotspot/share/opto/output.cpp @@ -570,6 +570,10 @@ void PhaseOutput::shorten_branches(uint* blk_starts) { blk_size += max_loop_pad; block_worst_case_pad[i + 1] = max_loop_pad; } + } else { + if (last_call_adr == blk_starts[i]+blk_size) { + blk_size += nop_size; + } } // Save block size; update total method size @@ -1795,17 +1799,25 @@ void PhaseOutput::fill_buffer(C2_MacroAssembler* masm, uint* blk_starts) { } // End for all instructions in block + auto emit_nops = [&](int nops_count) { + MachNode *nop = new MachNopNode(nops_count); + block->insert_node(nop, block->number_of_nodes()); + C->cfg()->map_node_to_block(nop, block); + nop->emit(masm, C->regalloc()); + current_offset = masm->offset(); + }; + // If the next block is the top of a loop, pad this block out to align // the loop top a little. Helps prevent pipe stalls at loop back branches. if (i < nblocks-1) { Block *nb = C->cfg()->get_block(i + 1); int padding = nb->alignment_padding(current_offset); if( padding > 0 ) { - MachNode *nop = new MachNopNode(padding / nop_size); - block->insert_node(nop, block->number_of_nodes()); - C->cfg()->map_node_to_block(nop, block); - nop->emit(masm, C->regalloc()); - current_offset = masm->offset(); + emit_nops(padding / nop_size); + } + } else { + if (last_call_offset == current_offset) { + emit_nops(1); } } // Verify that the distance for generated before forward From 7d1ece5024326dcbbb63322fdd5335d91ff4ffb1 Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Fri, 12 Sep 2025 16:42:46 +0100 Subject: [PATCH 04/16] Revert "Ensure stub code is not adjacent to a call" --- src/hotspot/share/opto/output.cpp | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp index 2047ff2b0e663..3cfeb80b48235 100644 --- a/src/hotspot/share/opto/output.cpp +++ b/src/hotspot/share/opto/output.cpp @@ -570,10 +570,6 @@ void PhaseOutput::shorten_branches(uint* blk_starts) { blk_size += max_loop_pad; block_worst_case_pad[i + 1] = max_loop_pad; } - } else { - if (last_call_adr == blk_starts[i]+blk_size) { - blk_size += nop_size; - } } // Save block size; update total method size @@ -1799,25 +1795,17 @@ void PhaseOutput::fill_buffer(C2_MacroAssembler* masm, uint* blk_starts) { } // End for all instructions in block - auto emit_nops = [&](int nops_count) { - MachNode *nop = new MachNopNode(nops_count); - block->insert_node(nop, block->number_of_nodes()); - C->cfg()->map_node_to_block(nop, block); - nop->emit(masm, C->regalloc()); - current_offset = masm->offset(); - }; - // If the next block is the top of a loop, pad this block out to align // the loop top a little. Helps prevent pipe stalls at loop back branches. if (i < nblocks-1) { Block *nb = C->cfg()->get_block(i + 1); int padding = nb->alignment_padding(current_offset); if( padding > 0 ) { - emit_nops(padding / nop_size); - } - } else { - if (last_call_offset == current_offset) { - emit_nops(1); + MachNode *nop = new MachNopNode(padding / nop_size); + block->insert_node(nop, block->number_of_nodes()); + C->cfg()->map_node_to_block(nop, block); + nop->emit(masm, C->regalloc()); + current_offset = masm->offset(); } } // Verify that the distance for generated before forward From 47178248f3ea73f973614105a48a26d52d688e38 Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Fri, 12 Sep 2025 16:56:00 +0100 Subject: [PATCH 05/16] Offset the deoptimization handler entry point Change-Id: I596317ec6a364b341e4642636fa5cf08f87ed722 --- src/hotspot/cpu/aarch64/aarch64.ad | 13 ++++++---- src/hotspot/cpu/arm/arm.ad | 19 ++++++++------- src/hotspot/cpu/ppc/ppc.ad | 14 +++++++++-- src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp | 18 +------------- src/hotspot/cpu/riscv/riscv.ad | 13 +++++++--- src/hotspot/cpu/s390/s390.ad | 13 ++++++++-- src/hotspot/cpu/x86/x86.ad | 21 ++++++++-------- src/hotspot/os/posix/signals_posix.cpp | 4 ++-- src/hotspot/os/windows/os_windows.cpp | 4 ++-- src/hotspot/share/code/nmethod.cpp | 24 +++++++++---------- src/hotspot/share/code/nmethod.hpp | 8 +++---- src/hotspot/share/code/nmethod.inline.hpp | 4 ++-- src/hotspot/share/runtime/frame.cpp | 8 +++---- src/hotspot/share/runtime/vmStructs.cpp | 4 ++-- .../classes/sun/jvm/hotspot/code/NMethod.java | 16 ++++++------- .../hotspot/runtime/aarch64/AARCH64Frame.java | 2 +- .../jvm/hotspot/runtime/ppc64/PPC64Frame.java | 2 +- .../hotspot/runtime/riscv64/RISCV64Frame.java | 2 +- .../sun/jvm/hotspot/runtime/x86/X86Frame.java | 2 +- 19 files changed, 105 insertions(+), 86 deletions(-) diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 5a40332a4f3af..a837fa43ac889 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -1,6 +1,7 @@ // // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved. // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved. +// Copyright 2025 Arm Limited and/or its affiliates. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -1197,7 +1198,7 @@ class HandlerImpl { static int emit_deopt_handler(C2_MacroAssembler* masm); static uint size_deopt_handler() { - // count one adr and one far branch instruction + // count one branch instruction and one far call instruction sequence return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size(); } }; @@ -2266,14 +2267,18 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) ciEnv::current()->record_failure("CodeCache is full"); return 0; // CodeBuffer::expand failed } + int offset = __ offset(); + Label start; + __ bind(start); + __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); - __ adr(lr, __ pc()); - __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); + int entry_point = __ offset(); + __ b(start); assert(__ offset() - offset == (int) size_deopt_handler(), "overflow"); __ end_a_stub(); - return offset; + return entry_point; } // REQUIRED MATCHER CODE diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad index e151267037268..f1bc9dc5df104 100644 --- a/src/hotspot/cpu/arm/arm.ad +++ b/src/hotspot/cpu/arm/arm.ad @@ -1,5 +1,6 @@ // // Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved. +// Copyright 2025 Arm Limited and/or its affiliates. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -108,7 +109,7 @@ class HandlerImpl { static int emit_deopt_handler(C2_MacroAssembler* masm); static uint size_deopt_handler() { - return ( 9 * 4 ); + return ( 4 * 4 ); } }; @@ -880,19 +881,21 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { } int offset = __ offset(); - address deopt_pc = __ pc(); - __ sub(SP, SP, wordSize); // make room for saved PC - __ push(LR); // save LR that may be live when we get here - __ mov_relative_address(LR, deopt_pc); - __ str(LR, Address(SP, wordSize)); // save deopt PC - __ pop(LR); // restore LR + Label start; + __ bind(start); + + // Store address of the entry point + __ push(PC); __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg); + int entry_point = __ offset(); + __ b(start); + assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow"); __ end_a_stub(); - return offset; + return entry_point; } bool Matcher::match_rule_supported(int opcode) { diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index e2af81a240874..85bab0ebc61ed 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -1,6 +1,7 @@ // // Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved. // Copyright (c) 2012, 2025 SAP SE. All rights reserved. +// Copyright 2025 Arm Limited and/or its affiliates. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -2092,7 +2093,7 @@ class HandlerImpl { static uint size_deopt_handler() { // The deopt_handler is a bl64_patchable. - return MacroAssembler::bl64_patchable_size; + return MacroAssembler::bl64_patchable_size + BytesPerInstWord; } }; @@ -2118,12 +2119,21 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { } int offset = __ offset(); + + Label start; + __ bind(start); + __ bl64_patchable((address)SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type); + + int entry_point = __ offset(); + + __ b(start); + assert(__ offset() - offset == (int) size_deopt_handler(), "must be fixed size"); __ end_a_stub(); - return offset; + return entry_point; } //============================================================================= diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp index 64bd8dc7812e0..69f2b9c94c95c 100644 --- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp +++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp @@ -83,7 +83,6 @@ class RegisterSaver { static OopMap* push_frame_reg_args_and_save_live_registers(MacroAssembler* masm, int* out_frame_size_in_bytes, bool generate_oop_map, - int return_pc_adjustment, ReturnPCLocation return_pc_location, bool save_vectors = false); static void restore_live_registers_and_pop_frame(MacroAssembler* masm, @@ -262,7 +261,6 @@ static const RegisterSaver::LiveRegType RegisterSaver_LiveVecRegs[] = { OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssembler* masm, int* out_frame_size_in_bytes, bool generate_oop_map, - int return_pc_adjustment, ReturnPCLocation return_pc_location, bool save_vectors) { // Push an abi_reg_args-frame and store all registers which may be live. @@ -271,7 +269,6 @@ OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssemble // propagated to the RegisterMap of the caller frame during // StackFrameStream construction (needed for deoptimization; see // compiledVFrame::create_stack_value). - // If return_pc_adjustment != 0 adjust the return pc by return_pc_adjustment. // Updated return pc is returned in R31 (if not return_pc_is_pre_saved). // calculate frame size @@ -305,14 +302,11 @@ OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssemble // Do the save_LR by hand and adjust the return pc if requested. switch (return_pc_location) { case return_pc_is_lr: __ mflr(R31); break; - case return_pc_is_pre_saved: assert(return_pc_adjustment == 0, "unsupported"); break; + case return_pc_is_pre_saved: break; case return_pc_is_thread_saved_exception_pc: __ ld(R31, thread_(saved_exception_pc)); break; default: ShouldNotReachHere(); } if (return_pc_location != return_pc_is_pre_saved) { - if (return_pc_adjustment != 0) { - __ addi(R31, R31, return_pc_adjustment); - } __ std(R31, frame_size_in_bytes + _abi0(lr), R1_SP); } @@ -2977,14 +2971,9 @@ void SharedRuntime::generate_deopt_blob() { // deopt_handler: call_deopt_stub // cur. return pc --> ... // - // So currently SR_LR points behind the call in the deopt handler. - // We adjust it such that it points to the start of the deopt handler. // The return_pc has been stored in the frame of the deoptee and // will replace the address of the deopt_handler in the call // to Deoptimization::fetch_unroll_info below. - // We can't grab a free register here, because all registers may - // contain live values, so let the RegisterSaver do the adjustment - // of the return pc. const int return_pc_adjustment_no_exception = -MacroAssembler::bl64_patchable_size; // Push the "unpack frame" @@ -2992,7 +2981,6 @@ void SharedRuntime::generate_deopt_blob() { map = RegisterSaver::push_frame_reg_args_and_save_live_registers(masm, &first_frame_size_in_bytes, /*generate_oop_map=*/ true, - return_pc_adjustment_no_exception, RegisterSaver::return_pc_is_lr); assert(map != nullptr, "OopMap must have been created"); @@ -3027,7 +3015,6 @@ void SharedRuntime::generate_deopt_blob() { RegisterSaver::push_frame_reg_args_and_save_live_registers(masm, &first_frame_size_in_bytes, /*generate_oop_map=*/ false, - /*return_pc_adjustment_exception=*/ 0, RegisterSaver::return_pc_is_pre_saved); // Deopt during an exception. Save exec mode for unpack_frames. @@ -3045,7 +3032,6 @@ void SharedRuntime::generate_deopt_blob() { RegisterSaver::push_frame_reg_args_and_save_live_registers(masm, &first_frame_size_in_bytes, /*generate_oop_map=*/ false, - /*return_pc_adjustment_reexecute=*/ 0, RegisterSaver::return_pc_is_pre_saved); __ li(exec_mode_reg, Deoptimization::Unpack_reexecute); #endif @@ -3336,7 +3322,6 @@ SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr) map = RegisterSaver::push_frame_reg_args_and_save_live_registers(masm, &frame_size_in_bytes, /*generate_oop_map=*/ true, - /*return_pc_adjustment=*/0, return_pc_location, save_vectors); // The following is basically a call_VM. However, we need the precise @@ -3437,7 +3422,6 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination map = RegisterSaver::push_frame_reg_args_and_save_live_registers(masm, &frame_size_in_bytes, /*generate_oop_map*/ true, - /*return_pc_adjustment*/ 0, RegisterSaver::return_pc_is_lr); // Use noreg as last_Java_pc, the return pc will be reconstructed diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index 0427521499486..b7088d99e22b7 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -2,6 +2,7 @@ // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved. // Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. // Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved. +// Copyright 2025 Arm Limited and/or its affiliates. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -1052,8 +1053,8 @@ class HandlerImpl { static int emit_deopt_handler(C2_MacroAssembler* masm); static uint size_deopt_handler() { - // count auipc + far branch - return NativeInstruction::instruction_size + MacroAssembler::far_branch_size(); + // count auipc + far branch + jal + return 2 * NativeInstruction::instruction_size + MacroAssembler::far_branch_size(); } }; @@ -1815,8 +1816,14 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) } int offset = __ offset(); + Label start; + __ bind(start); + __ auipc(ra, 0); - __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); + __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); + + int entry_point = __ offset(); + __ j(start); assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow"); __ end_a_stub(); diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index 16b0e373156f8..91c97c9f0f966 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -1,6 +1,7 @@ // // Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved. // Copyright (c) 2017, 2024 SAP SE. All rights reserved. +// Copyright 2025 Arm Limited and/or its affiliates. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -1652,7 +1653,7 @@ class HandlerImpl { static int emit_deopt_handler(C2_MacroAssembler* masm); static uint size_deopt_handler() { - return NativeCall::max_instruction_size(); + return NativeCall::max_instruction_size() + MacroAssembler::jump_pcrelative_size(); } }; @@ -1678,14 +1679,22 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { int offset = __ offset(); + Label start; + __ bind(start); + // Size_deopt_handler() must be exact on zarch, so for simplicity // we do not use load_const_opt here. __ load_const(Z_R1, SharedRuntime::deopt_blob()->unpack()); __ call(Z_R1); + + int entry_point = __ offset(); + + __ z_bru(start); + assert(__ offset() - offset == (int) size_deopt_handler(), "must be fixed size"); __ end_a_stub(); - return offset; + return entry_point; } //============================================================================= diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index c5fee5ec7197f..b1f2bb0f4e5a3 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -1,5 +1,6 @@ // // Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved. +// Copyright 2025 Arm Limited and/or its affiliates. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -1137,8 +1138,8 @@ class HandlerImpl { static int emit_deopt_handler(C2_MacroAssembler* masm); static uint size_deopt_handler() { - // three 5 byte instructions plus one move for unreachable address. - return 15+3; + // one call and one jmp. + return 10; } }; @@ -1243,20 +1244,20 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { int offset = __ offset(); address the_pc = (address) __ pc(); - Label next; + Label start; + __ bind(start); + // push a "the_pc" on the stack without destroying any registers // as they all may be live. + __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); + + int entry_point = __ offset(); - // push address of "next" - __ call(next, relocInfo::none); // reloc none is fine since it is a disp32 - __ bind(next); - // adjust it so it matches "the_pc" - __ subptr(Address(rsp, 0), __ offset() - offset); + __ jmp(start); - __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow %d", (__ offset() - offset)); __ end_a_stub(); - return offset; + return entry_point; } static Assembler::Width widthForType(BasicType bt) { diff --git a/src/hotspot/os/posix/signals_posix.cpp b/src/hotspot/os/posix/signals_posix.cpp index 0157d354f4027..84f5d3ff553ab 100644 --- a/src/hotspot/os/posix/signals_posix.cpp +++ b/src/hotspot/os/posix/signals_posix.cpp @@ -622,8 +622,8 @@ int JVM_HANDLE_XXX_SIGNAL(int sig, siginfo_t* info, nmethod* nm = cb->as_nmethod(); assert(nm->insts_contains_inclusive(pc), ""); address deopt = nm->is_method_handle_return(pc) ? - nm->deopt_mh_handler_begin() : - nm->deopt_handler_begin(); + nm->deopt_mh_handler_entry() : + nm->deopt_handler_entry(); assert(deopt != nullptr, ""); frame fr = os::fetch_frame_from_context(uc); diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp index f50677571250c..eebba6aaff84f 100644 --- a/src/hotspot/os/windows/os_windows.cpp +++ b/src/hotspot/os/windows/os_windows.cpp @@ -2795,8 +2795,8 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) { nmethod* nm = cb->as_nmethod(); frame fr = os::fetch_frame_from_context((void*)exceptionInfo->ContextRecord); address deopt = nm->is_method_handle_return(pc) ? - nm->deopt_mh_handler_begin() : - nm->deopt_handler_begin(); + nm->deopt_mh_handler_entry() : + nm->deopt_handler_entry(); assert(nm->insts_contains_inclusive(pc), ""); nm->set_original_pc(&fr, pc); // Set pc to handler diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index 7eb6b04adfc8c..fa62d3b443cbe 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -1316,8 +1316,8 @@ nmethod::nmethod( } // Native wrappers do not have deopt handlers. Make the values // something that will never match a pc like the nmethod vtable entry - _deopt_handler_offset = 0; - _deopt_mh_handler_offset = 0; + _deopt_handler_entry_offset = 0; + _deopt_mh_handler_entry_offset = 0; _unwind_handler_offset = 0; CHECKED_CAST(_oops_size, uint16_t, align_up(code_buffer->total_oop_size(), oopSize)); @@ -1461,14 +1461,14 @@ nmethod::nmethod( _exception_offset = -1; } if (offsets->value(CodeOffsets::Deopt) != -1) { - _deopt_handler_offset = code_offset() + offsets->value(CodeOffsets::Deopt); + _deopt_handler_entry_offset = code_offset() + offsets->value(CodeOffsets::Deopt); } else { - _deopt_handler_offset = -1; + _deopt_handler_entry_offset = -1; } if (offsets->value(CodeOffsets::DeoptMH) != -1) { - _deopt_mh_handler_offset = code_offset() + offsets->value(CodeOffsets::DeoptMH); + _deopt_mh_handler_entry_offset = code_offset() + offsets->value(CodeOffsets::DeoptMH); } else { - _deopt_mh_handler_offset = -1; + _deopt_mh_handler_entry_offset = -1; } } else #endif @@ -1485,11 +1485,11 @@ nmethod::nmethod( _exception_offset = -1; } - _deopt_handler_offset = _stub_offset + offsets->value(CodeOffsets::Deopt); + _deopt_handler_entry_offset = _stub_offset + offsets->value(CodeOffsets::Deopt); if (offsets->value(CodeOffsets::DeoptMH) != -1) { - _deopt_mh_handler_offset = _stub_offset + offsets->value(CodeOffsets::DeoptMH); + _deopt_mh_handler_entry_offset = _stub_offset + offsets->value(CodeOffsets::DeoptMH); } else { - _deopt_mh_handler_offset = -1; + _deopt_mh_handler_entry_offset = -1; } } if (offsets->value(CodeOffsets::UnwindHandler) != -1) { @@ -2714,7 +2714,7 @@ void nmethod::copy_scopes_pcs(PcDesc* pcs, int count) { break; } } - assert(has_method_handle_invokes() == (_deopt_mh_handler_offset != -1), "must have deopt mh handler"); + assert(has_method_handle_invokes() == (_deopt_mh_handler_entry_offset != -1), "must have deopt mh handler"); int size = count * sizeof(PcDesc); assert(scopes_pcs_size() >= size, "oob"); @@ -3725,12 +3725,12 @@ const char* nmethod::nmethod_section_label(address pos) const { if (pos == code_begin()) label = "[Instructions begin]"; if (pos == entry_point()) label = "[Entry Point]"; if (pos == verified_entry_point()) label = "[Verified Entry Point]"; - if (has_method_handle_invokes() && (pos == deopt_mh_handler_begin())) label = "[Deopt MH Handler Code]"; + if (has_method_handle_invokes() && (pos == deopt_mh_handler_entry())) label = "[Deopt MH Handler Entry Point]"; if (pos == consts_begin() && pos != insts_begin()) label = "[Constants]"; // Check stub_code before checking exception_handler or deopt_handler. if (pos == this->stub_begin()) label = "[Stub Code]"; if (JVMCI_ONLY(_exception_offset >= 0 &&) pos == exception_begin()) label = "[Exception Handler]"; - if (JVMCI_ONLY(_deopt_handler_offset != -1 &&) pos == deopt_handler_begin()) label = "[Deopt Handler Code]"; + if (JVMCI_ONLY(_deopt_handler_entry_offset != -1 &&) pos == deopt_handler_entry()) label = "[Deopt Handler Entry Point]"; return label; } diff --git a/src/hotspot/share/code/nmethod.hpp b/src/hotspot/share/code/nmethod.hpp index 301454b659159..74faecc8ef24c 100644 --- a/src/hotspot/share/code/nmethod.hpp +++ b/src/hotspot/share/code/nmethod.hpp @@ -226,10 +226,10 @@ class nmethod : public CodeBlob { int _exception_offset; // All deoptee's will resume execution at this location described by // this offset. - int _deopt_handler_offset; + int _deopt_handler_entry_offset; // All deoptee's at a MethodHandle call site will resume execution // at this location described by this offset. - int _deopt_mh_handler_offset; + int _deopt_mh_handler_entry_offset; // Offset (from insts_end) of the unwind handler if it exists int16_t _unwind_handler_offset; // Number of arguments passed on the stack @@ -606,8 +606,8 @@ class nmethod : public CodeBlob { address stub_begin () const { return header_begin() + _stub_offset ; } address stub_end () const { return code_end() ; } address exception_begin () const { return header_begin() + _exception_offset ; } - address deopt_handler_begin () const { return header_begin() + _deopt_handler_offset ; } - address deopt_mh_handler_begin() const { return header_begin() + _deopt_mh_handler_offset ; } + address deopt_handler_entry () const { return header_begin() + _deopt_handler_entry_offset ; } + address deopt_mh_handler_entry() const { return header_begin() + _deopt_mh_handler_entry_offset ; } address unwind_handler_begin () const { return _unwind_handler_offset != -1 ? (insts_end() - _unwind_handler_offset) : nullptr; } oop* oops_begin () const { return (oop*) data_begin(); } oop* oops_end () const { return (oop*) data_end(); } diff --git a/src/hotspot/share/code/nmethod.inline.hpp b/src/hotspot/share/code/nmethod.inline.hpp index 1e556b6825099..b56ec80c2ba8e 100644 --- a/src/hotspot/share/code/nmethod.inline.hpp +++ b/src/hotspot/share/code/nmethod.inline.hpp @@ -34,11 +34,11 @@ inline bool nmethod::is_deopt_pc(address pc) { return is_deopt_entry(pc) || is_deopt_mh_entry(pc); } inline bool nmethod::is_deopt_entry(address pc) { - return pc == deopt_handler_begin(); + return pc == deopt_handler_entry(); } inline bool nmethod::is_deopt_mh_entry(address pc) { - return pc == deopt_mh_handler_begin(); + return pc == deopt_mh_handler_entry(); } // class ExceptionCache methods diff --git a/src/hotspot/share/runtime/frame.cpp b/src/hotspot/share/runtime/frame.cpp index ba9aa3eb29ade..94d62094ee185 100644 --- a/src/hotspot/share/runtime/frame.cpp +++ b/src/hotspot/share/runtime/frame.cpp @@ -207,9 +207,9 @@ address frame::raw_pc() const { nmethod* nm = cb()->as_nmethod_or_null(); assert(nm != nullptr, "only nmethod is expected here"); if (nm->is_method_handle_return(pc())) - return nm->deopt_mh_handler_begin() - pc_return_offset; + return nm->deopt_mh_handler_entry() - pc_return_offset; else - return nm->deopt_handler_begin() - pc_return_offset; + return nm->deopt_handler_entry() - pc_return_offset; } else { return (pc() - pc_return_offset); } @@ -359,8 +359,8 @@ void frame::deoptimize(JavaThread* thread) { // If the call site is a MethodHandle call site use the MH deopt handler. nmethod* nm = _cb->as_nmethod(); address deopt = nm->is_method_handle_return(pc()) ? - nm->deopt_mh_handler_begin() : - nm->deopt_handler_begin(); + nm->deopt_mh_handler_entry() : + nm->deopt_handler_entry(); NativePostCallNop* inst = nativePostCallNop_at(pc()); diff --git a/src/hotspot/share/runtime/vmStructs.cpp b/src/hotspot/share/runtime/vmStructs.cpp index 6fc16f9b04520..14658bd266e15 100644 --- a/src/hotspot/share/runtime/vmStructs.cpp +++ b/src/hotspot/share/runtime/vmStructs.cpp @@ -541,8 +541,8 @@ nonstatic_field(nmethod, _osr_link, nmethod*) \ nonstatic_field(nmethod, _state, volatile signed char) \ nonstatic_field(nmethod, _exception_offset, int) \ - nonstatic_field(nmethod, _deopt_handler_offset, int) \ - nonstatic_field(nmethod, _deopt_mh_handler_offset, int) \ + nonstatic_field(nmethod, _deopt_handler_entry_offset, int) \ + nonstatic_field(nmethod, _deopt_mh_handler_entry_offset, int) \ nonstatic_field(nmethod, _orig_pc_offset, int) \ nonstatic_field(nmethod, _stub_offset, int) \ nonstatic_field(nmethod, _scopes_pcs_offset, int) \ diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/code/NMethod.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/code/NMethod.java index a3228765e910d..1d5bd8c96c95c 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/code/NMethod.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/code/NMethod.java @@ -85,8 +85,8 @@ private static void initialize(TypeDataBase db) { immutableDataField = type.getAddressField("_immutable_data"); immutableDataSizeField = type.getCIntegerField("_immutable_data_size"); exceptionOffsetField = type.getCIntegerField("_exception_offset"); - deoptHandlerOffsetField = type.getCIntegerField("_deopt_handler_offset"); - deoptMhHandlerOffsetField = type.getCIntegerField("_deopt_mh_handler_offset"); + deoptHandlerOffsetField = type.getCIntegerField("_deopt_handler_entry_offset"); + deoptMhHandlerOffsetField = type.getCIntegerField("_deopt_mh_handler_entry_offset"); origPCOffsetField = type.getCIntegerField("_orig_pc_offset"); stubOffsetField = type.getCIntegerField("_stub_offset"); scopesPCsOffsetField = type.getCIntegerField("_scopes_pcs_offset"); @@ -124,8 +124,8 @@ public Method getMethod() { public Address instsBegin() { return codeBegin(); } public Address instsEnd() { return headerBegin().addOffsetTo(getStubOffset()); } public Address exceptionBegin() { return headerBegin().addOffsetTo(getExceptionOffset()); } - public Address deoptHandlerBegin() { return headerBegin().addOffsetTo(getDeoptHandlerOffset()); } - public Address deoptMhHandlerBegin() { return headerBegin().addOffsetTo(getDeoptMhHandlerOffset()); } + public Address deoptHandlerEntry() { return headerBegin().addOffsetTo(getDeoptHandlerEntryOffset()); } + public Address deoptMhHandlerEntry() { return headerBegin().addOffsetTo(getDeoptMhHandlerEntryOffset()); } public Address stubBegin() { return headerBegin().addOffsetTo(getStubOffset()); } public Address stubEnd() { return dataBegin(); } public Address oopsBegin() { return dataBegin(); } @@ -264,8 +264,8 @@ public boolean isMethodHandleReturn(Address returnPc) { // Deopt // Return true is the PC is one would expect if the frame is being deopted. public boolean isDeoptPc (Address pc) { return isDeoptEntry(pc) || isDeoptMhEntry(pc); } - public boolean isDeoptEntry (Address pc) { return pc == deoptHandlerBegin(); } - public boolean isDeoptMhEntry (Address pc) { return pc == deoptMhHandlerBegin(); } + public boolean isDeoptEntry (Address pc) { return pc == deoptHandlerEntry(); } + public boolean isDeoptMhEntry (Address pc) { return pc == deoptMhHandlerEntry(); } /** Tells whether frames described by this nmethod can be deoptimized. Note: native wrappers cannot be deoptimized. */ @@ -493,8 +493,8 @@ public String getName() { private int getEntryBCI() { return (int) entryBCIField .getValue(addr); } private int getExceptionOffset() { return (int) exceptionOffsetField .getValue(addr); } - private int getDeoptHandlerOffset() { return (int) deoptHandlerOffsetField .getValue(addr); } - private int getDeoptMhHandlerOffset() { return (int) deoptMhHandlerOffsetField.getValue(addr); } + private int getDeoptHandlerEntryOffset() { return (int) deoptHandlerOffsetField .getValue(addr); } + private int getDeoptMhHandlerEntryOffset() { return (int) deoptMhHandlerOffsetField.getValue(addr); } private int getStubOffset() { return (int) stubOffsetField .getValue(addr); } private int getScopesDataOffset() { return (int) scopesDataOffsetField .getValue(addr); } private int getScopesPCsOffset() { return (int) scopesPCsOffsetField .getValue(addr); } diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/aarch64/AARCH64Frame.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/aarch64/AARCH64Frame.java index dca5f2efa3bb6..b63bee40ed5a1 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/aarch64/AARCH64Frame.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/aarch64/AARCH64Frame.java @@ -112,7 +112,7 @@ private void adjustForDeopt() { CodeBlob cb = VM.getVM().getCodeCache().findBlob(pc); if (cb != null && cb.isJavaMethod()) { NMethod nm = (NMethod) cb; - if (pc.equals(nm.deoptHandlerBegin())) { + if (pc.equals(nm.deoptHandlerEntry())) { if (Assert.ASSERTS_ENABLED) { Assert.that(this.getUnextendedSP() != null, "null SP in Java frame"); } diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/ppc64/PPC64Frame.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/ppc64/PPC64Frame.java index a47a632c286ef..1927ccbf5fcd8 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/ppc64/PPC64Frame.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/ppc64/PPC64Frame.java @@ -91,7 +91,7 @@ private void adjustForDeopt() { CodeBlob cb = VM.getVM().getCodeCache().findBlob(pc); if (cb != null && cb.isJavaMethod()) { NMethod nm = (NMethod) cb; - if (pc.equals(nm.deoptHandlerBegin())) { + if (pc.equals(nm.deoptHandlerEntry())) { if (Assert.ASSERTS_ENABLED) { Assert.that(this.getUnextendedSP() != null, "null SP in Java frame"); } diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/riscv64/RISCV64Frame.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/riscv64/RISCV64Frame.java index 948a3008016ea..c4f6a91136b1e 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/riscv64/RISCV64Frame.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/riscv64/RISCV64Frame.java @@ -107,7 +107,7 @@ private void adjustForDeopt() { CodeBlob cb = VM.getVM().getCodeCache().findBlob(pc); if (cb != null && cb.isJavaMethod()) { NMethod nm = (NMethod) cb; - if (pc.equals(nm.deoptHandlerBegin())) { + if (pc.equals(nm.deoptHandlerEntry())) { if (Assert.ASSERTS_ENABLED) { Assert.that(this.getUnextendedSP() != null, "null SP in Java frame"); } diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/x86/X86Frame.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/x86/X86Frame.java index 169ecea156591..f62545b2e9db5 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/x86/X86Frame.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/runtime/x86/X86Frame.java @@ -108,7 +108,7 @@ private void adjustForDeopt() { CodeBlob cb = VM.getVM().getCodeCache().findBlob(pc); if (cb != null && cb.isJavaMethod()) { NMethod nm = (NMethod) cb; - if (pc.equals(nm.deoptHandlerBegin())) { + if (pc.equals(nm.deoptHandlerEntry())) { if (Assert.ASSERTS_ENABLED) { Assert.that(this.getUnextendedSP() != null, "null SP in Java frame"); } From fe7db04570a112b3ae1627174c6a682bc622edd5 Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Fri, 26 Sep 2025 13:06:36 +0100 Subject: [PATCH 06/16] The patch is contributed by @TheRealMDoerr --- src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp | 10 +++++++--- src/hotspot/cpu/ppc/ppc.ad | 1 - src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp | 1 - 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index b548444bb4875..585fc26fdf443 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -267,13 +267,17 @@ int LIR_Assembler::emit_deopt_handler() { return -1; } - int offset = code_offset(); + Label start; + + __ bind(start); __ bl64_patchable(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type); + int entry_point = __ offset(); + __ b(start); - guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); + guarantee(code_offset() - entry_point <= deopt_handler_size(), "overflow"); __ end_a_stub(); - return offset; + return entry_point; } diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index 85bab0ebc61ed..4d22d37d0c281 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -1,7 +1,6 @@ // // Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved. // Copyright (c) 2012, 2025 SAP SE. All rights reserved. -// Copyright 2025 Arm Limited and/or its affiliates. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp index 69f2b9c94c95c..e988d653c879b 100644 --- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp +++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp @@ -2974,7 +2974,6 @@ void SharedRuntime::generate_deopt_blob() { // The return_pc has been stored in the frame of the deoptee and // will replace the address of the deopt_handler in the call // to Deoptimization::fetch_unroll_info below. - const int return_pc_adjustment_no_exception = -MacroAssembler::bl64_patchable_size; // Push the "unpack frame" // Save everything in sight. From 5107d025254eb0017d9f0ffcc53c49893c1c0c13 Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Fri, 26 Sep 2025 13:07:40 +0100 Subject: [PATCH 07/16] Address review comments --- .../cpu/aarch64/c1_LIRAssembler_aarch64.cpp | 10 ++++++++-- .../cpu/aarch64/c1_LIRAssembler_aarch64.hpp | 2 +- src/hotspot/cpu/arm/arm.ad | 12 ++++++++---- src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp | 10 ++++++++-- src/hotspot/cpu/arm/c1_LIRAssembler_arm.hpp | 2 +- src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp | 3 ++- src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp | 2 +- src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp | 12 +++++++++--- src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp | 2 +- src/hotspot/cpu/riscv/riscv.ad | 8 +++----- src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp | 15 +++++++++++++-- src/hotspot/cpu/s390/s390.ad | 1 - src/hotspot/cpu/s390/sharedRuntime_s390.cpp | 6 +----- src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 14 ++++++++++---- src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp | 2 +- src/hotspot/cpu/x86/x86.ad | 4 ---- 16 files changed, 67 insertions(+), 38 deletions(-) diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index 31f1e97002abf..1e6047db3e69a 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -453,12 +453,18 @@ int LIR_Assembler::emit_deopt_handler() { int offset = code_offset(); - __ adr(lr, pc()); + Label start; + __ bind(start); + __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); + + int entry_point = __ offset(); + __ b(start); + guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); __ end_a_stub(); - return offset; + return entry_point; } void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) { diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp index 12b941fc4f7e8..729cd2827b72a 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp @@ -71,7 +71,7 @@ friend class ArrayCopyStub; // CompiledDirectCall::to_trampoline_stub_size() _call_stub_size = 13 * NativeInstruction::instruction_size, _exception_handler_size = DEBUG_ONLY(1*K) NOT_DEBUG(175), - _deopt_handler_size = 7 * NativeInstruction::instruction_size + _deopt_handler_size = 4 * NativeInstruction::instruction_size }; public: diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad index f1bc9dc5df104..5e315671ceed2 100644 --- a/src/hotspot/cpu/arm/arm.ad +++ b/src/hotspot/cpu/arm/arm.ad @@ -1,6 +1,5 @@ // // Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved. -// Copyright 2025 Arm Limited and/or its affiliates. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -109,7 +108,7 @@ class HandlerImpl { static int emit_deopt_handler(C2_MacroAssembler* masm); static uint size_deopt_handler() { - return ( 4 * 4 ); + return ( 9 * 4 ); } }; @@ -885,11 +884,16 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { Label start; __ bind(start); - // Store address of the entry point - __ push(PC); __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg); int entry_point = __ offset(); + address deopt_pc = __ pc(); + // Preserve R0 and reserve space for the address of the entry point + __ push(RegisterSet(R0) | RegisterSet(R1)); + // Store the entry point address + __ mov_relative_address(R0, deopt_pc); + __ str(R0, Address(SP, wordSize)); + __ pop(R0); // restore R0 __ b(start); assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow"); diff --git a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp index 6a859a5875bbd..350c20b91869f 100644 --- a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp +++ b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp @@ -272,14 +272,20 @@ int LIR_Assembler::emit_deopt_handler() { int offset = code_offset(); + Label start; + __ bind(start); + + __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg); + + int entry_point = __ offset(); __ mov_relative_address(LR, __ pc()); __ push(LR); // stub expects LR to be saved - __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg); + __ b(start); assert(code_offset() - offset <= deopt_handler_size(), "overflow"); __ end_a_stub(); - return offset; + return entry_point; } diff --git a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.hpp b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.hpp index 77d1353268584..615d2f188ff61 100644 --- a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.hpp +++ b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.hpp @@ -54,7 +54,7 @@ enum { _call_stub_size = 16, _exception_handler_size = PRODUCT_ONLY(68) NOT_PRODUCT(68+60), - _deopt_handler_size = 16 + _deopt_handler_size = 20 }; public: diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index 585fc26fdf443..66be32b7c34b5 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -267,6 +267,7 @@ int LIR_Assembler::emit_deopt_handler() { return -1; } + int offset = code_offset(); Label start; __ bind(start); @@ -274,7 +275,7 @@ int LIR_Assembler::emit_deopt_handler() { int entry_point = __ offset(); __ b(start); - guarantee(code_offset() - entry_point <= deopt_handler_size(), "overflow"); + guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); __ end_a_stub(); return entry_point; diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp index e4de2eb5c4683..6a2f62648501c 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp @@ -63,7 +63,7 @@ enum { _static_call_stub_size = 4 * BytesPerInstWord + MacroAssembler::b64_patchable_size, // or smaller _call_stub_size = _static_call_stub_size + MacroAssembler::trampoline_stub_size, // or smaller _exception_handler_size = MacroAssembler::b64_patchable_size, // or smaller - _deopt_handler_size = MacroAssembler::bl64_patchable_size + _deopt_handler_size = MacroAssembler::bl64_patchable_size + BytesPerInstWord }; // '_static_call_stub_size' is only used on ppc (see LIR_Assembler::emit_static_call_stub() diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp index 81b829bde9a43..9c286129d3687 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp @@ -381,12 +381,18 @@ int LIR_Assembler::emit_deopt_handler() { int offset = code_offset(); - __ auipc(ra, 0); - __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); + Label start; + __ bind(start); + + __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); + + int entry_point = __ offset(); + __ j(start); + guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); __ end_a_stub(); - return offset; + return entry_point; } void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) { diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp index e4efb2c171dcb..c617c9ac6efd8 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp @@ -72,7 +72,7 @@ friend class ArrayCopyStub; // See emit_exception_handler for detail _exception_handler_size = DEBUG_ONLY(256) NOT_DEBUG(32), // or smaller // See emit_deopt_handler for detail - // auipc (1) + far_jump (2) + // far_jump (2) + j (1) _deopt_handler_size = 1 * MacroAssembler::instruction_size + 2 * MacroAssembler::instruction_size }; diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index b7088d99e22b7..0bb2696754235 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -2,7 +2,6 @@ // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved. // Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. // Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved. -// Copyright 2025 Arm Limited and/or its affiliates. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -1053,8 +1052,8 @@ class HandlerImpl { static int emit_deopt_handler(C2_MacroAssembler* masm); static uint size_deopt_handler() { - // count auipc + far branch + jal - return 2 * NativeInstruction::instruction_size + MacroAssembler::far_branch_size(); + // count far call + j + return NativeInstruction::instruction_size + MacroAssembler::far_branch_size(); } }; @@ -1819,7 +1818,6 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) Label start; __ bind(start); - __ auipc(ra, 0); __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); int entry_point = __ offset(); @@ -1827,7 +1825,7 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow"); __ end_a_stub(); - return offset; + return entry_point; } // REQUIRED MATCHER CODE diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp index 4d90f31326735..1c3e0cfb822e7 100644 --- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp @@ -276,14 +276,25 @@ int LIR_Assembler::emit_deopt_handler() { // Not enough space left for the handler. bailout("deopt handler overflow"); return -1; - } int offset = code_offset(); + } + + int offset = code_offset(); + + Label start; + __ bind(start); + // Size must be constant (see HandlerImpl::emit_deopt_handler). __ load_const(Z_R1_scratch, SharedRuntime::deopt_blob()->unpack()); __ call(Z_R1_scratch); + + int entry_point = __ offset(); + + __ z_bru(start); + guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); __ end_a_stub(); - return offset; + return entry_point; } void LIR_Assembler::jobject2reg(jobject o, Register reg) { diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index 91c97c9f0f966..fb9eae191ef3c 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -1,7 +1,6 @@ // // Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved. // Copyright (c) 2017, 2024 SAP SE. All rights reserved. -// Copyright 2025 Arm Limited and/or its affiliates. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp index 90f54b60611ea..36fe8c9b1891d 100644 --- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp +++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp @@ -2558,14 +2558,10 @@ void SharedRuntime::generate_deopt_blob() { // Normal entry (non-exception case) // // We have been called from the deopt handler of the deoptee. - // Z_R14 points behind the call in the deopt handler. We adjust - // it such that it points to the start of the deopt handler. + // Z_R14 points to the entry point of the deopt handler. // The return_pc has been stored in the frame of the deoptee and // will replace the address of the deopt_handler in the call // to Deoptimization::fetch_unroll_info below. - // The (int) cast is necessary, because -((unsigned int)14) - // is an unsigned int. - __ add2reg(Z_R14, -(int)NativeCall::max_instruction_size()); const Register exec_mode_reg = Z_tmp_1; diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index 0176ff967ce4b..eec12617c6abe 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -454,14 +454,20 @@ int LIR_Assembler::emit_deopt_handler() { } int offset = code_offset(); - InternalAddress here(__ pc()); - __ pushptr(here.addr(), rscratch1); - __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); + Label start; + __ bind(start); + + __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); + + int entry_point = __ offset(); + + __ jmp(start); + guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); __ end_a_stub(); - return offset; + return entry_point; } void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) { diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp index 8524dc90276f0..7a8fbc75ba71c 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp @@ -48,7 +48,7 @@ enum { _call_stub_size = 28, _exception_handler_size = DEBUG_ONLY(1*K) NOT_DEBUG(175), - _deopt_handler_size = 17 + _deopt_handler_size = 10 }; public: diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index b1f2bb0f4e5a3..62c61bf73314e 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -1,6 +1,5 @@ // // Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved. -// Copyright 2025 Arm Limited and/or its affiliates. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -1243,12 +1242,9 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { } int offset = __ offset(); - address the_pc = (address) __ pc(); Label start; __ bind(start); - // push a "the_pc" on the stack without destroying any registers - // as they all may be live. __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); int entry_point = __ offset(); From 18e36eca767a7dc25a0c86386186028682a4cedc Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Mon, 29 Sep 2025 12:13:16 +0100 Subject: [PATCH 08/16] Address review comments --- src/hotspot/cpu/aarch64/aarch64.ad | 4 ++-- src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp | 4 ++-- src/hotspot/cpu/arm/arm.ad | 4 ++-- src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp | 4 ++-- src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp | 4 ++-- src/hotspot/cpu/ppc/ppc.ad | 4 ++-- src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp | 4 ++-- src/hotspot/cpu/riscv/riscv.ad | 4 ++-- src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp | 4 ++-- src/hotspot/cpu/s390/s390.ad | 4 ++-- src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 4 ++-- src/hotspot/cpu/x86/x86.ad | 4 ++-- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index a837fa43ac889..6b2cdb4d36e64 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -2273,12 +2273,12 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) __ bind(start); __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); - int entry_point = __ offset(); + int entry_offset = __ offset(); __ b(start); assert(__ offset() - offset == (int) size_deopt_handler(), "overflow"); __ end_a_stub(); - return entry_point; + return entry_offset; } // REQUIRED MATCHER CODE diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index 1e6047db3e69a..0a0e96cf97115 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -458,13 +458,13 @@ int LIR_Assembler::emit_deopt_handler() { __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); - int entry_point = __ offset(); + int entry_offset = __ offset(); __ b(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); __ end_a_stub(); - return entry_point; + return entry_offset; } void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) { diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad index 5e315671ceed2..ec4d71e96a053 100644 --- a/src/hotspot/cpu/arm/arm.ad +++ b/src/hotspot/cpu/arm/arm.ad @@ -886,7 +886,7 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg); - int entry_point = __ offset(); + int entry_offset = __ offset(); address deopt_pc = __ pc(); // Preserve R0 and reserve space for the address of the entry point __ push(RegisterSet(R0) | RegisterSet(R1)); @@ -899,7 +899,7 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow"); __ end_a_stub(); - return entry_point; + return entry_offset; } bool Matcher::match_rule_supported(int opcode) { diff --git a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp index 350c20b91869f..1d843abd87b71 100644 --- a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp +++ b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp @@ -277,7 +277,7 @@ int LIR_Assembler::emit_deopt_handler() { __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg); - int entry_point = __ offset(); + int entry_offset = __ offset(); __ mov_relative_address(LR, __ pc()); __ push(LR); // stub expects LR to be saved __ b(start); @@ -285,7 +285,7 @@ int LIR_Assembler::emit_deopt_handler() { assert(code_offset() - offset <= deopt_handler_size(), "overflow"); __ end_a_stub(); - return entry_point; + return entry_offset; } diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index 66be32b7c34b5..e363bf41df538 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -272,13 +272,13 @@ int LIR_Assembler::emit_deopt_handler() { __ bind(start); __ bl64_patchable(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type); - int entry_point = __ offset(); + int entry_offset = __ offset(); __ b(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); __ end_a_stub(); - return entry_point; + return entry_offset; } diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index 4d22d37d0c281..9422fb51d5b13 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -2125,14 +2125,14 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { __ bl64_patchable((address)SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type); - int entry_point = __ offset(); + int entry_offset = __ offset(); __ b(start); assert(__ offset() - offset == (int) size_deopt_handler(), "must be fixed size"); __ end_a_stub(); - return entry_point; + return entry_offset; } //============================================================================= diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp index 9c286129d3687..e6d74c46c800f 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp @@ -386,13 +386,13 @@ int LIR_Assembler::emit_deopt_handler() { __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); - int entry_point = __ offset(); + int entry_offset = __ offset(); __ j(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); __ end_a_stub(); - return entry_point; + return entry_offset; } void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) { diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index 0bb2696754235..eb1ae4f86ac85 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -1820,12 +1820,12 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); - int entry_point = __ offset(); + int entry_offset = __ offset(); __ j(start); assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow"); __ end_a_stub(); - return entry_point; + return entry_offset; } // REQUIRED MATCHER CODE diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp index 1c3e0cfb822e7..e69b4a6b7f294 100644 --- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp @@ -287,14 +287,14 @@ int LIR_Assembler::emit_deopt_handler() { __ load_const(Z_R1_scratch, SharedRuntime::deopt_blob()->unpack()); __ call(Z_R1_scratch); - int entry_point = __ offset(); + int entry_offset = __ offset(); __ z_bru(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); __ end_a_stub(); - return entry_point; + return entry_offset; } void LIR_Assembler::jobject2reg(jobject o, Register reg) { diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index fb9eae191ef3c..cd8991996f0a7 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -1686,14 +1686,14 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { __ load_const(Z_R1, SharedRuntime::deopt_blob()->unpack()); __ call(Z_R1); - int entry_point = __ offset(); + int entry_offset = __ offset(); __ z_bru(start); assert(__ offset() - offset == (int) size_deopt_handler(), "must be fixed size"); __ end_a_stub(); - return entry_point; + return entry_offset; } //============================================================================= diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index eec12617c6abe..8e683daff17ca 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -460,14 +460,14 @@ int LIR_Assembler::emit_deopt_handler() { __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); - int entry_point = __ offset(); + int entry_offset = __ offset(); __ jmp(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); __ end_a_stub(); - return entry_point; + return entry_offset; } void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) { diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index 62c61bf73314e..ecc344616c34e 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -1247,13 +1247,13 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); - int entry_point = __ offset(); + int entry_offset = __ offset(); __ jmp(start); assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow %d", (__ offset() - offset)); __ end_a_stub(); - return entry_point; + return entry_offset; } static Assembler::Width widthForType(BasicType bt) { From 6e2411080eb6a0f6d8998fb7da5965fa204f720e Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Mon, 13 Oct 2025 12:26:30 +0100 Subject: [PATCH 09/16] Address review comments --- src/hotspot/share/ci/ciEnv.cpp | 1 - src/hotspot/share/code/nmethod.cpp | 1 - src/hotspot/share/runtime/sharedRuntime.cpp | 1 - 3 files changed, 3 deletions(-) diff --git a/src/hotspot/share/ci/ciEnv.cpp b/src/hotspot/share/ci/ciEnv.cpp index e30b58f4d0bb8..eb45a3a493ab4 100644 --- a/src/hotspot/share/ci/ciEnv.cpp +++ b/src/hotspot/share/ci/ciEnv.cpp @@ -1,6 +1,5 @@ /* * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved. - * Copyright 2025 Arm Limited and/or its affiliates. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index fa62d3b443cbe..a7faf0b94a4d9 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -1,6 +1,5 @@ /* * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. - * Copyright 2025 Arm Limited and/or its affiliates. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 7de7dc471e30d..82e3f88e5deb0 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -1,6 +1,5 @@ /* * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. - * Copyright 2025 Arm Limited and/or its affiliates. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From e9c817c8af31112093626e6104ee58b87acc27de Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Tue, 28 Oct 2025 22:12:56 +0000 Subject: [PATCH 10/16] Test for deoptimization in virtual threads Change-Id: I9ef51b426d34e9b44a3891f6a45307232f900e5a --- src/hotspot/share/runtime/deoptimization.cpp | 3 + .../jtreg/runtime/vthread/Deoptimization.java | 159 ++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 test/hotspot/jtreg/runtime/vthread/Deoptimization.java diff --git a/src/hotspot/share/runtime/deoptimization.cpp b/src/hotspot/share/runtime/deoptimization.cpp index 853c6554022f9..546f415d7f9a0 100644 --- a/src/hotspot/share/runtime/deoptimization.cpp +++ b/src/hotspot/share/runtime/deoptimization.cpp @@ -499,6 +499,9 @@ Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread RegisterMap::WalkContinuation::skip); // Now get the deoptee with a valid map frame deoptee = stub_frame.sender(&map); + if (exec_mode == Unpack_deopt) { + assert(deoptee.is_deoptimized_frame(), "frame is not marked for deoptimization"); + } // Set the deoptee nmethod assert(current->deopt_compiled_method() == nullptr, "Pending deopt!"); nmethod* nm = deoptee.cb()->as_nmethod_or_null(); diff --git a/test/hotspot/jtreg/runtime/vthread/Deoptimization.java b/test/hotspot/jtreg/runtime/vthread/Deoptimization.java new file mode 100644 index 0000000000000..050848c3a723a --- /dev/null +++ b/test/hotspot/jtreg/runtime/vthread/Deoptimization.java @@ -0,0 +1,159 @@ +/* + * Copyright 2025 Arm Limited and/or its affiliates. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test id=vthread-deopt-c1 + * @summary Deoptimization test for virtual threads (C1) + * @requires vm.continuations + * @requires vm.compiler1.enabled + * @requires vm.opt.TieredStopAtLevel != 0 + * @library /test/lib + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. + * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI + * -XX:-BackgroundCompilation + * -XX:TieredStopAtLevel=1 + * Deoptimization + */ + +/** + * @test id=vthread-deopt-c2 + * @summary Deoptimization test for virtual threads (C2) + * @requires vm.continuations + * @requires vm.compiler2.enabled + * @library /test/lib + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. + * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI + * -XX:-BackgroundCompilation + * -XX:-TieredCompilation + * Deoptimization + */ + +import java.lang.reflect.Method; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; +import java.util.Objects; +import jdk.test.whitebox.WhiteBox; + +public class Deoptimization { + static final WhiteBox white_box = WhiteBox.getWhiteBox(); + + static class TestTask implements Runnable { + CyclicBarrier start_barrier = null; + AtomicInteger completed_number = new AtomicInteger(0); + + public void reset(int barrier_parties) { + start_barrier = new CyclicBarrier(barrier_parties); + completed_number.set(0); + } + + public int getNumberWaiting() { + return start_barrier.getNumberWaiting(); + } + + public int getNumberCompleted() { + return completed_number.get(); + } + + public void await() throws BrokenBarrierException, InterruptedException { + start_barrier.await(); + } + + public void run() { + try { + await(); + } catch(BrokenBarrierException e) { + return; + } catch(InterruptedException e) { + return; + } + + completed_number.getAndIncrement(); + } + } + + static void test(TestTask task, Method method, int vthreads_num) throws Exception { + task.reset(vthreads_num + 1 /* 1 for the main thread */); + + Thread[] vthreads = new Thread[vthreads_num]; + for (int i = 0; i < vthreads_num; i++) { + vthreads[i] = Thread.startVirtualThread(task); + } + + while (task.getNumberWaiting() != vthreads_num) { + Thread.onSpinWait(); + } + + if (method != null) { + if (!white_box.isMethodCompiled(method, false)) { + throw new Error("Unexpectedly, it is not compiled."); + } + + white_box.deoptimizeMethod(method); + + if (white_box.isMethodCompiled(method, false)) { + throw new Error("Unexpectedly, it is compiled."); + } + } + + task.await(); + + for (int i = 0; i < vthreads_num; i++) { + vthreads[i].join(); + } + + if (task.getNumberCompleted() != vthreads_num) { + throw new Error("Some threads didn't reach completion"); + } + } + + static int getIntegerOption(String option_name) { + Object option_object = white_box.getVMFlag(option_name); + String option_string = Objects.toString(option_object); + return Integer.parseInt(option_string); + } + + public static void main(String[] args) throws Exception { + int tiered_stop_at_level = getIntegerOption("TieredStopAtLevel"); + + Method method_run = TestTask.class.getMethod("run"); + white_box.testSetDontInlineMethod(method_run, true); + + Method method_await = TestTask.class.getMethod("await"); + white_box.testSetDontInlineMethod(method_await, true); + + TestTask task = new TestTask(); + + // Warm-up + test(task, null, 2); + + white_box.enqueueMethodForCompilation(method_run, tiered_stop_at_level); + + // Deoptimization test + test(task, method_run, 10000); + } +} From de9bec59e0452005191eb429a77ae2e9f6f0cde7 Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Tue, 28 Oct 2025 22:16:30 +0000 Subject: [PATCH 11/16] Check for NOP and MOVK separately in NativePostCallNop --- .../cpu/aarch64/nativeInst_aarch64.cpp | 6 ----- .../cpu/aarch64/nativeInst_aarch64.hpp | 22 ++++++++++++++----- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp index 5a7fececafaaf..f2003dd9b5576 100644 --- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp @@ -394,12 +394,6 @@ void NativePostCallNop::make_deopt() { NativeDeoptInstruction::insert(addr_at(0)); } -#ifdef ASSERT -static bool is_movk_to_zr(uint32_t insn) { - return ((insn & 0xffe0001f) == 0xf280001f); -} -#endif - bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) { if (((oopmap_slot & 0xff) != oopmap_slot) || ((cb_offset & 0xffffff) != cb_offset)) { return false; // cannot encode diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp index df5d97c23764f..b7444347bf130 100644 --- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp @@ -526,14 +526,24 @@ inline NativeLdSt* NativeLdSt_at(address addr) { // can store an offset from the initial nop to the nmethod. class NativePostCallNop: public NativeInstruction { +private: + static bool is_movk_to_zr(uint32_t insn) { + return ((insn & 0xffe0001f) == 0xf280001f); + } + public: bool check() const { - uint64_t insns = *(uint64_t*)addr_at(0); - // Check for two instructions: nop; movk zr, xx - // These instructions only ever appear together in a post-call - // NOP, so it's unnecessary to check that the third instruction is - // a MOVK as well. - return (insns & 0xffe0001fffffffff) == 0xf280001fd503201f; + // Check the first instruction is NOP. + if (is_nop()) { + uint32_t insn = *(uint32_t*)addr_at(4); + // Check next instruction is MOVK zr, xx. + // These instructions only ever appear together in a post-call + // NOP, so it's unnecessary to check that the third instruction is + // a MOVK as well. + return is_movk_to_zr(insn); + } + + return false; } bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { From 462dbf62feb6053e01b0112872422af18f32f83f Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Tue, 28 Oct 2025 22:17:39 +0000 Subject: [PATCH 12/16] Address review comments and fix a mistype --- src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp | 2 +- src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index d31e2ab72d54f..2498a646e31f6 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -452,7 +452,7 @@ int LIR_Assembler::emit_deopt_handler() { Label start; __ bind(start); - __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); + __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); int entry_offset = __ offset(); __ b(start); diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp index c617c9ac6efd8..ed2ab0c4861bc 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp @@ -72,7 +72,7 @@ friend class ArrayCopyStub; // See emit_exception_handler for detail _exception_handler_size = DEBUG_ONLY(256) NOT_DEBUG(32), // or smaller // See emit_deopt_handler for detail - // far_jump (2) + j (1) + // far_call (2) + j (1) _deopt_handler_size = 1 * MacroAssembler::instruction_size + 2 * MacroAssembler::instruction_size }; From 359c2f185c7add1cac98523f4325a7896e8bd3e0 Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Mon, 3 Nov 2025 18:10:32 +0000 Subject: [PATCH 13/16] Rename deoptHandlerOffsetField -> deoptHandlerEntryOffsetField --- .../share/classes/sun/jvm/hotspot/code/NMethod.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/code/NMethod.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/code/NMethod.java index c5075e65e5b51..f935c56b536ee 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/code/NMethod.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/code/NMethod.java @@ -48,7 +48,7 @@ public class NMethod extends CodeBlob { /** Offsets for different nmethod parts */ private static CIntegerField exceptionOffsetField; - private static CIntegerField deoptHandlerOffsetField; + private static CIntegerField deoptHandlerEntryOffsetField; private static CIntegerField origPCOffsetField; private static CIntegerField stubOffsetField; private static CIntField handlerTableOffsetField; @@ -86,7 +86,7 @@ private static void initialize(TypeDataBase db) { immutableDataField = type.getAddressField("_immutable_data"); immutableDataSizeField = type.getCIntegerField("_immutable_data_size"); exceptionOffsetField = type.getCIntegerField("_exception_offset"); - deoptHandlerOffsetField = type.getCIntegerField("_deopt_handler_entry_offset"); + deoptHandlerEntryOffsetField = type.getCIntegerField("_deopt_handler_entry_offset"); origPCOffsetField = type.getCIntegerField("_orig_pc_offset"); stubOffsetField = type.getCIntegerField("_stub_offset"); scopesPCsOffsetField = type.getCIntegerField("_scopes_pcs_offset"); @@ -490,7 +490,7 @@ public String getName() { private int getEntryBCI() { return (int) entryBCIField .getValue(addr); } private int getExceptionOffset() { return (int) exceptionOffsetField .getValue(addr); } - private int getDeoptHandlerEntryOffset() { return (int) deoptHandlerOffsetField .getValue(addr); } + private int getDeoptHandlerEntryOffset() { return (int) deoptHandlerEntryOffsetField .getValue(addr); } private int getStubOffset() { return (int) stubOffsetField .getValue(addr); } private int getScopesDataOffset() { return (int) scopesDataOffsetField .getValue(addr); } private int getScopesPCsOffset() { return (int) scopesPCsOffsetField .getValue(addr); } From 7bb43523b3c9d1495d72a5bc75c3912c3f51e64c Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Fri, 7 Nov 2025 10:51:41 +0000 Subject: [PATCH 14/16] x86: Fix post-call NOP check access outside code blob Change-Id: I5f5fd681ba699f96859180d984841dd6df8524b5 --- src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp | 2 +- src/hotspot/cpu/x86/nativeInst_x86.hpp | 3 ++- src/hotspot/cpu/x86/x86.ad | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp index 7a8fbc75ba71c..33f7b063e77a6 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp @@ -48,7 +48,7 @@ enum { _call_stub_size = 28, _exception_handler_size = DEBUG_ONLY(1*K) NOT_DEBUG(175), - _deopt_handler_size = 10 + _deopt_handler_size = 7 }; public: diff --git a/src/hotspot/cpu/x86/nativeInst_x86.hpp b/src/hotspot/cpu/x86/nativeInst_x86.hpp index 3e76700648010..9e9bb5a0e50d8 100644 --- a/src/hotspot/cpu/x86/nativeInst_x86.hpp +++ b/src/hotspot/cpu/x86/nativeInst_x86.hpp @@ -73,6 +73,7 @@ class NativeInstruction { s_char sbyte_at(int offset) const { return *(s_char*) addr_at(offset); } u_char ubyte_at(int offset) const { return *(u_char*) addr_at(offset); } + jshort short_at(int offset) const { return *(jshort*) addr_at(offset); } jint int_at(int offset) const { return *(jint*) addr_at(offset); } intptr_t ptr_at(int offset) const { return *(intptr_t*) addr_at(offset); } @@ -581,7 +582,7 @@ class NativePostCallNop: public NativeInstruction { displacement_offset = 4 }; - bool check() const { return int_at(0) == 0x841f0f; } + bool check() const { return short_at(0) == 0x1f0f && short_at(2) == 0x0084; } bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { int32_t data = int_at(displacement_offset); if (data == 0) { diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index 240e180e220f5..a98d2e89a9255 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -2771,7 +2771,7 @@ class HandlerImpl { static uint size_deopt_handler() { // one call and one jmp. - return 10; + return 7; } }; From 20cc58a3649db0650da054809f64e0c4416d616f Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Thu, 13 Nov 2025 14:41:33 +0000 Subject: [PATCH 15/16] Add an assertion to detect out of bounds access in post-call NOP checks --- src/hotspot/cpu/aarch64/aarch64.ad | 2 ++ src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp | 2 ++ src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp | 9 ++++++++- src/hotspot/cpu/arm/arm.ad | 2 ++ src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp | 2 ++ src/hotspot/cpu/arm/nativeInst_arm_32.hpp | 7 +++++++ src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp | 2 ++ src/hotspot/cpu/ppc/nativeInst_ppc.hpp | 10 ++++++++-- src/hotspot/cpu/ppc/ppc.ad | 2 ++ src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp | 2 ++ src/hotspot/cpu/riscv/nativeInst_riscv.hpp | 9 ++++++++- src/hotspot/cpu/riscv/riscv.ad | 2 ++ src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp | 2 ++ src/hotspot/cpu/s390/nativeInst_s390.hpp | 7 +++++++ src/hotspot/cpu/s390/s390.ad | 2 ++ src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 2 ++ src/hotspot/cpu/x86/nativeInst_x86.hpp | 9 +++++++-- src/hotspot/cpu/x86/x86.ad | 2 ++ 18 files changed, 69 insertions(+), 6 deletions(-) diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 1e506edb63445..61cfef329888f 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -2277,6 +2277,8 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) __ b(start); assert(__ offset() - offset == (int) size_deopt_handler(), "overflow"); + assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; } diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index 2498a646e31f6..a8c55bb837fbd 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -458,6 +458,8 @@ int LIR_Assembler::emit_deopt_handler() { __ b(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); + guarantee(code_offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp index b7444347bf130..98b522b54b132 100644 --- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp @@ -532,10 +532,17 @@ class NativePostCallNop: public NativeInstruction { } public: + enum AArch64_specific_constants { + // The two parts should be checked separately to prevent out of bounds access in case + // the return address points to the deopt handler stub code entry point which could be + // at the end of page. + first_check_size = 4 + }; + bool check() const { // Check the first instruction is NOP. if (is_nop()) { - uint32_t insn = *(uint32_t*)addr_at(4); + uint32_t insn = *(uint32_t*)addr_at(first_check_size); // Check next instruction is MOVK zr, xx. // These instructions only ever appear together in a post-call // NOP, so it's unnecessary to check that the third instruction is diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad index eb9b0ed8fbafc..caf60dad57a98 100644 --- a/src/hotspot/cpu/arm/arm.ad +++ b/src/hotspot/cpu/arm/arm.ad @@ -897,6 +897,8 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { __ b(start); assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow"); + assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp index 1d7c1579502f0..f168a34f140a5 100644 --- a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp +++ b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp @@ -283,6 +283,8 @@ int LIR_Assembler::emit_deopt_handler() { __ b(start); assert(code_offset() - offset <= deopt_handler_size(), "overflow"); + assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/arm/nativeInst_arm_32.hpp b/src/hotspot/cpu/arm/nativeInst_arm_32.hpp index ee856bcfe6049..4c0c61d783c6e 100644 --- a/src/hotspot/cpu/arm/nativeInst_arm_32.hpp +++ b/src/hotspot/cpu/arm/nativeInst_arm_32.hpp @@ -430,6 +430,13 @@ inline NativeCall* nativeCall_before(address return_address) { class NativePostCallNop: public NativeInstruction { public: + enum arm_specific_constants { + // If the check is adjusted to read beyond size of the instruction sequence at the deopt + // handler stub code entry point, it has to happen in two stages - to prevent out of bounds + // access in case the return address points to the entry point which could be at + // the end of page. + first_check_size = 4 + }; bool check() const { return is_nop(); } bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { return false; } bool patch(int32_t oopmap_slot, int32_t cb_offset) { return false; } diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index 7898500cff256..53484bc15e8ba 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -272,6 +272,8 @@ int LIR_Assembler::emit_deopt_handler() { __ b(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); + guarantee(code_offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/ppc/nativeInst_ppc.hpp b/src/hotspot/cpu/ppc/nativeInst_ppc.hpp index dcb5c2bb3cbb1..885c0731e4670 100644 --- a/src/hotspot/cpu/ppc/nativeInst_ppc.hpp +++ b/src/hotspot/cpu/ppc/nativeInst_ppc.hpp @@ -51,8 +51,6 @@ class NativeInstruction { friend class Relocation; public: - bool is_post_call_nop() const { return MacroAssembler::is_post_call_nop(long_at(0)); } - bool is_jump() const { return Assembler::is_b(long_at(0)); } // See NativeGeneralJump. bool is_sigtrap_ic_miss_check() { @@ -531,6 +529,14 @@ class NativePostCallNop: public NativeInstruction { }; public: + enum ppc_specific_constants { + // If the check is adjusted to read beyond size of the instruction at the deopt handler stub + // code entry point, it has to happen in two stages - to prevent out of bounds access in case + // the return address points to the entry point which could be at the end of page. + first_check_size = 4 + }; + + bool is_post_call_nop() const { return MacroAssembler::is_post_call_nop(long_at(0)); } bool check() const { return is_post_call_nop(); } bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { uint32_t instr_bits = long_at(0); diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index 5488dbdb8c09c..fc95b34d0a218 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -2130,6 +2130,8 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { __ b(start); assert(__ offset() - offset == (int) size_deopt_handler(), "must be fixed size"); + assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp index b085dd7ac00b0..9e83a736a8925 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp @@ -386,6 +386,8 @@ int LIR_Assembler::emit_deopt_handler() { __ j(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); + guarantee(code_offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp index d990cfbc50d4f..12c15966e15c0 100644 --- a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp +++ b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp @@ -311,12 +311,19 @@ inline bool NativeInstruction::is_jump_or_nop() { // can store an offset from the initial nop to the nmethod. class NativePostCallNop: public NativeInstruction { public: + enum RISCV_specific_constants { + // The two parts should be checked separately to prevent out of bounds access in + // case the return address points to the deopt handler stub code entry point + // which could be at the end of page. + first_check_size = 4 + }; + bool check() const { // Check for two instructions: nop; lui zr, hi20 // These instructions only ever appear together in a post-call // NOP, so it's unnecessary to check that the third instruction is // an addiw as well. - return is_nop() && MacroAssembler::is_lui_to_zr_at(addr_at(4)); + return is_nop() && MacroAssembler::is_lui_to_zr_at(addr_at(first_check_size)); } bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const; bool patch(int32_t oopmap_slot, int32_t cb_offset); diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index d9dab3c5e0b19..2afd7ef899fa3 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -1844,6 +1844,8 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) __ j(start); assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow"); + assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp index ee6df63d21b37..28e49275901dd 100644 --- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp @@ -288,6 +288,8 @@ int LIR_Assembler::emit_deopt_handler() { __ z_bru(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); + guarantee(code_offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/s390/nativeInst_s390.hpp b/src/hotspot/cpu/s390/nativeInst_s390.hpp index 16400df3f2686..111c08a02c0b6 100644 --- a/src/hotspot/cpu/s390/nativeInst_s390.hpp +++ b/src/hotspot/cpu/s390/nativeInst_s390.hpp @@ -649,6 +649,13 @@ class NativeGeneralJump: public NativeInstruction { class NativePostCallNop: public NativeInstruction { public: + enum z_specific_constants { + // Once the check is implemented, this has to specify number of bytes checked on the first + // read. If the check would read beyond size of the instruction at the deopt handler stub + // code entry point, then it has to happen in two stages - to prevent out of bounds access + // in case the return address points to the entry point which could be at the end of page. + first_check_size = 0 + }; bool check() const { Unimplemented(); return false; } bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { return false; } bool patch(int32_t oopmap_slot, int32_t cb_offset) { Unimplemented(); return false; } diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index 0c4939d843283..7c890815937fc 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -1691,6 +1691,8 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { __ z_bru(start); assert(__ offset() - offset == (int) size_deopt_handler(), "must be fixed size"); + assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index 6600d8410506b..8d237e397fe49 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -461,6 +461,8 @@ int LIR_Assembler::emit_deopt_handler() { __ jmp(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); + guarantee(code_offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/x86/nativeInst_x86.hpp b/src/hotspot/cpu/x86/nativeInst_x86.hpp index 9e9bb5a0e50d8..ec7fc3b154a5d 100644 --- a/src/hotspot/cpu/x86/nativeInst_x86.hpp +++ b/src/hotspot/cpu/x86/nativeInst_x86.hpp @@ -579,10 +579,15 @@ class NativePostCallNop: public NativeInstruction { instruction_code = 0x0f, instruction_size = 8, instruction_offset = 0, - displacement_offset = 4 + displacement_offset = 4, + + // The two parts should be checked separately to prevent out of bounds access in case + // the return address points to the deopt handler stub code entry point which could be + // at the end of page. + first_check_size = 2 }; - bool check() const { return short_at(0) == 0x1f0f && short_at(2) == 0x0084; } + bool check() const { return short_at(0) == 0x1f0f && short_at(first_check_size) == 0x0084; } bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { int32_t data = int_at(displacement_offset); if (data == 0) { diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index a98d2e89a9255..918c8eddf4e5c 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -2885,6 +2885,8 @@ int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) { __ jmp(start); assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow %d", (__ offset() - offset)); + assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; } From 3a014376976c47f3a22f87bbf870e8315bf447c6 Mon Sep 17 00:00:00 2001 From: Ruben Ayrapetyan Date: Tue, 18 Nov 2025 18:55:24 +0000 Subject: [PATCH 16/16] Replace `guarantee` with `assert` in the C1 `emit_deopt_handler` --- src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp | 4 ++-- src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp | 4 ++-- src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp | 4 ++-- src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp | 4 ++-- src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index a8c55bb837fbd..37a6a130e0dd0 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -458,8 +458,8 @@ int LIR_Assembler::emit_deopt_handler() { __ b(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); - guarantee(code_offset() - entry_offset >= NativePostCallNop::first_check_size, - "out of bounds read in post-call NOP check"); + assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index 53484bc15e8ba..0b48653ae64ce 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -272,8 +272,8 @@ int LIR_Assembler::emit_deopt_handler() { __ b(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); - guarantee(code_offset() - entry_offset >= NativePostCallNop::first_check_size, - "out of bounds read in post-call NOP check"); + assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp index 9e83a736a8925..e77a2067e8933 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp @@ -386,8 +386,8 @@ int LIR_Assembler::emit_deopt_handler() { __ j(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); - guarantee(code_offset() - entry_offset >= NativePostCallNop::first_check_size, - "out of bounds read in post-call NOP check"); + assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp index 28e49275901dd..93d6051aa7658 100644 --- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp @@ -288,8 +288,8 @@ int LIR_Assembler::emit_deopt_handler() { __ z_bru(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); - guarantee(code_offset() - entry_offset >= NativePostCallNop::first_check_size, - "out of bounds read in post-call NOP check"); + assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset; diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index 8d237e397fe49..a2ea7af606d4f 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -461,8 +461,8 @@ int LIR_Assembler::emit_deopt_handler() { __ jmp(start); guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); - guarantee(code_offset() - entry_offset >= NativePostCallNop::first_check_size, - "out of bounds read in post-call NOP check"); + assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size, + "out of bounds read in post-call NOP check"); __ end_a_stub(); return entry_offset;