A new ObjectMonitorTable implementation

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp

@@ -31,6 +31,7 @@
 #include "opto/output.hpp"
 #include "opto/subnode.hpp"
 #include "runtime/stubRoutines.hpp"
+#include "runtime/synchronizer.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/powerOfTwo.hpp"
 
@@ -157,6 +158,8 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1,
   Label locked;
   // Finish fast lock unsuccessfully. MUST branch to with flag == NE
   Label slow_path;
+  // Finish fast lock unsuccessfully. Sets flag == NE
+  Label slow_path_set_ne;
 
   if (UseObjectMonitorTable) {
     // Clear cache in case fast locking succeeds or we need to take the slow-path.
@@ -222,36 +225,54 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1,
       assert(t1_monitor == t1_mark, "should be the same here");
     } else {
       Label monitor_found;
+      Label lookup_in_table;
+      Label found_in_cache;
 
       // Load cache address
       lea(t3_t, Address(rthread, JavaThread::om_cache_oops_offset()));
 
-      const int num_unrolled = 2;
+      const int num_unrolled = OMCache::CAPACITY;
       for (int i = 0; i < num_unrolled; i++) {
-        ldr(t1, Address(t3_t));
-        cmp(obj, t1);
-        br(Assembler::EQ, monitor_found);
+        ldr(t2, Address(t3_t));
+        cmp(obj, t2);
+        br(Assembler::EQ, found_in_cache);
         increment(t3_t, in_bytes(OMCache::oop_to_oop_difference()));
       }
 
-      Label loop;
+      b(lookup_in_table);
 
-      // Search for obj in cache.
-      bind(loop);
+      bind(found_in_cache);
+      ldr(t1_monitor, Address(t3_t, OMCache::oop_to_monitor_difference()));
+      b(monitor_found);
+
+      bind(lookup_in_table);
+
+      // Grab hash code
+      ubfx(t1_mark, t1_mark, markWord::hash_shift, markWord::hash_bits);
+
+      // Get the table and calculate bucket
+      lea(t3, ExternalAddress(ObjectMonitorTable::current_table_address()));
+      ldr(t3, Address(t3, 0));
+      ldr(t2, Address(t3, ObjectMonitorTable::table_capacity_mask_offset()));
+      ands(t1_mark, t1_mark, t2);
+      ldr(t3, Address(t3, ObjectMonitorTable::table_buckets_offset()));
 
-      // Check for match.
-      ldr(t1, Address(t3_t));
-      cmp(obj, t1);
-      br(Assembler::EQ, monitor_found);
+      // Read monitor from bucket
+      lsl(t1_mark, t1_mark, 3);
+      ldr(t1_monitor, Address(t3, t1_mark));
 
-      // Search until null encountered, guaranteed _null_sentinel at end.
-      increment(t3_t, in_bytes(OMCache::oop_to_oop_difference()));
-      cbnz(t1, loop);
-      // Cache Miss, NE set from cmp above, cbnz does not set flags
-      b(slow_path);
+      // Check if empty slot, removed slot or tomb stone
+      cmp(t1_monitor, (unsigned char)2);
+      br(Assembler::LS, slow_path_set_ne);
+
+      // Check if object matches
+      ldr(t3, Address(t1_monitor, ObjectMonitor::object_offset()));
+      BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
+      bs_asm->try_resolve_weak_handle_in_c2(this, t3, t2, slow_path_set_ne);
+      cmp(t3, obj);
+      br(Assembler::NE, slow_path);
 
       bind(monitor_found);
-      ldr(t1_monitor, Address(t3_t, OMCache::oop_to_monitor_difference()));
     }
 
     const Register t2_owner_addr = t2;
@@ -286,12 +307,18 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1,
 
   bind(locked);
 
+  Label the_end;
 #ifdef ASSERT
   // Check that locked label is reached with Flags == EQ.
   Label flag_correct;
   br(Assembler::EQ, flag_correct);
   stop("Fast Lock Flag != EQ");
 #endif
+  b(the_end);
+
+  bind(slow_path_set_ne);
+  // Set NE
+  cmp(rthread, zr);
 
   bind(slow_path);
 #ifdef ASSERT
@@ -300,6 +327,7 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1,
   stop("Fast Lock Flag != NE");
   bind(flag_correct);
 #endif
+  bind(the_end);
   // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
 }
 

src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp

@@ -441,6 +441,10 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na
   return opto_reg;
 }
 
+void BarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+  __ ldr(obj, Address(obj, 0));
+}
+
 #undef __
 #define __ _masm->
 

src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp

@@ -135,6 +135,7 @@ class BarrierSetAssembler: public CHeapObj<mtGC> {
                                                   OptoReg::Name opto_reg);
   OptoReg::Name refine_register(const Node* node,
                                 OptoReg::Name opto_reg);
+  virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
 #endif // COMPILER2
 };
 

src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp

@@ -1327,6 +1327,18 @@ void ZStoreBarrierStubC2Aarch64::emit_code(MacroAssembler& masm) {
   register_stub(this);
 }
 
+#undef __
+#define __ masm->
+
+void ZBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+  __ ldr(obj, Address(obj, 0));
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadBeforeMov);
+  __ movzw(tmp, barrier_Relocation::unpatched);
+  __ tst(obj, tmp);
+  __ br(Assembler::NE, slow_path);
+  __ lsr(obj, obj, ZPointerLoadShift);
+}
+
 #undef __
 
 #endif // COMPILER2

src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp

@@ -191,6 +191,7 @@ class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase {
                                      ZLoadBarrierStubC2* stub) const;
   void generate_c2_store_barrier_stub(MacroAssembler* masm,
                                       ZStoreBarrierStubC2* stub) const;
+  void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
 #endif // COMPILER2
 
   void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);

src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp

@@ -275,6 +275,10 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na
   return opto_reg;
 }
 
+void BarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+  __ ld(obj, 0, obj);
+}
+
 #undef __
 #define __ _masm->
 

src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2018, 2022 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -81,6 +81,8 @@ class BarrierSetAssembler: public CHeapObj<mtGC> {
 
 #ifdef COMPILER2
   OptoReg::Name refine_register(const Node* node, OptoReg::Name opto_reg) const;
+  virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj,
+                                             Register tmp, Label& slow_path);
 #endif // COMPILER2
 };
 

src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp

@@ -950,6 +950,21 @@ void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm,
   __ b(*stub->continuation());
 }
 
+void ZBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+  // Load the oop from the weak handle
+  __ ld(obj, 0, obj);
+
+  // Check if oop is okay
+  __ ld(tmp, in_bytes(ZThreadLocalData::mark_bad_mask_offset()), R16_thread);
+
+  // Test reference against bad mask. If mask bad, then we need to fix it up.
+  __ and_(tmp, obj, tmp);
+  __ bne(CR0, slow_path);
+
+  // Uncolor oop if okay
+  __ srdi(obj, obj, ZPointerLoadShift);
+}
+
 #undef __
 #endif // COMPILER2
 

src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2021, 2022 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -108,6 +108,8 @@ class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase {
   void generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const;
 
   void generate_c2_store_barrier_stub(MacroAssembler* masm, ZStoreBarrierStubC2* stub) const;
+
+  void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
 #endif // COMPILER2
 
   void store_barrier_fast(MacroAssembler* masm,

src/hotspot/cpu/ppc/macroAssembler_ppc.cpp

@@ -2756,39 +2756,60 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
       addi(owner_addr, mark, in_bytes(ObjectMonitor::owner_offset()) - monitor_tag);
       mark = noreg;
     } else {
+      const Register cache_addr = tmp2;
+      const Register tmp3_bucket = tmp3;
       Label monitor_found;
-      Register cache_addr = tmp2;
+      Label lookup_in_table;
+      Label found_in_cache;
 
       // Load cache address
       addi(cache_addr, R16_thread, in_bytes(JavaThread::om_cache_oops_offset()));
 
-      const int num_unrolled = 2;
+      const int num_unrolled = OMCache::CAPACITY;
       for (int i = 0; i < num_unrolled; i++) {
         ld(R0, 0, cache_addr);
         cmpd(CR0, R0, obj);
-        beq(CR0, monitor_found);
+        beq(CR0, found_in_cache);
         addi(cache_addr, cache_addr, in_bytes(OMCache::oop_to_oop_difference()));
       }
 
-      Label loop;
+      b(lookup_in_table);
 
-      // Search for obj in cache.
-      bind(loop);
+      bind(found_in_cache);
+      ld(monitor, in_bytes(OMCache::oop_to_monitor_difference()), cache_addr);
+      b(monitor_found);
+
+      bind(lookup_in_table);
+
+      // Grab hash code
+      srdi(mark, mark, markWord::hash_shift);
+
+      // Get the table and calculate bucket
+      load_const_optimized(tmp3, ObjectMonitorTable::current_table_address(), R0);
+      ld_ptr(tmp3, 0, tmp3);
+      ld(tmp2, in_bytes(ObjectMonitorTable::table_capacity_mask_offset()), tmp3);
+      andr(mark, mark, tmp2);
+      ld(tmp3, in_bytes(ObjectMonitorTable::table_buckets_offset()), tmp3);
 
-      // Check for match.
-      ld(R0, 0, cache_addr);
-      cmpd(CR0, R0, obj);
-      beq(CR0, monitor_found);
+      // Read monitor from bucket
+      sldi(mark, mark, LogBytesPerWord);
+      add(tmp3_bucket, tmp3, mark);
 
-      // Search until null encountered, guaranteed _null_sentinel at end.
-      addi(cache_addr, cache_addr, in_bytes(OMCache::oop_to_oop_difference()));
-      cmpdi(CR1, R0, 0);
-      bne(CR1, loop);
-      // Cache Miss, CR0.NE set from cmp above
-      b(slow_path);
+      // Read monitor from bucket
+      ld_ptr(monitor, 0, tmp3_bucket);
+
+      // Check if empty slot, removed slot or tomb stone
+      cmpdi(CR0, monitor, 3);
+      blt(CR0, slow_path);
+
+      // Check if object matches
+      ld(tmp3, in_bytes(ObjectMonitor::object_offset()), monitor);
+      BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
+      bs_asm->try_resolve_weak_handle_in_c2(this, tmp3, tmp2, slow_path);
+      cmpd(CR0, tmp3, obj);
+      bne(CR0, slow_path);
 
       bind(monitor_found);
-      ld(monitor, in_bytes(OMCache::oop_to_monitor_difference()), cache_addr);
 
       // Compute owner address.
       addi(owner_addr, monitor, in_bytes(ObjectMonitor::owner_offset()));

src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp

@@ -31,6 +31,7 @@
 #include "opto/output.hpp"
 #include "opto/subnode.hpp"
 #include "runtime/stubRoutines.hpp"
+#include "runtime/synchronizer.hpp"
 #include "utilities/globalDefinitions.hpp"
 
 #ifdef PRODUCT
@@ -123,35 +124,55 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box,
     if (!UseObjectMonitorTable) {
       assert(tmp1_monitor == tmp1_mark, "should be the same here");
     } else {
+      const Register tmp3_bucket = tmp3;
       Label monitor_found;
+      Label lookup_in_table;
+      Label found_in_cache;
 
       // Load cache address
       la(tmp3_t, Address(xthread, JavaThread::om_cache_oops_offset()));
 
-      const int num_unrolled = 2;
+      const int num_unrolled = OMCache::CAPACITY;
       for (int i = 0; i < num_unrolled; i++) {
         ld(tmp1, Address(tmp3_t));
-        beq(obj, tmp1, monitor_found);
+        beq(obj, tmp1, found_in_cache);
         add(tmp3_t, tmp3_t, in_bytes(OMCache::oop_to_oop_difference()));
       }
 
-      Label loop;
+      j(lookup_in_table);
 
-      // Search for obj in cache.
-      bind(loop);
+      bind(found_in_cache);
+      ld(tmp1_monitor, Address(tmp3_t, OMCache::oop_to_monitor_difference()));
+      j(monitor_found);
+
+      bind(lookup_in_table);
+
+      // Grab hash code
+      srli(tmp1, tmp1_mark, markWord::hash_shift);
 
-      // Check for match.
-      ld(tmp1, Address(tmp3_t));
-      beq(obj, tmp1, monitor_found);
+      // Get the table and calculate bucket
+      la(tmp3_t, ExternalAddress(ObjectMonitorTable::current_table_address()));
+      ld(tmp3_t, Address(tmp3_t));
+      ld(tmp2, Address(tmp3_t, ObjectMonitorTable::table_capacity_mask_offset()));
+      andr(tmp1, tmp1, tmp2);
+      ld(tmp3_t, Address(tmp3_t, ObjectMonitorTable::table_buckets_offset()));
+      slli(tmp1, tmp1, LogBytesPerWord);
+      add(tmp3_bucket, tmp3_t, tmp1);
 
-      // Search until null encountered, guaranteed _null_sentinel at end.
-      add(tmp3_t, tmp3_t, in_bytes(OMCache::oop_to_oop_difference()));
-      bnez(tmp1, loop);
-      // Cache Miss. Take the slowpath.
-      j(slow_path);
+      // Read monitor from bucket
+      ld(tmp1_monitor, Address(tmp3_bucket));
+
+      // Check if empty slot, removed slot or tomb stone
+      li(tmp2, 2);
+      bleu(tmp1_monitor, tmp2, slow_path);
+
+      // Check if object matches
+      ld(tmp3, Address(tmp1_monitor, ObjectMonitor::object_offset()));
+      BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
+      bs_asm->try_resolve_weak_handle_in_c2(this, tmp3, tmp2, slow_path);
+      bne(tmp3, obj, slow_path);
 
       bind(monitor_found);
-      ld(tmp1_monitor, Address(tmp3_t, OMCache::oop_to_monitor_difference()));
     }
 
     const Register tmp2_owner_addr = tmp2;

src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp

@@ -369,6 +369,10 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na
   return opto_reg;
 }
 
+void BarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+  __ ld(obj, Address(obj));
+}
+
 #undef __
 #define __ _masm->