8382711: Shenandoah: Support AOT code dumping

Reviewed-by: kvn, iveresov, wkemper, adinn

Author: shipilev

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

@@ -261,24 +261,32 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
 
   // Test for in-cset
   if (is_strong) {
-    __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
-    __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+    if (AOTCodeCache::is_on_for_dump()) {
+      __ lea(rscratch2, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
+      __ ldr(rscratch2, Address(rscratch2));
+      __ lea(rscratch1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
+      __ ldrw(rscratch1, Address(rscratch1));
+      __ lsrv(rscratch1, r0, rscratch1);
+    } else {
+      __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
+      __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+    }
     __ ldrb(rscratch2, Address(rscratch2, rscratch1));
     __ tbz(rscratch2, 0, not_cset);
   }
 
   __ push_call_clobbered_registers();
   if (is_strong) {
     if (is_narrow) {
-      __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
+      __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)));
     } else {
-      __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
+      __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)));
     }
   } else if (is_weak) {
     if (is_narrow) {
-      __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
+      __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)));
     } else {
-      __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
+      __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)));
     }
   } else {
     assert(is_phantom, "only remaining strength");
@@ -709,8 +717,16 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
 
   if (is_strong) {
     // Check for object in cset.
-    __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
-    __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+    if (AOTCodeCache::is_on_for_dump()) {
+      __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
+      __ ldr(tmp2, Address(tmp2));
+      __ lea(tmp1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
+      __ ldrw(tmp1, Address(tmp1));
+      __ lsrv(tmp1, res, tmp1);
+    } else {
+      __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
+      __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+    }
     __ ldrb(tmp2, Address(tmp2, tmp1));
     __ cbz(tmp2, *stub->continuation());
   }
@@ -795,25 +811,25 @@ void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_s
   bool is_native  = ShenandoahBarrierSet::is_native_access(decorators);
   if (is_strong) {
     if (is_native) {
-      __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
+      __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)));
     } else {
       if (UseCompressedOops) {
-        __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
+        __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)));
       } else {
-        __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
+        __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)));
       }
     }
   } else if (is_weak) {
     assert(!is_native, "weak must not be called off-heap");
     if (UseCompressedOops) {
-      __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
+      __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)));
     } else {
-      __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
+      __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)));
     }
   } else {
     assert(is_phantom, "only remaining strength");
     assert(is_native, "phantom must only be called off-heap");
-    __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
+    __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom)));
   }
   __ blr(lr);
   __ mov(rscratch1, r0);

src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp

@@ -312,9 +312,9 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
   if (is_strong) {
     // Test for object in cset
     // Allocate temporary registers
-    for (int i = 0; i < 8; i++) {
+    for (int i = 0; i < Register::available_gp_registers(); i++) {
       Register r = as_Register(i);
-      if (r != rsp && r != rbp && r != dst && r != src.base() && r != src.index()) {
+      if (r != rsp && r != rbp && r != rcx && r != dst && r != src.base() && r != src.index() ) {
         if (tmp1 == noreg) {
           tmp1 = r;
         } else {
@@ -333,8 +333,19 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
 
     // Optimized cset-test
     __ movptr(tmp1, dst);
-    __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
-    __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+    if (AOTCodeCache::is_on_for_dump()) {
+      assert_different_registers(tmp1, tmp2, rcx);
+      __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
+      __ push(rcx);
+      __ movb(rcx, Address(tmp2));
+      __ shrptr(tmp1);
+      __ pop(rcx);
+      __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
+      __ movptr(tmp2, Address(tmp2));
+    } else {
+      __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+      __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+    }
     __ movbool(tmp1, Address(tmp1, tmp2, Address::times_1));
     __ testbool(tmp1);
     __ jcc(Assembler::zero, not_cset);
@@ -886,8 +897,27 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
   if (is_strong) {
     // Check for object being in the collection set.
     __ mov(tmp1, res);
-    __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
-    __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+    if (AOTCodeCache::is_on_for_dump()) {
+      __ push(rcx);
+      __ lea(rcx, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
+      __ movl(rcx, Address(rcx));
+      if (tmp1 != rcx) {
+        __ mov(tmp1, res);
+        __ shrptr(tmp1);
+        __ pop(rcx);
+      } else {
+        assert_different_registers(tmp2, rcx);
+        __ mov(tmp2, res);
+        __ shrptr(tmp2);
+        __ pop(rcx);
+        __ movptr(tmp1, tmp2);
+      }
+      __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
+      __ movptr(tmp2, Address(tmp2));
+    } else {
+      __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+      __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+    }
     __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
     __ testbool(tmp2);
     __ jcc(Assembler::zero, *stub->continuation());

src/hotspot/share/code/aotCodeCache.cpp

@@ -63,6 +63,7 @@
 #include "gc/g1/g1HeapRegion.hpp"
 #endif
 #if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
 #include "gc/shenandoah/shenandoahRuntime.hpp"
 #endif
 #if INCLUDE_ZGC
@@ -2476,14 +2477,16 @@ void AOTRuntimeConstants::initialize_from_runtime() {
   BarrierSet* bs = BarrierSet::barrier_set();
   address card_table_base = nullptr;
   uint grain_shift = 0;
+  address cset_base = nullptr;
 #if INCLUDE_G1GC
   if (bs->is_a(BarrierSet::G1BarrierSet)) {
     grain_shift = G1HeapRegion::LogOfHRGrainBytes;
   } else
 #endif
 #if INCLUDE_SHENANDOAHGC
   if (bs->is_a(BarrierSet::ShenandoahBarrierSet)) {
-    grain_shift = 0;
+    grain_shift = ShenandoahHeapRegion::region_size_bytes_shift_jint();
+    cset_base = ShenandoahHeap::in_cset_fast_test_addr();
   } else
 #endif
   if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
@@ -2495,11 +2498,13 @@ void AOTRuntimeConstants::initialize_from_runtime() {
   }
   _aot_runtime_constants._card_table_base = card_table_base;
   _aot_runtime_constants._grain_shift = grain_shift;
+  _aot_runtime_constants._cset_base = cset_base;
 }
 
 address AOTRuntimeConstants::_field_addresses_list[] = {
   ((address)&_aot_runtime_constants._card_table_base),
   ((address)&_aot_runtime_constants._grain_shift),
+  ((address)&_aot_runtime_constants._cset_base),
   nullptr
 };
 

src/hotspot/share/code/aotCodeCache.hpp

@@ -688,6 +688,7 @@ class AOTRuntimeConstants {
  private:
   address _card_table_base;
   uint    _grain_shift;
+  address _cset_base;
   static address _field_addresses_list[];
   static AOTRuntimeConstants _aot_runtime_constants;
   // private constructor for unique singleton
@@ -703,13 +704,15 @@ class AOTRuntimeConstants {
   }
   static address card_table_base_address();
   static address grain_shift_address() { return (address)&_aot_runtime_constants._grain_shift; }
+  static address cset_base_address() { return (address)&_aot_runtime_constants._cset_base; }
   static address* field_addresses_list() {
     return _field_addresses_list;
   }
 #else
   static bool contains(address adr)        { return false; }
   static address card_table_base_address() { return nullptr; }
   static address grain_shift_address()     { return nullptr; }
+  static address cset_base_address()       { return nullptr; }
   static address* field_addresses_list()   { return nullptr; }
 #endif
 };

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

@@ -25,6 +25,7 @@
 
 
 #include "classfile/javaClasses.hpp"
+#include "code/aotCodeCache.hpp"
 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
@@ -928,12 +929,31 @@ void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl,
   PhaseIterGVN& igvn = phase->igvn();
 
   Node* raw_val        = new CastP2XNode(old_ctrl, val);
-  Node* cset_idx       = new URShiftXNode(raw_val, igvn.intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint()));
+  Node* region_size_shift = nullptr;
+  if (AOTCodeCache::is_on_for_dump()) {
+    Node* aot_addr = igvn.makecon(TypeRawPtr::make(AOTRuntimeConstants::grain_shift_address()));
+    region_size_shift = new LoadINode(old_ctrl, raw_mem, aot_addr,
+                                      DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(nullptr),
+                                      TypeInt::INT, MemNode::unordered);
+    phase->register_new_node(region_size_shift, old_ctrl);
+  } else {
+    region_size_shift = igvn.intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  }
+  Node* cset_idx       = new URShiftXNode(raw_val, region_size_shift);
 
   // Figure out the target cset address with raw pointer math.
   // This avoids matching AddP+LoadB that would emit inefficient code.
   // See JDK-8245465.
-  Node* cset_addr_ptr  = igvn.makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr()));
+  Node* cset_addr_ptr = nullptr;
+  if (AOTCodeCache::is_on_for_dump()) {
+    Node* aot_addr = igvn.makecon(TypeRawPtr::make(AOTRuntimeConstants::cset_base_address()));
+    cset_addr_ptr = new LoadPNode(old_ctrl, raw_mem, aot_addr,
+                                  DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(nullptr),
+                                  TypeRawPtr::NOTNULL, MemNode::unordered);
+    phase->register_new_node(cset_addr_ptr, old_ctrl);
+  } else {
+    cset_addr_ptr  = igvn.makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr()));
+  }
   Node* cset_addr      = new CastP2XNode(old_ctrl, cset_addr_ptr);
   Node* cset_load_addr = new AddXNode(cset_addr, cset_idx);
   Node* cset_load_ptr  = new CastX2PNode(cset_load_addr);