8376839: GenShen: Improve performance of evacuations into the old generation

Reviewed-by: kdnilsen

Author: William Kemper

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

@@ -112,6 +112,24 @@ void ShenandoahConcurrentGC::entry_concurrent_update_refs_prepare(ShenandoahHeap
   heap->concurrent_prepare_for_update_refs();
 }
 
+void ShenandoahConcurrentGC::entry_update_card_table() {
+  ShenandoahHeap* const heap = ShenandoahHeap::heap();
+  TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
+
+  static const char* msg = "Concurrent update cards";
+  ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_card_table);
+  EventMark em("%s", msg);
+
+  ShenandoahWorkerScope scope(heap->workers(),
+                              ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
+                              "concurrent update cards");
+
+  // Heap needs to be parsable here.
+  // Also, parallel heap region iterate must have a phase set.
+  assert(ShenandoahTimingsTracker::is_current_phase_valid(), "Current phase must be set");
+  ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table();
+}
+
 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   _generation->ref_processor()->set_soft_reference_policy(
@@ -206,6 +224,11 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
 
     // Perform update-refs phase.
     entry_concurrent_update_refs_prepare(heap);
+
+    if (ShenandoahHeap::heap()->mode()->is_generational()) {
+      entry_update_card_table();
+    }
+
     if (ShenandoahVerify) {
       vmop_entry_init_update_refs();
     }

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp

@@ -59,8 +59,6 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
   bool collect(GCCause::Cause cause) override;
   ShenandoahDegenPoint degen_point() const;
 
-  void entry_concurrent_update_refs_prepare(ShenandoahHeap* heap);
-
   // Return true if this cycle found enough immediate garbage to skip evacuation
   bool abbreviated() const { return _abbreviated; }
 
@@ -95,6 +93,8 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
   void entry_cleanup_early();
   void entry_evacuate();
   void entry_update_thread_roots();
+  void entry_update_card_table();
+  void entry_concurrent_update_refs_prepare(ShenandoahHeap* heap);
   void entry_update_refs();
   void entry_cleanup_complete();
 

src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp

@@ -277,6 +277,11 @@ void ShenandoahDegenGC::op_degenerated() {
         _abbreviated = true;
       }
 
+      // labs are retired, walk the old regions and update remembered set
+      if (ShenandoahHeap::heap()->mode()->is_generational()) {
+        ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table();
+      }
+
     case _degenerated_update_refs:
       if (heap->has_forwarded_objects()) {
         op_update_refs();

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp

@@ -1569,7 +1569,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah
   // We must call try_recycle_under_lock() even if !r->is_trash().  The reason is that if r is being recycled at this
   // moment by a GC worker thread, it may appear to be not trash even though it has not yet been fully recycled.  If
   // we proceed without waiting for the worker to finish recycling the region, the worker thread may overwrite the
-  // region's affiliation with FREE after we set the region's affiliation to req.afiliation() below
+  // region's affiliation with FREE after we set the region's affiliation to req.affiliation() below
   r->try_recycle_under_lock();
   in_new_region = r->is_empty();
   if (in_new_region) {
@@ -1585,7 +1585,6 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah
       // concurrent preparations for mixed evacuations are completed), we mark this region as not requiring any
       // coalesce-and-fill processing.
       r->end_preemptible_coalesce_and_fill();
-      _heap->old_generation()->clear_cards_for(r);
     }
 #ifdef ASSERT
     ShenandoahMarkingContext* const ctx = _heap->marking_context();

src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp

@@ -133,3 +133,4 @@ void ShenandoahGenerationalEvacuationTask::evacuate_and_promote_regions() {
     }
   }
 }
+

src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp

@@ -363,10 +363,6 @@ oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, uint
       // Record that the evacuation succeeded
       evac_tracker()->end_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
     }
-
-    if (TO_GENERATION == OLD_GENERATION) {
-      old_generation()->handle_evacuation(copy, size);
-    }
   }  else {
     // Failed to evacuate. We need to deal with the object that is left behind. Since this
     // new allocation is certainly after TAMS, it will be considered live in the next cycle.

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -1955,6 +1955,26 @@ void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const
   }
 }
 
+class ShenandoahHeapRegionIteratorTask : public WorkerTask {
+private:
+  ShenandoahRegionIterator _regions;
+  ShenandoahHeapRegionClosure* _closure;
+
+public:
+  ShenandoahHeapRegionIteratorTask(ShenandoahHeapRegionClosure* closure)
+    : WorkerTask("Shenandoah Heap Region Iterator")
+    , _closure(closure) {}
+
+  void work(uint worker_id) override {
+    ShenandoahParallelWorkerSession worker_session(worker_id);
+    ShenandoahHeapRegion* region = _regions.next();
+    while (region != nullptr) {
+      _closure->heap_region_do(region);
+      region = _regions.next();
+    }
+  }
+};
+
 class ShenandoahParallelHeapRegionTask : public WorkerTask {
 private:
   ShenandoahHeap* const _heap;
@@ -2011,6 +2031,11 @@ void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* b
   }
 }
 
+void ShenandoahHeap::heap_region_iterator(ShenandoahHeapRegionClosure* closure) const {
+  ShenandoahHeapRegionIteratorTask task(closure);
+  workers()->run_task(&task);
+}
+
 class ShenandoahRendezvousHandshakeClosure : public HandshakeClosure {
 public:
   inline ShenandoahRendezvousHandshakeClosure(const char* name) : HandshakeClosure(name) {}

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -298,6 +298,7 @@ class ShenandoahHeap : public CollectedHeap {
 
   void heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
   void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
+  void heap_region_iterator(ShenandoahHeapRegionClosure* blk) const;
 
   inline ShenandoahMmuTracker* mmu_tracker() { return &_mmu_tracker; };
 

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp

@@ -67,6 +67,7 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c
   _new_top(nullptr),
   _empty_time(os::elapsedTime()),
   _top_before_promoted(nullptr),
+  _top_at_evac_start(start),
   _state(committed ? _empty_committed : _empty_uncommitted),
   _top(start),
   _tlab_allocs(0),
@@ -565,12 +566,17 @@ void ShenandoahHeapRegion::recycle_internal() {
   assert(_recycling.is_set() && is_trash(), "Wrong state");
   ShenandoahHeap* heap = ShenandoahHeap::heap();
 
+  _top_at_evac_start = _bottom;
   _mixed_candidate_garbage_words = 0;
   set_top(bottom());
   clear_live_data();
   reset_alloc_metadata();
   heap->marking_context()->reset_top_at_mark_start(this);
   set_update_watermark(bottom());
+  if (is_old()) {
+    heap->old_generation()->clear_cards_for(this);
+  }
+
   if (ZapUnusedHeapArea) {
     SpaceMangler::mangle_region(MemRegion(bottom(), end()));
   }

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@@ -246,6 +246,7 @@ class ShenandoahHeapRegion {
   double _empty_time;
 
   HeapWord* _top_before_promoted;
+  HeapWord* _top_at_evac_start;
 
   // Seldom updated fields
   Atomic<RegionState> _state;
@@ -365,12 +366,15 @@ class ShenandoahHeapRegion {
   }
 
   // Returns true iff this region was promoted in place subsequent to the most recent start of concurrent old marking.
-  inline bool was_promoted_in_place() {
+  bool was_promoted_in_place() const {
     return _promoted_in_place;
   }
   inline void restore_top_before_promote();
   inline size_t garbage_before_padded_for_promote() const;
 
+  HeapWord* get_top_at_evac_start() const { return _top_at_evac_start; }
+  void record_top_at_evac_start()         { _top_at_evac_start = _top; }
+
   // If next available memory is not aligned on address that is multiple of alignment, fill the empty space
   // so that returned object is aligned on an address that is a multiple of alignment_in_bytes.  Requested
   // size is in words.  It is assumed that this->is_old().  A pad object is allocated, filled, and registered