< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

Print this page

   1 /*
   2  * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.

   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "memory/allocation.hpp"
  28 #include "memory/universe.hpp"
  29 
  30 #include "gc/shared/classUnloadingContext.hpp"
  31 #include "gc/shared/gcArguments.hpp"
  32 #include "gc/shared/gcTimer.hpp"
  33 #include "gc/shared/gcTraceTime.inline.hpp"
  34 #include "gc/shared/locationPrinter.inline.hpp"
  35 #include "gc/shared/memAllocator.hpp"
  36 #include "gc/shared/plab.hpp"
  37 #include "gc/shared/tlab_globals.hpp"
  38 



  39 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  40 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  41 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  42 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  43 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
  44 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  45 #include "gc/shenandoah/shenandoahControlThread.hpp"
  46 #include "gc/shenandoah/shenandoahFreeSet.hpp"



  47 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  48 #include "gc/shenandoah/shenandoahHeap.inline.hpp"

  49 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  50 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  51 #include "gc/shenandoah/shenandoahInitLogger.hpp"
  52 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  53 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  54 #include "gc/shenandoah/shenandoahMetrics.hpp"
  55 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"

  56 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  57 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  58 #include "gc/shenandoah/shenandoahPadding.hpp"
  59 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  60 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  61 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"

  62 #include "gc/shenandoah/shenandoahSTWMark.hpp"
  63 #include "gc/shenandoah/shenandoahUtils.hpp"
  64 #include "gc/shenandoah/shenandoahVerifier.hpp"
  65 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  66 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  67 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  68 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"


  69 #include "gc/shenandoah/mode/shenandoahIUMode.hpp"
  70 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
  71 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"


  72 #if INCLUDE_JFR
  73 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  74 #endif
  75 
  76 #include "classfile/systemDictionary.hpp"
  77 #include "code/codeCache.hpp"
  78 #include "memory/classLoaderMetaspace.hpp"
  79 #include "memory/metaspaceUtils.hpp"
  80 #include "oops/compressedOops.inline.hpp"
  81 #include "prims/jvmtiTagMap.hpp"
  82 #include "runtime/atomic.hpp"
  83 #include "runtime/globals.hpp"
  84 #include "runtime/interfaceSupport.inline.hpp"
  85 #include "runtime/java.hpp"
  86 #include "runtime/orderAccess.hpp"
  87 #include "runtime/safepointMechanism.hpp"
  88 #include "runtime/vmThread.hpp"
  89 #include "services/mallocTracker.hpp"
  90 #include "services/memTracker.hpp"
  91 #include "utilities/events.hpp"

 143 jint ShenandoahHeap::initialize() {
 144   //
 145   // Figure out heap sizing
 146   //
 147 
 148   size_t init_byte_size = InitialHeapSize;
 149   size_t min_byte_size  = MinHeapSize;
 150   size_t max_byte_size  = MaxHeapSize;
 151   size_t heap_alignment = HeapAlignment;
 152 
 153   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 154 
 155   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 156   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 157 
 158   _num_regions = ShenandoahHeapRegion::region_count();
 159   assert(_num_regions == (max_byte_size / reg_size_bytes),
 160          "Regions should cover entire heap exactly: " SIZE_FORMAT " != " SIZE_FORMAT "/" SIZE_FORMAT,
 161          _num_regions, max_byte_size, reg_size_bytes);
 162 
 163   // Now we know the number of regions, initialize the heuristics.
 164   initialize_heuristics();
 165 
 166   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 167   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 168   assert(num_committed_regions <= _num_regions, "sanity");
 169   _initial_size = num_committed_regions * reg_size_bytes;
 170 
 171   size_t num_min_regions = min_byte_size / reg_size_bytes;
 172   num_min_regions = MIN2(num_min_regions, _num_regions);
 173   assert(num_min_regions <= _num_regions, "sanity");
 174   _minimum_size = num_min_regions * reg_size_bytes;
 175 
 176   // Default to max heap size.
 177   _soft_max_size = _num_regions * reg_size_bytes;
 178 
 179   _committed = _initial_size;
 180 
 181   size_t heap_page_size   = UseLargePages ? os::large_page_size() : os::vm_page_size();
 182   size_t bitmap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 183   size_t region_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 184 
 185   //
 186   // Reserve and commit memory for heap
 187   //
 188 
 189   ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 190   initialize_reserved_region(heap_rs);
 191   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 192   _heap_region_special = heap_rs.special();
 193 
 194   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 195          "Misaligned heap: " PTR_FORMAT, p2i(base()));



 196 
 197 #if SHENANDOAH_OPTIMIZED_MARKTASK
 198   // The optimized ShenandoahMarkTask takes some bits away from the full object bits.
 199   // Fail if we ever attempt to address more than we can.
 200   if ((uintptr_t)heap_rs.end() >= ShenandoahMarkTask::max_addressable()) {
 201     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 202                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 203                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 204                 p2i(heap_rs.base()), p2i(heap_rs.end()), ShenandoahMarkTask::max_addressable());
 205     vm_exit_during_initialization("Fatal Error", buf);
 206   }
 207 #endif
 208 
 209   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 210   if (!_heap_region_special) {
 211     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 212                               "Cannot commit heap memory");
 213   }
 214 






















 215   //
 216   // Reserve and commit memory for bitmap(s)
 217   //
 218 
 219   _bitmap_size = ShenandoahMarkBitMap::compute_size(heap_rs.size());
 220   _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
 221 
 222   size_t bitmap_bytes_per_region = reg_size_bytes / ShenandoahMarkBitMap::heap_map_factor();
 223 
 224   guarantee(bitmap_bytes_per_region != 0,
 225             "Bitmap bytes per region should not be zero");
 226   guarantee(is_power_of_2(bitmap_bytes_per_region),
 227             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 228 
 229   if (bitmap_page_size > bitmap_bytes_per_region) {
 230     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 231     _bitmap_bytes_per_slice = bitmap_page_size;
 232   } else {
 233     _bitmap_regions_per_slice = 1;
 234     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 235   }
 236 
 237   guarantee(_bitmap_regions_per_slice >= 1,
 238             "Should have at least one region per slice: " SIZE_FORMAT,
 239             _bitmap_regions_per_slice);
 240 
 241   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 242             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 243             _bitmap_bytes_per_slice, bitmap_page_size);
 244 
 245   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);




 246   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 247   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 248   _bitmap_region_special = bitmap.special();
 249 
 250   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 251                               align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 252   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 253   if (!_bitmap_region_special) {
 254     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 255                               "Cannot commit bitmap memory");
 256   }
 257 
 258   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions, _max_workers);
 259 
 260   if (ShenandoahVerify) {
 261     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);




 262     if (!verify_bitmap.special()) {
 263       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 264                                 "Cannot commit verification bitmap memory");
 265     }
 266     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 267     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 268     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 269     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 270   }
 271 
 272   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 273   ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);





 274   MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
 275   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 276   _aux_bitmap_region_special = aux_bitmap.special();
 277   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 278 
 279   //
 280   // Create regions and region sets
 281   //
 282   size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
 283   size_t region_storage_size = align_up(region_align * _num_regions, region_page_size);
 284   region_storage_size = align_up(region_storage_size, os::vm_allocation_granularity());

 285 
 286   ReservedSpace region_storage(region_storage_size, region_page_size);



 287   MemTracker::record_virtual_memory_type(region_storage.base(), mtGC);
 288   if (!region_storage.special()) {
 289     os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
 290                               "Cannot commit region memory");
 291   }
 292 
 293   // Try to fit the collection set bitmap at lower addresses. This optimizes code generation for cset checks.
 294   // Go up until a sensible limit (subject to encoding constraints) and try to reserve the space there.
 295   // If not successful, bite a bullet and allocate at whatever address.
 296   {
 297     size_t cset_align = MAX2<size_t>(os::vm_page_size(), os::vm_allocation_granularity());
 298     size_t cset_size = align_up(((size_t) sh_rs.base() + sh_rs.size()) >> ShenandoahHeapRegion::region_size_bytes_shift(), cset_align);

 299 
 300     uintptr_t min = round_up_power_of_2(cset_align);
 301     uintptr_t max = (1u << 30u);

 302 
 303     for (uintptr_t addr = min; addr <= max; addr <<= 1u) {
 304       char* req_addr = (char*)addr;
 305       assert(is_aligned(req_addr, cset_align), "Should be aligned");
 306       ReservedSpace cset_rs(cset_size, cset_align, os::vm_page_size(), req_addr);
 307       if (cset_rs.is_reserved()) {
 308         assert(cset_rs.base() == req_addr, "Allocated where requested: " PTR_FORMAT ", " PTR_FORMAT, p2i(cset_rs.base()), addr);
 309         _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 310         break;
 311       }
 312     }
 313 
 314     if (_collection_set == nullptr) {
 315       ReservedSpace cset_rs(cset_size, cset_align, os::vm_page_size());
 316       _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 317     }




 318   }
 319 
 320   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);

 321   _free_set = new ShenandoahFreeSet(this, _num_regions);
 322 
 323   {
 324     ShenandoahHeapLocker locker(lock());
 325 
 326     for (size_t i = 0; i < _num_regions; i++) {
 327       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
 328       bool is_committed = i < num_committed_regions;
 329       void* loc = region_storage.base() + i * region_align;
 330 
 331       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
 332       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
 333 
 334       _marking_context->initialize_top_at_mark_start(r);
 335       _regions[i] = r;
 336       assert(!collection_set()->is_in(i), "New region should not be in collection set");


 337     }
 338 
 339     // Initialize to complete
 340     _marking_context->mark_complete();

 341 
 342     _free_set->rebuild();



 343   }
 344 
 345   if (AlwaysPreTouch) {
 346     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 347     // before initialize() below zeroes it with initializing thread. For any given region,
 348     // we touch the region and the corresponding bitmaps from the same thread.
 349     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 350 
 351     _pretouch_heap_page_size = heap_page_size;
 352     _pretouch_bitmap_page_size = bitmap_page_size;
 353 
 354 #ifdef LINUX
 355     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 356     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 357     // them into huge one. Therefore, we need to pretouch with smaller pages.
 358     if (UseTransparentHugePages) {
 359       _pretouch_heap_page_size = (size_t)os::vm_page_size();
 360       _pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 361     }
 362 #endif
 363 
 364     // OS memory managers may want to coalesce back-to-back pages. Make their jobs
 365     // simpler by pre-touching continuous spaces (heap and bitmap) separately.
 366 
 367     ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, _pretouch_bitmap_page_size);
 368     _workers->run_task(&bcl);
 369 
 370     ShenandoahPretouchHeapTask hcl(_pretouch_heap_page_size);
 371     _workers->run_task(&hcl);
 372   }
 373 
 374   //
 375   // Initialize the rest of GC subsystems
 376   //
 377 
 378   _liveness_cache = NEW_C_HEAP_ARRAY(ShenandoahLiveData*, _max_workers, mtGC);
 379   for (uint worker = 0; worker < _max_workers; worker++) {
 380     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(ShenandoahLiveData, _num_regions, mtGC);
 381     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
 382   }
 383 
 384   // There should probably be Shenandoah-specific options for these,
 385   // just as there are G1-specific options.
 386   {
 387     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 388     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 389     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 390   }
 391 
 392   _monitoring_support = new ShenandoahMonitoringSupport(this);
 393   _phase_timings = new ShenandoahPhaseTimings(max_workers());
 394   ShenandoahCodeRoots::initialize();
 395 
 396   if (ShenandoahPacing) {
 397     _pacer = new ShenandoahPacer(this);
 398     _pacer->setup_for_idle();
 399   } else {
 400     _pacer = nullptr;
 401   }
 402 
 403   _control_thread = new ShenandoahControlThread();
 404 
 405   ShenandoahInitLogger::print();
 406 
 407   return JNI_OK;
 408 }
 409 








 410 void ShenandoahHeap::initialize_mode() {
 411   if (ShenandoahGCMode != nullptr) {
 412     if (strcmp(ShenandoahGCMode, "satb") == 0) {
 413       _gc_mode = new ShenandoahSATBMode();
 414     } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
 415       _gc_mode = new ShenandoahIUMode();
 416     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 417       _gc_mode = new ShenandoahPassiveMode();


 418     } else {
 419       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 420     }
 421   } else {
 422     vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option (null)");
 423   }
 424   _gc_mode->initialize_flags();
 425   if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 426     vm_exit_during_initialization(
 427             err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 428                     _gc_mode->name()));
 429   }
 430   if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
 431     vm_exit_during_initialization(
 432             err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 433                     _gc_mode->name()));
 434   }
 435 }
 436 
 437 void ShenandoahHeap::initialize_heuristics() {
 438   assert(_gc_mode != nullptr, "Must be initialized");
 439   _heuristics = _gc_mode->initialize_heuristics();
 440 
 441   if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 442     vm_exit_during_initialization(
 443             err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 444                     _heuristics->name()));
 445   }
 446   if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
 447     vm_exit_during_initialization(
 448             err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 449                     _heuristics->name()));
 450   }
 451 }
 452 
 453 #ifdef _MSC_VER
 454 #pragma warning( push )
 455 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 456 #endif
 457 
 458 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 459   CollectedHeap(),


 460   _initial_size(0),
 461   _used(0),
 462   _committed(0),
 463   _bytes_allocated_since_gc_start(0),
 464   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
 465   _workers(nullptr),
 466   _safepoint_workers(nullptr),
 467   _heap_region_special(false),
 468   _num_regions(0),
 469   _regions(nullptr),
 470   _update_refs_iterator(this),
 471   _gc_state_changed(false),




 472   _control_thread(nullptr),


 473   _shenandoah_policy(policy),
 474   _gc_mode(nullptr),
 475   _heuristics(nullptr),
 476   _free_set(nullptr),
 477   _pacer(nullptr),
 478   _verifier(nullptr),
 479   _phase_timings(nullptr),
 480   _monitoring_support(nullptr),
 481   _memory_pool(nullptr),
 482   _stw_memory_manager("Shenandoah Pauses"),
 483   _cycle_memory_manager("Shenandoah Cycles"),
 484   _gc_timer(new ConcurrentGCTimer()),
 485   _soft_ref_policy(),
 486   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
 487   _ref_processor(new ShenandoahReferenceProcessor(MAX2(_max_workers, 1U))),
 488   _marking_context(nullptr),
 489   _bitmap_size(0),
 490   _bitmap_regions_per_slice(0),
 491   _bitmap_bytes_per_slice(0),
 492   _bitmap_region_special(false),
 493   _aux_bitmap_region_special(false),
 494   _liveness_cache(nullptr),
 495   _collection_set(nullptr)
 496 {
 497   // Initialize GC mode early, so we can adjust barrier support
 498   initialize_mode();
 499   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
 500 
 501   _max_workers = MAX2(_max_workers, 1U);
 502   _workers = new ShenandoahWorkerThreads("Shenandoah GC Threads", _max_workers);
 503   if (_workers == nullptr) {
 504     vm_exit_during_initialization("Failed necessary allocation.");
 505   } else {
 506     _workers->initialize_workers();
 507   }
 508 
 509   if (ParallelGCThreads > 1) {
 510     _safepoint_workers = new ShenandoahWorkerThreads("Safepoint Cleanup Thread",
 511                                                 ParallelGCThreads);
 512     _safepoint_workers->initialize_workers();
 513   }
 514 }
 515 
 516 #ifdef _MSC_VER
 517 #pragma warning( pop )
 518 #endif
 519 
 520 class ShenandoahResetBitmapTask : public WorkerTask {
 521 private:
 522   ShenandoahRegionIterator _regions;
 523 
 524 public:
 525   ShenandoahResetBitmapTask() :
 526     WorkerTask("Shenandoah Reset Bitmap") {}
 527 
 528   void work(uint worker_id) {
 529     ShenandoahHeapRegion* region = _regions.next();
 530     ShenandoahHeap* heap = ShenandoahHeap::heap();
 531     ShenandoahMarkingContext* const ctx = heap->marking_context();
 532     while (region != nullptr) {
 533       if (heap->is_bitmap_slice_committed(region)) {
 534         ctx->clear_bitmap(region);
 535       }
 536       region = _regions.next();
 537     }
 538   }
 539 };
 540 
 541 void ShenandoahHeap::reset_mark_bitmap() {
 542   assert_gc_workers(_workers->active_workers());
 543   mark_incomplete_marking_context();
 544 
 545   ShenandoahResetBitmapTask task;
 546   _workers->run_task(&task);
 547 }
 548 
 549 void ShenandoahHeap::print_on(outputStream* st) const {
 550   st->print_cr("Shenandoah Heap");
 551   st->print_cr(" " SIZE_FORMAT "%s max, " SIZE_FORMAT "%s soft max, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
 552                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 553                byte_size_in_proper_unit(soft_max_capacity()), proper_unit_for_byte_size(soft_max_capacity()),
 554                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 555                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 556   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
 557                num_regions(),
 558                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 559                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 560 
 561   st->print("Status: ");
 562   if (has_forwarded_objects())                 st->print("has forwarded objects, ");
 563   if (is_concurrent_mark_in_progress())        st->print("marking, ");





 564   if (is_evacuation_in_progress())             st->print("evacuating, ");
 565   if (is_update_refs_in_progress())            st->print("updating refs, ");
 566   if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
 567   if (is_full_gc_in_progress())                st->print("full gc, ");
 568   if (is_full_gc_move_in_progress())           st->print("full gc move, ");
 569   if (is_concurrent_weak_root_in_progress())   st->print("concurrent weak roots, ");
 570   if (is_concurrent_strong_root_in_progress() &&
 571       !is_concurrent_weak_root_in_progress())  st->print("concurrent strong roots, ");
 572 
 573   if (cancelled_gc()) {
 574     st->print("cancelled");
 575   } else {
 576     st->print("not cancelled");
 577   }
 578   st->cr();
 579 
 580   st->print_cr("Reserved region:");
 581   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 582                p2i(reserved_region().start()),
 583                p2i(reserved_region().end()));

 594   st->cr();
 595   MetaspaceUtils::print_on(st);
 596 
 597   if (Verbose) {
 598     st->cr();
 599     print_heap_regions_on(st);
 600   }
 601 }
 602 
 603 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 604 public:
 605   void do_thread(Thread* thread) {
 606     assert(thread != nullptr, "Sanity");
 607     assert(thread->is_Worker_thread(), "Only worker thread expected");
 608     ShenandoahThreadLocalData::initialize_gclab(thread);
 609   }
 610 };
 611 
 612 void ShenandoahHeap::post_initialize() {
 613   CollectedHeap::post_initialize();




 614   MutexLocker ml(Threads_lock);
 615 
 616   ShenandoahInitWorkerGCLABClosure init_gclabs;
 617   _workers->threads_do(&init_gclabs);
 618 
 619   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 620   // Now, we will let WorkerThreads to initialize gclab when new worker is created.
 621   _workers->set_initialize_gclab();
 622   if (_safepoint_workers != nullptr) {
 623     _safepoint_workers->threads_do(&init_gclabs);
 624     _safepoint_workers->set_initialize_gclab();
 625   }
 626 
 627   _heuristics->initialize();
 628 
 629   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
 630 }
 631 




 632 size_t ShenandoahHeap::used() const {
 633   return Atomic::load(&_used);
 634 }
 635 
 636 size_t ShenandoahHeap::committed() const {
 637   return Atomic::load(&_committed);
 638 }
 639 
 640 void ShenandoahHeap::increase_committed(size_t bytes) {
 641   shenandoah_assert_heaplocked_or_safepoint();
 642   _committed += bytes;
 643 }
 644 
 645 void ShenandoahHeap::decrease_committed(size_t bytes) {
 646   shenandoah_assert_heaplocked_or_safepoint();
 647   _committed -= bytes;
 648 }
 649 
 650 void ShenandoahHeap::increase_used(size_t bytes) {
 651   Atomic::add(&_used, bytes, memory_order_relaxed);









































 652 }
 653 
 654 void ShenandoahHeap::set_used(size_t bytes) {
 655   Atomic::store(&_used, bytes);



 656 }
 657 
 658 void ShenandoahHeap::decrease_used(size_t bytes) {
 659   assert(used() >= bytes, "never decrease heap size by more than we've left");
 660   Atomic::sub(&_used, bytes, memory_order_relaxed);


 661 }
 662 
 663 void ShenandoahHeap::increase_allocated(size_t bytes) {
 664   Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed);



 665 }
 666 
 667 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
 668   size_t bytes = words * HeapWordSize;
 669   if (!waste) {
 670     increase_used(bytes);
 671   }
 672   increase_allocated(bytes);


 673   if (ShenandoahPacing) {
 674     control_thread()->pacing_notify_alloc(words);
 675     if (waste) {
 676       pacer()->claim_for_alloc(words, true);
 677     }
 678   }
 679 }
 680 
 681 size_t ShenandoahHeap::capacity() const {
 682   return committed();
 683 }
 684 
 685 size_t ShenandoahHeap::max_capacity() const {
 686   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 687 }
 688 
 689 size_t ShenandoahHeap::soft_max_capacity() const {
 690   size_t v = Atomic::load(&_soft_max_size);
 691   assert(min_capacity() <= v && v <= max_capacity(),
 692          "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
 693          min_capacity(), v, max_capacity());
 694   return v;
 695 }
 696 
 697 void ShenandoahHeap::set_soft_max_capacity(size_t v) {
 698   assert(min_capacity() <= v && v <= max_capacity(),
 699          "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
 700          min_capacity(), v, max_capacity());
 701   Atomic::store(&_soft_max_size, v);
 702 }
 703 
 704 size_t ShenandoahHeap::min_capacity() const {
 705   return _minimum_size;
 706 }
 707 
 708 size_t ShenandoahHeap::initial_capacity() const {
 709   return _initial_size;
 710 }
 711 
 712 bool ShenandoahHeap::is_in(const void* p) const {
 713   HeapWord* heap_base = (HeapWord*) base();
 714   HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
 715   return p >= heap_base && p < last_region_end;





















 716 }
 717 
 718 void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) {
 719   assert (ShenandoahUncommit, "should be enabled");
 720 
 721   // Application allocates from the beginning of the heap, and GC allocates at
 722   // the end of it. It is more efficient to uncommit from the end, so that applications
 723   // could enjoy the near committed regions. GC allocations are much less frequent,
 724   // and therefore can accept the committing costs.
 725 
 726   size_t count = 0;
 727   for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
 728     ShenandoahHeapRegion* r = get_region(i - 1);
 729     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 730       ShenandoahHeapLocker locker(lock());
 731       if (r->is_empty_committed()) {
 732         if (committed() < shrink_until + ShenandoahHeapRegion::region_size_bytes()) {
 733           break;
 734         }
 735 
 736         r->make_uncommitted();
 737         count++;
 738       }
 739     }
 740     SpinPause(); // allow allocators to take the lock
 741   }
 742 
 743   if (count > 0) {
 744     control_thread()->notify_heap_changed();

















 745   }
















 746 }
 747 
 748 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 749   // New object should fit the GCLAB size
 750   size_t min_size = MAX2(size, PLAB::min_size());
 751 
 752   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 753   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;

 754   new_size = MIN2(new_size, PLAB::max_size());
 755   new_size = MAX2(new_size, PLAB::min_size());
 756 
 757   // Record new heuristic value even if we take any shortcut. This captures
 758   // the case when moderately-sized objects always take a shortcut. At some point,
 759   // heuristics should catch up with them.

 760   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 761 
 762   if (new_size < size) {
 763     // New size still does not fit the object. Fall back to shared allocation.
 764     // This avoids retiring perfectly good GCLABs, when we encounter a large object.

 765     return nullptr;
 766   }
 767 
 768   // Retire current GCLAB, and allocate a new one.
 769   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 770   gclab->retire();
 771 
 772   size_t actual_size = 0;
 773   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 774   if (gclab_buf == nullptr) {
 775     return nullptr;
 776   }
 777 
 778   assert (size <= actual_size, "allocation should fit");
 779 
 780   // ...and clear or zap just allocated TLAB, if needed.
 781   if (ZeroTLAB) {
 782     Copy::zero_to_words(gclab_buf, actual_size);
 783   } else if (ZapTLAB) {
 784     // Skip mangling the space corresponding to the object header to
 785     // ensure that the returned space is not considered parsable by
 786     // any concurrent GC thread.
 787     size_t hdr_size = oopDesc::header_size();
 788     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 789   }
 790   gclab->set_buf(gclab_buf, actual_size);
 791   return gclab->allocate(size);
 792 }
 793 

 794 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 795                                             size_t requested_size,
 796                                             size_t* actual_size) {
 797   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 798   HeapWord* res = allocate_memory(req);
 799   if (res != nullptr) {
 800     *actual_size = req.actual_size();
 801   } else {
 802     *actual_size = 0;
 803   }
 804   return res;
 805 }
 806 
 807 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 808                                              size_t word_size,
 809                                              size_t* actual_size) {
 810   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 811   HeapWord* res = allocate_memory(req);
 812   if (res != nullptr) {
 813     *actual_size = req.actual_size();

 815     *actual_size = 0;
 816   }
 817   return res;
 818 }
 819 
 820 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 821   intptr_t pacer_epoch = 0;
 822   bool in_new_region = false;
 823   HeapWord* result = nullptr;
 824 
 825   if (req.is_mutator_alloc()) {
 826     if (ShenandoahPacing) {
 827       pacer()->pace_for_alloc(req.size());
 828       pacer_epoch = pacer()->epoch();
 829     }
 830 
 831     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
 832       result = allocate_memory_under_lock(req, in_new_region);
 833     }
 834 
 835     // Allocation failed, block until control thread reacted, then retry allocation.
 836     //
 837     // It might happen that one of the threads requesting allocation would unblock
 838     // way later after GC happened, only to fail the second allocation, because
 839     // other threads have already depleted the free storage. In this case, a better
 840     // strategy is to try again, as long as GC makes progress (or until at least
 841     // one full GC has completed).
 842     size_t original_count = shenandoah_policy()->full_gc_count();
 843     while (result == nullptr
 844         && (_progress_last_gc.is_set() || original_count == shenandoah_policy()->full_gc_count())) {
 845       control_thread()->handle_alloc_failure(req);
 846       result = allocate_memory_under_lock(req, in_new_region);






























 847     }
 848   } else {
 849     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
 850     result = allocate_memory_under_lock(req, in_new_region);
 851     // Do not call handle_alloc_failure() here, because we cannot block.
 852     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
 853   }
 854 
 855   if (in_new_region) {
 856     control_thread()->notify_heap_changed();




 857   }
 858 




 859   if (result != nullptr) {
 860     size_t requested = req.size();
 861     size_t actual = req.actual_size();
 862 
 863     assert (req.is_lab_alloc() || (requested == actual),
 864             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
 865             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
 866 
 867     if (req.is_mutator_alloc()) {
 868       notify_mutator_alloc_words(actual, false);
 869 
 870       // If we requested more than we were granted, give the rest back to pacer.
 871       // This only matters if we are in the same pacing epoch: do not try to unpace
 872       // over the budget for the other phase.
 873       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
 874         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
 875       }
 876     } else {
 877       increase_used(actual*HeapWordSize);
 878     }
 879   }
 880 
 881   return result;
 882 }
 883 
 884 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
 885   // If we are dealing with mutator allocation, then we may need to block for safepoint.
 886   // We cannot block for safepoint for GC allocations, because there is a high chance
 887   // we are already running at safepoint or from stack watermark machinery, and we cannot
 888   // block again.
 889   ShenandoahHeapLocker locker(lock(), req.is_mutator_alloc());
 890   return _free_set->allocate(req, in_new_region);




































 891 }
 892 
 893 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
 894                                         bool*  gc_overhead_limit_was_exceeded) {
 895   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
 896   return allocate_memory(req);
 897 }
 898 
 899 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 900                                                              size_t size,
 901                                                              Metaspace::MetadataType mdtype) {
 902   MetaWord* result;
 903 
 904   // Inform metaspace OOM to GC heuristics if class unloading is possible.
 905   if (heuristics()->can_unload_classes()) {
 906     ShenandoahHeuristics* h = heuristics();
 907     h->record_metaspace_oom();
 908   }
 909 
 910   // Expand and retry allocation
 911   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 912   if (result != nullptr) {
 913     return result;
 914   }
 915 
 916   // Start full GC
 917   collect(GCCause::_metadata_GC_clear_soft_refs);
 918 
 919   // Retry allocation
 920   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
 921   if (result != nullptr) {
 922     return result;
 923   }
 924 
 925   // Expand and retry allocation
 926   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);

 983     while ((r =_cs->claim_next()) != nullptr) {
 984       assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
 985       _sh->marked_object_iterate(r, &cl);
 986 
 987       if (ShenandoahPacing) {
 988         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
 989       }
 990 
 991       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
 992         break;
 993       }
 994     }
 995   }
 996 };
 997 
 998 void ShenandoahHeap::evacuate_collection_set(bool concurrent) {
 999   ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1000   workers()->run_task(&task);
1001 }
1002 
























































































1003 void ShenandoahHeap::trash_cset_regions() {
1004   ShenandoahHeapLocker locker(lock());
1005 
1006   ShenandoahCollectionSet* set = collection_set();
1007   ShenandoahHeapRegion* r;
1008   set->clear_current_index();
1009   while ((r = set->next()) != nullptr) {
1010     r->make_trash();
1011   }
1012   collection_set()->clear();
1013 }
1014 
1015 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1016   st->print_cr("Heap Regions:");
1017   st->print_cr("Region state: EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HP=pinned humongous start");
1018   st->print_cr("              HC=humongous continuation, CS=collection set, TR=trash, P=pinned, CSP=pinned collection set");
1019   st->print_cr("BTE=bottom/top/end, TAMS=top-at-mark-start");
1020   st->print_cr("UWM=update watermark, U=used");
1021   st->print_cr("T=TLAB allocs, G=GCLAB allocs");
1022   st->print_cr("S=shared allocs, L=live data");
1023   st->print_cr("CP=critical pins");
1024 
1025   for (size_t i = 0; i < num_regions(); i++) {
1026     get_region(i)->print_on(st);
1027   }
1028 }
1029 
1030 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1031   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1032 
1033   oop humongous_obj = cast_to_oop(start->bottom());
1034   size_t size = humongous_obj->size();
1035   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1036   size_t index = start->index() + required_regions - 1;
1037 
1038   assert(!start->has_live(), "liveness must be zero");
1039 
1040   for(size_t i = 0; i < required_regions; i++) {
1041     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1042     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1043     ShenandoahHeapRegion* region = get_region(index --);
1044 
1045     assert(region->is_humongous(), "expect correct humongous start or continuation");
1046     assert(!region->is_cset(), "Humongous region should not be in collection set");
1047 
1048     region->make_trash_immediate();
1049   }

1050 }
1051 
1052 class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
1053 public:
1054   ShenandoahCheckCleanGCLABClosure() {}
1055   void do_thread(Thread* thread) {
1056     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1057     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1058     assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");






1059   }
1060 };
1061 
1062 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1063 private:
1064   bool const _resize;
1065 public:
1066   ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
1067   void do_thread(Thread* thread) {
1068     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1069     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1070     gclab->retire();
1071     if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1072       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1073     }













1074   }
1075 };
1076 
1077 void ShenandoahHeap::labs_make_parsable() {
1078   assert(UseTLAB, "Only call with UseTLAB");
1079 
1080   ShenandoahRetireGCLABClosure cl(false);
1081 
1082   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1083     ThreadLocalAllocBuffer& tlab = t->tlab();
1084     tlab.make_parsable();
1085     cl.do_thread(t);
1086   }
1087 
1088   workers()->threads_do(&cl);
1089 }
1090 
1091 void ShenandoahHeap::tlabs_retire(bool resize) {
1092   assert(UseTLAB, "Only call with UseTLAB");
1093   assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");

1155   }
1156   return nullptr;
1157 }
1158 
1159 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1160   ShenandoahHeapRegion* r = heap_region_containing(addr);
1161   return r->block_is_obj(addr);
1162 }
1163 
1164 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1165   return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1166 }
1167 
1168 void ShenandoahHeap::prepare_for_verify() {
1169   if (SafepointSynchronize::is_at_safepoint() && UseTLAB) {
1170     labs_make_parsable();
1171   }
1172 }
1173 
1174 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1175   tcl->do_thread(_control_thread);







1176   workers()->threads_do(tcl);
1177   if (_safepoint_workers != nullptr) {
1178     _safepoint_workers->threads_do(tcl);
1179   }
1180 }
1181 
1182 void ShenandoahHeap::print_tracing_info() const {
1183   LogTarget(Info, gc, stats) lt;
1184   if (lt.is_enabled()) {
1185     ResourceMark rm;
1186     LogStream ls(lt);
1187 
1188     phase_timings()->print_global_on(&ls);
1189 
1190     ls.cr();
1191     ls.cr();
1192 
1193     shenandoah_policy()->print_gc_stats(&ls);
1194 
1195     ls.cr();
1196     ls.cr();
1197   }
1198 }
1199 








































1200 void ShenandoahHeap::verify(VerifyOption vo) {
1201   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1202     if (ShenandoahVerify) {
1203       verifier()->verify_generic(vo);
1204     } else {
1205       // TODO: Consider allocating verification bitmaps on demand,
1206       // and turn this on unconditionally.
1207     }
1208   }
1209 }
1210 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1211   return _free_set->capacity();
1212 }
1213 
1214 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1215 private:
1216   MarkBitMap* _bitmap;
1217   ShenandoahScanObjectStack* _oop_stack;
1218   ShenandoahHeap* const _heap;
1219   ShenandoahMarkingContext* const _marking_context;

1514   const uint active_workers = workers()->active_workers();
1515   const size_t n_regions = num_regions();
1516   size_t stride = ShenandoahParallelRegionStride;
1517   if (stride == 0 && active_workers > 1) {
1518     // Automatically derive the stride to balance the work between threads
1519     // evenly. Do not try to split work if below the reasonable threshold.
1520     constexpr size_t threshold = 4096;
1521     stride = n_regions <= threshold ?
1522             threshold :
1523             (n_regions + active_workers - 1) / active_workers;
1524   }
1525 
1526   if (n_regions > stride && active_workers > 1) {
1527     ShenandoahParallelHeapRegionTask task(blk, stride);
1528     workers()->run_task(&task);
1529   } else {
1530     heap_region_iterate(blk);
1531   }
1532 }
1533 
1534 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1535 private:
1536   ShenandoahMarkingContext* const _ctx;
1537 public:
1538   ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1539 
1540   void heap_region_do(ShenandoahHeapRegion* r) {
1541     assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1542     if (r->is_active()) {
1543       // Check if region needs updating its TAMS. We have updated it already during concurrent
1544       // reset, so it is very likely we don't need to do another write here.
1545       if (_ctx->top_at_mark_start(r) != r->top()) {
1546         _ctx->capture_top_at_mark_start(r);
1547       }
1548     } else {
1549       assert(_ctx->top_at_mark_start(r) == r->top(),
1550              "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
1551     }
1552   }
1553 
1554   bool is_thread_safe() { return true; }
1555 };
1556 
1557 class ShenandoahRendezvousClosure : public HandshakeClosure {
1558 public:
1559   inline ShenandoahRendezvousClosure() : HandshakeClosure("ShenandoahRendezvous") {}
1560   inline void do_thread(Thread* thread) {}
1561 };
1562 
1563 void ShenandoahHeap::rendezvous_threads() {
1564   ShenandoahRendezvousClosure cl;
1565   Handshake::execute(&cl);
1566 }
1567 
1568 void ShenandoahHeap::recycle_trash() {
1569   free_set()->recycle_trash();
1570 }
1571 
1572 class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1573 private:
1574   ShenandoahMarkingContext* const _ctx;
1575 public:
1576   ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1577 
1578   void heap_region_do(ShenandoahHeapRegion* r) {
1579     if (r->is_active()) {
1580       // Reset live data and set TAMS optimistically. We would recheck these under the pause
1581       // anyway to capture any updates that happened since now.
1582       r->clear_live_data();
1583       _ctx->capture_top_at_mark_start(r);
1584     }
1585   }
1586 
1587   bool is_thread_safe() { return true; }
1588 };
1589 
1590 void ShenandoahHeap::prepare_gc() {
1591   reset_mark_bitmap();
1592 
1593   ShenandoahResetUpdateRegionStateClosure cl;
1594   parallel_heap_region_iterate(&cl);
1595 }
1596 
1597 class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1598 private:
1599   ShenandoahMarkingContext* const _ctx;
1600   ShenandoahHeapLock* const _lock;
1601 
1602 public:
1603   ShenandoahFinalMarkUpdateRegionStateClosure() :
1604     _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
1605 
1606   void heap_region_do(ShenandoahHeapRegion* r) {
1607     if (r->is_active()) {
1608       // All allocations past TAMS are implicitly live, adjust the region data.
1609       // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1610       HeapWord *tams = _ctx->top_at_mark_start(r);
1611       HeapWord *top = r->top();
1612       if (top > tams) {
1613         r->increase_live_data_alloc_words(pointer_delta(top, tams));
1614       }
1615 
1616       // We are about to select the collection set, make sure it knows about
1617       // current pinning status. Also, this allows trashing more regions that
1618       // now have their pinning status dropped.
1619       if (r->is_pinned()) {
1620         if (r->pin_count() == 0) {
1621           ShenandoahHeapLocker locker(_lock);
1622           r->make_unpinned();
1623         }
1624       } else {
1625         if (r->pin_count() > 0) {
1626           ShenandoahHeapLocker locker(_lock);
1627           r->make_pinned();
1628         }
1629       }
1630 
1631       // Remember limit for updating refs. It's guaranteed that we get no
1632       // from-space-refs written from here on.
1633       r->set_update_watermark_at_safepoint(r->top());
1634     } else {
1635       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1636       assert(_ctx->top_at_mark_start(r) == r->top(),
1637              "Region " SIZE_FORMAT " should have correct TAMS", r->index());
1638     }
1639   }
1640 
1641   bool is_thread_safe() { return true; }
1642 };
1643 
1644 void ShenandoahHeap::prepare_regions_and_collection_set(bool concurrent) {
1645   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
1646   {
1647     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states :
1648                                          ShenandoahPhaseTimings::degen_gc_final_update_region_states);
1649     ShenandoahFinalMarkUpdateRegionStateClosure cl;
1650     parallel_heap_region_iterate(&cl);
1651 
1652     assert_pinned_region_status();
1653   }
1654 
1655   {
1656     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset :
1657                                          ShenandoahPhaseTimings::degen_gc_choose_cset);
1658     ShenandoahHeapLocker locker(lock());
1659     _collection_set->clear();
1660     heuristics()->choose_collection_set(_collection_set);
1661   }
1662 
1663   {
1664     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset :
1665                                          ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
1666     ShenandoahHeapLocker locker(lock());
1667     _free_set->rebuild();
1668   }
1669 }
1670 
1671 void ShenandoahHeap::do_class_unloading() {
1672   _unloader.unload();



1673 }
1674 
1675 void ShenandoahHeap::stw_weak_refs(bool full_gc) {
1676   // Weak refs processing
1677   ShenandoahPhaseTimings::Phase phase = full_gc ? ShenandoahPhaseTimings::full_gc_weakrefs
1678                                                 : ShenandoahPhaseTimings::degen_gc_weakrefs;
1679   ShenandoahTimingsTracker t(phase);
1680   ShenandoahGCWorkerPhase worker_phase(phase);
1681   ref_processor()->process_references(phase, workers(), false /* concurrent */);

1682 }
1683 
1684 void ShenandoahHeap::prepare_update_heap_references(bool concurrent) {
1685   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1686 
1687   // Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to
1688   // make them parsable for update code to work correctly. Plus, we can compute new sizes
1689   // for future GCLABs here.
1690   if (UseTLAB) {
1691     ShenandoahGCPhase phase(concurrent ?
1692                             ShenandoahPhaseTimings::init_update_refs_manage_gclabs :
1693                             ShenandoahPhaseTimings::degen_gc_init_update_refs_manage_gclabs);
1694     gclabs_retire(ResizeTLAB);
1695   }
1696 
1697   _update_refs_iterator.reset();
1698 }
1699 
1700 void ShenandoahHeap::propagate_gc_state_to_java_threads() {
1701   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1702   if (_gc_state_changed) {
1703     _gc_state_changed = false;
1704     char state = gc_state();
1705     for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1706       ShenandoahThreadLocalData::set_gc_state(t, state);
1707     }
1708   }
1709 }
1710 
1711 void ShenandoahHeap::set_gc_state(uint mask, bool value) {
1712   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1713   _gc_state.set_cond(mask, value);
1714   _gc_state_changed = true;
1715 }
1716 
1717 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
1718   assert(!has_forwarded_objects(), "Not expected before/after mark phase");
1719   set_gc_state(MARKING, in_progress);
1720   ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
















































1721 }
1722 
1723 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
1724   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
1725   set_gc_state(EVACUATION, in_progress);
1726 }
1727 
1728 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
1729   if (in_progress) {
1730     _concurrent_strong_root_in_progress.set();
1731   } else {
1732     _concurrent_strong_root_in_progress.unset();
1733   }
1734 }
1735 
1736 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool cond) {
1737   set_gc_state(WEAK_ROOTS, cond);
1738 }
1739 
1740 GCTracer* ShenandoahHeap::tracer() {
1741   return shenandoah_policy()->tracer();
1742 }
1743 
1744 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
1745   return _free_set->used();
1746 }
1747 
1748 bool ShenandoahHeap::try_cancel_gc() {
1749   jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
1750   return prev == CANCELLABLE;
1751 }
1752 











1753 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
1754   if (try_cancel_gc()) {
1755     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
1756     log_info(gc)("%s", msg.buffer());
1757     Events::log(Thread::current(), "%s", msg.buffer());

1758   }
1759 }
1760 
1761 uint ShenandoahHeap::max_workers() {
1762   return _max_workers;
1763 }
1764 
1765 void ShenandoahHeap::stop() {
1766   // The shutdown sequence should be able to terminate when GC is running.
1767 
1768   // Step 0. Notify policy to disable event recording.
1769   _shenandoah_policy->record_shutdown();
1770 



1771   // Step 1. Notify control thread that we are in shutdown.
1772   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
1773   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
1774   control_thread()->prepare_for_graceful_shutdown();
1775 
1776   // Step 2. Notify GC workers that we are cancelling GC.
1777   cancel_gc(GCCause::_shenandoah_stop_vm);
1778 
1779   // Step 3. Wait until GC worker exits normally.
1780   control_thread()->stop();
1781 }
1782 
1783 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
1784   if (!unload_classes()) return;
1785   ClassUnloadingContext ctx(_workers->active_workers(),
1786                             true /* unregister_nmethods_during_purge */,
1787                             false /* lock_codeblob_free_separately */);
1788 
1789   // Unload classes and purge SystemDictionary.
1790   {

1861 }
1862 
1863 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
1864   set_gc_state(HAS_FORWARDED, cond);
1865 }
1866 
1867 void ShenandoahHeap::set_unload_classes(bool uc) {
1868   _unload_classes.set_cond(uc);
1869 }
1870 
1871 bool ShenandoahHeap::unload_classes() const {
1872   return _unload_classes.is_set();
1873 }
1874 
1875 address ShenandoahHeap::in_cset_fast_test_addr() {
1876   ShenandoahHeap* heap = ShenandoahHeap::heap();
1877   assert(heap->collection_set() != nullptr, "Sanity");
1878   return (address) heap->collection_set()->biased_map_address();
1879 }
1880 
1881 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
1882   return Atomic::load(&_bytes_allocated_since_gc_start);
1883 }
1884 
1885 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
1886   Atomic::store(&_bytes_allocated_since_gc_start, (size_t)0);





1887 }
1888 
1889 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
1890   _degenerated_gc_in_progress.set_cond(in_progress);
1891 }
1892 
1893 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
1894   _full_gc_in_progress.set_cond(in_progress);
1895 }
1896 
1897 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
1898   assert (is_full_gc_in_progress(), "should be");
1899   _full_gc_move_in_progress.set_cond(in_progress);
1900 }
1901 
1902 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
1903   set_gc_state(UPDATEREFS, in_progress);
1904 }
1905 
1906 void ShenandoahHeap::register_nmethod(nmethod* nm) {

1930     if (r->is_active()) {
1931       if (r->is_pinned()) {
1932         if (r->pin_count() == 0) {
1933           r->make_unpinned();
1934         }
1935       } else {
1936         if (r->pin_count() > 0) {
1937           r->make_pinned();
1938         }
1939       }
1940     }
1941   }
1942 
1943   assert_pinned_region_status();
1944 }
1945 
1946 #ifdef ASSERT
1947 void ShenandoahHeap::assert_pinned_region_status() {
1948   for (size_t i = 0; i < num_regions(); i++) {
1949     ShenandoahHeapRegion* r = get_region(i);
1950     assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
1951            "Region " SIZE_FORMAT " pinning status is inconsistent", i);



1952   }
1953 }
1954 #endif
1955 
1956 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
1957   return _gc_timer;
1958 }
1959 
1960 void ShenandoahHeap::prepare_concurrent_roots() {
1961   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
1962   assert(!is_stw_gc_in_progress(), "Only concurrent GC");
1963   set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
1964   set_concurrent_weak_root_in_progress(true);
1965   if (unload_classes()) {
1966     _unloader.prepare();
1967   }
1968 }
1969 
1970 void ShenandoahHeap::finish_concurrent_roots() {
1971   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
1972   assert(!is_stw_gc_in_progress(), "Only concurrent GC");
1973   if (unload_classes()) {
1974     _unloader.finish();
1975   }
1976 }
1977 
1978 #ifdef ASSERT
1979 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
1980   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
1981 
1982   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1983     if (UseDynamicNumberOfGCThreads) {
1984       assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
1985     } else {
1986       // Use ParallelGCThreads inside safepoints
1987       assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads within safepoints");
1988     }
1989   } else {
1990     if (UseDynamicNumberOfGCThreads) {
1991       assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
1992     } else {
1993       // Use ConcGCThreads outside safepoints
1994       assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
1995     }
1996   }
1997 }
1998 #endif
1999 
2000 ShenandoahVerifier* ShenandoahHeap::verifier() {
2001   guarantee(ShenandoahVerify, "Should be enabled");
2002   assert (_verifier != nullptr, "sanity");
2003   return _verifier;
2004 }
2005 
2006 template<bool CONCURRENT>
2007 class ShenandoahUpdateHeapRefsTask : public WorkerTask {
2008 private:
2009   ShenandoahHeap* _heap;
2010   ShenandoahRegionIterator* _regions;
2011 public:
2012   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) :
2013     WorkerTask("Shenandoah Update References"),
2014     _heap(ShenandoahHeap::heap()),
2015     _regions(regions) {
2016   }
2017 
2018   void work(uint worker_id) {
2019     if (CONCURRENT) {
2020       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2021       ShenandoahSuspendibleThreadSetJoiner stsj;
2022       do_work<ShenandoahConcUpdateRefsClosure>();
2023     } else {
2024       ShenandoahParallelWorkerSession worker_session(worker_id);
2025       do_work<ShenandoahSTWUpdateRefsClosure>();
2026     }
2027   }
2028 
2029 private:
2030   template<class T>
2031   void do_work() {












2032     T cl;
2033     ShenandoahHeapRegion* r = _regions->next();
2034     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2035     while (r != nullptr) {
2036       HeapWord* update_watermark = r->get_update_watermark();
2037       assert (update_watermark >= r->bottom(), "sanity");
2038       if (r->is_active() && !r->is_cset()) {
2039         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2040       }
2041       if (ShenandoahPacing) {
2042         _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2043       }
2044       if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
2045         return;
2046       }
2047       r = _regions->next();
2048     }
2049   }
2050 };
2051 
2052 void ShenandoahHeap::update_heap_references(bool concurrent) {
2053   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2054 
2055   if (concurrent) {
2056     ShenandoahUpdateHeapRefsTask<true> task(&_update_refs_iterator);
2057     workers()->run_task(&task);
2058   } else {
2059     ShenandoahUpdateHeapRefsTask<false> task(&_update_refs_iterator);
2060     workers()->run_task(&task);
2061   }
2062 }
2063 
2064 
2065 class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
2066 private:
2067   ShenandoahHeapLock* const _lock;
2068 
2069 public:
2070   ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {}
2071 
2072   void heap_region_do(ShenandoahHeapRegion* r) {
2073     // Drop unnecessary "pinned" state from regions that does not have CP marks
2074     // anymore, as this would allow trashing them.
2075 
2076     if (r->is_active()) {
2077       if (r->is_pinned()) {
2078         if (r->pin_count() == 0) {
2079           ShenandoahHeapLocker locker(_lock);
2080           r->make_unpinned();
2081         }
2082       } else {
2083         if (r->pin_count() > 0) {
2084           ShenandoahHeapLocker locker(_lock);
2085           r->make_pinned();
2086         }
2087       }
2088     }
2089   }
2090 
2091   bool is_thread_safe() { return true; }
2092 };
2093 
2094 void ShenandoahHeap::update_heap_region_states(bool concurrent) {
2095   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2096   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2097 
2098   {
2099     ShenandoahGCPhase phase(concurrent ?
2100                             ShenandoahPhaseTimings::final_update_refs_update_region_states :
2101                             ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states);
2102     ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl;
2103     parallel_heap_region_iterate(&cl);
2104 
2105     assert_pinned_region_status();
2106   }
2107 
2108   {
2109     ShenandoahGCPhase phase(concurrent ?
2110                             ShenandoahPhaseTimings::final_update_refs_trash_cset :
2111                             ShenandoahPhaseTimings::degen_gc_final_update_refs_trash_cset);
2112     trash_cset_regions();
2113   }
2114 }
2115 





2116 void ShenandoahHeap::rebuild_free_set(bool concurrent) {
2117   {
2118     ShenandoahGCPhase phase(concurrent ?
2119                             ShenandoahPhaseTimings::final_update_refs_rebuild_freeset :
2120                             ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset);
2121     ShenandoahHeapLocker locker(lock());
2122     _free_set->rebuild();




































2123   }
2124 }
2125 
2126 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2127   print_on(st);
2128   st->cr();
2129   print_heap_regions_on(st);
2130 }
2131 
2132 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2133   size_t slice = r->index() / _bitmap_regions_per_slice;
2134 
2135   size_t regions_from = _bitmap_regions_per_slice * slice;
2136   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2137   for (size_t g = regions_from; g < regions_to; g++) {
2138     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2139     if (skip_self && g == r->index()) continue;
2140     if (get_region(g)->is_committed()) {
2141       return true;
2142     }

2190   }
2191 
2192   // Uncommit the bitmap slice:
2193   size_t slice = r->index() / _bitmap_regions_per_slice;
2194   size_t off = _bitmap_bytes_per_slice * slice;
2195   size_t len = _bitmap_bytes_per_slice;
2196   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2197     return false;
2198   }
2199   return true;
2200 }
2201 
2202 void ShenandoahHeap::safepoint_synchronize_begin() {
2203   SuspendibleThreadSet::synchronize();
2204 }
2205 
2206 void ShenandoahHeap::safepoint_synchronize_end() {
2207   SuspendibleThreadSet::desynchronize();
2208 }
2209 
2210 void ShenandoahHeap::entry_uncommit(double shrink_before, size_t shrink_until) {
2211   static const char *msg = "Concurrent uncommit";
2212   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_uncommit, true /* log_heap_usage */);
2213   EventMark em("%s", msg);
2214 
2215   op_uncommit(shrink_before, shrink_until);
2216 }
2217 
2218 void ShenandoahHeap::try_inject_alloc_failure() {
2219   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2220     _inject_alloc_failure.set();
2221     os::naked_short_sleep(1);
2222     if (cancelled_gc()) {
2223       log_info(gc)("Allocation failure was successfully injected");
2224     }
2225   }
2226 }
2227 
2228 bool ShenandoahHeap::should_inject_alloc_failure() {
2229   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2230 }
2231 
2232 void ShenandoahHeap::initialize_serviceability() {
2233   _memory_pool = new ShenandoahMemoryPool(this);
2234   _cycle_memory_manager.add_pool(_memory_pool);
2235   _stw_memory_manager.add_pool(_memory_pool);
2236 }
2237 
2238 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2239   GrowableArray<GCMemoryManager*> memory_managers(2);
2240   memory_managers.append(&_cycle_memory_manager);
2241   memory_managers.append(&_stw_memory_manager);
2242   return memory_managers;
2243 }
2244 
2245 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2246   GrowableArray<MemoryPool*> memory_pools(1);
2247   memory_pools.append(_memory_pool);
2248   return memory_pools;
2249 }
2250 
2251 MemoryUsage ShenandoahHeap::memory_usage() {
2252   return _memory_pool->get_memory_usage();
2253 }
2254 
2255 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2256   _heap(ShenandoahHeap::heap()),
2257   _index(0) {}
2258 
2259 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2260   _heap(heap),
2261   _index(0) {}
2262 
2263 void ShenandoahRegionIterator::reset() {
2264   _index = 0;
2265 }
2266 
2267 bool ShenandoahRegionIterator::has_next() const {
2268   return _index < _heap->num_regions();
2269 }
2270 
2271 char ShenandoahHeap::gc_state() const {
2272   return _gc_state.raw_value();

2297   }
2298 }
2299 
2300 bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
2301   if (is_idle()) return false;
2302 
2303   // Objects allocated after marking start are implicitly alive, don't need any barriers during
2304   // marking phase.
2305   if (is_concurrent_mark_in_progress() &&
2306      !marking_context()->allocated_after_mark_start(obj)) {
2307     return true;
2308   }
2309 
2310   // Can not guarantee obj is deeply good.
2311   if (has_forwarded_objects()) {
2312     return true;
2313   }
2314 
2315   return false;
2316 }























   1 /*
   2  * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.
   4  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "memory/allocation.hpp"
  29 #include "memory/universe.hpp"
  30 
  31 #include "gc/shared/classUnloadingContext.hpp"
  32 #include "gc/shared/gcArguments.hpp"
  33 #include "gc/shared/gcTimer.hpp"
  34 #include "gc/shared/gcTraceTime.inline.hpp"
  35 #include "gc/shared/locationPrinter.inline.hpp"
  36 #include "gc/shared/memAllocator.hpp"
  37 #include "gc/shared/plab.hpp"
  38 #include "gc/shared/tlab_globals.hpp"
  39 
  40 #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
  41 #include "gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp"
  42 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
  43 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  44 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  45 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  46 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  47 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
  48 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  49 #include "gc/shenandoah/shenandoahControlThread.hpp"
  50 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  51 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
  52 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
  53 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
  54 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  55 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  56 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
  57 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  58 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  59 #include "gc/shenandoah/shenandoahInitLogger.hpp"
  60 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  61 #include "gc/shenandoah/shenandoahMemoryPool.hpp"

  62 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  63 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
  64 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  65 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  66 #include "gc/shenandoah/shenandoahPadding.hpp"
  67 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  68 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  69 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  70 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
  71 #include "gc/shenandoah/shenandoahSTWMark.hpp"
  72 #include "gc/shenandoah/shenandoahUtils.hpp"
  73 #include "gc/shenandoah/shenandoahVerifier.hpp"
  74 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  75 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  76 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  77 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  78 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
  79 #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
  80 #include "gc/shenandoah/mode/shenandoahIUMode.hpp"
  81 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
  82 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
  83 #include "utilities/globalDefinitions.hpp"
  84 
  85 #if INCLUDE_JFR
  86 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  87 #endif
  88 
  89 #include "classfile/systemDictionary.hpp"
  90 #include "code/codeCache.hpp"
  91 #include "memory/classLoaderMetaspace.hpp"
  92 #include "memory/metaspaceUtils.hpp"
  93 #include "oops/compressedOops.inline.hpp"
  94 #include "prims/jvmtiTagMap.hpp"
  95 #include "runtime/atomic.hpp"
  96 #include "runtime/globals.hpp"
  97 #include "runtime/interfaceSupport.inline.hpp"
  98 #include "runtime/java.hpp"
  99 #include "runtime/orderAccess.hpp"
 100 #include "runtime/safepointMechanism.hpp"
 101 #include "runtime/vmThread.hpp"
 102 #include "services/mallocTracker.hpp"
 103 #include "services/memTracker.hpp"
 104 #include "utilities/events.hpp"

 156 jint ShenandoahHeap::initialize() {
 157   //
 158   // Figure out heap sizing
 159   //
 160 
 161   size_t init_byte_size = InitialHeapSize;
 162   size_t min_byte_size  = MinHeapSize;
 163   size_t max_byte_size  = MaxHeapSize;
 164   size_t heap_alignment = HeapAlignment;
 165 
 166   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 167 
 168   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 169   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 170 
 171   _num_regions = ShenandoahHeapRegion::region_count();
 172   assert(_num_regions == (max_byte_size / reg_size_bytes),
 173          "Regions should cover entire heap exactly: " SIZE_FORMAT " != " SIZE_FORMAT "/" SIZE_FORMAT,
 174          _num_regions, max_byte_size, reg_size_bytes);
 175 



 176   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 177   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 178   assert(num_committed_regions <= _num_regions, "sanity");
 179   _initial_size = num_committed_regions * reg_size_bytes;
 180 
 181   size_t num_min_regions = min_byte_size / reg_size_bytes;
 182   num_min_regions = MIN2(num_min_regions, _num_regions);
 183   assert(num_min_regions <= _num_regions, "sanity");
 184   _minimum_size = num_min_regions * reg_size_bytes;
 185 
 186   // Default to max heap size.
 187   _soft_max_size = _num_regions * reg_size_bytes;
 188 
 189   _committed = _initial_size;
 190 
 191   size_t heap_page_size   = UseLargePages ? os::large_page_size() : os::vm_page_size();
 192   size_t bitmap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 193   size_t region_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 194 
 195   //
 196   // Reserve and commit memory for heap
 197   //
 198 
 199   ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 200   initialize_reserved_region(heap_rs);
 201   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 202   _heap_region_special = heap_rs.special();
 203 
 204   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 205          "Misaligned heap: " PTR_FORMAT, p2i(base()));
 206   os::trace_page_sizes_for_requested_size("Heap",
 207                                           max_byte_size, heap_rs.page_size(), heap_alignment,
 208                                           heap_rs.base(), heap_rs.size());
 209 
 210 #if SHENANDOAH_OPTIMIZED_MARKTASK
 211   // The optimized ShenandoahMarkTask takes some bits away from the full object bits.
 212   // Fail if we ever attempt to address more than we can.
 213   if ((uintptr_t)heap_rs.end() >= ShenandoahMarkTask::max_addressable()) {
 214     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 215                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 216                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 217                 p2i(heap_rs.base()), p2i(heap_rs.end()), ShenandoahMarkTask::max_addressable());
 218     vm_exit_during_initialization("Fatal Error", buf);
 219   }
 220 #endif
 221 
 222   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 223   if (!_heap_region_special) {
 224     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 225                               "Cannot commit heap memory");
 226   }
 227 
 228   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this, _heap_region));
 229 
 230   // Now we know the number of regions and heap sizes, initialize the heuristics.
 231   initialize_heuristics();
 232 
 233   assert(_heap_region.byte_size() == heap_rs.size(), "Need to know reserved size for card table");
 234 
 235   //
 236   // Worker threads must be initialized after the barrier is configured
 237   //
 238   _workers = new ShenandoahWorkerThreads("Shenandoah GC Threads", _max_workers);
 239   if (_workers == nullptr) {
 240     vm_exit_during_initialization("Failed necessary allocation.");
 241   } else {
 242     _workers->initialize_workers();
 243   }
 244 
 245   if (ParallelGCThreads > 1) {
 246     _safepoint_workers = new ShenandoahWorkerThreads("Safepoint Cleanup Thread", ParallelGCThreads);
 247     _safepoint_workers->initialize_workers();
 248   }
 249 
 250   //
 251   // Reserve and commit memory for bitmap(s)
 252   //
 253 
 254   size_t bitmap_size_orig = ShenandoahMarkBitMap::compute_size(heap_rs.size());
 255   _bitmap_size = align_up(bitmap_size_orig, bitmap_page_size);
 256 
 257   size_t bitmap_bytes_per_region = reg_size_bytes / ShenandoahMarkBitMap::heap_map_factor();
 258 
 259   guarantee(bitmap_bytes_per_region != 0,
 260             "Bitmap bytes per region should not be zero");
 261   guarantee(is_power_of_2(bitmap_bytes_per_region),
 262             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 263 
 264   if (bitmap_page_size > bitmap_bytes_per_region) {
 265     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 266     _bitmap_bytes_per_slice = bitmap_page_size;
 267   } else {
 268     _bitmap_regions_per_slice = 1;
 269     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 270   }
 271 
 272   guarantee(_bitmap_regions_per_slice >= 1,
 273             "Should have at least one region per slice: " SIZE_FORMAT,
 274             _bitmap_regions_per_slice);
 275 
 276   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 277             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 278             _bitmap_bytes_per_slice, bitmap_page_size);
 279 
 280   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
 281   os::trace_page_sizes_for_requested_size("Mark Bitmap",
 282                                           bitmap_size_orig, bitmap.page_size(), bitmap_page_size,
 283                                           bitmap.base(),
 284                                           bitmap.size());
 285   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 286   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 287   _bitmap_region_special = bitmap.special();
 288 
 289   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 290     align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 291   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 292   if (!_bitmap_region_special) {
 293     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 294                               "Cannot commit bitmap memory");
 295   }
 296 
 297   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
 298 
 299   if (ShenandoahVerify) {
 300     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
 301     os::trace_page_sizes_for_requested_size("Verify Bitmap",
 302                                             bitmap_size_orig, verify_bitmap.page_size(), bitmap_page_size,
 303                                             verify_bitmap.base(),
 304                                             verify_bitmap.size());
 305     if (!verify_bitmap.special()) {
 306       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 307                                 "Cannot commit verification bitmap memory");
 308     }
 309     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 310     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 311     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 312     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 313   }
 314 
 315   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 316   size_t aux_bitmap_page_size = bitmap_page_size;
 317 
 318   ReservedSpace aux_bitmap(_bitmap_size, aux_bitmap_page_size);
 319   os::trace_page_sizes_for_requested_size("Aux Bitmap",
 320                                           bitmap_size_orig, aux_bitmap.page_size(), aux_bitmap_page_size,
 321                                           aux_bitmap.base(), aux_bitmap.size());
 322   MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
 323   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 324   _aux_bitmap_region_special = aux_bitmap.special();
 325   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 326 
 327   //
 328   // Create regions and region sets
 329   //
 330   size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
 331   size_t region_storage_size_orig = region_align * _num_regions;
 332   size_t region_storage_size = align_up(region_storage_size_orig,
 333                                         MAX2(region_page_size, os::vm_allocation_granularity()));
 334 
 335   ReservedSpace region_storage(region_storage_size, region_page_size);
 336   os::trace_page_sizes_for_requested_size("Region Storage",
 337                                           region_storage_size_orig, region_storage.page_size(), region_page_size,
 338                                           region_storage.base(), region_storage.size());
 339   MemTracker::record_virtual_memory_type(region_storage.base(), mtGC);
 340   if (!region_storage.special()) {
 341     os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
 342                               "Cannot commit region memory");
 343   }
 344 
 345   // Try to fit the collection set bitmap at lower addresses. This optimizes code generation for cset checks.
 346   // Go up until a sensible limit (subject to encoding constraints) and try to reserve the space there.
 347   // If not successful, bite a bullet and allocate at whatever address.
 348   {
 349     const size_t cset_align = MAX2<size_t>(os::vm_page_size(), os::vm_allocation_granularity());
 350     const size_t cset_size = align_up(((size_t) sh_rs.base() + sh_rs.size()) >> ShenandoahHeapRegion::region_size_bytes_shift(), cset_align);
 351     const size_t cset_page_size = os::vm_page_size();
 352 
 353     uintptr_t min = round_up_power_of_2(cset_align);
 354     uintptr_t max = (1u << 30u);
 355     ReservedSpace cset_rs;
 356 
 357     for (uintptr_t addr = min; addr <= max; addr <<= 1u) {
 358       char* req_addr = (char*)addr;
 359       assert(is_aligned(req_addr, cset_align), "Should be aligned");
 360       cset_rs = ReservedSpace(cset_size, cset_align, cset_page_size, req_addr);
 361       if (cset_rs.is_reserved()) {
 362         assert(cset_rs.base() == req_addr, "Allocated where requested: " PTR_FORMAT ", " PTR_FORMAT, p2i(cset_rs.base()), addr);
 363         _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 364         break;
 365       }
 366     }
 367 
 368     if (_collection_set == nullptr) {
 369       cset_rs = ReservedSpace(cset_size, cset_align, os::vm_page_size());
 370       _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 371     }
 372     os::trace_page_sizes_for_requested_size("Collection Set",
 373                                             cset_size, cset_rs.page_size(), cset_page_size,
 374                                             cset_rs.base(),
 375                                             cset_rs.size());
 376   }
 377 
 378   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
 379   _affiliations = NEW_C_HEAP_ARRAY(uint8_t, _num_regions, mtGC);
 380   _free_set = new ShenandoahFreeSet(this, _num_regions);
 381 
 382   {
 383     ShenandoahHeapLocker locker(lock());
 384 
 385     for (size_t i = 0; i < _num_regions; i++) {
 386       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
 387       bool is_committed = i < num_committed_regions;
 388       void* loc = region_storage.base() + i * region_align;
 389 
 390       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
 391       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
 392 
 393       _marking_context->initialize_top_at_mark_start(r);
 394       _regions[i] = r;
 395       assert(!collection_set()->is_in(i), "New region should not be in collection set");
 396 
 397       _affiliations[i] = ShenandoahAffiliation::FREE;
 398     }
 399 
 400     // Initialize to complete
 401     _marking_context->mark_complete();
 402     size_t young_cset_regions, old_cset_regions;
 403 
 404     // We are initializing free set.  We ignore cset region tallies.
 405     size_t first_old, last_old, num_old;
 406     _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
 407     _free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old);
 408   }
 409 
 410   if (AlwaysPreTouch) {
 411     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 412     // before initialize() below zeroes it with initializing thread. For any given region,
 413     // we touch the region and the corresponding bitmaps from the same thread.
 414     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 415 
 416     _pretouch_heap_page_size = heap_page_size;
 417     _pretouch_bitmap_page_size = bitmap_page_size;
 418 










 419     // OS memory managers may want to coalesce back-to-back pages. Make their jobs
 420     // simpler by pre-touching continuous spaces (heap and bitmap) separately.
 421 
 422     ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, _pretouch_bitmap_page_size);
 423     _workers->run_task(&bcl);
 424 
 425     ShenandoahPretouchHeapTask hcl(_pretouch_heap_page_size);
 426     _workers->run_task(&hcl);
 427   }
 428 
 429   //
 430   // Initialize the rest of GC subsystems
 431   //
 432 
 433   _liveness_cache = NEW_C_HEAP_ARRAY(ShenandoahLiveData*, _max_workers, mtGC);
 434   for (uint worker = 0; worker < _max_workers; worker++) {
 435     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(ShenandoahLiveData, _num_regions, mtGC);
 436     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
 437   }
 438 
 439   // There should probably be Shenandoah-specific options for these,
 440   // just as there are G1-specific options.
 441   {
 442     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 443     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 444     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 445   }
 446 
 447   _monitoring_support = new ShenandoahMonitoringSupport(this);
 448   _phase_timings = new ShenandoahPhaseTimings(max_workers());
 449   ShenandoahCodeRoots::initialize();
 450 
 451   if (ShenandoahPacing) {
 452     _pacer = new ShenandoahPacer(this);
 453     _pacer->setup_for_idle();


 454   }
 455 
 456   initialize_controller();
 457 
 458   print_init_logger();
 459 
 460   return JNI_OK;
 461 }
 462 
 463 void ShenandoahHeap::initialize_controller() {
 464   _control_thread = new ShenandoahControlThread();
 465 }
 466 
 467 void ShenandoahHeap::print_init_logger() const {
 468   ShenandoahInitLogger::print();
 469 }
 470 
 471 void ShenandoahHeap::initialize_mode() {
 472   if (ShenandoahGCMode != nullptr) {
 473     if (strcmp(ShenandoahGCMode, "satb") == 0) {
 474       _gc_mode = new ShenandoahSATBMode();
 475     } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
 476       _gc_mode = new ShenandoahIUMode();
 477     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 478       _gc_mode = new ShenandoahPassiveMode();
 479     } else if (strcmp(ShenandoahGCMode, "generational") == 0) {
 480       _gc_mode = new ShenandoahGenerationalMode();
 481     } else {
 482       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 483     }
 484   } else {
 485     vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option (null)");
 486   }
 487   _gc_mode->initialize_flags();
 488   if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 489     vm_exit_during_initialization(
 490             err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 491                     _gc_mode->name()));
 492   }
 493   if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
 494     vm_exit_during_initialization(
 495             err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 496                     _gc_mode->name()));
 497   }
 498 }
 499 
 500 void ShenandoahHeap::initialize_heuristics() {
 501   _global_generation = new ShenandoahGlobalGeneration(mode()->is_generational(), max_workers(), max_capacity(), max_capacity());
 502   _global_generation->initialize_heuristics(mode());











 503 }
 504 
 505 #ifdef _MSC_VER
 506 #pragma warning( push )
 507 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 508 #endif
 509 
 510 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 511   CollectedHeap(),
 512   _gc_generation(nullptr),
 513   _active_generation(nullptr),
 514   _initial_size(0),

 515   _committed(0),
 516   _max_workers(MAX3(ConcGCThreads, ParallelGCThreads, 1U)),

 517   _workers(nullptr),
 518   _safepoint_workers(nullptr),
 519   _heap_region_special(false),
 520   _num_regions(0),
 521   _regions(nullptr),
 522   _affiliations(nullptr),
 523   _gc_state_changed(false),
 524   _gc_no_progress_count(0),
 525   _cancel_requested_time(0),
 526   _update_refs_iterator(this),
 527   _global_generation(nullptr),
 528   _control_thread(nullptr),
 529   _young_generation(nullptr),
 530   _old_generation(nullptr),
 531   _shenandoah_policy(policy),
 532   _gc_mode(nullptr),

 533   _free_set(nullptr),
 534   _pacer(nullptr),
 535   _verifier(nullptr),
 536   _phase_timings(nullptr),
 537   _monitoring_support(nullptr),
 538   _memory_pool(nullptr),
 539   _stw_memory_manager("Shenandoah Pauses"),
 540   _cycle_memory_manager("Shenandoah Cycles"),
 541   _gc_timer(new ConcurrentGCTimer()),
 542   _soft_ref_policy(),
 543   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),

 544   _marking_context(nullptr),
 545   _bitmap_size(0),
 546   _bitmap_regions_per_slice(0),
 547   _bitmap_bytes_per_slice(0),
 548   _bitmap_region_special(false),
 549   _aux_bitmap_region_special(false),
 550   _liveness_cache(nullptr),
 551   _collection_set(nullptr)
 552 {
 553   // Initialize GC mode early, many subsequent initialization procedures depend on it
 554   initialize_mode();















 555 }
 556 
 557 #ifdef _MSC_VER
 558 #pragma warning( pop )
 559 #endif
 560 





























 561 void ShenandoahHeap::print_on(outputStream* st) const {
 562   st->print_cr("Shenandoah Heap");
 563   st->print_cr(" " SIZE_FORMAT "%s max, " SIZE_FORMAT "%s soft max, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
 564                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 565                byte_size_in_proper_unit(soft_max_capacity()), proper_unit_for_byte_size(soft_max_capacity()),
 566                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 567                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 568   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
 569                num_regions(),
 570                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 571                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 572 
 573   st->print("Status: ");
 574   if (has_forwarded_objects())                 st->print("has forwarded objects, ");
 575   if (!mode()->is_generational()) {
 576     if (is_concurrent_mark_in_progress())      st->print("marking,");
 577   } else {
 578     if (is_concurrent_old_mark_in_progress())    st->print("old marking, ");
 579     if (is_concurrent_young_mark_in_progress())  st->print("young marking, ");
 580   }
 581   if (is_evacuation_in_progress())             st->print("evacuating, ");
 582   if (is_update_refs_in_progress())            st->print("updating refs, ");
 583   if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
 584   if (is_full_gc_in_progress())                st->print("full gc, ");
 585   if (is_full_gc_move_in_progress())           st->print("full gc move, ");
 586   if (is_concurrent_weak_root_in_progress())   st->print("concurrent weak roots, ");
 587   if (is_concurrent_strong_root_in_progress() &&
 588       !is_concurrent_weak_root_in_progress())  st->print("concurrent strong roots, ");
 589 
 590   if (cancelled_gc()) {
 591     st->print("cancelled");
 592   } else {
 593     st->print("not cancelled");
 594   }
 595   st->cr();
 596 
 597   st->print_cr("Reserved region:");
 598   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 599                p2i(reserved_region().start()),
 600                p2i(reserved_region().end()));

 611   st->cr();
 612   MetaspaceUtils::print_on(st);
 613 
 614   if (Verbose) {
 615     st->cr();
 616     print_heap_regions_on(st);
 617   }
 618 }
 619 
 620 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 621 public:
 622   void do_thread(Thread* thread) {
 623     assert(thread != nullptr, "Sanity");
 624     assert(thread->is_Worker_thread(), "Only worker thread expected");
 625     ShenandoahThreadLocalData::initialize_gclab(thread);
 626   }
 627 };
 628 
 629 void ShenandoahHeap::post_initialize() {
 630   CollectedHeap::post_initialize();
 631 
 632   // Schedule periodic task to report on gc thread CPU utilization
 633   _mmu_tracker.initialize();
 634 
 635   MutexLocker ml(Threads_lock);
 636 
 637   ShenandoahInitWorkerGCLABClosure init_gclabs;
 638   _workers->threads_do(&init_gclabs);
 639 
 640   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 641   // Now, we will let WorkerThreads to initialize gclab when new worker is created.
 642   _workers->set_initialize_gclab();
 643   if (_safepoint_workers != nullptr) {
 644     _safepoint_workers->threads_do(&init_gclabs);
 645     _safepoint_workers->set_initialize_gclab();
 646   }
 647 


 648   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
 649 }
 650 
 651 ShenandoahHeuristics* ShenandoahHeap::heuristics() {
 652   return _global_generation->heuristics();
 653 }
 654 
 655 size_t ShenandoahHeap::used() const {
 656   return global_generation()->used();
 657 }
 658 
 659 size_t ShenandoahHeap::committed() const {
 660   return Atomic::load(&_committed);
 661 }
 662 
 663 void ShenandoahHeap::increase_committed(size_t bytes) {
 664   shenandoah_assert_heaplocked_or_safepoint();
 665   _committed += bytes;
 666 }
 667 
 668 void ShenandoahHeap::decrease_committed(size_t bytes) {
 669   shenandoah_assert_heaplocked_or_safepoint();
 670   _committed -= bytes;
 671 }
 672 
 673 // For tracking usage based on allocations, it should be the case that:
 674 // * The sum of regions::used == heap::used
 675 // * The sum of a generation's regions::used == generation::used
 676 // * The sum of a generation's humongous regions::free == generation::humongous_waste
 677 // These invariants are checked by the verifier on GC safepoints.
 678 //
 679 // Additional notes:
 680 // * When a mutator's allocation request causes a region to be retired, the
 681 //   free memory left in that region is considered waste. It does not contribute
 682 //   to the usage, but it _does_ contribute to allocation rate.
 683 // * The bottom of a PLAB must be aligned on card size. In some cases this will
 684 //   require padding in front of the PLAB (a filler object). Because this padding
 685 //   is included in the region's used memory we include the padding in the usage
 686 //   accounting as waste.
 687 // * Mutator allocations are used to compute an allocation rate. They are also
 688 //   sent to the Pacer for those purposes.
 689 // * There are three sources of waste:
 690 //  1. The padding used to align a PLAB on card size
 691 //  2. Region's free is less than minimum TLAB size and is retired
 692 //  3. The unused portion of memory in the last region of a humongous object
 693 void ShenandoahHeap::increase_used(const ShenandoahAllocRequest& req) {
 694   size_t actual_bytes = req.actual_size() * HeapWordSize;
 695   size_t wasted_bytes = req.waste() * HeapWordSize;
 696   ShenandoahGeneration* generation = generation_for(req.affiliation());
 697 
 698   if (req.is_gc_alloc()) {
 699     assert(wasted_bytes == 0 || req.type() == ShenandoahAllocRequest::_alloc_plab, "Only PLABs have waste");
 700     increase_used(generation, actual_bytes + wasted_bytes);
 701   } else {
 702     assert(req.is_mutator_alloc(), "Expected mutator alloc here");
 703     // padding and actual size both count towards allocation counter
 704     generation->increase_allocated(actual_bytes + wasted_bytes);
 705 
 706     // only actual size counts toward usage for mutator allocations
 707     increase_used(generation, actual_bytes);
 708 
 709     // notify pacer of both actual size and waste
 710     notify_mutator_alloc_words(req.actual_size(), req.waste());
 711 
 712     if (wasted_bytes > 0 && ShenandoahHeapRegion::requires_humongous(req.actual_size())) {
 713       increase_humongous_waste(generation,wasted_bytes);
 714     }
 715   }
 716 }
 717 
 718 void ShenandoahHeap::increase_humongous_waste(ShenandoahGeneration* generation, size_t bytes) {
 719   generation->increase_humongous_waste(bytes);
 720   if (!generation->is_global()) {
 721     global_generation()->increase_humongous_waste(bytes);
 722   }
 723 }
 724 
 725 void ShenandoahHeap::decrease_humongous_waste(ShenandoahGeneration* generation, size_t bytes) {
 726   generation->decrease_humongous_waste(bytes);
 727   if (!generation->is_global()) {
 728     global_generation()->decrease_humongous_waste(bytes);
 729   }
 730 }
 731 
 732 void ShenandoahHeap::increase_used(ShenandoahGeneration* generation, size_t bytes) {
 733   generation->increase_used(bytes);
 734   if (!generation->is_global()) {
 735     global_generation()->increase_used(bytes);
 736   }
 737 }
 738 
 739 void ShenandoahHeap::decrease_used(ShenandoahGeneration* generation, size_t bytes) {
 740   generation->decrease_used(bytes);
 741   if (!generation->is_global()) {
 742     global_generation()->decrease_used(bytes);
 743   }
 744 }
 745 
 746 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, size_t waste) {
 747   if (ShenandoahPacing) {
 748     control_thread()->pacing_notify_alloc(words);
 749     if (waste > 0) {
 750       pacer()->claim_for_alloc<true>(waste);
 751     }
 752   }
 753 }
 754 
 755 size_t ShenandoahHeap::capacity() const {
 756   return committed();
 757 }
 758 
 759 size_t ShenandoahHeap::max_capacity() const {
 760   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 761 }
 762 
 763 size_t ShenandoahHeap::soft_max_capacity() const {
 764   size_t v = Atomic::load(&_soft_max_size);
 765   assert(min_capacity() <= v && v <= max_capacity(),
 766          "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
 767          min_capacity(), v, max_capacity());
 768   return v;
 769 }
 770 
 771 void ShenandoahHeap::set_soft_max_capacity(size_t v) {
 772   assert(min_capacity() <= v && v <= max_capacity(),
 773          "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT,
 774          min_capacity(), v, max_capacity());
 775   Atomic::store(&_soft_max_size, v);
 776 }
 777 
 778 size_t ShenandoahHeap::min_capacity() const {
 779   return _minimum_size;
 780 }
 781 
 782 size_t ShenandoahHeap::initial_capacity() const {
 783   return _initial_size;
 784 }
 785 
 786 void ShenandoahHeap::maybe_uncommit(double shrink_before, size_t shrink_until) {
 787   assert (ShenandoahUncommit, "should be enabled");
 788 
 789   // Determine if there is work to do. This avoids taking heap lock if there is
 790   // no work available, avoids spamming logs with superfluous logging messages,
 791   // and minimises the amount of work while locks are taken.
 792 
 793   if (committed() <= shrink_until) return;
 794 
 795   bool has_work = false;
 796   for (size_t i = 0; i < num_regions(); i++) {
 797     ShenandoahHeapRegion* r = get_region(i);
 798     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 799       has_work = true;
 800       break;
 801     }
 802   }
 803 
 804   if (has_work) {
 805     static const char* msg = "Concurrent uncommit";
 806     ShenandoahConcurrentPhase gcPhase(msg, ShenandoahPhaseTimings::conc_uncommit, true /* log_heap_usage */);
 807     EventMark em("%s", msg);
 808 
 809     op_uncommit(shrink_before, shrink_until);
 810   }
 811 }
 812 
 813 void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) {
 814   assert (ShenandoahUncommit, "should be enabled");
 815 
 816   // Application allocates from the beginning of the heap, and GC allocates at
 817   // the end of it. It is more efficient to uncommit from the end, so that applications
 818   // could enjoy the near committed regions. GC allocations are much less frequent,
 819   // and therefore can accept the committing costs.
 820 
 821   size_t count = 0;
 822   for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
 823     ShenandoahHeapRegion* r = get_region(i - 1);
 824     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 825       ShenandoahHeapLocker locker(lock());
 826       if (r->is_empty_committed()) {
 827         if (committed() < shrink_until + ShenandoahHeapRegion::region_size_bytes()) {
 828           break;
 829         }
 830 
 831         r->make_uncommitted();
 832         count++;
 833       }
 834     }
 835     SpinPause(); // allow allocators to take the lock
 836   }
 837 
 838   if (count > 0) {
 839     notify_heap_changed();
 840   }
 841 }
 842 
 843 bool ShenandoahHeap::check_soft_max_changed() {
 844   size_t new_soft_max = Atomic::load(&SoftMaxHeapSize);
 845   size_t old_soft_max = soft_max_capacity();
 846   if (new_soft_max != old_soft_max) {
 847     new_soft_max = MAX2(min_capacity(), new_soft_max);
 848     new_soft_max = MIN2(max_capacity(), new_soft_max);
 849     if (new_soft_max != old_soft_max) {
 850       log_info(gc)("Soft Max Heap Size: " SIZE_FORMAT "%s -> " SIZE_FORMAT "%s",
 851                    byte_size_in_proper_unit(old_soft_max), proper_unit_for_byte_size(old_soft_max),
 852                    byte_size_in_proper_unit(new_soft_max), proper_unit_for_byte_size(new_soft_max)
 853       );
 854       set_soft_max_capacity(new_soft_max);
 855       return true;
 856     }
 857   }
 858   return false;
 859 }
 860 
 861 void ShenandoahHeap::notify_heap_changed() {
 862   // Update monitoring counters when we took a new region. This amortizes the
 863   // update costs on slow path.
 864   monitoring_support()->notify_heap_changed();
 865   _heap_changed.try_set();
 866 }
 867 
 868 void ShenandoahHeap::set_forced_counters_update(bool value) {
 869   monitoring_support()->set_forced_counters_update(value);
 870 }
 871 
 872 void ShenandoahHeap::handle_force_counters_update() {
 873   monitoring_support()->handle_force_counters_update();
 874 }
 875 
 876 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 877   // New object should fit the GCLAB size
 878   size_t min_size = MAX2(size, PLAB::min_size());
 879 
 880   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 881   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
 882 
 883   new_size = MIN2(new_size, PLAB::max_size());
 884   new_size = MAX2(new_size, PLAB::min_size());
 885 
 886   // Record new heuristic value even if we take any shortcut. This captures
 887   // the case when moderately-sized objects always take a shortcut. At some point,
 888   // heuristics should catch up with them.
 889   log_debug(gc, free)("Set new GCLAB size: " SIZE_FORMAT, new_size);
 890   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 891 
 892   if (new_size < size) {
 893     // New size still does not fit the object. Fall back to shared allocation.
 894     // This avoids retiring perfectly good GCLABs, when we encounter a large object.
 895     log_debug(gc, free)("New gclab size (" SIZE_FORMAT ") is too small for " SIZE_FORMAT, new_size, size);
 896     return nullptr;
 897   }
 898 
 899   // Retire current GCLAB, and allocate a new one.
 900   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 901   gclab->retire();
 902 
 903   size_t actual_size = 0;
 904   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 905   if (gclab_buf == nullptr) {
 906     return nullptr;
 907   }
 908 
 909   assert (size <= actual_size, "allocation should fit");
 910 
 911   // ...and clear or zap just allocated TLAB, if needed.
 912   if (ZeroTLAB) {
 913     Copy::zero_to_words(gclab_buf, actual_size);
 914   } else if (ZapTLAB) {
 915     // Skip mangling the space corresponding to the object header to
 916     // ensure that the returned space is not considered parsable by
 917     // any concurrent GC thread.
 918     size_t hdr_size = oopDesc::header_size();
 919     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 920   }
 921   gclab->set_buf(gclab_buf, actual_size);
 922   return gclab->allocate(size);
 923 }
 924 
 925 // Called from stubs in JIT code or interpreter
 926 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 927                                             size_t requested_size,
 928                                             size_t* actual_size) {
 929   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 930   HeapWord* res = allocate_memory(req);
 931   if (res != nullptr) {
 932     *actual_size = req.actual_size();
 933   } else {
 934     *actual_size = 0;
 935   }
 936   return res;
 937 }
 938 
 939 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 940                                              size_t word_size,
 941                                              size_t* actual_size) {
 942   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 943   HeapWord* res = allocate_memory(req);
 944   if (res != nullptr) {
 945     *actual_size = req.actual_size();

 947     *actual_size = 0;
 948   }
 949   return res;
 950 }
 951 
 952 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 953   intptr_t pacer_epoch = 0;
 954   bool in_new_region = false;
 955   HeapWord* result = nullptr;
 956 
 957   if (req.is_mutator_alloc()) {
 958     if (ShenandoahPacing) {
 959       pacer()->pace_for_alloc(req.size());
 960       pacer_epoch = pacer()->epoch();
 961     }
 962 
 963     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
 964       result = allocate_memory_under_lock(req, in_new_region);
 965     }
 966 
 967     // Check that gc overhead is not exceeded.
 968     //
 969     // Shenandoah will grind along for quite a while allocating one
 970     // object at a time using shared (non-tlab) allocations. This check
 971     // is testing that the GC overhead limit has not been exceeded.
 972     // This will notify the collector to start a cycle, but will raise
 973     // an OOME to the mutator if the last Full GCs have not made progress.
 974     // gc_no_progress_count is incremented following each degen or full GC that fails to achieve is_good_progress().
 975     if ((result == nullptr) && !req.is_lab_alloc() && (get_gc_no_progress_count() > ShenandoahNoProgressThreshold)) {
 976       control_thread()->handle_alloc_failure(req, false);
 977       req.set_actual_size(0);
 978       return nullptr;
 979     }
 980 
 981     if (result == nullptr) {
 982       // Block until control thread reacted, then retry allocation.
 983       //
 984       // It might happen that one of the threads requesting allocation would unblock
 985       // way later after GC happened, only to fail the second allocation, because
 986       // other threads have already depleted the free storage. In this case, a better
 987       // strategy is to try again, until at least one full GC has completed.
 988       //
 989       // Stop retrying and return nullptr to cause OOMError exception if our allocation failed even after:
 990       //   a) We experienced a GC that had good progress, or
 991       //   b) We experienced at least one Full GC (whether or not it had good progress)
 992 
 993       size_t original_count = shenandoah_policy()->full_gc_count();
 994       while ((result == nullptr) && (original_count == shenandoah_policy()->full_gc_count())) {
 995         control_thread()->handle_alloc_failure(req, true);
 996         result = allocate_memory_under_lock(req, in_new_region);
 997       }
 998       if (result != nullptr) {
 999         // If our allocation request has been satisifed after it initially failed, we count this as good gc progress
1000         notify_gc_progress();
1001       }
1002       if (log_develop_is_enabled(Debug, gc, alloc)) {
1003         ResourceMark rm;
1004         log_debug(gc, alloc)("Thread: %s, Result: " PTR_FORMAT ", Request: %s, Size: " SIZE_FORMAT
1005                              ", Original: " SIZE_FORMAT ", Latest: " SIZE_FORMAT,
1006                              Thread::current()->name(), p2i(result), req.type_string(), req.size(),
1007                              original_count, get_gc_no_progress_count());
1008       }
1009     }
1010   } else {
1011     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
1012     result = allocate_memory_under_lock(req, in_new_region);
1013     // Do not call handle_alloc_failure() here, because we cannot block.
1014     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
1015   }
1016 
1017   if (in_new_region) {
1018     notify_heap_changed();
1019   }
1020 
1021   if (result == nullptr) {
1022     req.set_actual_size(0);
1023   }
1024 
1025   // This is called regardless of the outcome of the allocation to account
1026   // for any waste created by retiring regions with this request.
1027   increase_used(req);
1028 
1029   if (result != nullptr) {
1030     size_t requested = req.size();
1031     size_t actual = req.actual_size();
1032 
1033     assert (req.is_lab_alloc() || (requested == actual),
1034             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
1035             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
1036 
1037     if (req.is_mutator_alloc()) {


1038       // If we requested more than we were granted, give the rest back to pacer.
1039       // This only matters if we are in the same pacing epoch: do not try to unpace
1040       // over the budget for the other phase.
1041       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
1042         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
1043       }


1044     }
1045   }
1046 
1047   return result;
1048 }
1049 
1050 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
1051   // If we are dealing with mutator allocation, then we may need to block for safepoint.
1052   // We cannot block for safepoint for GC allocations, because there is a high chance
1053   // we are already running at safepoint or from stack watermark machinery, and we cannot
1054   // block again.
1055   ShenandoahHeapLocker locker(lock(), req.is_mutator_alloc());
1056 
1057   // Make sure the old generation has room for either evacuations or promotions before trying to allocate.
1058   if (req.is_old() && !old_generation()->can_allocate(req)) {
1059     return nullptr;
1060   }
1061 
1062   // If TLAB request size is greater than available, allocate() will attempt to downsize request to fit within available
1063   // memory.
1064   HeapWord* result = _free_set->allocate(req, in_new_region);
1065 
1066   // Record the plab configuration for this result and register the object.
1067   if (result != nullptr && req.is_old()) {
1068     old_generation()->configure_plab_for_current_thread(req);
1069     if (req.type() == ShenandoahAllocRequest::_alloc_shared_gc) {
1070       // Register the newly allocated object while we're holding the global lock since there's no synchronization
1071       // built in to the implementation of register_object().  There are potential races when multiple independent
1072       // threads are allocating objects, some of which might span the same card region.  For example, consider
1073       // a card table's memory region within which three objects are being allocated by three different threads:
1074       //
1075       // objects being "concurrently" allocated:
1076       //    [-----a------][-----b-----][--------------c------------------]
1077       //            [---- card table memory range --------------]
1078       //
1079       // Before any objects are allocated, this card's memory range holds no objects.  Note that allocation of object a
1080       // wants to set the starts-object, first-start, and last-start attributes of the preceding card region.
1081       // Allocation of object b wants to set the starts-object, first-start, and last-start attributes of this card region.
1082       // Allocation of object c also wants to set the starts-object, first-start, and last-start attributes of this
1083       // card region.
1084       //
1085       // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as
1086       // last-start representing object b while first-start represents object c.  This is why we need to require all
1087       // register_object() invocations to be "mutually exclusive" with respect to each card's memory range.
1088       old_generation()->card_scan()->register_object(result);
1089     }
1090   }
1091 
1092   return result;
1093 }
1094 
1095 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
1096                                         bool*  gc_overhead_limit_was_exceeded) {
1097   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
1098   return allocate_memory(req);
1099 }
1100 
1101 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
1102                                                              size_t size,
1103                                                              Metaspace::MetadataType mdtype) {
1104   MetaWord* result;
1105 
1106   // Inform metaspace OOM to GC heuristics if class unloading is possible.
1107   ShenandoahHeuristics* h = global_generation()->heuristics();
1108   if (h->can_unload_classes()) {
1109     h->record_metaspace_oom();
1110   }
1111 
1112   // Expand and retry allocation
1113   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
1114   if (result != nullptr) {
1115     return result;
1116   }
1117 
1118   // Start full GC
1119   collect(GCCause::_metadata_GC_clear_soft_refs);
1120 
1121   // Retry allocation
1122   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
1123   if (result != nullptr) {
1124     return result;
1125   }
1126 
1127   // Expand and retry allocation
1128   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);

1185     while ((r =_cs->claim_next()) != nullptr) {
1186       assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
1187       _sh->marked_object_iterate(r, &cl);
1188 
1189       if (ShenandoahPacing) {
1190         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
1191       }
1192 
1193       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
1194         break;
1195       }
1196     }
1197   }
1198 };
1199 
1200 void ShenandoahHeap::evacuate_collection_set(bool concurrent) {
1201   ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1202   workers()->run_task(&task);
1203 }
1204 
1205 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1206   assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1207   if (ShenandoahThreadLocalData::is_oom_during_evac(thread)) {
1208     // This thread went through the OOM during evac protocol. It is safe to return
1209     // the forward pointer. It must not attempt to evacuate any other objects.
1210     return ShenandoahBarrierSet::resolve_forwarded(p);
1211   }
1212 
1213   assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
1214 
1215   ShenandoahHeapRegion* r = heap_region_containing(p);
1216   assert(!r->is_humongous(), "never evacuate humongous objects");
1217 
1218   ShenandoahAffiliation target_gen = r->affiliation();
1219   return try_evacuate_object(p, thread, r, target_gen);
1220 }
1221 
1222 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1223                                                ShenandoahAffiliation target_gen) {
1224   assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1225   assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1226   bool alloc_from_lab = true;
1227   HeapWord* copy = nullptr;
1228   size_t size = p->size();
1229 
1230 #ifdef ASSERT
1231   if (ShenandoahOOMDuringEvacALot &&
1232       (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
1233     copy = nullptr;
1234   } else {
1235 #endif
1236     if (UseTLAB) {
1237       copy = allocate_from_gclab(thread, size);
1238     }
1239     if (copy == nullptr) {
1240       // If we failed to allocate in LAB, we'll try a shared allocation.
1241       ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen);
1242       copy = allocate_memory(req);
1243       alloc_from_lab = false;
1244     }
1245 #ifdef ASSERT
1246   }
1247 #endif
1248 
1249   if (copy == nullptr) {
1250     control_thread()->handle_alloc_failure_evac(size);
1251 
1252     _oom_evac_handler.handle_out_of_memory_during_evacuation();
1253 
1254     return ShenandoahBarrierSet::resolve_forwarded(p);
1255   }
1256 
1257   // Copy the object:
1258   Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
1259 
1260   // Try to install the new forwarding pointer.
1261   oop copy_val = cast_to_oop(copy);
1262   oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
1263   if (result == copy_val) {
1264     // Successfully evacuated. Our copy is now the public one!
1265     ContinuationGCSupport::relativize_stack_chunk(copy_val);
1266     shenandoah_assert_correct(nullptr, copy_val);
1267     return copy_val;
1268   }  else {
1269     // Failed to evacuate. We need to deal with the object that is left behind. Since this
1270     // new allocation is certainly after TAMS, it will be considered live in the next cycle.
1271     // But if it happens to contain references to evacuated regions, those references would
1272     // not get updated for this stale copy during this cycle, and we will crash while scanning
1273     // it the next cycle.
1274     if (alloc_from_lab) {
1275       // For LAB allocations, it is enough to rollback the allocation ptr. Either the next
1276       // object will overwrite this stale copy, or the filler object on LAB retirement will
1277       // do this.
1278       ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
1279     } else {
1280       // For non-LAB allocations, we have no way to retract the allocation, and
1281       // have to explicitly overwrite the copy with the filler object. With that overwrite,
1282       // we have to keep the fwdptr initialized and pointing to our (stale) copy.
1283       assert(size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size");
1284       fill_with_object(copy, size);
1285       shenandoah_assert_correct(nullptr, copy_val);
1286       // For non-LAB allocations, the object has already been registered
1287     }
1288     shenandoah_assert_correct(nullptr, result);
1289     return result;
1290   }
1291 }
1292 
1293 void ShenandoahHeap::trash_cset_regions() {
1294   ShenandoahHeapLocker locker(lock());
1295 
1296   ShenandoahCollectionSet* set = collection_set();
1297   ShenandoahHeapRegion* r;
1298   set->clear_current_index();
1299   while ((r = set->next()) != nullptr) {
1300     r->make_trash();
1301   }
1302   collection_set()->clear();
1303 }
1304 
1305 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1306   st->print_cr("Heap Regions:");
1307   st->print_cr("Region state: EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HP=pinned humongous start");
1308   st->print_cr("              HC=humongous continuation, CS=collection set, TR=trash, P=pinned, CSP=pinned collection set");
1309   st->print_cr("BTE=bottom/top/end, TAMS=top-at-mark-start");
1310   st->print_cr("UWM=update watermark, U=used");
1311   st->print_cr("T=TLAB allocs, G=GCLAB allocs");
1312   st->print_cr("S=shared allocs, L=live data");
1313   st->print_cr("CP=critical pins");
1314 
1315   for (size_t i = 0; i < num_regions(); i++) {
1316     get_region(i)->print_on(st);
1317   }
1318 }
1319 
1320 size_t ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1321   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1322 
1323   oop humongous_obj = cast_to_oop(start->bottom());
1324   size_t size = humongous_obj->size();
1325   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1326   size_t index = start->index() + required_regions - 1;
1327 
1328   assert(!start->has_live(), "liveness must be zero");
1329 
1330   for(size_t i = 0; i < required_regions; i++) {
1331     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1332     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1333     ShenandoahHeapRegion* region = get_region(index --);
1334 
1335     assert(region->is_humongous(), "expect correct humongous start or continuation");
1336     assert(!region->is_cset(), "Humongous region should not be in collection set");
1337 
1338     region->make_trash_immediate();
1339   }
1340   return required_regions;
1341 }
1342 
1343 class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
1344 public:
1345   ShenandoahCheckCleanGCLABClosure() {}
1346   void do_thread(Thread* thread) {
1347     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1348     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1349     assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");
1350 
1351     if (ShenandoahHeap::heap()->mode()->is_generational()) {
1352       PLAB* plab = ShenandoahThreadLocalData::plab(thread);
1353       assert(plab != nullptr, "PLAB should be initialized for %s", thread->name());
1354       assert(plab->words_remaining() == 0, "PLAB should not need retirement");
1355     }
1356   }
1357 };
1358 
1359 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1360 private:
1361   bool const _resize;
1362 public:
1363   ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
1364   void do_thread(Thread* thread) {
1365     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1366     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1367     gclab->retire();
1368     if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1369       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1370     }
1371 
1372     if (ShenandoahHeap::heap()->mode()->is_generational()) {
1373       PLAB* plab = ShenandoahThreadLocalData::plab(thread);
1374       assert(plab != nullptr, "PLAB should be initialized for %s", thread->name());
1375 
1376       // There are two reasons to retire all plabs between old-gen evacuation passes.
1377       //  1. We need to make the plab memory parsable by remembered-set scanning.
1378       //  2. We need to establish a trustworthy UpdateWaterMark value within each old-gen heap region
1379       ShenandoahGenerationalHeap::heap()->retire_plab(plab, thread);
1380       if (_resize && ShenandoahThreadLocalData::plab_size(thread) > 0) {
1381         ShenandoahThreadLocalData::set_plab_size(thread, 0);
1382       }
1383     }
1384   }
1385 };
1386 
1387 void ShenandoahHeap::labs_make_parsable() {
1388   assert(UseTLAB, "Only call with UseTLAB");
1389 
1390   ShenandoahRetireGCLABClosure cl(false);
1391 
1392   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1393     ThreadLocalAllocBuffer& tlab = t->tlab();
1394     tlab.make_parsable();
1395     cl.do_thread(t);
1396   }
1397 
1398   workers()->threads_do(&cl);
1399 }
1400 
1401 void ShenandoahHeap::tlabs_retire(bool resize) {
1402   assert(UseTLAB, "Only call with UseTLAB");
1403   assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");

1465   }
1466   return nullptr;
1467 }
1468 
1469 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1470   ShenandoahHeapRegion* r = heap_region_containing(addr);
1471   return r->block_is_obj(addr);
1472 }
1473 
1474 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1475   return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1476 }
1477 
1478 void ShenandoahHeap::prepare_for_verify() {
1479   if (SafepointSynchronize::is_at_safepoint() && UseTLAB) {
1480     labs_make_parsable();
1481   }
1482 }
1483 
1484 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1485   if (_shenandoah_policy->is_at_shutdown()) {
1486     return;
1487   }
1488 
1489   if (_control_thread != nullptr) {
1490     tcl->do_thread(_control_thread);
1491   }
1492 
1493   workers()->threads_do(tcl);
1494   if (_safepoint_workers != nullptr) {
1495     _safepoint_workers->threads_do(tcl);
1496   }
1497 }
1498 
1499 void ShenandoahHeap::print_tracing_info() const {
1500   LogTarget(Info, gc, stats) lt;
1501   if (lt.is_enabled()) {
1502     ResourceMark rm;
1503     LogStream ls(lt);
1504 
1505     phase_timings()->print_global_on(&ls);
1506 
1507     ls.cr();
1508     ls.cr();
1509 
1510     shenandoah_policy()->print_gc_stats(&ls);
1511 
1512     ls.cr();
1513     ls.cr();
1514   }
1515 }
1516 
1517 void ShenandoahHeap::set_gc_generation(ShenandoahGeneration* generation) {
1518   shenandoah_assert_control_or_vm_thread_at_safepoint();
1519   _gc_generation = generation;
1520 }
1521 
1522 // Active generation may only be set by the VM thread at a safepoint.
1523 void ShenandoahHeap::set_active_generation() {
1524   assert(Thread::current()->is_VM_thread(), "Only the VM Thread");
1525   assert(SafepointSynchronize::is_at_safepoint(), "Only at a safepoint!");
1526   assert(_gc_generation != nullptr, "Will set _active_generation to nullptr");
1527   _active_generation = _gc_generation;
1528 }
1529 
1530 void ShenandoahHeap::on_cycle_start(GCCause::Cause cause, ShenandoahGeneration* generation) {
1531   shenandoah_policy()->record_collection_cause(cause);
1532 
1533   assert(gc_cause()  == GCCause::_no_gc, "Over-writing cause");
1534   assert(_gc_generation == nullptr, "Over-writing _gc_generation");
1535 
1536   set_gc_cause(cause);
1537   set_gc_generation(generation);
1538 
1539   generation->heuristics()->record_cycle_start();
1540 }
1541 
1542 void ShenandoahHeap::on_cycle_end(ShenandoahGeneration* generation) {
1543   assert(gc_cause() != GCCause::_no_gc, "cause wasn't set");
1544   assert(_gc_generation != nullptr, "_gc_generation wasn't set");
1545 
1546   generation->heuristics()->record_cycle_end();
1547   if (mode()->is_generational() && generation->is_global()) {
1548     // If we just completed a GLOBAL GC, claim credit for completion of young-gen and old-gen GC as well
1549     young_generation()->heuristics()->record_cycle_end();
1550     old_generation()->heuristics()->record_cycle_end();
1551   }
1552 
1553   set_gc_generation(nullptr);
1554   set_gc_cause(GCCause::_no_gc);
1555 }
1556 
1557 void ShenandoahHeap::verify(VerifyOption vo) {
1558   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1559     if (ShenandoahVerify) {
1560       verifier()->verify_generic(vo);
1561     } else {
1562       // TODO: Consider allocating verification bitmaps on demand,
1563       // and turn this on unconditionally.
1564     }
1565   }
1566 }
1567 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1568   return _free_set->capacity();
1569 }
1570 
1571 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1572 private:
1573   MarkBitMap* _bitmap;
1574   ShenandoahScanObjectStack* _oop_stack;
1575   ShenandoahHeap* const _heap;
1576   ShenandoahMarkingContext* const _marking_context;

1871   const uint active_workers = workers()->active_workers();
1872   const size_t n_regions = num_regions();
1873   size_t stride = ShenandoahParallelRegionStride;
1874   if (stride == 0 && active_workers > 1) {
1875     // Automatically derive the stride to balance the work between threads
1876     // evenly. Do not try to split work if below the reasonable threshold.
1877     constexpr size_t threshold = 4096;
1878     stride = n_regions <= threshold ?
1879             threshold :
1880             (n_regions + active_workers - 1) / active_workers;
1881   }
1882 
1883   if (n_regions > stride && active_workers > 1) {
1884     ShenandoahParallelHeapRegionTask task(blk, stride);
1885     workers()->run_task(&task);
1886   } else {
1887     heap_region_iterate(blk);
1888   }
1889 }
1890 























1891 class ShenandoahRendezvousClosure : public HandshakeClosure {
1892 public:
1893   inline ShenandoahRendezvousClosure() : HandshakeClosure("ShenandoahRendezvous") {}
1894   inline void do_thread(Thread* thread) {}
1895 };
1896 
1897 void ShenandoahHeap::rendezvous_threads() {
1898   ShenandoahRendezvousClosure cl;
1899   Handshake::execute(&cl);
1900 }
1901 
1902 void ShenandoahHeap::recycle_trash() {
1903   free_set()->recycle_trash();
1904 }
1905 



































































































1906 void ShenandoahHeap::do_class_unloading() {
1907   _unloader.unload();
1908   if (mode()->is_generational()) {
1909     old_generation()->set_parsable(false);
1910   }
1911 }
1912 
1913 void ShenandoahHeap::stw_weak_refs(bool full_gc) {
1914   // Weak refs processing
1915   ShenandoahPhaseTimings::Phase phase = full_gc ? ShenandoahPhaseTimings::full_gc_weakrefs
1916                                                 : ShenandoahPhaseTimings::degen_gc_weakrefs;
1917   ShenandoahTimingsTracker t(phase);
1918   ShenandoahGCWorkerPhase worker_phase(phase);
1919   shenandoah_assert_generations_reconciled();
1920   gc_generation()->ref_processor()->process_references(phase, workers(), false /* concurrent */);
1921 }
1922 
1923 void ShenandoahHeap::prepare_update_heap_references(bool concurrent) {
1924   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1925 
1926   // Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to
1927   // make them parsable for update code to work correctly. Plus, we can compute new sizes
1928   // for future GCLABs here.
1929   if (UseTLAB) {
1930     ShenandoahGCPhase phase(concurrent ?
1931                             ShenandoahPhaseTimings::init_update_refs_manage_gclabs :
1932                             ShenandoahPhaseTimings::degen_gc_init_update_refs_manage_gclabs);
1933     gclabs_retire(ResizeTLAB);
1934   }
1935 
1936   _update_refs_iterator.reset();
1937 }
1938 
1939 void ShenandoahHeap::propagate_gc_state_to_java_threads() {
1940   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1941   if (_gc_state_changed) {
1942     _gc_state_changed = false;
1943     char state = gc_state();
1944     for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1945       ShenandoahThreadLocalData::set_gc_state(t, state);
1946     }
1947   }
1948 }
1949 
1950 void ShenandoahHeap::set_gc_state(uint mask, bool value) {
1951   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
1952   _gc_state.set_cond(mask, value);
1953   _gc_state_changed = true;
1954 }
1955 
1956 void ShenandoahHeap::set_concurrent_young_mark_in_progress(bool in_progress) {
1957   uint mask;
1958   assert(!has_forwarded_objects(), "Young marking is not concurrent with evacuation");
1959   if (!in_progress && is_concurrent_old_mark_in_progress()) {
1960     assert(mode()->is_generational(), "Only generational GC has old marking");
1961     assert(_gc_state.is_set(MARKING), "concurrent_old_marking_in_progress implies MARKING");
1962     // If old-marking is in progress when we turn off YOUNG_MARKING, leave MARKING (and OLD_MARKING) on
1963     mask = YOUNG_MARKING;
1964   } else {
1965     mask = MARKING | YOUNG_MARKING;
1966   }
1967   set_gc_state(mask, in_progress);
1968   manage_satb_barrier(in_progress);
1969 }
1970 
1971 void ShenandoahHeap::set_concurrent_old_mark_in_progress(bool in_progress) {
1972 #ifdef ASSERT
1973   // has_forwarded_objects() iff UPDATEREFS or EVACUATION
1974   bool has_forwarded = has_forwarded_objects();
1975   bool updating_or_evacuating = _gc_state.is_set(UPDATEREFS | EVACUATION);
1976   bool evacuating = _gc_state.is_set(EVACUATION);
1977   assert ((has_forwarded == updating_or_evacuating) || (evacuating && !has_forwarded && collection_set()->is_empty()),
1978           "Updating or evacuating iff has forwarded objects, or if evacuation phase is promoting in place without forwarding");
1979 #endif
1980   if (!in_progress && is_concurrent_young_mark_in_progress()) {
1981     // If young-marking is in progress when we turn off OLD_MARKING, leave MARKING (and YOUNG_MARKING) on
1982     assert(_gc_state.is_set(MARKING), "concurrent_young_marking_in_progress implies MARKING");
1983     set_gc_state(OLD_MARKING, in_progress);
1984   } else {
1985     set_gc_state(MARKING | OLD_MARKING, in_progress);
1986   }
1987   manage_satb_barrier(in_progress);
1988 }
1989 
1990 bool ShenandoahHeap::is_prepare_for_old_mark_in_progress() const {
1991   return old_generation()->is_preparing_for_mark();
1992 }
1993 
1994 void ShenandoahHeap::manage_satb_barrier(bool active) {
1995   if (is_concurrent_mark_in_progress()) {
1996     // Ignore request to deactivate barrier while concurrent mark is in progress.
1997     // Do not attempt to re-activate the barrier if it is already active.
1998     if (active && !ShenandoahBarrierSet::satb_mark_queue_set().is_active()) {
1999       ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active);
2000     }
2001   } else {
2002     // No concurrent marking is in progress so honor request to deactivate,
2003     // but only if the barrier is already active.
2004     if (!active && ShenandoahBarrierSet::satb_mark_queue_set().is_active()) {
2005       ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active);
2006     }
2007   }
2008 }
2009 
2010 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
2011   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
2012   set_gc_state(EVACUATION, in_progress);
2013 }
2014 
2015 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
2016   if (in_progress) {
2017     _concurrent_strong_root_in_progress.set();
2018   } else {
2019     _concurrent_strong_root_in_progress.unset();
2020   }
2021 }
2022 
2023 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool cond) {
2024   set_gc_state(WEAK_ROOTS, cond);
2025 }
2026 
2027 GCTracer* ShenandoahHeap::tracer() {
2028   return shenandoah_policy()->tracer();
2029 }
2030 
2031 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2032   return _free_set->used();
2033 }
2034 
2035 bool ShenandoahHeap::try_cancel_gc() {
2036   jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
2037   return prev == CANCELLABLE;
2038 }
2039 
2040 void ShenandoahHeap::cancel_concurrent_mark() {
2041   if (mode()->is_generational()) {
2042     young_generation()->cancel_marking();
2043     old_generation()->cancel_marking();
2044   }
2045 
2046   global_generation()->cancel_marking();
2047 
2048   ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
2049 }
2050 
2051 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
2052   if (try_cancel_gc()) {
2053     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
2054     log_info(gc)("%s", msg.buffer());
2055     Events::log(Thread::current(), "%s", msg.buffer());
2056     _cancel_requested_time = os::elapsedTime();
2057   }
2058 }
2059 
2060 uint ShenandoahHeap::max_workers() {
2061   return _max_workers;
2062 }
2063 
2064 void ShenandoahHeap::stop() {
2065   // The shutdown sequence should be able to terminate when GC is running.
2066 
2067   // Step 0. Notify policy to disable event recording.
2068   _shenandoah_policy->record_shutdown();
2069 
2070   // Step 0a. Stop reporting on gc thread cpu utilization
2071   mmu_tracker()->stop();
2072 
2073   // Step 1. Notify control thread that we are in shutdown.
2074   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2075   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2076   control_thread()->prepare_for_graceful_shutdown();
2077 
2078   // Step 2. Notify GC workers that we are cancelling GC.
2079   cancel_gc(GCCause::_shenandoah_stop_vm);
2080 
2081   // Step 3. Wait until GC worker exits normally.
2082   control_thread()->stop();
2083 }
2084 
2085 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
2086   if (!unload_classes()) return;
2087   ClassUnloadingContext ctx(_workers->active_workers(),
2088                             true /* unregister_nmethods_during_purge */,
2089                             false /* lock_codeblob_free_separately */);
2090 
2091   // Unload classes and purge SystemDictionary.
2092   {

2163 }
2164 
2165 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2166   set_gc_state(HAS_FORWARDED, cond);
2167 }
2168 
2169 void ShenandoahHeap::set_unload_classes(bool uc) {
2170   _unload_classes.set_cond(uc);
2171 }
2172 
2173 bool ShenandoahHeap::unload_classes() const {
2174   return _unload_classes.is_set();
2175 }
2176 
2177 address ShenandoahHeap::in_cset_fast_test_addr() {
2178   ShenandoahHeap* heap = ShenandoahHeap::heap();
2179   assert(heap->collection_set() != nullptr, "Sanity");
2180   return (address) heap->collection_set()->biased_map_address();
2181 }
2182 




2183 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2184   if (mode()->is_generational()) {
2185     young_generation()->reset_bytes_allocated_since_gc_start();
2186     old_generation()->reset_bytes_allocated_since_gc_start();
2187   }
2188 
2189   global_generation()->reset_bytes_allocated_since_gc_start();
2190 }
2191 
2192 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2193   _degenerated_gc_in_progress.set_cond(in_progress);
2194 }
2195 
2196 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2197   _full_gc_in_progress.set_cond(in_progress);
2198 }
2199 
2200 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2201   assert (is_full_gc_in_progress(), "should be");
2202   _full_gc_move_in_progress.set_cond(in_progress);
2203 }
2204 
2205 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2206   set_gc_state(UPDATEREFS, in_progress);
2207 }
2208 
2209 void ShenandoahHeap::register_nmethod(nmethod* nm) {

2233     if (r->is_active()) {
2234       if (r->is_pinned()) {
2235         if (r->pin_count() == 0) {
2236           r->make_unpinned();
2237         }
2238       } else {
2239         if (r->pin_count() > 0) {
2240           r->make_pinned();
2241         }
2242       }
2243     }
2244   }
2245 
2246   assert_pinned_region_status();
2247 }
2248 
2249 #ifdef ASSERT
2250 void ShenandoahHeap::assert_pinned_region_status() {
2251   for (size_t i = 0; i < num_regions(); i++) {
2252     ShenandoahHeapRegion* r = get_region(i);
2253     shenandoah_assert_generations_reconciled();
2254     if (gc_generation()->contains(r)) {
2255       assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2256              "Region " SIZE_FORMAT " pinning status is inconsistent", i);
2257     }
2258   }
2259 }
2260 #endif
2261 
2262 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
2263   return _gc_timer;
2264 }
2265 
2266 void ShenandoahHeap::prepare_concurrent_roots() {
2267   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2268   assert(!is_stw_gc_in_progress(), "Only concurrent GC");
2269   set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
2270   set_concurrent_weak_root_in_progress(true);
2271   if (unload_classes()) {
2272     _unloader.prepare();
2273   }
2274 }
2275 
2276 void ShenandoahHeap::finish_concurrent_roots() {
2277   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2278   assert(!is_stw_gc_in_progress(), "Only concurrent GC");
2279   if (unload_classes()) {
2280     _unloader.finish();
2281   }
2282 }
2283 
2284 #ifdef ASSERT
2285 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2286   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2287 
2288   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2289     // Use ParallelGCThreads inside safepoints
2290     assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads (%u) within safepoint, not %u",
2291            ParallelGCThreads, nworkers);



2292   } else {
2293     // Use ConcGCThreads outside safepoints
2294     assert(nworkers == ConcGCThreads, "Use ConcGCThreads (%u) outside safepoints, %u",
2295            ConcGCThreads, nworkers);



2296   }
2297 }
2298 #endif
2299 
2300 ShenandoahVerifier* ShenandoahHeap::verifier() {
2301   guarantee(ShenandoahVerify, "Should be enabled");
2302   assert (_verifier != nullptr, "sanity");
2303   return _verifier;
2304 }
2305 
2306 template<bool CONCURRENT>
2307 class ShenandoahUpdateHeapRefsTask : public WorkerTask {
2308 private:
2309   ShenandoahHeap* _heap;
2310   ShenandoahRegionIterator* _regions;
2311 public:
2312   explicit ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) :
2313     WorkerTask("Shenandoah Update References"),
2314     _heap(ShenandoahHeap::heap()),
2315     _regions(regions) {
2316   }
2317 
2318   void work(uint worker_id) {
2319     if (CONCURRENT) {
2320       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2321       ShenandoahSuspendibleThreadSetJoiner stsj;
2322       do_work<ShenandoahConcUpdateRefsClosure>(worker_id);
2323     } else {
2324       ShenandoahParallelWorkerSession worker_session(worker_id);
2325       do_work<ShenandoahSTWUpdateRefsClosure>(worker_id);
2326     }
2327   }
2328 
2329 private:
2330   template<class T>
2331   void do_work(uint worker_id) {
2332     if (CONCURRENT && (worker_id == 0)) {
2333       // We ask the first worker to replenish the Mutator free set by moving regions previously reserved to hold the
2334       // results of evacuation.  These reserves are no longer necessary because evacuation has completed.
2335       size_t cset_regions = _heap->collection_set()->count();
2336 
2337       // Now that evacuation is done, we can reassign any regions that had been reserved to hold the results of evacuation
2338       // to the mutator free set.  At the end of GC, we will have cset_regions newly evacuated fully empty regions from
2339       // which we will be able to replenish the Collector free set and the OldCollector free set in preparation for the
2340       // next GC cycle.
2341       _heap->free_set()->move_regions_from_collector_to_mutator(cset_regions);
2342     }
2343     // If !CONCURRENT, there's no value in expanding Mutator free set
2344     T cl;
2345     ShenandoahHeapRegion* r = _regions->next();

2346     while (r != nullptr) {
2347       HeapWord* update_watermark = r->get_update_watermark();
2348       assert (update_watermark >= r->bottom(), "sanity");
2349       if (r->is_active() && !r->is_cset()) {
2350         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2351         if (ShenandoahPacing) {
2352           _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2353         }
2354       }
2355       if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
2356         return;
2357       }
2358       r = _regions->next();
2359     }
2360   }
2361 };
2362 
2363 void ShenandoahHeap::update_heap_references(bool concurrent) {
2364   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2365 
2366   if (concurrent) {
2367     ShenandoahUpdateHeapRefsTask<true> task(&_update_refs_iterator);
2368     workers()->run_task(&task);
2369   } else {
2370     ShenandoahUpdateHeapRefsTask<false> task(&_update_refs_iterator);
2371     workers()->run_task(&task);
2372   }
2373 }
2374 






























2375 void ShenandoahHeap::update_heap_region_states(bool concurrent) {
2376   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2377   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2378 
2379   {
2380     ShenandoahGCPhase phase(concurrent ?
2381                             ShenandoahPhaseTimings::final_update_refs_update_region_states :
2382                             ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states);
2383 
2384     final_update_refs_update_region_states();
2385 
2386     assert_pinned_region_status();
2387   }
2388 
2389   {
2390     ShenandoahGCPhase phase(concurrent ?
2391                             ShenandoahPhaseTimings::final_update_refs_trash_cset :
2392                             ShenandoahPhaseTimings::degen_gc_final_update_refs_trash_cset);
2393     trash_cset_regions();
2394   }
2395 }
2396 
2397 void ShenandoahHeap::final_update_refs_update_region_states() {
2398   ShenandoahSynchronizePinnedRegionStates cl;
2399   parallel_heap_region_iterate(&cl);
2400 }
2401 
2402 void ShenandoahHeap::rebuild_free_set(bool concurrent) {
2403   ShenandoahGCPhase phase(concurrent ?
2404                           ShenandoahPhaseTimings::final_update_refs_rebuild_freeset :
2405                           ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset);
2406   ShenandoahHeapLocker locker(lock());
2407   size_t young_cset_regions, old_cset_regions;
2408   size_t first_old_region, last_old_region, old_region_count;
2409   _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count);
2410   // If there are no old regions, first_old_region will be greater than last_old_region
2411   assert((first_old_region > last_old_region) ||
2412          ((last_old_region + 1 - first_old_region >= old_region_count) &&
2413           get_region(first_old_region)->is_old() && get_region(last_old_region)->is_old()),
2414          "sanity: old_region_count: " SIZE_FORMAT ", first_old_region: " SIZE_FORMAT ", last_old_region: " SIZE_FORMAT,
2415          old_region_count, first_old_region, last_old_region);
2416 
2417   if (mode()->is_generational()) {
2418 #ifdef ASSERT
2419     if (ShenandoahVerify) {
2420       verifier()->verify_before_rebuilding_free_set();
2421     }
2422 #endif
2423 
2424     // The computation of bytes_of_allocation_runway_before_gc_trigger is quite conservative so consider all of this
2425     // available for transfer to old. Note that transfer of humongous regions does not impact available.
2426     ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap();
2427     size_t allocation_runway = gen_heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(young_cset_regions);
2428     gen_heap->compute_old_generation_balance(allocation_runway, old_cset_regions);
2429 
2430     // Total old_available may have been expanded to hold anticipated promotions.  We trigger if the fragmented available
2431     // memory represents more than 16 regions worth of data.  Note that fragmentation may increase when we promote regular
2432     // regions in place when many of these regular regions have an abundant amount of available memory within them.  Fragmentation
2433     // will decrease as promote-by-copy consumes the available memory within these partially consumed regions.
2434     //
2435     // We consider old-gen to have excessive fragmentation if more than 12.5% of old-gen is free memory that resides
2436     // within partially consumed regions of memory.
2437   }
2438   // Rebuild free set based on adjusted generation sizes.
2439   _free_set->finish_rebuild(young_cset_regions, old_cset_regions, old_region_count);
2440 
2441   if (mode()->is_generational()) {
2442     ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap();
2443     ShenandoahOldGeneration* old_gen = gen_heap->old_generation();
2444     old_gen->heuristics()->evaluate_triggers(first_old_region, last_old_region, old_region_count, num_regions());
2445   }
2446 }
2447 
2448 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2449   print_on(st);
2450   st->cr();
2451   print_heap_regions_on(st);
2452 }
2453 
2454 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2455   size_t slice = r->index() / _bitmap_regions_per_slice;
2456 
2457   size_t regions_from = _bitmap_regions_per_slice * slice;
2458   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2459   for (size_t g = regions_from; g < regions_to; g++) {
2460     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2461     if (skip_self && g == r->index()) continue;
2462     if (get_region(g)->is_committed()) {
2463       return true;
2464     }

2512   }
2513 
2514   // Uncommit the bitmap slice:
2515   size_t slice = r->index() / _bitmap_regions_per_slice;
2516   size_t off = _bitmap_bytes_per_slice * slice;
2517   size_t len = _bitmap_bytes_per_slice;
2518   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2519     return false;
2520   }
2521   return true;
2522 }
2523 
2524 void ShenandoahHeap::safepoint_synchronize_begin() {
2525   SuspendibleThreadSet::synchronize();
2526 }
2527 
2528 void ShenandoahHeap::safepoint_synchronize_end() {
2529   SuspendibleThreadSet::desynchronize();
2530 }
2531 








2532 void ShenandoahHeap::try_inject_alloc_failure() {
2533   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2534     _inject_alloc_failure.set();
2535     os::naked_short_sleep(1);
2536     if (cancelled_gc()) {
2537       log_info(gc)("Allocation failure was successfully injected");
2538     }
2539   }
2540 }
2541 
2542 bool ShenandoahHeap::should_inject_alloc_failure() {
2543   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2544 }
2545 
2546 void ShenandoahHeap::initialize_serviceability() {
2547   _memory_pool = new ShenandoahMemoryPool(this);
2548   _cycle_memory_manager.add_pool(_memory_pool);
2549   _stw_memory_manager.add_pool(_memory_pool);
2550 }
2551 
2552 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2553   GrowableArray<GCMemoryManager*> memory_managers(2);
2554   memory_managers.append(&_cycle_memory_manager);
2555   memory_managers.append(&_stw_memory_manager);
2556   return memory_managers;
2557 }
2558 
2559 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2560   GrowableArray<MemoryPool*> memory_pools(1);
2561   memory_pools.append(_memory_pool);
2562   return memory_pools;
2563 }
2564 
2565 MemoryUsage ShenandoahHeap::memory_usage() {
2566   return MemoryUsage(_initial_size, used(), committed(), max_capacity());
2567 }
2568 
2569 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2570   _heap(ShenandoahHeap::heap()),
2571   _index(0) {}
2572 
2573 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2574   _heap(heap),
2575   _index(0) {}
2576 
2577 void ShenandoahRegionIterator::reset() {
2578   _index = 0;
2579 }
2580 
2581 bool ShenandoahRegionIterator::has_next() const {
2582   return _index < _heap->num_regions();
2583 }
2584 
2585 char ShenandoahHeap::gc_state() const {
2586   return _gc_state.raw_value();

2611   }
2612 }
2613 
2614 bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
2615   if (is_idle()) return false;
2616 
2617   // Objects allocated after marking start are implicitly alive, don't need any barriers during
2618   // marking phase.
2619   if (is_concurrent_mark_in_progress() &&
2620      !marking_context()->allocated_after_mark_start(obj)) {
2621     return true;
2622   }
2623 
2624   // Can not guarantee obj is deeply good.
2625   if (has_forwarded_objects()) {
2626     return true;
2627   }
2628 
2629   return false;
2630 }
2631 
2632 ShenandoahGeneration* ShenandoahHeap::generation_for(ShenandoahAffiliation affiliation) const {
2633   if (!mode()->is_generational()) {
2634     return global_generation();
2635   } else if (affiliation == YOUNG_GENERATION) {
2636     return young_generation();
2637   } else if (affiliation == OLD_GENERATION) {
2638     return old_generation();
2639   }
2640 
2641   ShouldNotReachHere();
2642   return nullptr;
2643 }
2644 
2645 void ShenandoahHeap::log_heap_status(const char* msg) const {
2646   if (mode()->is_generational()) {
2647     young_generation()->log_status(msg);
2648     old_generation()->log_status(msg);
2649   } else {
2650     global_generation()->log_status(msg);
2651   }
2652 }
< prev index next >