Skip to content

Commit

Permalink
Equalize promoted (#64206)
Browse files Browse the repository at this point in the history 
I have observed that in server GC scenarios, the amount of promoted memory is often very uneven between heaps, which leads to suboptimal work distribution between server GC threads.

The PR introduces a new method equalize_promoted_bytes that attemps to even out the promoted memory between heaps by moving regions from heaps with lots of promotion of heaps with less promotion.

The algorithm used removes regions from heaps that have more than average promotion. These surplus regions are arranged into size classes by the amount of promoted memory in them. The heaps are also arranged into size classes by how much their promoted memory is short of the average promoted memory per heap.

Then we repeatedly move the surplus region with the most promoted memory to the heap with the biggest deficit. If the heap still has a deficit, it is reconsidered under its new deficit size class. Otherwise, it now has average or more promoted memory and is removed from further consideration.

Because regions may now move between heaps, it is no longer true that the finalization queue entry for an object is always on the same heap as the object itself. This necessitated moving the call to finalize_queue->UpdatePromotedGenerations to a later point in time when all threads have finished updating the final generation for a region.
  • Loading branch information
@PeterSolMS
PeterSolMS authored Feb 24, 2022
1 parent 1b75f42 commit 85b4f0e
Show file tree
Hide file tree
Showing 2 changed files with 333 additions and 12 deletions.
333 changes: 321 additions & 12 deletions src/coreclr/gc/gc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22468,6 +22468,314 @@ void gc_heap::sync_promoted_bytes()
}
}
}

#ifdef MULTIPLE_HEAPS
void gc_heap::set_heap_for_contained_basic_regions (heap_segment* region, gc_heap* hp)
{
uint8_t* region_start = get_region_start (region);
uint8_t* region_end = heap_segment_reserved (region);

int num_basic_regions = (int)((region_end - region_start) >> min_segment_size_shr);
for (int i = 0; i < num_basic_regions; i++)
{
uint8_t* basic_region_start = region_start + ((size_t)i << min_segment_size_shr);
heap_segment* basic_region = get_region_info (basic_region_start);
heap_segment_heap (basic_region) = hp;
}
}

heap_segment* gc_heap::unlink_first_rw_region (int gen_idx)
{
generation* gen = generation_of (gen_idx);
heap_segment* prev_region = generation_tail_ro_region (gen);
heap_segment* region = nullptr;
if (prev_region)
{
assert (heap_segment_read_only_p (prev_region));
region = heap_segment_next (prev_region);
assert (region != nullptr);
// don't remove the last region in the generation
if (heap_segment_next (region) == nullptr)
{
assert (region == generation_tail_region (gen));
return nullptr;
}
heap_segment_next (prev_region) = heap_segment_next (region);
}
else
{
region = generation_start_segment (gen);
assert (region != nullptr);
// don't remove the last region in the generation
if (heap_segment_next (region) == nullptr)
{
assert (region == generation_tail_region (gen));
return nullptr;
}
generation_start_segment (gen) = heap_segment_next (region);
}
assert (region != generation_tail_region (gen));
assert (!heap_segment_read_only_p (region));
dprintf (REGIONS_LOG, ("unlink_first_rw_region on heap: %d gen: %d region: %Ix", heap_number, gen_idx, heap_segment_mem (region)));

set_heap_for_contained_basic_regions (region, nullptr);

return region;
}

void gc_heap::thread_rw_region_front (int gen_idx, heap_segment* region)
{
generation* gen = generation_of (gen_idx);
assert (!heap_segment_read_only_p (region));
heap_segment* prev_region = generation_tail_ro_region (gen);
if (prev_region)
{
heap_segment_next (region) = heap_segment_next (prev_region);
heap_segment_next (prev_region) = region;
}
else
{
heap_segment_next (region) = generation_start_segment (gen);
generation_start_segment (gen) = region;
}
dprintf (REGIONS_LOG, ("thread_rw_region_front on heap: %d gen: %d region: %Ix", heap_number, gen_idx, heap_segment_mem (region)));

set_heap_for_contained_basic_regions (region, this);
}
#endif // MULTIPLE_HEAPS

void gc_heap::equalize_promoted_bytes()
{
#ifdef MULTIPLE_HEAPS
// algorithm to roughly balance promoted bytes across heaps by moving regions between heaps
// goal is just to balance roughly, while keeping computational complexity low
// hope is to achieve better work balancing in relocate and compact phases
//
int condemned_gen_number = settings.condemned_generation;
int highest_gen_number = ((condemned_gen_number == max_generation) ?
(total_generation_count - 1) : condemned_gen_number);
int stop_gen_idx = get_stop_generation_index (condemned_gen_number);

for (int gen_idx = highest_gen_number; gen_idx >= stop_gen_idx; gen_idx--)
{
// step 1:
// compute total promoted bytes per gen
size_t total_surv = 0;
size_t max_surv_per_heap = 0;
size_t surv_per_heap[MAX_SUPPORTED_CPUS];
for (int i = 0; i < n_heaps; i++)
{
surv_per_heap[i] = 0;

gc_heap* hp = g_heaps[i];

generation* condemned_gen = hp->generation_of (gen_idx);
heap_segment* current_region = heap_segment_rw (generation_start_segment (condemned_gen));

while (current_region)
{
total_surv += heap_segment_survived (current_region);
surv_per_heap[i] += heap_segment_survived (current_region);
current_region = heap_segment_next (current_region);
}

max_surv_per_heap = max (max_surv_per_heap, surv_per_heap[i]);

dprintf (REGIONS_LOG, ("gen: %d heap %d surv: %Id", gen_idx, i, surv_per_heap[i]));
}
// compute average promoted bytes per heap and per gen
// be careful to round up
size_t avg_surv_per_heap = (total_surv + n_heaps - 1) / n_heaps;

if (avg_surv_per_heap != 0)
{
dprintf (REGIONS_LOG, ("before equalize: gen: %d avg surv: %Id max_surv: %Id imbalance: %d", gen_idx, avg_surv_per_heap, max_surv_per_heap, max_surv_per_heap*100/avg_surv_per_heap));
}
//
// step 2:
// remove regions from surplus heaps until all heaps are <= average
// put removed regions into surplus regions
//
// step 3:
// put regions into size classes by survivorship
// put deficit heaps into size classes by deficit
//
// step 4:
// while (surplus regions is non-empty)
// get surplus region from biggest size class
// put it into heap from biggest deficit size class
// re-insert heap by resulting deficit size class

heap_segment* surplus_regions = nullptr;
size_t max_deficit = 0;
size_t max_survived = 0;

// go through all the heaps
for (int i = 0; i < n_heaps; i++)
{
// remove regions from this heap until it has average or less survivorship
while (surv_per_heap[i] > avg_surv_per_heap)
{
heap_segment* region = g_heaps[i]->unlink_first_rw_region (gen_idx);
if (region == nullptr)
{
break;
}
assert (surv_per_heap[i] >= (size_t)heap_segment_survived (region));
dprintf (REGIONS_LOG, ("heap: %d surv: %Id - %Id = %Id",
i,
surv_per_heap[i],
heap_segment_survived (region),
surv_per_heap[i] - heap_segment_survived (region)));

surv_per_heap[i] -= heap_segment_survived (region);

heap_segment_next (region) = surplus_regions;
surplus_regions = region;

max_survived = max (max_survived, (size_t)heap_segment_survived (region));
}
if (surv_per_heap[i] < avg_surv_per_heap)
{
size_t deficit = avg_surv_per_heap - surv_per_heap[i];
max_deficit = max (max_deficit, deficit);
}
}

// we arrange both surplus regions and deficit heaps by size classes
const int NUM_SIZE_CLASSES = 16;
heap_segment* surplus_regions_by_size_class[NUM_SIZE_CLASSES];
memset (surplus_regions_by_size_class, 0, sizeof(surplus_regions_by_size_class));
double survived_scale_factor = ((double)NUM_SIZE_CLASSES) / (max_survived + 1);

heap_segment* next_region;
for (heap_segment* region = surplus_regions; region != nullptr; region = next_region)
{
int size_class = (int)(heap_segment_survived (region)*survived_scale_factor);
assert ((0 <= size_class) && (size_class < NUM_SIZE_CLASSES));
next_region = heap_segment_next (region);
heap_segment_next (region) = surplus_regions_by_size_class[size_class];
surplus_regions_by_size_class[size_class] = region;
}

int next_heap_in_size_class[MAX_SUPPORTED_CPUS];
int heaps_by_deficit_size_class[NUM_SIZE_CLASSES];
for (int i = 0; i < NUM_SIZE_CLASSES; i++)
{
heaps_by_deficit_size_class[i] = -1;
}
double deficit_scale_factor = ((double)NUM_SIZE_CLASSES) / (max_deficit + 1);

for (int i = 0; i < n_heaps; i++)
{
if (avg_surv_per_heap > surv_per_heap[i])
{
size_t deficit = avg_surv_per_heap - surv_per_heap[i];
int size_class = (int)(deficit*deficit_scale_factor);
assert ((0 <= size_class) && (size_class < NUM_SIZE_CLASSES));
next_heap_in_size_class[i] = heaps_by_deficit_size_class[size_class];
heaps_by_deficit_size_class[size_class] = i;
}
}

int region_size_class = NUM_SIZE_CLASSES - 1;
int heap_size_class = NUM_SIZE_CLASSES - 1;
while (region_size_class >= 0)
{
// obtain a region from the biggest size class
heap_segment* region = surplus_regions_by_size_class[region_size_class];
if (region == nullptr)
{
region_size_class--;
continue;
}
// and a heap from the biggest deficit size class
int heap_num;
while (true)
{
if (heap_size_class < 0)
{
// put any remaining regions on heap 0
// rare case, but there may be some 0 surv size regions
heap_num = 0;
break;
}
heap_num = heaps_by_deficit_size_class[heap_size_class];
if (heap_num >= 0)
{
break;
}
heap_size_class--;
}

// now move the region to the heap
surplus_regions_by_size_class[region_size_class] = heap_segment_next (region);
g_heaps[heap_num]->thread_rw_region_front (gen_idx, region);

// adjust survival for this heap
dprintf (REGIONS_LOG, ("heap: %d surv: %Id + %Id = %Id",
heap_num,
surv_per_heap[heap_num],
heap_segment_survived (region),
surv_per_heap[heap_num] + heap_segment_survived (region)));

surv_per_heap[heap_num] += heap_segment_survived (region);

if (heap_size_class < 0)
{
// no need to update size classes for heaps -
// just work down the remaining regions, if any
continue;
}

// is this heap now average or above?
if (surv_per_heap[heap_num] >= avg_surv_per_heap)
{
// if so, unlink from the current size class
heaps_by_deficit_size_class[heap_size_class] = next_heap_in_size_class[heap_num];
continue;
}

// otherwise compute the updated deficit
size_t new_deficit = avg_surv_per_heap - surv_per_heap[heap_num];

// check if this heap moves to a differenct deficit size class
int new_heap_size_class = (int)(new_deficit*deficit_scale_factor);
if (new_heap_size_class != heap_size_class)
{
// the new deficit size class should be smaller and in range
assert (new_heap_size_class < heap_size_class);
assert ((0 <= new_heap_size_class) && (new_heap_size_class < NUM_SIZE_CLASSES));

// if so, unlink from the current size class
heaps_by_deficit_size_class[heap_size_class] = next_heap_in_size_class[heap_num];

// and link to the new size class
next_heap_in_size_class[heap_num] = heaps_by_deficit_size_class[new_heap_size_class];
heaps_by_deficit_size_class[new_heap_size_class] = heap_num;
}
}
// we will generally be left with some heaps with deficits here, but that's ok

// check we didn't screw up the data structures
for (int i = 0; i < n_heaps; i++)
{
g_heaps[i]->verify_regions (gen_idx, false);
}
#ifdef TRACE_GC
max_surv_per_heap = 0;
for (int i = 0; i < n_heaps; i++)
{
max_surv_per_heap = max (max_surv_per_heap, surv_per_heap[i]);
}
if (avg_surv_per_heap != 0)
{
dprintf (REGIONS_LOG, ("after equalize: gen: %d avg surv: %Id max_surv: %Id imbalance: %d", gen_idx, avg_surv_per_heap, max_surv_per_heap, max_surv_per_heap*100/avg_surv_per_heap));
}
#endif // TRACE_GC
}
#endif //MULTIPLE_HEAPS
}
#endif //USE_REGIONS

#if !defined(USE_REGIONS) || defined(_DEBUG)
Expand Down Expand Up @@ -25629,6 +25937,7 @@ void gc_heap::mark_phase (int condemned_gen_number, BOOL mark_only_p)

#ifdef USE_REGIONS
sync_promoted_bytes();
equalize_promoted_bytes();
#endif //USE_REGIONS

#ifdef MULTIPLE_HEAPS
Expand Down Expand Up @@ -29397,11 +29706,6 @@ void gc_heap::plan_phase (int condemned_gen_number)
#endif //!USE_REGIONS

{
#ifdef FEATURE_PREMORTEM_FINALIZATION
finalize_queue->UpdatePromotedGenerations (condemned_gen_number,
(!settings.demotion && settings.promotion));
#endif // FEATURE_PREMORTEM_FINALIZATION

#ifdef MULTIPLE_HEAPS
dprintf(3, ("Joining after end of compaction"));
gc_t_join.join(this, gc_join_adjust_handle_age_compact);
Expand All @@ -29424,6 +29728,11 @@ void gc_heap::plan_phase (int condemned_gen_number)
}
#endif //MULTIPLE_HEAPS

#ifdef FEATURE_PREMORTEM_FINALIZATION
finalize_queue->UpdatePromotedGenerations (condemned_gen_number,
(!settings.demotion && settings.promotion));
#endif // FEATURE_PREMORTEM_FINALIZATION

ScanContext sc;
sc.thread_number = heap_number;
sc.promotion = FALSE;
Expand Down Expand Up @@ -29603,13 +29912,6 @@ void gc_heap::plan_phase (int condemned_gen_number)
generation_free_obj_space (generation_of (max_generation))));
}

#ifdef FEATURE_PREMORTEM_FINALIZATION
if (!special_sweep_p)
{
finalize_queue->UpdatePromotedGenerations (condemned_gen_number, TRUE);
}
#endif // FEATURE_PREMORTEM_FINALIZATION

#ifdef MULTIPLE_HEAPS
dprintf(3, ("Joining after end of sweep"));
gc_t_join.join(this, gc_join_adjust_handle_age_sweep);
Expand Down Expand Up @@ -29651,6 +29953,13 @@ void gc_heap::plan_phase (int condemned_gen_number)
#endif //MULTIPLE_HEAPS
}

#ifdef FEATURE_PREMORTEM_FINALIZATION
if (!special_sweep_p)
{
finalize_queue->UpdatePromotedGenerations (condemned_gen_number, TRUE);
}
#endif // FEATURE_PREMORTEM_FINALIZATION

if (!special_sweep_p)
{
clear_gen1_cards();
Expand Down
12 changes: 12 additions & 0 deletions src/coreclr/gc/gcpriv.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2276,6 +2276,18 @@ class gc_heap
#ifdef USE_REGIONS
PER_HEAP_ISOLATED
void sync_promoted_bytes();

PER_HEAP
void set_heap_for_contained_basic_regions (heap_segment* region, gc_heap* hp);

PER_HEAP
heap_segment* unlink_first_rw_region (int gen_idx);

PER_HEAP
void thread_rw_region_front (int gen_idx, heap_segment* region);

PER_HEAP_ISOLATED
void equalize_promoted_bytes();
#endif //USE_REGIONS

#if !defined(USE_REGIONS) || defined(_DEBUG)
Expand Down

0 comments on commit 85b4f0e

Please sign in to comment.