Index: common/collection_scheduler.cpp =================================================================== --- common/collection_scheduler.cpp (revision 698467) +++ common/collection_scheduler.cpp (working copy) @@ -31,24 +31,15 @@ return; } -void gc_update_collection_scheduler(GC* gc, int64 time_mutator, int64 time_collection) -{ - if(gc_is_specify_con_gc()){ - gc_update_con_collection_scheduler(gc, time_mutator, time_collection); - } - return; -} Boolean gc_sched_collection(GC* gc, unsigned int gc_cause) { /*collection scheduler only schedules concurrent collection now.*/ if(GC_CAUSE_CONCURRENT_GC == gc_cause){ assert(gc_is_specify_con_gc()); - return gc_sched_con_collection(gc, gc_cause); + return gc_con_perform_collection( gc ); }else{ return FALSE; } } - - Index: common/collection_scheduler.h =================================================================== --- common/collection_scheduler.h (revision 698467) +++ common/collection_scheduler.h (working copy) @@ -26,12 +26,7 @@ void collection_scheduler_initialize(GC* gc); void collection_scheduler_destruct(GC* gc); -void gc_update_collection_scheduler(GC* gc, int64 time_mutator, int64 time_collection); Boolean gc_sched_collection(GC* gc, unsigned int gc_cause); #endif - - - - Index: common/concurrent_collection_scheduler.cpp =================================================================== --- common/concurrent_collection_scheduler.cpp (revision 698467) +++ common/concurrent_collection_scheduler.cpp (working copy) @@ -22,7 +22,7 @@ #include "collection_scheduler.h" #include "concurrent_collection_scheduler.h" #include "gc_concurrent.h" -#include "../thread/marker.h" +#include "../thread/conclctor.h" #include "../verify/verify_live_heap.h" #define NUM_TRIAL_COLLECTION 2 @@ -53,6 +53,7 @@ Boolean gc_use_space_scheduler() { return cc_scheduler_kind & SPACE_BASED_SCHEDULER; } + static int64 time_delay_to_start_mark = MAX_DELAY_TIME; static POINTER_SIZE_INT space_threshold_to_start_mark = MAX_SPACE_THRESHOLD; @@ -75,6 +76,7 @@ STD_FREE(gc->collection_scheduler); } + void gc_decide_cc_scheduler_kind(char* cc_scheduler) { string_to_upper(cc_scheduler); @@ -93,281 +95,247 @@ gc_enable_time_scheduler(); } -static Boolean time_to_start_mark(GC* gc) -{ - if(!gc_use_time_scheduler()) return FALSE; - - int64 time_current = time_now(); - return (time_current - get_collection_end_time()) > time_delay_to_start_mark; -} +/*====================== new scheduler ===================*/ +extern unsigned int NUM_CON_MARKERS; +extern unsigned int NUM_CON_SWEEPERS; +unsigned int gc_get_mutator_number(GC *gc); -static Boolean space_to_start_mark(GC* gc) -{ - if(!gc_use_space_scheduler()) return FALSE; +#define MOSTLY_CON_MARKER_DIVISION 0.5 +unsigned int mostly_con_final_marker_num=1; +unsigned int mostly_con_long_marker_num=1; - POINTER_SIZE_INT size_new_obj = gc_get_new_object_size(gc,FALSE); - return (size_new_obj > space_threshold_to_start_mark); -} +unsigned int gc_get_marker_number(GC* gc) { + unsigned int mutator_num = gc_get_mutator_number(gc); + unsigned int marker_specified = NUM_CON_MARKERS; + if(marker_specified == 0) { + if( gc_is_kind(ALGO_CON_OTF_OBJ) || gc_is_kind(ALGO_CON_OTF_REF) ) { + marker_specified = min(gc->num_conclctors, mutator_num>>1); + INFO2("gc.con.scheduler", "[Marker Num] mutator num="<num_window_slots; i++){ - TRACE2("gc.con.cs",i+1<<"--"<alloc_rate_window[i]); - } - - TRACE2("gc.con.cs","Tracing Rate: "); - for(i = 0; i < cc_scheduler->num_window_slots; i++){ - TRACE2("gc.con.cs",i+1<<"--"<trace_rate_window[i]); - } - - float average_alloc_rate = sum_alloc_rate / cc_scheduler->num_window_slots; - float average_trace_rate = sum_trace_rate / cc_scheduler->num_window_slots; - float average_space_util_ratio = sum_space_util_ratio / cc_scheduler->num_window_slots; - - TRACE2("gc.con.cs","averAllocRate: "<size_free_space * average_space_util_ratio) / average_alloc_rate; - float time_trace_expected = space_stat->num_live_obj / average_trace_rate; - TRACE2("gc.con.cs","[GC][Con] expected alloc time "< time_trace_expected){ - if(gc_is_kind(ALGO_CON_OTF_OBJ)||gc_is_kind(ALGO_CON_OTF_REF)){ - float time_correction = gc_sweep_is_concurrent()? TIME_CORRECTION_OTF_MARK_SWEEP : TIME_CORRECTION_OTF_MARK; - cc_scheduler->time_delay_to_start_mark = (int64)((time_alloc_expected - time_trace_expected)*time_correction); - }else if(gc_is_kind(ALGO_CON_MOSTLY)){ - cc_scheduler->time_delay_to_start_mark = (int64)(time_mutator* TIME_CORRECTION_MOSTLY_MARK); - } - }else{ - cc_scheduler->time_delay_to_start_mark = MIN_DELAY_TIME; + float util_rate = con_collection_stat->heap_utilization_rate; + unsigned int space_threshold = 0; + if( gc_is_kind(ALGO_CON_OTF_OBJ) || gc_is_kind(ALGO_CON_OTF_REF) ) { + if( con_collection_stat->trace_rate == 0 ) //for initial iteration + con_collection_stat->trace_rate = con_collection_stat->alloc_rate*20; + unsigned int alloc_rate = con_collection_stat->alloc_rate; + if(alloc_ratetrace_rate) { // THRESHOLD = Heap*utilization_rate*(1-alloc_rate/marking_rate), accurate formaler + float alloc_marking_rate_ratio = (float)(alloc_rate)/con_collection_stat->trace_rate; + + space_threshold = (unsigned int)(heap_size*util_rate*(1-alloc_marking_rate_ratio)*conservative_factor); + } else { //use default + unsigned int alloc_while_marking = DEFAULT_MARKING_TIME*con_collection_stat->alloc_rate; + space_threshold = (unsigned int)(heap_size*util_rate) -alloc_while_marking; } - - cc_scheduler->space_threshold_to_start_mark = - (POINTER_SIZE_INT)(space_stat->size_free_space * ((time_alloc_expected - time_trace_expected) / time_alloc_expected)); - - time_delay_to_start_mark = cc_scheduler->time_delay_to_start_mark + time_this_collection_correction; - space_threshold_to_start_mark = cc_scheduler->space_threshold_to_start_mark; + } else if(gc_is_kind(ALGO_CON_MOSTLY)) { + unsigned int alloc_while_marking = DEFAULT_MARKING_TIME*con_collection_stat->alloc_rate; + space_threshold = (unsigned int)(heap_size*util_rate) -alloc_while_marking; } - TRACE2("gc.con.cs","[GC][Con] concurrent marking will delay "<<(unsigned int)(time_delay_to_start_mark>>10)<<" ms "); - TRACE2("gc.con.cs","[GC][Con] time correction "<<(unsigned int)(time_this_collection_correction>>10)<<" ms "); + if( space_threshold > con_collection_stat->surviving_size_at_gc_end ) + alloc_space_threshold = space_threshold - con_collection_stat->surviving_size_at_gc_end; + else + alloc_space_threshold = MIN_SPACE_THRESHOLD; + + //INFO2("gc.con.info", "[Threshold] alloc_space_threshold=" << alloc_space_threshold); + return space_threshold; } -void gc_update_con_collection_scheduler(GC* gc, int64 time_mutator, int64 time_collection) +/* this parameters are updated at end of GC */ +void gc_update_scheduler_parameter( GC *gc ) { - assert(gc_is_specify_con_gc()); - if(GC_CAUSE_RUNTIME_FORCE_GC == gc->cause) return; - - con_collection_scheduler_update_stat(gc, time_mutator, time_collection); - con_collection_scheduler_update_start_point(gc, time_mutator, time_collection); + Con_Collection_Statistics *con_collection_stat = gc_ms_get_con_collection_stat((GC_MS*)gc); + last_check_time_point = time_now(); + + unsigned int alloc_rate = con_collection_stat->alloc_rate; + space_check_stage_1 = alloc_rate * trans_time_unit(SPACE_CHECK_STAGE_ONE_TIME); + space_check_stage_2 = alloc_rate * trans_time_unit(SPACE_CHECK_STAGE_TWO_TIME); + //INFO2( "gc.con.scheduler", "space_check_stage_1=["<gc_start_time - con_collection_stat->gc_end_time)>>2; + //INFO2("gc.con.scheduler", "next check time = [" << trans_time_unit(check_delay_time) << "] ms" ); + if(gc_is_specify_con_sweep()) { + conservative_factor = CONSERCATIVE_FACTOR_FULLY_CONCURRENT; + } + calculate_start_con_space_threshold(con_collection_stat, gc->committed_heap_size); } -Boolean gc_sched_con_collection(GC* gc, unsigned int gc_cause) +void gc_force_update_scheduler_parameter( GC *gc ) { - if(!try_lock(gc->lock_collect_sched)) return FALSE; - vm_gc_lock_enum(); + last_check_time_point = time_now(); + //check_delay_time = SPACE_CHECK_STAGE_ONE_TIME; + check_delay_time = time_now(); + //INFO2("gc.con.scheduler", "next check time = [" << trans_time_unit(check_delay_time) << "] ms" ); + Con_Collection_Statistics *con_collection_stat = gc_ms_get_con_collection_stat((GC_MS*)gc); + con_collection_stat->alloc_rate = DEFAULT_ALLOC_RATE; +} - gc_try_finish_con_phase(gc); - if(gc_need_start_con_enum(gc)){ - /*TODO:Concurrent rootset enumeration.*/ - assert(0); - } - - if(gc_need_start_con_mark(gc)){ - INFO2("gc.con.info", "[GC][Con] concurrent mark start ..."); - gc_start_con_mark(gc); - vm_gc_unlock_enum(); - unlock(gc->lock_collect_sched); - return TRUE; - } - if(gc_need_start_con_sweep(gc)){ - gc->num_collections++; - INFO2("gc.con.info", "[GC][Con] collection number:"<< gc->num_collections<<" "); - gc_start_con_sweep(gc); - vm_gc_unlock_enum(); - unlock(gc->lock_collect_sched); - return TRUE; - } +static inline Boolean check_start_mark( GC *gc ) +{ + unsigned int new_object_occupied_size = gc_get_mutator_new_obj_size(gc); + Con_Collection_Statistics *con_collection_stat = gc_ms_get_con_collection_stat((GC_MS*)gc); + /*just debugging*/ + float used_rate = (float)(con_collection_stat->surviving_size_at_gc_end + new_object_occupied_size)/gc->committed_heap_size; + if( alloc_space_threshold < new_object_occupied_size ) { + INFO2( "gc.con.info", "[Start Con] check has been delayed " << check_delay_time << " us, until ratio at start point="<SPACE_CHECK_STAGE_TWO_TIME ) { //if time interval is too small, the alloc rate will not be updated + unsigned int interval_time = trans_time_unit(time_now() - con_collection_stat->gc_end_time); + unsigned int interval_space = new_object_occupied_size; + con_collection_stat->alloc_rate = interval_space/interval_time; + } + check_delay_time = ((alloc_space_threshold - new_object_occupied_size)/con_collection_stat->alloc_rate)<<9; + } + last_check_time_point = time_now(); - if(gc_need_reset_after_con_collect(gc)){ - int64 pause_start = time_now(); - int disable_count = vm_suspend_all_threads(); - gc_reset_after_con_collect(gc); - gc_start_mutator_time_measure(gc); - set_collection_end_time(); - vm_resume_all_threads(disable_count); - vm_gc_unlock_enum(); - INFO2("gc.con.time","[GC][Con]pause(reset collection): "<<((unsigned int)((time_now()-pause_start)>>10))<<" ms "); - unlock(gc->lock_collect_sched); - return TRUE; + //INFO2("gc.con.info", "[GC Scheduler] check has been delayed=" << last_check_delay << " us, used_rate=" << used_rate << ", free_space=" << free_space << " bytes, next delay=" << check_delay_time << " us" ); + return FALSE; +} + +static SpinLock check_lock; +static inline Boolean space_should_start_mark( GC *gc) +{ + if( ( time_now() -last_check_time_point ) > check_delay_time && try_lock(check_lock) ) { //first condition is checked frequently, second condition is for synchronization + Boolean should_start = check_start_mark(gc); + unlock(check_lock); + return should_start; } - vm_gc_unlock_enum(); - unlock(gc->lock_collect_sched); return FALSE; } -extern unsigned int NUM_MARKERS; +inline static Boolean gc_con_start_condition( GC* gc ) { + return space_should_start_mark(gc); +} -unsigned int gc_decide_marker_number(GC* gc) -{ - unsigned int num_active_marker; - Con_Collection_Scheduler* cc_scheduler = (Con_Collection_Scheduler*)gc->collection_scheduler; - /*If the number of markers is specfied, just return the specified value.*/ - if(NUM_MARKERS != 0) return NUM_MARKERS; +void gc_reset_after_con_collection(GC *gc); +void gc_merge_free_list_global(GC *gc); +void gc_con_stat_information_out(GC *gc); - /*If the number of markers isn't specified, we decide the value dynamically.*/ - if(NUM_TRIAL_COLLECTION == 0 || gc->num_collections < NUM_TRIAL_COLLECTION){ - /*Start trial cycle, collection set to 1 in trial cycle and */ - num_active_marker = 1; - }else{ - num_active_marker = cc_scheduler->last_marker_num; - int64 c_time = cc_scheduler->last_collector_time; - int64 m_time = cc_scheduler->last_mutator_time; - int64 d_time = cc_scheduler->time_delay_to_start_mark; +unsigned int sub_time = 0; +int64 pause_time = 0; +/* + concurrent collection entry function, it may start proper phase according to the current state. +*/ +Boolean gc_con_perform_collection( GC* gc ) { + int disable_count; + int64 pause_start; + Con_Collection_Statistics *con_collection_stat = gc_ms_get_con_collection_stat((GC_MS*)gc); + switch( gc->gc_concurrent_status ) { + case GC_CON_NIL : + if( !gc_con_start_condition(gc) ) + return FALSE; + if( !state_transformation( gc, GC_CON_NIL, GC_CON_STW_ENUM ) ) + return FALSE; + + gc->num_collections++; + gc->cause = GC_CAUSE_CONCURRENT_GC; - if(num_active_marker == 0) num_active_marker = 1; + con_collection_stat->gc_start_time = time_now(); + disable_count = hythread_reset_suspend_disable(); + + gc_start_con_enumeration(gc); //now, it is a stw enumeration + con_collection_stat->marking_start_time = time_now(); + state_transformation( gc, GC_CON_STW_ENUM, GC_CON_START_MARKERS ); + gc_start_con_marking(gc); - if((c_time + d_time) > m_time || (float)d_time < (m_time * 0.25)){ - TRACE2("gc.con.cs","[GC][Con] increase marker number."); - num_active_marker ++; - if(num_active_marker > gc->num_markers) num_active_marker = gc->num_markers; - }else if((float)d_time > (m_time * 0.6)){ - TRACE2("gc.con.cs","[GC][Con] decrease marker number."); - num_active_marker --; - if(num_active_marker == 0) num_active_marker = 1; - } - - TRACE2("gc.con.cs","[GC][Con] ctime "<<(unsigned)(c_time>>10)<<" mtime "<<(unsigned)(m_time>>10)<<" dtime "<<(unsigned)(d_time>>10)); - TRACE2("gc.con.cs","[GC][Con] marker num : "<last_marker_num = num_active_marker; - return num_active_marker; + return TRUE; } Index: common/concurrent_collection_scheduler.h =================================================================== --- common/concurrent_collection_scheduler.h (revision 698467) +++ common/concurrent_collection_scheduler.h (working copy) @@ -20,6 +20,7 @@ #define STAT_SAMPLE_WINDOW_SIZE 5 +struct GC_MS; typedef struct Con_Collection_Scheduler { /*common field*/ GC* gc; @@ -46,10 +47,16 @@ void con_collection_scheduler_initialize(GC* gc); void con_collection_scheduler_destruct(GC* gc); +void gc_update_scheduler_parameter( GC *gc ); +void gc_force_update_scheduler_parameter( GC *gc ); +Boolean gc_con_perform_collection( GC* gc ); Boolean gc_sched_con_collection(GC* gc, unsigned int gc_cause); -void gc_update_con_collection_scheduler(GC* gc, int64 time_mutator, int64 time_collection); void gc_decide_cc_scheduler_kind(char* cc_scheduler); void gc_set_default_cc_scheduler_kind(); + +extern unsigned int mostly_con_final_marker_num; +extern unsigned int mostly_con_long_marker_num; + #endif Index: common/gc_common.cpp =================================================================== --- common/gc_common.cpp (revision 698467) +++ common/gc_common.cpp (working copy) @@ -22,7 +22,7 @@ #include "gc_common.h" #include "gc_metadata.h" #include "../thread/mutator.h" -#include "../thread/marker.h" +#include "../thread/conclctor.h" #include "../finalizer_weakref/finalizer_weakref.h" #include "../gen/gen.h" #include "../mark_sweep/gc_ms.h" @@ -74,11 +74,19 @@ static int64 collection_start_time = time_now(); static int64 collection_end_time = time_now(); -int64 get_collection_end_time() +int64 get_gc_start_time() +{ return collection_start_time; } + +void set_gc_start_time() +{ collection_start_time = time_now(); } + +int64 get_gc_end_time() { return collection_end_time; } -void set_collection_end_time() -{ collection_end_time = time_now(); } +void set_gc_end_time() +{ + collection_end_time = time_now(); +} void gc_decide_collection_kind(GC* gc, unsigned int cause) { @@ -93,17 +101,17 @@ } -void gc_update_space_stat(GC_MS* gc) +void gc_update_space_stat(GC* gc) { #ifdef USE_UNIQUE_MARK_SWEEP_GC - gc_ms_update_space_stat((GC_MS*)gc); + gc_ms_update_space_stat((GC_MS *)gc); #endif } -void gc_reset_space_stat(GC_MS* gc) +void gc_reset_space_stat(GC* gc) { #ifdef USE_UNIQUE_MARK_SWEEP_GC - gc_ms_reset_space_stat((GC_MS*)gc); + gc_ms_reset_space_stat((GC_MS *)gc); #endif } @@ -118,7 +126,7 @@ gc_set_rootset(gc); } -void gc_reset_after_collection(GC* gc, int64 time_mutator, int64 time_collection) +void gc_reset_after_collection(GC* gc) { if(gc_is_gen_mode()) gc_prepare_mutator_remset(gc); @@ -139,12 +147,10 @@ #endif } - gc_update_space_stat((GC_MS*)gc); + gc_update_space_stat(gc); - gc_update_collection_scheduler(gc, time_mutator, time_collection); + gc_reset_space_stat(gc); - gc_reset_space_stat((GC_MS*)gc); - gc_reset_collector_state(gc); gc_clear_dirty_set(gc); @@ -154,23 +160,36 @@ } +void gc_prepare_root_in_STW(GC *gc, hythread_iterator_t iterator) +{ + + INFO2("gc.process", "GC Verifier: enumerate rootset ... \n"); + gc_clear_rootset(gc); + gc_reset_rootset(gc); + vm_enumerate_root_set(iterator); + gc_copy_interior_pointer_table_to_rootset(); + gc_set_rootset(gc); +} + +void set_check_delay( int64 mutator_time ); + void gc_reclaim_heap(GC* gc, unsigned int gc_cause) { INFO2("gc.process", "\nGC: GC start ...\n"); - collection_start_time = time_now(); - int64 time_mutator = collection_start_time - collection_end_time; - - gc->num_collections++; gc->cause = gc_cause; if(gc_is_specify_con_gc()){ - gc_finish_con_GC(gc, time_mutator); - collection_end_time = time_now(); + gc_wait_con_finish(gc); INFO2("gc.process", "GC: GC end\n"); return; } + set_gc_start_time(); + int64 time_mutator = get_gc_start_time() - get_gc_end_time(); + + gc->num_collections++; + /* FIXME:: before mutators suspended, the ops below should be very careful to avoid racing with mutators. */ @@ -207,16 +226,16 @@ gc_gen_reclaim_heap((GC_Gen*)gc, collection_start_time); #endif - collection_end_time = time_now(); + set_gc_end_time(); - int64 time_collection = collection_end_time - collection_start_time; + int64 time_collection = get_gc_end_time() - get_gc_start_time(); #if !defined(USE_UNIQUE_MARK_SWEEP_GC)&&!defined(USE_UNIQUE_MOVE_COMPACT_GC) gc_gen_collection_verbose_info((GC_Gen*)gc, time_collection, time_mutator); gc_gen_space_verbose_info((GC_Gen*)gc); #endif - gc_reset_after_collection(gc, time_mutator, time_collection); + gc_reset_after_collection(gc); gc_assign_free_area_to_mutators(gc); @@ -229,7 +248,3 @@ - - - - Index: common/gc_common.h =================================================================== --- common/gc_common.h (revision 698467) +++ common/gc_common.h (working copy) @@ -39,8 +39,10 @@ #include "../common/gc_for_barrier.h" -/* + #define USE_UNIQUE_MARK_SWEEP_GC //define it to only use Mark-Sweep GC (no NOS, no LOS). + + /* #define USE_UNIQUE_MOVE_COMPACT_GC //define it to only use Move-Compact GC (no NOS, no LOS). */ @@ -336,20 +338,8 @@ return TRUE; } -extern volatile Boolean obj_alloced_live; -inline Boolean is_obj_alloced_live() -{ return obj_alloced_live; } -inline void gc_enable_alloc_obj_live() -{ - obj_alloced_live = TRUE; -} -inline void gc_disable_alloc_obj_live() -{ - obj_alloced_live = FALSE; -} - /***************************************************************/ inline Boolean obj_is_survivor(Partial_Reveal_Object* p_obj) @@ -391,7 +381,7 @@ /***************************************************************/ /* all GCs inherit this GC structure */ -struct Marker; +struct Conclctor; struct Mutator; struct Collector; struct GC_Metadata; @@ -421,9 +411,12 @@ unsigned int num_collectors; unsigned int num_active_collectors; /* not all collectors are working */ - Marker** markers; - unsigned int num_markers; + /*concurrent markers and collectors*/ + Conclctor** conclctors; + unsigned int num_conclctors; + //unsigned int num_active_conclctors; unsigned int num_active_markers; + unsigned int num_active_sweepers; /* metadata is the pool for rootset, tracestack, etc. */ GC_Metadata* metadata; @@ -443,7 +436,7 @@ Space_Tuner* tuner; - unsigned int gc_concurrent_status; /*concurrent GC status: only support CONCURRENT_MARK_PHASE now*/ + volatile unsigned int gc_concurrent_status; /*concurrent GC status: only support CONCURRENT_MARK_PHASE now*/ Collection_Scheduler* collection_scheduler; SpinLock lock_con_mark; @@ -488,12 +481,16 @@ GC* gc_parse_options(); void gc_reclaim_heap(GC* gc, unsigned int gc_cause); +void gc_relaim_heap_con_mode( GC *gc); void gc_prepare_rootset(GC* gc); -int64 get_collection_end_time(); -void set_collection_end_time(); +int64 get_gc_start_time(); +void set_gc_start_time(); +int64 get_gc_end_time(); +void set_gc_end_time(); + /* generational GC related */ extern Boolean NOS_PARTIAL_FORWARD; Index: common/gc_concurrent.cpp =================================================================== --- common/gc_concurrent.cpp (revision 698467) +++ common/gc_concurrent.cpp (working copy) @@ -17,325 +17,597 @@ #include "gc_common.h" #include "gc_metadata.h" #include "../thread/mutator.h" -#include "../thread/marker.h" +#include "../thread/conclctor.h" #include "../thread/collector.h" #include "../finalizer_weakref/finalizer_weakref.h" #include "../gen/gen.h" #include "../mark_sweep/gc_ms.h" +#include "../mark_sweep/wspace_mark_sweep.h" #include "interior_pointer.h" #include "collection_scheduler.h" #include "gc_concurrent.h" #include "../common/gc_for_barrier.h" +#include "concurrent_collection_scheduler.h" +#include "../verify/verify_live_heap.h" -volatile Boolean concurrent_in_marking = FALSE; -volatile Boolean concurrent_in_sweeping = FALSE; -volatile Boolean mark_is_concurrent = FALSE; -volatile Boolean sweep_is_concurrent = FALSE; +struct Con_Collection_Statistics; volatile Boolean gc_sweep_global_normal_chunk = FALSE; -static void gc_check_con_mark(GC* gc) +//just debugging +inline void gc_ms_get_current_heap_usage(GC_MS *gc) { - if(!is_mark_finished(gc)){ - lock(gc->lock_con_mark); - if(gc_is_kind(ALGO_CON_OTF_OBJ)){ - gc_ms_start_con_mark((GC_MS*)gc, MIN_NUM_MARKERS); - }else if(gc_is_kind(ALGO_CON_OTF_REF)){ - gc_ms_start_con_mark((GC_MS*)gc, MIN_NUM_MARKERS); - }else if(gc_is_kind(ALGO_CON_MOSTLY)){ - //ignore. - } - unlock(gc->lock_con_mark); - } + Con_Collection_Statistics *con_collection_stat = gc_ms_get_con_collection_stat(gc); + unsigned int new_obj_size = gc_get_mutator_new_obj_size((GC *)gc); + unsigned int current_size = con_collection_stat->surviving_size_at_gc_end + new_obj_size; + INFO2("gc.con.scheduler", "[Heap Usage]surviving_size("<surviving_size_at_gc_end<<")+new_obj_size("<committed_heap_size<<")"); } -static void gc_wait_con_mark_finish(GC* gc) +void gc_con_update_stat_before_enable_alloc_live(GC *gc) { - wait_mark_finish(gc); - gc_set_barrier_function(WB_REM_NIL); - gc_set_concurrent_status(gc,GC_CON_STATUS_NIL); + Con_Collection_Statistics *con_collection_stat = gc_ms_get_con_collection_stat((GC_MS *)gc); + con_collection_stat->alloc_size_before_alloc_live = gc_get_mutator_new_obj_size(gc); } + +volatile Boolean obj_alloced_live; -unsigned int gc_decide_marker_number(GC* gc); +void gc_enable_alloc_obj_live(GC *gc) +{ + gc_con_update_stat_before_enable_alloc_live(gc); + obj_alloced_live = TRUE; +} -void gc_start_con_mark(GC* gc) +void gc_mostly_con_update_stat_after_final_marking(GC *gc) { - int disable_count; - unsigned int num_marker; - - if(!try_lock(gc->lock_con_mark) || gc_mark_is_concurrent()) return; - - lock(gc->lock_enum); - disable_count = hythread_reset_suspend_disable(); - int64 pause_start = time_now(); - gc_set_rootset_type(ROOTSET_IS_OBJ); - gc_prepare_rootset(gc); - - gc_set_concurrent_status(gc, GC_CON_MARK_PHASE); + POINTER_SIZE_INT num_live_obj = 0; + POINTER_SIZE_INT size_live_obj = 0; + POINTER_SIZE_INT num_dirty_obj_traced = 0; - num_marker = gc_decide_marker_number(gc); - - /*start concurrent mark*/ - if(gc_is_kind(ALGO_CON_OTF_OBJ)){ - gc_set_barrier_function(WB_REM_OBJ_SNAPSHOT); - gc_ms_start_con_mark((GC_MS*)gc, num_marker); - }else if(gc_is_kind(ALGO_CON_MOSTLY)){ - gc_set_barrier_function(WB_REM_SOURCE_OBJ); - gc_ms_start_mostly_con_mark((GC_MS*)gc, num_marker); - }else if(gc_is_kind(ALGO_CON_OTF_REF)){ - gc_set_barrier_function(WB_REM_OLD_VAR); - gc_ms_start_con_mark((GC_MS*)gc, num_marker); + unsigned int num_conclctors = gc->num_conclctors; + for( unsigned int i=0; iconclctors[i]; + if( conclctor->role != CONCLCTOR_ROLE_MARKER ) + continue; + num_live_obj += conclctor->live_obj_num; + size_live_obj += conclctor->live_obj_size; + num_dirty_obj_traced += conclctor->num_dirty_slots_traced; + conclctor->live_obj_num = 0; + conclctor->live_obj_size = 0; + conclctor->num_dirty_slots_traced = 0; } - unlock(gc->lock_enum); - INFO2("gc.con.time","[GC][Con]pause(enumeration root): "<<((unsigned int)((time_now()-pause_start)>>10))<<" ms "); - vm_resume_threads_after(); - assert(hythread_is_suspend_enabled()); - hythread_set_suspend_disable(disable_count); - - unlock(gc->lock_con_mark); + Con_Collection_Statistics * con_collection_stat = gc_ms_get_con_collection_stat((GC_MS*)gc); + con_collection_stat->live_size_marked += size_live_obj; + INFO2("gc.con.scheduler", "[Final Mark Finish] live_marked_size: "<live_size_marked<<" bytes"); + } -void mostly_con_mark_terminate_reset(); -void terminate_mostly_con_mark(); - -void gc_finish_con_mark(GC* gc, Boolean need_STW) +unsigned int gc_get_conclcor_num(GC* gc, unsigned int req_role); +//called by the marker when it finishes +void gc_con_update_stat_after_marking(GC *gc) { - gc_check_con_mark(gc); - - if(gc_is_kind(ALGO_CON_MOSTLY)) - terminate_mostly_con_mark(); - - gc_wait_con_mark_finish(gc); + POINTER_SIZE_INT num_live_obj = 0; + POINTER_SIZE_INT size_live_obj = 0; + POINTER_SIZE_INT num_dirty_obj_traced = 0; - int disable_count; - if(need_STW){ - /*suspend the mutators.*/ - lock(gc->lock_enum); - if(gc_is_kind(ALGO_CON_MOSTLY)){ - /*In mostly concurrent algorithm, there's a final marking pause. - Prepare root set for final marking.*/ - disable_count = hythread_reset_suspend_disable(); - gc_set_rootset_type(ROOTSET_IS_OBJ); - gc_prepare_rootset(gc); - }else{ - disable_count = vm_suspend_all_threads(); - } + unsigned int num_conclctors = gc->num_conclctors; + for( unsigned int i=0; iconclctors[i]; + if( conclctor->role != CONCLCTOR_ROLE_MARKER ) + continue; + num_live_obj += conclctor->live_obj_num; + size_live_obj += conclctor->live_obj_size; + num_dirty_obj_traced += conclctor->num_dirty_slots_traced; + conclctor->live_obj_num = 0; + conclctor->live_obj_size = 0; + conclctor->num_dirty_slots_traced = 0; } - if(gc_is_kind(ALGO_CON_MOSTLY)){ - /*In mostly concurrent algorithm, there's a final marking pause. - Suspend the mutators once again and finish the marking phase.*/ + unsigned int write_barrier_marked_size = gc_get_mutator_write_barrier_marked_size(gc); + Con_Collection_Statistics * con_collection_stat = gc_ms_get_con_collection_stat((GC_MS*)gc); + con_collection_stat->live_size_marked = size_live_obj + write_barrier_marked_size; + //INFO2("gc.con.scheduler", "[Mark Finish] live_marked_size: "<live_size_marked<<" bytes"); + + /*statistics information update (marking_end_time, trace_rate) */ + con_collection_stat->marking_end_time = time_now(); + int64 marking_time = (unsigned int)(con_collection_stat->marking_end_time - con_collection_stat->marking_start_time); - /*prepare dirty object*/ - gc_prepare_dirty_set(gc); - - gc_set_weakref_sets(gc); - - /*start STW mark*/ - gc_ms_start_mostly_con_final_mark((GC_MS*)gc, MIN_NUM_MARKERS); - - mostly_con_mark_terminate_reset(); - gc_clear_dirty_set(gc); - } + unsigned int heap_size = + con_collection_stat->surviving_size_at_gc_end + + gc_get_mutator_new_obj_size(gc); + + con_collection_stat->trace_rate = heap_size/trans_time_unit(marking_time); + - gc_reset_dirty_set(gc); - - if(need_STW){ - unlock(gc->lock_enum); - if(gc_is_kind(ALGO_CON_MOSTLY)){ - vm_resume_threads_after(); - assert(hythread_is_suspend_enabled()); - hythread_set_suspend_disable(disable_count); - }else{ - vm_resume_all_threads(disable_count); - } - } - + + /* + //statistics just for debugging + unsigned int marker_num = gc_get_conclcor_num(gc, CONCLCTOR_ROLE_MARKER); + float heap_used_rate = (float)heap_size/gc->committed_heap_size; + unsigned int new_obj_size_marking = gc_get_mutator_new_obj_size(gc) - con_collection_stat->alloc_size_before_alloc_live; + unsigned int alloc_rate_marking = new_obj_size_marking/trans_time_unit(con_collection_stat->marking_end_time - con_collection_stat->marking_start_time); + INFO2("gc.con.scheduler", "[Mark Finish] tracing time=" <live_size_marked<<" bytes"); + INFO2("gc.con.scheduler", "[Mark Finish] alloc_rate: "<alloc_rate<<" b/ms"); + INFO2("gc.con.scheduler", "[Mark Finish] trace_rate: "<trace_rate<<" b/ms"); } -int64 gc_get_con_mark_time(GC* gc) +//Called only when heap is exhuaset +void gc_con_update_stat_heap_exhausted(GC* gc) { - int64 time_mark = 0; - Marker** markers = gc->markers; + unsigned int new_obj_size = gc_get_mutator_new_obj_size(gc); + Con_Collection_Statistics * con_collection_stat = gc_ms_get_con_collection_stat((GC_MS*)gc); + con_collection_stat->heap_utilization_rate = (float)(con_collection_stat->surviving_size_at_gc_end + new_obj_size)/gc->committed_heap_size; + //INFO2("gc.con.scheduler", "[Heap exhausted] surviving size="<surviving_size_at_gc_end<<" bytes, new_obj_size="<surviving_size_at_gc_end<<" bytes"); + INFO2("gc.con.scheduler", "[Reset] alloc_rate: "<alloc_rate<<" b/ms"); + INFO2("gc.con.scheduler", "[Reset] utilization_rate: "<heap_utilization_rate); + INFO2("gc.con.scheduler", "[Reset] trace_rate: "<trace_rate<<" b/ms"); + INFO2("gc.con.scheduler", "[Reset] sweeping time: "<sweeping_time<<" us"); + INFO2("gc.con.scheduler", "[Reset] gc time: "<< trans_time_unit(con_collection_stat->gc_end_time - con_collection_stat->gc_start_time) ); + INFO2("gc.con.scheduler","=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+="); } -void gc_wait_con_sweep_finish(GC* gc) +void gc_reset_after_con_collection(GC* gc) { - wait_collection_finish(gc); - gc_set_concurrent_status(gc,GC_CON_STATUS_NIL); + assert(gc_is_specify_con_gc()); + int64 reset_start = time_now(); + if(!IGNORE_FINREF ){ + INFO2("gc.process", "GC: finref process after collection ...\n"); + gc_put_finref_to_vm(gc); + gc_reset_finref_metadata(gc); + gc_activate_finref_threads((GC*)gc); +#ifndef BUILD_IN_REFERENT + } else { + gc_clear_weakref_pools(gc); + gc_clear_finref_repset_pool(gc); +#endif + } + reset_start = time_now(); + gc_reset_con_space_stat(gc); + gc_clear_conclctor_role(gc); + vm_reclaim_native_objs(); } -void gc_finish_con_sweep(GC * gc) + + +void gc_set_default_con_algo() { - gc_wait_con_sweep_finish(gc); + assert((GC_PROP & ALGO_CON_MASK) == 0); + GC_PROP |= ALGO_CON_OTF_OBJ; } -void gc_try_finish_con_phase(GC * gc) +void gc_decide_con_algo(char* concurrent_algo) { - /*Note: we do not finish concurrent mark here if we do not want to start concurrent sweep.*/ - if(gc_con_is_in_marking(gc) && is_mark_finished(gc)){ - /*Although all conditions above are satisfied, we can not guarantee concurrent marking is finished. - Because, sometimes, the concurrent marking has not started yet. We check the concurrent mark lock - here to guarantee this occasional case.*/ - if(try_lock(gc->lock_con_mark)){ - unlock(gc->lock_con_mark); - gc_finish_con_mark(gc, TRUE); - } + string_to_upper(concurrent_algo); + GC_PROP &= ~ALGO_CON_MASK; + if(!strcmp(concurrent_algo, "OTF_OBJ")){ + GC_PROP |= ALGO_CON_OTF_OBJ; + }else if(!strcmp(concurrent_algo, "MOSTLY_CON")){ + GC_PROP |= ALGO_CON_MOSTLY; + }else if(!strcmp(concurrent_algo, "OTF_SLOT")){ + GC_PROP |= ALGO_CON_OTF_REF; } - - if(gc_con_is_in_sweeping(gc) && is_collector_finished(gc)){ - //The reason is same as concurrent mark above. - if(try_lock(gc->lock_con_sweep)){ - unlock(gc->lock_con_sweep); - gc_finish_con_sweep(gc); - } - } } -void gc_reset_after_collection(GC* gc, int64 time_mutator, int64 time_collection); -void gc_reset_after_con_collect(GC* gc) +/* + gc start enumeration phase, now, it is in a stop-the-world manner +*/ +void gc_start_con_enumeration(GC * gc) { - assert(gc_is_specify_con_gc()); - - int64 time_mutator = gc_get_mutator_time(gc); - int64 time_collection = gc_get_collector_time(gc) + gc_get_marker_time(gc); + gc_set_rootset_type(ROOTSET_IS_OBJ); + gc_prepare_rootset(gc); +} - gc_reset_interior_pointer_table(); +//unsigned int gc_decide_marker_number(GC* gc); +unsigned int gc_get_marker_number(GC* gc); +/* gc start marking phase */ +void gc_start_con_marking(GC *gc) +{ + unsigned int num_marker; + num_marker = gc_get_marker_number(gc); - gc_reset_after_collection(gc, time_mutator, time_collection); - - if(gc_mark_is_concurrent()){ - gc_reset_con_mark(gc); + if(gc_is_kind(ALGO_CON_OTF_OBJ)) { + gc_enable_alloc_obj_live(gc); + gc_set_barrier_function(WB_REM_OBJ_SNAPSHOT); + gc_ms_start_con_mark((GC_MS*)gc, num_marker); + } else if(gc_is_kind(ALGO_CON_MOSTLY)) { + gc_set_barrier_function(WB_REM_SOURCE_OBJ); + gc_ms_start_mostly_con_mark((GC_MS*)gc, num_marker); + } else if(gc_is_kind(ALGO_CON_OTF_REF)) { + gc_enable_alloc_obj_live(gc); + gc_set_barrier_function(WB_REM_OLD_VAR); + gc_ms_start_con_mark((GC_MS*)gc, num_marker); } +} - if(gc_sweep_is_concurrent()){ - gc_reset_con_sweep(gc); + +/* + gc start sweeping phase +*/ +void gc_prepare_sweeping(GC *gc) { + INFO2("gc.con.info", "Concurrent collection, current collection = " << gc->num_collections ); + /*FIXME: enable finref*/ + if(!IGNORE_FINREF ){ + gc_set_obj_with_fin(gc); + Collector* collector = gc->collectors[0]; + collector_identify_finref(collector); + #ifndef BUILD_IN_REFERENT + } else { + conclctor_set_weakref_sets(gc); + gc_update_weakref_ignore_finref(gc); + #endif } + gc_identify_dead_weak_roots(gc); } -void gc_finish_con_GC(GC* gc, int64 time_mutator) -{ +int64 get_last_check_point(); +// for the case pure stop the world +static void gc_partial_con_PSTW( GC *gc) { int64 time_collection_start = time_now(); - + INFO2("gc.space.stat","Stop-the-world collection = "<num_collections<<""); + INFO2("gc.con.info", "from last check point =" << (unsigned int)(time_collection_start -get_last_check_point()) ); + // stop the world enumeration gc->num_collections++; - - lock(gc->lock_enum); - int disable_count = hythread_reset_suspend_disable(); gc_set_rootset_type(ROOTSET_IS_REF); gc_prepare_rootset(gc); - unlock(gc->lock_enum); - - if(gc_sweep_is_concurrent()){ - if(gc_con_is_in_sweeping()) - gc_finish_con_sweep(gc); - }else{ - if(gc_con_is_in_marking()){ - gc_finish_con_mark(gc, FALSE); - } - gc->in_collection = TRUE; - gc_reset_mutator_context(gc); - if(!IGNORE_FINREF ) gc_set_obj_with_fin(gc); - gc_ms_reclaim_heap((GC_MS*)gc); + + if(gc->cause != GC_CAUSE_RUNTIME_FORCE_GC ) { + unsigned int new_obj_size = gc_get_mutator_new_obj_size(gc); + Con_Collection_Statistics * con_collection_stat = gc_ms_get_con_collection_stat((GC_MS*)gc); + con_collection_stat->heap_utilization_rate = (float)(con_collection_stat->surviving_size_at_gc_end + new_obj_size)/gc->committed_heap_size; } - int64 time_collection = 0; - if(gc_mark_is_concurrent()){ - time_collection = gc_get_con_mark_time(gc); - gc_reset_con_mark(gc); - }else{ - time_collection = time_now()-time_collection_start; - } + //reclaim heap + gc_reset_mutator_context(gc); + if(!IGNORE_FINREF ) gc_set_obj_with_fin(gc); + gc_ms_reclaim_heap((GC_MS*)gc); - if(gc_sweep_is_concurrent()){ - gc_reset_con_sweep(gc); + //update live size + gc_PSTW_update_stat_after_marking(gc); + + // reset the collection and resume mutators + gc_reset_after_con_collection(gc); + + set_con_nil(gc); // concurrent scheduling will continue after mutators are resumed + vm_resume_threads_after(); + assert(hythread_is_suspend_enabled()); + hythread_set_suspend_disable(disable_count); +} + +void terminate_mostly_con_mark(); +void wspace_mostly_con_final_mark( GC *gc ); + +// for the case concurrent marking is not finished before heap is exhausted +static void gc_partial_con_PMSS(GC *gc) { + INFO2("gc.con.info", "[PMSS] Heap has been exhuasted, current collection = " << gc->num_collections ); + // wait concurrent marking finishes + int64 wait_start = time_now(); + gc_disable_alloc_obj_live(gc); // in the STW manner, so we can disable it at anytime before the mutators are resumed + //in the stop the world phase (only conclctors is running at the moment), so the spin lock will not lose more performance + while( gc->gc_concurrent_status == GC_CON_START_MARKERS || + gc->gc_concurrent_status == GC_CON_TRACING || + gc->gc_concurrent_status == GC_CON_TRACE_DONE) + { + vm_thread_yield(); //let the unfinished marker run } - gc_reset_after_collection(gc, time_mutator, time_collection); + /*just debugging*/ + gc_ms_get_current_heap_usage((GC_MS *)gc); + int64 pause_time = time_now() - wait_start; + INFO2("gc.con.info", "[PMSS]wait marking time="<marking_end_time - wait_start); + INFO2("gc.con.info", "[PMSS] marking late time [" << marking_time_shortage << "] us" ); + /* + //re-enumerate + gc_set_rootset_type(ROOTSET_IS_REF); + gc_prepare_root_in_STW(gc, iterator); + //verify marking phase + verify_heap_after_con_gc(gc); + */ + + // start STW reclaiming heap + gc_con_update_stat_heap_exhausted(gc); // calculate util rate + gc_reset_mutator_context(gc); + if(!IGNORE_FINREF ) gc_set_obj_with_fin(gc); + gc_ms_reclaim_heap((GC_MS*)gc); - gc_start_mutator_time_measure(gc); + // reset after partial stop the world collection + gc_reset_after_con_collection(gc); + set_con_nil(gc); +} + +// only when current sweep is set to false +static void gc_partial_con_CMSS(GC *gc) { - vm_resume_threads_after(); - assert(hythread_is_suspend_enabled()); - hythread_set_suspend_disable(disable_count); - int64 pause_time = time_now()-time_collection_start; + INFO2("gc.con.info", "[CMSS] Heap has been exhuasted, current collection = " << gc->num_collections ); + gc_disable_alloc_obj_live(gc); // in the STW manner, so we can disable it at anytime before the mutators are resumed - if(GC_CAUSE_RUNTIME_FORCE_GC == gc->cause){ - INFO2("gc.con.time","[GC][Con]pause( Forcing GC ): "<<((unsigned int)(pause_time>>10))<<" ms "); - }else{ - INFO2("gc.con.time","[GC][Con]pause( Heap exhuasted ): "<<((unsigned int)(pause_time>>10))<<" ms "); + /*just debugging*/ + gc_ms_get_current_heap_usage((GC_MS *)gc); + Con_Collection_Statistics *con_collection_stat = gc_ms_get_con_collection_stat((GC_MS*)gc); + unsigned int from_marking_end = (unsigned int)(time_now() - con_collection_stat->marking_end_time); + INFO2("gc.con.info", "[CMSS] marking early time [" << from_marking_end << "] us" ); + + /*[con_verifier] + //re-enumerate + gc_set_rootset_type(ROOTSET_IS_REF); + gc_prepare_root_in_STW(gc, iterator); + //verify marking phase + verify_heap_after_con_gc(gc); + */ + + gc_con_update_stat_heap_exhausted(gc); // calculate util rate + + // start reclaiming heap, it will skip the marking phase + gc_reset_mutator_context(gc); + if(!IGNORE_FINREF ) gc_set_obj_with_fin(gc); + gc_ms_reclaim_heap((GC_MS*)gc); + + // reset after partial stop the world collection + gc_reset_after_con_collection(gc); + set_con_nil(gc); +} + +void gc_merge_free_list_global(GC *gc); +//for the case concurrent marking and partial concurrent sweeping +static void gc_partial_con_CMPS( GC *gc ) { + + while(gc->gc_concurrent_status == GC_CON_SWEEPING || gc->gc_concurrent_status == GC_CON_SWEEP_DONE) { + vm_thread_yield(); //let the unfinished sweeper run } - return; + gc_merge_free_list_global(gc); + // reset after partial stop the world collection + gc_reset_after_con_collection(gc); + set_con_nil(gc); } -void gc_set_default_con_algo() + +inline static void partial_stop_the_world_info( unsigned int type, unsigned int pause_time ) { + switch( type ) { + case GC_PARTIAL_PSTW : + INFO2("gc.con.time","[PT] pause ( Heap exhuasted ), PSTW=" << pause_time << " us"); + break; + case GC_PARTIAL_PMSS : + INFO2("gc.con.time","[PT] pause ( Heap exhuasted ), PMSS=" << pause_time << " us"); + break; + case GC_PARTIAL_CMPS : + INFO2("gc.con.time","[PT] pause ( Heap exhuasted ), CMPS=" << pause_time << " us"); + break; + case GC_PARTIAL_CMSS : + INFO2("gc.con.time","[PT] pause ( Heap exhuasted ), CMSS=" << pause_time << " us"); + break; + case GC_PARTIAL_FCSR : + INFO2("gc.con.time","[PT] pause ( Heap exhuasted ), FCSR=" << pause_time << " us"); + break; + } +} + +static unsigned int gc_con_heap_full_mostly_con( GC *gc ) { - assert((GC_PROP & ALGO_CON_MASK) == 0); - GC_PROP |= ALGO_CON_OTF_OBJ; + while( gc->gc_concurrent_status == GC_CON_START_MARKERS ) { // we should enumerate rootset after old rootset is traced + vm_thread_yield(); + } + + int64 final_start = time_now(); + int disable_count = hythread_reset_suspend_disable(); + gc_set_rootset_type(ROOTSET_IS_OBJ); + gc_prepare_rootset(gc); + + gc_set_barrier_function(WB_REM_NIL); //in stw phase, so we can remove write barrier at any time + terminate_mostly_con_mark(); // terminate current mostly concurrent marking + + //in the stop the world phase (only conclctors is running at the moment), so the spin lock will not lose more performance + while(gc->gc_concurrent_status == GC_CON_TRACING) { + vm_thread_yield(); //let the unfinished marker run + } + + //final marking phase + gc_clear_conclctor_role(gc); + wspace_mostly_con_final_mark(gc); + + /*just debugging*/ + int64 final_time = time_now() - final_start; + INFO2("gc.scheduler", "[MOSTLY_CON] final marking time=" << final_time << " us"); + gc_ms_get_current_heap_usage((GC_MS *)gc); + + // start STW reclaiming heap + gc_con_update_stat_heap_exhausted(gc); // calculate util rate + gc_reset_mutator_context(gc); + if(!IGNORE_FINREF ) gc_set_obj_with_fin(gc); + gc_ms_reclaim_heap((GC_MS*)gc); + + // reset after partial stop the world collection + gc_reset_after_con_collection(gc); + set_con_nil(gc); + + vm_resume_threads_after(); + hythread_set_suspend_disable(disable_count); + return GC_PARTIAL_PMSS; + } -void gc_decide_con_algo(char* concurrent_algo) +static unsigned int gc_con_heap_full_otf( GC *gc ) { - string_to_upper(concurrent_algo); - GC_PROP &= ~ALGO_CON_MASK; - if(!strcmp(concurrent_algo, "OTF_OBJ")){ - GC_PROP |= ALGO_CON_OTF_OBJ; - }else if(!strcmp(concurrent_algo, "MOSTLY_CON")){ - GC_PROP |= ALGO_CON_MOSTLY; - }else if(!strcmp(concurrent_algo, "OTF_SLOT")){ - GC_PROP |= ALGO_CON_OTF_REF; + unsigned int partial_type; //for time measuring and debugging + int disable_count = vm_suspend_all_threads(); + Con_Collection_Statistics *con_collection_stat = gc_ms_get_con_collection_stat((GC_MS*)gc); + con_collection_stat->pause_start_time = time_now(); + switch(gc->gc_concurrent_status) { + case GC_CON_START_MARKERS : + case GC_CON_TRACING : + case GC_CON_TRACE_DONE : + partial_type = GC_PARTIAL_PMSS; + gc_partial_con_PMSS(gc); + break; + case GC_CON_BEFORE_SWEEP : // only when current sweep is set to false + partial_type = GC_PARTIAL_CMSS; + gc_partial_con_CMSS(gc); + break; + case GC_CON_SWEEPING : + case GC_CON_SWEEP_DONE : + partial_type = GC_PARTIAL_CMPS; + gc_partial_con_CMPS(gc); + break; + case GC_CON_BEFORE_FINISH : //heap can be exhausted when sweeping finishes, very rare + partial_type = GC_PARTIAL_FCSR; + gc_merge_free_list_global(gc); + gc_reset_after_con_collection(gc); + set_con_nil(gc); + break; + case GC_CON_RESET : + case GC_CON_NIL : + case GC_CON_STW_ENUM : + /*do nothing, if still in gc_con_reset, will wait to finish after resuming. this case happens rarely*/ + partial_type = GC_PARTIAL_FCSR; + break; + /* other state is illegal here */ + default: + INFO2("gc.con.info", "illegal state when the heap is out [" << gc->gc_concurrent_status << "]"); + RAISE_ERROR + } + vm_resume_all_threads(disable_count); + return partial_type; +} + +void gc_con_stat_information_out(GC *gc); +/* +this method is called before STW gc start, there is a big lock outside +*/ +void gc_wait_con_finish( GC* gc ) { + int64 time_collection_start = time_now(); + unsigned int partial_type; //for time measuring and debugging + + /* cocurrent gc is idle */ + if( state_transformation( gc, GC_CON_NIL, GC_CON_DISABLE ) ) { // for the race condition of con schduling and STW gc + Con_Collection_Statistics *con_collection_stat = gc_ms_get_con_collection_stat((GC_MS*)gc); + con_collection_stat->gc_start_time = time_now(); + con_collection_stat->pause_start_time = con_collection_stat->gc_start_time; + partial_type = GC_PARTIAL_PSTW; + gc_partial_con_PSTW( gc ); + } else { + while(gc->gc_concurrent_status == GC_CON_STW_ENUM ) { //wait concurrent gc finish enumeration + hythread_safe_point(); + vm_thread_yield(); + } + if( gc_is_kind(ALGO_CON_MOSTLY) ) + partial_type = gc_con_heap_full_mostly_con(gc); + else if( gc_is_kind(ALGO_CON_OTF_OBJ) || gc_is_kind(ALGO_CON_OTF_REF) ) { + partial_type = gc_con_heap_full_otf(gc); + if(gc->gc_concurrent_status == GC_CON_RESET) { + while( gc->gc_concurrent_status == GC_CON_RESET ) { //wait concurrent to finish + hythread_safe_point(); + vm_thread_yield(); + } + } + } + else + RAISE_ERROR + } + + int64 pause_time = time_now()-time_collection_start; + gc_con_stat_information_out(gc); + if(GC_CAUSE_RUNTIME_FORCE_GC == gc->cause) { + INFO2("gc.con.time","[GC][Con]pause( Forcing GC ): "<<(unsigned int)(pause_time)<<" us "); + } else { + partial_stop_the_world_info( partial_type, (unsigned int)pause_time ); } } + + Index: common/gc_concurrent.h =================================================================== --- common/gc_concurrent.h (revision 698467) +++ common/gc_concurrent.h (working copy) @@ -19,21 +19,69 @@ #define _GC_CONCURRENT_H_ #include "gc_common.h" -enum GC_CONCURRENT_STATUS{ - GC_CON_STATUS_NIL = 0x00, - GC_CON_MARK_PHASE = 0x01, - GC_MOSTLY_CON_FINAL_MARK_PHASE = 0x11, // for mostly concurrent only. - GC_CON_SWEEP_PHASE = 0x02 + +#define RATE_CALCULATE_DENOMINATOR_FACTOR 10; //trans us to ms +inline unsigned int trans_time_unit(int64 x) +{ + int64 result = x>>10; + if(result) return (unsigned int)result; + return 1; +} + +#define RAISE_ERROR __asm int 3; +/* concurrent collection states in new design */ +enum GC_CONCURRENT_STATUS { + GC_CON_NIL = 0x00, + GC_CON_STW_ENUM = 0x01, + GC_CON_START_MARKERS = 0x02, + GC_CON_TRACING = 0x03, + GC_CON_TRACE_DONE = 0x04, + GC_CON_BEFORE_SWEEP = 0x05, + GC_CON_SWEEPING = 0x06, + GC_CON_SWEEP_DONE = 0x07, + GC_CON_BEFORE_FINISH = 0x08, + GC_CON_RESET = 0x09, + GC_CON_DISABLE = 0x0A, }; +// this type is just for debugging and time measuring +enum GC_PARTIAL_STW_TYPE { + GC_PARTIAL_PSTW = 0x00, //pure stop the world + GC_PARTIAL_PMSS = 0x01, //concurrent marking has finished and stop the world sweeping + GC_PARTIAL_CMSS = 0x02, // partial concurrent marking and stop the world sweeping + GC_PARTIAL_CMPS = 0x03, //concurrent marking and sweeping + GC_PARTIAL_FCSR = 0x04, //fully concurrent marking and sweeping, but stw finish reset +}; + enum HANDSHAKE_SINGAL{ HSIG_MUTATOR_SAFE = 0x0, - HSIG_DISABLE_SWEEP_LOCAL_CHUNKS = 0x01, HSIG_DISABLE_SWEEP_GLOBAL_CHUNKS = 0x02, HSIG_MUTATOR_ENTER_ALLOC_MARK = 0x03, }; +typedef struct Con_Collection_Statistics { + POINTER_SIZE_INT live_size_marked; //marked objects size + POINTER_SIZE_INT alloc_size_before_alloc_live; //alloc objects size before marking + POINTER_SIZE_INT live_alloc_size; + POINTER_SIZE_INT surviving_size_at_gc_end; //total live object size when gc is ended + + POINTER_SIZE_INT trace_rate; //bytes per ms + POINTER_SIZE_INT alloc_rate; //bytes per ms + + float heap_utilization_rate; + + int64 gc_start_time; + int64 gc_end_time; + + int64 marking_start_time; + int64 marking_end_time; + + int64 sweeping_time; + int64 pause_start_time; + +} Con_Space_Statistics; + inline void gc_set_con_gc(unsigned int con_phase) { GC_PROP |= con_phase; } @@ -58,107 +106,101 @@ inline Boolean gc_is_specify_con_sweep() { return (GC_PROP & ALGO_CON_SWEEP) == ALGO_CON_SWEEP; } -extern volatile Boolean concurrent_in_marking; -extern volatile Boolean concurrent_in_sweeping; -extern volatile Boolean mark_is_concurrent; -extern volatile Boolean sweep_is_concurrent; -inline Boolean gc_mark_is_concurrent() -{ - return mark_is_concurrent; -} +extern volatile Boolean obj_alloced_live; -inline void gc_mark_set_concurrent() -{ - if(gc_is_kind(ALGO_CON_OTF_OBJ) || gc_is_kind(ALGO_CON_OTF_REF)) - gc_enable_alloc_obj_live(); - mark_is_concurrent = TRUE; -} +inline Boolean is_obj_alloced_live() +{ return obj_alloced_live; } -inline void gc_mark_unset_concurrent() -{ - gc_disable_alloc_obj_live(); - mark_is_concurrent = FALSE; +inline void gc_disable_alloc_obj_live(GC *gc) +{ + obj_alloced_live = FALSE; } -inline Boolean gc_con_is_in_marking() +void gc_enable_alloc_obj_live(GC * gc); + +/* + tranform the states across the collection process, + which should be a atomic operation because there are several collector run parallel +*/ +inline Boolean state_transformation( GC* gc, unsigned int from_state, unsigned int to_state ) { - return concurrent_in_marking; + unsigned int old_state = apr_atomic_cas32( &gc->gc_concurrent_status, to_state, from_state ); + if( old_state != from_state ) + return FALSE; + else + return TRUE; } -inline Boolean gc_con_is_in_marking(GC* gc) -{ - return gc->gc_concurrent_status == GC_CON_MARK_PHASE; +/* set concurrent to idle, + Or enable concurrent gc, called when STW gc finishes + */ +inline void set_con_nil( GC *gc ) { + apr_atomic_set32( &gc->gc_concurrent_status, GC_CON_NIL ); } -inline Boolean gc_sweep_is_concurrent() -{ - return sweep_is_concurrent; + +/* gc start enumeration phase, now, it is in a stop-the-world manner */ +void gc_start_con_enumeration(GC * gc); + +/* gc start marking phase */ +void gc_start_con_marking(GC *gc); + + +/* prepare for sweeping */ +void gc_prepare_sweeping(GC *gc); + +/* gc start sweeping phase */ +void gc_start_con_sweeping(GC *gc); + +/* gc finish concurrent collection */ +void gc_con_final_work(GC* gc); + + +/* gc wait cocurrent collection finishes */ +void gc_wait_con_finish( GC* gc ); + +/* is in gc marking phase */ +inline Boolean in_con_marking_phase( GC *gc ) { + unsigned int status = gc->gc_concurrent_status; + return (status == GC_CON_TRACING) || (status == GC_CON_TRACE_DONE); } -inline void gc_sweep_set_concurrent() -{ - sweep_is_concurrent = TRUE; +/* is in gc sweeping phase */ +inline Boolean in_con_sweeping_phase( GC *gc ) { + unsigned int status = gc->gc_concurrent_status; + return (status == GC_CON_SWEEPING) || (status == GC_CON_SWEEP_DONE); } -inline void gc_sweep_unset_concurrent() -{ - sweep_is_concurrent = FALSE; +inline Boolean in_con_idle( GC *gc ) { + return gc->gc_concurrent_status == GC_CON_NIL; } -inline Boolean gc_con_is_in_sweeping() -{ - return concurrent_in_sweeping; +inline Boolean gc_con_is_in_STW( GC *gc ) { + return gc->gc_concurrent_status == GC_CON_DISABLE; } -inline Boolean gc_con_is_in_sweeping(GC* gc) -{ - return gc->gc_concurrent_status == GC_CON_SWEEP_PHASE; +/* is gc ready to sweeping */ +inline Boolean in_con_ready_sweep( GC *gc ) { + return gc->gc_concurrent_status == GC_CON_BEFORE_SWEEP; } -inline void gc_set_concurrent_status(GC*gc, unsigned int status) -{ - /*Reset status*/ - concurrent_in_marking = FALSE; - concurrent_in_sweeping = FALSE; - - gc->gc_concurrent_status = status; - switch(status){ - case GC_CON_MARK_PHASE: - gc_mark_set_concurrent(); - concurrent_in_marking = TRUE; - break; - case GC_CON_SWEEP_PHASE: - gc_sweep_set_concurrent(); - concurrent_in_sweeping = TRUE; - break; - default: - assert(!concurrent_in_marking && !concurrent_in_sweeping); - } +/* is gc sweeping */ +inline Boolean in_con_sweep( GC *gc ) { + return ( gc->gc_concurrent_status == GC_CON_SWEEPING || gc->gc_concurrent_status == GC_CON_SWEEP_DONE ); - return; } -void gc_reset_con_mark(GC* gc); -void gc_start_con_mark(GC* gc); -void gc_finish_con_mark(GC* gc, Boolean need_STW); -int64 gc_get_con_mark_time(GC* gc); +void gc_con_update_stat_after_marking( GC *gc ); -void gc_start_con_sweep(GC* gc); -void gc_finish_con_sweep(GC * gc); -void gc_reset_after_con_collect(GC* gc); -void gc_try_finish_con_phase(GC * gc); - void gc_decide_con_algo(char* concurrent_algo); void gc_set_default_con_algo(); -void gc_reset_con_sweep(GC* gc); -void gc_finish_con_GC(GC* gc, int64 time_mutator); - extern volatile Boolean gc_sweep_global_normal_chunk; + inline Boolean gc_is_sweep_global_normal_chunk() { return gc_sweep_global_normal_chunk; } Index: common/gc_for_barrier.cpp =================================================================== --- common/gc_for_barrier.cpp (revision 698467) +++ common/gc_for_barrier.cpp (working copy) @@ -18,13 +18,17 @@ /** * @author Xiao-Feng Li, 2006/10/05 */ - + +#include +#include #include "../gen/gen.h" #include "../thread/mutator.h" #include "gc_for_barrier.h" #include "../mark_sweep/wspace_mark_sweep.h" #include "../common/gc_concurrent.h" +#include "../common/gc_common.h" #include "../finalizer_weakref/finalizer_weakref.h" +#include "../verify/verify_live_heap.h" /* All the write barrier interfaces need cleanup */ @@ -117,10 +121,8 @@ Mutator *mutator = (Mutator *)gc_get_tls(); //FIXME: Release lock. - lock(mutator->dirty_set_lock); obj_dirty_in_table((Partial_Reveal_Object *) p_obj_holding_ref); - mutator_dirtyset_add_entry(mutator, (Partial_Reveal_Object*)p_obj_holding_ref); - unlock(mutator->dirty_set_lock); + mutator_dirtyset_add_entry(mutator, (Partial_Reveal_Object*)p_obj_holding_ref); } } @@ -204,7 +206,8 @@ mutator_dirtyset_add_entry(mutator, obj_to_snapshot); } } - obj_mark_black_in_table((Partial_Reveal_Object *) p_obj_holding_ref); + obj_mark_gray_in_table((Partial_Reveal_Object *) p_obj_holding_ref); // now, the black-only obj (no gray bit been set) will also be scaned by marker, here mark it to gray to prevent this, just a workaround + obj_mark_black_in_table((Partial_Reveal_Object *) p_obj_holding_ref, mutator); obj_dirty_in_table((Partial_Reveal_Object *) p_obj_holding_ref); } } @@ -215,33 +218,142 @@ REF* p_obj_slot = (REF*) p_slot ; Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*)read_slot(p_obj_slot); if(p_obj && obj_need_remember_oldvar(p_obj)){ + mutator->dirty_obj_num++; mutator_dirtyset_add_entry(mutator, p_obj); } } +/* +static void write_barrier_for_check(Managed_Object_Handle p_obj_holding_ref,Managed_Object_Handle *p_slot, Managed_Object_Handle p_target) +{ + //Mutator *mutator = (Mutator *)gc_get_tls(); + + Partial_Reveal_Object* src_obj = (Partial_Reveal_Object*)p_obj_holding_ref; + Partial_Reveal_Object* sub_obj = (Partial_Reveal_Object*)read_slot((REF*) p_slot); + Partial_Reveal_Object* target_obj = (Partial_Reveal_Object*)p_target; + + if(src_obj && (!obj_is_mark_black_in_table(src_obj))){ + INFO2("gc.verifier", "[write_barrier_for_check] [Src]"); + analyze_bad_obj(src_obj); + RAISE_ERROR; + } + + if(sub_obj && (!obj_is_mark_black_in_table(sub_obj))){ + INFO2("gc.verifier", "[write_barrier_for_check] [Sub]"); + analyze_bad_obj(sub_obj); + INFO2("gc.verifier", "[source object]"); + analyze_bad_obj(src_obj); + //RAISE_ERROR; + return; + } + + if(target_obj && (!obj_is_mark_black_in_table(target_obj))){ + INFO2("gc.verifier", "[write_barrier_for_check] [Target]"); + analyze_bad_obj(target_obj); + RAISE_ERROR; + } + + *p_slot = p_target; +} +*/ //=========================================== /* The following routines were supposed to be the only way to alter any value in gc heap. */ void gc_heap_write_ref (Managed_Object_Handle p_obj_holding_ref, unsigned offset, Managed_Object_Handle p_target) { assert(0); } -void gc_heap_wrote_object (Managed_Object_Handle p_obj_written) + +Boolean gc_heap_copy_object_array(Managed_Object_Handle src_array, unsigned int src_start, Managed_Object_Handle dst_array, unsigned int dst_start, unsigned int length) { - /*Concurrent Mark: Since object clone and array copy do not modify object slots, - we treat it as an new object. It has already been marked when dest object was created. - We use WB_REM_SOURCE_OBJ function here to debug. - */ - if(WB_REM_SOURCE_OBJ == write_barrier_function){ - Mutator *mutator = (Mutator *)gc_get_tls(); - lock(mutator->dirty_set_lock); - - obj_dirty_in_table((Partial_Reveal_Object *) p_obj_written); - mutator_dirtyset_add_entry(mutator, (Partial_Reveal_Object*)p_obj_written); - - unlock(mutator->dirty_set_lock); - } + GC_VTable_Info *src_gcvt = obj_get_gcvt((Partial_Reveal_Object*)src_array); + GC_VTable_Info *dst_gcvt = obj_get_gcvt((Partial_Reveal_Object*)dst_array); + + Class_Handle src_class = src_gcvt->gc_clss; + Class_Handle dst_class = dst_gcvt->gc_clss; + + + //element size of src should be same as element size of dst + assert(src_gcvt->array_elem_size == dst_gcvt->array_elem_size); + unsigned int elem_size = src_gcvt->array_elem_size; + unsigned int src_first_elem_offset = array_first_element_offset((Partial_Reveal_Array*)src_array); + unsigned int dst_first_elem_offset = array_first_element_offset((Partial_Reveal_Array*)dst_array); + /* + #ifdef COMPRESS_REFERENCE + COMPRESSED_REFERENCE *src_copy_body = (COMPRESSED_REFERENCE *)((POINTER_SIZE_INT)src_array + src_first_elem_offset + elem_size*src_start); + COMPRESSED_REFERENCE *dst_copy_body = (COMPRESSED_REFERENCE *)((POINTER_SIZE_INT)dst_array + dst_first_elem_offset + elem_size*dst_start); + #else + #endif + */ + REF* src_copy_body = (REF*)((POINTER_SIZE_INT)src_array + src_first_elem_offset + elem_size*src_start); + REF* dst_copy_body = (REF*)((POINTER_SIZE_INT)dst_array + dst_first_elem_offset + elem_size*dst_start); + + + if(class_is_instanceof(src_class, dst_class)) { + //rem obj before is for OTF GC barriers + if(WB_REM_OLD_VAR == write_barrier_function) { + for (unsigned int count = 0; count < length; count++) { + write_barrier_rem_slot_oldvar((Managed_Object_Handle *)dst_copy_body+count); + } + } else if(WB_REM_OBJ_SNAPSHOT == write_barrier_function) { + write_barrier_rem_obj_snapshot(dst_array); + } + + memmove(dst_copy_body, src_copy_body, length * elem_size); + + } else { //for the condition src is not the type of dst + Class_Handle dst_elem_clss = class_get_array_element_class(dst_class); + if(WB_REM_OBJ_SNAPSHOT == write_barrier_function) { + write_barrier_rem_obj_snapshot(dst_array); + } + + for (unsigned int count = 0; count < length; count++) { + // 1, null elements copy direct + if (src_copy_body[count] == NULL) { + if(WB_REM_OLD_VAR == write_barrier_function) { + write_barrier_rem_slot_oldvar((Managed_Object_Handle *)dst_copy_body+count); + } + dst_copy_body[count] = NULL; + continue; + } + + // 2, For non-null elements check if types are compatible. +/* +#ifdef COMPRESS_REFERENCE + ManagedObject *src_elem = (ManagedObject *)uncompress_compressed_reference(src_elem_offset); + Class_Handle src_elem_clss = src_elem->vt()->clss; +#else +#endif +*/ + Class_Handle src_elem_clss = obj_get_gcvt(src_copy_body[count])->gc_clss; + + if (!class_is_instanceof(src_elem_clss, dst_elem_clss)) { + if(WB_REM_SOURCE_OBJ == write_barrier_function) { + write_barrier_rem_source_obj(dst_array); + } + return FALSE; + } + + if(WB_REM_OLD_VAR == write_barrier_function) { + write_barrier_rem_slot_oldvar((Managed_Object_Handle *)dst_copy_body+count); + } + dst_copy_body[count] = src_copy_body[count]; + } + } + + //rem obj after is for mostly concurrent + if(WB_REM_SOURCE_OBJ == write_barrier_function) { + write_barrier_rem_source_obj(dst_array); + } + + return TRUE; +} + + +void gc_heap_wrote_object (Managed_Object_Handle p_obj_written ) +{ + if( !gc_is_gen_mode() || !object_has_ref_field((Partial_Reveal_Object*)p_obj_written)) return; @@ -283,6 +395,13 @@ write_barrier_rem_slot_oldvar(p_slot); *p_slot = p_target; break; + //just debugging + /* + case WB_CON_DEBUG: + write_barrier_for_check(p_obj_holding_ref, p_slot, p_target); + //*p_slot = p_target; + break; + */ default: assert(0); return; Index: common/gc_for_barrier.h =================================================================== --- common/gc_for_barrier.h (revision 698467) +++ common/gc_for_barrier.h (working copy) @@ -32,7 +32,8 @@ WB_REM_SOURCE_REF = 0x02, WB_REM_OLD_VAR = 0x03, WB_REM_NEW_VAR = 0x04, - WB_REM_OBJ_SNAPSHOT = 0x05 + WB_REM_OBJ_SNAPSHOT = 0x05, + WB_CON_DEBUG = 0x06 }; inline void gc_set_barrier_function(unsigned int wb_function) @@ -42,5 +43,3 @@ #endif /* _GC_FOR_BARRIER_H_ */ - - Index: common/gc_for_class.cpp =================================================================== --- common/gc_for_class.cpp (revision 698467) +++ common/gc_for_class.cpp (working copy) @@ -202,5 +202,3 @@ - - Index: common/gc_for_vm.cpp =================================================================== --- common/gc_for_vm.cpp (revision 698467) +++ common/gc_for_vm.cpp (working copy) @@ -30,7 +30,7 @@ #include "../mark_sweep/gc_ms.h" #include "../move_compact/gc_mc.h" #include "interior_pointer.h" -#include "../thread/marker.h" +#include "../thread/conclctor.h" #include "../thread/collector.h" #include "../verify/verify_live_heap.h" #include "../finalizer_weakref/finalizer_weakref.h" @@ -115,7 +115,10 @@ collection_scheduler_initialize(gc); if(gc_is_specify_con_gc()){ - marker_initialize(gc); + gc->gc_concurrent_status = GC_CON_NIL; + conclctor_initialize(gc); + } else { + gc->gc_concurrent_status = GC_CON_DISABLE; } collector_initialize(gc); @@ -134,6 +137,9 @@ { INFO2("gc.process", "GC: call GC wrapup ...."); GC* gc = p_global_gc; + // destruct threads first, and then destruct data structures + conclctor_destruct(gc); + collector_destruct(gc); #if defined(USE_UNIQUE_MARK_SWEEP_GC) gc_ms_destruct((GC_MS*)gc); @@ -148,8 +154,6 @@ #ifndef BUILD_IN_REFERENT gc_finref_metadata_destruct(gc); #endif - collector_destruct(gc); - marker_destruct(gc); if( verify_live_heap ){ gc_terminate_heap_verification(gc); @@ -445,5 +449,3 @@ - - Index: common/gc_metadata.cpp =================================================================== --- common/gc_metadata.cpp (revision 698467) +++ common/gc_metadata.cpp (working copy) @@ -26,6 +26,7 @@ #include "compressed_ref.h" #include "../utils/sync_stack.h" #include "../gen/gen.h" +#include "../verify/verify_live_heap.h" #define GC_METADATA_SIZE_BYTES (1*MB) #define GC_METADATA_EXTEND_SIZE_BYTES (1*MB) @@ -74,6 +75,7 @@ } gc_metadata.gc_rootset_pool = sync_pool_create(); + //gc_metadata.gc_verifier_rootset_pool = sync_pool_create(); gc_metadata.gc_uncompressed_rootset_pool = sync_pool_create(); gc_metadata.mutator_remset_pool = sync_pool_create(); gc_metadata.collector_remset_pool = sync_pool_create(); @@ -552,6 +554,7 @@ Boolean obj_is_mark_black_in_table(Partial_Reveal_Object* p_obj); #endif +void analyze_bad_obj(Partial_Reveal_Object *p_obj); void gc_reset_dirty_set(GC* gc) { GC_Metadata* metadata = gc->metadata; @@ -559,18 +562,14 @@ Mutator *mutator = gc->mutator_list; while (mutator) { Vector_Block* local_dirty_set = mutator->dirty_set; - assert(local_dirty_set); - if(!vector_block_is_empty(local_dirty_set)){ -#ifdef _DEBUG + if(!vector_block_is_empty(local_dirty_set)) { POINTER_SIZE_INT* iter = vector_block_iterator_init(local_dirty_set); while(!vector_block_iterator_end(local_dirty_set,iter)){ Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*) *iter; iter = vector_block_iterator_advance(local_dirty_set, iter); -#ifdef USE_UNIQUE_MARK_SWEEP_GC - assert(obj_is_mark_black_in_table(p_obj)); -#endif + analyze_bad_obj(p_obj); } -#endif + RAISE_ERROR; vector_block_clear(mutator->dirty_set); } mutator = mutator->next; @@ -581,26 +580,27 @@ if(!pool_is_empty(global_dirty_set_pool)){ Vector_Block* dirty_set = pool_get_entry(global_dirty_set_pool); - while(dirty_set != NULL){ + while(dirty_set != NULL) { if(!vector_block_is_empty(dirty_set)){ -#ifdef _DEBUG - POINTER_SIZE_INT* iter = vector_block_iterator_init(dirty_set); - while(!vector_block_iterator_end(dirty_set,iter)){ - Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*) *iter; - iter = vector_block_iterator_advance(dirty_set, iter); -#ifdef USE_UNIQUE_MARK_SWEEP_GC - assert(obj_is_mark_black_in_table(p_obj)); -#endif - } -#endif + /* + POINTER_SIZE_INT* iter = vector_block_iterator_init(dirty_set); + while(!vector_block_iterator_end(dirty_set,iter)){ + Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*) *iter; + iter = vector_block_iterator_advance(dirty_set, iter); + analyze_bad_obj(p_obj); + }*/ + RAISE_ERROR; + vector_block_clear(dirty_set); + pool_put_entry(metadata->free_set_pool,dirty_set); + } else { + pool_put_entry(metadata->free_set_pool,dirty_set); } - vector_block_clear(dirty_set); - pool_put_entry(metadata->free_set_pool,dirty_set); - dirty_set = pool_get_entry(global_dirty_set_pool); - } + dirty_set = pool_get_entry(global_dirty_set_pool); } + } } + void gc_prepare_dirty_set(GC* gc) { GC_Metadata* metadata = gc->metadata; @@ -610,14 +610,59 @@ Mutator *mutator = gc->mutator_list; while (mutator) { + if( vector_block_is_empty(mutator->dirty_set) ) { + mutator = mutator->next; + continue; + } //FIXME: temproray solution for mostly concurrent. - lock(mutator->dirty_set_lock); + //lock(mutator->dirty_set_lock); pool_put_entry(gc_dirty_set_pool, mutator->dirty_set); mutator->dirty_set = free_set_pool_get_entry(metadata); + //unlock(mutator->dirty_set_lock); + mutator = mutator->next; + } + unlock(gc->mutator_list_lock); +} +void gc_copy_local_dirty_set_to_global(GC *gc) +{ + + GC_Metadata* metadata = gc->metadata; + if(!pool_is_empty(metadata->gc_dirty_set_pool)) //only when the global dirty is empty + return; + + Pool* gc_dirty_set_pool = metadata->gc_dirty_set_pool; + Vector_Block* dirty_copy = free_set_pool_get_entry(metadata); + unsigned int i = 0; + Vector_Block* local_dirty_set = NULL; + + lock(gc->mutator_list_lock); + Mutator *mutator = gc->mutator_list; + + while (mutator) { + lock(mutator->dirty_set_lock); + local_dirty_set = mutator->dirty_set; + if( vector_block_is_empty(local_dirty_set) ) { + unlock(mutator->dirty_set_lock); + mutator = mutator->next; + continue; + } + unsigned int dirty_set_size = vector_block_entry_count(local_dirty_set); + for(i=0; idirty_set_lock); mutator = mutator->next; } unlock(gc->mutator_list_lock); + if( !vector_block_is_empty(dirty_copy) ) + pool_put_entry(gc_dirty_set_pool, dirty_copy); + else + free_set_pool_put_entry(dirty_copy, metadata); } void gc_clear_dirty_set(GC* gc) @@ -636,7 +681,11 @@ } void free_set_pool_put_entry(Vector_Block* block, GC_Metadata *metadata) -{ pool_put_entry(metadata->free_set_pool, block); } +{ + if(!vector_block_is_empty(block)) + RAISE_ERROR + pool_put_entry(metadata->free_set_pool, block); +} void gc_reset_collectors_rem_set(GC *gc) @@ -654,5 +703,3 @@ } } - - Index: common/gc_metadata.h =================================================================== --- common/gc_metadata.h (revision 698467) +++ common/gc_metadata.h (working copy) @@ -67,6 +67,7 @@ void gc_clear_remset(GC* gc); void gc_prepare_dirty_set(GC* gc); +void gc_copy_local_dirty_set_to_global(GC *gc); void gc_reset_dirty_set(GC* gc); void gc_clear_dirty_set(GC* gc); @@ -118,14 +119,14 @@ while(!block) block = gc_metadata_extend(metadata->free_task_pool); - assert(vector_stack_is_empty(block)); + assert(vector_block_is_empty(block)); return block; } inline void mutator_remset_add_entry(Mutator* mutator, REF* p_ref) { assert( p_ref >= gc_heap_base_address() && p_ref < gc_heap_ceiling_address()); - + Vector_Block* root_set = mutator->rem_set; vector_block_add_entry(root_set, (POINTER_SIZE_INT)p_ref); @@ -139,17 +140,22 @@ inline void mutator_dirtyset_add_entry(Mutator* mutator, Partial_Reveal_Object* p_obj) { Vector_Block* dirty_set = mutator->dirty_set; + mutator->dirty_obj_slot_num++; vector_block_add_entry(dirty_set, (POINTER_SIZE_INT)p_obj); - if( !vector_block_is_full(dirty_set) ) return; - - vector_block_set_full(dirty_set); - - if(vector_block_set_exclusive(dirty_set)){ - pool_put_entry(gc_metadata.gc_dirty_set_pool, dirty_set); + if( !vector_block_is_full(dirty_set) ) { + return; } - + + lock(mutator->dirty_set_lock); + if( vector_block_is_empty(dirty_set) ) { + vector_block_clear(dirty_set); + unlock(mutator->dirty_set_lock); + return; + } + pool_put_entry(gc_metadata.gc_dirty_set_pool, dirty_set); mutator->dirty_set = free_set_pool_get_entry(&gc_metadata); + unlock(mutator->dirty_set_lock); } inline void collector_repset_add_entry(Collector* collector, Partial_Reveal_Object** p_ref) Index: common/gc_options.cpp =================================================================== --- common/gc_options.cpp (revision 698467) +++ common/gc_options.cpp (working copy) @@ -37,7 +37,10 @@ extern Boolean FORCE_FULL_COMPACT; -extern unsigned int NUM_MARKERS; +extern unsigned int NUM_CONCLCTORS; +extern unsigned int NUM_CON_MARKERS; +extern unsigned int NUM_CON_SWEEPERS; + extern unsigned int NUM_COLLECTORS; extern unsigned int MINOR_COLLECTORS; extern unsigned int MAJOR_COLLECTORS; @@ -59,12 +62,12 @@ GC* gc_mc_create(); GC* gc_ms_create(); -static GC* gc_decide_collection_algo(char* unique_algo, Boolean has_los) +static GC* gc_unique_decide_collection_algo(char* unique_algo, Boolean has_los) { /* if unique_algo is not set, gc_gen_decide_collection_algo is called. */ assert(unique_algo); - GC_PROP = ALGO_POOL_SHARE | ALGO_DEPTH_FIRST; + GC_PROP = ALGO_POOL_SHARE | ALGO_DEPTH_FIRST | ALGO_IS_UNIQUE; assert(!has_los); /* currently unique GCs don't use LOS */ if(has_los) @@ -84,7 +87,7 @@ GC_PROP |= ALGO_MS_NORMAL; gc = gc_ms_create(); }else{ - LWARN(48, "GC algorithm setting incorrect. Will use default value."); + LWARN(48, "\nGC algorithm setting incorrect. Will use default value.\n"); GC_PROP |= ALGO_COMPACT_MOVE; gc = gc_mc_create(); } @@ -159,7 +162,7 @@ if(minor_algo || major_algo){ LWARN(60, "Generational options cannot be set with unique_algo, ignored."); } - gc = gc_decide_collection_algo(unique_algo, has_los); + gc = gc_unique_decide_collection_algo(unique_algo, has_los); vm_properties_destroy_value(unique_algo); }else{ /* default */ gc = gc_gen_decide_collection_algo(minor_algo, major_algo, has_los); @@ -225,7 +228,7 @@ if (min_heap_size > max_heap_size){ max_heap_size = min_heap_size; - LWARN(61, "Max heap size you set is too small, reset to {0}MB" << max_heap_size/MB); + LWARN(61, "Max heap size is too small, reset to {0}MB" << max_heap_size/MB); } min_heap_size_bytes = min_heap_size; @@ -248,11 +251,25 @@ NUM_COLLECTORS = (num==0)? NUM_COLLECTORS:num; } - if (vm_property_is_set("gc.num_markers", VM_PROPERTIES) == 1) { - unsigned int num = vm_property_get_integer("gc.num_markers"); - NUM_MARKERS = (num==0)? NUM_MARKERS:num; + if (vm_property_is_set("gc.num_conclctors", VM_PROPERTIES) == 1) { + unsigned int num = vm_property_get_integer("gc.num_conclctors"); + NUM_CONCLCTORS = (num==0)? NUM_CONCLCTORS:num; } + // for concurrent GC debug + if (vm_property_is_set("gc.num_con_markers", VM_PROPERTIES) == 1) { + unsigned int num = vm_property_get_integer("gc.num_con_markers"); + NUM_CON_MARKERS = (num==0)? NUM_CON_MARKERS:num; + } + + if (vm_property_is_set("gc.num_con_sweepers", VM_PROPERTIES) == 1) { + unsigned int num = vm_property_get_integer("gc.num_con_sweepers"); + NUM_CON_SWEEPERS = (num==0)? NUM_CON_SWEEPERS:num; + } + + + + if (vm_property_is_set("gc.tospace_size", VM_PROPERTIES) == 1) { TOSPACE_SIZE = vm_property_get_size("gc.tospace_size"); } @@ -411,5 +428,3 @@ - - Index: common/gc_platform.h =================================================================== --- common/gc_platform.h (revision 698467) +++ common/gc_platform.h (working copy) @@ -66,7 +66,7 @@ #define prefetchnta(pref_addr) _mm_prefetch((char*)(pref_addr), _MM_HINT_NTA ) #endif /*ALLOC_PREFETCH*/ -#elif defined (__linux__) || defined(FREEBSD) +#elif defined (__linux__) #define FORCE_INLINE inline __attribute__((always_inline)) #ifdef PREFETCH_SUPPORTED @@ -154,6 +154,13 @@ return disable_count; } +inline int vm_suspend_all_threads( hythread_iterator_t *thread_iterator ) +{ + int disable_count = hythread_reset_suspend_disable(); + hythread_suspend_all(thread_iterator, NULL); + hythread_suspend_disable(); + return disable_count; +} inline void vm_resume_all_threads(int disable_count) { hythread_suspend_enable(); Index: common/gc_properties.h =================================================================== --- common/gc_properties.h (revision 698467) +++ common/gc_properties.h (working copy) @@ -54,7 +54,8 @@ enum GC_Property{ ALGO_HAS_NOS = 0x1, ALGO_HAS_LOS = 0x2, - ALGO_IS_GEN = 0x4, + ALGO_IS_UNIQUE = 0x4, + ALGO_IS_GEN = 0x8, ALGO_COPY_FORWARD = 0x10, ALGO_COPY_SEMISPACE = 0x20, @@ -107,6 +108,21 @@ GC_PROP &= ~ALGO_IS_GEN; } +FORCE_INLINE Boolean gc_is_unique_space() +{ + return gc_is_kind(ALGO_IS_UNIQUE); +} + +FORCE_INLINE Boolean gc_is_unique_move_compact() +{ + return gc_is_kind(ALGO_IS_UNIQUE) && gc_is_kind(ALGO_COMPACT_MOVE); +} + +FORCE_INLINE Boolean gc_is_unique_mark_sweep() +{ + return gc_is_kind(ALGO_IS_UNIQUE) && gc_is_kind(ALGO_MS_NORMAL); +} + FORCE_INLINE Boolean gc_is_gen_mode() { return gc_is_kind(ALGO_IS_GEN); Index: finalizer_weakref/finalizer_weakref.cpp =================================================================== --- finalizer_weakref/finalizer_weakref.cpp (revision 698467) +++ finalizer_weakref/finalizer_weakref.cpp (working copy) @@ -178,7 +178,7 @@ } else { assert(major_is_marksweep()); p_ref_or_obj = p_obj; - if(!gc_mark_is_concurrent()) + if( gc->gc_concurrent_status == GC_CON_NIL ) trace_object = trace_obj_in_ms_marking; else trace_object = trace_obj_in_ms_concurrent_mark; @@ -855,5 +855,3 @@ - - Index: finalizer_weakref/finalizer_weakref_metadata.cpp =================================================================== --- finalizer_weakref/finalizer_weakref_metadata.cpp (revision 698467) +++ finalizer_weakref/finalizer_weakref_metadata.cpp (working copy) @@ -22,6 +22,7 @@ #include "finalizer_weakref_metadata.h" #include "../thread/mutator.h" #include "../thread/collector.h" +#include "../thread/conclctor.h" #define FINREF_METADATA_SEG_SIZE_BIT_SHIFT 20 #define FINREF_METADATA_SEG_SIZE_BYTES (1 << FINREF_METADATA_SEG_SIZE_BIT_SHIFT) @@ -235,20 +236,6 @@ collector->weakref_set= NULL; collector->phanref_set= NULL; } - - if(gc_mark_is_concurrent() && !gc_sweep_is_concurrent()){ - unsigned int num_active_markers = gc->num_active_markers; - for(unsigned int i = 0; i < num_active_markers; i++) - { - Collector *marker = (Collector*)gc->markers[i]; - pool_put_entry(metadata->softref_pool, marker->softref_set); - pool_put_entry(metadata->weakref_pool, marker->weakref_set); - pool_put_entry(metadata->phanref_pool, marker->phanref_set); - marker->softref_set = NULL; - marker->weakref_set= NULL; - marker->phanref_set= NULL; - } - } return; } Index: gen/gen.h =================================================================== --- gen/gen.h (revision 698467) +++ gen/gen.h (working copy) @@ -76,9 +76,12 @@ unsigned int num_collectors; unsigned int num_active_collectors; /* not all collectors are working */ - Marker** markers; - unsigned int num_markers; + /*concurrent markers and collectors*/ + Conclctor** conclctors; + unsigned int num_conclctors; + //unsigned int num_active_conclctors; unsigned int num_active_markers; + unsigned int num_active_sweepers; /* metadata is the pool for rootset, markstack, etc. */ GC_Metadata* metadata; @@ -99,7 +102,7 @@ //For_LOS_extend Space_Tuner* tuner; - unsigned int gc_concurrent_status; + volatile unsigned int gc_concurrent_status; Collection_Scheduler* collection_scheduler; SpinLock lock_con_mark; @@ -218,5 +221,3 @@ - - Index: los/lspace_alloc_collect.cpp =================================================================== --- los/lspace_alloc_collect.cpp (revision 698467) +++ los/lspace_alloc_collect.cpp (working copy) @@ -543,5 +543,3 @@ - - Index: mark_sweep/gc_ms.cpp =================================================================== --- mark_sweep/gc_ms.cpp (revision 698467) +++ mark_sweep/gc_ms.cpp (working copy) @@ -18,10 +18,11 @@ #include "../common/gc_common.h" #include "gc_ms.h" +#include "../common/gc_concurrent.h" #include "wspace_mark_sweep.h" #include "../finalizer_weakref/finalizer_weakref.h" #include "../common/compressed_ref.h" -#include "../thread/marker.h" +#include "../thread/conclctor.h" #include "../verify/verify_live_heap.h" #ifdef USE_32BITS_HASHCODE #include "../common/hashcode.h" @@ -69,7 +70,7 @@ void gc_ms_reclaim_heap(GC_MS *gc) { - if(verify_live_heap) gc_verify_heap((GC*)gc, TRUE); + //if(verify_live_heap) gc_verify_heap((GC*)gc, TRUE); Wspace *wspace = gc_ms_get_wspace(gc); @@ -77,32 +78,48 @@ wspace_reset_after_collection(wspace); - if(verify_live_heap) gc_verify_heap((GC*)gc, FALSE); + //if(verify_live_heap) gc_verify_heap((GC*)gc, FALSE); } -void wspace_mark_scan_concurrent(Marker* marker); +void wspace_mark_scan_concurrent(Conclctor* marker); +void wspace_last_mc_marker_work(Conclctor *last_marker); +void wspace_last_otf_marker_work(Conclctor *last_marker); + void gc_ms_start_con_mark(GC_MS* gc, unsigned int num_markers) { if(gc->num_active_markers == 0) pool_iterator_init(gc->metadata->gc_rootset_pool); - - marker_execute_task_concurrent((GC*)gc,(TaskType)wspace_mark_scan_concurrent,(Space*)gc->wspace, num_markers); + + set_marker_final_func( (TaskType)wspace_last_otf_marker_work ); + conclctor_execute_task_concurrent((GC*)gc,(TaskType)wspace_mark_scan_concurrent,(Space*)gc->wspace, num_markers, CONCLCTOR_ROLE_MARKER); } -void wspace_mark_scan_mostly_concurrent(Marker* marker); +void wspace_mark_scan_mostly_concurrent(Conclctor* marker); +void wspace_last_mc_marker_work(Conclctor* marker); + void gc_ms_start_mostly_con_mark(GC_MS* gc, unsigned int num_markers) { if(gc->num_active_markers == 0) pool_iterator_init(gc->metadata->gc_rootset_pool); - marker_execute_task_concurrent((GC*)gc,(TaskType)wspace_mark_scan_mostly_concurrent,(Space*)gc->wspace, num_markers); + set_marker_final_func( (TaskType)wspace_last_mc_marker_work ); + conclctor_execute_task_concurrent((GC*)gc,(TaskType)wspace_mark_scan_mostly_concurrent,(Space*)gc->wspace, num_markers, CONCLCTOR_ROLE_MARKER); } + +void wspace_final_mark_scan_mostly_concurrent( Conclctor *marker ); +void conclctor_execute_task_synchronized(GC* gc, TaskType task_func, Space* space, unsigned int num_markers, unsigned int role); + void gc_ms_start_mostly_con_final_mark(GC_MS* gc, unsigned int num_markers) { pool_iterator_init(gc->metadata->gc_rootset_pool); - marker_execute_task((GC*)gc,(TaskType)wspace_mark_scan_mostly_concurrent,(Space*)gc->wspace); + conclctor_execute_task_synchronized( (GC*)gc,(TaskType)wspace_final_mark_scan_mostly_concurrent,(Space*)gc->wspace, num_markers, CONCLCTOR_ROLE_MARKER ); + + /* + collector_execute_task( (GC*)gc,(TaskType)wspace_mark_scan_mostly_concurrent,(Space*)gc->wspace ); + collector_set_weakref_sets( (GC*)gc ); + */ } /*FIXME: move this function out of this file.*/ @@ -119,32 +136,54 @@ unlock(gc->mutator_list_lock); } -void wspace_sweep_concurrent(Collector* collector); -void gc_ms_start_con_sweep(GC_MS* gc, unsigned int num_collectors) + +void wspace_sweep_concurrent(Conclctor* collector); +void wspace_last_sweeper_work(Conclctor *last_sweeper); +//void gc_con_print_stat_heap_utilization_rate(GC *gc); + void gc_ms_get_current_heap_usage(GC_MS *gc); + +void gc_ms_start_con_sweep(GC_MS* gc, unsigned int num_conclctors) { ops_color_flip(); mem_fence(); gc_check_mutator_allocation((GC*)gc); - gc_disable_alloc_obj_live(); + gc_disable_alloc_obj_live((GC*)gc); + //just debugging + //gc_con_print_stat_heap_utilization_rate((GC*)gc); + //INFO2("gc.scheduler", "=== Start Con Sweeping ==="); + Con_Collection_Statistics *con_collection_stat = gc_ms_get_con_collection_stat(gc); + con_collection_stat->sweeping_time = time_now(); + + gc_ms_get_current_heap_usage(gc); + gc_clear_conclctor_role((GC*)gc); wspace_init_pfc_pool_iterator(gc->wspace); - - collector_execute_task_concurrent((GC*)gc, (TaskType)wspace_sweep_concurrent, (Space*)gc->wspace, num_collectors); + set_sweeper_final_func( (TaskType)wspace_last_sweeper_work ); + conclctor_execute_task_concurrent((GC*)gc, (TaskType)wspace_sweep_concurrent, (Space*)gc->wspace, num_conclctors, CONCLCTOR_ROLE_SWEEPER); - collector_release_weakref_sets((GC*)gc, num_collectors); + //conclctor_release_weakref_sets((GC*)gc); } -void gc_ms_start_con_mark(GC_MS* gc) +unsigned int gc_ms_get_live_object_size(GC_MS* gc) { - pool_iterator_init(gc->metadata->gc_rootset_pool); + POINTER_SIZE_INT num_live_obj = 0; + POINTER_SIZE_INT size_live_obj = 0; - marker_execute_task_concurrent((GC*)gc,(TaskType)wspace_mark_scan_concurrent,(Space*)gc->wspace); + unsigned int num_collectors = gc->num_active_collectors; + Collector** collectors = gc->collectors; + unsigned int i; + for(i = 0; i < num_collectors; i++){ + Collector* collector = collectors[i]; + num_live_obj += collector->live_obj_num; + size_live_obj += collector->live_obj_size; + } + return size_live_obj; } + void gc_ms_update_space_stat(GC_MS* gc) { POINTER_SIZE_INT num_live_obj = 0; - POINTER_SIZE_INT size_live_obj = 0; - POINTER_SIZE_INT new_obj_size = 0; + POINTER_SIZE_INT size_live_obj = 0; Space_Statistics* wspace_stat = gc->wspace->space_statistic; @@ -157,22 +196,20 @@ size_live_obj += collector->live_obj_size; } - new_obj_size = gc_get_new_object_size((GC*)gc, TRUE); - - wspace_stat->size_new_obj = new_obj_size; - + wspace_stat->size_new_obj = gc_get_mutator_new_obj_size( (GC*)gc ); wspace_stat->num_live_obj = num_live_obj; wspace_stat->size_live_obj = size_live_obj; wspace_stat->last_size_free_space = wspace_stat->size_free_space; wspace_stat->size_free_space = gc->committed_heap_size - size_live_obj;/*TODO:inaccurate value.*/ wspace_stat->space_utilization_ratio = (float)wspace_stat->size_new_obj / wspace_stat->last_size_free_space; - + INFO2("gc.space.stat","[GC][Space Stat] num_live_obj : "<num_live_obj<<" "); INFO2("gc.space.stat","[GC][Space Stat] size_live_obj : "<size_live_obj<<" "); INFO2("gc.space.stat","[GC][Space Stat] size_free_space : "<size_free_space<<" "); INFO2("gc.space.stat","[GC][Space Stat] last_size_free_space: "<last_size_free_space<<" "); INFO2("gc.space.stat","[GC][Space Stat] size_new_obj : "<size_new_obj<<" "); INFO2("gc.space.stat","[GC][Space Stat] utilization_ratio : "<space_utilization_ratio<<" "); + } void gc_ms_reset_space_stat(GC_MS* gc) @@ -180,10 +217,12 @@ Space_Statistics* wspace_stat = gc->wspace->space_statistic; wspace_stat->size_new_obj = 0; wspace_stat->num_live_obj = 0; - wspace_stat->size_live_obj = 0; + wspace_stat->size_live_obj = 0; wspace_stat->space_utilization_ratio = 0; } void gc_ms_iterate_heap(GC_MS *gc) { } + + Index: mark_sweep/gc_ms.h =================================================================== --- mark_sweep/gc_ms.h (revision 698467) +++ mark_sweep/gc_ms.h (working copy) @@ -47,10 +47,12 @@ unsigned int num_collectors; unsigned int num_active_collectors; /* not all collectors are working */ - Marker** markers; - unsigned int num_markers; + /*concurrent markers and collectors*/ + Conclctor** conclctors; + unsigned int num_conclctors; unsigned int num_active_markers; - + unsigned int num_active_sweepers; + /* metadata is the pool for rootset, markstack, etc. */ GC_Metadata *metadata; Finref_Metadata *finref_metadata; @@ -70,7 +72,7 @@ //For_LOS_extend Space_Tuner *tuner; - unsigned int gc_concurrent_status; + volatile unsigned int gc_concurrent_status; Collection_Scheduler* collection_scheduler; SpinLock lock_con_mark; @@ -91,10 +93,22 @@ ////////////////////////////////////////////////////////////////////////////////////////// inline void *gc_ms_fast_alloc(unsigned size, Allocator *allocator) -{ return wspace_thread_local_alloc(size, allocator); } +{ + void *p_obj = wspace_thread_local_alloc(size, allocator); + if(p_obj) { + ((Mutator*)allocator)->new_obj_size += size; + ((Mutator*)allocator)->new_obj_num++; + } + return p_obj; + } inline void *gc_ms_alloc(unsigned size, Allocator *allocator) -{ return wspace_alloc(size, allocator); } +{ + void * p_obj = wspace_alloc(size, allocator); + if(p_obj) + ((Mutator*)allocator)->new_obj_num++; + return p_obj; +} inline Wspace *gc_ms_get_wspace(GC_MS *gc) { return gc->wspace; } @@ -108,6 +122,8 @@ inline POINTER_SIZE_INT gc_ms_total_memory_size(GC_MS *gc) { return space_committed_size((Space*)gc_ms_get_wspace(gc)); } +void gc_ms_print_detail_stat(GC_MS *gc); + ///////////////////////////////////////////////////////////////////////////////////////// void gc_ms_initialize(GC_MS *gc, POINTER_SIZE_INT initial_heap_size, POINTER_SIZE_INT final_heap_size); @@ -123,4 +139,11 @@ void gc_ms_start_mostly_con_final_mark(GC_MS* gc, unsigned int num_markers); void gc_ms_reset_space_stat(GC_MS* gc); +unsigned int gc_ms_get_live_object_size(GC_MS* gc); + +FORCE_INLINE Con_Collection_Statistics *gc_ms_get_con_collection_stat(GC_MS* gc) +{ + return gc->wspace->con_collection_statistics; +} + #endif // _GC_MS_H_ Index: mark_sweep/wspace.cpp =================================================================== --- mark_sweep/wspace.cpp (revision 698467) +++ mark_sweep/wspace.cpp (working copy) @@ -63,6 +63,10 @@ memset(wspace->space_statistic, 0, sizeof(Space_Statistics)); wspace->space_statistic->size_free_space = commit_size; + wspace->con_collection_statistics = (Con_Collection_Statistics*)STD_MALLOC(sizeof(Con_Collection_Statistics)); + memset(wspace->con_collection_statistics, 0, sizeof(Con_Collection_Statistics)); + wspace->con_collection_statistics->heap_utilization_rate = DEFAULT_HEAP_UTILIZATION_RATE; + #ifdef USE_UNIQUE_MARK_SWEEP_GC gc_ms_set_wspace((GC_MS*)gc, wspace); #else @@ -207,18 +211,7 @@ } } -#ifdef USE_UNIQUE_MARK_SWEEP_GC -void wspace_set_space_statistic(Wspace *wspace) -{ - GC_MS *gc = (GC_MS*)wspace->gc; - for(unsigned int i = 0; i < gc->num_collectors; ++i){ - wspace->surviving_obj_num += gc->collectors[i]->live_obj_num; - wspace->surviving_obj_size += gc->collectors[i]->live_obj_size; - } -} -#endif - extern void wspace_decide_compaction_need(Wspace *wspace); extern void mark_sweep_wspace(Collector *collector); Index: mark_sweep/wspace.h =================================================================== --- mark_sweep/wspace.h (revision 698467) +++ mark_sweep/wspace.h (working copy) @@ -24,12 +24,14 @@ #include "../common/gc_common.h" #include "../common/gc_concurrent.h" +#define DEFAULT_HEAP_UTILIZATION_RATE 0.92f /* * The sweep space accomodates objects collected by mark-sweep */ struct Size_Segment; struct Free_Chunk_List; +struct Con_Collection_Statistics; typedef struct Wspace { /* <-- first couple of fields are overloadded as Space */ @@ -71,14 +73,11 @@ Free_Chunk_List *aligned_free_chunk_lists; Free_Chunk_List *unaligned_free_chunk_lists; Free_Chunk_List *hyper_free_chunk_list; - POINTER_SIZE_INT surviving_obj_num; - POINTER_SIZE_INT surviving_obj_size; + + Con_Collection_Statistics *con_collection_statistics; + } Wspace; -#ifdef USE_UNIQUE_MARK_SWEEP_GC -void wspace_set_space_statistic(Wspace *wspace); -#endif - Wspace *wspace_initialize(GC *gc, void *start, POINTER_SIZE_INT wspace_size, POINTER_SIZE_INT commit_size); void wspace_destruct(Wspace *wspace); void wspace_reset_after_collection(Wspace *wspace); Index: mark_sweep/wspace_alloc.cpp =================================================================== --- mark_sweep/wspace_alloc.cpp (revision 698467) +++ mark_sweep/wspace_alloc.cpp (working copy) @@ -120,8 +120,9 @@ #ifdef SSPACE_VERIFY wspace_verify_alloc(p_obj, size); #endif - - if(p_obj) ((Mutator*)allocator)->new_obj_size += size; + if(p_obj) { + ((Mutator*)allocator)->new_obj_occupied_size+=size; + } return p_obj; } @@ -149,6 +150,7 @@ //if(!chunk) chunk = wspace_steal_pfc(wspace, seg_index, index); if(!chunk){ mutator_post_signal((Mutator*) allocator,HSIG_MUTATOR_SAFE); + //INFO2("gc.wspace", "[Local Alloc Failed] alloc obj with size" << size << " bytes" ); return NULL; } chunk->status |= CHUNK_IN_USE; @@ -184,6 +186,7 @@ //if(!chunk) chunk = wspace_steal_pfc(wspace, seg_index, index); if(!chunk) { mutator_post_signal((Mutator*) allocator,HSIG_MUTATOR_SAFE); + //INFO2("gc.wspace", "[Non-Local Alloc Failed] alloc obj with size" << size << " bytes" ); return NULL; } p_obj = alloc_in_chunk(chunk); @@ -201,7 +204,9 @@ mutator_post_signal((Mutator*) allocator,HSIG_MUTATOR_SAFE); } - + if(p_obj) { + ((Mutator*)allocator)->new_obj_occupied_size+=size; + } return p_obj; } @@ -224,7 +229,7 @@ mutator_post_signal((Mutator*) allocator,HSIG_MUTATOR_ENTER_ALLOC_MARK); if(is_obj_alloced_live()){ - chunk->table[0] |= cur_mark_black_color ; + chunk->table[0] |= cur_mark_black_color; // just for debugging, mark new object } mutator_post_signal((Mutator*) allocator,HSIG_MUTATOR_SAFE); @@ -234,6 +239,8 @@ chunk->status = CHUNK_ABNORMAL| CHUNK_USED; wspace_reg_used_chunk(wspace, chunk); assert(get_obj_info_raw((Partial_Reveal_Object*)chunk->base) & SUPER_OBJ_MASK); + + ((Mutator*)allocator)->new_obj_occupied_size+=chunk_size; return chunk->base; } @@ -257,14 +264,21 @@ #ifdef WSPACE_CONCURRENT_GC_STATS if(p_obj && gc_con_is_in_marking()) ((Partial_Reveal_Object*)p_obj)->obj_info |= NEW_OBJ_MASK; #endif - + + return p_obj; } +Free_Chunk_List *get_hyper_free_chunk_list(); + /* FIXME:: the collection should be seperated from the alloation */ void *wspace_alloc(unsigned size, Allocator *allocator) { void *p_obj = NULL; + /* + if( get_hyper_free_chunk_list()->head == NULL ) + INFO2("gc.wspace", "[BEFORE ALLOC]hyper free chunk is EMPTY!!"); + */ if(gc_is_specify_con_gc()) gc_sched_collection(allocator->gc, GC_CAUSE_CONCURRENT_GC); @@ -273,6 +287,10 @@ p_obj = wspace_try_alloc(size, allocator); if(p_obj){ ((Mutator*)allocator)->new_obj_size += size; + /* + if( get_hyper_free_chunk_list()->head == NULL ) + INFO2("gc.wspace", "[AFTER FIRST ALLOC]hyper free chunk is EMPTY!!"); + */ return p_obj; } @@ -284,9 +302,21 @@ if(p_obj){ vm_gc_unlock_enum(); ((Mutator*)allocator)->new_obj_size += size; + /* + if( get_hyper_free_chunk_list()->head == NULL ) + INFO2("gc.wspace", "[AFTER SECOND ALLOC]hyper free chunk is EMPTY!!"); + */ return p_obj; } - gc_reclaim_heap(allocator->gc, GC_CAUSE_MOS_IS_FULL); + + INFO2("gc.con.info", "[Exhausted Cause] Allocation size is :" << size << " bytes"); + GC *gc = allocator->gc; + /* + gc->cause = GC_CAUSE_MOS_IS_FULL; + if(gc_is_specify_con_gc()) + gc_relaim_heap_con_mode(gc); + else*/ + gc_reclaim_heap(gc, GC_CAUSE_MOS_IS_FULL); vm_gc_unlock_enum(); #ifdef SSPACE_CHUNK_INFO @@ -294,7 +324,10 @@ #endif p_obj = wspace_try_alloc(size, allocator); - + /* + if( get_hyper_free_chunk_list()->head == NULL ) + INFO2("gc.wspace", "[AFTER COLLECTION ALLOC]hyper free chunk is EMPTY!!"); + */ if(p_obj) ((Mutator*)allocator)->new_obj_size += size; return p_obj; Index: mark_sweep/wspace_alloc.h =================================================================== --- mark_sweep/wspace_alloc.h (revision 698467) +++ mark_sweep/wspace_alloc.h (working copy) @@ -184,8 +184,9 @@ void *p_obj = (void*)((POINTER_SIZE_INT)chunk->base + ((POINTER_SIZE_INT)chunk->slot_size * slot_index)); /*mark black is placed here because of race condition between ops color flip. */ - if(p_obj && is_obj_alloced_live()) + if(p_obj && is_obj_alloced_live()) { obj_mark_black_in_table((Partial_Reveal_Object*)p_obj, chunk->slot_size); + } alloc_slot_in_table(table, slot_index); if(chunk->status & CHUNK_NEED_ZEROING) Index: mark_sweep/wspace_chunk.cpp =================================================================== --- mark_sweep/wspace_chunk.cpp (revision 698467) +++ mark_sweep/wspace_chunk.cpp (working copy) @@ -29,6 +29,9 @@ static Free_Chunk_List unaligned_free_chunk_lists[NUM_UNALIGNED_FREE_CHUNK_BUCKET]; static Free_Chunk_List hyper_free_chunk_list; +Free_Chunk_List *get_hyper_free_chunk_list() { + return &hyper_free_chunk_list; +} static void init_size_segment(Size_Segment *seg, unsigned int size_min, unsigned int size_max, unsigned int gran_shift_bits, Boolean local_alloc) { @@ -313,15 +316,20 @@ { POINTER_SIZE_INT chunk_size = CHUNK_SIZE(chunk); assert(!(chunk_size % CHUNK_GRANULARITY)); - + + Free_Chunk_List *free_list = NULL; if(chunk_size > HYPER_OBJ_THRESHOLD){ - lock(wspace->hyper_free_chunk_list->lock); + free_list = wspace->hyper_free_chunk_list; + lock(free_list->lock); list_put_hyper_free_chunk_to_tail(wspace->hyper_free_chunk_list, chunk); - unlock(wspace->hyper_free_chunk_list->lock); - }else if(!((POINTER_SIZE_INT)chunk & NORMAL_CHUNK_LOW_MASK) && !(chunk_size & NORMAL_CHUNK_LOW_MASK)) - list_put_free_chunk_to_tail(&wspace->aligned_free_chunk_lists[ALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)], chunk); - else - list_put_free_chunk_to_tail(&wspace->unaligned_free_chunk_lists[UNALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)], chunk); + unlock(free_list->lock); + }else if(!((POINTER_SIZE_INT)chunk & NORMAL_CHUNK_LOW_MASK) && !(chunk_size & NORMAL_CHUNK_LOW_MASK)) { + free_list = &wspace->aligned_free_chunk_lists[ALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)]; + list_put_free_chunk_to_tail(free_list, chunk); + } else { + free_list = &wspace->unaligned_free_chunk_lists[UNALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)]; + list_put_free_chunk_to_tail(free_list, chunk); + } } @@ -410,9 +418,11 @@ assert(!chunk); /* search in the hyper free chunk list */ + chunk = wspace_get_hyper_free_chunk(wspace, NORMAL_CHUNK_SIZE_BYTES, TRUE); assert(!((POINTER_SIZE_INT)chunk & NORMAL_CHUNK_LOW_MASK)); - + /*if(chunk == NULL ) + INFO2("gc.wspace", "return from hyper free chunk list");*/ return chunk; } @@ -476,7 +486,12 @@ Free_Chunk *prev_chunk = NULL; Free_Chunk *chunk = list->head; + /* + if( chunk == NULL ) + INFO2("gc.wspace", "NO free hyper chunk now!!!" ); + */ while(chunk){ + if(CHUNK_SIZE(chunk) >= chunk_size){ Free_Chunk *next_chunk = chunk->next; if(prev_chunk) @@ -488,6 +503,8 @@ else list->tail = prev_chunk; break; + } else { + //INFO2("gc.wspace", "check chunk with SIZE "<next; @@ -954,5 +971,3 @@ #endif - - Index: mark_sweep/wspace_chunk.h =================================================================== --- mark_sweep/wspace_chunk.h (revision 698467) +++ mark_sweep/wspace_chunk.h (working copy) @@ -204,6 +204,35 @@ assert(list->lock == FREE_LOCK); } +inline void free_chunk_list_add_tail(Free_Chunk_List *list, Free_Chunk *chunk) +{ + chunk->next = NULL; + if(list->head) { + list->tail->next = chunk; + chunk->prev = list->tail; + list->tail = chunk; + } else { + chunk->prev = NULL; + list->head = list->tail = chunk; + } + list->chunk_num++; +} + +inline void free_chunk_list_add_head(Free_Chunk_List *list, Free_Chunk *chunk) +{ + chunk->prev = NULL; + if(list->head) { + list->head->prev = chunk; + chunk->next = list->head; + list->head = chunk; + } else { + chunk->next = NULL; + list->head = list->tail = chunk; + } + list->chunk_num++; +} + + inline void free_list_detach_chunk(Free_Chunk_List *list, Free_Chunk *chunk) { if(chunk->prev) @@ -218,6 +247,18 @@ --list->chunk_num; } +inline Boolean chunk_is_in_list(Free_Chunk_List *from_list, Free_Chunk *chunk) +{ + Free_Chunk *pro_chunk = from_list->head; + while(pro_chunk) { + if(pro_chunk == chunk) + return TRUE; + pro_chunk = pro_chunk->next; + } + return FALSE; +} + + inline void move_free_chunks_between_lists(Free_Chunk_List *to_list, Free_Chunk_List *from_list) { if(to_list->tail){ @@ -229,8 +270,10 @@ from_list->tail->next = to_list->head; to_list->head = from_list->head; } + //to_list->chunk_num += from_list->chunk_num; from_list->head = NULL; from_list->tail = NULL; + from_list->chunk_num = 0; } /* Padding the last index word in table to facilitate allocation */ @@ -402,7 +445,7 @@ Chunk_Header *chunk = (Chunk_Header*)pool_get_entry(pfc_pool); /*2. If in concurrent sweeping phase, search PFC backup pool*/ - if(!chunk && gc_con_is_in_sweeping()){ + if(!chunk && in_con_sweeping_phase(wspace->gc)){ pfc_pool = wspace->pfc_pools_backup[seg_index][index]; chunk = (Chunk_Header*)pool_get_entry(pfc_pool); } Index: mark_sweep/wspace_compact.cpp =================================================================== --- mark_sweep/wspace_compact.cpp (revision 698467) +++ mark_sweep/wspace_compact.cpp (working copy) @@ -33,7 +33,7 @@ float free_mem_ratio = (float)free_mem_size / wspace->committed_heap_size; #ifdef USE_UNIQUE_MARK_SWEEP_GC - if(!gc_mark_is_concurrent() && (free_mem_ratio > WSPACE_COMPACT_RATIO) && (wspace->gc->cause != GC_CAUSE_RUNTIME_FORCE_GC)){ + if( gc_con_is_in_STW(wspace->gc) && (free_mem_ratio > WSPACE_COMPACT_RATIO) && (wspace->gc->cause != GC_CAUSE_RUNTIME_FORCE_GC)){ #else if(collect_is_major()){ #endif @@ -279,5 +279,3 @@ - - Index: mark_sweep/wspace_mark_mostly_concurrent.cpp =================================================================== --- mark_sweep/wspace_mark_mostly_concurrent.cpp (revision 698467) +++ mark_sweep/wspace_mark_mostly_concurrent.cpp (working copy) @@ -17,9 +17,12 @@ #include "wspace_mark_sweep.h" #include "../finalizer_weakref/finalizer_weakref.h" -#include "../thread/marker.h" +#include "../thread/conclctor.h" +#include "gc_ms.h" volatile Boolean need_terminate_mostly_con_mark; +extern unsigned int mostly_con_final_marker_num; +extern unsigned int mostly_con_long_marker_num; Boolean obj_is_marked_in_table(Partial_Reveal_Object *obj); @@ -35,7 +38,7 @@ } } -static FORCE_INLINE void scan_object(Marker* marker, Partial_Reveal_Object *p_obj) +static FORCE_INLINE void scan_object(Conclctor* marker, Partial_Reveal_Object *p_obj) { assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); if(obj_is_dirty_in_table(p_obj)){ @@ -67,21 +70,22 @@ } #ifndef BUILD_IN_REFERENT - scan_weak_reference((Collector*)marker, p_obj, scan_slot); + //scan_weak_reference((Collector*)marker, p_obj, scan_slot); + scan_weak_reference_direct((Collector*)marker, p_obj, scan_slot); #endif } -static void trace_object(Marker* marker, Partial_Reveal_Object *p_obj) +static void trace_object(Conclctor* marker, Partial_Reveal_Object *p_obj) { scan_object(marker, p_obj); - obj_mark_black_in_table(p_obj); + obj_mark_black_in_table(p_obj, marker); Vector_Block *trace_stack = marker->trace_stack; while(!vector_stack_is_empty(trace_stack)){ p_obj = (Partial_Reveal_Object*)vector_stack_pop(trace_stack); scan_object(marker, p_obj); - obj_mark_black_in_table(p_obj); + obj_mark_black_in_table(p_obj, marker); trace_stack = marker->trace_stack; } } @@ -91,34 +95,31 @@ { need_terminate_mostly_con_mark = FALSE; } void terminate_mostly_con_mark() -{ need_terminate_mostly_con_mark = TRUE; } +{ need_terminate_mostly_con_mark = TRUE; } -static Boolean concurrent_mark_need_terminating(GC* gc) +static Boolean concurrent_mark_need_terminating_mc(GC* gc) { - if(need_terminate_mostly_con_mark) return TRUE; - + return need_terminate_mostly_con_mark; + /* GC_Metadata *metadata = gc->metadata; return pool_is_empty(metadata->gc_dirty_set_pool); + */ } static volatile unsigned int num_active_markers = 0; - -void wspace_mark_scan_mostly_concurrent(Marker* marker) +static SpinLock info_lock; +void wspace_mark_scan_mostly_concurrent(Conclctor* marker) { - int64 time_mark_start = time_now(); GC *gc = marker->gc; GC_Metadata *metadata = gc->metadata; - /* reset the num_finished_collectors to be 0 by one collector. This is necessary for the barrier later. */ - unsigned int current_thread_id = atomic_inc32(&num_active_markers); + unsigned int num_dirtyset_slot = 0; - unsigned int num_dirtyset_slot = 0; - marker->trace_stack = free_task_pool_get_entry(metadata); + /* first step: copy all root objects to mark tasks.*/ Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool); - - /* first step: copy all root objects to mark tasks.*/ + while(root_set){ POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); while(!vector_block_iterator_end(root_set,iter)){ @@ -132,15 +133,28 @@ } root_set = pool_iterator_next(metadata->gc_rootset_pool); } + /* put back the last trace_stack task */ pool_put_entry(metadata->mark_task_pool, marker->trace_stack); - marker->trace_stack = free_task_pool_get_entry(metadata); + /* following code has such concerns: + 1, current_thread_id should be unique + 2, mostly concurrent do not need adding new marker dynamically + 3, when the heap is exhausted, final marking will enumeration rootset, it should be after above actions + */ + unsigned int current_thread_id = atomic_inc32(&num_active_markers); + + if((current_thread_id+1) == gc->num_active_markers ) + state_transformation( gc, GC_CON_START_MARKERS, GC_CON_TRACING); + + while( gc->gc_concurrent_status == GC_CON_START_MARKERS ); + retry: + + /*second step: mark dirty pool*/ - Vector_Block* dirty_set = pool_get_entry(metadata->gc_dirty_set_pool); while(dirty_set){ @@ -152,7 +166,7 @@ assert(p_obj!=NULL); //FIXME: restrict condition? obj_clear_dirty_in_table(p_obj); - obj_clear_mark_in_table(p_obj); + obj_clear_mark_in_table(p_obj, marker); if(obj_mark_gray_in_table(p_obj)) collector_tracestack_push((Collector*)marker, p_obj); @@ -186,41 +200,186 @@ mark_task = pool_get_entry(metadata->mark_task_pool); } + /* if(current_thread_id == 0){ gc_prepare_dirty_set(marker->gc); - } + }*/ + gc_copy_local_dirty_set_to_global(gc); + /* conditions to terminate mark: 1.All thread finished current job. 2.Flag is set to terminate concurrent mark. */ atomic_dec32(&num_active_markers); - while(num_active_markers != 0 || !concurrent_mark_need_terminating(gc)){ + while(num_active_markers != 0 || !concurrent_mark_need_terminating_mc(gc) ) { + if(!pool_is_empty(metadata->mark_task_pool) || !pool_is_empty(metadata->gc_dirty_set_pool)) { + atomic_inc32(&num_active_markers); + goto retry; + } else if( current_thread_id >= mostly_con_long_marker_num ) { + break; + } + apr_sleep(15000); + } + + /* + while(num_active_markers != 0 || !concurrent_mark_need_terminating_mc(gc)){ if(!pool_is_empty(metadata->mark_task_pool) || !pool_is_empty(metadata->gc_dirty_set_pool)){ atomic_inc32(&num_active_markers); goto retry; } + }*/ + + /* put back the last mark stack to the free pool */ + mark_task = (Vector_Block*)marker->trace_stack; + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + marker->trace_stack = NULL; + marker->num_dirty_slots_traced = num_dirtyset_slot; + + /* + if(num_dirtyset_slot!=0) { + lock(info_lock); + INFO2("gc.marker", "marker ["<< current_thread_id <<"] processed dirty slot="<gc; + GC_Metadata *metadata = gc->metadata; + + unsigned int num_dirtyset_slot = 0; + + marker->trace_stack = free_task_pool_get_entry(metadata); + Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool); + + /* first step: copy all root objects to mark tasks.*/ + while(root_set){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); + while(!vector_block_iterator_end(root_set,iter)){ + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)*iter; + iter = vector_block_iterator_advance(root_set,iter); + + assert(p_obj!=NULL); + assert(address_belongs_to_gc_heap(p_obj, gc)); + if(obj_mark_gray_in_table(p_obj)) + collector_tracestack_push((Collector*)marker, p_obj); + } + root_set = pool_iterator_next(metadata->gc_rootset_pool); } + /* put back the last trace_stack task */ + pool_put_entry(metadata->mark_task_pool, marker->trace_stack); + marker->trace_stack = free_task_pool_get_entry(metadata); + + + /*second step: mark dirty pool*/ + Vector_Block* dirty_set = pool_get_entry(metadata->gc_dirty_set_pool); + + while(dirty_set){ + POINTER_SIZE_INT* iter = vector_block_iterator_init(dirty_set); + while(!vector_block_iterator_end(dirty_set,iter)){ + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)*iter; + iter = vector_block_iterator_advance(dirty_set,iter); + + assert(p_obj!=NULL); //FIXME: restrict condition? + + obj_clear_dirty_in_table(p_obj); + obj_clear_mark_in_table(p_obj, marker); + + if(obj_mark_gray_in_table(p_obj)) + collector_tracestack_push((Collector*)marker, p_obj); + + num_dirtyset_slot ++; + } + vector_block_clear(dirty_set); + pool_put_entry(metadata->free_set_pool, dirty_set); + dirty_set = pool_get_entry(metadata->gc_dirty_set_pool); + } + /* put back the last trace_stack task */ + pool_put_entry(metadata->mark_task_pool, marker->trace_stack); + + /* third step: iterate over the mark tasks and scan objects */ + marker->trace_stack = free_task_pool_get_entry(metadata); + + Vector_Block *mark_task = pool_get_entry(metadata->mark_task_pool); + while(mark_task){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task); + while(!vector_block_iterator_end(mark_task,iter)){ + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)*iter; + iter = vector_block_iterator_advance(mark_task,iter); + trace_object(marker, p_obj); + } + /* run out one task, put back to the pool and grab another task */ + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + mark_task = pool_get_entry(metadata->mark_task_pool); + } + /* put back the last mark stack to the free pool */ mark_task = (Vector_Block*)marker->trace_stack; vector_stack_clear(mark_task); pool_put_entry(metadata->free_task_pool, mark_task); marker->trace_stack = NULL; + + //marker->time_mark += time_mark; + marker->num_dirty_slots_traced = num_dirtyset_slot; + //INFO2("gc.marker", "[final marker] processed dirty slot="<time_mark += time_mark; - marker->num_dirty_slots_traced += num_dirtyset_slot; return; } + + + +void wspace_last_mc_marker_work( Conclctor *last_marker ) { + + GC *gc = last_marker->gc; + if( gc->gc_concurrent_status != GC_CON_TRACING ) + return; + + gc_con_update_stat_after_marking(gc); //calculate marked size + //just debugging + Con_Collection_Statistics *con_collection_stat = gc_ms_get_con_collection_stat((GC_MS*)gc); + con_collection_stat->marking_end_time = time_now(); + int64 con_marking_time = con_collection_stat->marking_end_time - con_collection_stat->marking_start_time; + INFO2("gc.scheduler", "[MOSTLY_CON] con marking time=" << con_marking_time << " us"); + + state_transformation( gc, GC_CON_TRACING, GC_CON_TRACE_DONE ); + //INFO2("gc.con.info", " first marking thread finished its job, GC is waiting for all the marking threads finish, current marker num is [" << gc->num_active_markers << "]" ); +} + +void gc_mostly_con_update_stat_after_final_marking(GC *gc); +void wspace_mostly_con_final_mark( GC *gc ) { + + /*init the root set pool*/ + pool_iterator_init(gc->metadata->gc_rootset_pool); + /*prepare dirty object*/ + gc_prepare_dirty_set(gc); + /*new asssign thread may reuse the one just finished in the same phase*/ + conclctor_set_weakref_sets(gc); + + /*start final mostly concurrent mark */ + gc_ms_start_mostly_con_final_mark((GC_MS*)gc, mostly_con_final_marker_num); + + mostly_con_mark_terminate_reset(); + gc_mostly_con_update_stat_after_final_marking(gc); + + gc_reset_dirty_set(gc); + gc_clear_rootset(gc); + gc_prepare_sweeping(gc); + state_transformation( gc, GC_CON_TRACE_DONE, GC_CON_BEFORE_SWEEP ); +} + void trace_obj_in_ms_mostly_concurrent_mark(Collector *collector, void *p_obj) { obj_mark_gray_in_table((Partial_Reveal_Object*)p_obj); - trace_object((Marker*)collector, (Partial_Reveal_Object *)p_obj); + trace_object((Conclctor*)collector, (Partial_Reveal_Object *)p_obj); } - - - Index: mark_sweep/wspace_mark_otf_concurrent.cpp =================================================================== --- mark_sweep/wspace_mark_otf_concurrent.cpp (revision 698467) +++ mark_sweep/wspace_mark_otf_concurrent.cpp (working copy) @@ -16,7 +16,11 @@ */ #include "wspace_mark_sweep.h" #include "../finalizer_weakref/finalizer_weakref.h" -#include "../thread/marker.h" +#include "../thread/conclctor.h" +#include "gc_ms.h" +struct GC_MS; +struct Wspace; +struct Space_Statistics; Boolean obj_is_marked_in_table(Partial_Reveal_Object *obj); @@ -32,7 +36,7 @@ } } -static FORCE_INLINE void scan_object(Marker* marker, Partial_Reveal_Object *p_obj) +static FORCE_INLINE void scan_object(Conclctor* marker, Partial_Reveal_Object *p_obj) { assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); @@ -71,42 +75,59 @@ } -static void trace_object(Marker* marker, Partial_Reveal_Object *p_obj) +static void trace_object(Conclctor* marker, Partial_Reveal_Object *p_obj) { scan_object(marker, p_obj); - obj_mark_black_in_table(p_obj); + //obj_mark_black_in_table(p_obj); + obj_mark_black_in_table(p_obj, marker); Vector_Block *trace_stack = marker->trace_stack; while(!vector_stack_is_empty(trace_stack)){ p_obj = (Partial_Reveal_Object*)vector_stack_pop(trace_stack); scan_object(marker, p_obj); - obj_mark_black_in_table(p_obj); + //obj_mark_black_in_table(p_obj); + obj_mark_black_in_table(p_obj, marker); trace_stack = marker->trace_stack; } } -static Boolean concurrent_mark_need_terminating(GC* gc) +static Boolean dirty_set_is_empty(GC *gc) { + lock(gc->mutator_list_lock); + Mutator *mutator = gc->mutator_list; + while (mutator) { + Vector_Block* local_dirty_set = mutator->dirty_set; + if(!vector_block_is_empty(local_dirty_set)){ + unlock(gc->mutator_list_lock); + return FALSE; + } + mutator = mutator->next; + } GC_Metadata *metadata = gc->metadata; - return gc_local_dirtyset_is_empty(gc) && pool_is_empty(metadata->gc_dirty_set_pool); + Boolean is_empty = pool_is_empty(metadata->gc_dirty_set_pool); + unlock(gc->mutator_list_lock); //unlock put here to prevent creating new mutators before checking global dirty set + return is_empty; } +static Boolean concurrent_mark_need_terminating_otf(GC* gc) +{ + return dirty_set_is_empty(gc); +} /* for marking phase termination detection */ static volatile unsigned int num_active_markers = 0; +//static volatile unsigned int root_set_obj_size = 0; -void wspace_mark_scan_concurrent(Marker* marker) +void wspace_mark_scan_concurrent(Conclctor* marker) { - marker->time_measurement_start = time_now(); + //marker->time_measurement_start = time_now(); GC *gc = marker->gc; GC_Metadata *metadata = gc->metadata; /* reset the num_finished_collectors to be 0 by one collector. This is necessary for the barrier later. */ - atomic_inc32(&num_active_markers); - + unsigned int current_thread_id = atomic_inc32(&num_active_markers); marker->trace_stack = free_task_pool_get_entry(metadata); - Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool); - + /* first step: copy all root objects to mark tasks.*/ while(root_set){ POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); @@ -116,6 +137,7 @@ assert(p_obj!=NULL); assert(address_belongs_to_gc_heap(p_obj, gc)); + //if(obj_mark_gray_in_table(p_obj, &root_set_obj_size)) if(obj_mark_gray_in_table(p_obj)) collector_tracestack_push((Collector*)marker, p_obj); } @@ -126,8 +148,10 @@ marker->trace_stack = free_task_pool_get_entry(metadata); + state_transformation( gc, GC_CON_START_MARKERS, GC_CON_TRACING); retry: + gc_copy_local_dirty_set_to_global(marker->gc); /*second step: mark dirty object snapshot pool*/ Vector_Block* dirty_set = pool_get_entry(metadata->gc_dirty_set_pool); @@ -137,7 +161,10 @@ Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)*iter; iter = vector_block_iterator_advance(dirty_set,iter); - assert(p_obj!=NULL); //FIXME: restrict? + if(p_obj==NULL) { //FIXME: restrict? + RAISE_ERROR; + } + marker->num_dirty_slots_traced++; if(obj_mark_gray_in_table(p_obj)) collector_tracestack_push((Collector*)marker, p_obj); } @@ -175,39 +202,15 @@ 3.global snapshot pool is empty. */ atomic_dec32(&num_active_markers); - while(num_active_markers != 0 || !concurrent_mark_need_terminating(gc)){ - if(!pool_is_empty(metadata->mark_task_pool) || !pool_is_empty(metadata->gc_dirty_set_pool)){ - atomic_inc32(&num_active_markers); - goto retry; - }else{ - /*grab a block from mutator and begin tracing*/ - POINTER_SIZE_INT thread_num = (POINTER_SIZE_INT)marker->thread_handle; - Vector_Block* local_dirty_set = gc_get_local_dirty_set(gc, (unsigned int)(thread_num + 1)); - /*1. If local_dirty_set has been set full bit, the block is full and will no longer be put into global snapshot pool; - so it should be checked again to see if there're remaining entries unscanned in it. In this case, the - share bit in local_dirty_set should not be cleared, beacause of rescanning exclusively. - 2. If local_dirty_set has not been set full bit, the block is used by mutator and has the chance to be put into - global snapshot pool. In this case, we simply clear the share bit in local_dirty_set. - */ - if(local_dirty_set != NULL){ - atomic_inc32(&num_active_markers); - do{ - while(!vector_block_is_empty(local_dirty_set)){ //|| !vector_block_not_full_set_unshared(local_dirty_set)){ - Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*) vector_block_get_entry(local_dirty_set); - if(!obj_belongs_to_gc_heap(p_obj)) { - assert(0); - } - - if(obj_mark_gray_in_table(p_obj)){ - collector_tracestack_push((Collector*)marker, p_obj); - } - } - }while(!vector_block_not_full_set_unshared(local_dirty_set) && !vector_block_is_empty(local_dirty_set)); - goto retry; - } + while(num_active_markers != 0 || !concurrent_mark_need_terminating_otf(gc)){ + if(!pool_is_empty(metadata->mark_task_pool) || !concurrent_mark_need_terminating_otf(gc)){ + atomic_inc32(&num_active_markers); + goto retry; } + apr_sleep(15000); } - + + state_transformation( gc, GC_CON_TRACING, GC_CON_TRACE_DONE ); /* put back the last mark stack to the free pool */ mark_task = (Vector_Block*)marker->trace_stack; vector_stack_clear(mark_task); @@ -215,19 +218,33 @@ marker->trace_stack = NULL; assert(pool_is_empty(metadata->gc_dirty_set_pool)); - marker->time_measurement_end = time_now(); - marker->time_mark = marker->time_measurement_end - marker->time_measurement_start; - + //INFO2("gc.con.info", "first marker finishes its job"); + return; } +void wspace_last_otf_marker_work( Conclctor *last_marker ) { + GC *gc = last_marker->gc; + + gc_reset_dirty_set(gc); + gc_set_barrier_function(WB_REM_NIL); + + //INFO2("gc.con.info", "all markers finish "); + gc_con_update_stat_after_marking(gc); //calculate marked size + + gc_clear_rootset(gc); + + gc_prepare_sweeping(gc); + state_transformation( gc, GC_CON_TRACE_DONE, GC_CON_BEFORE_SWEEP ); +} + + void trace_obj_in_ms_concurrent_mark(Collector *collector, void *p_obj) { obj_mark_gray_in_table((Partial_Reveal_Object*)p_obj); - trace_object((Marker*)collector, (Partial_Reveal_Object *)p_obj); + trace_object((Conclctor*)collector, (Partial_Reveal_Object *)p_obj); } - Index: mark_sweep/wspace_mark_sweep.cpp =================================================================== --- mark_sweep/wspace_mark_sweep.cpp (revision 698467) +++ mark_sweep/wspace_mark_sweep.cpp (working copy) @@ -21,6 +21,7 @@ #include "gc_ms.h" #include "../gen/gen.h" #include "../thread/collector.h" +#include "../thread/conclctor.h" #include "../finalizer_weakref/finalizer_weakref.h" #include "../common/fix_repointed_refs.h" #include "../common/gc_concurrent.h" @@ -269,13 +270,14 @@ Mark all live objects in heap ****************************/ atomic_cas32(&num_marking_collectors, 0, num_active_collectors+1); - if(!gc_mark_is_concurrent()){ + //if mark has been done in a concurrent manner, skip this mark + if( gc_con_is_in_STW(gc) ) { if(collect_is_fallback()) wspace_fallback_mark_scan(collector, wspace); else wspace_mark_scan(collector, wspace); } - + unsigned int old_num = atomic_inc32(&num_marking_collectors); if( ++old_num == num_active_collectors ){ /* last collector's world here */ @@ -292,8 +294,9 @@ #endif gc_identify_dead_weak_roots(gc); gc_init_chunk_for_sweep(gc, wspace); - /* let other collectors go */ - num_marking_collectors++; + + /* let other collectors go */ + num_marking_collectors++; } while(num_marking_collectors != num_active_collectors + 1); @@ -302,8 +305,8 @@ atomic_cas32( &num_sweeping_collectors, 0, num_active_collectors+1); wspace_sweep(collector, wspace); - old_num = atomic_inc32(&num_sweeping_collectors); + //INFO2("gc.con.scheduler", "[SWEEPER NUM] num_sweeping_collectors = " << num_sweeping_collectors); if( ++old_num == num_active_collectors ){ #ifdef SSPACE_TIME wspace_sweep_time(FALSE, wspace->need_compact); @@ -392,10 +395,6 @@ if(!collect_is_major()) wspace_merge_free_chunks(gc, wspace); -#ifdef USE_UNIQUE_MARK_SWEEP_GC - wspace_set_space_statistic(wspace); -#endif - #ifdef SSPACE_VERIFY wspace_verify_after_collection(gc); #endif Index: mark_sweep/wspace_mark_sweep.h =================================================================== --- mark_sweep/wspace_mark_sweep.h (revision 698467) +++ mark_sweep/wspace_mark_sweep.h (working copy) @@ -20,6 +20,7 @@ #include "wspace_chunk.h" #include "wspace_verify.h" +#include "../thread/conclctor.h" #define PFC_REUSABLE_RATIO 0.1 #define WSPACE_COMPACT_RATIO 0.06 @@ -27,6 +28,8 @@ inline Boolean chunk_is_reusable(Chunk_Header *chunk) { return (float)(chunk->slot_num-chunk->alloc_num)/chunk->slot_num > PFC_REUSABLE_RATIO; } +struct Conclctor; + #define OBJ_ALLOC_BIT_IN_TABLE 0x01 #define OBJ_BLACK_BIT_IN_TABLE 0x02 #define OBJ_GRAY_BIT_IN_TABLE 0x04 @@ -172,7 +175,53 @@ } +//just debugging for root set size +FORCE_INLINE Boolean obj_mark_gray_in_table(Partial_Reveal_Object *obj, volatile unsigned int *slot_size) +{ + volatile POINTER_SIZE_INT *p_color_word; + Chunk_Header *chunk; + unsigned int slot_index; + + if(is_super_obj(obj)){ + chunk = ABNORMAL_CHUNK_HEADER(obj); + slot_index = 0; + } else { + chunk = NORMAL_CHUNK_HEADER(obj); + slot_index = slot_addr_to_index(chunk, obj); + } + + unsigned int word_index = slot_index >> SLOT_NUM_PER_WORD_SHIT; + unsigned int index_in_word = COLOR_BITS_PER_OBJ * (slot_index & (((unsigned int)(SLOT_NUM_PER_WORD_IN_TABLE-1)))); + p_color_word = &chunk->table[word_index]; + + assert(p_color_word); + + //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); + POINTER_SIZE_INT mark_color = cur_mark_gray_color << index_in_word; + + POINTER_SIZE_INT old_word = *p_color_word; + if(old_word & mark_color) return FALSE; /*already marked gray*/ + apr_atomic_add32(slot_size, chunk->slot_size); + + //POINTER_SIZE_INT new_word = (old_word & color_bits_mask) | mark_color; + POINTER_SIZE_INT new_word = old_word | mark_color; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return TRUE; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + if(old_word & mark_color) return FALSE; /*already marked gray*/ + + //new_word = (old_word & color_bits_mask) | mark_color; + new_word = old_word | mark_color; + } + + return FALSE; +} + + FORCE_INLINE Boolean obj_mark_gray_in_table(Partial_Reveal_Object *obj) { volatile POINTER_SIZE_INT *p_color_word; @@ -184,7 +233,7 @@ POINTER_SIZE_INT mark_color = cur_mark_gray_color << index_in_word; POINTER_SIZE_INT old_word = *p_color_word; - if(old_word & mark_color) return FALSE; /*already marked gray or black.*/ + if(old_word & mark_color) return FALSE; /*already marked gray*/ //POINTER_SIZE_INT new_word = (old_word & color_bits_mask) | mark_color; POINTER_SIZE_INT new_word = old_word | mark_color; @@ -194,7 +243,7 @@ return TRUE; /*returning true does not mean it's marked by this thread. */ } old_word = *p_color_word; - if(old_word & mark_color) return FALSE; /*already marked gray or black.*/ + if(old_word & mark_color) return FALSE; /*already marked gray*/ //new_word = (old_word & color_bits_mask) | mark_color; new_word = old_word | mark_color; @@ -203,6 +252,37 @@ return FALSE; } +FORCE_INLINE Boolean obj_mark_black_in_table(Partial_Reveal_Object *obj) +{ + //assert(obj_is_mark_in_table(obj)); + volatile POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + assert(p_color_word); + + //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); + POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; + + POINTER_SIZE_INT old_word = *p_color_word; + if(old_word & mark_black_color) return FALSE; /*already marked black*/ + + POINTER_SIZE_INT new_word = old_word | mark_black_color; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return TRUE; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + if(old_word & mark_black_color) return FALSE; /*already marked black*/ + + new_word = old_word | mark_black_color; + } + + return FALSE; + +} + + FORCE_INLINE Boolean obj_mark_black_in_table(Partial_Reveal_Object *obj, unsigned int size) { //assert(obj_is_mark_in_table(obj)); @@ -212,7 +292,7 @@ assert(p_color_word); //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); - POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; + POINTER_SIZE_INT mark_black_color = (OBJ_DIRTY_BIT_IN_TABLE|cur_mark_black_color) << index_in_word; //just debugging, to mark new object POINTER_SIZE_INT old_word = *p_color_word; if(old_word & mark_black_color) return FALSE; /*already marked black*/ @@ -233,12 +313,28 @@ } -FORCE_INLINE Boolean obj_mark_black_in_table(Partial_Reveal_Object *obj) +FORCE_INLINE Boolean obj_mark_black_in_table(Partial_Reveal_Object *obj, Conclctor* marker) { // assert(obj_is_mark_in_table(obj)); volatile POINTER_SIZE_INT *p_color_word; - unsigned int index_in_word; - p_color_word = get_color_word_in_table(obj, index_in_word); + Chunk_Header *chunk; + unsigned int slot_index; + unsigned int obj_ocuppied_size = 0; + + if(is_super_obj(obj)){ + chunk = ABNORMAL_CHUNK_HEADER(obj); + slot_index = 0; + obj_ocuppied_size = CHUNK_SIZE(chunk); + } else { + chunk = NORMAL_CHUNK_HEADER(obj); + slot_index = slot_addr_to_index(chunk, obj); + obj_ocuppied_size = chunk->slot_size; + } + + unsigned int word_index = slot_index >> SLOT_NUM_PER_WORD_SHIT; + unsigned int index_in_word = COLOR_BITS_PER_OBJ * (slot_index & (((unsigned int)(SLOT_NUM_PER_WORD_IN_TABLE-1)))); + p_color_word = &chunk->table[word_index]; + assert(p_color_word); //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); @@ -246,6 +342,9 @@ POINTER_SIZE_INT old_word = *p_color_word; if(obj_is_mark_black_in_table(obj)) return FALSE; /*already marked black*/ + + marker->live_obj_num++; + marker->live_obj_size+=obj_ocuppied_size; POINTER_SIZE_INT new_word = old_word | mark_black_color; while(new_word != old_word) { @@ -262,6 +361,55 @@ return FALSE; } +static volatile unsigned int mutator_marked = 0; + +FORCE_INLINE Boolean obj_mark_black_in_table(Partial_Reveal_Object *obj, Mutator *mutator) +{ + // assert(obj_is_mark_in_table(obj)); + volatile POINTER_SIZE_INT *p_color_word; + Chunk_Header *chunk; + unsigned int slot_index; + unsigned int obj_size = 0; + + if(is_super_obj(obj)){ + chunk = ABNORMAL_CHUNK_HEADER(obj); + slot_index = 0; + obj_size = CHUNK_SIZE(chunk); + } else { + chunk = NORMAL_CHUNK_HEADER(obj); + slot_index = slot_addr_to_index(chunk, obj); + obj_size = chunk->slot_size; + } + + unsigned int word_index = slot_index >> SLOT_NUM_PER_WORD_SHIT; + unsigned int index_in_word = COLOR_BITS_PER_OBJ * (slot_index & (((unsigned int)(SLOT_NUM_PER_WORD_IN_TABLE-1)))); + p_color_word = &chunk->table[word_index]; + assert(p_color_word); + + //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); + POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; + + POINTER_SIZE_INT old_word = *p_color_word; + if(obj_is_mark_black_in_table(obj)) return FALSE; /*already marked black*/ + + //mutator->new_obj_size += vm_object_size(obj); + mutator->write_barrier_marked_size += obj_size; + + POINTER_SIZE_INT new_word = old_word | mark_black_color; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return TRUE; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + if(obj_is_mark_black_in_table(obj)) return FALSE; /*already marked black*/ + + new_word = old_word | mark_black_color; + } + + return FALSE; +} + FORCE_INLINE Boolean obj_dirty_in_table(Partial_Reveal_Object *obj) { volatile POINTER_SIZE_INT *p_color_word; @@ -304,12 +452,29 @@ return FALSE; } -FORCE_INLINE Boolean obj_clear_mark_in_table(Partial_Reveal_Object *obj) +FORCE_INLINE Boolean obj_clear_mark_in_table(Partial_Reveal_Object *obj, Conclctor *marker) { - volatile POINTER_SIZE_INT *p_color_word; - unsigned int index_in_word; - p_color_word = get_color_word_in_table(obj, index_in_word); + volatile POINTER_SIZE_INT *p_color_word; + Chunk_Header *chunk; + unsigned int slot_index; + + if(is_super_obj(obj)){ + chunk = ABNORMAL_CHUNK_HEADER(obj); + slot_index = 0; + } else { + chunk = NORMAL_CHUNK_HEADER(obj); + slot_index = slot_addr_to_index(chunk, obj); + } + + unsigned int word_index = slot_index >> SLOT_NUM_PER_WORD_SHIT; + unsigned int index_in_word = COLOR_BITS_PER_OBJ * (slot_index & (((unsigned int)(SLOT_NUM_PER_WORD_IN_TABLE-1)))); + p_color_word = &chunk->table[word_index]; assert(p_color_word); + + if(obj_is_mark_black_in_table(obj)) { + marker->live_obj_num--; + marker->live_obj_size-=chunk->slot_size; + } //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); POINTER_SIZE_INT mark_color = (cur_mark_black_color|cur_mark_gray_color) << index_in_word; @@ -417,7 +582,7 @@ inline void ops_color_flip(void) { POINTER_SIZE_INT temp = cur_alloc_color; - cur_alloc_color = cur_mark_black_color; + cur_alloc_color = cur_mark_black_color; //can not use mark = alloc, otherwise some obj alloc when swapping may be lost cur_mark_black_color = temp; cur_alloc_mask = (~cur_alloc_mask) & FLIP_COLOR_MASK_IN_TABLE; cur_mark_mask = (~cur_mark_mask) & FLIP_COLOR_MASK_IN_TABLE; Index: mark_sweep/wspace_sweep.cpp =================================================================== --- mark_sweep/wspace_sweep.cpp (revision 698467) +++ mark_sweep/wspace_sweep.cpp (working copy) @@ -231,6 +231,7 @@ void wspace_merge_free_chunks(GC *gc, Wspace *wspace) { + Free_Chunk_List free_chunk_list; free_chunk_list.head = NULL; free_chunk_list.tail = NULL; Index: mark_sweep/wspace_sweep_concurrent.cpp =================================================================== --- mark_sweep/wspace_sweep_concurrent.cpp (revision 698467) +++ mark_sweep/wspace_sweep_concurrent.cpp (working copy) @@ -2,10 +2,61 @@ #include "wspace_chunk.h" #include "wspace_mark_sweep.h" #include "gc_ms.h" +#include "../thread/conclctor.h" #include "../gen/gen.h" -static void collector_sweep_normal_chunk_con(Collector *collector, Wspace *wspace, Chunk_Header *chunk) + +static void wspace_check_free_list_chunks(Free_Chunk_List* free_list) { + Free_Chunk* chunk = free_list->head; + while(chunk ){ + assert(!(chunk->status & (CHUNK_TO_MERGE |CHUNK_MERGED) )); + chunk = chunk->next; + } +} + +static void wspace_check_free_chunks_status(Wspace* wspace) +{ + unsigned int i; + + for(i = NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) + wspace_check_free_list_chunks(&wspace->aligned_free_chunk_lists[i]); + + for(i = NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) + wspace_check_free_list_chunks(&wspace->unaligned_free_chunk_lists[i]); + + wspace_check_free_list_chunks(wspace->hyper_free_chunk_list); + +} + +inline static void check_list(Free_Chunk_List *chunk_list) +{ + Free_Chunk *chunk = chunk_list->head; + unsigned int count = 0; + while(chunk) { + count++; + chunk = chunk->next; + } + assert( count == chunk_list->chunk_num ); +} + +inline static void collector_add_free_chunk(Conclctor *sweeper, Free_Chunk *chunk) +{ + Free_Chunk_List *list = sweeper->free_chunk_list; + + chunk->status = CHUNK_FREE | CHUNK_TO_MERGE; + chunk->next = list->head; + chunk->prev = NULL; + if(list->head) + list->head->prev = chunk; + else + list->tail = chunk; + list->head = chunk; + list->chunk_num++; +} + +static void collector_sweep_normal_chunk_con(Conclctor *sweeper, Wspace *wspace, Chunk_Header *chunk) +{ unsigned int slot_num = chunk->slot_num; unsigned int live_num = 0; unsigned int first_free_word_index = MAX_SLOT_INDEX; @@ -13,20 +64,24 @@ unsigned int index_word_num = (slot_num + SLOT_NUM_PER_WORD_IN_TABLE - 1) / SLOT_NUM_PER_WORD_IN_TABLE; for(unsigned int i=0; ilive_obj_size += live_num * chunk->slot_size; - collector->live_obj_num += live_num; + sweeper->live_obj_size += live_num * chunk->slot_size; + sweeper->live_obj_num += live_num; if(!live_num){ /* all objects in this chunk are dead */ - collector_add_free_chunk(collector, (Free_Chunk*)chunk); + collector_add_free_chunk(sweeper, (Free_Chunk*)chunk); } else { chunk->alloc_num = live_num; if(!chunk_is_reusable(chunk)){ /* most objects in this chunk are swept, add chunk to pfc list*/ @@ -37,172 +92,173 @@ } } -static inline void collector_sweep_abnormal_chunk_con(Collector *collector, Wspace *wspace, Chunk_Header *chunk) +static inline void collector_sweep_abnormal_chunk_con(Conclctor *sweeper, Wspace *wspace, Chunk_Header *chunk) { assert(chunk->status == (CHUNK_ABNORMAL | CHUNK_USED)); POINTER_SIZE_INT *table = chunk->table; table[0] &= cur_alloc_mask; if(!table[0]){ - collector_add_free_chunk(collector, (Free_Chunk*)chunk); + collector_add_free_chunk(sweeper, (Free_Chunk*)chunk); } else { wspace_reg_live_abnormal_chunk(wspace, chunk); - collector->live_obj_size += CHUNK_SIZE(chunk); - collector->live_obj_num++; + sweeper->live_obj_size += CHUNK_SIZE(chunk); + sweeper->live_obj_num++; } } -static void wspace_sweep_chunk_con(Wspace* wspace, Collector* collector, Chunk_Header_Basic* chunk) +static void wspace_sweep_chunk_con(Wspace* wspace, Conclctor* sweeper, Chunk_Header_Basic* chunk) { if(chunk->status & CHUNK_NORMAL){ /* chunk is used as a normal sized obj chunk */ assert(chunk->status == (CHUNK_NORMAL | CHUNK_USED)); - collector_sweep_normal_chunk_con(collector, wspace, (Chunk_Header*)chunk); + collector_sweep_normal_chunk_con(sweeper, wspace, (Chunk_Header*)chunk); } else { /* chunk is used as a super obj chunk */ assert(chunk->status == (CHUNK_ABNORMAL | CHUNK_USED)); - collector_sweep_abnormal_chunk_con(collector, wspace, (Chunk_Header*)chunk); + collector_sweep_abnormal_chunk_con(sweeper, wspace, (Chunk_Header*)chunk); } } -static Free_Chunk_List* wspace_get_free_chunk_list(Wspace* wspace) +//used in last sweeper and final stw reset +Free_Chunk_List merged_free_chunk_list; +Free_Chunk_List free_chunk_list_from_sweepers; +Free_Chunk_List global_free_chunk_list; + +static Free_Chunk_List* wspace_collect_free_chunks_from_sweepers(GC *gc) { - GC* gc = wspace->gc; - Free_Chunk_List* free_chunk_list = (Free_Chunk_List*) STD_MALLOC(sizeof(Free_Chunk_List)); + Free_Chunk_List* free_chunk_list = &free_chunk_list_from_sweepers; assert(free_chunk_list); - memset(free_chunk_list, 0, sizeof(Free_Chunk_List)); + free_chunk_list_init(free_chunk_list); - /* Collect free chunks from collectors to one list */ - for(unsigned int i=0; inum_collectors; ++i){ - Free_Chunk_List *list = gc->collectors[i]->free_chunk_list; + for( unsigned int i=0; inum_conclctors; i++ ) { + Conclctor *conclctor = gc->conclctors[i]; + if( conclctor->role != CONCLCTOR_ROLE_SWEEPER ) + continue; + Free_Chunk_List *list = conclctor->free_chunk_list; move_free_chunks_between_lists(free_chunk_list, list); } - return free_chunk_list; } -Boolean wspace_get_free_chunk_concurrent(Wspace *wspace, Free_Chunk* chunk) + +static void wspace_reset_free_list_chunks(Free_Chunk_List* free_list) { - POINTER_SIZE_INT chunk_size = CHUNK_SIZE(chunk); - assert(!(chunk_size % CHUNK_GRANULARITY)); + Free_Chunk* chunk = free_list->head; + while(chunk ){ + assert(chunk->status & CHUNK_FREE); + chunk->status = CHUNK_FREE; + chunk = chunk->next; + } +} - Free_Chunk_List* free_list = NULL; +static void wspace_reset_free_list_chunks(Free_Chunk_List* free_list, Chunk_Status_t status) +{ + Free_Chunk* chunk = free_list->head; + while(chunk ){ + assert(chunk->status & CHUNK_FREE); + chunk->status = status; + chunk = chunk->next; + } +} - /*Find list*/ - if(chunk_size > HYPER_OBJ_THRESHOLD) - free_list = wspace->hyper_free_chunk_list; - else if(!((POINTER_SIZE_INT)chunk & NORMAL_CHUNK_LOW_MASK) && !(chunk_size & NORMAL_CHUNK_LOW_MASK)) - free_list = &wspace->aligned_free_chunk_lists[ALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)]; - else - free_list = &wspace->unaligned_free_chunk_lists[UNALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)]; +static unsigned int get_to_merge_length(Free_Chunk_List *free_list) +{ + Free_Chunk* chunk = free_list->head; + unsigned int counter = 0; + while(chunk) { + if(chunk->status&CHUNK_MERGED) { + return counter; + } + counter++; + chunk = chunk->next; + } + return counter; +} - /*Lock this free list*/ - lock(free_list->lock); - - /*Search free list for chunk*/ - Free_Chunk* chunk_iter = free_list->head; - while((POINTER_SIZE_INT)chunk_iter){ - if((POINTER_SIZE_INT)chunk_iter == (POINTER_SIZE_INT)chunk){ - /*Find chunk and delete from list.*/ - free_list_detach_chunk(free_list, chunk); - unlock(free_list->lock); - return TRUE; - } - chunk_iter = chunk_iter->next; - } - - unlock(free_list->lock); - - return FALSE; +static unsigned int get_length(Free_Chunk_List *free_list) +{ + Free_Chunk* chunk = free_list->head; + unsigned int counter = 0; + while(chunk) { + counter++; + chunk = chunk->next; + } + return counter; } -void wspace_merge_adj_free_chunks(Wspace* wspace,Free_Chunk* chunk) +static void wspace_merge_free_list(Wspace* wspace, Free_Chunk_List *free_list) { + int64 merge_start = time_now(); Free_Chunk *wspace_ceiling = (Free_Chunk*)space_heap_end((Space*)wspace); + Free_Chunk *chunk = free_list->head; + while(chunk && !(chunk->status &CHUNK_MERGED)) { - /* Check if the back adjcent chunks are free */ - Free_Chunk *back_chunk = (Free_Chunk*)chunk->adj_next; - while(back_chunk < wspace_ceiling && (back_chunk->status & CHUNK_FREE)){ - assert(chunk < back_chunk); - /* Remove back_chunk from list */ - if(wspace_get_free_chunk_concurrent(wspace,back_chunk)){ + free_list->head = chunk->next; + free_list->chunk_num--; + if(free_list->head) + free_list->head->prev = NULL; + /* Check if the back adjcent chunks are free */ + Free_Chunk *back_chunk = (Free_Chunk*)chunk->adj_next; + while(back_chunk < wspace_ceiling && (back_chunk->status & (CHUNK_TO_MERGE|CHUNK_MERGED))) { + assert(chunk < back_chunk); + /* Remove back_chunk from list */ + free_list_detach_chunk(free_list, back_chunk); back_chunk = (Free_Chunk*)back_chunk->adj_next; chunk->adj_next = (Chunk_Header_Basic*)back_chunk; - }else{ - break; } - } + if(back_chunk < wspace_ceiling) + back_chunk->adj_prev = (Chunk_Header_Basic*)chunk; - chunk->status = CHUNK_FREE | CHUNK_MERGED; - /* put the free chunk to the according free chunk list */ - wspace_put_free_chunk_to_tail(wspace, chunk); - -} - -static void wspace_merge_list_concurrent(Wspace* wspace, Free_Chunk_List* free_list) -{ - lock(free_list->lock); - Free_Chunk* chunk = free_list->head; - - while(chunk && !is_free_chunk_merged(chunk)){ - free_list_detach_chunk(free_list, chunk); - unlock(free_list->lock); - - wspace_merge_adj_free_chunks(wspace, chunk); - - lock(free_list->lock); + //INFO2("gc.con.info", "the iteration merges [" << counter << "] chunks, to merge length=" << get_to_merge_length(free_list)); + chunk->status = CHUNK_FREE | CHUNK_MERGED; + free_chunk_list_add_tail(free_list, chunk); chunk = free_list->head; } - - unlock(free_list->lock); + //INFO2("gc.con.info", "after "<< counter <<" mergings, chunks num [" << get_length(free_list) << "], time=" << (time_now()-merge_start) << " us"); } + -static void wspace_merge_free_chunks_concurrent(Wspace* wspace, Free_Chunk_List* free_list) +static inline Free_Chunk_List * gc_collect_global_free_chunk_list(Wspace *wspace, GC *gc) { - Free_Chunk *chunk = free_list->head; - - /*merge free list*/ - wspace_merge_list_concurrent(wspace, free_list); - /*check free pool*/ + free_chunk_list_init(&global_free_chunk_list); + Free_Chunk_List *global_free_list = &global_free_chunk_list; unsigned int i; - for(i = NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) - wspace_merge_list_concurrent(wspace, &wspace->aligned_free_chunk_lists[i]); - + move_free_chunks_between_lists(global_free_list, &wspace->aligned_free_chunk_lists[i]); for(i = NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) - wspace_merge_list_concurrent(wspace, &wspace->unaligned_free_chunk_lists[i]); + move_free_chunks_between_lists(global_free_list, &wspace->unaligned_free_chunk_lists[i]); - wspace_merge_list_concurrent(wspace, wspace->hyper_free_chunk_list); + move_free_chunks_between_lists(global_free_list, wspace->hyper_free_chunk_list); + move_free_chunks_between_lists(global_free_list, &free_chunk_list_from_sweepers); + + wspace_reset_free_list_chunks(global_free_list, CHUNK_FREE|CHUNK_TO_MERGE); + + return global_free_list; } -static void wspace_reset_free_list_chunks(Wspace* wspace, Free_Chunk_List* free_list) -{ - lock(free_list->lock); - Free_Chunk* chunk = free_list->head; +//final remerge in a STW manner, this can reduce the lock of merging global free list +void gc_merge_free_list_global(GC *gc) { + Wspace *wspace = gc_get_wspace(gc); + int64 start_merge = time_now(); - while(chunk ){ - assert(chunk->status & CHUNK_FREE); - chunk->status = CHUNK_FREE; - chunk = chunk->next; + Free_Chunk_List *global_free_list = gc_collect_global_free_chunk_list(wspace, gc); + wspace_merge_free_list(wspace, global_free_list); + wspace_reset_free_list_chunks(global_free_list); + + //put to global list + Free_Chunk *chunk = global_free_list->head; + while(chunk) { + global_free_list->head = chunk->next; + if(global_free_list->head) + global_free_list->head->prev = NULL; + wspace_put_free_chunk(wspace, chunk); + chunk = global_free_list->head; } + //INFO2("gc.merge", "[merge global] time=" << (time_now()-start_merge) << " us" ); - unlock(free_list->lock); } -static void wspace_reset_free_chunks_status(Wspace* wspace) -{ - unsigned int i; - - for(i = NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) - wspace_reset_free_list_chunks(wspace, &wspace->aligned_free_chunk_lists[i]); - - for(i = NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) - wspace_reset_free_list_chunks(wspace, &wspace->unaligned_free_chunk_lists[i]); - - wspace_reset_free_list_chunks(wspace, wspace->hyper_free_chunk_list); - -} - static void allocator_sweep_local_chunks(Allocator *allocator) { Wspace *wspace = gc_get_wspace(allocator->gc); @@ -243,9 +299,7 @@ static void gc_sweep_mutator_local_chunks(GC *gc) { -#ifdef USE_UNIQUE_MARK_SWEEP_GC lock(gc->mutator_list_lock); // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - /* release local chunks of each mutator in unique mark-sweep GC */ Mutator *mutator = gc->mutator_list; while(mutator){ @@ -253,9 +307,7 @@ allocator_sweep_local_chunks((Allocator*)mutator); mutator = mutator->next; } - unlock(gc->mutator_list_lock); -#endif } static void gc_wait_mutator_signal(GC *gc, unsigned int handshake_signal) @@ -279,19 +331,15 @@ The mark bit and alloc bit is exchanged before entering this function. This function is to clear the mark bit and merge the free chunks concurrently. */ -void wspace_sweep_concurrent(Collector* collector) +void wspace_sweep_concurrent(Conclctor* sweeper) { - collector->time_measurement_start = time_now(); - GC *gc = collector->gc; + GC *gc = sweeper->gc; + Wspace *wspace = gc_get_wspace(gc); - collector->live_obj_size = 0; - collector->live_obj_num = 0; + sweeper->live_obj_size = 0; + sweeper->live_obj_num = 0; - unsigned int num_active_collectors = gc->num_active_collectors; - - atomic_cas32(&num_sweeping_collectors, 0, num_active_collectors+1); - Pool* used_chunk_pool = wspace->used_chunk_pool; Chunk_Header_Basic* chunk_to_sweep; @@ -299,7 +347,7 @@ /*1. Grab chunks from used list, sweep the chunk and push back to PFC backup list & free list.*/ chunk_to_sweep = chunk_pool_get_chunk(used_chunk_pool); while(chunk_to_sweep != NULL){ - wspace_sweep_chunk_con(wspace, collector, chunk_to_sweep); + wspace_sweep_chunk_con(wspace, sweeper, chunk_to_sweep); chunk_to_sweep = chunk_pool_get_chunk(used_chunk_pool); } @@ -312,7 +360,7 @@ while(chunk_to_sweep != NULL){ assert(chunk_to_sweep->status == (CHUNK_NORMAL | CHUNK_NEED_ZEROING)); chunk_to_sweep->status = CHUNK_NORMAL | CHUNK_USED; - wspace_sweep_chunk_con(wspace, collector, chunk_to_sweep); + wspace_sweep_chunk_con(wspace, sweeper, chunk_to_sweep); chunk_to_sweep = chunk_pool_get_chunk(pfc_pool); } } @@ -320,12 +368,23 @@ pfc_pool = wspace_grab_next_pfc_pool(wspace); } - unsigned int old_num = atomic_inc32(&num_sweeping_collectors); - if( ++old_num == num_active_collectors ){ - - /*3. Check the local chunk of mutator*/ - gc_sweep_mutator_local_chunks(wspace->gc); +} + +//final work should be done by the last sweeper +void wspace_last_sweeper_work( Conclctor *last_sweeper ) { + + GC *gc = last_sweeper->gc; + Wspace *wspace = gc_get_wspace(gc); + Chunk_Header_Basic* chunk_to_sweep; + Pool* used_chunk_pool = wspace->used_chunk_pool; + + /* all but one sweeper finishes its job*/ + state_transformation( gc, GC_CON_SWEEPING, GC_CON_SWEEP_DONE ); + + /*3. Check the local chunk of mutator*/ + gc_sweep_mutator_local_chunks(wspace->gc); + /*4. Sweep gloabl alloc normal chunks again*/ gc_set_sweep_global_normal_chunk(); gc_wait_mutator_signal(wspace->gc, HSIG_MUTATOR_SAFE); @@ -337,27 +396,27 @@ while(chunk_to_sweep != NULL){ assert(chunk_to_sweep->status == (CHUNK_NORMAL | CHUNK_NEED_ZEROING)); chunk_to_sweep->status = CHUNK_NORMAL | CHUNK_USED; - wspace_sweep_chunk_con(wspace, collector, chunk_to_sweep); + wspace_sweep_chunk_con(wspace, last_sweeper, chunk_to_sweep); chunk_to_sweep = chunk_pool_get_chunk(pfc_pool); } } /*grab more pfc pools*/ pfc_pool = wspace_grab_next_pfc_pool(wspace); } - - /*4. Check the used list again.*/ + + /*5. Check the used list again.*/ chunk_to_sweep = chunk_pool_get_chunk(used_chunk_pool); while(chunk_to_sweep != NULL){ - wspace_sweep_chunk_con(wspace, collector, chunk_to_sweep); + wspace_sweep_chunk_con(wspace, last_sweeper, chunk_to_sweep); chunk_to_sweep = chunk_pool_get_chunk(used_chunk_pool); } - /*5. Switch the PFC backup list to PFC list.*/ + /*6. Switch the PFC backup list to PFC list.*/ wspace_exchange_pfc_pool(wspace); gc_unset_sweep_global_normal_chunk(); - /*6. Put back live abnormal chunk and normal unreusable chunk*/ + /*7. Put back live abnormal chunk and normal unreusable chunk*/ Chunk_Header* used_abnormal_chunk = wspace_get_live_abnormal_chunk(wspace); while(used_abnormal_chunk){ used_abnormal_chunk->status = CHUNK_USED | CHUNK_ABNORMAL; @@ -373,18 +432,13 @@ unreusable_normal_chunk = wspace_get_unreusable_normal_chunk(wspace); } pool_empty(wspace->unreusable_normal_chunk_pool); - - - /*7. Merge free chunks*/ - Free_Chunk_List* free_chunk_list = wspace_get_free_chunk_list(wspace); - wspace_merge_free_chunks_concurrent(wspace, free_chunk_list); - wspace_reset_free_chunks_status(wspace); - - /* let other collectors go */ - num_sweeping_collectors++; - } - while(num_sweeping_collectors != num_active_collectors + 1); - collector->time_measurement_end = time_now(); + + /*8. Merge free chunks from sweepers*/ + Free_Chunk_List *free_list_from_sweeper = wspace_collect_free_chunks_from_sweepers(gc); + wspace_merge_free_list(wspace, free_list_from_sweeper); + + /* last sweeper will transform the state to before_finish */ + state_transformation( gc, GC_CON_SWEEP_DONE, GC_CON_BEFORE_FINISH ); } Index: move_compact/gc_mc.cpp =================================================================== --- move_compact/gc_mc.cpp (revision 698467) +++ move_compact/gc_mc.cpp (working copy) @@ -19,6 +19,56 @@ GC* gc_mc_create() { - assert(0); - return NULL; + GC* gc = (GC*)STD_MALLOC(sizeof(GC_MC)); + assert(gc); + memset(gc, 0, sizeof(GC_MC)); + return gc; } + +void gc_mc_initialize(GC_MC *gc_mc, POINTER_SIZE_INT min_heap_size, POINTER_SIZE_INT max_heap_size) +{ + assert(gc_mc); + + max_heap_size = round_down_to_size(max_heap_size, SPACE_ALLOC_UNIT); + min_heap_size = round_up_to_size(min_heap_size, SPACE_ALLOC_UNIT); + assert(max_heap_size <= max_heap_size_bytes); + assert(max_heap_size >= min_heap_size_bytes); + + void *cspace_base; + cspace_base = vm_reserve_mem(0, max_heap_size); + assert(cspace_base); + cspace_initialize((GC*)gc_mc, cspace_base, max_heap_size, max_heap_size); + + HEAP_BASE = (POINTER_SIZE_INT)cspace_base; + + gc_mc->heap_start = cspace_base; + gc_mc->heap_end = (void*)((POINTER_SIZE_INT)cspace_base + max_heap_size); + gc_mc->reserved_heap_size = max_heap_size; + gc_mc->committed_heap_size = max_heap_size; + gc_mc->num_collections = 0; + gc_mc->time_collections = 0; +} + +void gc_mc_destruct(GC_MC *gc_mc) +{ + Cspace *cspace = gc_mc->cspace; + void *cspace_start = cspace->heap_start; + cspace_destruct(cspace); + gc_mc->cspace = NULL; + vm_unmap_mem(cspace_start, space_committed_size((Space*)cspace)); +} + +void gc_mc_reclaim_heap(GC_MC *gc) +{ + cspace_collection(gc->cspace); +} + +void gc_mc_iterate_heap(GC_MC *gc) +{ +} + +Space *gc_mc_get_cspace(GC_MC *gc) +{ return (Space*)gc->cspace; } + +void gc_mc_set_cspace(GC_MC *gc, Cspace *cspace) +{ gc->cspace = cspace; } Index: move_compact/gc_mc.h =================================================================== --- move_compact/gc_mc.h (revision 698467) +++ move_compact/gc_mc.h (working copy) @@ -18,7 +18,98 @@ #ifndef _GC_MC_H_ #define _GC_MC_H_ -#include "../common/gc_common.h" +#include "cspace.h" +extern POINTER_SIZE_INT min_heap_size_bytes; +extern POINTER_SIZE_INT max_heap_size_bytes; -#endif /* ifndef _GC_MC_H_ */ +typedef struct GC_MC { + void* physical_start; + void* heap_start; + void* heap_end; + POINTER_SIZE_INT reserved_heap_size; + POINTER_SIZE_INT committed_heap_size; + unsigned int num_collections; + Boolean in_collection; + int64 time_collections; + float survive_ratio; + + /* mutation related info */ + Mutator *mutator_list; + SpinLock mutator_list_lock; + unsigned int num_mutators; + + /* collection related info */ + Collector** collectors; + unsigned int num_collectors; + unsigned int num_active_collectors; /* not all collectors are working */ + + /*concurrent markers and collectors*/ + Conclctor** conclctors; + unsigned int num_conclctors; + //unsigned int num_active_conclctors; + unsigned int num_active_markers; + unsigned int num_active_sweepers; + + /* metadata is the pool for rootset, tracestack, etc. */ + GC_Metadata* metadata; + Finref_Metadata *finref_metadata; + + unsigned int collect_kind; /* MAJOR or MINOR */ + unsigned int last_collect_kind; + unsigned int cause;/*GC_CAUSE_LOS_IS_FULL, GC_CAUSE_NOS_IS_FULL, or GC_CAUSE_RUNTIME_FORCE_GC*/ + Boolean collect_result; /* succeed or fail */ + + Boolean generate_barrier; + + /* FIXME:: this is wrong! root_set belongs to mutator */ + Vector_Block* root_set; + Vector_Block* weakroot_set; + Vector_Block* uncompressed_root_set; + + Space_Tuner* tuner; + + volatile unsigned int gc_concurrent_status; /*concurrent GC status: only support CONCURRENT_MARK_PHASE now*/ + Collection_Scheduler* collection_scheduler; + + SpinLock lock_con_mark; + SpinLock lock_enum; + SpinLock lock_con_sweep; + SpinLock lock_collect_sched; + + /* system info */ + unsigned int _system_alloc_unit; + unsigned int _machine_page_size_bytes; + unsigned int _num_processors; + + /* END of GC --> */ + + Cspace *cspace; + +} GC_MC; + + +////////////////////////////////////////////////////////////////////////////////////////// + +inline void *gc_mc_alloc(unsigned size, Allocator *allocator) +{ return cspace_alloc(size, allocator); } + +Space *gc_mc_get_cspace(GC_MC *gc); +void gc_mc_set_cspace(GC_MC *gc, Cspace *cspace); + +inline POINTER_SIZE_INT gc_mc_free_memory_size(GC_MC *gc) +{ return cspace_free_memory_size((Cspace*)gc_mc_get_cspace(gc)); } + +inline POINTER_SIZE_INT gc_mc_total_memory_size(GC_MC *gc) +{ return space_committed_size(gc_mc_get_cspace(gc)); } + +///////////////////////////////////////////////////////////////////////////////////////// + +void gc_mc_initialize(GC_MC *gc, POINTER_SIZE_INT initial_heap_size, POINTER_SIZE_INT final_heap_size); +void gc_mc_destruct(GC_MC *gc); +void gc_mc_reclaim_heap(GC_MC *gc); +void gc_mc_iterate_heap(GC_MC *gc); + + +#endif // _GC_MS_H_ + Index: thread/collector.cpp =================================================================== --- thread/collector.cpp (revision 698467) +++ thread/collector.cpp (working copy) @@ -366,5 +366,3 @@ - - Index: thread/mutator.cpp =================================================================== --- thread/mutator.cpp (revision 698467) +++ thread/mutator.cpp (working copy) @@ -208,30 +208,86 @@ return time_mutator; } -static POINTER_SIZE_INT size_new_obj_desturcted_mutator_alloced; +static POINTER_SIZE_INT desturcted_mutator_alloced_size; +static POINTER_SIZE_INT desturcted_mutator_alloced_num; +static POINTER_SIZE_INT desturcted_mutator_alloced_occupied_size; +static POINTER_SIZE_INT desturcted_mutator_write_barrier_marked_size; void mutator_register_new_obj_size(Mutator * mutator) { - size_new_obj_desturcted_mutator_alloced += mutator->new_obj_size; + desturcted_mutator_alloced_size += mutator->new_obj_size; + desturcted_mutator_alloced_num += mutator->new_obj_num; + desturcted_mutator_alloced_occupied_size += mutator->new_obj_occupied_size; + desturcted_mutator_write_barrier_marked_size += mutator->write_barrier_marked_size; } -POINTER_SIZE_INT gc_get_new_object_size(GC* gc, Boolean need_reset) + +unsigned int gc_get_mutator_write_barrier_marked_size(GC* gc) { - POINTER_SIZE_INT new_obj_size = 0; + POINTER_SIZE_INT write_barrier_marked_size = 0; + lock(gc->mutator_list_lock); + Mutator* mutator = gc->mutator_list; + while (mutator) { + write_barrier_marked_size += mutator->write_barrier_marked_size; + mutator = mutator->next; + } + unlock(gc->mutator_list_lock); + return write_barrier_marked_size; +} +unsigned int gc_get_mutator_dirty_obj_num(GC *gc) +{ + POINTER_SIZE_INT dirty_obj_num = 0; + lock(gc->mutator_list_lock); + Mutator* mutator = gc->mutator_list; + while (mutator) { + dirty_obj_num += mutator->dirty_obj_num; + mutator = mutator->next; + } + unlock(gc->mutator_list_lock); + return dirty_obj_num; +} +unsigned int gc_get_mutator_new_obj_size(GC* gc) +{ + POINTER_SIZE_INT new_obj_occupied_size = 0; lock(gc->mutator_list_lock); Mutator* mutator = gc->mutator_list; while (mutator) { - new_obj_size += mutator->new_obj_size; - if(need_reset) mutator->new_obj_size = 0; + new_obj_occupied_size += mutator->new_obj_occupied_size; mutator = mutator->next; } unlock(gc->mutator_list_lock); - new_obj_size += size_new_obj_desturcted_mutator_alloced; - if(need_reset) size_new_obj_desturcted_mutator_alloced = 0; - return new_obj_size; + return new_obj_occupied_size + desturcted_mutator_alloced_occupied_size; + } +unsigned int gc_reset_mutator_new_obj_size(GC * gc) +{ + POINTER_SIZE_INT new_obj_occupied_size = 0; + lock(gc->mutator_list_lock); + Mutator* mutator = gc->mutator_list; + while (mutator) { + new_obj_occupied_size += mutator->new_obj_occupied_size; + mutator->new_obj_size = 0; + mutator->new_obj_num = 0; + mutator->new_obj_occupied_size = 0; + mutator->write_barrier_marked_size = 0; + mutator->dirty_obj_num = 0; + mutator = mutator->next; + } + new_obj_occupied_size += desturcted_mutator_alloced_occupied_size; + desturcted_mutator_alloced_size = 0; + desturcted_mutator_alloced_num = 0; + desturcted_mutator_alloced_occupied_size = 0; + unlock(gc->mutator_list_lock); + + return new_obj_occupied_size; +} +unsigned int gc_get_mutator_number(GC *gc) +{ + return gc->num_mutators; +} + Index: thread/mutator.h =================================================================== --- thread/mutator.h (revision 698467) +++ thread/mutator.h (working copy) @@ -50,7 +50,14 @@ SpinLock dirty_set_lock; unsigned int dirty_obj_slot_num; //only ON_THE_FLY unsigned int dirty_obj_num; //concurrent mark + + /* obj alloc information */ POINTER_SIZE_INT new_obj_size; + /* accurate object number and total size*/ + POINTER_SIZE_INT new_obj_num; + POINTER_SIZE_INT new_obj_occupied_size; + POINTER_SIZE_INT write_barrier_marked_size; + } Mutator; void mutator_initialize(GC* gc, void* tls_gc_info); @@ -64,8 +71,12 @@ Vector_Block* gc_get_local_dirty_set(GC* gc, unsigned int shared_id); void gc_start_mutator_time_measure(GC* gc); int64 gc_get_mutator_time(GC* gc); -POINTER_SIZE_INT gc_get_new_object_size(GC* gc, Boolean need_reset); +unsigned int gc_get_mutator_write_barrier_marked_size( GC *gc ); +unsigned int gc_get_mutator_dirty_obj_num(GC *gc); +unsigned int gc_get_mutator_new_obj_size( GC* gc ); +unsigned int gc_reset_mutator_new_obj_size( GC* gc ); + inline void mutator_post_signal(Mutator* mutator, unsigned int handshake_signal) { mem_fence(); Index: thread/mutator_alloc.cpp =================================================================== --- thread/mutator_alloc.cpp (revision 698467) +++ thread/mutator_alloc.cpp (working copy) @@ -31,8 +31,6 @@ //#define GC_OBJ_SIZE_STATISTIC -volatile Boolean obj_alloced_live = FALSE; - #ifdef GC_OBJ_SIZE_STATISTIC #define GC_OBJ_SIZE_STA_MAX 256*KB unsigned int obj_size_distribution_map[GC_OBJ_SIZE_STA_MAX>>10]; Index: trace_forward/fspace_alloc.cpp =================================================================== --- trace_forward/fspace_alloc.cpp (revision 698467) +++ trace_forward/fspace_alloc.cpp (working copy) @@ -96,5 +96,3 @@ } - - Index: verify/verifier_common.cpp =================================================================== --- verify/verifier_common.cpp (revision 698467) +++ verify/verifier_common.cpp (working copy) @@ -209,7 +209,7 @@ } } -void verifier_log_start(const char* message) +void verifier_log_start(char* message) { printf("------------------------------%-16s------------------------------\n", message); } @@ -217,7 +217,7 @@ void verifier_collect_kind_log(Heap_Verifier* heap_verifier) { GC* gc = heap_verifier->gc; - const char* gc_kind; + char* gc_kind; if(collect_is_minor()){ gc_kind = " minor collection."; }else if(collect_is_fallback()){ Index: verify/verifier_common.h =================================================================== --- verify/verifier_common.h (revision 698467) +++ verify/verifier_common.h (working copy) @@ -77,7 +77,7 @@ Boolean verifier_parse_options(Heap_Verifier* heap_verifier, char* options); void verifier_log_before_gc(Heap_Verifier* heap_verifier); void verifier_log_after_gc(Heap_Verifier* heap_verifier); -void verifier_log_start(const char* message); +void verifier_log_start(char* message); Boolean verify_rootset_slot(REF* p_ref, Heap_Verifier* heap_verifier); @@ -128,7 +128,7 @@ if(!obj_is_alloc_in_color_table(p_obj)) printf("\nERROR: obj after GC should be set its alloc color!\n"); }else{ - if(gc_mark_is_concurrent()) + if( !in_con_idle(heap_verifier->gc) ) assert(obj_is_mark_black_in_table(p_obj)); } #endif Index: verify/verifier_scanner.cpp =================================================================== --- verify/verifier_scanner.cpp (revision 698467) +++ verify/verifier_scanner.cpp (working copy) @@ -444,5 +444,3 @@ - - Index: verify/verify_concurrent_mark.cpp =================================================================== --- verify/verify_concurrent_mark.cpp (revision 0) +++ verify/verify_concurrent_mark.cpp (revision 0) @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "verify_live_heap.h" +#include "verifier_common.h" +#include "verify_gc_effect.h" +#include "verify_mutator_effect.h" +#include "../finalizer_weakref/finalizer_weakref.h" +#include "../mark_sweep/wspace_mark_sweep.h" +void analyze_bad_obj(Partial_Reveal_Object *p_obj) +{ + Chunk_Header *chunk; + unsigned int slot_index; + unsigned int obj_size = 0; + + if(is_super_obj(p_obj)){ + chunk = ABNORMAL_CHUNK_HEADER(p_obj); + slot_index = 0; + obj_size = CHUNK_SIZE(chunk); + INFO2("gc.verifier", "[super bad obj]=" << p_obj << " size=" << obj_size << ", chunk" << chunk); + } else { + chunk = NORMAL_CHUNK_HEADER(p_obj); + slot_index = slot_addr_to_index(chunk, p_obj); + obj_size = chunk->slot_size; + INFO2("gc.verifier", "[normal bad obj]=" << p_obj << ", size=" << obj_size << ", chunk[" << chunk <<"] slot index[" <gc_class_name << " jlC=" << vt->jlC); + INFO2( "gc.verifier", "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"); + +} + + + /*<--------live objects scanner begin-------->*/ +static FORCE_INLINE void scan_slot(Heap_Verifier* heap_verifier, REF*p_ref) +{ + GC_Verifier* gc_verifier = heap_verifier->gc_verifier; + Partial_Reveal_Object *p_obj = read_slot(p_ref); + if(p_obj == NULL) return; + assert(address_belongs_to_gc_heap(p_obj, heap_verifier->gc)); + verifier_tracestack_push(p_obj, gc_verifier->trace_stack); + return; +} + +static void scan_object(Heap_Verifier* heap_verifier, Partial_Reveal_Object *p_obj) +{ + GC_Verifier* gc_verifier = heap_verifier->gc_verifier; + if(!obj_mark_in_vt(p_obj)) return; + assert(obj_is_mark_black_in_table(p_obj)); + if(!obj_is_mark_black_in_table(p_obj)) { + analyze_bad_obj(p_obj); + } + verify_object_header(p_obj, heap_verifier); + verifier_update_verify_info(p_obj, heap_verifier); + + if (!object_has_ref_field(p_obj)) return; + REF* p_ref; + if (object_is_array(p_obj)) { + Partial_Reveal_Array* array = (Partial_Reveal_Array*)p_obj; + unsigned int array_length = array->array_len; + //INFO2("gc.verifier","\tscan array "<< p_obj <<"(" << array_length << ")"); + p_ref = (REF*)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); + for (unsigned int i = 0; i < array_length; i++) { + scan_slot(heap_verifier, p_ref+i); + } + }else{ + unsigned int num_refs = object_ref_field_num(p_obj); + int* ref_iterator = object_ref_iterator_init(p_obj); + + //INFO2("gc.verifier","\tscan object "<< p_obj <<"(" << num_refs << ")"); + for(unsigned int i=0; i "); + scan_object(heap_verifier, p_obj); + GC_Verifier* gc_verifier = heap_verifier->gc_verifier; + Vector_Block* trace_stack = (Vector_Block*)gc_verifier->trace_stack; + Partial_Reveal_Object *sub_obj = NULL; + while( !vector_stack_is_empty(trace_stack)){ + sub_obj = (Partial_Reveal_Object *)vector_stack_pop(trace_stack); + scan_object(heap_verifier, sub_obj); + trace_stack = (Vector_Block*)gc_verifier->trace_stack; + } + return; +} + +void con_verifier_trace_from_rootsets(Heap_Verifier* heap_verifier, Pool* root_set_pool) +{ + Heap_Verifier_Metadata* verifier_metadata = heap_verifier->heap_verifier_metadata; + GC_Verifier* gc_verifier = heap_verifier->gc_verifier; + gc_verifier->objects_set = verifier_free_set_pool_get_entry(verifier_metadata->free_set_pool); + gc_verifier->trace_stack = verifier_free_task_pool_get_entry(verifier_metadata->free_task_pool); + gc_verifier->hashcode_set = verifier_free_set_pool_get_entry(verifier_metadata->free_set_pool); + pool_iterator_init(root_set_pool); + Vector_Block* root_set = pool_iterator_next(root_set_pool); + + /* first step: copy all root objects to trace tasks. */ + while(root_set){ + POINTER_SIZE_INT* iter = vector_block_iterator_init(root_set); + while(!vector_block_iterator_end(root_set,iter)){ + REF* p_ref = (REF* )*iter; + iter = vector_block_iterator_advance(root_set,iter); + Partial_Reveal_Object* p_obj = read_slot(p_ref); + verifier_tracestack_push(p_obj, gc_verifier->trace_stack); + } + root_set = pool_iterator_next(root_set_pool); + } + /* put back the last trace_stack task */ + pool_put_entry(verifier_metadata->mark_task_pool, gc_verifier->trace_stack); + + /* second step: iterate over the trace tasks and forward objects */ + gc_verifier->trace_stack = verifier_free_task_pool_get_entry(verifier_metadata->free_task_pool); + + Vector_Block* trace_task = pool_get_entry(verifier_metadata->mark_task_pool); + + while(trace_task){ + POINTER_SIZE_INT* iter = vector_block_iterator_init(trace_task); + while(!vector_block_iterator_end(trace_task,iter)){ + Partial_Reveal_Object* p_obj = (Partial_Reveal_Object* )*iter; + iter = vector_block_iterator_advance(trace_task,iter); + trace_object(heap_verifier, p_obj); + } + vector_stack_clear(trace_task); + pool_put_entry(verifier_metadata->free_task_pool, trace_task); + trace_task = pool_get_entry(verifier_metadata->mark_task_pool); + } + + pool_put_entry(verifier_metadata->objects_pool_before_gc, gc_verifier->objects_set); + + vector_stack_clear(gc_verifier->trace_stack); + pool_put_entry(verifier_metadata->free_task_pool, gc_verifier->trace_stack); + gc_verifier->trace_stack = NULL; + +} + +unsigned int clear_objs_mark_bit(Heap_Verifier* heap_verifier) +{ + Heap_Verifier_Metadata* verifier_metadata = heap_verifier->heap_verifier_metadata; + Pool* marked_objs_pool = verifier_metadata->objects_pool_before_gc; + + pool_iterator_init(marked_objs_pool); + Vector_Block* objs_set = pool_iterator_next(marked_objs_pool); + unsigned int clear_counter = 0; + while(objs_set){ + POINTER_SIZE_INT* iter = vector_block_iterator_init(objs_set); + while(!vector_block_iterator_end(objs_set,iter)){ + Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*)*iter; + iter = vector_block_iterator_advance(objs_set,iter); + assert(p_obj != NULL); + assert(obj_is_marked_in_vt(p_obj)); + clear_counter++; + obj_unmark_in_vt(p_obj); + } + objs_set = pool_iterator_next(marked_objs_pool); + } + return clear_counter; +} + +void verifier_rescan_after_con(Heap_Verifier* heap_verifier) +{ + INFO2("gc.con.verify", "start scan live object %%%%%%%%%%%%%%%%%%%%%%"); + Heap_Verifier_Metadata* verifier_metadata = heap_verifier->heap_verifier_metadata; + con_verifier_trace_from_rootsets(heap_verifier, verifier_metadata->root_set_pool); + clear_objs_mark_bit(heap_verifier); + INFO2("gc.con.verify", "end of scan live object %%%%%%%%%%%%%%%%%%%%%%"); +} + +void verify_gc_reset(Heap_Verifier* heap_verifier); +void verify_heap_after_con_gc(GC *gc) +{ + Heap_Verifier *heap_verifier = get_heap_verifier(); + int64 verify_start_time = time_now(); + verifier_copy_rootsets(gc, heap_verifier); + verifier_rescan_after_con(heap_verifier); + INFO2("gc.verifier", "[Verifier] verifier marked num=" << heap_verifier->gc_verifier->num_live_objects_before_gc ); + verify_gc_reset(heap_verifier); + unsigned int verify_time = trans_time_unit(time_now() - verify_start_time); + INFO2("gc.verifier", "[Verifier] verify time = [" << verify_time << "] ms") +} + Index: verify/verify_gc_effect.cpp =================================================================== --- verify/verify_gc_effect.cpp (revision 698467) +++ verify/verify_gc_effect.cpp (working copy) @@ -328,7 +328,7 @@ if(!obj_is_alloc_in_color_table(p_obj)) printf("\nERROR: obj after GC should be set its alloc color!\n"); }else{ - if(gc_mark_is_concurrent()) + if( !in_con_idle(heap_verifier->gc) ) assert(obj_is_mark_black_in_table(p_obj)); } #endif Index: verify/verify_live_heap.cpp =================================================================== --- verify/verify_live_heap.cpp (revision 698467) +++ verify/verify_live_heap.cpp (working copy) @@ -156,5 +156,3 @@ - -