Index: src/common/collection_scheduler.cpp =================================================================== --- src/common/collection_scheduler.cpp (revision 0) +++ src/common/collection_scheduler.cpp (revision 0) @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gc_common.h" +#include "../gen/gen.h" +#include "../mark_sweep/gc_ms.h" +#include "../mark_sweep/sspace.h" +#include "collection_scheduler.h" +#include "gc_concurrent.h" +static int64 time_delay_to_start_mark = 0; + +void collection_scheduler_initialize(GC* gc) +{ + + Collection_Scheduler* collection_scheduler = (Collection_Scheduler*) STD_MALLOC(sizeof(Collection_Scheduler)); + assert(collection_scheduler); + memset(collection_scheduler, 0, sizeof(Collection_Scheduler)); + + collection_scheduler->gc = gc; + gc->collection_scheduler = collection_scheduler; + time_delay_to_start_mark = 0; + + return; +} +void collection_scheduler_destruct(GC* gc) +{ + STD_FREE(gc->collection_scheduler); +} + +Boolean gc_need_start_concurrent_mark(GC* gc) +{ + if(!USE_CONCURRENT_GC) return FALSE; + //FIXME: GEN mode also needs the support of starting mark after thread resume. +#ifdef USE_MARK_SWEEP_GC + if(gc_is_concurrent_mark_phase() ) return FALSE; + + int64 time_current = time_now(); + if( time_current - get_collection_end_time() > time_delay_to_start_mark) + return TRUE; + else return FALSE; +#else + /*FIXME: concurrent mark is not support in GC_GEN*/ + assert(0); + if(gc_next_collection_kind((GC_Gen*)gc) == MAJOR_COLLECTION) + return TRUE; + else + return FALSE; +#endif +} + + + +void gc_update_collection_scheduler(GC* gc, int64 mutator_time, int64 mark_time) +{ + //FIXME: GEN GC should be supportted. +#ifdef USE_MARK_SWEEP_GC + + Collection_Scheduler* collection_scheduler = gc->collection_scheduler; + Space* space = NULL; + + space = (Space*) gc_get_sspace(gc); + + Space_Statistics* sspace_stat = space->space_statistic; + + unsigned int slot_index = collection_scheduler->last_slot_index_in_window; + unsigned int num_slot = collection_scheduler->num_slot_in_window; + + collection_scheduler->num_obj_traced_window[slot_index] = sspace_stat->num_live_obj; + collection_scheduler->size_alloced_window[slot_index] = sspace_stat->last_size_free_space; + + int64 time_mutator = mutator_time; + int64 time_mark = mark_time; + + collection_scheduler->alloc_rate_window[slot_index] + = time_mutator == 0 ? 0 : (float)collection_scheduler->size_alloced_window[slot_index] / time_mutator; + + collection_scheduler->trace_rate_window[slot_index] + = time_mark == 0 ? 0 : (float)collection_scheduler->num_obj_traced_window[slot_index] / time_mark; + + collection_scheduler->num_slot_in_window = num_slot >= STATISTICS_SAMPLING_WINDOW_SIZE ? num_slot : (++num_slot); + collection_scheduler->last_slot_index_in_window = (++slot_index)% STATISTICS_SAMPLING_WINDOW_SIZE; + + float sum_alloc_rate = 0; + float sum_trace_rate = 0; + + unsigned int i; + for(i = 0; i < collection_scheduler->num_slot_in_window; i++){ + sum_alloc_rate += collection_scheduler->alloc_rate_window[i]; + sum_trace_rate += collection_scheduler->trace_rate_window[i]; + + } + + float average_alloc_rate = sum_alloc_rate / collection_scheduler->num_slot_in_window; + float average_trace_rate = sum_trace_rate / collection_scheduler->num_slot_in_window; + + if(average_alloc_rate == 0 || average_trace_rate == 0){ + time_delay_to_start_mark = 0; + }else{ + float expected_time_alloc = sspace_stat->size_free_space / average_alloc_rate; + float expected_time_trace = sspace_stat->num_live_obj / average_trace_rate; + + + if(expected_time_alloc > expected_time_trace) + collection_scheduler->time_delay_to_start_mark = (int64)((expected_time_alloc - expected_time_trace)*0.7); + else + collection_scheduler->time_delay_to_start_mark = 0; + + time_delay_to_start_mark = collection_scheduler->time_delay_to_start_mark; + } +#endif + return; + +} + Index: src/common/collection_scheduler.h =================================================================== --- src/common/collection_scheduler.h (revision 0) +++ src/common/collection_scheduler.h (revision 0) @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _COLLECTION_SCHEDULER_H_ +#define _COLLECTION_SCHEDULER_H_ + +#define STATISTICS_SAMPLING_WINDOW_SIZE 5 + +typedef struct Collection_Scheduler { + /*common field*/ + GC* gc; + + /*mark schedule */ + int64 time_delay_to_start_mark; + + /**/ + unsigned int num_slot_in_window; + unsigned int last_slot_index_in_window; + + float alloc_rate_window[STATISTICS_SAMPLING_WINDOW_SIZE]; + float trace_rate_window[STATISTICS_SAMPLING_WINDOW_SIZE]; + POINTER_SIZE_INT num_obj_traced_window[STATISTICS_SAMPLING_WINDOW_SIZE]; + POINTER_SIZE_INT size_alloced_window[STATISTICS_SAMPLING_WINDOW_SIZE]; +} Collection_Scheduler; + +void collection_scheduler_initialize(GC* gc); +void collection_scheduler_destruct(GC* gc); + +void gc_update_collection_scheduler(GC* gc, int64 mutator_time, int64 mark_time); +Boolean gc_need_start_concurrent_mark(GC* gc); + + +#endif + Index: src/common/gc_common.cpp =================================================================== --- src/common/gc_common.cpp (revision 569956) +++ src/common/gc_common.cpp (working copy) @@ -22,11 +22,14 @@ #include "gc_common.h" #include "gc_metadata.h" #include "../thread/mutator.h" +#include "../thread/marker.h" #include "../finalizer_weakref/finalizer_weakref.h" #include "../gen/gen.h" #include "../mark_sweep/gc_ms.h" #include "../common/space_tuner.h" #include "interior_pointer.h" +#include "collection_scheduler.h" +#include "gc_concurrent.h" unsigned int Cur_Mark_Bit = 0x1; unsigned int Cur_Forward_Bit = 0x2; @@ -43,6 +46,7 @@ extern Boolean MINOR_ALGORITHM; extern Boolean MAJOR_ALGORITHM; +extern unsigned int NUM_MARKERS; extern unsigned int NUM_COLLECTORS; extern unsigned int MINOR_COLLECTORS; extern unsigned int MAJOR_COLLECTORS; @@ -56,6 +60,7 @@ extern Boolean JVMTI_HEAP_ITERATION ; extern Boolean IS_MOVE_COMPACT; +extern Boolean USE_CONCURRENT_GC; static int get_int_property(const char *property_name) { @@ -189,6 +194,11 @@ NUM_COLLECTORS = (num==0)? NUM_COLLECTORS:num; } + if (is_property_set("gc.num_markers", VM_PROPERTIES) == 1) { + unsigned int num = get_int_property("gc.num_markers"); + NUM_MARKERS = (num==0)? NUM_MARKERS:num; + } + /* GC algorithm decision */ /* Step 1: */ char* minor_algo = NULL; @@ -265,89 +275,107 @@ large_page_hint = strdup(value); destroy_property_value(value); } + + if (is_property_set("gc.concurrent_gc", VM_PROPERTIES) == 1){ + USE_CONCURRENT_GC= get_boolean_property("gc.concurrent_gc"); + } return; } void gc_assign_free_area_to_mutators(GC* gc) { +#ifndef USE_MARK_SWEEP_GC gc_gen_assign_free_area_to_mutators((GC_Gen*)gc); +#endif } -void gc_adjust_heap_size(GC* gc, int64 pause_time) -{ - gc_gen_adjust_heap_size((GC_Gen*)gc, pause_time); -} - void gc_copy_interior_pointer_table_to_rootset(); /*used for computing collection time and mutator time*/ -static int64 collection_start_time = time_now(); +static int64 collection_start_time = time_now(); static int64 collection_end_time = time_now(); + +int64 get_collection_end_time() +{ return collection_end_time; } + void gc_reclaim_heap(GC* gc, unsigned int gc_cause) -{ +{ INFO2("gc.process", "\nGC: GC start ...\n"); - - collection_start_time = time_now(); - int64 mutator_time = collection_start_time -collection_end_time; - + + collection_start_time = time_now(); + int64 mutator_time = collection_start_time - collection_end_time; + /* FIXME:: before mutators suspended, the ops below should be very careful to avoid racing with mutators. */ - gc->num_collections++; + gc->num_collections++; gc->cause = gc_cause; gc_decide_collection_kind((GC_Gen*)gc, gc_cause); -#ifndef USE_MARK_SWEEP_GC - gc_gen_update_space_before_gc((GC_Gen*)gc); - gc_compute_space_tune_size_before_marking(gc, gc_cause); -#endif - #ifdef MARK_BIT_FLIPPING if(gc_match_kind(gc, MINOR_COLLECTION)) mark_bit_flip(); #endif - gc_metadata_verify(gc, TRUE); + if(!USE_CONCURRENT_GC){ + gc_metadata_verify(gc, TRUE); #ifndef BUILD_IN_REFERENT - gc_finref_metadata_verify((GC*)gc, TRUE); + gc_finref_metadata_verify((GC*)gc, TRUE); #endif - + } /* Stop the threads and collect the roots. */ + lock(gc->enumerate_rootset_lock); INFO2("gc.process", "GC: stop the threads and enumerate rootset ...\n"); - gc_reset_rootset(gc); + gc_clear_rootset(gc); + gc_reset_rootset(gc); vm_enumerate_root_set_all_threads(); gc_copy_interior_pointer_table_to_rootset(); - gc_set_rootset(gc); + gc_set_rootset(gc); + unlock(gc->enumerate_rootset_lock); + if(USE_CONCURRENT_GC && gc_mark_is_concurrent()){ + gc_finish_concurrent_mark(gc); + } + /* this has to be done after all mutators are suspended */ gc_reset_mutator_context(gc); - + if(!IGNORE_FINREF ) gc_set_obj_with_fin(gc); #ifndef USE_MARK_SWEEP_GC - gc_gen_reclaim_heap((GC_Gen*)gc); + gc_gen_reclaim_heap((GC_Gen*)gc, collection_start_time); #else gc_ms_reclaim_heap((GC_MS*)gc); #endif + + /* FIXME:: clear root set here to support verify. */ +#ifdef COMPRESS_REFERENCE + gc_set_pool_clear(gc->metadata->gc_uncompressed_rootset_pool); +#endif + + gc_reset_interior_pointer_table(); - gc_reset_interior_pointer_table(); - gc_metadata_verify(gc, FALSE); - + collection_end_time = time_now(); - int64 pause_time = collection_end_time - collection_start_time; - gc->time_collections += pause_time; - #ifndef USE_MARK_SWEEP_GC - gc_gen_collection_verbose_info((GC_Gen*)gc, pause_time, mutator_time); + gc_gen_collection_verbose_info((GC_Gen*)gc, collection_end_time - collection_start_time, mutator_time); gc_gen_space_verbose_info((GC_Gen*)gc); - gc_adjust_heap_size(gc, pause_time); - - gc_gen_adapt((GC_Gen*)gc, pause_time); #endif if(gc_is_gen_mode()) gc_prepare_mutator_remset(gc); + int64 mark_time = 0; + if(USE_CONCURRENT_GC && gc_mark_is_concurrent()){ + gc_reset_concurrent_mark(gc); + mark_time = gc_get_concurrent_mark_time(gc); + } + +#ifndef USE_MARK_SWEEP_GC + if(USE_CONCURRENT_GC && gc_need_start_concurrent_mark(gc)) + gc_start_concurrent_mark(gc); +#endif + if(!IGNORE_FINREF ){ INFO2("gc.process", "GC: finref process after collection ...\n"); gc_put_finref_to_vm(gc); @@ -359,18 +387,16 @@ #endif } -#ifndef USE_MARK_SWEEP_GC - gc_space_tuner_reset(gc); - gc_gen_update_space_after_gc((GC_Gen*)gc); - gc_assign_free_area_to_mutators(gc); +#ifdef USE_MARK_SWEEP_GC + gc_ms_update_space_statistics((GC_MS*)gc); #endif - vm_reclaim_native_objs(); + gc_assign_free_area_to_mutators(gc); + + if(USE_CONCURRENT_GC) gc_update_collection_scheduler(gc, mutator_time, mark_time); + + vm_reclaim_native_objs(); vm_resume_threads_after(); INFO2("gc.process", "GC: GC end\n"); return; } - - - - Index: src/common/gc_common.h =================================================================== --- src/common/gc_common.h (revision 569956) +++ src/common/gc_common.h (working copy) @@ -92,24 +92,39 @@ /* major collection */ MAJOR_COMPACT_SLIDE, - MAJOR_COMPACT_MOVE + MAJOR_COMPACT_MOVE, + MAJOR_MARK_SWEEP }; /* Possible combinations: * MINOR_COLLECTION - * MAJOR_COLLECTION + * NORMAL_MAJOR_COLLECTION * FALLBACK_COLLECTION - * MAJOR_COLLECTION | EXTEND_COLLECTION + * NORMAL_MAJOR_COLLECTION | EXTEND_COLLECTION * FALLBACK_COLLECTION | EXTEND_COLLECTION + * MS_COLLECTION + * MS_COMPACT_COLLECTION */ enum Collection_Kind { - MINOR_COLLECTION = 0x1, - MAJOR_COLLECTION = 0x2, - FALLBACK_COLLECTION = 0x4, - EXTEND_COLLECTION = 0x8, - MARK_SWEEP_GC = 0x10, - SWEEP_COMPACT_GC = 0x20 + /* Two main kinds: generational GC and mark-sweep GC; this is decided at compiling time */ + GEN_GC = 0x1, + MARK_SWEEP_GC = 0x2, + /* Mask of bits standing for two basic kinds */ + GC_BASIC_KIND_MASK = ~(unsigned int)0x7, + + /* Sub-kinds of generational GC use the 4~7th LSB */ + MINOR_COLLECTION = 0x11, /* 0x10 & GEN_GC */ + MAJOR_COLLECTION = 0x21, /* 0x20 & GEN_GC */ + + /* Sub-kinds of major collection use the 8~11th LSB */ + NORMAL_MAJOR_COLLECTION = 0x121, /* 0x100 & MAJOR_COLLECTION */ + FALLBACK_COLLECTION = 0x221, /* 0x200 & MAJOR_COLLECTION */ + EXTEND_COLLECTION = 0x421, /* 0x400 & MAJOR_COLLECTION */ + + /* Sub-kinds of mark-sweep GC use the 12~15th LSB */ + MS_COLLECTION = 0x1002, /* 0x1000 & MARK_SWEEP_GC */ + MS_COMPACT_COLLECTION = 0x2002 /* 0x2000 & MARK_SWEEP_GC */ }; extern Boolean IS_FALLBACK_COMPACTION; /* only for mark/fw bits debugging purpose */ @@ -118,7 +133,7 @@ GC_CAUSE_NIL, GC_CAUSE_NOS_IS_FULL, GC_CAUSE_LOS_IS_FULL, - GC_CAUSE_POS_IS_FULL, + GC_CAUSE_SSPACE_IS_FULL, GC_CAUSE_RUNTIME_FORCE_GC }; @@ -356,13 +371,35 @@ #endif /* MARK_BIT_FLIPPING */ +inline Boolean obj_is_dirty_in_oi(Partial_Reveal_Object* p_obj) +{ + Obj_Info_Type info = get_obj_info_raw(p_obj); + return (Boolean)(info & OBJ_DIRTY_BIT); +} + +inline Boolean obj_dirty_in_oi(Partial_Reveal_Object* p_obj) +{ + Obj_Info_Type info = get_obj_info_raw(p_obj); + if( info & OBJ_DIRTY_BIT ) return FALSE; + + Obj_Info_Type new_info = info | OBJ_DIRTY_BIT; + while(info != atomic_cas32((volatile unsigned int *)get_obj_info_addr(p_obj),new_info, info)){ + info = get_obj_info_raw(p_obj); + if( info & OBJ_DIRTY_BIT ) return FALSE; + new_info = info |OBJ_DIRTY_BIT; + } + return TRUE; +} + /* all GCs inherit this GC structure */ +struct Marker; struct Mutator; struct Collector; struct GC_Metadata; struct Finref_Metadata; struct Vector_Block; struct Space_Tuner; +struct Collection_Scheduler; typedef struct GC{ void* heap_start; @@ -382,6 +419,10 @@ Collector** collectors; unsigned int num_collectors; unsigned int num_active_collectors; /* not all collectors are working */ + + Marker** markers; + unsigned int num_markers; + unsigned int num_active_markers; /* metadata is the pool for rootset, tracestack, etc. */ GC_Metadata* metadata; @@ -400,6 +441,13 @@ Vector_Block* uncompressed_root_set; Space_Tuner* tuner; + + unsigned int gc_concurrent_status; /*concurrent GC status: only support CONCURRENT_MARK_PHASE now*/ + Collection_Scheduler* collection_scheduler; + + SpinLock concurrent_mark_lock; + SpinLock enumerate_rootset_lock; + /* system info */ unsigned int _system_alloc_unit; @@ -422,17 +470,29 @@ return (addr >= gc_heap_base(gc) && addr < gc_heap_ceiling(gc)); } +/* gc must match exactly that kind if returning TRUE */ inline Boolean gc_match_kind(GC *gc, unsigned int kind) { assert(gc->collect_kind && kind); - return gc->collect_kind & kind; + return (Boolean)((gc->collect_kind & kind) == kind); } +/* multi_kinds is a combination of multi collect kinds + * gc must match one of them. + */ +inline Boolean gc_match_either_kind(GC *gc, unsigned int multi_kinds) +{ + multi_kinds &= GC_BASIC_KIND_MASK; + assert(gc->collect_kind && multi_kinds); + return (Boolean)(gc->collect_kind & multi_kinds); +} inline unsigned int gc_get_processor_num(GC* gc) { return gc->_num_processors; } void gc_parse_options(GC* gc); void gc_reclaim_heap(GC* gc, unsigned int gc_cause); +int64 get_collection_end_time(); + /* generational GC related */ extern Boolean NOS_PARTIAL_FORWARD; Index: src/common/gc_concurrent.cpp =================================================================== --- src/common/gc_concurrent.cpp (revision 0) +++ src/common/gc_concurrent.cpp (revision 0) @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gc_common.h" +#include "gc_metadata.h" +#include "../thread/mutator.h" +#include "../thread/marker.h" +#include "../finalizer_weakref/finalizer_weakref.h" +#include "../gen/gen.h" +#include "../mark_sweep/gc_ms.h" +#include "interior_pointer.h" +#include "collection_scheduler.h" +#include "gc_concurrent.h" + +Boolean USE_CONCURRENT_GC = FALSE; + +volatile Boolean concurrent_mark_phase = FALSE; +volatile Boolean mark_is_concurrent = FALSE; + + +static void gc_check_concurrent_mark(GC* gc) +{ + if(!is_mark_finished(gc)){ + lock(gc->concurrent_mark_lock); +#ifndef USE_MARK_SWEEP_GC + gc_gen_start_concurrent_mark((GC_Gen*)gc); +#else + gc_ms_start_concurrent_mark((GC_MS*)gc, MAX_NUM_MARKERS); +#endif + unlock(gc->concurrent_mark_lock); + } +} + +static void gc_wait_concurrent_mark_finish(GC* gc) +{ + wait_mark_finish(gc); + gc_set_concurrent_status(gc,GC_CONCURRENT_STATUS_NIL); + gc_reset_snaptshot(gc); +} + +void gc_start_concurrent_mark(GC* gc) +{ + if(!try_lock(gc->concurrent_mark_lock) || gc_mark_is_concurrent()) return; + + /*prepare rootset*/ + if(TRUE){ + lock(gc->enumerate_rootset_lock); + gc_metadata_verify(gc, TRUE); + gc_reset_rootset(gc); + vm_enumerate_root_set_all_threads(); + gc_copy_interior_pointer_table_to_rootset(); + gc_set_rootset(gc); + }else{ + gc_clear_remset((GC*)gc); + if(!IGNORE_FINREF){ + gc_copy_finaliable_obj_to_rootset(gc); + } + gc->root_set = NULL; + } + gc_set_concurrent_status(gc, GC_CONCURRENT_MARK_PHASE); + + gc_decide_collection_kind((GC_Gen*)gc, GC_CAUSE_NIL); + + /*start concurrent mark*/ +#ifndef USE_MARK_SWEEP_GC + gc_gen_start_concurrent_mark((GC_Gen*)gc); +#else + gc_ms_start_concurrent_mark((GC_MS*)gc, MIN_NUM_MARKERS); +#endif + + if(TRUE){ + unlock(gc->enumerate_rootset_lock); + vm_resume_threads_after(); + } + + unlock(gc->concurrent_mark_lock); +} + +void gc_finish_concurrent_mark(GC* gc) +{ + gc_check_concurrent_mark(gc); + gc_wait_concurrent_mark_finish(gc); +} + +void gc_reset_concurrent_mark(GC* gc) +{ + gc->num_active_markers = 0; + gc_mark_unset_concurrent(); +} + +int64 gc_get_concurrent_mark_time(GC* gc) +{ + int64 time_mark = 0; + Marker** markers = gc->markers; + unsigned int i; + for(i = 0; i < gc->num_active_markers; i++){ + Marker* marker = markers[i]; + if(marker->time_mark > time_mark){ + time_mark = marker->time_mark; + } + } + return time_mark; +} + + Index: src/common/gc_concurrent.h =================================================================== --- src/common/gc_concurrent.h (revision 0) +++ src/common/gc_concurrent.h (revision 0) @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _GC_CONCURRENT_H_ +#define _GC_CONCURRENT_H_ +#include "gc_common.h" + +enum GC_CONCURRENT_STATUS{ + GC_CONCURRENT_STATUS_NIL = 0x00, + GC_CONCURRENT_MARK_PHASE = 0x01, +}; + +extern volatile Boolean concurrent_mark_phase; +extern volatile Boolean mark_is_concurrent; +extern Boolean USE_CONCURRENT_GC; + +inline Boolean gc_mark_is_concurrent() +{ + return mark_is_concurrent; +} + +inline void gc_mark_set_concurrent() +{ + mark_is_concurrent = TRUE; +} + +inline void gc_mark_unset_concurrent() +{ + mark_is_concurrent = FALSE; +} + +inline Boolean gc_is_concurrent_mark_phase() +{ + return concurrent_mark_phase; +} + +inline Boolean gc_is_concurrent_mark_phase(GC* gc) +{ + return gc->gc_concurrent_status == GC_CONCURRENT_MARK_PHASE; +} + +inline void gc_set_concurrent_status(GC*gc, unsigned int status) +{ + gc->gc_concurrent_status = status; + if(status == GC_CONCURRENT_MARK_PHASE){ + concurrent_mark_phase = TRUE; + gc_mark_set_concurrent(); + }else{ + concurrent_mark_phase = FALSE; + } +} + +void gc_reset_concurrent_mark(GC* gc); +void gc_start_concurrent_mark(GC* gc); +void gc_finish_concurrent_mark(GC* gc); +int64 gc_get_concurrent_mark_time(GC* gc); + + +#endif Index: src/common/gc_for_class.h =================================================================== --- src/common/gc_for_class.h (revision 569956) +++ src/common/gc_for_class.h (working copy) @@ -65,6 +65,8 @@ #endif /* else MARK_BIT_FLIPPING */ +#define OBJ_DIRTY_BIT 0x20 + /*emt64 related!*/ #define COMPRESS_VTABLE Index: src/common/gc_for_vm.cpp =================================================================== --- src/common/gc_for_vm.cpp (revision 569956) +++ src/common/gc_for_vm.cpp (working copy) @@ -27,9 +27,12 @@ #include "../gen/gen.h" #include "../mark_sweep/gc_ms.h" #include "interior_pointer.h" +#include "../thread/marker.h" #include "../thread/collector.h" #include "../verify/verify_live_heap.h" #include "../finalizer_weakref/finalizer_weakref.h" +#include "collection_scheduler.h" +#include "gc_concurrent.h" #ifdef USE_32BITS_HASHCODE #include "hashcode.h" #endif @@ -51,7 +54,7 @@ } int gc_init() -{ +{ INFO2("gc.process", "GC: call GC init...\n"); assert(p_global_gc == NULL); @@ -85,7 +88,13 @@ #ifndef BUILD_IN_REFERENT gc_finref_metadata_initialize(gc); #endif + if(USE_CONCURRENT_GC){ + collection_scheduler_initialize(gc); + marker_initialize(gc); + } + collector_initialize(gc); + gc_init_heap_verification(gc); mutator_need_block = FALSE; @@ -111,6 +120,7 @@ gc_finref_metadata_destruct(gc); #endif collector_destruct(gc); + marker_destruct(gc); if( verify_live_heap ){ gc_terminate_heap_verification(gc); @@ -173,19 +183,19 @@ return VTABLE_TRACING; } -void gc_add_weak_root_set_entry(Managed_Object_Handle *ref, Boolean is_pinned, Boolean is_short_weak) +void gc_add_weak_root_set_entry(Managed_Object_Handle *ref, Boolean is_pinned, Boolean is_short_weak) { //assert(is_short_weak == FALSE); //Currently no need for short_weak_roots Partial_Reveal_Object** p_ref = (Partial_Reveal_Object**)ref; Partial_Reveal_Object* p_obj = *p_ref; /* we don't enumerate NULL reference and nos_boundary - FIXME:: nos_boundary is a static field in GCHelper.java for fast write barrier, not a real object reference + FIXME:: nos_boundary is a static field in GCHelper.java for fast write barrier, not a real object reference this should be fixed that magic Address field should not be enumerated. */ #ifdef COMPRESS_REFERENCE if (p_obj == (Partial_Reveal_Object*)HEAP_NULL || p_obj == NULL || p_obj == nos_boundary ) return; #else if (p_obj == NULL || p_obj == nos_boundary ) return; -#endif +#endif assert( !obj_is_marked_in_vt(p_obj)); assert( address_belongs_to_gc_heap(p_obj, p_global_gc)); gc_weak_rootset_add_entry(p_global_gc, p_ref, is_short_weak); Index: src/common/gc_metadata.cpp =================================================================== --- src/common/gc_metadata.cpp (revision 569956) +++ src/common/gc_metadata.cpp (working copy) @@ -76,6 +76,7 @@ gc_metadata.mutator_remset_pool = sync_pool_create(); gc_metadata.collector_remset_pool = sync_pool_create(); gc_metadata.collector_repset_pool = sync_pool_create(); + gc_metadata.dirty_obj_snaptshot_pool = sync_pool_create(); gc_metadata.weak_roots_pool = sync_pool_create(); #ifdef USE_32BITS_HASHCODE gc_metadata.collector_hashcode_pool = sync_pool_create(); @@ -98,6 +99,7 @@ sync_pool_destruct(metadata->mutator_remset_pool); sync_pool_destruct(metadata->collector_remset_pool); sync_pool_destruct(metadata->collector_repset_pool); + sync_pool_destruct(metadata->dirty_obj_snaptshot_pool); sync_pool_destruct(metadata->weak_roots_pool); #ifdef USE_32BITS_HASHCODE sync_pool_destruct(metadata->collector_hashcode_pool); @@ -234,7 +236,7 @@ update_rootset_interior_pointer(); /* it was pointing to the last root_set entry in gc_rootset_pool (before rem_sets). */ - gc->root_set = NULL; + //gc->root_set = NULL; return; } @@ -262,9 +264,9 @@ // gc->root_set = NULL; if(vector_block_is_empty(gc->weak_root_set)) - pool_put_entry(free_set_pool, gc->weak_root_set); - else - pool_put_entry(metadata->weak_roots_pool, gc->weak_root_set); + pool_put_entry(free_set_pool, gc->weak_root_set); + else + pool_put_entry(metadata->weak_roots_pool, gc->weak_root_set); gc->weak_root_set = NULL; if(!gc_is_gen_mode()) return; @@ -287,8 +289,9 @@ pool_put_entry(metadata->collector_remset_pool, collector->rem_set); collector->rem_set = NULL; } - - if( !gc_match_kind(gc, MINOR_COLLECTION )){ + + assert(gc_match_either_kind(gc, MINOR_COLLECTION|NORMAL_MAJOR_COLLECTION)); + if( gc_match_kind(gc, NORMAL_MAJOR_COLLECTION )){ /* all the remsets are useless now */ /* clean and put back mutator remsets */ root_set = pool_get_entry( mutator_remset_pool ); @@ -330,7 +333,8 @@ void gc_reset_rootset(GC* gc) { assert(pool_is_empty(gc_metadata.gc_rootset_pool)); - assert(gc->root_set == NULL); + ///TODO: check the statements below assert(gc->root_set == NULL); + if(gc->root_set != NULL) gc->root_set = NULL; gc->root_set = free_set_pool_get_entry(&gc_metadata); assert(vector_block_is_empty(gc->root_set)); @@ -349,6 +353,17 @@ return; } +void gc_clear_rootset(GC* gc) +{ + gc_reset_interior_pointer_table(); + gc_set_pool_clear(gc->metadata->gc_rootset_pool); +#ifdef COMPRESS_REFERENCE + gc_set_pool_clear(gc->metadata->gc_uncompressed_rootset_pool); +#endif + gc->root_set = NULL; +} + + void gc_clear_remset(GC* gc) { assert(gc->root_set != NULL); @@ -391,7 +406,63 @@ return; } +#ifdef _DEBUG +Boolean obj_is_mark_black_in_table(Partial_Reveal_Object* p_obj); +#endif +void gc_reset_snaptshot(GC* gc) +{ + GC_Metadata* metadata = gc->metadata; + /*reset mutator local snapshot block*/ + Mutator *mutator = gc->mutator_list; + while (mutator) { + Vector_Block* local_snapshot = mutator->dirty_obj_snapshot; + assert(local_snapshot); + if(!vector_block_is_empty(local_snapshot)){ +#ifdef _DEBUG + POINTER_SIZE_INT* iter = vector_block_iterator_init(local_snapshot); + while(!vector_block_iterator_end(local_snapshot,iter)){ + Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*) *iter; + iter = vector_block_iterator_advance(local_snapshot, iter); +#ifdef USE_MARK_SWEEP_GC + assert(obj_is_mark_black_in_table(p_obj)); +#endif + } +#endif + vector_block_clear(mutator->dirty_obj_snapshot); + } + mutator = mutator->next; + } + /*reset global snapshot pool*/ + Pool* global_snapshot = metadata->dirty_obj_snaptshot_pool; + if(!pool_is_empty(global_snapshot)){ + Vector_Block* snapshot_block = pool_get_entry(global_snapshot); + while(snapshot_block != NULL){ + if(!vector_block_is_empty(snapshot_block)){ +#ifdef _DEBUG + POINTER_SIZE_INT* iter = vector_block_iterator_init(snapshot_block); + while(!vector_block_iterator_end(snapshot_block,iter)){ + Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*) *iter; + iter = vector_block_iterator_advance(snapshot_block, iter); +#ifdef USE_MARK_SWEEP_GC + assert(obj_is_mark_black_in_table(p_obj)); +#endif + } +#endif + } + vector_block_clear(snapshot_block); + pool_put_entry(metadata->free_set_pool,snapshot_block); + snapshot_block = pool_get_entry(global_snapshot); + } + } + + + +} + + + + Index: src/common/gc_metadata.h =================================================================== --- src/common/gc_metadata.h (revision 569956) +++ src/common/gc_metadata.h (working copy) @@ -48,7 +48,9 @@ #ifdef USE_32BITS_HASHCODE Pool* collector_hashcode_pool; #endif - + + Pool* dirty_obj_snaptshot_pool; + }GC_Metadata; extern GC_Metadata gc_metadata; @@ -59,12 +61,16 @@ void gc_set_rootset(GC* gc); void gc_reset_rootset(GC* gc); +void gc_clear_rootset(GC* gc); void gc_fix_rootset(Collector* collector); +void gc_clear_remset(GC* gc); +void gc_reset_snaptshot(GC* gc); void identify_dead_weak_roots(GC *gc, Pool *pool); void gc_update_weak_roots_pool(GC *gc); void gc_clear_remset(GC* gc); + inline void gc_task_pool_clear(Pool* task_pool) { Vector_Block* task = pool_get_entry(task_pool); @@ -130,6 +136,23 @@ assert(mutator->rem_set); } +inline void mutator_snapshotset_add_entry(Mutator* mutator, Partial_Reveal_Object* p_obj) +{ + Vector_Block* dirty_obj_snapshot = mutator->dirty_obj_snapshot; + vector_block_add_entry(dirty_obj_snapshot, (POINTER_SIZE_INT)p_obj); + + if( !vector_block_is_full(dirty_obj_snapshot) ) return; + + vector_block_set_full(dirty_obj_snapshot); + + if(vector_block_set_exclusive(dirty_obj_snapshot)){ + //?vector_block_set_full(dirty_obj_snapshot); //ynhe + pool_put_entry(gc_metadata.dirty_obj_snaptshot_pool, dirty_obj_snapshot); + } + + mutator->dirty_obj_snapshot = free_set_pool_get_entry(&gc_metadata); +} + inline void collector_repset_add_entry(Collector* collector, Partial_Reveal_Object** p_ref) { // assert( p_ref >= gc_heap_base_address() && p_ref < gc_heap_ceiling_address()); Index: src/common/gc_platform.h =================================================================== --- src/common/gc_platform.h (revision 569956) +++ src/common/gc_platform.h (working copy) @@ -24,6 +24,7 @@ #include "port_vmem.h" #include "port_atomic.h" +#include "port_malloc.h" #include @@ -64,7 +65,19 @@ #define VmThreadHandle void* #define VmEventHandle hysem_t #define THREAD_OK TM_ERROR_NONE +#define THREAD_GROUP hythread_group_t +extern THREAD_GROUP gc_thread_group; + +inline THREAD_GROUP get_gc_thread_group () { + if (!gc_thread_group) { + IDATA UNUSED stat = hythread_group_create(&gc_thread_group); + assert(stat == TM_ERROR_NONE); + } + return gc_thread_group; +} + + inline int vm_wait_event(VmEventHandle event) { int stat = (int)hysem_wait(event); assert(stat == THREAD_OK); return stat; @@ -91,13 +104,12 @@ inline int vm_create_thread(int (*func)(void*), void *data) { - hythread_t* ret_thread = NULL; + hythread_t ret_thread = (hythread_t)STD_CALLOC(1,hythread_get_struct_size());; UDATA stacksize = 0; - UDATA priority = 0; - UDATA suspend = 0; + UDATA priority = 5; - return (int)hythread_create(ret_thread, stacksize, priority, suspend, - (hythread_entrypoint_t)func, data); + return (int)hythread_create_with_group(ret_thread, get_gc_thread_group(), stacksize, priority, + (hythread_entrypoint_t)func, data); } inline void *atomic_casptr(volatile void **mem, void *with, const void *cmp) Index: src/common/gc_space.h =================================================================== --- src/common/gc_space.h (revision 569956) +++ src/common/gc_space.h (working copy) @@ -26,6 +26,13 @@ extern unsigned int SPACE_ALLOC_UNIT; +typedef struct Space_Statistics{ + POINTER_SIZE_INT num_live_obj; + POINTER_SIZE_INT size_live_obj; + POINTER_SIZE_INT size_free_space; + POINTER_SIZE_INT last_size_free_space; +}Space_Statistics; + struct GC; /* all Spaces inherit this Space structure */ typedef struct Space{ @@ -40,6 +47,8 @@ GC* gc; Boolean move_object; + Space_Statistics* space_statistic; + /* Size allocted since last minor collection. */ volatile uint64 last_alloced_size; /* Size allocted since last major collection. */ @@ -53,6 +62,9 @@ uint64 period_surviving_size; }Space; +struct Allocator; +typedef void *(*Space_Alloc_Func)(unsigned, Allocator *); + inline POINTER_SIZE_INT space_committed_size(Space* space){ return space->committed_heap_size;} inline void* space_heap_start(Space* space){ return space->heap_start; } inline void* space_heap_end(Space* space){ return space->heap_end; } @@ -81,6 +93,8 @@ GC* gc; Boolean move_object; + Space_Statistics* space_statistic; + /* Size allocted since last minor collection. */ volatile uint64 last_alloced_size; /* Size allocted since last major collection. */ @@ -105,12 +119,14 @@ unsigned int num_used_blocks; unsigned int num_managed_blocks; unsigned int num_total_blocks; + + volatile Block_Header* block_iterator; /* END of Blocked_Space --> */ }Blocked_Space; -inline Boolean space_has_free_block(Blocked_Space* space){ return space->free_block_idx <= space->ceiling_block_idx; } -inline unsigned int space_free_memory_size(Blocked_Space* space){ return GC_BLOCK_SIZE_BYTES * (space->ceiling_block_idx - space->free_block_idx + 1); } -inline Boolean space_used_memory_size(Blocked_Space* space){ return GC_BLOCK_SIZE_BYTES * (space->free_block_idx - space->first_block_idx); } +inline Boolean blocked_space_has_free_block(Blocked_Space *space){ return space->free_block_idx <= space->ceiling_block_idx; } +inline unsigned int blocked_space_free_mem_size(Blocked_Space *space){ return GC_BLOCK_SIZE_BYTES * (space->ceiling_block_idx - space->free_block_idx + 1); } +inline Boolean blocked_space_used_mem_size(Blocked_Space *space){ return GC_BLOCK_SIZE_BYTES * (space->free_block_idx - space->first_block_idx); } inline void space_init_blocks(Blocked_Space* space) { @@ -198,4 +214,31 @@ space->num_managed_blocks = (unsigned int)(space->committed_heap_size >> GC_BLOCK_SHIFT_COUNT); } +inline void blocked_space_block_iterator_init(Blocked_Space *space) +{ space->block_iterator = (Block_Header*)space->blocks; } + +inline void blocked_space_block_iterator_init_free(Blocked_Space *space) +{ space->block_iterator = (Block_Header*)&space->blocks[space->free_block_idx - space->first_block_idx]; } + +inline Block_Header *blocked_space_block_iterator_get(Blocked_Space *space) +{ return (Block_Header*)space->block_iterator; } + +inline Block_Header *blocked_space_block_iterator_next(Blocked_Space *space) +{ + Block_Header *cur_block = (Block_Header*)space->block_iterator; + + while(cur_block != NULL){ + Block_Header *next_block = cur_block->next; + + Block_Header *temp = (Block_Header*)atomic_casptr((volatile void **)&space->block_iterator, next_block, cur_block); + if(temp != cur_block){ + cur_block = (Block_Header*)space->block_iterator; + continue; + } + return cur_block; + } + /* run out space blocks */ + return NULL; +} + #endif //#ifndef _GC_SPACE_H_ Index: src/common/object_status.h =================================================================== --- src/common/object_status.h (revision 0) +++ src/common/object_status.h (revision 0) @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _OBJECT_STATUS_H_ +#define _OBJECT_STATUS_H_ + +#include "../gen/gen.h" +#include "../mark_sweep/gc_ms.h" + + +inline Boolean obj_is_dead_in_gen_minor_gc(Partial_Reveal_Object *p_obj) +{ + /* + * The first condition is for supporting switch between nongen and gen minor collection. + * With this kind of switch dead objects in MOS & LOS may be set the mark or fw bit in oi. + * The second condition is for supporting partially forwarding NOS. + * In partially forwarding situation live objects in the non-forwarding half NOS will only be marked but not forwarded. + */ + return obj_belongs_to_nos(p_obj) && !obj_is_marked_or_fw_in_oi(p_obj); +} + +inline Boolean obj_is_dead_in_nongen_minor_gc(Partial_Reveal_Object *p_obj) +{ + return (obj_belongs_to_nos(p_obj) && !obj_is_fw_in_oi(p_obj)) + || (!obj_belongs_to_nos(p_obj) && !obj_is_marked_in_oi(p_obj)); +} + +extern Boolean obj_is_mark_black_in_table(Partial_Reveal_Object *obj); + +/* The caller must be in places where alloc color and mark color haven't been flipped */ +inline Boolean obj_is_dead_in_sweep_major_gc(Partial_Reveal_Object *p_obj) +{ + return (obj_belongs_to_nos(p_obj) && !obj_is_marked_in_vt(p_obj)) + || (!obj_belongs_to_nos(p_obj) && !obj_is_mark_black_in_table(p_obj)); +} + +inline Boolean obj_is_dead_in_compact_major_gc(Partial_Reveal_Object *p_obj) +{ + return !obj_is_marked_in_vt(p_obj); +} + +#ifdef USE_MARK_SWEEP_GC +inline Boolean obj_is_dead_in_mark_sweep_gc(Partial_Reveal_Object *p_obj) +{ + return !obj_is_mark_black_in_table(p_obj); +} +#endif + +inline Boolean gc_obj_is_dead(GC *gc, Partial_Reveal_Object *p_obj) +{ + assert(p_obj); + +#ifdef USE_MARK_SWEEP_GC + return obj_is_dead_in_mark_sweep_gc(p_obj); +#endif + + if(gc_match_kind(gc, MINOR_COLLECTION)){ + if(gc_is_gen_mode()) + return obj_is_dead_in_gen_minor_gc(p_obj); + else + return obj_is_dead_in_nongen_minor_gc(p_obj); + } else if(gc_get_mos((GC_Gen*)gc)->collect_algorithm == MAJOR_MARK_SWEEP){ + return obj_is_dead_in_sweep_major_gc(p_obj); + } else { + return obj_is_dead_in_compact_major_gc(p_obj); + } +} + +inline Boolean fspace_obj_to_be_forwarded(Partial_Reveal_Object *p_obj) +{ + if(!obj_belongs_to_nos(p_obj)) return FALSE; + return forward_first_half ? (p_obj < object_forwarding_boundary) : (p_obj>=object_forwarding_boundary); +} +inline Boolean obj_need_move(GC *gc, Partial_Reveal_Object *p_obj) +{ + /* assert(!gc_obj_is_dead(gc, p_obj)); commented out for weakroot */ + +#ifdef USE_MARK_SWEEP_GC + Sspace *sspace = gc_ms_get_sspace((GC_MS*)gc); + return sspace->move_object; +#endif + + if(gc_is_gen_mode() && gc_match_kind(gc, MINOR_COLLECTION)) + return fspace_obj_to_be_forwarded(p_obj); + + Space *space = space_of_addr(gc, p_obj); + return space->move_object; +} + + +#endif /* _OBJECT_STATUS_H_ */ Index: src/common/space_tuner.cpp =================================================================== --- src/common/space_tuner.cpp (revision 569956) +++ src/common/space_tuner.cpp (working copy) @@ -33,7 +33,7 @@ /* Calculate speed of allocation and waste memory of specific space respectively, * then decide whether to execute a space tuning according to the infomation.*/ -void gc_decide_space_tune(GC* gc, unsigned int cause) +static void gc_decide_space_tune(GC* gc) { Blocked_Space* mspace = (Blocked_Space*)gc_get_mos((GC_Gen*)gc); Blocked_Space* fspace = (Blocked_Space*)gc_get_nos((GC_Gen*)gc); @@ -94,11 +94,11 @@ /* The tuning size computing before marking is not precise. We only estimate the probable direction of space tuning. * If this function decide to set TRANS_NOTHING, then we just call the normal marking function. * Else, we call the marking function for space tuning. */ -void gc_compute_space_tune_size_before_marking(GC* gc, unsigned int cause) +void gc_compute_space_tune_size_before_marking(GC* gc) { if(gc_match_kind(gc, MINOR_COLLECTION)) return; - gc_decide_space_tune(gc, cause); + gc_decide_space_tune(gc); Space_Tuner* tuner = gc->tuner; assert((tuner->speed_los != 0) && ( tuner->speed_mos != 0)) ; @@ -440,7 +440,7 @@ void gc_space_tuner_reset(GC* gc) { Space_Tuner* tuner = gc->tuner; - if( !gc_match_kind(gc, MINOR_COLLECTION)){ + if( gc_match_kind(gc, MAJOR_COLLECTION)){ /*Clear the fields every major collection except the wast area statistic.*/ tuner->tuning_size = 0; tuner->interim_blocks = NULL; @@ -538,6 +538,3 @@ STD_FREE(tuner->interim_blocks); return; } - - - Index: src/common/space_tuner.h =================================================================== --- src/common/space_tuner.h (revision 569956) +++ src/common/space_tuner.h (working copy) @@ -75,11 +75,14 @@ POINTER_SIZE_INT min_tuning_size; }Space_Tuner; -void gc_compute_space_tune_size_before_marking(GC* gc, unsigned int cause); +void gc_compute_space_tune_size_before_marking(GC* gc); void gc_compute_space_tune_size_after_marking(GC *gc); void gc_space_tuner_reset(GC* gc); void gc_space_tuner_initialize(GC* gc); void gc_space_tuner_init_fake_blocks_for_los_shrink(GC* gc); void gc_space_tuner_release_fake_blocks_for_los_shrink(GC* gc); +inline Boolean gc_has_space_tuner(GC* gc) +{ return (Boolean)(POINTER_SIZE_INT)gc->tuner; } + #endif /* _SPACE_TUNER_H_ */ Index: src/common/weak_roots.cpp =================================================================== --- src/common/weak_roots.cpp (revision 569956) +++ src/common/weak_roots.cpp (working copy) @@ -16,7 +16,7 @@ #include "gc_common.h" #include "gc_metadata.h" -#include "../gen/gen.h" +#include "object_status.h" void identify_dead_weak_roots(GC *gc, Pool *pool) { Index: src/finalizer_weakref/finalizer_weakref.cpp =================================================================== --- src/finalizer_weakref/finalizer_weakref.cpp (revision 569956) +++ src/finalizer_weakref/finalizer_weakref.cpp (working copy) @@ -29,6 +29,8 @@ #include "../gen/gen.h" #include "../mark_sweep/gc_ms.h" #include "../common/space_tuner.h" +#include "../common/compressed_ref.h" +#include "../common/object_status.h" Boolean IGNORE_FINREF = FALSE; Boolean DURING_RESURRECTION = FALSE; @@ -102,7 +104,7 @@ } gc_put_finalizable_objects(gc); - if(gc_match_kind(gc, MAJOR_COLLECTION|FALLBACK_COLLECTION|SWEEP_COMPACT_GC)) + if(gc_match_either_kind(gc, MAJOR_COLLECTION|MS_COMPACT_COLLECTION)) finref_add_repset_from_pool(gc, obj_with_fin_pool); } @@ -133,9 +135,9 @@ trace_object = trace_obj_in_gen_fw; else trace_object = trace_obj_in_nongen_fw; - } else if(gc_match_kind(gc, MAJOR_COLLECTION)){ + } else if(gc_match_kind(gc, NORMAL_MAJOR_COLLECTION)){ p_ref_or_obj = p_obj; - if(gc->tuner->kind != TRANS_NOTHING){ + if(gc_has_space_tuner(gc) && (gc->tuner->kind != TRANS_NOTHING)){ trace_object = trace_obj_in_space_tune_marking; unsigned int obj_size = vm_object_size(p_obj); #ifdef USE_32BITS_HASHCODE @@ -147,13 +149,15 @@ } else { collector->los_live_obj_size += round_up_to_size(obj_size, KB); } - }else{ + } else if(gc_get_mos((GC_Gen*)gc)->collect_algorithm == MAJOR_MARK_SWEEP){ + trace_object = trace_obj_in_ms_marking; + } else { trace_object = trace_obj_in_normal_marking; } } else if(gc_match_kind(gc, FALLBACK_COLLECTION)){ trace_object = trace_obj_in_fallback_marking; } else { - assert(gc_match_kind(gc, MARK_SWEEP_GC|SWEEP_COMPACT_GC)); + assert(gc_match_kind(gc, MARK_SWEEP_GC)); p_ref_or_obj = p_obj; trace_object = trace_obj_in_ms_marking; } @@ -168,9 +172,8 @@ POINTER_SIZE_INT *iter = vector_block_iterator_init(task_block); while(!vector_block_iterator_end(task_block, iter)){ void *p_ref_or_obj = (void*)*iter; - assert((gc_match_kind(gc, MINOR_COLLECTION | FALLBACK_COLLECTION) && *(Partial_Reveal_Object **)p_ref_or_obj) - || (gc_match_kind(gc, MAJOR_COLLECTION) && p_ref_or_obj) - || (gc_match_kind(gc, MARK_SWEEP_GC|SWEEP_COMPACT_GC) && p_ref_or_obj)); + assert((gc_match_either_kind(gc, MINOR_COLLECTION|FALLBACK_COLLECTION) && *(Partial_Reveal_Object **)p_ref_or_obj) + || (gc_match_either_kind(gc, NORMAL_MAJOR_COLLECTION|MS_COLLECTION|MS_COMPACT_COLLECTION) && p_ref_or_obj)); trace_object(collector, p_ref_or_obj); if(collector->result == FALSE) break; /* Resurrection fallback happens; force return */ @@ -235,7 +238,7 @@ * Because it is outside heap, we can't update it in ref fixing. * In minor collection p_ref of the root dead obj is automatically updated while tracing. */ - if(gc_match_kind(gc, MAJOR_COLLECTION|FALLBACK_COLLECTION|SWEEP_COMPACT_GC)) + if(gc_match_either_kind(gc, MAJOR_COLLECTION|MS_COMPACT_COLLECTION)) finref_add_repset_from_pool(gc, finalizable_obj_pool); metadata->pending_finalizers = TRUE; @@ -246,7 +249,7 @@ static void identify_dead_refs(GC *gc, Pool *pool) { - if(gc_match_kind(gc, MAJOR_COLLECTION|FALLBACK_COLLECTION|SWEEP_COMPACT_GC)) + if(gc_match_either_kind(gc, MAJOR_COLLECTION|MS_COMPACT_COLLECTION)) finref_reset_repset(gc); pool_iterator_init(pool); Vector_Block *block = pool_iterator_next(pool); @@ -270,7 +273,7 @@ if(gc_match_kind(gc, MINOR_COLLECTION)){ assert(obj_is_fw_in_oi(p_referent)); write_slot(p_referent_field, (obj_get_fw_in_oi(p_referent))); - } else if(!gc_match_kind(gc, MARK_SWEEP_GC)){ + } else if(!gc_match_kind(gc, MS_COLLECTION)){ finref_repset_add_entry(gc, p_referent_field); } } @@ -283,7 +286,7 @@ block = pool_iterator_next(pool); } - if(gc_match_kind(gc, MAJOR_COLLECTION|FALLBACK_COLLECTION|SWEEP_COMPACT_GC)){ + if(gc_match_either_kind(gc, MAJOR_COLLECTION|MS_COMPACT_COLLECTION)){ finref_put_repset(gc); finref_add_repset_from_pool(gc, pool); } @@ -319,7 +322,7 @@ Finref_Metadata *metadata = gc->finref_metadata; Pool *phanref_pool = metadata->phanref_pool; - if(gc_match_kind(gc, MAJOR_COLLECTION|FALLBACK_COLLECTION|SWEEP_COMPACT_GC)) + if(gc_match_either_kind(gc, MAJOR_COLLECTION|MS_COMPACT_COLLECTION)) finref_reset_repset(gc); // collector_reset_repset(collector); pool_iterator_init(phanref_pool); @@ -344,7 +347,7 @@ if(gc_match_kind(gc, MINOR_COLLECTION)){ assert(obj_is_fw_in_oi(p_referent)); write_slot(p_referent_field, (obj_get_fw_in_oi(p_referent))); - } else if(!gc_match_kind(gc, MARK_SWEEP_GC)){ + } else if(!gc_match_kind(gc, MS_COLLECTION)){ finref_repset_add_entry(gc, p_referent_field); } *p_ref = (REF)NULL; @@ -362,7 +365,7 @@ block = pool_iterator_next(phanref_pool); } // collector_put_repset(collector); - if(gc_match_kind(gc, MAJOR_COLLECTION|FALLBACK_COLLECTION|SWEEP_COMPACT_GC)){ + if(gc_match_either_kind(gc, MAJOR_COLLECTION|MS_COMPACT_COLLECTION)){ finref_put_repset(gc); finref_add_repset_from_pool(gc, phanref_pool); } @@ -631,14 +634,14 @@ { Finref_Metadata *metadata = gc->finref_metadata; - if(gc_match_kind(gc, MAJOR_COLLECTION|FALLBACK_COLLECTION|SWEEP_COMPACT_GC)) + if(gc_match_either_kind(gc, MAJOR_COLLECTION|MS_COMPACT_COLLECTION)) finref_reset_repset(gc); - if(!gc_match_kind(gc, MARK_SWEEP_GC)){ + if(!gc_match_kind(gc, MS_COLLECTION)){ update_referent_field_ignore_finref(gc, metadata->softref_pool); update_referent_field_ignore_finref(gc, metadata->weakref_pool); update_referent_field_ignore_finref(gc, metadata->phanref_pool); } - if(gc_match_kind(gc, MAJOR_COLLECTION|FALLBACK_COLLECTION|SWEEP_COMPACT_GC)) + if(gc_match_either_kind(gc, MAJOR_COLLECTION|MS_COMPACT_COLLECTION)) finref_put_repset(gc); } @@ -667,7 +670,11 @@ *p_ref = obj_get_fw_in_table(p_obj); } -static inline void sweep_compaction_update_ref(GC *gc, REF *p_ref) +/* In two cases mark-sweep needs fixing repointed refs: + * 1. ms with compaction + * 2. ms as a mos collection algorithm + */ +static inline void moving_mark_sweep_update_ref(GC *gc, REF *p_ref) { /* There are only two kinds of p_ref being added into finref_repset_pool: * 1. p_ref is in a vector block from one finref pool; @@ -696,40 +703,6 @@ extern Boolean IS_MOVE_COMPACT; /* parameter pointer_addr_in_pool means it is p_ref or p_obj in pool */ -static void destructively_fix_finref_pool(GC *gc, Pool *pool, Boolean pointer_addr_in_pool) -{ - Finref_Metadata *metadata = gc->finref_metadata; - REF *p_ref; - Partial_Reveal_Object *p_obj; - - /* NOTE:: this is destructive to the root sets. */ - Vector_Block *repset = pool_get_entry(pool); - while(repset){ - POINTER_SIZE_INT *iter = vector_block_iterator_init(repset); - for(; !vector_block_iterator_end(repset,iter); iter = vector_block_iterator_advance(repset,iter)){ - if(pointer_addr_in_pool) - p_ref = (REF*)*iter; - else - p_ref = (REF*)iter; - p_obj = read_slot(p_ref); - - if(IS_MOVE_COMPACT){ - move_compaction_update_ref(gc, p_ref); - } else if(gc_match_kind(gc, SWEEP_COMPACT_GC)){ - if(obj_is_fw_in_oi(p_obj)) - sweep_compaction_update_ref(gc, p_ref); - } else { - assert((obj_is_marked_in_vt(p_obj) && obj_is_fw_in_oi(p_obj))); - write_slot(p_ref , obj_get_fw_in_oi(p_obj)); - } - } - vector_block_clear(repset); - pool_put_entry(metadata->free_pool, repset); - repset = pool_get_entry(pool); - } -} - -/* parameter pointer_addr_in_pool means it is p_ref or p_obj in pool */ static void nondestructively_fix_finref_pool(GC *gc, Pool *pool, Boolean pointer_addr_in_pool) { Finref_Metadata *metadata = gc->finref_metadata; @@ -748,11 +721,11 @@ p_ref = (REF*)iter; p_obj = read_slot(p_ref); - if(!IS_MOVE_COMPACT){ + if(IS_MOVE_COMPACT){ move_compaction_update_ref(gc, p_ref); - } else if(gc_match_kind(gc, SWEEP_COMPACT_GC)){ + } else if(gc_match_kind(gc, MS_COMPACT_COLLECTION) || gc_get_mos((GC_Gen*)gc)->collect_algorithm==MAJOR_MARK_SWEEP){ if(obj_is_fw_in_oi(p_obj)) - sweep_compaction_update_ref(gc, p_ref); + moving_mark_sweep_update_ref(gc, p_ref); } else { assert((obj_is_marked_in_vt(p_obj) && obj_is_fw_in_oi(p_obj))); write_slot(p_ref , obj_get_fw_in_oi(p_obj)); @@ -770,7 +743,7 @@ Pool *repset_pool = metadata->repset_pool; Pool *fallback_ref_pool = metadata->fallback_ref_pool; - destructively_fix_finref_pool(gc, repset_pool, TRUE); + nondestructively_fix_finref_pool(gc, repset_pool, TRUE); if(!pool_is_empty(fallback_ref_pool)){ assert(IS_FALLBACK_COMPACTION); nondestructively_fix_finref_pool(gc, fallback_ref_pool, FALSE); @@ -787,3 +760,26 @@ vm_hint_finalize(); } } + +static void finref_copy_pool_to_rootset(GC *gc, Pool *src_pool) +{ + pool_iterator_init(src_pool); + while(Vector_Block *root_set = pool_iterator_next(src_pool)){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); + while(!vector_block_iterator_end(root_set, iter)){ + gc_compressed_rootset_add_entry(gc, (REF*)iter); + iter = vector_block_iterator_advance(root_set, iter); + } + } +} + +void gc_copy_finaliable_obj_to_rootset(GC *gc) +{ + Pool *finalizable_obj_pool = gc->finref_metadata->finalizable_obj_pool; + Pool *finalizable_obj_pool_copy = gc->finref_metadata->finalizable_obj_pool_copy; + Pool *free_pool = gc->metadata->gc_rootset_pool; + finref_metadata_clear_pool(finalizable_obj_pool_copy); + finref_copy_pool(finalizable_obj_pool, finalizable_obj_pool_copy, gc); + finref_copy_pool_to_rootset(gc, finalizable_obj_pool_copy); +} + Index: src/finalizer_weakref/finalizer_weakref.h =================================================================== --- src/finalizer_weakref/finalizer_weakref.h (revision 569956) +++ src/finalizer_weakref/finalizer_weakref.h (working copy) @@ -93,7 +93,9 @@ extern void gc_put_finref_to_vm(GC *gc); extern void put_all_fin_on_exit(GC *gc); -extern void gc_update_finref_repointed_refs(GC* gc); +extern void gc_update_finref_repointed_refs(GC *gc); extern void gc_activate_finref_threads(GC *gc); +void gc_copy_finaliable_obj_to_rootset(GC *gc); + #endif // _FINREF_H_ Index: src/finalizer_weakref/finalizer_weakref_metadata.cpp =================================================================== --- src/finalizer_weakref/finalizer_weakref_metadata.cpp (revision 569956) +++ src/finalizer_weakref/finalizer_weakref_metadata.cpp (working copy) @@ -56,6 +56,7 @@ finref_metadata.free_pool = sync_pool_create(); finref_metadata.obj_with_fin_pool = sync_pool_create(); finref_metadata.finalizable_obj_pool = sync_pool_create(); + finref_metadata.finalizable_obj_pool_copy = sync_pool_create(); finref_metadata.softref_pool = sync_pool_create(); finref_metadata.weakref_pool = sync_pool_create(); finref_metadata.phanref_pool = sync_pool_create(); @@ -89,6 +90,7 @@ sync_pool_destruct(metadata->free_pool); sync_pool_destruct(metadata->obj_with_fin_pool); sync_pool_destruct(metadata->finalizable_obj_pool); + sync_pool_destruct(metadata->finalizable_obj_pool_copy); sync_pool_destruct(metadata->softref_pool); sync_pool_destruct(metadata->weakref_pool); sync_pool_destruct(metadata->phanref_pool); @@ -204,6 +206,7 @@ collector->phanref_set= finref_get_free_block(gc); } +#include "../common/gc_concurrent.h" /* put back last weak references block of each collector */ void gc_set_weakref_sets(GC *gc) { @@ -220,6 +223,20 @@ collector->weakref_set= NULL; collector->phanref_set= NULL; } + + if(gc_mark_is_concurrent()){ + unsigned int num_active_markers = gc->num_active_markers; + for(unsigned int i = 0; i < num_active_markers; i++) + { + Collector *marker = (Collector*)gc->markers[i]; + pool_put_entry(metadata->softref_pool, marker->softref_set); + pool_put_entry(metadata->weakref_pool, marker->weakref_set); + pool_put_entry(metadata->phanref_pool, marker->phanref_set); + marker->softref_set = NULL; + marker->weakref_set= NULL; + marker->phanref_set= NULL; + } + } return; } @@ -233,10 +250,14 @@ assert(pool_is_empty(metadata->softref_pool)); assert(pool_is_empty(metadata->weakref_pool)); assert(pool_is_empty(metadata->phanref_pool)); - assert(pool_is_empty(metadata->repset_pool)); assert(metadata->finalizable_obj_set == NULL); assert(metadata->repset == NULL); + if(!pool_is_empty(metadata->repset_pool)) + finref_metadata_clear_pool(metadata->repset_pool); + if(!pool_is_empty(metadata->fallback_ref_pool)) + finref_metadata_clear_pool(metadata->fallback_ref_pool); + /* Extract empty blocks in obj_with_fin_pool and put them into free_pool */ Vector_Block *block = pool_get_entry(obj_with_fin_pool); while(block){ @@ -378,10 +399,9 @@ return pool_has_no_ref(gc->finref_metadata->repset_pool); } -static void finref_metadata_clear_pool(Pool *pool) +void finref_metadata_clear_pool(Pool *pool) { - while(Vector_Block* block = pool_get_entry(pool)) - { + while(Vector_Block* block = pool_get_entry(pool)){ vector_block_clear(block); pool_put_entry(finref_metadata.free_pool, block); } @@ -393,3 +413,19 @@ finref_metadata_clear_pool(gc->finref_metadata->weakref_pool); finref_metadata_clear_pool(gc->finref_metadata->phanref_pool); } + +Boolean finref_copy_pool(Pool *src_pool, Pool *dest_pool, GC *gc) +{ + Vector_Block *dest_block = finref_get_free_block(gc); + pool_iterator_init(src_pool); + while(Vector_Block *src_block = pool_iterator_next(src_pool)){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(src_block); + while(!vector_block_iterator_end(src_block, iter)){ + assert(*iter); + finref_metadata_add_entry(gc, dest_block, dest_pool, *iter); + iter = vector_block_iterator_advance(src_block, iter); + } + } + return TRUE; +} + Index: src/finalizer_weakref/finalizer_weakref_metadata.h =================================================================== --- src/finalizer_weakref/finalizer_weakref_metadata.h (revision 569956) +++ src/finalizer_weakref/finalizer_weakref_metadata.h (working copy) @@ -38,6 +38,8 @@ Pool *obj_with_fin_pool; // list of objects that have finalizers // these objects are added in when they are being allocated Pool *finalizable_obj_pool; // temporary buffer for finalizable objects identified during one single GC + + Pool *finalizable_obj_pool_copy; Pool *softref_pool; // temporary buffer for soft references identified during one single GC Pool *weakref_pool; // temporary buffer for weak references identified during one single GC @@ -84,6 +86,10 @@ extern Boolean phanref_pool_is_empty(GC *gc); extern Boolean finref_repset_pool_is_empty(GC *gc); +extern void finref_metadata_clear_pool(Pool *pool); +extern Boolean finref_copy_pool(Pool* src_pool, Pool* dest_pool, GC* gc); + + extern void gc_clear_weakref_pools(GC *gc); extern Vector_Block *finref_metadata_extend(void); Index: src/gen/gc_for_barrier.cpp =================================================================== --- src/gen/gc_for_barrier.cpp (revision 569956) +++ src/gen/gc_for_barrier.cpp (working copy) @@ -22,7 +22,10 @@ #include "../gen/gen.h" #include "../thread/mutator.h" #include "gc_for_barrier.h" +#include "../mark_sweep/sspace_mark_sweep.h" +#include "../common/gc_concurrent.h" + /* All the write barrier interfaces need cleanup */ Boolean gen_mode; @@ -81,6 +84,8 @@ void gc_heap_wrote_object (Managed_Object_Handle p_obj_written) { + /*concurrent mark: since object clone and array copy do not modify object slot, + we treat it as an new object. It has already been marked when dest object was created.*/ if( !gc_is_gen_mode() ) return; if( object_has_ref_field((Partial_Reveal_Object*)p_obj_written)){ /* for array copy and object clone */ @@ -92,18 +97,66 @@ void gc_heap_write_ref (Managed_Object_Handle p_obj_holding_ref, unsigned offset, Managed_Object_Handle p_target) { assert(0); } +/*This function is for concurrent mark.*/ +static void gc_dirty_object_write_barrier(Managed_Object_Handle p_obj_holding_ref) +{ + Mutator *mutator = (Mutator *)gc_get_tls(); + REF* p_obj_slot; + if(obj_need_take_snaptshot((Partial_Reveal_Object*)p_obj_holding_ref)){ + if (object_is_array((Partial_Reveal_Object*)p_obj_holding_ref)) { + Partial_Reveal_Object* array = (Partial_Reveal_Object*)p_obj_holding_ref; + assert(!obj_is_primitive_array(array)); + + Partial_Reveal_Object* obj_to_snapshot; + + int32 array_length = vector_get_length((Vector_Handle) array); + for (int i = 0; i < array_length; i++) { + p_obj_slot = (REF*)vector_get_element_address_ref((Vector_Handle) array, i); + obj_to_snapshot = (Partial_Reveal_Object*)read_slot(p_obj_slot); + if (obj_to_snapshot != NULL) + mutator_snapshotset_add_entry(mutator, obj_to_snapshot); + } + }else{ + /* scan non-array object */ + Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*)p_obj_holding_ref; + unsigned int num_refs = object_ref_field_num(p_obj); + int *ref_iterator = object_ref_iterator_init(p_obj); + + Partial_Reveal_Object* obj_to_snapshot; + + for(unsigned int i=0; i_num_processors; + + POINTER_SIZE_INT min_nos_size_threshold = min_heap_size>>5; + if(min_nos_size_bytes > min_nos_size_threshold) + min_nos_size_bytes = round_down_to_size(min_nos_size_threshold, SPACE_ALLOC_UNIT); + + if(MIN_NOS_SIZE) min_nos_size_bytes = MIN_NOS_SIZE; +} +static POINTER_SIZE_INT determine_los_size(POINTER_SIZE_INT min_heap_size) +{ + POINTER_SIZE_INT los_size = min_heap_size >> 7; + if(INIT_LOS_SIZE) los_size = INIT_LOS_SIZE; + if(los_size < min_los_size_bytes ) + los_size = min_los_size_bytes; + + los_size = round_down_to_size(los_size, SPACE_ALLOC_UNIT); + return los_size; +} + +void *alloc_large_pages(size_t size, const char *hint); + void gc_gen_initial_verbose_info(GC_Gen *gc); -void gc_gen_initialize(GC_Gen *gc_gen, POINTER_SIZE_INT min_heap_size, POINTER_SIZE_INT max_heap_size) +void gc_gen_initialize(GC_Gen *gc_gen, POINTER_SIZE_INT min_heap_size, POINTER_SIZE_INT max_heap_size) { TRACE2("gc.process", "GC: GC_Gen heap init ... \n"); assert(gc_gen); - + max_heap_size = round_down_to_size(max_heap_size, SPACE_ALLOC_UNIT); min_heap_size = round_up_to_size(min_heap_size, SPACE_ALLOC_UNIT); assert(max_heap_size <= max_heap_size_bytes); assert(max_heap_size >= min_heap_size_bytes); - - min_nos_size_bytes *= gc_gen->_num_processors; - - POINTER_SIZE_INT min_nos_size_threshold = min_heap_size>>5; - if(min_nos_size_bytes > min_nos_size_threshold){ - min_nos_size_bytes = round_down_to_size(min_nos_size_threshold,SPACE_ALLOC_UNIT); - } - if( MIN_NOS_SIZE ) min_nos_size_bytes = MIN_NOS_SIZE; - - POINTER_SIZE_INT los_size = min_heap_size >> 7; - if(INIT_LOS_SIZE) los_size = INIT_LOS_SIZE; - if(los_size < min_los_size_bytes ) - los_size = min_los_size_bytes ; + determine_min_nos_size(gc_gen, min_heap_size); - los_size = round_down_to_size(los_size, SPACE_ALLOC_UNIT); - + POINTER_SIZE_INT los_size = 0; + if(MAJOR_ALGO == MAJOR_MARK_SWEEP) + min_los_size_bytes = 0; + else + los_size = determine_los_size(min_heap_size); + /* let's compute and reserve the space for committing */ /* heuristic nos + mos + LOS = max, and nos*ratio = mos */ - POINTER_SIZE_INT nos_reserve_size, nos_commit_size; - POINTER_SIZE_INT mos_reserve_size, mos_commit_size; - POINTER_SIZE_INT los_mos_size; + POINTER_SIZE_INT nos_reserve_size, nos_commit_size; + POINTER_SIZE_INT mos_reserve_size, mos_commit_size; + POINTER_SIZE_INT los_mos_reserve_size; - /*Give GC a hint of gc survive ratio. And the last_survive_ratio field is used in heap size adjustment*/ + /* Give GC a hint of gc survive ratio. And the last_survive_ratio field is used in heap size adjustment */ gc_gen->survive_ratio = 0.2f; - + + los_mos_reserve_size = max_heap_size - NOS_SIZE; + mos_reserve_size = los_mos_reserve_size - min_los_size_bytes; if(NOS_SIZE){ - los_mos_size = min_heap_size - NOS_SIZE; - mos_reserve_size = los_mos_size - los_size; - - nos_commit_size = NOS_SIZE; - nos_reserve_size = NOS_SIZE; - - }else{ - los_mos_size = min_heap_size; - mos_reserve_size = max_heap_size_bytes - min_los_size_bytes; + nos_reserve_size = nos_commit_size = NOS_SIZE; + } else { + nos_reserve_size = mos_reserve_size; nos_commit_size = (POINTER_SIZE_INT)(((float)(min_heap_size - los_size))/(1.0f + gc_gen->survive_ratio)); - nos_reserve_size = mos_reserve_size; } - - nos_commit_size = round_down_to_size(nos_commit_size, SPACE_ALLOC_UNIT); + nos_commit_size = round_down_to_size(nos_commit_size, SPACE_ALLOC_UNIT); mos_commit_size = min_heap_size - los_size - nos_commit_size; + + /* Reserve memory for spaces of gc_gen */ + void *reserved_base; + void *reserved_end; + void *nos_base; - /* allocate memory for gc_gen */ - void* reserved_base; - void* reserved_end; - void* nos_base; - #ifdef STATIC_NOS_MAPPING - //FIXME: no large page support in static nos mapping + //FIXME:: no large page support in static nos mapping assert(large_page_hint==NULL); - assert((POINTER_SIZE_INT)nos_boundary%SPACE_ALLOC_UNIT == 0); + assert(!((POINTER_SIZE_INT)nos_boundary % SPACE_ALLOC_UNIT)); nos_base = vm_reserve_mem(nos_boundary, nos_reserve_size); if( nos_base != nos_boundary ){ DIE2("gc.base","Warning: Static NOS mapping: Can't reserve memory at address"<= nos_base){ los_mos_base = (void*)((POINTER_SIZE_INT)los_mos_base - SPACE_ALLOC_UNIT); if(los_mos_base < RESERVE_BOTTOM){ DIE2("gc.base","Static NOS mapping: Can't reserve memory at address"<reserved_heap_size = los_size + nos_reserve_size + mos_reserve_size; + + gc_gen->heap_start = reserved_base; + gc_gen->heap_end = reserved_end; +#ifdef STATIC_NOS_MAPPING + gc_gen->reserved_heap_size = los_mos_reserve_size + nos_reserve_size; #else - gc_gen->reserved_heap_size = max_heap_size_bytes; + gc_gen->reserved_heap_size = max_heap_size; #endif - gc_gen->heap_start = reserved_base; - gc_gen->heap_end = reserved_end; - gc_gen->blocks = (Block*)reserved_base; + /* Commented out for that the frontmost reserved mem size in los is not counted in los' committed size. + * gc_gen->committed_heap_size = min_heap_size; + */ gc_gen->num_collections = 0; gc_gen->time_collections = 0; + gc_gen->blocks = (Block*)reserved_base; gc_gen->force_major_collect = FALSE; gc_gen->force_gen_mode = FALSE; gc_los_initialize(gc_gen, reserved_base, los_size); - - reserved_base = (void*)((POINTER_SIZE_INT)reserved_base + los_size); - gc_mos_initialize(gc_gen, reserved_base, mos_reserve_size, mos_commit_size); - - gc_nos_initialize(gc_gen, nos_base, nos_reserve_size, nos_commit_size); - - /* connect mos and nos, so that they can be compacted as one space */ - Blocked_Space* mos = (Blocked_Space*)gc_get_mos(gc_gen); - Blocked_Space* nos = (Blocked_Space*)gc_get_nos(gc_gen); - Block_Header* mos_last_block = (Block_Header*)&mos->blocks[mos->num_managed_blocks-1]; - Block_Header* nos_first_block = (Block_Header*)&nos->blocks[0]; - mos_last_block->next = nos_first_block; + gc_mos_initialize(gc_gen, (void*)((POINTER_SIZE_INT)reserved_base + los_size), mos_reserve_size, mos_commit_size); + gc_nos_initialize(gc_gen, nos_base, nos_reserve_size, nos_commit_size); - nos->collect_algorithm = MINOR_ALGO; - mos->collect_algorithm = MAJOR_ALGO; - - gc_space_tuner_initialize((GC*)gc_gen); - - gc_gen_mode_adapt_init(gc_gen); + gc_gen->committed_heap_size = space_committed_size(gc_get_nos(gc_gen)) + + space_committed_size(gc_get_mos(gc_gen)) + + space_committed_size(gc_get_los(gc_gen)); + + if(MAJOR_ALGO != MAJOR_MARK_SWEEP){ + Blocked_Space *nos = (Blocked_Space*)gc_get_nos(gc_gen); + Blocked_Space *mos = (Blocked_Space*)gc_get_mos(gc_gen); + /* Connect mos and nos, so that they can be compacted as one space */ + Block_Header *mos_last_block = (Block_Header*)&mos->blocks[mos->num_managed_blocks-1]; + Block_Header *nos_first_block = (Block_Header*)&nos->blocks[0]; + mos_last_block->next = nos_first_block; - gc_gen->committed_heap_size = space_committed_size((Space*)gc_gen->nos) + - space_committed_size((Space*)gc_gen->mos) + - space_committed_size((Space*)gc_gen->los); + gc_space_tuner_initialize((GC*)gc_gen); + gc_gen_mode_adapt_init(gc_gen); + } #ifdef GC_GEN_STATS gc_gen_stats_initialize(gc_gen); @@ -235,33 +239,27 @@ return; } -void gc_gen_destruct(GC_Gen *gc_gen) +void gc_gen_destruct(GC_Gen *gc_gen) { TRACE2("gc.process", "GC: GC_Gen heap destruct ......"); - Space* nos = (Space*)gc_gen->nos; - Space* mos = (Space*)gc_gen->mos; - Space* los = (Space*)gc_gen->los; - - POINTER_SIZE_INT nos_size = space_committed_size(nos); - POINTER_SIZE_INT mos_size = space_committed_size(mos); - POINTER_SIZE_INT los_size = space_committed_size(los); - - void* nos_start = nos->heap_start; - void* mos_start = mos->heap_start; - void* los_start = los->heap_start; + Space *nos = gc_gen->nos; gc_nos_destruct(gc_gen); + vm_unmap_mem(nos->heap_start, space_committed_size(nos)); gc_gen->nos = NULL; - gc_mos_destruct(gc_gen); + Space *mos = gc_gen->mos; + gc_mos_destruct(gc_gen); + vm_unmap_mem(mos->heap_start, space_committed_size(mos)); gc_gen->mos = NULL; + + if(MAJOR_ALGO != MAJOR_MARK_SWEEP){ + Space *los = gc_gen->los; + gc_los_destruct(gc_gen); + vm_unmap_mem(los->heap_start, space_committed_size(los)); + gc_gen->los = NULL; + } - gc_los_destruct(gc_gen); - gc_gen->los = NULL; - - vm_unmap_mem(nos_start, nos_size); - vm_unmap_mem(mos_start, mos_size); - vm_unmap_mem(los_start, los_size); #ifdef GC_GEN_STATS gc_gen_stats_destruct(gc_gen); #endif @@ -269,31 +267,94 @@ return; } -Space* gc_get_nos(GC_Gen* gc){ return (Space*)gc->nos;} -Space* gc_get_mos(GC_Gen* gc){ return (Space*)gc->mos;} -Space* gc_get_los(GC_Gen* gc){ return (Space*)gc->los;} +Space *gc_get_nos(GC_Gen *gc){ return gc->nos; } +Space *gc_get_mos(GC_Gen *gc){ return gc->mos; } +Space *gc_get_los(GC_Gen *gc){ return gc->los; } -void gc_set_nos(GC_Gen* gc, Space* nos){ gc->nos = (Fspace*)nos;} -void gc_set_mos(GC_Gen* gc, Space* mos){ gc->mos = (Mspace*)mos;} -void gc_set_los(GC_Gen* gc, Space* los){ gc->los = (Lspace*)los;} +void gc_set_nos(GC_Gen *gc, Space *nos){ gc->nos = nos; } +void gc_set_mos(GC_Gen *gc, Space *mos){ gc->mos = mos; } +void gc_set_los(GC_Gen *gc, Space *los){ gc->los = los; } -void* mos_alloc(unsigned size, Allocator *allocator){return mspace_alloc(size, allocator);} +Space_Alloc_Func mos_alloc; +//void* mos_alloc(unsigned size, Allocator *allocator){return mspace_alloc(size, allocator);} void* nos_alloc(unsigned size, Allocator *allocator){return fspace_alloc(size, allocator);} -void* los_alloc(unsigned size, Allocator *allocator){return lspace_alloc(size, allocator);} +Space_Alloc_Func los_alloc; +//void* los_alloc(unsigned size, Allocator *allocator){return lspace_alloc(size, allocator);} void* los_try_alloc(POINTER_SIZE_INT size, GC* gc){ return lspace_try_alloc((Lspace*)((GC_Gen*)gc)->los, size); } +void gc_nos_initialize(GC_Gen *gc, void *start, POINTER_SIZE_INT nos_size, POINTER_SIZE_INT commit_size) +{ + Space *nos = (Space*)fspace_initialize((GC*)gc, start, nos_size, commit_size); + gc_set_nos(gc, nos); + nos->collect_algorithm = MINOR_ALGO; +} +void gc_nos_destruct(GC_Gen *gc) +{ fspace_destruct((Fspace*)gc->nos); } + +void gc_mos_initialize(GC_Gen *gc, void *start, POINTER_SIZE_INT mos_size, POINTER_SIZE_INT commit_size) +{ + Space *mos; + if(MAJOR_ALGO == MAJOR_MARK_SWEEP){ + mos = (Space*)sspace_initialize((GC*)gc, start, mos_size, commit_size); + mos_alloc = sspace_alloc; + } else { + mos = (Space*)mspace_initialize((GC*)gc, start, mos_size, commit_size); + mos_alloc = mspace_alloc; + } + gc_set_mos(gc, mos); + mos->collect_algorithm = MAJOR_ALGO; +} + +void gc_mos_destruct(GC_Gen *gc) +{ + if(MAJOR_ALGO == MAJOR_MARK_SWEEP) + sspace_destruct((Sspace*)gc->mos); + else + mspace_destruct((Mspace*)gc->mos); +} + +void gc_los_initialize(GC_Gen *gc, void *start, POINTER_SIZE_INT los_size) +{ + Space *los; + if(MAJOR_ALGO == MAJOR_MARK_SWEEP){ + assert(los_size == 0); + los = NULL; + los_alloc = sspace_alloc; + } else { + los = (Space*)lspace_initialize((GC*)gc, start, los_size); + los_alloc = lspace_alloc; + } + gc_set_los(gc, los); +} + +void gc_los_destruct(GC_Gen *gc) +{ + if(MAJOR_ALGO != MAJOR_MARK_SWEEP) + lspace_destruct((Lspace*)gc->los); +} + + Boolean FORCE_FULL_COMPACT = FALSE; Boolean IGNORE_VTABLE_TRACING = TRUE; -Boolean VTABLE_TRACING = FALSE; +Boolean VTABLE_TRACING = FALSE; +unsigned int gc_next_collection_kind(GC_Gen* gc) +{ + if(gc->force_major_collect || FORCE_FULL_COMPACT) + return MAJOR_COLLECTION; + else + return MINOR_COLLECTION; +} + + void gc_decide_collection_kind(GC_Gen* gc, unsigned int cause) { /* this is for debugging. */ gc->last_collect_kind = gc->collect_kind; if(gc->force_major_collect || cause== GC_CAUSE_LOS_IS_FULL || FORCE_FULL_COMPACT) - gc->collect_kind = MAJOR_COLLECTION; + gc->collect_kind = NORMAL_MAJOR_COLLECTION; else gc->collect_kind = MINOR_COLLECTION; @@ -303,7 +364,7 @@ VTABLE_TRACING = TRUE; #ifdef USE_MARK_SWEEP_GC - gc->collect_kind = MARK_SWEEP_GC; + gc->collect_kind = MS_COLLECTION; #endif return; } @@ -332,17 +393,20 @@ } if(!major_algo){ - MAJOR_ALGO= MAJOR_COMPACT_MOVE; + MAJOR_ALGO = MAJOR_COMPACT_MOVE; }else{ string_to_upper(major_algo); if(!strcmp(major_algo, "MAJOR_COMPACT_SLIDE")){ - MAJOR_ALGO= MAJOR_COMPACT_SLIDE; + MAJOR_ALGO = MAJOR_COMPACT_SLIDE; }else if(!strcmp(major_algo, "MAJOR_COMPACT_MOVE")){ - MAJOR_ALGO= MAJOR_COMPACT_MOVE; + MAJOR_ALGO = MAJOR_COMPACT_MOVE; + }else if(!strcmp(major_algo, "MAJOR_MARK_SWEEP")){ + MAJOR_ALGO = MAJOR_MARK_SWEEP; + }else{ WARN2("gc.base","\nWarning: GC algorithm setting incorrect. Will use default value.\n"); @@ -356,7 +420,7 @@ void gc_gen_assign_free_area_to_mutators(GC_Gen* gc) { if(gc->cause == GC_CAUSE_LOS_IS_FULL){ - Lspace* los = gc->los; + Lspace* los = (Lspace*)gc->los; los->success_ptr = los_try_alloc(los->failure_size, (GC*)gc); los->failure_size = 0; @@ -372,14 +436,14 @@ return; } -void gc_gen_adjust_heap_size(GC_Gen* gc, int64 pause_time) +static void gc_gen_adjust_heap_size(GC_Gen* gc) { if(gc_match_kind((GC*)gc, MINOR_COLLECTION)) return; if(gc->committed_heap_size == max_heap_size_bytes - LOS_HEAD_RESERVE_FOR_HEAP_NULL) return; - Mspace* mos = gc->mos; - Fspace* nos = gc->nos; - Lspace* los = gc->los; + Mspace* mos = (Mspace*)gc->mos; + Fspace* nos = (Fspace*)gc->nos; + Lspace* los = (Lspace*)gc->los; /* We can not tolerate gc->survive_ratio be greater than threshold twice continuously. * Or, we must adjust heap size */ @@ -393,18 +457,20 @@ assert(heap_total_size > heap_surviving_size); float heap_survive_ratio = (float)heap_surviving_size / (float)heap_total_size; + float non_los_survive_ratio = (float)mos->period_surviving_size / (float)(mos->committed_heap_size + nos->committed_heap_size); float threshold_survive_ratio = 0.3f; float regular_survive_ratio = 0.125f; POINTER_SIZE_INT new_heap_total_size = 0; POINTER_SIZE_INT adjust_size = 0; - if(heap_survive_ratio < threshold_survive_ratio) return; + if( (heap_survive_ratio < threshold_survive_ratio) && (non_los_survive_ratio < threshold_survive_ratio) )return; if(++tolerate < 2) return; tolerate = 0; - new_heap_total_size = (POINTER_SIZE_INT)((float)heap_surviving_size / regular_survive_ratio); + new_heap_total_size = max((POINTER_SIZE_INT)((float)heap_surviving_size / regular_survive_ratio), + (POINTER_SIZE_INT)((float)mos->period_surviving_size / regular_survive_ratio + los->committed_heap_size)); new_heap_total_size = round_down_to_size(new_heap_total_size, SPACE_ALLOC_UNIT); @@ -469,25 +535,119 @@ } +void gc_gen_start_concurrent_mark(GC_Gen* gc) +{ + assert(0); +} + +static inline void nos_collection(Space *nos) +{ fspace_collection((Fspace*)nos); } + +static inline void mos_collection(Space *mos) +{ + if(MAJOR_ALGO == MAJOR_MARK_SWEEP) + sspace_collection((Sspace*)mos); + else + mspace_collection((Mspace*)mos); +} + +static inline void los_collection(Space *los) +{ + if(MAJOR_ALGO != MAJOR_MARK_SWEEP) + lspace_collection((Lspace*)los); +} + +static void gc_gen_update_space_info_before_gc(GC_Gen *gc) +{ + Fspace *nos = (Fspace*)gc->nos; + Mspace *mos = (Mspace*)gc->mos; + Lspace *los = (Lspace*)gc->los; + + /* Update before every GC to avoid the atomic operation in every fspace_alloc_block */ + assert( nos->free_block_idx >= nos->first_block_idx ); + nos->num_used_blocks = nos->free_block_idx - nos->first_block_idx; + nos->last_alloced_size = GC_BLOCK_SIZE_BYTES * nos->num_used_blocks; + nos->accumu_alloced_size += nos->last_alloced_size; + + mos->num_used_blocks = mos->free_block_idx - mos->first_block_idx; + + if(los){ + assert(MAJOR_ALGO != MAJOR_MARK_SWEEP); + los->accumu_alloced_size += los->last_alloced_size; + } +} + +static void gc_gen_update_space_info_after_gc(GC_Gen *gc) +{ + Space *nos = gc_get_nos(gc); + Space *mos = gc_get_mos(gc); + Space *los = gc_get_los(gc); + + /* Minor collection, but also can be every n minor collections, use fspace->num_collections to identify. */ + if (gc_match_kind((GC*)gc, MINOR_COLLECTION)){ + mos->accumu_alloced_size += mos->last_alloced_size; + /* The alloced_size reset operation of mos and nos is not necessary, because they are not accumulated. + * But los->last_alloced_size must be reset, because it is accumulated. */ + if(los){ + assert(MAJOR_ALGO != MAJOR_MARK_SWEEP); + los->last_alloced_size = 0; + } + /* Major collection, but also can be every n major collections, use mspace->num_collections to identify. */ + } else { + mos->total_alloced_size += mos->accumu_alloced_size; + mos->last_alloced_size = 0; + mos->accumu_alloced_size = 0; + + nos->total_alloced_size += nos->accumu_alloced_size; + nos->last_alloced_size = 0; + nos->accumu_alloced_size = 0; + + if(los){ + assert(MAJOR_ALGO != MAJOR_MARK_SWEEP); + los->total_alloced_size += los->accumu_alloced_size; + los->last_alloced_size = 0; + los->accumu_alloced_size = 0; + } + } +} + +static void nos_reset_after_collection(Space *nos) +{ + fspace_reset_after_collection((Fspace*)nos); +} + +static void mos_reset_after_collection(Space *mos) +{ + if(MAJOR_ALGO != MAJOR_MARK_SWEEP) + mspace_reset_after_collection((Mspace*)mos); + else + sspace_reset_after_collection((Sspace*)mos); +} + Boolean IS_FALLBACK_COMPACTION = FALSE; /* only for debugging, don't use it. */ -static unsigned int mspace_num_used_blocks_before_minor; -static unsigned int mspace_num_used_blocks_after_minor; void gc_gen_stats_verbose(GC_Gen* gc); -void gc_gen_reclaim_heap(GC_Gen* gc) + +void gc_gen_reclaim_heap(GC_Gen *gc, int64 gc_start_time) { INFO2("gc.process", "GC: start GC_Gen ...\n"); - - if(verify_live_heap) gc_verify_heap((GC*)gc, TRUE); - - Blocked_Space* fspace = (Blocked_Space*)gc->nos; - Blocked_Space* mspace = (Blocked_Space*)gc->mos; - mspace->num_used_blocks = mspace->free_block_idx - mspace->first_block_idx; - fspace->num_used_blocks = fspace->free_block_idx - fspace->first_block_idx; - + + Space *nos = gc->nos; + Space *mos = gc->mos; + Space *los = gc->los; + + + if(verify_live_heap && (MAJOR_ALGO != MAJOR_MARK_SWEEP)) + gc_verify_heap((GC*)gc, TRUE); + + if(MAJOR_ALGO != MAJOR_MARK_SWEEP){ + gc_gen_update_space_info_before_gc(gc); + gc_compute_space_tune_size_before_marking((GC*)gc); + } + gc->collect_result = TRUE; #ifdef GC_GEN_STATS - gc_gen_stats_reset_before_collection((GC_Gen*)gc); - gc_gen_collector_stats_reset((GC_Gen*)gc); + gc_gen_stats_reset_before_collection(gc); + gc_gen_collector_stats_reset(gc); #endif if(gc_match_kind((GC*)gc, MINOR_COLLECTION)){ @@ -495,42 +655,50 @@ INFO2("gc.process", "GC: start minor collection ...\n"); /* FIXME:: move_object is only useful for nongen_slide_copy */ - gc->mos->move_object = 0; - /* This is for compute mspace->last_alloced_size */ + mos->move_object = FALSE; + + /* This is for compute mos->last_alloced_size */ + unsigned int mos_used_blocks_before_minor, mos_used_blocks_after_minor; /* only used for non MAJOR_MARK_SWEEP collection */ + if(MAJOR_ALGO != MAJOR_MARK_SWEEP) + mos_used_blocks_before_minor = ((Blocked_Space*)mos)->free_block_idx - ((Blocked_Space*)mos)->first_block_idx; + + nos_collection(nos); - mspace_num_used_blocks_before_minor = mspace->free_block_idx - mspace->first_block_idx; - fspace_collection(gc->nos); - #ifdef GC_GEN_STATS gc_gen_collector_stats_verbose_minor_collection(gc); #endif - mspace_num_used_blocks_after_minor = mspace->free_block_idx - mspace->first_block_idx; - assert( mspace_num_used_blocks_before_minor <= mspace_num_used_blocks_after_minor ); - mspace->last_alloced_size = GC_BLOCK_SIZE_BYTES * ( mspace_num_used_blocks_after_minor - mspace_num_used_blocks_before_minor ); - /*If the current minor collection failed, i.e. there happens a fallback, we should not do the minor sweep of LOS*/ + if(MAJOR_ALGO != MAJOR_MARK_SWEEP){ + mos_used_blocks_after_minor = ((Blocked_Space*)mos)->free_block_idx - ((Blocked_Space*)mos)->first_block_idx; + assert( mos_used_blocks_before_minor <= mos_used_blocks_after_minor ); + ((Blocked_Space*)mos)->last_alloced_size = GC_BLOCK_SIZE_BYTES * ( mos_used_blocks_after_minor - mos_used_blocks_before_minor ); + } + + /* If the minor collection failed, i.e. there happens a fallback, we should not do the minor sweep of LOS. */ if(gc->collect_result != FALSE && !gc_is_gen_mode()) { #ifdef GC_GEN_STATS gc->stats->num_minor_collections++; #endif - lspace_collection(gc->los); + los_collection(los); } - gc->mos->move_object = 1; + + mos->move_object = TRUE; INFO2("gc.process", "GC: end of minor collection ...\n"); - }else{ + } else { INFO2("gc.process", "GC: start major collection ...\n"); - /* process mos and nos together in one compaction */ - gc->los->move_object = 1; + if(MAJOR_ALGO != MAJOR_MARK_SWEEP) + los->move_object = TRUE; + + mos_collection(mos); /* collect mos and nos together */ + los_collection(los); + + if(MAJOR_ALGO != MAJOR_MARK_SWEEP) + los->move_object = FALSE; - mspace_collection(gc->mos); /* fspace collection is included */ - lspace_collection(gc->los); - - gc->los->move_object = 0; - #ifdef GC_GEN_STATS gc->stats->num_major_collections++; gc_gen_collector_stats_verbose_major_collection(gc); @@ -538,33 +706,39 @@ INFO2("gc.process", "GC: end of major collection ...\n"); } - + if(gc->collect_result == FALSE && gc_match_kind((GC*)gc, MINOR_COLLECTION)){ - + INFO2("gc.process", "GC: Minor collection failed, transform to fallback collection ..."); - - if(gc_is_gen_mode()) gc_clear_remset((GC*)gc); - /* runout mspace in minor collection */ - assert(mspace->free_block_idx == mspace->ceiling_block_idx + 1); - mspace->num_used_blocks = mspace->num_managed_blocks; - + if(gc_is_gen_mode()) gc_clear_remset((GC*)gc); + + /* runout mos in minor collection */ + if(MAJOR_ALGO != MAJOR_MARK_SWEEP){ + assert(((Blocked_Space*)mos)->free_block_idx == ((Blocked_Space*)mos)->ceiling_block_idx + 1); + ((Blocked_Space*)mos)->num_used_blocks = ((Blocked_Space*)mos)->num_managed_blocks; + } + IS_FALLBACK_COMPACTION = TRUE; gc_reset_collect_result((GC*)gc); - gc->collect_kind = FALLBACK_COLLECTION; + gc->collect_kind = FALLBACK_COLLECTION; + #ifdef GC_GEN_STATS /*since stats is changed in minor collection, we need to reset stats before fallback collection*/ gc_gen_stats_reset_before_collection((GC_Gen*)gc); gc_gen_collector_stats_reset((GC_Gen*)gc); #endif - if(verify_live_heap) event_gc_collect_kind_changed((GC*)gc); + if(verify_live_heap && (MAJOR_ALGO != MAJOR_MARK_SWEEP)) + event_gc_collect_kind_changed((GC*)gc); - gc->los->move_object = 1; - mspace_collection(gc->mos); /* fspace collection is included */ - lspace_collection(gc->los); - gc->los->move_object = 0; - + if(MAJOR_ALGO != MAJOR_MARK_SWEEP) + los->move_object = TRUE; + mos_collection(mos); /* collect both mos and nos */ + los_collection(los); + if(MAJOR_ALGO != MAJOR_MARK_SWEEP) + los->move_object = FALSE; + IS_FALLBACK_COMPACTION = FALSE; #ifdef GC_GEN_STATS @@ -582,59 +756,36 @@ exit(0); } - if(verify_live_heap) gc_verify_heap((GC*)gc, FALSE); + nos_reset_after_collection(nos); + if(gc_match_kind((GC*)gc, MAJOR_COLLECTION)) + mos_reset_after_collection(mos); + + if(verify_live_heap && (MAJOR_ALGO != MAJOR_MARK_SWEEP)) + gc_verify_heap((GC*)gc, FALSE); + + assert(MAJOR_ALGO == MAJOR_MARK_SWEEP || !los->move_object); + + if(MAJOR_ALGO != MAJOR_MARK_SWEEP){ + gc_gen_adjust_heap_size(gc); + + int64 pause_time = time_now() - gc_start_time; + gc->time_collections += pause_time; + gc_gen_adapt(gc, pause_time); + gc_space_tuner_reset((GC*)gc); + } + + gc_gen_update_space_info_after_gc(gc); - /* FIXME:: clear root set here to support verify. */ -#ifdef COMPRESS_REFERENCE - gc_set_pool_clear(gc->metadata->gc_uncompressed_rootset_pool); -#endif - - assert(!gc->los->move_object); #ifdef GC_GEN_STATS - gc_gen_stats_update_after_collection((GC_Gen*)gc); + gc_gen_stats_update_after_collection(gc); gc_gen_stats_verbose(gc); #endif INFO2("gc.process", "GC: end of GC_Gen\n"); - + return; } -void gc_gen_update_space_before_gc(GC_Gen *gc) -{ - /* Update before every GC to avoid the atomic operation in every fspace_alloc_block */ - assert( gc->nos->free_block_idx >= gc->nos->first_block_idx ); - gc->nos->last_alloced_size = GC_BLOCK_SIZE_BYTES * ( gc->nos->free_block_idx - gc->nos->first_block_idx ); - - gc->nos->accumu_alloced_size += gc->nos->last_alloced_size; - gc->los->accumu_alloced_size += gc->los->last_alloced_size; -} - -void gc_gen_update_space_after_gc(GC_Gen *gc) -{ - /* Minor collection, but also can be every n minor collections, use fspace->num_collections to identify. */ - if (gc_match_kind((GC*)gc, MINOR_COLLECTION)){ - gc->mos->accumu_alloced_size += gc->mos->last_alloced_size; - /* The alloced_size reset operation of mos and nos is not necessary, because they are not accumulated. - * But los->last_alloced_size must be reset, because it is accumulated. */ - gc->los->last_alloced_size = 0; - /* Major collection, but also can be every n major collections, use mspace->num_collections to identify. */ - }else{ - gc->mos->total_alloced_size += gc->mos->accumu_alloced_size; - gc->mos->last_alloced_size = 0; - gc->mos->accumu_alloced_size = 0; - - gc->nos->total_alloced_size += gc->nos->accumu_alloced_size; - gc->nos->last_alloced_size = 0; - gc->nos->accumu_alloced_size = 0; - - gc->los->total_alloced_size += gc->los->accumu_alloced_size; - gc->los->last_alloced_size = 0; - gc->los->accumu_alloced_size = 0; - - } -} - void gc_gen_iterate_heap(GC_Gen *gc) { /** the function is called after stoped the world **/ @@ -646,7 +797,7 @@ mutator = mutator->next; } - Mspace* mspace = gc->mos; + Blocked_Space *mspace = (Blocked_Space*)gc->mos; Block_Header *curr_block = (Block_Header*)mspace->blocks; Block_Header *space_end = (Block_Header*)&mspace->blocks[mspace->free_block_idx - mspace->first_block_idx]; while(curr_block < space_end) { @@ -665,7 +816,7 @@ if(curr_block == NULL) break; } - Fspace* fspace = gc->nos; + Blocked_Space *fspace = (Blocked_Space*)gc->nos; curr_block = (Block_Header*)fspace->blocks; space_end = (Block_Header*)&fspace->blocks[fspace->free_block_idx - fspace->first_block_idx]; while(curr_block < space_end) { @@ -680,7 +831,7 @@ if(curr_block == NULL) break; } - Lspace* lspace = gc->los; + Lspace *lspace = (Lspace*)gc->los; POINTER_SIZE_INT lspace_obj = (POINTER_SIZE_INT)lspace->heap_start; POINTER_SIZE_INT lspace_end = (POINTER_SIZE_INT)lspace->heap_end; unsigned int hash_extend_size = 0; @@ -699,6 +850,14 @@ } } +void gc_gen_hook_for_collector_init(Collector *collector) +{ + if(MAJOR_ALGO == MAJOR_MARK_SWEEP){ + allocator_init_local_chunks((Allocator*)collector); + collector_init_free_chunk_list(collector); + } +} + void gc_gen_collection_verbose_info(GC_Gen *gc, int64 pause_time, int64 mutator_time) { @@ -719,10 +878,10 @@ INFO2("gc.collect","GC: current minor collection num: "<stats->num_minor_collections); #endif break; - case MAJOR_COLLECTION: - INFO2("gc.collect","GC: collection type: major"); + case NORMAL_MAJOR_COLLECTION: + INFO2("gc.collect","GC: collection type: normal major"); #ifdef GC_GEN_STATS - INFO2("gc.collect","GC: current major collection num: "<stats->num_major_collections); + INFO2("gc.collect","GC: current normal major collection num: "<stats->num_major_collections); #endif break; case FALLBACK_COLLECTION: @@ -752,9 +911,9 @@ { INFO2("gc.space","GC: Heap info after GC["<num_collections<<"]:" <<"\nGC: Heap size: "<committed_heap_size)<<", free size:"<los->committed_heap_size)<<", free size:"<los)) - <<"\nGC: MOS size: "<mos->committed_heap_size)<<", free size:"<mos)) - <<"\nGC: NOS size: "<nos->committed_heap_size)<<", free size:"<nos))<<"\n"); + <<"\nGC: LOS size: "<los->committed_heap_size)<<", free size:"<los)) + <<"\nGC: MOS size: "<mos->committed_heap_size)<<", free size:"<mos)) + <<"\nGC: NOS size: "<nos->committed_heap_size)<<", free size:"<nos))<<"\n"); } inline void gc_gen_initial_verbose_info(GC_Gen *gc) @@ -789,6 +948,3 @@ <<"\nGC: total appliction execution time: "<total_mutator_time<<"\n"); #endif } - - - Index: src/gen/gen.h =================================================================== --- src/gen/gen.h (revision 569956) +++ src/gen/gen.h (working copy) @@ -28,6 +28,7 @@ #include "../trace_forward/fspace.h" #include "../mark_compact/mspace.h" #include "../los/lspace.h" +#include "../mark_sweep/sspace.h" #include "../finalizer_weakref/finalizer_weakref_metadata.h" #ifdef GC_GEN_STATS @@ -74,6 +75,10 @@ unsigned int num_collectors; unsigned int num_active_collectors; /* not all collectors are working */ + Marker** markers; + unsigned int num_markers; + unsigned int num_active_markers; + /* metadata is the pool for rootset, markstack, etc. */ GC_Metadata* metadata; Finref_Metadata *finref_metadata; @@ -93,6 +98,13 @@ //For_LOS_extend Space_Tuner* tuner; + unsigned int gc_concurrent_status; + Collection_Scheduler* collection_scheduler; + + SpinLock concurrent_mark_lock; + SpinLock enumerate_rootset_lock; + + /* system info */ unsigned int _system_alloc_unit; unsigned int _machine_page_size_bytes; @@ -100,9 +112,9 @@ /* END of GC --> */ Block* blocks; - Fspace *nos; - Mspace *mos; - Lspace *los; + Space *nos; + Space *mos; + Space *los; Boolean force_major_collect; Gen_Mode_Adaptor* gen_mode_adaptor; @@ -124,35 +136,24 @@ void gc_gen_wrapup_verbose(GC_Gen* gc); inline POINTER_SIZE_INT gc_gen_free_memory_size(GC_Gen* gc) -{ return space_free_memory_size((Blocked_Space*)gc->nos) + - space_free_memory_size((Blocked_Space*)gc->mos) + - lspace_free_memory_size(gc->los); } +{ return blocked_space_free_mem_size((Blocked_Space*)gc->nos) + + blocked_space_free_mem_size((Blocked_Space*)gc->mos) + + lspace_free_memory_size((Lspace*)gc->los); } inline POINTER_SIZE_INT gc_gen_total_memory_size(GC_Gen* gc) { return space_committed_size((Space*)gc->nos) + space_committed_size((Space*)gc->mos) + - lspace_committed_size(gc->los); } + lspace_committed_size((Lspace*)gc->los); } ///////////////////////////////////////////////////////////////////////////////////////// -inline void gc_nos_initialize(GC_Gen* gc, void* start, POINTER_SIZE_INT nos_size, POINTER_SIZE_INT commit_size) -{ fspace_initialize((GC*)gc, start, nos_size, commit_size); } +void gc_nos_initialize(GC_Gen *gc, void *start, POINTER_SIZE_INT nos_size, POINTER_SIZE_INT commit_size); +void gc_nos_destruct(GC_Gen *gc); +void gc_mos_initialize(GC_Gen *gc, void *start, POINTER_SIZE_INT mos_size, POINTER_SIZE_INT commit_size); +void gc_mos_destruct(GC_Gen *gc); +void gc_los_initialize(GC_Gen *gc, void *start, POINTER_SIZE_INT los_size); +void gc_los_destruct(GC_Gen *gc); -inline void gc_nos_destruct(GC_Gen* gc) -{ fspace_destruct(gc->nos); } - -inline void gc_mos_initialize(GC_Gen* gc, void* start, POINTER_SIZE_INT mos_size, POINTER_SIZE_INT commit_size) -{ mspace_initialize((GC*)gc, start, mos_size, commit_size); } - -inline void gc_mos_destruct(GC_Gen* gc) -{ mspace_destruct(gc->mos); } - -inline void gc_los_initialize(GC_Gen* gc, void* start, POINTER_SIZE_INT los_size) -{ lspace_initialize((GC*)gc, start, los_size); } - -inline void gc_los_destruct(GC_Gen* gc) -{ lspace_destruct(gc->los); } - inline Space* space_of_addr(GC* gc, void* addr) { assert(address_belongs_to_gc_heap(addr, gc)); @@ -161,9 +162,9 @@ return (Space*)((GC_Gen*)gc)->los; } -void* mos_alloc(unsigned size, Allocator *allocator); +extern Space_Alloc_Func mos_alloc; void* nos_alloc(unsigned size, Allocator *allocator); -void* los_alloc(unsigned size, Allocator *allocator); +extern Space_Alloc_Func los_alloc; void* los_try_alloc(POINTER_SIZE_INT size, GC* gc); Space* gc_get_nos(GC_Gen* gc); @@ -176,10 +177,11 @@ void gc_decide_collection_algorithm(GC_Gen* gc, char* minor_algo, char* major_algo); void gc_decide_collection_kind(GC_Gen* gc, unsigned int cause); +unsigned int gc_next_collection_kind(GC_Gen* gc); void gc_gen_adapt(GC_Gen* gc, int64 pause_time); -void gc_gen_reclaim_heap(GC_Gen* gc); +void gc_gen_reclaim_heap(GC_Gen* gc, int64 gc_start_time); void gc_gen_assign_free_area_to_mutators(GC_Gen* gc); @@ -192,61 +194,9 @@ void gc_gen_iterate_heap(GC_Gen *gc); +void gc_gen_start_concurrent_mark(GC_Gen* gc); + extern Boolean GEN_NONGEN_SWITCH ; -inline Boolean obj_is_dead_in_gen_minor_gc(Partial_Reveal_Object *p_obj) -{ - /* - * The first condition is for supporting switch between nongen and gen minor collection - * With this kind of switch dead objects in MOS & LOS may be set the mark or fw bit in oi - */ - return obj_belongs_to_nos(p_obj) && !obj_is_marked_or_fw_in_oi(p_obj); -} - -inline Boolean obj_is_dead_in_nongen_minor_gc(Partial_Reveal_Object *p_obj) -{ - return (obj_belongs_to_nos(p_obj) && !obj_is_fw_in_oi(p_obj)) - || (!obj_belongs_to_nos(p_obj) && !obj_is_marked_in_oi(p_obj)); -} - -inline Boolean obj_is_dead_in_major_gc(Partial_Reveal_Object *p_obj) -{ - return !obj_is_marked_in_vt(p_obj); -} - -// clear the two least significant bits of p_obj first -inline Boolean gc_obj_is_dead(GC *gc, Partial_Reveal_Object *p_obj) -{ - assert(p_obj); - if(gc_match_kind(gc, MINOR_COLLECTION)){ - if(gc_is_gen_mode()) - return obj_is_dead_in_gen_minor_gc(p_obj); - else - return obj_is_dead_in_nongen_minor_gc(p_obj); - } else { - return obj_is_dead_in_major_gc(p_obj); - } -} - -extern Boolean forward_first_half; -extern void* object_forwarding_boundary; - -inline Boolean fspace_obj_to_be_forwarded(Partial_Reveal_Object *p_obj) -{ - if(!obj_belongs_to_nos(p_obj)) return FALSE; - return forward_first_half? (p_obj < object_forwarding_boundary):(p_obj>=object_forwarding_boundary); -} - -inline Boolean obj_need_move(GC *gc, Partial_Reveal_Object *p_obj) -{ - if(gc_is_gen_mode() && gc_match_kind(gc, MINOR_COLLECTION)) - return fspace_obj_to_be_forwarded(p_obj); - - Space *space = space_of_addr(gc, p_obj); - return space->move_object; -} - #endif /* ifndef _GC_GEN_H_ */ - - Index: src/gen/gen_adapt.cpp =================================================================== --- src/gen/gen_adapt.cpp (revision 569956) +++ src/gen/gen_adapt.cpp (working copy) @@ -101,11 +101,11 @@ Blocked_Space* mspace = (Blocked_Space*)gc->mos; Gen_Mode_Adaptor* gen_mode_adaptor = gc->gen_mode_adaptor; - POINTER_SIZE_INT mos_free_size = space_free_memory_size(mspace); - POINTER_SIZE_INT nos_free_size = space_free_memory_size(fspace); + POINTER_SIZE_INT mos_free_size = blocked_space_free_mem_size(mspace); + POINTER_SIZE_INT nos_free_size = blocked_space_free_mem_size(fspace); POINTER_SIZE_INT total_free_size = mos_free_size + nos_free_size; - if(!gc_match_kind((GC*)gc, MINOR_COLLECTION)) { + if(gc_match_kind((GC*)gc, MAJOR_COLLECTION)) { assert(!gc_is_gen_mode()); if(gen_mode_adaptor->major_survive_ratio_threshold != 0 && mspace->survive_ratio > gen_mode_adaptor->major_survive_ratio_threshold){ @@ -195,20 +195,20 @@ float survive_ratio = 0.2f; - POINTER_SIZE_INT mos_free_size = space_free_memory_size(mspace); - POINTER_SIZE_INT nos_free_size = space_free_memory_size(fspace); + POINTER_SIZE_INT mos_free_size = blocked_space_free_mem_size(mspace); + POINTER_SIZE_INT nos_free_size = blocked_space_free_mem_size(fspace); assert(nos_free_size == space_committed_size((Space*)fspace)); POINTER_SIZE_INT total_free_size = mos_free_size + nos_free_size; - if(!gc_match_kind((GC*)gc, MINOR_COLLECTION)) gc->force_gen_mode = FALSE; + if(gc_match_kind((GC*)gc, MAJOR_COLLECTION)) gc->force_gen_mode = FALSE; if(!gc->force_gen_mode){ /*Major collection:*/ - if(!gc_match_kind((GC*)gc, MINOR_COLLECTION)){ + if(gc_match_kind((GC*)gc, MAJOR_COLLECTION)){ mspace->time_collections += pause_time; Tslow = (float)pause_time; SMax = total_free_size; /*If fall back happens, and nos_boundary is up to heap_ceiling, then we force major.*/ - if(gc->nos->num_managed_blocks == 0) + if(((Fspace*)gc->nos)->num_managed_blocks == 0) gc->force_major_collect = TRUE; else gc->force_major_collect = FALSE; @@ -293,8 +293,7 @@ POINTER_SIZE_INT new_mos_size; POINTER_SIZE_INT curr_nos_size = space_committed_size((Space*)fspace); - POINTER_SIZE_INT used_mos_size = space_used_memory_size(mspace); - POINTER_SIZE_INT free_mos_size = space_committed_size((Space*)mspace) - used_mos_size; + POINTER_SIZE_INT used_mos_size = blocked_space_used_mem_size(mspace); POINTER_SIZE_INT total_size; @@ -455,10 +454,10 @@ <blocks[mspace->free_block_idx - mspace->first_block_idx]; Index: src/los/lspace.cpp =================================================================== --- src/los/lspace.cpp (revision 569956) +++ src/los/lspace.cpp (working copy) @@ -29,7 +29,7 @@ extern POINTER_SIZE_INT min_los_size_bytes; extern POINTER_SIZE_INT min_none_los_size_bytes; -void lspace_initialize(GC* gc, void* start, POINTER_SIZE_INT lspace_size) +Lspace *lspace_initialize(GC* gc, void* start, POINTER_SIZE_INT lspace_size) { Lspace* lspace = (Lspace*)STD_MALLOC( sizeof(Lspace)); assert(lspace); @@ -62,18 +62,16 @@ lspace->num_collections = 0; lspace->time_collections = 0; lspace->survive_ratio = 0.5f; - lspace->last_alloced_size = 0; lspace->accumu_alloced_size = 0; lspace->total_alloced_size = 0; lspace->last_surviving_size = 0; lspace->period_surviving_size = 0; - gc_set_los((GC_Gen*)gc, (Space*)lspace); p_global_lspace_move_obj = &(lspace->move_object); los_boundary = lspace->heap_end; - return; + return lspace; } void lspace_destruct(Lspace* lspace) @@ -130,4 +128,3 @@ { return lspace->failure_size; } - Index: src/los/lspace.h =================================================================== --- src/los/lspace.h (revision 569956) +++ src/los/lspace.h (working copy) @@ -50,6 +50,8 @@ /*LOS_Shrink:This field stands for sliding compact to lspace */ Boolean move_object; + Space_Statistics* space_statistic; + /* Size allocted since last collection. */ volatile uint64 last_alloced_size; /* Size allocted since last major collection. */ @@ -75,9 +77,9 @@ void* scompact_fa_end; }Lspace; -void lspace_initialize(GC* gc, void* reserved_base, POINTER_SIZE_INT lspace_size); +Lspace *lspace_initialize(GC* gc, void* reserved_base, POINTER_SIZE_INT lspace_size); void lspace_destruct(Lspace* lspace); -Managed_Object_Handle lspace_alloc(POINTER_SIZE_INT size, Allocator* allocator); +Managed_Object_Handle lspace_alloc(unsigned size, Allocator* allocator); void* lspace_try_alloc(Lspace* lspace, POINTER_SIZE_INT alloc_size); void lspace_sliding_compact(Collector* collector, Lspace* lspace); void lspace_compute_object_target(Collector* collector, Lspace* lspace); @@ -85,12 +87,21 @@ void lspace_collection(Lspace* lspace); inline POINTER_SIZE_INT lspace_free_memory_size(Lspace* lspace) -{ /* FIXME:: */ +{ + if(!lspace) return 0; + /* FIXME:: */ assert(lspace->committed_heap_size > (POINTER_SIZE_INT)lspace->last_surviving_size + (POINTER_SIZE_INT)lspace->last_alloced_size); return (lspace->committed_heap_size - (POINTER_SIZE_INT)lspace->last_surviving_size - (POINTER_SIZE_INT)lspace->last_alloced_size); } -inline POINTER_SIZE_INT lspace_committed_size(Lspace* lspace){ return lspace->committed_heap_size; } +inline POINTER_SIZE_INT lspace_committed_size(Lspace* lspace) +{ + if(lspace) + return lspace->committed_heap_size; + else + return 0; +} + inline Partial_Reveal_Object* lspace_get_next_marked_object( Lspace* lspace, unsigned int* iterate_index) { POINTER_SIZE_INT next_area_start = (POINTER_SIZE_INT)lspace->heap_start + (*iterate_index) * KB; Index: src/los/lspace_alloc_collect.cpp =================================================================== --- src/los/lspace_alloc_collect.cpp (revision 569956) +++ src/los/lspace_alloc_collect.cpp (working copy) @@ -22,7 +22,8 @@ #include "lspace.h" #include "../gen/gen.h" #include "../common/space_tuner.h" - +#include "../common/gc_concurrent.h" +#include "../common/collection_scheduler.h" #ifdef GC_GEN_STATS #include "../gen/gen_stats.h" #endif @@ -197,13 +198,17 @@ return p_result; } -void* lspace_alloc(POINTER_SIZE_INT size, Allocator *allocator) +void* lspace_alloc(unsigned size, Allocator *allocator) { unsigned int try_count = 0; void* p_result = NULL; POINTER_SIZE_INT alloc_size = ALIGN_UP_TO_KILO(size); Lspace* lspace = (Lspace*)gc_get_los((GC_Gen*)allocator->gc); Free_Area_Pool* pool = lspace->free_pool; + + if(gc_need_start_concurrent_mark(allocator->gc)) + gc_start_concurrent_mark(allocator->gc); + while( try_count < 2 ){ if(p_result = lspace_try_alloc(lspace, alloc_size)) @@ -334,7 +339,7 @@ /*Lspace collection in major collection must move object*/ assert(lspace->move_object); //debug_minor_sweep - Block* mos_first_block = ((GC_Gen*)gc)->mos->blocks; + Block* mos_first_block = ((Blocked_Space*)((GC_Gen*)gc)->mos)->blocks; lspace->heap_end = (void*)mos_first_block; assert(!(tuner->tuning_size % GC_BLOCK_SIZE_BYTES)); new_fa_size = (POINTER_SIZE_INT)lspace->scompact_fa_end - (POINTER_SIZE_INT)lspace->scompact_fa_start + tuner->tuning_size; @@ -346,7 +351,7 @@ case TRANS_FROM_LOS_TO_MOS:{ assert(lspace->move_object); assert(tuner->tuning_size); - Block* mos_first_block = ((GC_Gen*)gc)->mos->blocks; + Block* mos_first_block = ((Blocked_Space*)((GC_Gen*)gc)->mos)->blocks; assert( (POINTER_SIZE_INT)lspace->heap_end - trans_size == (POINTER_SIZE_INT)mos_first_block ); lspace->heap_end = (void*)mos_first_block; lspace->committed_heap_size -= trans_size; @@ -475,5 +480,3 @@ TRACE2("gc.process", "GC: end of lspace sweep algo ...\n"); return; } - - Index: src/mark_compact/mspace.cpp =================================================================== --- src/mark_compact/mspace.cpp (revision 569956) +++ src/mark_compact/mspace.cpp (working copy) @@ -35,7 +35,7 @@ extern void gc_set_mos(GC_Gen* gc, Space* space); extern Space* gc_get_nos(GC_Gen* gc); -void mspace_initialize(GC* gc, void* start, POINTER_SIZE_INT mspace_size, POINTER_SIZE_INT commit_size) +Mspace *mspace_initialize(GC* gc, void* start, POINTER_SIZE_INT mspace_size, POINTER_SIZE_INT commit_size) { Mspace* mspace = (Mspace*)STD_MALLOC( sizeof(Mspace)); assert(mspace); @@ -69,6 +69,11 @@ space_init_blocks((Blocked_Space*)mspace); + mspace->space_statistic = (Space_Statistics*)STD_MALLOC(sizeof(Space_Statistics)); + assert(mspace->space_statistic); + memset(mspace->space_statistic, 0, sizeof(Space_Statistics)); + + mspace->num_collections = 0; mspace->time_collections = 0; mspace->survive_ratio = 0.2f; @@ -84,9 +89,7 @@ mspace->expected_threshold_ratio = 0.5f; - gc_set_mos((GC_Gen*)gc, (Space*)mspace); - - return; + return mspace; } @@ -97,47 +100,52 @@ STD_FREE(mspace); } -void mspace_block_iterator_init_free(Mspace* mspace) +void mspace_reset_after_collection(Mspace* mspace) { - mspace->block_iterator = (Block_Header*)&mspace->blocks[mspace->free_block_idx - mspace->first_block_idx]; -} - -#include "../common/space_tuner.h" -void mspace_block_iterator_init(Mspace* mspace) -{ - mspace->block_iterator = (Block_Header*)mspace->blocks; - return; -} - -Block_Header* mspace_block_iterator_get(Mspace* mspace) -{ - return (Block_Header*)mspace->block_iterator; -} - -Block_Header* mspace_block_iterator_next(Mspace* mspace) -{ - Block_Header* cur_block = (Block_Header*)mspace->block_iterator; + unsigned int old_num_used = mspace->num_used_blocks; + unsigned int new_num_used = mspace->free_block_idx - mspace->first_block_idx; + unsigned int num_used = old_num_used>new_num_used? old_num_used:new_num_used; - while(cur_block != NULL){ - Block_Header* next_block = cur_block->next; + Block* blocks = mspace->blocks; + unsigned int i; + for(i=0; i < num_used; i++){ + Block_Header* block = (Block_Header*)&(blocks[i]); + assert(!((POINTER_SIZE_INT)block % GC_BLOCK_SIZE_BYTES)); + block->status = BLOCK_USED; + block->free = block->new_free; + block->new_free = block->base; + block->src = NULL; + block->next_src = NULL; + assert(!block->dest_counter); - Block_Header* temp = (Block_Header*)atomic_casptr((volatile void **)&mspace->block_iterator, next_block, cur_block); - if(temp != cur_block){ - cur_block = (Block_Header*)mspace->block_iterator; - continue; + if(i >= new_num_used){ + block->status = BLOCK_FREE; + block->free = GC_BLOCK_BODY(block); } - return cur_block; } - /* run out space blocks */ - return NULL; + mspace->num_used_blocks = new_num_used; + /*For_statistic mos infomation*/ + mspace->period_surviving_size = new_num_used * GC_BLOCK_SIZE_BYTES; + + /* we should clear the remaining blocks which are set to be BLOCK_COMPACTED or BLOCK_TARGET */ + for(; i < mspace->num_managed_blocks; i++){ + Block_Header* block = (Block_Header*)&(blocks[i]); + assert(block->status& (BLOCK_COMPACTED|BLOCK_TARGET|BLOCK_DEST)); + block->status = BLOCK_FREE; + block->src = NULL; + block->next_src = NULL; + block->free = GC_BLOCK_BODY(block); + assert(!block->dest_counter); + } } + #include "../common/fix_repointed_refs.h" void mspace_fix_after_copy_nursery(Collector* collector, Mspace* mspace) { //the first block is not set yet - Block_Header* curr_block = mspace_block_iterator_next(mspace); + Block_Header* curr_block = blocked_space_block_iterator_next((Blocked_Space*)mspace); unsigned int first_block_idx = mspace->first_block_idx; unsigned int old_num_used = mspace->num_used_blocks; unsigned int old_free_idx = first_block_idx + old_num_used; @@ -155,7 +163,7 @@ else /* for blocks used for nos copy */ block_fix_ref_after_copying(curr_block); - curr_block = mspace_block_iterator_next(mspace); + curr_block = blocked_space_block_iterator_next((Blocked_Space*)mspace); } return; @@ -172,4 +180,3 @@ { return mspace->expected_threshold_ratio; } - Index: src/mark_compact/mspace.h =================================================================== --- src/mark_compact/mspace.h (revision 569956) +++ src/mark_compact/mspace.h (working copy) @@ -38,6 +38,8 @@ GC* gc; Boolean move_object; + Space_Statistics* space_statistic; + /* Size allocted since last minor collection. */ volatile uint64 last_alloced_size; /* Size allocted since last major collection. */ @@ -61,24 +63,21 @@ unsigned int num_used_blocks; unsigned int num_managed_blocks; unsigned int num_total_blocks; + + volatile Block_Header* block_iterator; /* END of Blocked_Space --> */ - volatile Block_Header* block_iterator; /*Threshold computed by NOS adaptive*/ float expected_threshold_ratio; }Mspace; -void mspace_initialize(GC* gc, void* reserved_base, POINTER_SIZE_INT mspace_size, POINTER_SIZE_INT commit_size); +Mspace *mspace_initialize(GC* gc, void* reserved_base, POINTER_SIZE_INT mspace_size, POINTER_SIZE_INT commit_size); void mspace_destruct(Mspace* mspace); void* mspace_alloc(unsigned size, Allocator *allocator); void mspace_collection(Mspace* mspace); +void mspace_reset_after_collection(Mspace* mspace); -void mspace_block_iterator_init(Mspace* mspace); -void mspace_block_iterator_init_free(Mspace* mspace); -Block_Header* mspace_block_iterator_next(Mspace* mspace); -Block_Header* mspace_block_iterator_get(Mspace* mspace); - void mspace_fix_after_copy_nursery(Collector* collector, Mspace* mspace); void mspace_set_expected_threshold_ratio(Mspace* mspace, float threshold_ratio); Index: src/mark_compact/mspace_collect_compact.cpp =================================================================== --- src/mark_compact/mspace_collect_compact.cpp (revision 569956) +++ src/mark_compact/mspace_collect_compact.cpp (working copy) @@ -33,6 +33,7 @@ #ifdef GC_GEN_STATS #include "../gen/gen_stats.h" #endif + void mspace_update_info_after_space_tuning(Mspace* mspace) { Space_Tuner *tuner = mspace->gc->tuner; @@ -60,45 +61,7 @@ Space* gc_get_nos(GC_Gen* gc); -void mspace_reset_after_compaction(Mspace* mspace) -{ - unsigned int old_num_used = mspace->num_used_blocks; - unsigned int new_num_used = mspace->free_block_idx - mspace->first_block_idx; - unsigned int num_used = old_num_used>new_num_used? old_num_used:new_num_used; - - Block* blocks = mspace->blocks; - unsigned int i; - for(i=0; i < num_used; i++){ - Block_Header* block = (Block_Header*)&(blocks[i]); - assert(!((POINTER_SIZE_INT)block % GC_BLOCK_SIZE_BYTES)); - block->status = BLOCK_USED; - block->free = block->new_free; - block->new_free = block->base; - block->src = NULL; - block->next_src = NULL; - assert(!block->dest_counter); - if(i >= new_num_used){ - block->status = BLOCK_FREE; - block->free = GC_BLOCK_BODY(block); - } - } - mspace->num_used_blocks = new_num_used; - /*For_statistic mos infomation*/ - mspace->period_surviving_size = new_num_used * GC_BLOCK_SIZE_BYTES; - - /* we should clear the remaining blocks which are set to be BLOCK_COMPACTED or BLOCK_TARGET */ - for(; i < mspace->num_managed_blocks; i++){ - Block_Header* block = (Block_Header*)&(blocks[i]); - assert(block->status& (BLOCK_COMPACTED|BLOCK_TARGET|BLOCK_DEST)); - block->status = BLOCK_FREE; - block->src = NULL; - block->next_src = NULL; - block->free = GC_BLOCK_BODY(block); - assert(!block->dest_counter); - } -} - void gc_reset_block_for_collectors(GC* gc, Mspace* mspace) { unsigned int free_blk_idx = mspace->first_block_idx; @@ -337,27 +300,27 @@ switch(mspace->collect_algorithm){ case MAJOR_COMPACT_SLIDE: - TRACE2("gc.process", "GC: slide compact algo start ... \n"); - collector_execute_task(gc, (TaskType)slide_compact_mspace, (Space*)mspace); - TRACE2("gc.process", "\nGC: end of slide compact algo ... \n"); + TRACE2("gc.process", "GC: slide compact algo start ... \n"); + collector_execute_task(gc, (TaskType)slide_compact_mspace, (Space*)mspace); + TRACE2("gc.process", "\nGC: end of slide compact algo ... \n"); #ifdef GC_GEN_STATS - gc_gen_stats_set_los_collected_flag((GC_Gen*)gc, true); - gc_gen_stats_set_mos_algo((GC_Gen*)gc, MAJOR_COMPACT_SLIDE); + gc_gen_stats_set_los_collected_flag((GC_Gen*)gc, true); + gc_gen_stats_set_mos_algo((GC_Gen*)gc, MAJOR_COMPACT_SLIDE); #endif break; case MAJOR_COMPACT_MOVE: IS_MOVE_COMPACT = TRUE; - - TRACE2("gc.process", "GC: move compact algo start ... \n"); - collector_execute_task(gc, (TaskType)move_compact_mspace, (Space*)mspace); - TRACE2("gc.process", "\nGC: end of move compact algo ... \n"); - IS_MOVE_COMPACT = FALSE; + + TRACE2("gc.process", "GC: move compact algo start ... \n"); + collector_execute_task(gc, (TaskType)move_compact_mspace, (Space*)mspace); + TRACE2("gc.process", "\nGC: end of move compact algo ... \n"); + IS_MOVE_COMPACT = FALSE; #ifdef GC_GEN_STATS - gc_gen_stats_set_mos_algo((GC_Gen*)gc, MAJOR_COMPACT_MOVE); + gc_gen_stats_set_mos_algo((GC_Gen*)gc, MAJOR_COMPACT_MOVE); #endif - break; - + break; + default: DIE2("gc.collect", "The speficied major collection algorithm doesn't exist!"); exit(0); Index: src/mark_compact/mspace_collect_compact.h =================================================================== --- src/mark_compact/mspace_collect_compact.h (revision 569956) +++ src/mark_compact/mspace_collect_compact.h (working copy) @@ -30,7 +30,6 @@ void gc_init_block_for_collectors(GC* gc, Mspace* mspace); void mspace_update_info_after_space_tuning(Mspace* mspace); -void mspace_reset_after_compaction(Mspace* mspace); Block_Header* mspace_get_first_compact_block(Mspace* mspace); Block_Header* mspace_get_first_target_block(Mspace* mspace); Index: src/mark_compact/mspace_extend_compact.cpp =================================================================== --- src/mark_compact/mspace_extend_compact.cpp (revision 569956) +++ src/mark_compact/mspace_extend_compact.cpp (working copy) @@ -34,15 +34,15 @@ static void set_first_and_end_block_to_move(Collector *collector, unsigned int mem_changed_size) { GC_Gen *gc_gen = (GC_Gen *)collector->gc; - Mspace *mspace = gc_gen->mos; - Fspace *fspace = gc_gen->nos; + Space *nos = gc_gen->nos; assert (!(mem_changed_size % SPACE_ALLOC_UNIT)); unsigned int mos_added_block_num = mem_changed_size >> GC_BLOCK_SHIFT_COUNT; // block number needing moving first_block_to_move = nos_first_free_block - mos_added_block_num; - if(first_block_to_move < (Block *)space_heap_start((Space *)fspace)) - first_block_to_move = (Block *)space_heap_start((Space *)fspace); + Block *nos_start_block = (Block*)space_heap_start(nos); + if(first_block_to_move < nos_start_block) + first_block_to_move = nos_start_block; } static POINTER_SIZE_INT fspace_shrink(Fspace *fspace) @@ -252,8 +252,7 @@ static void move_compacted_blocks_to_mspace(Collector *collector, unsigned int addr_diff) { GC_Gen *gc_gen = (GC_Gen *)collector->gc; - Mspace *mspace = gc_gen->mos; - Fspace *fspace = gc_gen->nos; + Mspace *mspace = (Mspace *)gc_gen->mos; while(Block_Header *block = mspace_block_iter_next_for_extension(mspace, (Block_Header *)nos_first_free_block)){ Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)block->base; @@ -272,9 +271,8 @@ void mspace_extend_compact(Collector *collector) { GC_Gen *gc_gen = (GC_Gen *)collector->gc; - Mspace *mspace = gc_gen->mos; - Fspace *fspace = gc_gen->nos; - Lspace *lspace = gc_gen->los; + Mspace *mspace = (Mspace *)gc_gen->mos; + Fspace *fspace = (Fspace *)gc_gen->nos; /*For_LOS adaptive: when doing EXTEND_COLLECTION, mspace->survive_ratio should not be updated in gc_decide_next_collect( )*/ gc_gen->collect_kind |= EXTEND_COLLECTION; Index: src/mark_compact/mspace_move_compact.cpp =================================================================== --- src/mark_compact/mspace_move_compact.cpp (revision 569956) +++ src/mark_compact/mspace_move_compact.cpp (working copy) @@ -150,13 +150,13 @@ static void mspace_fix_repointed_refs(Collector *collector, Mspace *mspace) { - Block_Header* curr_block = mspace_block_iterator_next(mspace); + Block_Header* curr_block = blocked_space_block_iterator_next((Blocked_Space*)mspace); while( curr_block){ if(curr_block->block_idx >= mspace->free_block_idx) break; curr_block->free = curr_block->new_free; // block_fix_ref_after_marking(curr_block); - curr_block = mspace_block_iterator_next(mspace); + curr_block = blocked_space_block_iterator_next((Blocked_Space*)mspace); } return; @@ -233,7 +233,7 @@ } gc_reset_block_for_collectors(gc, mspace); - mspace_block_iterator_init(mspace); + blocked_space_block_iterator_init((Blocked_Space*)mspace); num_moving_collectors++; } while(num_moving_collectors != num_active_collectors + 1); @@ -291,8 +291,6 @@ TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"] finished"); return; } - mspace_reset_after_compaction(mspace); - fspace_reset_for_allocation(fspace); gc_set_pool_clear(gc->metadata->gc_rootset_pool); gc_set_pool_clear(gc->metadata->weak_roots_pool); Index: src/mark_compact/mspace_slide_compact.cpp =================================================================== --- src/mark_compact/mspace_slide_compact.cpp (revision 569956) +++ src/mark_compact/mspace_slide_compact.cpp (working copy) @@ -156,14 +156,14 @@ #include "../common/fix_repointed_refs.h" -static void mspace_fix_repointed_refs(Collector* collector, Mspace* mspace) +static void mspace_fix_repointed_refs(Collector *collector, Mspace *mspace) { - Block_Header* curr_block = mspace_block_iterator_next(mspace); + Block_Header *curr_block = blocked_space_block_iterator_next((Blocked_Space*)mspace); /* for MAJOR_COLLECTION, we must iterate over all compact blocks */ while( curr_block){ block_fix_ref_after_repointing(curr_block); - curr_block = mspace_block_iterator_next(mspace); + curr_block = blocked_space_block_iterator_next((Blocked_Space*)mspace); } return; @@ -181,7 +181,7 @@ { assert(!next_block_for_dest); - Block_Header *block = mspace_block_iterator_get(mspace); + Block_Header *block = blocked_space_block_iterator_get((Blocked_Space*)mspace); if(block->status != BLOCK_DEST) return block; @@ -248,7 +248,7 @@ cur_dest_block = cur_dest_block->next; } } else { - cur_dest_block = mspace_block_iterator_get(mspace); + cur_dest_block = blocked_space_block_iterator_get((Blocked_Space*)mspace); } unsigned int total_dest_counter = 0; @@ -312,7 +312,7 @@ POINTER_SIZE_INT tuning_size = tuner->tuning_size; /*If LOS_Shrink, we just fix the repointed refs from the start of old mspace.*/ if((tuner->kind == TRANS_NOTHING) || (tuner->kind == TRANS_FROM_LOS_TO_MOS)){ - mspace_block_iterator_init(mspace); + blocked_space_block_iterator_init((Blocked_Space*)mspace); return; }else{ /*If LOS_Extend, we fix from the new start of mspace, because the block list is start from there.*/ @@ -333,7 +333,7 @@ if( tuner->kind == TRANS_NOTHING ){ /*If space is not tuned, we just start from mspace->heap_start.*/ - mspace_block_iterator_init(mspace); + blocked_space_block_iterator_init((Blocked_Space*)mspace); return; }else if (tuner->kind == TRANS_FROM_MOS_TO_LOS){ /*If LOS_Extend, we compact from the new start of mspace, because the block list is start from there.*/ @@ -399,16 +399,6 @@ } -/*For LOS_Extend*/ -static void mspace_restore_block_chain(Mspace* mspace) -{ - GC* gc = mspace->gc; - Fspace* fspace = (Fspace*)gc_get_nos((GC_Gen*)gc); - if(gc->tuner->kind == TRANS_FROM_MOS_TO_LOS) { - Block_Header* fspace_last_block = (Block_Header*)&fspace->blocks[fspace->num_managed_blocks - 1]; - fspace_last_block->next = NULL; - } -} static volatile unsigned int num_marking_collectors = 0; static volatile unsigned int num_repointing_collectors = 0; @@ -560,7 +550,8 @@ old_num = atomic_inc32(&num_restoring_collectors); if( ++old_num == num_active_collectors ){ - if(gc->tuner->kind != TRANS_NOTHING) mspace_update_info_after_space_tuning(mspace); + if(gc->tuner->kind != TRANS_NOTHING) + mspace_update_info_after_space_tuning(mspace); num_restoring_collectors++; } while(num_restoring_collectors != num_active_collectors + 1); @@ -585,12 +576,6 @@ /* Leftover: ************************************************** */ - mspace_reset_after_compaction(mspace); - fspace_reset_for_allocation(fspace); - - /*For LOS_Extend*/ - mspace_restore_block_chain(mspace); - gc_set_pool_clear(gc->metadata->gc_rootset_pool); gc_set_pool_clear(gc->metadata->weak_roots_pool); Index: src/mark_sweep/gc_ms.cpp =================================================================== --- src/mark_sweep/gc_ms.cpp (revision 569956) +++ src/mark_sweep/gc_ms.cpp (working copy) @@ -22,6 +22,8 @@ #include "gc_ms.h" #include "../finalizer_weakref/finalizer_weakref.h" #include "../common/compressed_ref.h" +#include "../thread/marker.h" +#include "../verify/verify_live_heap.h" #ifdef USE_32BITS_HASHCODE #include "../common/hashcode.h" #endif @@ -61,14 +63,51 @@ void gc_ms_reclaim_heap(GC_MS *gc) { + if(verify_live_heap) gc_verify_heap((GC*)gc, TRUE); + sspace_collection(gc_ms_get_sspace(gc)); - /* FIXME:: clear root set here to support verify */ -#ifdef COMPRESS_REFERENCE - gc_set_pool_clear(gc->metadata->gc_uncompressed_rootset_pool); -#endif + if(verify_live_heap) gc_verify_heap((GC*)gc, FALSE); } +void sspace_mark_scan_concurrent(Marker* marker); +void gc_ms_start_concurrent_mark(GC_MS* gc, unsigned int num_markers) +{ + if(gc->num_active_markers == 0) + pool_iterator_init(gc->metadata->gc_rootset_pool); + + marker_execute_task_concurrent((GC*)gc,(TaskType)sspace_mark_scan_concurrent,(Space*)gc->sspace, num_markers); +} + +void gc_ms_start_concurrent_mark(GC_MS* gc) +{ + pool_iterator_init(gc->metadata->gc_rootset_pool); + + marker_execute_task_concurrent((GC*)gc,(TaskType)sspace_mark_scan_concurrent,(Space*)gc->sspace); +} + +void gc_ms_update_space_statistics(GC_MS* gc) +{ + POINTER_SIZE_INT num_live_obj = 0; + POINTER_SIZE_INT size_live_obj = 0; + + Space_Statistics* sspace_stat = gc->sspace->space_statistic; + + unsigned int num_collectors = gc->num_active_collectors; + Collector** collectors = gc->collectors; + unsigned int i; + for(i = 0; i < num_collectors; i++){ + Collector* collector = collectors[i]; + num_live_obj += collector->live_obj_num; + size_live_obj += collector->live_obj_size; + } + + sspace_stat->num_live_obj = num_live_obj; + sspace_stat->size_live_obj = size_live_obj; + sspace_stat->last_size_free_space = sspace_stat->size_free_space; + sspace_stat->size_free_space = gc->committed_heap_size - size_live_obj;/*TODO:inaccurate value.*/ +} + void gc_ms_iterate_heap(GC_MS *gc) { } Index: src/mark_sweep/gc_ms.h =================================================================== --- src/mark_sweep/gc_ms.h (revision 569956) +++ src/mark_sweep/gc_ms.h (working copy) @@ -47,6 +47,10 @@ unsigned int num_collectors; unsigned int num_active_collectors; /* not all collectors are working */ + Marker** markers; + unsigned int num_markers; + unsigned int num_active_markers; + /* metadata is the pool for rootset, markstack, etc. */ GC_Metadata *metadata; Finref_Metadata *finref_metadata; @@ -64,7 +68,12 @@ //For_LOS_extend Space_Tuner *tuner; - + + unsigned int gc_concurrent_status; + Collection_Scheduler* collection_scheduler; + + SpinLock concurrent_mark_lock; + SpinLock enumerate_rootset_lock; /* system info */ unsigned int _system_alloc_unit; unsigned int _machine_page_size_bytes; @@ -102,6 +111,9 @@ void gc_ms_reclaim_heap(GC_MS *gc); void gc_ms_iterate_heap(GC_MS *gc); +void gc_ms_start_concurrent_mark(GC_MS* gc); +void gc_ms_start_concurrent_mark(GC_MS* gc, unsigned int num_markers); +void gc_ms_update_space_statistics(GC_MS* gc); #endif // USE_MARK_SWEEP_GC Index: src/mark_sweep/sspace.cpp =================================================================== --- src/mark_sweep/sspace.cpp (revision 569956) +++ src/mark_sweep/sspace.cpp (working copy) @@ -23,7 +23,7 @@ struct GC_Gen; -void sspace_initialize(GC *gc, void *start, unsigned int sspace_size, unsigned int commit_size) +Sspace *sspace_initialize(GC *gc, void *start, POINTER_SIZE_INT sspace_size, POINTER_SIZE_INT commit_size) { /* With sspace in the heap, the heap must be composed of a single sspace or a sspace and a NOS. * In either case, the reserved size and committed size of sspace must be the same. @@ -58,6 +58,10 @@ sspace_init_chunks(sspace); + sspace->space_statistic = (Space_Statistics*)STD_MALLOC(sizeof(Space_Statistics)); + assert(sspace->space_statistic); + memset(sspace->space_statistic, 0, sizeof(Space_Statistics)); + #ifdef USE_MARK_SWEEP_GC gc_ms_set_sspace((GC_MS*)gc, sspace); #else @@ -67,7 +71,7 @@ #ifdef SSPACE_VERIFY sspace_verify_init(gc); #endif - return; + return sspace; } static void sspace_destruct_chunks(Sspace *sspace) { return; } @@ -79,6 +83,13 @@ STD_FREE(sspace); } +void sspace_reset_after_collection(Sspace *sspace) +{ + sspace->move_object = FALSE; + sspace->need_compact = FALSE; + sspace->need_fix = FALSE; +} + void allocator_init_local_chunks(Allocator *allocator) { Sspace *sspace = gc_get_sspace(allocator->gc); @@ -110,6 +121,27 @@ allocator->local_chunks = local_chunks; } +void allocator_clear_local_chunks(Allocator *allocator, Boolean reuse_pfc) +{ + Sspace *sspace = gc_get_sspace(allocator->gc); + Size_Segment **size_segs = sspace->size_segments; + Chunk_Header ***local_chunks = allocator->local_chunks; + + for(unsigned int i = SIZE_SEGMENT_NUM; i--;){ + if(!size_segs[i]->local_alloc){ + assert(!local_chunks[i]); + continue; + } + Chunk_Header **chunks = local_chunks[i]; + assert(chunks); + for(unsigned int j = size_segs[i]->chunk_num; j--;){ + if(chunks[j] && reuse_pfc) + sspace_put_pfc(sspace, chunks[j]); + chunks[j] = NULL; + } + } +} + void allocactor_destruct_local_chunks(Allocator *allocator) { Sspace *sspace = gc_get_sspace(allocator->gc); @@ -141,6 +173,18 @@ STD_FREE(local_chunks); } +#ifdef USE_MARK_SWEEP_GC +void sspace_set_space_statistic(Sspace *sspace) +{ + GC_MS* gc = (GC_MS*)sspace->gc; + + for(unsigned int i=0; inum_collectors; ++i){ + sspace->surviving_obj_num += gc->collectors[i]->live_obj_num; + sspace->surviving_obj_size += gc->collectors[i]->live_obj_size; + } +} +#endif + extern void sspace_decide_compaction_need(Sspace *sspace); extern void mark_sweep_sspace(Collector *collector); @@ -157,8 +201,12 @@ #endif sspace_decide_compaction_need(sspace); - if(sspace->need_compact) - gc->collect_kind = SWEEP_COMPACT_GC; + if(sspace->need_compact && gc_match_kind(gc, MARK_SWEEP_GC)){ + assert(gc_match_kind(gc, MS_COLLECTION)); + gc->collect_kind = MS_COMPACT_COLLECTION; + } + if(sspace->need_compact || gc_match_kind(gc, MAJOR_COLLECTION)) + sspace->need_fix = TRUE; //printf("\n\n>>>>>>>>%s>>>>>>>>>>>>\n\n", sspace->need_compact ? "SWEEP COMPACT" : "MARK SWEEP"); #ifdef SSPACE_VERIFY sspace_verify_before_collection(gc); Index: src/mark_sweep/sspace.h =================================================================== --- src/mark_sweep/sspace.h (revision 569956) +++ src/mark_sweep/sspace.h (working copy) @@ -34,8 +34,8 @@ /* <-- first couple of fields are overloadded as Space */ void *heap_start; void *heap_end; - unsigned int reserved_heap_size; - unsigned int committed_heap_size; + POINTER_SIZE_INT reserved_heap_size; + POINTER_SIZE_INT committed_heap_size; unsigned int num_collections; int64 time_collections; float survive_ratio; @@ -43,6 +43,8 @@ GC *gc; Boolean move_object; + Space_Statistics* space_statistic; + /* Size allocted since last minor collection. */ volatile POINTER_SIZE_INT last_alloced_size; /* Size allocted since last major collection. */ @@ -58,15 +60,23 @@ /* END of Space --> */ Boolean need_compact; + Boolean need_fix; /* There are repointed ref needing fixing */ Size_Segment **size_segments; Pool ***pfc_pools; Free_Chunk_List *aligned_free_chunk_lists; Free_Chunk_List *unaligned_free_chunk_lists; Free_Chunk_List *hyper_free_chunk_list; + POINTER_SIZE_INT surviving_obj_num; + POINTER_SIZE_INT surviving_obj_size; } Sspace; -void sspace_initialize(GC *gc, void *start, unsigned int sspace_size, unsigned int commit_size); +#ifdef USE_MARK_SWEEP_GC +void sspace_set_space_statistic(Sspace *sspace); +#endif + +Sspace *sspace_initialize(GC *gc, void *start, POINTER_SIZE_INT sspace_size, POINTER_SIZE_INT commit_size); void sspace_destruct(Sspace *sspace); +void sspace_reset_after_collection(Sspace *sspace); void *sspace_thread_local_alloc(unsigned size, Allocator *allocator); void *sspace_alloc(unsigned size, Allocator *allocator); Index: src/mark_sweep/sspace_alloc.cpp =================================================================== --- src/mark_sweep/sspace_alloc.cpp (revision 569956) +++ src/mark_sweep/sspace_alloc.cpp (working copy) @@ -130,6 +130,8 @@ } p_obj = alloc_in_chunk(chunks[index]); } else { + if(gc_need_start_concurrent_mark(allocator->gc)) + gc_start_concurrent_mark(allocator->gc); chunk = sspace_get_pfc(sspace, seg_index, index); if(!chunk){ chunk = (Chunk_Header*)sspace_get_normal_free_chunk(sspace); @@ -137,8 +139,6 @@ } //if(!chunk) chunk = sspace_steal_pfc(sspace, seg_index, index); if(!chunk) return NULL; - assert(chunk->alloc_num < chunk->slot_num); - ++chunk->alloc_num; p_obj = alloc_in_chunk(chunk); if(chunk) sspace_put_pfc(sspace, chunk); @@ -151,6 +151,9 @@ { assert(size > SUPER_OBJ_THRESHOLD); + if(gc_need_start_concurrent_mark(allocator->gc)) + gc_start_concurrent_mark(allocator->gc); + unsigned int chunk_size = SUPER_SIZE_ROUNDUP(size); assert(chunk_size > SUPER_OBJ_THRESHOLD); assert(!(chunk_size & CHUNK_GRANULARITY_LOW_MASK)); @@ -187,6 +190,7 @@ if(p_obj) sspace_verify_alloc(p_obj, size); #endif + if(p_obj && gc_is_concurrent_mark_phase()) obj_mark_black_in_table((Partial_Reveal_Object*)p_obj,size); return p_obj; } @@ -206,7 +210,7 @@ vm_gc_unlock_enum(); return p_obj; } - gc_reclaim_heap(allocator->gc, GC_CAUSE_POS_IS_FULL); + gc_reclaim_heap(allocator->gc, GC_CAUSE_SSPACE_IS_FULL); vm_gc_unlock_enum(); #ifdef SSPACE_CHUNK_INFO Index: src/mark_sweep/sspace_alloc.h =================================================================== --- src/mark_sweep/sspace_alloc.h (revision 569956) +++ src/mark_sweep/sspace_alloc.h (working copy) @@ -19,10 +19,11 @@ #define _SSPACE_ALLOC_H_ #include "sspace_chunk.h" +#include "sspace_mark_sweep.h" +#include "../common/gc_concurrent.h" +#include "../common/collection_scheduler.h" - extern POINTER_SIZE_INT cur_alloc_color; -extern POINTER_SIZE_INT cur_mark_color; extern POINTER_SIZE_INT cur_alloc_mask; extern POINTER_SIZE_INT cur_mark_mask; @@ -36,6 +37,20 @@ return (Boolean)(table[word_index] & (cur_alloc_color << index_in_word)); } +#ifdef _DEBUG +static Boolean slot_is_free_in_table(POINTER_SIZE_INT *table, unsigned int slot_index) +{ + assert(!slot_is_alloc_in_table(table, slot_index)); + + unsigned int color_bits_index = slot_index * COLOR_BITS_PER_OBJ; + unsigned int word_index = color_bits_index / BITS_PER_WORD; + unsigned int index_in_word = color_bits_index % BITS_PER_WORD; + + return !(table[word_index] & cur_alloc_color << index_in_word); + +} +#endif + inline unsigned int composed_slot_index(unsigned int word_index, unsigned int index_in_word) { unsigned int color_bits_index = word_index*BITS_PER_WORD + index_in_word; @@ -158,7 +173,13 @@ POINTER_SIZE_INT *table = chunk->table; unsigned int slot_index = chunk->slot_index; + assert(chunk->alloc_num < chunk->slot_num); + ++chunk->alloc_num; + assert(chunk->base); void *p_obj = (void*)((POINTER_SIZE_INT)chunk->base + ((POINTER_SIZE_INT)chunk->slot_size * slot_index)); +#ifdef _DEBUG + slot_is_free_in_table(table, slot_index); +#endif alloc_slot_in_table(table, slot_index); if(chunk->status & CHUNK_NEED_ZEROING) memset(p_obj, 0, chunk->slot_size); @@ -172,6 +193,9 @@ chunk->slot_index = (slot_index < chunk->slot_num) ? slot_index : MAX_SLOT_INDEX; } else #endif + + if(p_obj && gc_is_concurrent_mark_phase()) obj_mark_black_in_table((Partial_Reveal_Object*)p_obj,chunk->slot_size); + chunk->slot_index = next_free_slot_index_in_table(table, slot_index, chunk->slot_num); if(chunk->slot_index == MAX_SLOT_INDEX){ chunk->status = CHUNK_USED | CHUNK_NORMAL; Index: src/mark_sweep/sspace_chunk.cpp =================================================================== --- src/mark_sweep/sspace_chunk.cpp (revision 569956) +++ src/mark_sweep/sspace_chunk.cpp (working copy) @@ -124,25 +124,26 @@ free_chunk_list_clear(&unaligned_free_chunk_lists[i]); free_chunk_list_clear(&hyper_free_chunk_list); - - /* release small obj chunks of each mutator */ + +#ifdef USE_MARK_SWEEP_GC + /* release local chunks of each mutator in unique mark-sweep GC */ Mutator *mutator = gc->mutator_list; while(mutator){ - Chunk_Header ***local_chunks = mutator->local_chunks; - for(i = SIZE_SEGMENT_NUM; i--;){ - if(!size_segments[i]->local_alloc){ - assert(!local_chunks[i]); - continue; - } - Chunk_Header **chunks = local_chunks[i]; - assert(chunks); - for(j = size_segments[i]->chunk_num; j--;) - chunks[j] = NULL; - } + allocator_clear_local_chunks((Allocator*)mutator, FALSE); mutator = mutator->next; } +#endif } +void gc_clear_collector_local_chunks(GC *gc) +{ + assert(gc_match_kind(gc, MAJOR_COLLECTION)); + /* release local chunks of each collector in gen GC */ + for(unsigned int i = gc->num_collectors; i--;){ + allocator_clear_local_chunks((Allocator*)gc->collectors[i], TRUE); + } +} + /* Simply put the free chunk to the according list * Don't merge continuous free chunks * The merging job is taken by sweeping @@ -150,7 +151,6 @@ static void list_put_free_chunk(Free_Chunk_List *list, Free_Chunk *chunk) { chunk->status = CHUNK_FREE; - chunk->adj_prev = NULL; chunk->prev = NULL; lock(list->lock); @@ -158,6 +158,8 @@ if(list->head) list->head->prev = chunk; list->head = chunk; + if(!list->tail) + list->tail = chunk; assert(list->chunk_num < ~((unsigned int)0)); ++list->chunk_num; unlock(list->lock); @@ -171,6 +173,8 @@ list->head = chunk->next; if(list->head) list->head->prev = NULL; + else + list->tail = NULL; assert(list->chunk_num); --list->chunk_num; assert(chunk->status == CHUNK_FREE); @@ -336,21 +340,22 @@ Free_Chunk_List *list = sspace->hyper_free_chunk_list; lock(list->lock); - Free_Chunk **p_next = &list->head; + Free_Chunk *prev_chunk = NULL; Free_Chunk *chunk = list->head; while(chunk){ if(CHUNK_SIZE(chunk) >= chunk_size){ Free_Chunk *next_chunk = chunk->next; - *p_next = next_chunk; - if(next_chunk){ - if(chunk != list->head) - next_chunk->prev = (Free_Chunk *)p_next; /* utilize an assumption: next is the first field of Free_Chunk */ - else - next_chunk->prev = NULL; - } + if(prev_chunk) + prev_chunk->next = next_chunk; + else + list->head = next_chunk; + if(next_chunk) + next_chunk->prev = prev_chunk; + else + list->tail = prev_chunk; break; } - p_next = &chunk->next; + prev_chunk = chunk; chunk = chunk->next; } unlock(list->lock); @@ -367,6 +372,26 @@ return chunk; } +void sspace_collect_free_chunks_to_list(Sspace *sspace, Free_Chunk_List *list) +{ + unsigned int i; + + for(i = NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) + move_free_chunks_between_lists(list, &sspace->aligned_free_chunk_lists[i]); + + for(i = NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) + move_free_chunks_between_lists(list, &sspace->unaligned_free_chunk_lists[i]); + + move_free_chunks_between_lists(list, sspace->hyper_free_chunk_list); + + Free_Chunk *chunk = list->head; + while(chunk){ + chunk->status = CHUNK_FREE | CHUNK_TO_MERGE; + chunk = chunk->next; + } +} + + typedef struct PFC_Pool_Iterator { volatile unsigned int seg_index; volatile unsigned int chunk_index; Index: src/mark_sweep/sspace_chunk.h =================================================================== --- src/mark_sweep/sspace_chunk.h (revision 569956) +++ src/mark_sweep/sspace_chunk.h (working copy) @@ -36,17 +36,19 @@ typedef struct Chunk_Header_Basic { Chunk_Header_Basic *next; + Chunk_Header_Basic *prev; Chunk_Status_t status; + Chunk_Header_Basic *adj_next; // adjacent next chunk Chunk_Header_Basic *adj_prev; // adjacent previous chunk, for merging continuous free chunks - Chunk_Header_Basic *adj_next; // adjacent next chunk } Chunk_Header_Basic; typedef struct Chunk_Header { /* Beginning of Chunk_Header_Basic */ Chunk_Header *next; /* pointing to the next pfc in the pfc pool */ + Chunk_Header *prev; /* pointing to the prev pfc in the pfc pool */ Chunk_Status_t status; + Chunk_Header_Basic *adj_next; // adjacent next chunk Chunk_Header_Basic *adj_prev; // adjacent previous chunk, for merging continuous free chunks - Chunk_Header_Basic *adj_next; // adjacent next chunk /* End of Chunk_Header_Basic */ void *base; unsigned int slot_size; @@ -65,7 +67,7 @@ #define ABNORMAL_CHUNK_HEADER(addr) ((Chunk_Header*)((POINTER_SIZE_INT)addr & CHUNK_GRANULARITY_HIGH_MASK)) #define MAX_SLOT_INDEX 0xFFffFFff -#define COLOR_BITS_PER_OBJ 2 // should be powers of 2 +#define COLOR_BITS_PER_OBJ 4 // should be powers of 2 #define SLOT_NUM_PER_WORD_IN_TABLE (BITS_PER_WORD /COLOR_BITS_PER_OBJ) /* Two equations: @@ -101,11 +103,11 @@ typedef struct Free_Chunk { /* Beginning of Chunk_Header_Basic */ Free_Chunk *next; /* pointing to the next free Free_Chunk */ + Free_Chunk *prev; /* pointing to the prev free Free_Chunk */ Chunk_Status_t status; + Chunk_Header_Basic *adj_next; // adjacent next chunk Chunk_Header_Basic *adj_prev; // adjacent previous chunk, for merging continuous free chunks - Chunk_Header_Basic *adj_next; // adjacent next chunk /* End of Chunk_Header_Basic */ - Free_Chunk *prev; /* pointing to the prev free Free_Chunk */ } Free_Chunk; typedef struct Free_Chunk_List { @@ -139,6 +141,31 @@ assert(list->lock == FREE_LOCK); } +inline void free_list_detach_chunk(Free_Chunk_List *list, Free_Chunk *chunk) +{ + if(chunk->prev) + chunk->prev->next = chunk->next; + else // chunk is the head + list->head = chunk->next; + if(chunk->next) + chunk->next->prev = chunk->prev; +} + +inline void move_free_chunks_between_lists(Free_Chunk_List *to_list, Free_Chunk_List *from_list) +{ + if(to_list->tail){ + to_list->head->prev = from_list->tail; + } else { + to_list->tail = from_list->tail; + } + if(from_list->head){ + from_list->tail->next = to_list->head; + to_list->head = from_list->head; + } + from_list->head = NULL; + from_list->tail = NULL; +} + /* Padding the last index word in table to facilitate allocation */ inline void chunk_pad_last_index_word(Chunk_Header *chunk, POINTER_SIZE_INT alloc_mask) { @@ -167,7 +194,7 @@ inline void normal_chunk_init(Chunk_Header *chunk, unsigned int slot_size) { assert(chunk->status == CHUNK_FREE); - assert((POINTER_SIZE_INT)chunk->adj_next == (POINTER_SIZE_INT)chunk + NORMAL_CHUNK_SIZE_BYTES); + assert(CHUNK_SIZE(chunk) == NORMAL_CHUNK_SIZE_BYTES); chunk->next = NULL; chunk->status = CHUNK_FRESH | CHUNK_NORMAL | CHUNK_NEED_ZEROING; @@ -184,7 +211,7 @@ inline void abnormal_chunk_init(Chunk_Header *chunk, unsigned int chunk_size, unsigned int obj_size) { assert(chunk->status == CHUNK_FREE); - assert((POINTER_SIZE_INT)chunk->adj_next == (POINTER_SIZE_INT)chunk + chunk_size); + assert(CHUNK_SIZE(chunk) == chunk_size); chunk->next = NULL; chunk->status = CHUNK_ABNORMAL; @@ -231,7 +258,7 @@ #define ALIGNED_CHUNK_INDEX_TO_SIZE(index) (((index) + 1) << NORMAL_CHUNK_SHIFT_COUNT) #define UNALIGNED_CHUNK_INDEX_TO_SIZE(index) (((index) + 1) << CHUNK_GRANULARITY_BITS) -#define SUPER_OBJ_MASK ((Obj_Info_Type)0x1) /* the lowest bit in obj info */ +#define SUPER_OBJ_MASK ((Obj_Info_Type)0x20) /* the 4th bit in obj info */ #define PFC_STEAL_NUM 3 #define PFC_STEAL_THRESHOLD 3 @@ -274,6 +301,7 @@ inline void sspace_put_pfc(Sspace *sspace, Chunk_Header *chunk) { unsigned int size = chunk->slot_size; + assert(chunk->base && chunk->alloc_num); assert(chunk && (size <= SUPER_OBJ_THRESHOLD)); assert(chunk->slot_index < chunk->slot_num); @@ -309,5 +337,9 @@ extern void zeroing_free_chunk(Free_Chunk *chunk); +extern void allocator_clear_local_chunks(Allocator *allocator, Boolean reuse_pfc); +extern void gc_clear_collector_local_chunks(GC *gc); +extern void sspace_collect_free_chunks_to_list(Sspace *sspace, Free_Chunk_List *list); + #endif //#ifndef _SSPACE_CHUNK_H_ Index: src/mark_sweep/sspace_compact.cpp =================================================================== --- src/mark_sweep/sspace_compact.cpp (revision 569956) +++ src/mark_sweep/sspace_compact.cpp (working copy) @@ -19,18 +19,20 @@ #include "sspace_alloc.h" #include "sspace_mark_sweep.h" #include "sspace_verify.h" -#include "../common/fix_repointed_refs.h" #define PFC_SORT_NUM 8 -static Chunk_Header_Basic *volatile next_chunk_for_fixing; - void sspace_decide_compaction_need(Sspace *sspace) { POINTER_SIZE_INT free_mem_size = free_mem_in_sspace(sspace, FALSE); float free_mem_ratio = (float)free_mem_size / sspace->committed_heap_size; + +#ifdef USE_MARK_SWEEP_GC if((free_mem_ratio > SSPACE_COMPACT_RATIO) && (sspace->gc->cause != GC_CAUSE_RUNTIME_FORCE_GC)){ +#else + if(gc_match_kind(sspace->gc, MAJOR_COLLECTION)){ +#endif sspace->need_compact = sspace->move_object = TRUE; } else { sspace->need_compact = sspace->move_object = FALSE; @@ -39,7 +41,7 @@ static inline void sorted_chunk_bucket_add_entry(Chunk_Header **head, Chunk_Header **tail, Chunk_Header *chunk) { - chunk->adj_prev = NULL; /* Field adj_prev is used as prev */ + chunk->prev = NULL; /* Field adj_prev is used as prev */ if(!*head){ assert(!*tail); @@ -50,7 +52,7 @@ assert(*tail); chunk->next = *head; - (*head)->adj_prev = (Chunk_Header_Basic*)chunk; + (*head)->prev = chunk; *head = chunk; } @@ -112,7 +114,7 @@ tail = bucket_tail[i]; } else { tail->next = bucket_head[i]; - bucket_head[i]->adj_prev = (Chunk_Header_Basic*)tail; + bucket_head[i]->prev = tail; tail = bucket_tail[i]; } } @@ -133,7 +135,7 @@ Chunk_Header *result = *least_free_chunk; *least_free_chunk = (*least_free_chunk)->next; if(*least_free_chunk) - (*least_free_chunk)->adj_prev = NULL; + (*least_free_chunk)->prev = NULL; else *most_free_chunk = NULL; return result; @@ -145,11 +147,12 @@ return NULL; } Chunk_Header *result = *most_free_chunk; - *most_free_chunk = (Chunk_Header*)(*most_free_chunk)->adj_prev; + *most_free_chunk = (*most_free_chunk)->prev; if(*most_free_chunk) (*most_free_chunk)->next = NULL; else *least_free_chunk = NULL; + assert(!result->next); return result; } @@ -175,7 +178,6 @@ } /* dest might be set to NULL, so we use *dest_ptr here */ - (*dest_ptr)->alloc_num += src->alloc_num - alloc_num; assert((*dest_ptr)->alloc_num <= (*dest_ptr)->slot_num); src->alloc_num = alloc_num; if(!dest){ @@ -185,7 +187,7 @@ } } -static void sspace_move_objects(Collector *collector, Sspace *sspace) +void sspace_compact(Collector *collector, Sspace *sspace) { Chunk_Header *least_free_chunk, *most_free_chunk; Pool *pfc_pool = sspace_grab_next_pfc_pool(sspace); @@ -225,111 +227,3 @@ } } -static void sspace_init_chunk_for_ref_fixing(Sspace *sspace) -{ - next_chunk_for_fixing = (Chunk_Header_Basic*)space_heap_start((Space*)sspace); - next_chunk_for_fixing->adj_prev = NULL; -} - -static void normal_chunk_fix_repointed_refs(Chunk_Header *chunk) -{ - /* Init field slot_index and depad the last index word in table for fixing */ - chunk->slot_index = 0; - chunk_depad_last_index_word(chunk); - - unsigned int alloc_num = chunk->alloc_num; - assert(alloc_num); - - /* After compaction, many chunks are filled with objects. - * For these chunks, we needn't find the allocated slot one by one by calling next_alloc_slot_in_chunk. - * That is a little time consuming. - * We'd like to fix those objects by incrementing their addr to find the next. - */ - if(alloc_num == chunk->slot_num){ /* Filled with objects */ - unsigned int slot_size = chunk->slot_size; - Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)slot_index_to_addr(chunk, 0); - for(unsigned int i = alloc_num; i--;){ - object_fix_ref_slots(p_obj); -#ifdef SSPACE_VERIFY - sspace_verify_fix_in_compact(); -#endif - p_obj = (Partial_Reveal_Object*)((POINTER_SIZE_INT)p_obj + slot_size); - } - } else { /* Chunk is not full */ - while(alloc_num){ - Partial_Reveal_Object *p_obj = next_alloc_slot_in_chunk(chunk); - assert(p_obj); - object_fix_ref_slots(p_obj); -#ifdef SSPACE_VERIFY - sspace_verify_fix_in_compact(); -#endif - --alloc_num; - } - } - - if(chunk->alloc_num != chunk->slot_num){ - chunk_pad_last_index_word(chunk, cur_alloc_mask); - pfc_reset_slot_index(chunk); - } -} - -static void abnormal_chunk_fix_repointed_refs(Chunk_Header *chunk) -{ - object_fix_ref_slots((Partial_Reveal_Object*)chunk->base); -#ifdef SSPACE_VERIFY - sspace_verify_fix_in_compact(); -#endif -} - -static void sspace_fix_repointed_refs(Collector *collector, Sspace *sspace) -{ - Chunk_Header_Basic *chunk = sspace_grab_next_chunk(sspace, &next_chunk_for_fixing, TRUE); - - while(chunk){ - if(chunk->status & CHUNK_NORMAL) - normal_chunk_fix_repointed_refs((Chunk_Header*)chunk); - else if(chunk->status & CHUNK_ABNORMAL) - abnormal_chunk_fix_repointed_refs((Chunk_Header*)chunk); - - chunk = sspace_grab_next_chunk(sspace, &next_chunk_for_fixing, TRUE); - } -} - -static volatile unsigned int num_moving_collectors = 0; -static volatile unsigned int num_fixing_collectors = 0; - -void compact_sspace(Collector *collector, Sspace *sspace) -{ - GC *gc = collector->gc; - - unsigned int num_active_collectors = gc->num_active_collectors; - - /* Pass 1: ************************************************** - move live objects between pfcs with the same size *****************/ - atomic_cas32(&num_moving_collectors, 0, num_active_collectors+1); - - sspace_move_objects(collector, sspace); - - unsigned int old_num = atomic_inc32(&num_moving_collectors); - if( ++old_num == num_active_collectors ){ - /* last collector's world here */ -#ifdef SSPACE_TIME - sspace_compact_time(FALSE); -#endif - sspace_init_chunk_for_ref_fixing(sspace); - /* let other collectors go */ - num_moving_collectors++; - } - while(num_moving_collectors != num_active_collectors + 1); - - /* Pass 2: ************************************************** - sweep dead objects ***************************************/ - atomic_cas32( &num_fixing_collectors, 0, num_active_collectors); - - sspace_fix_repointed_refs(collector, sspace); - - atomic_inc32(&num_fixing_collectors); - while(num_fixing_collectors != num_active_collectors); - -} - Index: src/mark_sweep/sspace_fallback_mark.cpp =================================================================== --- src/mark_sweep/sspace_fallback_mark.cpp (revision 0) +++ src/mark_sweep/sspace_fallback_mark.cpp (revision 0) @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sspace_mark_sweep.h" +#include "../finalizer_weakref/finalizer_weakref.h" + +static Sspace *sspace_in_fallback_marking; +static FORCE_INLINE Boolean obj_mark(Partial_Reveal_Object *obj) +{ + if(obj_belongs_to_space(obj, (Space*)sspace_in_fallback_marking)) + return obj_mark_black_in_table(obj); + else + return obj_mark_in_vt(obj); +} + +static FORCE_INLINE void scan_slot(Collector *collector, REF *p_ref) +{ + if( read_slot(p_ref) == NULL) return; + + collector_tracestack_push(collector, p_ref); +} + +static FORCE_INLINE void scan_object(Collector *collector, REF *p_ref) +{ + Partial_Reveal_Object *p_obj = read_slot(p_ref); + assert(p_obj); + assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); + + if(obj_belongs_to_nos(p_obj) && obj_is_fw_in_oi(p_obj)){ + assert(obj_get_vt(p_obj) == obj_get_vt(obj_get_fw_in_oi(p_obj))); + p_obj = obj_get_fw_in_oi(p_obj); + assert(p_obj); + write_slot(p_ref, p_obj); + } + + if(!obj_mark(p_obj)) + return; + + if(!object_has_ref_field(p_obj)) return; + + if(object_is_array(p_obj)){ /* scan array object */ + Partial_Reveal_Array *array = (Partial_Reveal_Array*)p_obj; + unsigned int array_length = array->array_len; + + REF *p_ref = (REF *)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); + for (unsigned int i = 0; i < array_length; i++) + scan_slot(collector, p_ref+i); + + return; + } + + /* scan non-array object */ + unsigned int num_refs = object_ref_field_num(p_obj); + int *ref_iterator = object_ref_iterator_init(p_obj); + + for(unsigned int i=0; itrace_stack; + while(!vector_stack_is_empty(trace_stack)){ + p_ref = (REF*)vector_stack_pop(trace_stack); + scan_object(collector, p_ref); + trace_stack = collector->trace_stack; + } +} + +/* NOTE:: This is another marking version: marking in color bitmap table. + Originally, we have to mark the object before put it into markstack, to + guarantee there is only one occurrance of an object in markstack. This is to + guarantee there is only one occurrance of a repointed ref slot in repset (they + are put to the set when the object is scanned). If the same object is put to + markstack twice, they will be scanned twice and their ref slots will be recorded twice. + Problem occurs when the ref slot is updated first time with new position, + the second time the value in the ref slot is not the old position as expected. + It needs to read the original obj header for forwarding pointer. With the new value, + it will read something nonsense since the obj is not moved yet. + This can be worked around if we want. + To do this we have to use atomic instruction for marking, which is undesirable. + So we abondoned this design. We no longer use the repset to remember repointed slots. +*/ + +/* for marking phase termination detection */ +static volatile unsigned int num_finished_collectors = 0; + +void sspace_fallback_mark_scan(Collector *collector, Sspace *sspace) +{ + GC *gc = collector->gc; + GC_Metadata *metadata = gc->metadata; + sspace_in_fallback_marking = sspace; + + /* reset the num_finished_collectors to be 0 by one collector. This is necessary for the barrier later. */ + unsigned int num_active_collectors = gc->num_active_collectors; + atomic_cas32(&num_finished_collectors, 0, num_active_collectors); + + collector->trace_stack = free_task_pool_get_entry(metadata); + + Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool); + + /* first step: copy all root objects to mark tasks. + FIXME:: can be done sequentially before coming here to eliminate atomic ops */ + while(root_set){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); + while(!vector_block_iterator_end(root_set,iter)){ + REF *p_ref = (REF*)*iter; + iter = vector_block_iterator_advance(root_set,iter); + + /* root ref can't be NULL, (remset may have NULL ref entry, but this function is only for MAJOR_COLLECTION */ + assert(read_slot(p_ref) != NULL); + /* we have to mark the object before putting it into marktask, because + it is possible to have two slots containing a same object. They will + be scanned twice and their ref slots will be recorded twice. Problem + occurs after the ref slot is updated first time with new position + and the second time the value is the ref slot is the old position as expected. + This can be worked around if we want. + */ + collector_tracestack_push(collector, p_ref); + } + root_set = pool_iterator_next(metadata->gc_rootset_pool); + } + /* put back the last trace_stack task */ + pool_put_entry(metadata->mark_task_pool, collector->trace_stack); + + /* second step: iterate over the mark tasks and scan objects */ + /* get a task buf for the mark stack */ + collector->trace_stack = free_task_pool_get_entry(metadata); + +retry: + Vector_Block *mark_task = pool_get_entry(metadata->mark_task_pool); + + while(mark_task){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task); + while(!vector_block_iterator_end(mark_task, iter)){ + REF *p_ref = (REF*)*iter; + iter = vector_block_iterator_advance(mark_task, iter); + + /* FIXME:: we should not let mark_task empty during working, , other may want to steal it. + degenerate my stack into mark_task, and grab another mark_task */ + trace_object(collector, p_ref); + } + /* run out one task, put back to the pool and grab another task */ + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + mark_task = pool_get_entry(metadata->mark_task_pool); + } + + /* termination detection. This is also a barrier. + NOTE:: We can simply spin waiting for num_finished_collectors, because each + generated new task would surely be processed by its generating collector eventually. + So code below is only for load balance optimization. */ + atomic_inc32(&num_finished_collectors); + while(num_finished_collectors != num_active_collectors){ + if(!pool_is_empty(metadata->mark_task_pool)){ + atomic_dec32(&num_finished_collectors); + goto retry; + } + } + + /* put back the last mark stack to the free pool */ + mark_task = (Vector_Block*)collector->trace_stack; + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + collector->trace_stack = NULL; + + return; +} + +void trace_obj_in_ms_fallback_marking(Collector *collector, void *p_ref) +{ + trace_object(collector, (REF*)p_ref); +} Index: src/mark_sweep/sspace_mark.cpp =================================================================== --- src/mark_sweep/sspace_mark.cpp (revision 569956) +++ src/mark_sweep/sspace_mark.cpp (working copy) @@ -18,6 +18,25 @@ #include "sspace_mark_sweep.h" #include "../finalizer_weakref/finalizer_weakref.h" +static Sspace *sspace_in_marking; +static FORCE_INLINE Boolean obj_mark_gray(Partial_Reveal_Object *obj) +{ + if(obj_belongs_to_space(obj, (Space*)sspace_in_marking)) + return obj_mark_gray_in_table(obj); + else + return obj_mark_in_vt(obj); +} + +static FORCE_INLINE Boolean obj_mark_black(Partial_Reveal_Object *obj) +{ + if(obj_belongs_to_space(obj, (Space*)sspace_in_marking)) + return obj_mark_black_in_table(obj); + else + return obj_mark_in_vt(obj); +} + + +/* The caller must be in places where alloc color and mark color haven't been flipped */ Boolean obj_is_marked_in_table(Partial_Reveal_Object *obj) { unsigned int index_in_word; @@ -25,7 +44,7 @@ assert(p_color_word); POINTER_SIZE_INT color_word = *p_color_word; - POINTER_SIZE_INT mark_color = cur_mark_color << index_in_word; + POINTER_SIZE_INT mark_color = cur_mark_gray_color << index_in_word; return (Boolean)(color_word & mark_color); } @@ -36,7 +55,7 @@ if( p_obj == NULL) return; assert(address_belongs_to_gc_heap(p_obj, collector->gc)); - if(obj_mark_in_table(p_obj)){ + if(obj_mark_gray(p_obj)){ assert(p_obj); collector_tracestack_push(collector, p_obj); } @@ -50,7 +69,7 @@ if(VTABLE_TRACING) if(vtable->vtmark == VT_UNMARKED) { vtable->vtmark = VT_MARKED; - if(obj_mark_in_table(vtable->jlC)) + if(obj_mark_black(vtable->jlC)) collector_tracestack_push(collector, vtable->jlC); } @@ -87,11 +106,13 @@ static void trace_object(Collector *collector, Partial_Reveal_Object *p_obj) { scan_object(collector, p_obj); + obj_mark_black(p_obj); Vector_Block *trace_stack = collector->trace_stack; while(!vector_stack_is_empty(trace_stack)){ p_obj = (Partial_Reveal_Object*)vector_stack_pop(trace_stack); scan_object(collector, p_obj); + obj_mark_black(p_obj); trace_stack = collector->trace_stack; } } @@ -114,10 +135,11 @@ /* for marking phase termination detection */ static volatile unsigned int num_finished_collectors = 0; -void sspace_mark_scan(Collector *collector) +void sspace_mark_scan(Collector *collector, Sspace *sspace) { GC *gc = collector->gc; GC_Metadata *metadata = gc->metadata; + sspace_in_marking = sspace; /* reset the num_finished_collectors to be 0 by one collector. This is necessary for the barrier later. */ unsigned int num_active_collectors = gc->num_active_collectors; @@ -132,12 +154,12 @@ while(root_set){ POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); while(!vector_block_iterator_end(root_set,iter)){ - REF *p_ref = (REF *)*iter; + REF *p_ref = (REF*)*iter; iter = vector_block_iterator_advance(root_set,iter); Partial_Reveal_Object *p_obj = read_slot(p_ref); /* root ref can't be NULL, (remset may have NULL ref entry, but this function is only for MAJOR_COLLECTION */ - assert(p_obj!=NULL); + assert(p_obj != NULL); /* we have to mark the object before putting it into marktask, because it is possible to have two slots containing a same object. They will be scanned twice and their ref slots will be recorded twice. Problem @@ -146,7 +168,7 @@ This can be worked around if we want. */ assert(address_belongs_to_gc_heap(p_obj, gc)); - if(obj_mark_in_table(p_obj)) + if(obj_mark_gray(p_obj)) collector_tracestack_push(collector, p_obj); } root_set = pool_iterator_next(metadata->gc_rootset_pool); @@ -163,9 +185,9 @@ while(mark_task){ POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task); - while(!vector_block_iterator_end(mark_task,iter)){ + while(!vector_block_iterator_end(mark_task, iter)){ Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)*iter; - iter = vector_block_iterator_advance(mark_task,iter); + iter = vector_block_iterator_advance(mark_task, iter); /* FIXME:: we should not let mark_task empty during working, , other may want to steal it. degenerate my stack into mark_task, and grab another mark_task */ @@ -200,6 +222,6 @@ void trace_obj_in_ms_marking(Collector *collector, void *p_obj) { - obj_mark_in_table((Partial_Reveal_Object*)p_obj); - trace_object(collector, (Partial_Reveal_Object *)p_obj); + obj_mark_gray((Partial_Reveal_Object*)p_obj); + trace_object(collector, (Partial_Reveal_Object*)p_obj); } Index: src/mark_sweep/sspace_mark_concurrent.cpp =================================================================== --- src/mark_sweep/sspace_mark_concurrent.cpp (revision 0) +++ src/mark_sweep/sspace_mark_concurrent.cpp (revision 0) @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sspace_mark_sweep.h" +#include "../finalizer_weakref/finalizer_weakref.h" +#include "../thread/marker.h" + +Boolean obj_is_marked_in_table(Partial_Reveal_Object *obj); + +static FORCE_INLINE void scan_slot(Collector* marker, REF *p_ref) +{ + Partial_Reveal_Object *p_obj = read_slot(p_ref); + if( p_obj == NULL) return; + + assert(address_belongs_to_gc_heap(p_obj, marker->gc)); + if(obj_mark_gray_in_table(p_obj)){ + assert(p_obj); + collector_tracestack_push((Collector*)marker, p_obj); + } +} + +static FORCE_INLINE void scan_object(Marker* marker, Partial_Reveal_Object *p_obj) +{ + assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); + + if(obj_is_dirty_in_table(p_obj)){ + assert(obj_is_mark_black_in_table(p_obj)); + return; + } + + if(!object_has_ref_field(p_obj)) return; + + REF *p_ref; + + if(object_is_array(p_obj)){ /* scan array object */ + Partial_Reveal_Array *array = (Partial_Reveal_Array*)p_obj; + unsigned int array_length = array->array_len; + + p_ref = (REF *)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); + for (unsigned int i = 0; i < array_length; i++) + scan_slot((Collector*)marker, p_ref+i); + + return; + } + + /* scan non-array object */ + unsigned int num_refs = object_ref_field_num(p_obj); + int *ref_iterator = object_ref_iterator_init(p_obj); + + for(unsigned int i=0; itrace_stack; + while(!vector_stack_is_empty(trace_stack)){ + p_obj = (Partial_Reveal_Object*)vector_stack_pop(trace_stack); + scan_object(marker, p_obj); + obj_mark_black_in_table(p_obj); + trace_stack = marker->trace_stack; + } +} + +static Boolean concurrent_mark_need_terminating(GC* gc) +{ + GC_Metadata *metadata = gc->metadata; + return gc_local_snapshot_is_empty(gc) && pool_is_empty(metadata->dirty_obj_snaptshot_pool); +} + +/* for marking phase termination detection */ +static volatile unsigned int num_active_markers = 0; + +void sspace_mark_scan_concurrent(Marker* marker) +{ + int64 time_mark_start = time_now(); + GC *gc = marker->gc; + GC_Metadata *metadata = gc->metadata; + + /* reset the num_finished_collectors to be 0 by one collector. This is necessary for the barrier later. */ + atomic_inc32(&num_active_markers); + + marker->trace_stack = free_task_pool_get_entry(metadata); + + Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool); + + /* first step: copy all root objects to mark tasks. + FIXME:: can be done sequentially before coming here to eliminate atomic ops */ + while(root_set){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); + while(!vector_block_iterator_end(root_set,iter)){ + REF *p_ref = (REF *)*iter; + iter = vector_block_iterator_advance(root_set,iter); + + Partial_Reveal_Object *p_obj = read_slot(p_ref); + /* root ref can't be NULL, (remset may have NULL ref entry, but this function is only for MAJOR_COLLECTION */ + assert(p_obj!=NULL); + /* we have to mark the object before putting it into marktask, because + it is possible to have two slots containing a same object. They will + be scanned twice and their ref slots will be recorded twice. Problem + occurs after the ref slot is updated first time with new position + and the second time the value is the ref slot is the old position as expected. + This can be worked around if we want. + */ + assert(address_belongs_to_gc_heap(p_obj, gc)); + if(obj_mark_gray_in_table(p_obj)) + collector_tracestack_push((Collector*)marker, p_obj); + } + root_set = pool_iterator_next(metadata->gc_rootset_pool); + } + /* put back the last trace_stack task */ + pool_put_entry(metadata->mark_task_pool, marker->trace_stack); + + marker_notify_mark_root_done(marker); + + /*second step: mark dirty object snapshot pool*/ + marker->trace_stack = free_task_pool_get_entry(metadata); + +retry: + + Vector_Block* snapshot_set = pool_get_entry(metadata->dirty_obj_snaptshot_pool); + + while(snapshot_set){ + POINTER_SIZE_INT* iter = vector_block_iterator_init(snapshot_set); + while(!vector_block_iterator_end(snapshot_set,iter)){ + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)*iter; + iter = vector_block_iterator_advance(snapshot_set,iter); + + assert(p_obj!=NULL); //ynhe, restrict? + if(obj_mark_gray_in_table(p_obj)) + collector_tracestack_push((Collector*)marker, p_obj); + } + vector_block_clear(snapshot_set); + pool_put_entry(metadata->free_set_pool, snapshot_set); + snapshot_set = pool_get_entry(metadata->dirty_obj_snaptshot_pool); + } + + /* put back the last trace_stack task */ + pool_put_entry(metadata->mark_task_pool, marker->trace_stack); + + /* third step: iterate over the mark tasks and scan objects */ + /* get a task buf for the mark stack */ + marker->trace_stack = free_task_pool_get_entry(metadata); + + + Vector_Block *mark_task = pool_get_entry(metadata->mark_task_pool); + + while(mark_task){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task); + while(!vector_block_iterator_end(mark_task,iter)){ + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)*iter; + iter = vector_block_iterator_advance(mark_task,iter); + trace_object(marker, p_obj); + } + /* run out one task, put back to the pool and grab another task */ + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + mark_task = pool_get_entry(metadata->mark_task_pool); + } + + /* termination condition: + 1.all thread finished current job. + 2.local snapshot vectors are empty. + 3.global snapshot pool is empty. + */ + atomic_dec32(&num_active_markers); + while(num_active_markers != 0 || !concurrent_mark_need_terminating(gc)){ + if(!pool_is_empty(metadata->mark_task_pool) || !pool_is_empty(metadata->dirty_obj_snaptshot_pool)){ + atomic_inc32(&num_active_markers); + goto retry; + }else{ + /*grab a block from mutator and begin tracing*/ + POINTER_SIZE_INT thread_num = (POINTER_SIZE_INT)marker->thread_handle; + Vector_Block* local_snapshot_set = gc_get_local_snapshot(gc, (unsigned int)(thread_num + 1)); + /*1. If local_snapshot_set has been set full bit, the block is full and will no longer be put into global snapshot pool; + so it should be checked again to see if there're remaining entrys unscanned in it. In this case, the + share bit in local_snapshot_set should not be clear, beacause of rescanning exclusively. + 2. If local_snapshot_set has not been set full bit, the block is used by mutator and has the chance to be put into + global snapshot pool. In this case, we simply cleared the share bit in local_snapshot_set. + */ + if(local_snapshot_set != NULL){ + atomic_inc32(&num_active_markers); + while(!vector_block_is_empty(local_snapshot_set) || !vector_block_not_full_set_unshared(local_snapshot_set)){ + Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*) vector_block_get_entry(local_snapshot_set); + if(obj_mark_gray_in_table(p_obj)) + collector_tracestack_push((Collector*)marker, p_obj); + } + goto retry; + } + } + } + + /* put back the last mark stack to the free pool */ + mark_task = (Vector_Block*)marker->trace_stack; + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + marker->trace_stack = NULL; + assert(pool_is_empty(metadata->dirty_obj_snaptshot_pool)); + + int64 time_mark = time_now() - time_mark_start; + marker->time_mark = time_mark; + + return; +} + +void trace_obj_in_ms_concurrent_mark(Marker *marker, void *p_obj) +{ + obj_mark_gray_in_table((Partial_Reveal_Object*)p_obj); + trace_object(marker, (Partial_Reveal_Object *)p_obj); +} + Index: src/mark_sweep/sspace_mark_sweep.cpp =================================================================== --- src/mark_sweep/sspace_mark_sweep.cpp (revision 569956) +++ src/mark_sweep/sspace_mark_sweep.cpp (working copy) @@ -15,26 +15,31 @@ * limitations under the License. */ +#include "sspace_alloc.h" #include "sspace_mark_sweep.h" #include "sspace_verify.h" #include "gc_ms.h" #include "../gen/gen.h" #include "../thread/collector.h" #include "../finalizer_weakref/finalizer_weakref.h" +#include "../common/fix_repointed_refs.h" +#include "../common/gc_concurrent.h" - +POINTER_SIZE_INT cur_alloc_mask = (~MARK_MASK_IN_TABLE) & FLIP_COLOR_MASK_IN_TABLE; +POINTER_SIZE_INT cur_mark_mask = MARK_MASK_IN_TABLE; POINTER_SIZE_INT cur_alloc_color = OBJ_COLOR_WHITE; -POINTER_SIZE_INT cur_mark_color = OBJ_COLOR_BLACK; -POINTER_SIZE_INT cur_alloc_mask = ~BLACK_MASK_IN_TABLE; -POINTER_SIZE_INT cur_mark_mask = BLACK_MASK_IN_TABLE; +POINTER_SIZE_INT cur_mark_gray_color = OBJ_COLOR_GRAY; +POINTER_SIZE_INT cur_mark_black_color = OBJ_COLOR_BLACK; +static Chunk_Header_Basic *volatile next_chunk_for_fixing; + static void ops_color_flip(void) { POINTER_SIZE_INT temp = cur_alloc_color; - cur_alloc_color = cur_mark_color; - cur_mark_color = temp; - cur_alloc_mask = ~cur_alloc_mask; - cur_mark_mask = ~cur_mark_mask; + cur_alloc_color = cur_mark_black_color; + cur_mark_black_color = temp; + cur_alloc_mask = (~cur_alloc_mask) & FLIP_COLOR_MASK_IN_TABLE; + cur_mark_mask = (~cur_mark_mask) & FLIP_COLOR_MASK_IN_TABLE; } void collector_init_free_chunk_list(Collector *collector) @@ -66,21 +71,134 @@ } +static void sspace_init_chunk_for_ref_fixing(Sspace *sspace) +{ + next_chunk_for_fixing = (Chunk_Header_Basic*)space_heap_start((Space*)sspace); + next_chunk_for_fixing->adj_prev = NULL; +} + +static void nos_init_block_for_forwarding(GC_Gen *gc_gen) +{ blocked_space_block_iterator_init((Blocked_Space*)gc_get_nos(gc_gen)); } + +static inline void block_forward_live_objects(Collector *collector, Sspace *sspace, Block_Header *cur_block) +{ + void *start_pos; + Partial_Reveal_Object *p_obj = block_get_first_marked_object(cur_block, &start_pos); + + while(p_obj ){ + assert(obj_is_marked_in_vt(p_obj)); + obj_clear_dual_bits_in_vt(p_obj); + Partial_Reveal_Object *p_target_obj = collector_forward_object(collector, p_obj); /* Could be implemented with a optimized function */ + if(!p_target_obj){ + assert(collector->gc->collect_result == FALSE); + printf("Out of mem in forwarding nos!\n"); + exit(0); + } + p_obj = block_get_next_marked_object(cur_block, &start_pos); + } +} + +static void collector_forward_nos_to_sspace(Collector *collector, Sspace *sspace) +{ + Blocked_Space *nos = (Blocked_Space*)gc_get_nos((GC_Gen*)collector->gc); + Block_Header *cur_block = blocked_space_block_iterator_next(nos); + + /* We must iterate over all nos blocks to forward live objects in them */ + while(cur_block){ + block_forward_live_objects(collector, sspace, cur_block); + cur_block = blocked_space_block_iterator_next(nos); + } +} + +static void normal_chunk_fix_repointed_refs(Chunk_Header *chunk) +{ + /* Init field slot_index and depad the last index word in table for fixing */ + chunk->slot_index = 0; + chunk_depad_last_index_word(chunk); + + unsigned int alloc_num = chunk->alloc_num; + assert(alloc_num); + + /* After compaction, many chunks are filled with objects. + * For these chunks, we needn't find the allocated slot one by one by calling next_alloc_slot_in_chunk. + * That is a little time consuming. + * We'd like to fix those objects by incrementing their addr to find the next. + */ + if(alloc_num == chunk->slot_num){ /* Filled with objects */ + unsigned int slot_size = chunk->slot_size; + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)slot_index_to_addr(chunk, 0); + for(unsigned int i = alloc_num; i--;){ + object_fix_ref_slots(p_obj); +#ifdef SSPACE_VERIFY + sspace_verify_fix_in_compact(); +#endif + p_obj = (Partial_Reveal_Object*)((POINTER_SIZE_INT)p_obj + slot_size); + } + } else { /* Chunk is not full */ + while(alloc_num){ + Partial_Reveal_Object *p_obj = next_alloc_slot_in_chunk(chunk); + assert(p_obj); + object_fix_ref_slots(p_obj); +#ifdef SSPACE_VERIFY + sspace_verify_fix_in_compact(); +#endif + --alloc_num; + } + } + + if(chunk->alloc_num != chunk->slot_num){ + chunk_pad_last_index_word(chunk, cur_alloc_mask); + pfc_reset_slot_index(chunk); + } +} + +static void abnormal_chunk_fix_repointed_refs(Chunk_Header *chunk) +{ + object_fix_ref_slots((Partial_Reveal_Object*)chunk->base); +#ifdef SSPACE_VERIFY + sspace_verify_fix_in_compact(); +#endif +} + +static void sspace_fix_repointed_refs(Collector *collector, Sspace *sspace) +{ + Chunk_Header_Basic *chunk = sspace_grab_next_chunk(sspace, &next_chunk_for_fixing, TRUE); + + while(chunk){ + if(chunk->status & CHUNK_NORMAL) + normal_chunk_fix_repointed_refs((Chunk_Header*)chunk); + else if(chunk->status & CHUNK_ABNORMAL) + abnormal_chunk_fix_repointed_refs((Chunk_Header*)chunk); + + chunk = sspace_grab_next_chunk(sspace, &next_chunk_for_fixing, TRUE); + } +} + + static volatile unsigned int num_marking_collectors = 0; static volatile unsigned int num_sweeping_collectors = 0; +static volatile unsigned int num_compacting_collectors = 0; +static volatile unsigned int num_forwarding_collectors = 0; +static volatile unsigned int num_fixing_collectors = 0; void mark_sweep_sspace(Collector *collector) { GC *gc = collector->gc; Sspace *sspace = gc_get_sspace(gc); + Space *nos = NULL; + if(gc_match_kind(gc, MAJOR_COLLECTION)) + nos = gc_get_nos((GC_Gen*)gc); unsigned int num_active_collectors = gc->num_active_collectors; /* Pass 1: ************************************************** - mark all live objects in heap ****************************/ + Mark all live objects in heap ****************************/ atomic_cas32(&num_marking_collectors, 0, num_active_collectors+1); - sspace_mark_scan(collector); + if(gc_match_kind(gc, FALLBACK_COLLECTION)) + sspace_fallback_mark_scan(collector, sspace); + else + sspace_mark_scan(collector, sspace); unsigned int old_num = atomic_inc32(&num_marking_collectors); if( ++old_num == num_active_collectors ){ @@ -104,7 +222,7 @@ while(num_marking_collectors != num_active_collectors + 1); /* Pass 2: ************************************************** - sweep dead objects ***************************************/ + Sweep dead objects ***************************************/ atomic_cas32( &num_sweeping_collectors, 0, num_active_collectors+1); sspace_sweep(collector, sspace); @@ -121,37 +239,80 @@ #ifdef SSPACE_VERIFY sspace_verify_after_sweep(gc); #endif - if(sspace->need_compact){ + + sspace_merge_free_chunks(gc, sspace); + + if(gc_match_kind(gc, MAJOR_COLLECTION)) + nos_init_block_for_forwarding((GC_Gen*)gc); + if(sspace->need_compact) sspace_init_pfc_pool_iterator(sspace); - } + if(sspace->need_fix) + sspace_init_chunk_for_ref_fixing(sspace); /* let other collectors go */ num_sweeping_collectors++; } while(num_sweeping_collectors != num_active_collectors + 1); - if(sspace->need_compact) - compact_sspace(collector, sspace); + /* Optional Pass: ******************************************* + Forward live obj in nos to mos (sspace) ******************/ + if(gc_match_kind(gc, MAJOR_COLLECTION)){ + atomic_cas32( &num_forwarding_collectors, 0, num_active_collectors); + + collector_forward_nos_to_sspace(collector, sspace); + + atomic_inc32(&num_forwarding_collectors); + while(num_forwarding_collectors != num_active_collectors); + } + /* Optional Pass: ******************************************* + Compact pfcs with the same size **************************/ + if(sspace->need_compact){ + atomic_cas32(&num_compacting_collectors, 0, num_active_collectors+1); + + sspace_compact(collector, sspace); + + /* If we need forward nos to mos, i.e. in major collection, an extra fixing phase after compaction is needed. */ + old_num = atomic_inc32(&num_compacting_collectors); + if( ++old_num == num_active_collectors ){ + sspace_remerge_free_chunks(gc, sspace); + /* let other collectors go */ + num_compacting_collectors++; + } + while(num_compacting_collectors != num_active_collectors + 1); + } + + /* Optional Pass: ******************************************* + Fix repointed refs ***************************************/ + if(sspace->need_fix){ + atomic_cas32( &num_fixing_collectors, 0, num_active_collectors); + + sspace_fix_repointed_refs(collector, sspace); + + atomic_inc32(&num_fixing_collectors); + while(num_fixing_collectors != num_active_collectors); + } + if( collector->thread_handle != 0 ) return; - if(sspace->need_compact){ + /* Leftover: *************************************************/ + + if(sspace->need_fix){ gc_fix_rootset(collector); #ifdef SSPACE_TIME sspace_fix_time(FALSE); #endif } - gc_collect_free_chunks(gc, sspace); -#ifdef SSPACE_TIME - sspace_merge_time(FALSE); -#endif - - /* Leftover: ************************************************ */ - - gc->root_set = NULL; // FIXME:: should be placed to a more appopriate place + //gc->root_set = NULL; // FIXME:: should be placed to a more appopriate place gc_set_pool_clear(gc->metadata->gc_rootset_pool); +#ifdef USE_MARK_SWEEP_GC + sspace_set_space_statistic(sspace); +#endif + if(gc_match_kind(gc, MAJOR_COLLECTION)) + gc_clear_collector_local_chunks(gc); + #ifdef SSPACE_VERIFY sspace_verify_after_collection(gc); #endif Index: src/mark_sweep/sspace_mark_sweep.h =================================================================== --- src/mark_sweep/sspace_mark_sweep.h (revision 569956) +++ src/mark_sweep/sspace_mark_sweep.h (working copy) @@ -27,22 +27,34 @@ inline Boolean chunk_is_reusable(Chunk_Header *chunk) { return (float)(chunk->slot_num-chunk->alloc_num)/chunk->slot_num > PFC_REUSABLE_RATIO; } +#define OBJ_ALLOC_BIT_IN_TABLE 0x01 +#define OBJ_BLACK_BIT_IN_TABLE 0x02 +#define OBJ_GRAY_BIT_IN_TABLE 0x04 +#define OBJ_COLOR_BIT_IN_TABLE 0x06 +#define OBJ_DIRTY_BIT_IN_TABLE 0x08 + enum Obj_Color { - OBJ_COLOR_BLUE = 0x0, - OBJ_COLOR_WHITE = 0x1, - OBJ_COLOR_BLACK = 0x2, - OBJ_COLOR_GRAY = 0x3, - OBJ_COLOR_MASK = 0x3 + OBJ_COLOR_BLUE = 0x0, + OBJ_COLOR_WHITE = OBJ_ALLOC_BIT_IN_TABLE, + OBJ_COLOR_GRAY = OBJ_GRAY_BIT_IN_TABLE, + OBJ_COLOR_BLACK = OBJ_BLACK_BIT_IN_TABLE, + OBJ_COLOR_MASK = OBJ_COLOR_BIT_IN_TABLE }; #ifdef POINTER64 - #define BLACK_MASK_IN_TABLE ((POINTER_SIZE_INT)0xAAAAAAAAAAAAAAAA) + //#define BLACK_MASK_IN_TABLE ((POINTER_SIZE_INT)0xAAAAAAAAAAAAAAAA) + #define MARK_MASK_IN_TABLE ((POINTER_SIZE_INT)0x2222222222222222) + #define FLIP_COLOR_MASK_IN_TABLE ((POINTER_SIZE_INT)0x3333333333333333) + //#define DIRY_MASK_IN_TABLE ((POINTER_SIZE_INT)0x4444444444444444) #else - #define BLACK_MASK_IN_TABLE ((POINTER_SIZE_INT)0xAAAAAAAA) + #define MARK_MASK_IN_TABLE ((POINTER_SIZE_INT)0x22222222) + #define FLIP_COLOR_MASK_IN_TABLE ((POINTER_SIZE_INT)0x33333333) + //#define DIRY_MASK_IN_TABLE ((POINTER_SIZE_INT)0x44444444) #endif extern POINTER_SIZE_INT cur_alloc_color; -extern POINTER_SIZE_INT cur_mark_color; +extern POINTER_SIZE_INT cur_mark_gray_color; +extern POINTER_SIZE_INT cur_mark_black_color; extern POINTER_SIZE_INT cur_alloc_mask; extern POINTER_SIZE_INT cur_mark_mask; @@ -73,7 +85,25 @@ return &chunk->table[word_index]; } +inline POINTER_SIZE_INT *get_color_word_in_table(Partial_Reveal_Object *obj, unsigned int &index_in_word, unsigned int size) +{ + Chunk_Header *chunk; + unsigned int index; + + if(size > SUPER_OBJ_THRESHOLD){ + chunk = ABNORMAL_CHUNK_HEADER(obj); + index = 0; + } else { + chunk = NORMAL_CHUNK_HEADER(obj); + index = slot_addr_to_index(chunk, obj); + } + unsigned int word_index = index >> 3; + index_in_word = COLOR_BITS_PER_OBJ * (index & (((unsigned int)(SLOT_NUM_PER_WORD_IN_TABLE-1)))); + + return &chunk->table[word_index]; +} +#if 0 /* Accurate marking: TRUE stands for being marked by this collector, and FALSE for another collector */ inline Boolean obj_mark_in_table(Partial_Reveal_Object *obj) { @@ -106,6 +136,207 @@ return FALSE; } +#endif + +inline Boolean obj_is_mark_gray_in_table(Partial_Reveal_Object *obj) +{ + POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + POINTER_SIZE_INT current_word = *p_color_word; + POINTER_SIZE_INT mark_gray_color = cur_mark_gray_color << index_in_word; + POINTER_SIZE_INT mark_black_color = cur_mark_black_color<< index_in_word; + + + if(current_word & mark_gray_color && !(current_word & mark_black_color)) + return TRUE; + else + return FALSE; +} + +inline Boolean obj_is_mark_black_in_table(Partial_Reveal_Object *obj) +{ + POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + POINTER_SIZE_INT current_word = *p_color_word; + POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; + + if(current_word & mark_black_color) + return TRUE; + else + return FALSE; + +} + +inline Boolean obj_is_mark_black_in_table(Partial_Reveal_Object *obj, unsigned int size) +{ + POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word, size); + POINTER_SIZE_INT current_word = *p_color_word; + POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; + + if(current_word & mark_black_color) + return TRUE; + else + return FALSE; + +} + + +inline Boolean obj_mark_gray_in_table(Partial_Reveal_Object *obj) +{ + volatile POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + assert(p_color_word); + + //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); + POINTER_SIZE_INT mark_color = cur_mark_gray_color << index_in_word; + + POINTER_SIZE_INT old_word = *p_color_word; + if(old_word & mark_color) return FALSE; /*already marked gray or black.*/ + + //POINTER_SIZE_INT new_word = (old_word & color_bits_mask) | mark_color; + POINTER_SIZE_INT new_word = old_word | mark_color; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return TRUE; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + if(old_word & mark_color) return FALSE; /*already marked gray or black.*/ + + //new_word = (old_word & color_bits_mask) | mark_color; + new_word = old_word | mark_color; + } + + return FALSE; +} + +inline Boolean obj_mark_black_in_table(Partial_Reveal_Object *obj, unsigned int size) +{ + //assert(obj_is_mark_in_table(obj)); + volatile POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word, size); + assert(p_color_word); + + //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); + POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; + + POINTER_SIZE_INT old_word = *p_color_word; + if(old_word & mark_black_color) return FALSE; /*already marked black*/ + + POINTER_SIZE_INT new_word = old_word | mark_black_color; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return TRUE; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + if(old_word & mark_black_color) return FALSE; /*already marked black*/ + + new_word = old_word | mark_black_color; + } + + return FALSE; + +} + +inline Boolean obj_mark_black_in_table(Partial_Reveal_Object *obj) +{ + // assert(obj_is_mark_in_table(obj)); + volatile POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + assert(p_color_word); + + POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); + POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; + + POINTER_SIZE_INT old_word = *p_color_word; + if(obj_is_mark_black_in_table(obj)) return FALSE; /*already marked black*/ + + POINTER_SIZE_INT new_word = old_word | mark_black_color; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return TRUE; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + if(obj_is_mark_black_in_table(obj)) return FALSE; /*already marked black*/ + + new_word = old_word | mark_black_color; + } + + return FALSE; +} + +inline Boolean obj_dirty_in_table(Partial_Reveal_Object *obj) +{ + volatile POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + assert(p_color_word); + + POINTER_SIZE_INT obj_dirty_bit_in_word = OBJ_DIRTY_BIT_IN_TABLE<< index_in_word; + + POINTER_SIZE_INT old_word = *p_color_word; + if(old_word & obj_dirty_bit_in_word) return FALSE; + + POINTER_SIZE_INT new_word = old_word | obj_dirty_bit_in_word; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return TRUE; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + if(old_word & obj_dirty_bit_in_word) return FALSE; + + new_word = old_word | obj_dirty_bit_in_word; + } + + return FALSE; +} + +inline Boolean obj_is_dirty_in_table(Partial_Reveal_Object *obj) +{ + POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + POINTER_SIZE_INT current_word = *p_color_word; + POINTER_SIZE_INT obj_dirty_bit_in_word = OBJ_DIRTY_BIT_IN_TABLE<< index_in_word; + + + if(current_word & obj_dirty_bit_in_word) + return TRUE; + else + return FALSE; +} + +inline Boolean obj_is_alloc_color_in_table(Partial_Reveal_Object *obj) +{ + POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + POINTER_SIZE_INT current_word = *p_color_word; + POINTER_SIZE_INT obj_alloc_color_bit_in_word = cur_alloc_color<< index_in_word; + + + if(current_word & obj_alloc_color_bit_in_word) + return TRUE; + else + return FALSE; + +} + +inline Boolean obj_need_take_snaptshot(Partial_Reveal_Object *obj) +{ + return !obj_is_mark_black_in_table(obj) && !obj_is_dirty_in_table(obj); +} + inline void collector_add_free_chunk(Collector *collector, Free_Chunk *chunk) { Free_Chunk_List *list = collector->free_chunk_list; @@ -121,11 +352,13 @@ } -extern void sspace_mark_scan(Collector *collector); +extern void sspace_mark_scan(Collector *collector, Sspace *sspace); +extern void sspace_fallback_mark_scan(Collector *collector, Sspace *sspace); extern void gc_init_chunk_for_sweep(GC *gc, Sspace *sspace); extern void sspace_sweep(Collector *collector, Sspace *sspace); -extern void compact_sspace(Collector *collector, Sspace *sspace); -extern void gc_collect_free_chunks(GC *gc, Sspace *sspace); +extern void sspace_compact(Collector *collector, Sspace *sspace); +extern void sspace_merge_free_chunks(GC *gc, Sspace *sspace); +extern void sspace_remerge_free_chunks(GC *gc, Sspace *sspace); extern Chunk_Header_Basic *sspace_grab_next_chunk(Sspace *sspace, Chunk_Header_Basic *volatile *shared_next_chunk, Boolean need_construct); extern void pfc_set_slot_index(Chunk_Header *chunk, unsigned int first_free_word_index, POINTER_SIZE_INT alloc_color); Index: src/mark_sweep/sspace_sweep.cpp =================================================================== --- src/mark_sweep/sspace_sweep.cpp (revision 569956) +++ src/mark_sweep/sspace_sweep.cpp (working copy) @@ -76,7 +76,7 @@ assert(live_num >= slot_index); live_num -= slot_index; POINTER_SIZE_INT index_word = table[word_index]; - POINTER_SIZE_INT mark_color = cur_mark_color << (COLOR_BITS_PER_OBJ * (slot_index % SLOT_NUM_PER_WORD_IN_TABLE)); + POINTER_SIZE_INT mark_color = cur_mark_black_color << (COLOR_BITS_PER_OBJ * (slot_index % SLOT_NUM_PER_WORD_IN_TABLE)); for(; slot_index < slot_num; ++slot_index){ assert(!(index_word & ~cur_mark_mask)); if(index_word & mark_color){ @@ -100,7 +100,7 @@ } mark_color <<= COLOR_BITS_PER_OBJ; if(!mark_color){ - mark_color = cur_mark_color; + mark_color = cur_mark_black_color; ++word_index; index_word = table[word_index]; while(index_word == cur_mark_mask && cur_free_slot_num == 0 && slot_index < slot_num){ @@ -137,14 +137,14 @@ live_num += live_num_in_word; if((first_free_word_index == MAX_SLOT_INDEX) && (live_num_in_word < SLOT_NUM_PER_WORD_IN_TABLE)){ first_free_word_index = i; - pfc_set_slot_index((Chunk_Header*)chunk, first_free_word_index, cur_mark_color); + pfc_set_slot_index((Chunk_Header*)chunk, first_free_word_index, cur_mark_black_color); } } assert(live_num <= slot_num); chunk->alloc_num = live_num; -#ifdef SSPACE_VERIFY + collector->live_obj_size += live_num * chunk->slot_size; collector->live_obj_num += live_num; -#endif + if(!live_num){ /* all objects in this chunk are dead */ collector_add_free_chunk(collector, (Free_Chunk*)chunk); } else if(chunk_is_reusable(chunk)){ /* most objects in this chunk are swept, add chunk to pfc list*/ @@ -163,19 +163,17 @@ if(!table[0]){ collector_add_free_chunk(collector, (Free_Chunk*)chunk); } -#ifdef SSPACE_VERIFY else { + collector->live_obj_size += CHUNK_SIZE(chunk); collector->live_obj_num++; } -#endif } void sspace_sweep(Collector *collector, Sspace *sspace) { Chunk_Header_Basic *chunk; -#ifdef SSPACE_VERIFY + collector->live_obj_size = 0; collector->live_obj_num = 0; -#endif chunk = sspace_grab_next_chunk(sspace, &next_chunk_for_sweep, TRUE); while(chunk){ @@ -192,70 +190,77 @@ } } -static void free_list_detach_chunk(Free_Chunk_List *list, Free_Chunk *chunk) -{ - if(chunk->prev) - chunk->prev->next = chunk->next; - else // chunk is the head - list->head = chunk->next; - if(chunk->next) - chunk->next->prev = chunk->prev; -} +/************ For merging free chunks in sspace ************/ -void gc_collect_free_chunks(GC *gc, Sspace *sspace) +static void merge_free_chunks_in_list(Sspace *sspace, Free_Chunk_List *list) { Free_Chunk *sspace_ceiling = (Free_Chunk*)space_heap_end((Space*)sspace); + Free_Chunk *chunk = list->head; - Free_Chunk_List free_chunk_list; - free_chunk_list.head = NULL; - free_chunk_list.tail = NULL; - - /* Collect free chunks from collectors to one list */ - for(unsigned int i=0; inum_collectors; ++i){ - Free_Chunk_List *list = gc->collectors[i]->free_chunk_list; - if(free_chunk_list.tail){ - free_chunk_list.head->prev = list->tail; - } else { - free_chunk_list.tail = list->tail; - } - if(list->head){ - list->tail->next = free_chunk_list.head; - free_chunk_list.head = list->head; - } - list->head = NULL; - list->tail = NULL; - } - - Free_Chunk *chunk = free_chunk_list.head; while(chunk){ assert(chunk->status == (CHUNK_FREE | CHUNK_TO_MERGE)); /* Remove current chunk from the chunk list */ - free_chunk_list.head = chunk->next; - if(free_chunk_list.head) - free_chunk_list.head->prev = NULL; - /* Check if the back adjcent chunks are free */ - Free_Chunk *back_chunk = (Free_Chunk*)chunk->adj_next; - while(back_chunk < sspace_ceiling && back_chunk->status == (CHUNK_FREE | CHUNK_TO_MERGE)){ - /* Remove back_chunk from list */ - free_list_detach_chunk(&free_chunk_list, back_chunk); - chunk->adj_next = back_chunk->adj_next; - back_chunk = (Free_Chunk*)chunk->adj_next; - } + list->head = chunk->next; + if(list->head) + list->head->prev = NULL; /* Check if the prev adjacent chunks are free */ Free_Chunk *prev_chunk = (Free_Chunk*)chunk->adj_prev; while(prev_chunk && prev_chunk->status == (CHUNK_FREE | CHUNK_TO_MERGE)){ + assert(prev_chunk < chunk); /* Remove prev_chunk from list */ - free_list_detach_chunk(&free_chunk_list, prev_chunk); + free_list_detach_chunk(list, prev_chunk); prev_chunk->adj_next = chunk->adj_next; chunk = prev_chunk; prev_chunk = (Free_Chunk*)chunk->adj_prev; } + /* Check if the back adjcent chunks are free */ + Free_Chunk *back_chunk = (Free_Chunk*)chunk->adj_next; + while(back_chunk < sspace_ceiling && back_chunk->status == (CHUNK_FREE | CHUNK_TO_MERGE)){ + assert(chunk < back_chunk); + /* Remove back_chunk from list */ + free_list_detach_chunk(list, back_chunk); + back_chunk = (Free_Chunk*)back_chunk->adj_next; + chunk->adj_next = (Chunk_Header_Basic*)back_chunk; + } + if(back_chunk < sspace_ceiling) + back_chunk->adj_prev = (Chunk_Header_Basic*)chunk; - //zeroing_free_chunk(chunk); - /* put the free chunk to the according free chunk list */ sspace_put_free_chunk(sspace, chunk); - chunk = free_chunk_list.head; + chunk = list->head; } } + +void sspace_merge_free_chunks(GC *gc, Sspace *sspace) +{ + Free_Chunk_List free_chunk_list; + free_chunk_list.head = NULL; + free_chunk_list.tail = NULL; + + /* Collect free chunks from collectors to one list */ + for(unsigned int i=0; inum_collectors; ++i){ + Free_Chunk_List *list = gc->collectors[i]->free_chunk_list; + move_free_chunks_between_lists(&free_chunk_list, list); + } + + merge_free_chunks_in_list(sspace, &free_chunk_list); +} + +void sspace_remerge_free_chunks(GC *gc, Sspace *sspace) +{ + Free_Chunk_List free_chunk_list; + free_chunk_list.head = NULL; + free_chunk_list.tail = NULL; + + /* Collect free chunks from sspace free chunk lists to one list */ + sspace_collect_free_chunks_to_list(sspace, &free_chunk_list); + + /* Collect free chunks from collectors to one list */ + for(unsigned int i=0; inum_collectors; ++i){ + Free_Chunk_List *list = gc->collectors[i]->free_chunk_list; + move_free_chunks_between_lists(&free_chunk_list, list); + } + + merge_free_chunks_in_list(sspace, &free_chunk_list); +} Index: src/mark_sweep/sspace_verify.cpp =================================================================== --- src/mark_sweep/sspace_verify.cpp (revision 569956) +++ src/mark_sweep/sspace_verify.cpp (working copy) @@ -345,6 +345,41 @@ return live_num; } +static void allocator_verify_local_chunks(Allocator *allocator) +{ + Sspace *sspace = gc_get_sspace(allocator->gc); + Size_Segment **size_segs = sspace->size_segments; + Chunk_Header ***local_chunks = allocator->local_chunks; + + for(unsigned int i = SIZE_SEGMENT_NUM; i--;){ + if(!size_segs[i]->local_alloc){ + assert(!local_chunks[i]); + continue; + } + Chunk_Header **chunks = local_chunks[i]; + assert(chunks); + for(unsigned int j = size_segs[i]->chunk_num; j--;){ + assert(!chunks[j]); + } + } +} + +static void gc_verify_allocator_local_chunks(GC *gc) +{ + if(gc_match_kind(gc, MARK_SWEEP_GC)){ + Mutator *mutator = gc->mutator_list; + while(mutator){ + allocator_verify_local_chunks((Allocator*)mutator); + mutator = mutator->next; + } + } + + if(gc_match_kind(gc, MAJOR_COLLECTION)) + for(unsigned int i = gc->num_collectors; i--;){ + allocator_verify_local_chunks((Allocator*)gc->collectors[i]); + } +} + void sspace_verify_before_collection(GC *gc) { printf("Allocated obj: %d\n", alloc_obj_num); @@ -374,6 +409,7 @@ POINTER_SIZE_INT total_live_obj = sspace_live_obj_num(sspace, TRUE); printf("Live obj after collection: %d\n", total_live_obj); check_and_clear_mark_cards(); + gc_verify_allocator_local_chunks(gc); } /* @@ -398,6 +434,7 @@ /* sspace verify marking with vtable marking in advance */ +Sspace *sspace_in_verifier; static Pool *trace_pool = NULL; static Vector_Block *trace_stack = NULL; POINTER_SIZE_INT live_obj_in_verify_marking = 0; @@ -429,7 +466,7 @@ Partial_Reveal_Object *p_obj = read_slot(p_ref); if( p_obj == NULL) return; - if(obj_mark_in_vtable(gc, p_obj)) + if(obj_belongs_to_space(p_obj, (Space*)sspace_in_verifier) && obj_mark_in_vtable(gc, p_obj)) tracestack_push(p_obj); return; @@ -479,6 +516,7 @@ void sspace_verify_vtable_mark(GC *gc) { + sspace_in_verifier = gc_get_sspace(gc); GC_Metadata *metadata = gc->metadata; Pool *rootset_pool = metadata->gc_rootset_pool; @@ -496,7 +534,7 @@ Partial_Reveal_Object *p_obj = read_slot(p_ref); assert(p_obj!=NULL); - if(obj_mark_in_vtable(gc, p_obj)) + if(obj_belongs_to_space(p_obj, (Space*)sspace_in_verifier) && obj_mark_in_vtable(gc, p_obj)) tracestack_push(p_obj); } root_set = pool_iterator_next(metadata->gc_rootset_pool); Index: src/thread/collector.cpp =================================================================== --- src/thread/collector.cpp (revision 569956) +++ src/thread/collector.cpp (working copy) @@ -103,7 +103,7 @@ #ifndef USE_MARK_SWEEP_GC /*For LOS_Shrink and LOS_Extend*/ - if(collector->gc->tuner->kind != TRANS_NOTHING){ + if(gc_has_space_tuner(collector->gc) && collector->gc->tuner->kind != TRANS_NOTHING){ collector->non_los_live_obj_size = 0; collector->los_live_obj_size = 0; } @@ -137,9 +137,9 @@ { /* FIXME:: to adaptively identify the num_collectors_to_activate */ if( MINOR_COLLECTORS && gc_match_kind(gc, MINOR_COLLECTION)){ - gc->num_active_collectors = MINOR_COLLECTORS; - }else if ( MAJOR_COLLECTORS && !gc_match_kind(gc, MINOR_COLLECTION)){ - gc->num_active_collectors = MAJOR_COLLECTORS; + gc->num_active_collectors = MINOR_COLLECTORS; + }else if ( MAJOR_COLLECTORS && gc_match_kind(gc, MAJOR_COLLECTION)){ + gc->num_active_collectors = MAJOR_COLLECTORS; }else{ gc->num_active_collectors = gc->num_collectors; } @@ -291,6 +291,8 @@ #ifdef USE_MARK_SWEEP_GC collector_init_free_chunk_list(collector); +#else + gc_gen_hook_for_collector_init(collector); #endif #ifdef GC_GEN_STATS Index: src/thread/collector.h =================================================================== --- src/thread/collector.h (revision 569956) +++ src/thread/collector.h (working copy) @@ -23,7 +23,6 @@ #define _COLLECTOR_H_ #include "../common/gc_space.h" -#include "../mark_sweep/sspace_verify.h" struct Block_Header; struct Stealable_Stack; @@ -70,9 +69,9 @@ Block_Header* cur_target_block; Free_Chunk_List *free_chunk_list; -#ifdef SSPACE_VERIFY + + POINTER_SIZE_INT live_obj_size; POINTER_SIZE_INT live_obj_num; -#endif void(*task_func)(void*) ; /* current task */ @@ -100,6 +99,10 @@ void collector_attach_hashcode(Collector *collector); #endif +#ifndef USE_MARK_SWEEP_GC +void gc_gen_hook_for_collector_init(Collector *collector); +#endif + inline Boolean gc_collection_result(GC* gc) { Boolean result = TRUE; @@ -122,5 +125,4 @@ } - #endif //#ifndef _COLLECTOR_H_ Index: src/thread/collector_alloc.h =================================================================== --- src/thread/collector_alloc.h (revision 569956) +++ src/thread/collector_alloc.h (working copy) @@ -27,7 +27,7 @@ #include "../common/hashcode.h" #endif -void* mos_alloc(unsigned size, Allocator *allocator); +extern Space_Alloc_Func mos_alloc; /* NOS forward obj to MOS in MINOR_COLLECTION */ FORCE_INLINE Partial_Reveal_Object* collector_forward_object(Collector* collector, Partial_Reveal_Object* p_obj) Index: src/thread/gc_thread.cpp =================================================================== --- src/thread/gc_thread.cpp (revision 569956) +++ src/thread/gc_thread.cpp (working copy) @@ -23,7 +23,9 @@ static hythread_tls_key_t tls_gc_key; POINTER_SIZE_INT tls_gc_offset; +hythread_group_t gc_thread_group = NULL; + void gc_tls_init() { hythread_tls_alloc(&tls_gc_key); Index: src/thread/marker.cpp =================================================================== --- src/thread/marker.cpp (revision 0) +++ src/thread/marker.cpp (revision 0) @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "marker.h" +#include "../finalizer_weakref/finalizer_weakref.h" + +#include "../gen/gen.h" +#include "../mark_sweep/gc_ms.h" + +unsigned int NUM_MARKERS = 0; +static volatile unsigned int live_marker_num = 0; + +void notify_marker_to_work(Marker* marker) +{ + vm_set_event(marker->task_assigned_event); +} + +void wait_marker_to_finish(Marker* marker) +{ + vm_wait_event(marker->task_finished_event); +} + +void marker_wait_for_task(Marker* marker) +{ + vm_wait_event(marker->task_assigned_event); +} + +void marker_notify_work_done(Marker* marker) +{ + vm_set_event(marker->task_finished_event); +} + +void wait_marker_finish_mark_root(Marker* marker) +{ + vm_wait_event(marker->markroot_finished_event); +} + +void marker_notify_mark_root_done(Marker* marker) +{ + vm_set_event(marker->markroot_finished_event); +} + +static int marker_thread_func(void *arg) +{ + Marker* marker = (Marker *)arg; + assert(marker); + + while(true){ + /* Waiting for newly assigned task */ + marker_wait_for_task(marker); + marker->marker_is_active = TRUE; + + /* waken up and check for new task */ + TaskType task_func = marker->task_func; + if(task_func == NULL){ + atomic_dec32(&live_marker_num); + return 1; + } + + task_func(marker); + + marker_notify_work_done(marker); + + marker->marker_is_active = FALSE; + } + + return 0; +} + +static void marker_init_thread(Marker* marker) +{ + int status = vm_create_event(&marker->task_assigned_event); + assert(status == THREAD_OK); + + status = vm_create_event(&marker->task_finished_event); + assert(status == THREAD_OK); + + status = vm_create_event(&marker->markroot_finished_event); + assert(status == THREAD_OK); + + status = (unsigned int)vm_create_thread(marker_thread_func, (void*)marker); + + assert(status == THREAD_OK); + + return; +} + +void marker_initialize(GC* gc) +{ + unsigned int num_processors = gc_get_processor_num(gc); + + unsigned int nthreads = max(NUM_MARKERS,num_processors); + + unsigned int size = sizeof(Marker*) * nthreads; + gc->markers = (Marker **) STD_MALLOC(size); + memset(gc->markers, 0, size); + + size = sizeof(Marker); + for (unsigned int i = 0; i < nthreads; i++) { + Marker* marker = (Marker *)STD_MALLOC(size); + memset(marker, 0, size); + + /* FIXME:: thread_handle is for temporary control */ + marker->thread_handle = (VmThreadHandle)(POINTER_SIZE_INT)i; + marker->gc = gc; + marker->marker_is_active = FALSE; + marker_init_thread(marker); + + gc->markers[i] = marker; + } + + gc->num_markers = NUM_MARKERS? NUM_MARKERS:num_processors; + live_marker_num = NUM_MARKERS; + + return; + +} + +void marker_terminate_thread(Marker* marker) +{ + assert(live_marker_num); + unsigned int old_live_marker_num = live_marker_num; + marker->task_func = NULL; /* NULL to notify thread exit */ + if(marker->marker_is_active) wait_marker_to_finish(marker); + notify_marker_to_work(marker); + while(old_live_marker_num == live_marker_num) + vm_thread_yield(); /* give marker time to die */ + + delete marker->trace_stack; + return; + +} + +void marker_destruct(GC* gc) +{ + for(unsigned int i=0; inum_markers; i++) + { + Marker* marker = gc->markers[i]; + marker_terminate_thread(marker); + STD_FREE(marker); + + } + assert(live_marker_num == 0); + + STD_FREE(gc->markers); + return; +} + +void wait_mark_finish(GC* gc) +{ + unsigned int num_active_marker = gc->num_active_markers; + for(unsigned int i=0; imarkers[i]; + wait_marker_to_finish(marker); + } + return; +} + +Boolean is_mark_finished(GC* gc) +{ + unsigned int num_active_marker = gc->num_active_markers; + unsigned int i = 0; + for(; imarkers[i]; + if(marker->marker_is_active){ + return FALSE; + } + } + return TRUE; +} + + +void marker_reset_thread(Marker* marker) +{ + marker->task_func = NULL; + +#ifndef BUILD_IN_REFERENT + collector_reset_weakref_sets((Collector*)marker); +#endif + + return; + +} + +void assign_marker_with_task(GC* gc, TaskType task_func, Space* space) +{ + for(unsigned int i=0; inum_markers; i++) + { + Marker* marker = gc->markers[i]; + + marker_reset_thread(marker); + marker->task_func = task_func; + marker->mark_space= space; + notify_marker_to_work(marker); + } + return; +} + +void assign_marker_with_task(GC* gc, TaskType task_func, Space* space, unsigned int num_markers) +{ + unsigned int i = gc->num_active_markers; + gc->num_active_markers += num_markers; + for(; i < gc->num_active_markers; i++) + { + printf("start mark thread %d \n", i); + Marker* marker = gc->markers[i]; + + marker_reset_thread(marker); + marker->task_func = task_func; + marker->mark_space= space; + notify_marker_to_work(marker); + } + return; +} + +void marker_execute_task(GC* gc, TaskType task_func, Space* space) +{ + assign_marker_with_task(gc, task_func, space); + wait_mark_finish(gc); + return; +} + +void wait_mark_root_finish(GC* gc) +{ + unsigned int num_marker = gc->num_markers; + for(unsigned int i=0; imarkers[i]; + wait_marker_finish_mark_root(marker); + } + return; +} + +void wait_mark_root_finish(GC* gc, unsigned int num_markers) +{ + unsigned int num_active_marker = gc->num_active_markers; + unsigned int i= num_active_marker - num_markers; + for(; i < num_active_marker; i++) + { + Marker* marker = gc->markers[i]; + wait_marker_finish_mark_root(marker); + } + return; +} + +void marker_execute_task_concurrent(GC* gc, TaskType task_func, Space* space) +{ + assign_marker_with_task(gc, task_func, space); + wait_mark_root_finish(gc); + return; +} + +void marker_execute_task_concurrent(GC* gc, TaskType task_func, Space* space, unsigned int num_markers) +{ + unsigned int num_free_markers = gc->num_markers - gc->num_active_markers; + if(num_markers > num_free_markers) + num_markers = num_free_markers; + assign_marker_with_task(gc, task_func, space,num_markers); + wait_mark_root_finish(gc, num_markers); + return; +} + Index: src/thread/marker.h =================================================================== --- src/thread/marker.h (revision 0) +++ src/thread/marker.h (revision 0) @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _MARKER_H_ +#define _MARKER_H_ + +#include "../common/gc_space.h" +#include "../mark_sweep/sspace_chunk.h" + +typedef struct Marker{ + /* <-- first couple of fields are overloaded as Allocator */ + void *free; + void *ceiling; + void *end; + void *alloc_block; + Chunk_Header ***local_chunks; + Space* alloc_space; + GC* gc; + VmThreadHandle thread_handle; /* This thread; */ + /* End of Allocator --> */ + + /* FIXME:: for testing */ + Space* mark_space; + + Vector_Block *trace_stack; + + Vector_Block* rep_set; /* repointed set */ + Vector_Block* rem_set; +#ifdef USE_32BITS_HASHCODE + Vector_Block* hashcode_set; +#endif + + Vector_Block *softref_set; + Vector_Block *weakref_set; + Vector_Block *phanref_set; + + VmEventHandle task_assigned_event; + VmEventHandle task_finished_event; + + Block_Header* cur_compact_block; + Block_Header* cur_target_block; + + Free_Chunk_List *free_chunk_list; + + POINTER_SIZE_INT live_obj_size; + POINTER_SIZE_INT live_obj_num; + + void(*task_func)(void*) ; /* current task */ + + POINTER_SIZE_INT non_los_live_obj_size; + POINTER_SIZE_INT los_live_obj_size; + POINTER_SIZE_INT segment_live_size[NORMAL_SIZE_SEGMENT_NUM]; + unsigned int result; + + VmEventHandle markroot_finished_event; + + Boolean marker_is_active; + int64 time_mark; + Marker* next; +} Marker; + +typedef Marker* Marker_List; + +#define MAX_NUM_MARKERS 0xff +#define MIN_NUM_MARKERS 0x01 + +void marker_initialize(GC* gc); +void marker_destruct(GC* gc); + +void marker_execute_task(GC* gc, TaskType task_func, Space* space); +void marker_execute_task_concurrent(GC* gc, TaskType task_func, Space* space, unsigned int num_markers); +void marker_execute_task_concurrent(GC* gc, TaskType task_func, Space* space); + +void marker_notify_mark_root_done(Marker* marker); +void wait_mark_finish(GC* gc); +Boolean is_mark_finished(GC* gc); + + + +#endif //_MARKER_H_ + Index: src/thread/mutator.cpp =================================================================== --- src/thread/mutator.cpp (revision 569956) +++ src/thread/mutator.cpp (working copy) @@ -38,6 +38,7 @@ mutator->rem_set = free_set_pool_get_entry(gc->metadata); assert(vector_block_is_empty(mutator->rem_set)); } + mutator->dirty_obj_snapshot = free_set_pool_get_entry(gc->metadata); if(!IGNORE_FINREF ) mutator->obj_with_fin = finref_get_free_block(gc); @@ -97,6 +98,13 @@ pool_put_entry(gc->finref_metadata->obj_with_fin_pool, mutator->obj_with_fin); mutator->obj_with_fin = NULL; } + + if( mutator->dirty_obj_snapshot != NULL){ + if(vector_block_is_empty(mutator->dirty_obj_snapshot)) + pool_put_entry(gc->metadata->free_set_pool, mutator->dirty_obj_snapshot); + else /* FIXME:: this condition may be released. */ + pool_put_entry(gc->metadata->dirty_obj_snaptshot_pool, mutator->dirty_obj_snapshot); + } //gc_set_tls(NULL); @@ -124,4 +132,57 @@ return; } +/* +Boolean gc_local_snapshot_is_empty(GC* gc) +{ + lock(gc->mutator_list_lock); + Mutator *mutator = gc->mutator_list; + while (mutator) { + if(mutator->concurrent_mark_handshake_status != LOCAL_SNAPSHOT_CONTAINER_IS_EMPTY){ + unlock(gc->mutator_list_lock); + return FALSE; + } + mutator = mutator->next; + } + + unlock(gc->mutator_list_lock); + return TRUE; +}*/ + +Boolean gc_local_snapshot_is_empty(GC* gc) +{ + lock(gc->mutator_list_lock); + + Mutator *mutator = gc->mutator_list; + while (mutator) { + Vector_Block* local_snapshot_set = mutator->dirty_obj_snapshot; + if(!vector_block_is_empty(local_snapshot_set)){ + unlock(gc->mutator_list_lock); + return FALSE; + } + mutator = mutator->next; + } + + unlock(gc->mutator_list_lock); + return TRUE; +} + +Vector_Block* gc_get_local_snapshot(GC* gc, unsigned int shared_id) +{ + lock(gc->mutator_list_lock); + + Mutator *mutator = gc->mutator_list; + while (mutator) { + Vector_Block* local_snapshot = mutator->dirty_obj_snapshot; + if(!vector_block_is_empty(local_snapshot) && vector_block_set_shared(local_snapshot,shared_id)){ + unlock(gc->mutator_list_lock); + return local_snapshot; + } + mutator = mutator->next; + } + + unlock(gc->mutator_list_lock); + return NULL; +} + Index: src/thread/mutator.h =================================================================== --- src/thread/mutator.h (revision 569956) +++ src/thread/mutator.h (working copy) @@ -42,6 +42,7 @@ Vector_Block* rem_set; Vector_Block* obj_with_fin; Mutator* next; /* The gc info area associated with the next active thread. */ + Vector_Block* dirty_obj_snapshot; } Mutator; void mutator_initialize(GC* gc, void* tls_gc_info); @@ -51,4 +52,6 @@ void gc_reset_mutator_context(GC* gc); void gc_prepare_mutator_remset(GC* gc); +Boolean gc_local_snapshot_is_empty(GC* gc); +Vector_Block* gc_get_local_snapshot(GC* gc, unsigned int shared_id); #endif /*ifndef _MUTATOR_H_ */ Index: src/trace_forward/fspace.cpp =================================================================== --- src/trace_forward/fspace.cpp (revision 569956) +++ src/trace_forward/fspace.cpp (working copy) @@ -38,7 +38,7 @@ struct GC_Gen; void gc_set_nos(GC_Gen* gc, Space* space); -void fspace_initialize(GC* gc, void* start, POINTER_SIZE_INT fspace_size, POINTER_SIZE_INT commit_size) +Fspace *fspace_initialize(GC* gc, void* start, POINTER_SIZE_INT fspace_size, POINTER_SIZE_INT commit_size) { assert( (fspace_size%GC_BLOCK_SIZE_BYTES) == 0 ); Fspace* fspace = (Fspace *)STD_MALLOC(sizeof(Fspace)); @@ -84,7 +84,6 @@ fspace->period_surviving_size = 0; fspace->gc = gc; - gc_set_nos((GC_Gen*)gc, (Space*)fspace); /* above is same as Mspace init --> */ forward_first_half = TRUE; @@ -97,7 +96,7 @@ else object_forwarding_boundary = (void*)&fspace->blocks[fspace->num_managed_blocks]; - return; + return fspace; } void fspace_destruct(Fspace *fspace) @@ -106,12 +105,12 @@ STD_FREE(fspace); } -void fspace_reset_for_allocation(Fspace* fspace) +void fspace_reset_after_collection(Fspace* fspace) { unsigned int first_idx = fspace->first_block_idx; unsigned int marked_start_idx = 0; //was for oi markbit reset, now useless unsigned int marked_last_idx = 0; - Boolean is_major_collection = !gc_match_kind(fspace->gc, MINOR_COLLECTION); + Boolean is_major_collection = gc_match_kind(fspace->gc, MAJOR_COLLECTION); Boolean gen_mode = gc_is_gen_mode(); if( is_major_collection || @@ -152,7 +151,16 @@ block->free = block->base; } - + + /* For los extension + * num_managed_blocks of fspace might be 0. + * In this case, the last block we found is mos' last block. + * And this implementation depends on the fact that mos and nos are continuous. + */ + int last_block_index = fspace->num_managed_blocks - 1; + Block_Header *fspace_last_block = (Block_Header*)&fspace->blocks[last_block_index]; + fspace_last_block->next = NULL; + return; } @@ -196,7 +204,7 @@ fspace->num_collections++; GC* gc = fspace->gc; - mspace_free_block_idx = ((GC_Gen*)gc)->mos->free_block_idx; + mspace_free_block_idx = ((Blocked_Space*)((GC_Gen*)gc)->mos)->free_block_idx; if(gc_is_gen_mode()){ fspace->collect_algorithm = MINOR_GEN_FORWARD_POOL; @@ -210,32 +218,32 @@ switch(fspace->collect_algorithm){ #ifdef MARK_BIT_FLIPPING - -case MINOR_NONGEN_FORWARD_POOL: - TRACE2("gc.process", "GC: nongen_forward_pool algo start ... \n"); - collector_execute_task(gc, (TaskType)nongen_forward_pool, (Space*)fspace); - TRACE2("gc.process", "\nGC: end of nongen forward algo ... \n"); + + case MINOR_NONGEN_FORWARD_POOL: + TRACE2("gc.process", "GC: nongen_forward_pool algo start ... \n"); + collector_execute_task(gc, (TaskType)nongen_forward_pool, (Space*)fspace); + TRACE2("gc.process", "\nGC: end of nongen forward algo ... \n"); #ifdef GC_GEN_STATS - gc_gen_stats_set_nos_algo((GC_Gen*)gc, MINOR_NONGEN_FORWARD_POOL); + gc_gen_stats_set_nos_algo((GC_Gen*)gc, MINOR_NONGEN_FORWARD_POOL); #endif - break; - + break; + #endif /*#ifdef MARK_BIT_FLIPPING */ -case MINOR_GEN_FORWARD_POOL: - TRACE2("gc.process", "gen_forward_pool algo start ... \n"); - collector_execute_task(gc, (TaskType)gen_forward_pool, (Space*)fspace); - TRACE2("gc.process", "\nGC: end of gen forward algo ... \n"); + case MINOR_GEN_FORWARD_POOL: + TRACE2("gc.process", "gen_forward_pool algo start ... \n"); + collector_execute_task(gc, (TaskType)gen_forward_pool, (Space*)fspace); + TRACE2("gc.process", "\nGC: end of gen forward algo ... \n"); #ifdef GC_GEN_STATS - gc_gen_stats_set_nos_algo((GC_Gen*)gc, MINOR_NONGEN_FORWARD_POOL); + gc_gen_stats_set_nos_algo((GC_Gen*)gc, MINOR_NONGEN_FORWARD_POOL); #endif - break; - -default: - DIE2("gc.collection","Specified minor collection algorithm doesn't exist!"); - exit(0); - break; + break; + + default: + DIE2("gc.collection","Specified minor collection algorithm doesn't exist!"); + exit(0); + break; } - + return; } Index: src/trace_forward/fspace.h =================================================================== --- src/trace_forward/fspace.h (revision 569956) +++ src/trace_forward/fspace.h (working copy) @@ -36,12 +36,12 @@ typedef Blocked_Space Fspace; -void fspace_initialize(GC* gc, void* start, POINTER_SIZE_INT fspace_size, POINTER_SIZE_INT commit_size); +Fspace *fspace_initialize(GC* gc, void* start, POINTER_SIZE_INT fspace_size, POINTER_SIZE_INT commit_size); void fspace_destruct(Fspace *fspace); void* fspace_alloc(unsigned size, Allocator *allocator); -void fspace_reset_for_allocation(Fspace* fspace); +void fspace_reset_after_collection(Fspace* fspace); /* gen mode */ void gen_forward_pool(Collector* collector); Index: src/trace_forward/fspace_alloc.cpp =================================================================== --- src/trace_forward/fspace_alloc.cpp (revision 569956) +++ src/trace_forward/fspace_alloc.cpp (working copy) @@ -20,6 +20,8 @@ */ #include "fspace.h" +#include "../common/gc_concurrent.h" +#include "../common/collection_scheduler.h" static Boolean fspace_alloc_block(Fspace* fspace, Allocator* allocator) { @@ -59,13 +61,15 @@ p_return = thread_local_alloc(size, allocator); if (p_return) return p_return; + if(gc_need_start_concurrent_mark(allocator->gc)) + gc_start_concurrent_mark(allocator->gc); /* ran out local block, grab a new one*/ Fspace* fspace = (Fspace*)allocator->alloc_space; int attempts = 0; while( !fspace_alloc_block(fspace, allocator)){ vm_gc_lock_enum(); /* after holding lock, try if other thread collected already */ - if ( !space_has_free_block((Blocked_Space*)fspace) ) { + if ( !blocked_space_has_free_block((Blocked_Space*)fspace) ) { if(attempts < 2) { #ifdef GC_GEN_STATS GC_Gen* gc = (GC_Gen*)allocator->gc; @@ -93,4 +97,3 @@ return p_return; } - Index: src/trace_forward/fspace_gen_forward_pool.cpp =================================================================== --- src/trace_forward/fspace_gen_forward_pool.cpp (revision 569956) +++ src/trace_forward/fspace_gen_forward_pool.cpp (working copy) @@ -354,8 +354,6 @@ gc_fix_rootset(collector); - fspace_reset_for_allocation(space); - TRACE2("gc.process", "GC: collector[0] finished"); return; Index: src/trace_forward/fspace_nongen_forward_pool.cpp =================================================================== --- src/trace_forward/fspace_nongen_forward_pool.cpp (revision 569956) +++ src/trace_forward/fspace_nongen_forward_pool.cpp (working copy) @@ -308,8 +308,6 @@ gc_fix_rootset(collector); - fspace_reset_for_allocation(space); - TRACE2("gc.process", "GC: collector[0] finished"); return; Index: src/utils/vector_block.h =================================================================== --- src/utils/vector_block.h (revision 569956) +++ src/utils/vector_block.h (working copy) @@ -64,7 +64,7 @@ */ inline Boolean vector_block_is_empty(Vector_Block* block) -{ return block->tail == block->entries; } +{ return block->tail == block->head; } inline void vector_block_add_entry(Vector_Block* block, POINTER_SIZE_INT value) { @@ -75,6 +75,95 @@ *(block->tail++) = value; } +inline POINTER_SIZE_INT vector_block_get_entry(Vector_Block* block) +{ +#ifdef _DEBUG + assert(!vector_block_is_empty(block)); +#endif + + POINTER_SIZE_INT value = *(block->head++); + +#ifdef _DEBUG + assert(value); +#endif + return value; +} + +#define VECTOR_BLOCK_SHARE_BIT 0x01 +#define VECTOR_BLOCK_FULL_BIT 0x02 +#define VECTOR_BLOCK_SHARE_ID_SHIFT 0x05 +#define VECTOR_BLOCK_EXCLUSIVE_ID 0xFFff +inline Boolean vector_block_is_shared(Vector_Block* block) +{ + return (Boolean)((POINTER_SIZE_INT)block->next & VECTOR_BLOCK_SHARE_BIT); +} + +inline Boolean vector_block_is_set_full(Vector_Block* block) +{ + return (Boolean)((POINTER_SIZE_INT)block->next & VECTOR_BLOCK_FULL_BIT); +} + +inline Boolean vector_block_set_full(Vector_Block* block) +{ + POINTER_SIZE_INT old_shared_var = (POINTER_SIZE_INT) block->next; + POINTER_SIZE_INT new_shared_var = old_shared_var | VECTOR_BLOCK_FULL_BIT; + + while(TRUE){ + POINTER_SIZE_INT old_var = (POINTER_SIZE_INT)atomic_casptr((volatile void **)& block->next ,(void*) new_shared_var,(void*) old_shared_var); + if(old_var == old_shared_var) return TRUE; + old_shared_var = (POINTER_SIZE_INT) block->next; + new_shared_var = old_shared_var | VECTOR_BLOCK_FULL_BIT; + } + assert(0); + return FALSE; + +} + +inline Boolean vector_block_set_shared(Vector_Block* block, unsigned int share_id) +{ + POINTER_SIZE_INT new_shared_var = (POINTER_SIZE_INT)((share_id << VECTOR_BLOCK_SHARE_ID_SHIFT) | VECTOR_BLOCK_SHARE_BIT); + POINTER_SIZE_INT old_shared_var = (POINTER_SIZE_INT) block->next; + if(old_shared_var != 0) return FALSE; + + while(TRUE){ + POINTER_SIZE_INT old_var = (POINTER_SIZE_INT)atomic_casptr((volatile void **)& block->next ,(void*) new_shared_var,(void*) old_shared_var); + if(old_var == old_shared_var) return TRUE; + old_shared_var = (POINTER_SIZE_INT) block->next; + if(old_shared_var != 0) return FALSE; + } +} + +inline Boolean vector_block_not_full_set_unshared(Vector_Block* block) +{ + POINTER_SIZE_INT new_shared_var = (POINTER_SIZE_INT) 0; + POINTER_SIZE_INT old_shared_var = (POINTER_SIZE_INT) block->next; + + if(old_shared_var & VECTOR_BLOCK_FULL_BIT) return FALSE; + + while(TRUE){ + POINTER_SIZE_INT old_var = (POINTER_SIZE_INT)atomic_casptr((volatile void **)& block->next ,(void*) new_shared_var,(void*) old_shared_var); + if(old_var == old_shared_var) return TRUE; + old_shared_var = (POINTER_SIZE_INT) block->next; + if(old_shared_var & VECTOR_BLOCK_FULL_BIT) return FALSE; + } + assert(0); + return FALSE; +} + +inline Boolean vector_block_set_exclusive(Vector_Block* block) +{ + POINTER_SIZE_INT new_shared_var = (POINTER_SIZE_INT)((VECTOR_BLOCK_EXCLUSIVE_ID << VECTOR_BLOCK_SHARE_ID_SHIFT) | VECTOR_BLOCK_SHARE_BIT); + POINTER_SIZE_INT old_shared_var = (POINTER_SIZE_INT) block->next; + if(old_shared_var & VECTOR_BLOCK_SHARE_BIT) return FALSE; + + while(TRUE){ + POINTER_SIZE_INT old_var = (POINTER_SIZE_INT)atomic_casptr((volatile void **)& block->next ,(void*) new_shared_var,(void*) old_shared_var); + if(old_var == old_shared_var) return TRUE; + old_shared_var = (POINTER_SIZE_INT) block->next; + if(old_shared_var & VECTOR_BLOCK_SHARE_BIT) return FALSE; + } +} + inline void vector_block_clear(Vector_Block* block) { block->head = (POINTER_SIZE_INT*)block->entries; Index: src/verify/verifier_common.cpp =================================================================== --- src/verify/verifier_common.cpp (revision 569956) +++ src/verify/verifier_common.cpp (working copy) @@ -83,13 +83,15 @@ Boolean verify_rootset_slot(REF* p_ref, Heap_Verifier* heap_verifier) { + Partial_Reveal_Object* p_obj = read_slot(p_ref); + assert(address_belongs_to_gc_heap(p_obj,heap_verifier->gc)); +#ifndef USE_MARK_SWEEP_GC GC_Gen* gc = (GC_Gen*)heap_verifier->gc; Space* mspace = gc_get_mos(gc); Space* lspace = gc_get_los(gc); - Partial_Reveal_Object* p_obj = read_slot(p_ref); if(p_obj == NULL){ - if(gc->collect_kind !=MINOR_COLLECTION ||(!heap_verifier->gc_is_gen_mode && !NOS_PARTIAL_FORWARD)){ + if(gc_match_kind((GC*)gc, MAJOR_COLLECTION) ||(!heap_verifier->gc_is_gen_mode && !NOS_PARTIAL_FORWARD)){ assert(0); return FALSE; }else{ @@ -106,12 +108,14 @@ } assert(address_belongs_to_gc_heap(p_obj,heap_verifier->gc)); if(heap_verifier->is_before_gc){ +#endif //if(!address_belongs_to_gc_heap(p_ref) && address_belongs_to_gc_heap(p_obj)){ if(!address_belongs_to_gc_heap(p_obj, heap_verifier->gc)){ printf("\nERROR: obj referenced by rootset is outside the heap error!\n"); assert(0); return FALSE; } +#ifndef USE_MARK_SWEEP_GC }else{ if(heap_verifier->gc_verifier->is_before_fallback_collection){ if(!address_belongs_to_gc_heap(p_obj, heap_verifier->gc)){ @@ -128,6 +132,7 @@ return FALSE; } } +#endif return TRUE; } @@ -211,10 +216,12 @@ gc_kind = " fallback collection."; }else if(gc_match_kind(gc, EXTEND_COLLECTION)){ gc_kind = " extend collection."; - }else if(gc_match_kind(gc, MAJOR_COLLECTION)){ + }else if(gc_match_kind(gc, NORMAL_MAJOR_COLLECTION)){ if(gc->tuner->kind == TRANS_NOTHING) gc_kind = "major collection (normal)"; else if(gc->tuner->kind == TRANS_FROM_LOS_TO_MOS) gc_kind = "major collection (LOS shrink)"; else if(gc->tuner->kind == TRANS_FROM_MOS_TO_LOS) gc_kind = "major collection (LOS extend)"; + }else if(gc_match_kind(gc, MARK_SWEEP_GC)){ + gc_kind = " mark sweep collection."; } printf(" GC_kind: %s\n", gc_kind); } Index: src/verify/verifier_common.h =================================================================== --- src/verify/verifier_common.h (revision 569956) +++ src/verify/verifier_common.h (working copy) @@ -22,10 +22,15 @@ #include "../common/gc_common.h" #include "../common/gc_space.h" #include "../gen/gen.h" +#include "../mark_sweep/gc_ms.h" #include "../common/space_tuner.h" #ifdef USE_32BITS_HASHCODE #include "../common/hashcode.h" #endif +#ifdef USE_MARK_SWEEP_GC +#include "../mark_sweep/sspace_mark_sweep.h" +#endif +#include "../common/gc_concurrent.h" struct Heap_Verifier; struct Allocation_Verifier; @@ -113,6 +118,22 @@ assert(p_obj); assert(uncompress_vt(obj_get_vt(p_obj))); assert(!address_belongs_to_gc_heap(uncompress_vt(obj_get_vt(p_obj)), (GC*)heap_verifier->gc)); + +//ynhe + +#ifdef USE_MARK_SWEEP_GC + GC_MS* gc = (GC_MS*)heap_verifier->gc; + if(!heap_verifier->is_before_gc){ + /*in GC_MS mark sweep algorithm, all live objects should be set their mark bit*/ + assert(obj_is_alloc_color_in_table(p_obj)); + if(!obj_is_alloc_color_in_table(p_obj)) + printf("\nERROR: obj after GC should be set its alloc color!\n"); + }else{ + //ynhe + if(gc_mark_is_concurrent()) + assert(obj_is_mark_black_in_table(p_obj)); + } +#endif } inline void verify_all_object_slot(REF* p_ref, Heap_Verifier* heap_verifier) Index: src/verify/verifier_scanner.cpp =================================================================== --- src/verify/verifier_scanner.cpp (revision 569956) +++ src/verify/verifier_scanner.cpp (working copy) @@ -36,6 +36,7 @@ static FORCE_INLINE void scan_object(Heap_Verifier* heap_verifier, Partial_Reveal_Object *p_obj) { GC_Verifier* gc_verifier = heap_verifier->gc_verifier; +#ifndef USE_MARK_SWEEP_GC if(gc_verifier->is_before_fallback_collection) { if(obj_belongs_to_nos(p_obj) && obj_is_fw_in_oi(p_obj)){ assert(obj_get_vt(p_obj) == obj_get_vt(obj_get_fw_in_oi(p_obj))); @@ -43,6 +44,7 @@ assert(p_obj); } } +#endif if(!obj_mark_in_vt(p_obj)) return; verify_object_header(p_obj, heap_verifier); @@ -204,9 +206,10 @@ void verifier_scan_resurrect_objects(Heap_Verifier* heap_verifier) { - GC_Gen* gc = (GC_Gen*)heap_verifier->gc; + GC* gc = heap_verifier->gc; Heap_Verifier_Metadata* verifier_metadata = heap_verifier->heap_verifier_metadata; verifier_update_info_before_resurrect(heap_verifier); + return; #ifndef BUILD_IN_REFERENT heap_verifier->gc_verifier->is_tracing_resurrect_obj = TRUE; if(heap_verifier->is_before_gc){ @@ -361,6 +364,7 @@ void verifier_scan_all_objects(Heap_Verifier* heap_verifier) { +#ifndef USE_MARK_SWEEP_GC GC_Gen* gc = (GC_Gen*)heap_verifier->gc; Space* fspace = gc_get_nos(gc); Space* mspace = gc_get_mos(gc); @@ -369,6 +373,9 @@ verifier_scan_nos_mos_objects(fspace, heap_verifier); verifier_scan_nos_mos_objects(mspace, heap_verifier); verifier_scan_los_objects(lspace, heap_verifier); +#else + assert(0); +#endif } /*<--------all objects scanner end--------->*/ @@ -405,6 +412,7 @@ void verifier_scan_unreachable_objects(Heap_Verifier* heap_verifier) { +#ifndef USE_MARK_SWEEP_GC if(heap_verifier->is_before_gc) return; GC_Gen* gc = (GC_Gen*)heap_verifier->gc; Space* mspace = gc_get_mos(gc); @@ -412,7 +420,9 @@ verifier_scan_mos_unreachable_objects(mspace, heap_verifier); verifier_scan_los_unreachable_objects(lspace, heap_verifier); - +#else + return; +#endif } /*<--------unreachable objects scanner end------>*/ Index: src/verify/verify_gc_effect.cpp =================================================================== --- src/verify/verify_gc_effect.cpp (revision 569956) +++ src/verify/verify_gc_effect.cpp (working copy) @@ -17,6 +17,9 @@ #include "verifier_common.h" #include "verify_gc_effect.h" +#ifdef USE_MARK_SWEEP_GC +#include "../mark_sweep/sspace_mark_sweep.h" +#endif static POINTER_SIZE_INT hash_obj_distance = 0; @@ -144,7 +147,7 @@ GC_Verifier* gc_verifier = heap_verifier->gc_verifier; assert(obj_belongs_to_space(p_obj, (Space*)fspace)); unsigned int forwarded_first_part; - if(!gc_verifier->gc_collect_kind == MINOR_COLLECTION || !NOS_PARTIAL_FORWARD || heap_verifier->gc_is_gen_mode) + if(!(gc_verifier->gc_collect_kind == MINOR_COLLECTION) || !NOS_PARTIAL_FORWARD || heap_verifier->gc_is_gen_mode) forwarded_first_part = true; else forwarded_first_part = forward_first_half^1; @@ -293,6 +296,7 @@ Heap_Verifier_Metadata* verifier_metadata = heap_verifier->heap_verifier_metadata; GC_Verifier* gc_verifier = heap_verifier->gc_verifier; +#ifndef USE_MARK_SWEEP_GC GC_Gen* gc = (GC_Gen*)heap_verifier->gc; Space* mspace = gc_get_mos(gc); Space* nspace = gc_get_nos(gc); @@ -313,6 +317,20 @@ } } } +#else + GC_MS* gc = (GC_MS*)heap_verifier->gc; + if(!heap_verifier->is_before_gc){ + /*in GC_MS mark sweep algorithm, all live objects should be set their mark bit*/ + assert(obj_is_alloc_color_in_table(p_obj)); + if(!obj_is_alloc_color_in_table(p_obj)) + printf("\nERROR: obj after GC should be set its alloc color!\n"); + }else{ + //ynhe + if(gc_mark_is_concurrent()) + assert(obj_is_mark_black_in_table(p_obj)); + } +#endif + /*store the object information*/ void* p_obj_information = verifier_copy_obj_information(p_obj); void* obj_hash_info = verifier_copy_hashcode(p_obj, heap_verifier, heap_verifier->is_before_gc); Index: src/verify/verify_live_heap.cpp =================================================================== --- src/verify/verify_live_heap.cpp (revision 569956) +++ src/verify/verify_live_heap.cpp (working copy) @@ -115,7 +115,7 @@ void event_gc_collect_kind_changed(GC* gc) { /*GC collection kind were changed from normal MINOR or MAJOR to FALLBACK MAJOR*/ - assert(gc->collect_kind == FALLBACK_COLLECTION); + assert(gc_match_kind(gc, FALLBACK_COLLECTION)); if(!heap_verifier->need_verify_gc) return; /*finish the fallbacked gc verify*/ Index: src/verify/verify_mutator_effect.cpp =================================================================== --- src/verify/verify_mutator_effect.cpp (revision 569956) +++ src/verify/verify_mutator_effect.cpp (working copy) @@ -330,7 +330,7 @@ void verifier_mark_wb_slots(Heap_Verifier* heap_verifier) { GC_Gen* gc = (GC_Gen*)(heap_verifier->gc); - if(gc->collect_kind != MINOR_COLLECTION ||!gc_is_gen_mode()) return; + if(gc_match_kind((GC*)gc, MAJOR_COLLECTION) ||!gc_is_gen_mode()) return; GC_Metadata*gc_metadata = gc->metadata; Space* nspace = gc_get_nos(gc); @@ -363,7 +363,7 @@ void verify_write_barrier(REF* p_ref, Heap_Verifier* heap_verifier) { GC_Gen* gc = (GC_Gen*)heap_verifier->gc; - if(gc->collect_kind != MINOR_COLLECTION ||!gc_is_gen_mode()) return; + if(gc_match_kind((GC*)gc, MAJOR_COLLECTION) ||!gc_is_gen_mode()) return; Space* nspace = gc_get_nos(gc); assert(address_belongs_to_gc_heap((void*)p_ref, (GC *) gc));