Index: src/common/collection_scheduler.cpp =================================================================== --- src/common/collection_scheduler.cpp (revision 606795) +++ src/common/collection_scheduler.cpp (working copy) @@ -18,9 +18,11 @@ #include "gc_common.h" #include "../gen/gen.h" #include "../mark_sweep/gc_ms.h" -#include "../mark_sweep/sspace.h" +#include "../mark_sweep/wspace.h" #include "collection_scheduler.h" #include "gc_concurrent.h" +#include "../verify/verify_live_heap.h" + static int64 time_delay_to_start_mark = 0; void collection_scheduler_initialize(GC* gc) @@ -43,10 +45,10 @@ Boolean gc_need_start_concurrent_mark(GC* gc) { - if(!USE_CONCURRENT_GC) return FALSE; + if(!USE_CONCURRENT_MARK) return FALSE; //FIXME: GEN mode also needs the support of starting mark after thread resume. #ifdef USE_MARK_SWEEP_GC - if(gc_is_concurrent_mark_phase() ) return FALSE; + if(gc_is_concurrent_mark_phase() || gc_mark_is_concurrent()) return FALSE; int64 time_current = time_now(); if( time_current - get_collection_end_time() > time_delay_to_start_mark) @@ -62,25 +64,50 @@ #endif } +Boolean gc_need_start_concurrent_sweep(GC* gc) +{ + if(!USE_CONCURRENT_SWEEP) return FALSE; + if(gc_sweep_is_concurrent()) return FALSE; + /*if mark is concurrent and STW GC has not started, we should start concurrent sweep*/ + if(gc_mark_is_concurrent() && !gc_is_concurrent_mark_phase(gc)) + return TRUE; + else + return FALSE; +} + +Boolean gc_need_reset_status(GC* gc) +{ + if(gc_sweep_is_concurrent() && !gc_is_concurrent_sweep_phase(gc)) + return TRUE; + else + return FALSE; +} + +Boolean gc_need_prepare_rootset(GC* gc) +{ + /*TODO: support on-the-fly root set enumeration.*/ + return FALSE; +} + void gc_update_collection_scheduler(GC* gc, int64 mutator_time, int64 mark_time) { - //FIXME: GEN GC should be supportted. + //FIXME: support GEN GC. #ifdef USE_MARK_SWEEP_GC Collection_Scheduler* collection_scheduler = gc->collection_scheduler; Space* space = NULL; - space = (Space*) gc_get_sspace(gc); + space = (Space*) gc_get_wspace(gc); - Space_Statistics* sspace_stat = space->space_statistic; + Space_Statistics* space_stat = space->space_statistic; unsigned int slot_index = collection_scheduler->last_slot_index_in_window; unsigned int num_slot = collection_scheduler->num_slot_in_window; - collection_scheduler->num_obj_traced_window[slot_index] = sspace_stat->num_live_obj; - collection_scheduler->size_alloced_window[slot_index] = sspace_stat->last_size_free_space; + collection_scheduler->num_obj_traced_window[slot_index] = space_stat->num_live_obj; + collection_scheduler->size_alloced_window[slot_index] = space_stat->last_size_free_space; int64 time_mutator = mutator_time; int64 time_mark = mark_time; @@ -90,7 +117,7 @@ collection_scheduler->trace_rate_window[slot_index] = time_mark == 0 ? 0 : (float)collection_scheduler->num_obj_traced_window[slot_index] / time_mark; - + collection_scheduler->num_slot_in_window = num_slot >= STATISTICS_SAMPLING_WINDOW_SIZE ? num_slot : (++num_slot); collection_scheduler->last_slot_index_in_window = (++slot_index)% STATISTICS_SAMPLING_WINDOW_SIZE; @@ -110,19 +137,59 @@ if(average_alloc_rate == 0 || average_trace_rate == 0){ time_delay_to_start_mark = 0; }else{ - float expected_time_alloc = sspace_stat->size_free_space / average_alloc_rate; - float expected_time_trace = sspace_stat->num_live_obj / average_trace_rate; + float time_alloc_expected = space_stat->size_free_space / average_alloc_rate; + float time_trace_expected = space_stat->num_live_obj / average_trace_rate; - if(expected_time_alloc > expected_time_trace) - collection_scheduler->time_delay_to_start_mark = (int64)((expected_time_alloc - expected_time_trace)*0.7); - else - collection_scheduler->time_delay_to_start_mark = 0; + if(time_alloc_expected > time_trace_expected){ + if(gc_concurrent_match_algorithm(OTF_REM_OBJ_SNAPSHOT_ALGO)||gc_concurrent_match_algorithm(OTF_REM_NEW_TARGET_ALGO)){ + collection_scheduler->time_delay_to_start_mark = (int64)((time_alloc_expected - time_trace_expected)*0.65); + }else if(gc_concurrent_match_algorithm(MOSTLY_CONCURRENT_ALGO)){ + collection_scheduler->time_delay_to_start_mark = (int64)(mutator_time * 0.6); + } + + }else{ + collection_scheduler->time_delay_to_start_mark = 0; + } time_delay_to_start_mark = collection_scheduler->time_delay_to_start_mark; - } + } + //[DEBUG] set to 0 for debugging. + //time_delay_to_start_mark = 0; #endif return; } +Boolean gc_try_schedule_collection(GC* gc, unsigned int gc_cause) +{ + gc_check_concurrent_phase(gc); + + if(gc_need_prepare_rootset(gc)){ + /*TODO:Enable concurrent rootset enumeration.*/ + assert(0); + } + + if(gc_need_start_concurrent_mark(gc)){ + gc_start_concurrent_mark(gc); + return TRUE; + } + + if(gc_need_start_concurrent_sweep(gc)){ + gc->num_collections++; + gc_start_concurrent_sweep(gc); + return TRUE; + } + + if(gc_need_reset_status(gc)){ + int disable_count = hythread_reset_suspend_disable(); + gc_reset_after_concurrent_collection(gc); + vm_resume_threads_after(); + hythread_set_suspend_disable(disable_count); + } + + return FALSE; + +} + + Index: src/common/collection_scheduler.h =================================================================== --- src/common/collection_scheduler.h (revision 606795) +++ src/common/collection_scheduler.h (working copy) @@ -41,8 +41,10 @@ void collection_scheduler_destruct(GC* gc); void gc_update_collection_scheduler(GC* gc, int64 mutator_time, int64 mark_time); +Boolean gc_try_schedule_collection(GC* gc, unsigned int gc_cause); Boolean gc_need_start_concurrent_mark(GC* gc); #endif + Index: src/common/fix_repointed_refs.h =================================================================== --- src/common/fix_repointed_refs.h (revision 606795) +++ src/common/fix_repointed_refs.h (working copy) @@ -32,24 +32,36 @@ Partial_Reveal_Object* p_obj = read_slot(p_ref); if(!p_obj) return; +#ifdef USE_UNIQUE_MOVE_COMPACT_GC + p_obj = obj_get_fw_in_table(p_obj); + assert(obj_belongs_to_gc_heap(p_obj)); + write_slot(p_ref, p_obj); + return; + +#endif + if(IS_MOVE_COMPACT){ /* This condition is removed because we do los sliding compaction at every major compaction after add los minor sweep. */ //if(obj_is_moved(p_obj)) /*Fixme: los_boundery ruined the modularity of gc_common.h*/ if(p_obj < los_boundary){ - write_slot(p_ref, obj_get_fw_in_oi(p_obj)); + p_obj = obj_get_fw_in_oi(p_obj); }else{ - *p_ref = obj_get_fw_in_table(p_obj); + p_obj = obj_get_fw_in_table(p_obj); } - }else{ + + write_slot(p_ref, p_obj); + + }else{ /* slide compact */ if(obj_is_fw_in_oi(p_obj)){ /* Condition obj_is_moved(p_obj) is for preventing mistaking previous mark bit of large obj as fw bit when fallback happens. * Because until fallback happens, perhaps the large obj hasn't been marked. So its mark bit remains as the last time. * This condition is removed because we do los sliding compaction at every major compaction after add los minor sweep. * In major collection condition obj_is_fw_in_oi(p_obj) can be omitted, * since those which can be scanned in MOS & NOS must have been set fw bit in oi. */ - assert((POINTER_SIZE_INT)obj_get_fw_in_oi(p_obj) > DUAL_MARKBITS); - write_slot(p_ref, obj_get_fw_in_oi(p_obj)); + p_obj = obj_get_fw_in_oi(p_obj); + assert(obj_belongs_to_gc_heap(p_obj)); + write_slot(p_ref, p_obj); } } Index: src/common/gc_block.cpp =================================================================== --- src/common/gc_block.cpp (revision 0) +++ src/common/gc_block.cpp (revision 0) @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gc_space.h" + +void space_init_blocks(Blocked_Space* space) +{ + Block* blocks = (Block*)space->heap_start; + Block_Header* last_block = (Block_Header*)blocks; + unsigned int start_idx = space->first_block_idx; + for(unsigned int i=0; i < space->num_managed_blocks; i++){ + Block_Header* block = (Block_Header*)&(blocks[i]); + block_init(block); + block->block_idx = i + start_idx; + last_block->next = block; + last_block = block; + } + last_block->next = NULL; + space->blocks = blocks; + + return; +} + +void space_desturct_blocks(Blocked_Space* space) +{ + Block* blocks = (Block*)space->heap_start; + unsigned int i=0; + for(; i < space->num_managed_blocks; i++){ + Block_Header* block = (Block_Header*)&(blocks[i]); + block_destruct(block); + } +} + +void blocked_space_shrink(Blocked_Space* space, unsigned int changed_size) +{ + unsigned int block_dec_count = changed_size >> GC_BLOCK_SHIFT_COUNT; + void* new_base = (void*)&(space->blocks[space->num_managed_blocks - block_dec_count]); + + void* decommit_base = (void*)round_down_to_size((POINTER_SIZE_INT)new_base, SPACE_ALLOC_UNIT); + + assert( ((Block_Header*)decommit_base)->block_idx >= space->free_block_idx); + + void* old_end = (void*)&space->blocks[space->num_managed_blocks]; + POINTER_SIZE_INT decommit_size = (POINTER_SIZE_INT)old_end - (POINTER_SIZE_INT)decommit_base; + assert(decommit_size && !(decommit_size%GC_BLOCK_SIZE_BYTES)); + + Boolean result = vm_decommit_mem(decommit_base, decommit_size); + assert(result == TRUE); + + space->committed_heap_size = (POINTER_SIZE_INT)decommit_base - (POINTER_SIZE_INT)space->heap_start; + space->num_managed_blocks = (unsigned int)(space->committed_heap_size >> GC_BLOCK_SHIFT_COUNT); + + Block_Header* new_last_block = (Block_Header*)&space->blocks[space->num_managed_blocks - 1]; + space->ceiling_block_idx = new_last_block->block_idx; + new_last_block->next = NULL; +} + +void blocked_space_extend(Blocked_Space* space, unsigned int changed_size) +{ + unsigned int block_inc_count = changed_size >> GC_BLOCK_SHIFT_COUNT; + + void* old_base = (void*)&space->blocks[space->num_managed_blocks]; + void* commit_base = (void*)round_down_to_size((POINTER_SIZE_INT)old_base, SPACE_ALLOC_UNIT); + unsigned int block_diff_count = (unsigned int)(((POINTER_SIZE_INT)old_base - (POINTER_SIZE_INT)commit_base) >> GC_BLOCK_SHIFT_COUNT); + block_inc_count += block_diff_count; + + POINTER_SIZE_INT commit_size = block_inc_count << GC_BLOCK_SHIFT_COUNT; + void* result = vm_commit_mem(commit_base, commit_size); + assert(result == commit_base); + + void* new_end = (void*)((POINTER_SIZE_INT)commit_base + commit_size); + space->committed_heap_size = (POINTER_SIZE_INT)new_end - (POINTER_SIZE_INT)space->heap_start; + /*Fixme: For_Heap_Adjust, but need fix if static mapping.*/ + space->heap_end = new_end; + /* init the grown blocks */ + Block_Header* block = (Block_Header*)commit_base; + Block_Header* last_block = (Block_Header*)((Block*)block -1); + unsigned int start_idx = last_block->block_idx + 1; + unsigned int i; + for(i=0; block < new_end; i++){ + block_init(block); + block->block_idx = start_idx + i; + last_block->next = block; + last_block = block; + block = (Block_Header*)((Block*)block + 1); + } + last_block->next = NULL; + space->ceiling_block_idx = last_block->block_idx; + space->num_managed_blocks = (unsigned int)(space->committed_heap_size >> GC_BLOCK_SHIFT_COUNT); +} + +void blocked_space_block_iterator_init(Blocked_Space *space) +{ space->block_iterator = (Block_Header*)space->blocks; } + +void blocked_space_block_iterator_init_free(Blocked_Space *space) +{ space->block_iterator = (Block_Header*)&space->blocks[space->free_block_idx - space->first_block_idx]; } + +Block_Header *blocked_space_block_iterator_get(Blocked_Space *space) +{ return (Block_Header*)space->block_iterator; } + +Block_Header *blocked_space_block_iterator_next(Blocked_Space *space) +{ + Block_Header *cur_block = (Block_Header*)space->block_iterator; + + while(cur_block != NULL){ + Block_Header *next_block = cur_block->next; + + Block_Header *temp = (Block_Header*)atomic_casptr((volatile void **)&space->block_iterator, next_block, cur_block); + if(temp != cur_block){ + cur_block = (Block_Header*)space->block_iterator; + continue; + } + return cur_block; + } + /* run out space blocks */ + return NULL; +} + +/* ================================================ */ + +#ifdef USE_32BITS_HASHCODE +Partial_Reveal_Object* block_get_first_marked_object_extend(Block_Header* block, void** start_pos) +{ + Partial_Reveal_Object* cur_obj = (Partial_Reveal_Object*)block->base; + Partial_Reveal_Object* block_end = (Partial_Reveal_Object*)block->free; + + Partial_Reveal_Object* first_marked_obj = next_marked_obj_in_block(cur_obj, block_end); + if(!first_marked_obj) + return NULL; + + *start_pos = obj_end_extend(first_marked_obj); + + return first_marked_obj; +} +#endif /* #ifdef USE_32BITS_HASHCODE */ + +Partial_Reveal_Object* block_get_first_marked_object(Block_Header* block, void** start_pos) +{ + Partial_Reveal_Object* cur_obj = (Partial_Reveal_Object*)block->base; + Partial_Reveal_Object* block_end = (Partial_Reveal_Object*)block->free; + + Partial_Reveal_Object* first_marked_obj = next_marked_obj_in_block(cur_obj, block_end); + if(!first_marked_obj) + return NULL; + + *start_pos = obj_end(first_marked_obj); + + return first_marked_obj; +} + +Partial_Reveal_Object* block_get_next_marked_object(Block_Header* block, void** start_pos) +{ + Partial_Reveal_Object* cur_obj = *(Partial_Reveal_Object**)start_pos; + Partial_Reveal_Object* block_end = (Partial_Reveal_Object*)block->free; + + Partial_Reveal_Object* next_marked_obj = next_marked_obj_in_block(cur_obj, block_end); + if(!next_marked_obj) + return NULL; + + *start_pos = obj_end(next_marked_obj); + + return next_marked_obj; +} + +Partial_Reveal_Object *block_get_first_marked_obj_prefetch_next(Block_Header *block, void **start_pos) +{ + Partial_Reveal_Object *cur_obj = (Partial_Reveal_Object *)block->base; + Partial_Reveal_Object *block_end = (Partial_Reveal_Object *)block->free; + + Partial_Reveal_Object *first_marked_obj = next_marked_obj_in_block(cur_obj, block_end); + if(!first_marked_obj) + return NULL; + + Partial_Reveal_Object *next_obj = obj_end(first_marked_obj); + *start_pos = next_obj; + + if(next_obj >= block_end) + return first_marked_obj; + + Partial_Reveal_Object *next_marked_obj = next_marked_obj_in_block(next_obj, block_end); + + if(next_marked_obj){ + if(next_marked_obj != next_obj) + obj_set_prefetched_next_pointer(next_obj, next_marked_obj); + } else { + obj_set_prefetched_next_pointer(next_obj, 0); + } + return first_marked_obj; +} + +Partial_Reveal_Object *block_get_first_marked_obj_after_prefetch(Block_Header *block, void **start_pos) +{ +#ifdef USE_32BITS_HASHCODE + return block_get_first_marked_object_extend(block, start_pos); +#else + return block_get_first_marked_object(block, start_pos); +#endif +} + +Partial_Reveal_Object *block_get_next_marked_obj_prefetch_next(Block_Header *block, void **start_pos) +{ + Partial_Reveal_Object *cur_obj = *(Partial_Reveal_Object **)start_pos; + Partial_Reveal_Object *block_end = (Partial_Reveal_Object *)block->free; + + if(cur_obj >= block_end) + return NULL; + + Partial_Reveal_Object *cur_marked_obj; + + if(obj_is_marked_in_vt(cur_obj)) + cur_marked_obj = cur_obj; + else + cur_marked_obj = (Partial_Reveal_Object *)obj_get_prefetched_next_pointer(cur_obj); + + if(!cur_marked_obj) + return NULL; + + Partial_Reveal_Object *next_obj = obj_end(cur_marked_obj); + *start_pos = next_obj; + + if(next_obj >= block_end) + return cur_marked_obj; + + Partial_Reveal_Object *next_marked_obj = next_marked_obj_in_block(next_obj, block_end); + + if(next_marked_obj){ + if(next_marked_obj != next_obj) + obj_set_prefetched_next_pointer(next_obj, next_marked_obj); + } else { + obj_set_prefetched_next_pointer(next_obj, 0); + } + + return cur_marked_obj; +} + +Partial_Reveal_Object *block_get_next_marked_obj_after_prefetch(Block_Header *block, void **start_pos) +{ + Partial_Reveal_Object *cur_obj = (Partial_Reveal_Object *)(*start_pos); + Partial_Reveal_Object *block_end = (Partial_Reveal_Object *)block->free; + + if(cur_obj >= block_end) + return NULL; + + Partial_Reveal_Object *cur_marked_obj; + + if(obj_is_marked_in_vt(cur_obj)) + cur_marked_obj = cur_obj; + else if (obj_is_fw_in_oi(cur_obj)) + /* why we need this obj_is_fw_in_oi(cur_obj) check. It's because of one source block with two dest blocks. + In that case, the second half of it might have been copied by another thread, so the live objects's + markbit is cleared. When the thread for the first half reaches the first object of the second half, + it finds there is no markbit in vt. But it still want to get the forward pointer of this object, + and figure out that, the target address (forward pointer) is in another dest block than current one; + so it knows it finishes its part of the source block, and will get next source source block. + Without this check, it will get prefetch pointer from oi, and finds the forward pointer (and with fwd_bit). + It's not the next live object, so it's wrong. + + Now I simply let it to return NULL when it finds ! obj_is_marked_in_vt(cur_obj) but obj_is_fw_in_oi(cur_obj). + This is the same in logic but clearer. + Change from original code: + if(obj_is_marked_in_vt(cur_obj) || obj_is_fw_in_oi(cur_obj) ) + cur_marked_obj = cur_obj; + else + cur_marked_obj = obj_get_prefetched_next_pointer(cur_obj); + + To current code: + if(obj_is_marked_in_vt(cur_obj) ) + cur_marked_obj = cur_obj; + else if (obj_is_fw_in_oi(cur_obj) ) + return NULL; + else + cur_marked_obj = obj_get_prefetched_next_pointer(cur_obj); + */ + return NULL; + else + cur_marked_obj = obj_get_prefetched_next_pointer(cur_obj); + + if(!cur_marked_obj) + return NULL; + +#ifdef USE_32BITS_HASHCODE + Partial_Reveal_Object *next_obj = obj_end_extend(cur_marked_obj); +#else + Partial_Reveal_Object *next_obj = obj_end(cur_marked_obj); +#endif + + *start_pos = next_obj; + + return cur_marked_obj; +} Index: src/common/gc_block.h =================================================================== --- src/common/gc_block.h (revision 606795) +++ src/common/gc_block.h (working copy) @@ -47,14 +47,23 @@ void* ceiling; void* new_free; /* used only during compaction */ unsigned int block_idx; +#ifdef USE_UNIQUE_MOVE_COMPACT_GC + unsigned int num_multi_block; /*number of block in large block*/ +#endif volatile unsigned int status; + + /* following three fields are used only in parallel sliding compaction */ volatile unsigned int dest_counter; + Partial_Reveal_Object* src; + Partial_Reveal_Object* next_src; + #ifdef USE_32BITS_HASHCODE Hashcode_Buf* hashcode_buf; /*hash code entry list*/ #endif - Partial_Reveal_Object* src; - Partial_Reveal_Object* next_src; Block_Header* next; +#ifdef USE_UNIQUE_MOVE_COMPACT_GC + Block_Header* next_large_block; /*used to link free super large block in gc_mc*/ +#endif POINTER_SIZE_INT table[1]; /* entry num == OFFSET_TABLE_SIZE_WORDS */ }Block_Header; @@ -95,12 +104,18 @@ #define ADDRESS_OFFSET_TO_BLOCK_HEADER(addr) ((unsigned int)((POINTER_SIZE_INT)addr&GC_BLOCK_LOW_MASK)) #define ADDRESS_OFFSET_IN_BLOCK_BODY(addr) ((unsigned int)(ADDRESS_OFFSET_TO_BLOCK_HEADER(addr)- GC_BLOCK_HEADER_SIZE_BYTES)) +#ifdef USE_UNIQUE_MOVE_COMPACT_GC +#define NUM_BLOCKS_PER_LARGE_OBJECT(size) (((size)+GC_BLOCK_HEADER_SIZE_BYTES+ GC_BLOCK_SIZE_BYTES-1)/GC_BLOCK_SIZE_BYTES) +#endif inline void block_init(Block_Header* block) { block->free = (void*)((POINTER_SIZE_INT)block + GC_BLOCK_HEADER_SIZE_BYTES); block->ceiling = (void*)((POINTER_SIZE_INT)block + GC_BLOCK_SIZE_BYTES); block->base = block->free; block->new_free = block->free; +#ifdef USE_UNIQUE_MOVE_COMPACT_GC + block->num_multi_block = 0; +#endif block->status = BLOCK_FREE; block->dest_counter = 0; block->src = NULL; @@ -139,185 +154,66 @@ } #endif -inline void obj_set_prefetched_next_pointer(Partial_Reveal_Object* obj, Partial_Reveal_Object* raw_prefetched_next){ - /*Fixme: em64t: This may be not necessary!*/ - if(raw_prefetched_next == 0){ - *((POINTER_SIZE_INT*)obj + 1) = 0; - return; - } - REF ref = obj_ptr_to_ref(raw_prefetched_next); - *( (REF*)((POINTER_SIZE_INT*)obj + 1) ) = ref; -} +/*FIXME: We use native pointer and put into oi, assuming VT and OI takes two POINTER_SIZE_INTs. + This is untrue if: + 1. compressed_ref && compressed_vt (vt and oi: 32bit. this can be true when heap<4G in 64bit machine) + Fortunately, current real design gives both vt (padded) and oi 64 bit even in this case. + + This is true in 64bit machine if either vt or oi is not compressed: + 2. compressed_ref && ! compressed_vt (vt: 64bit, oi: 64bit) + 3. !compressed_ref && ! compressed_vt (vt: 64bit, oi: 64bit) + 4. !compressed_ref && compressed_vt (with padded vt) (vt: 64bit, oi: 64bit) -inline Partial_Reveal_Object* obj_get_prefetched_next_pointer(Partial_Reveal_Object* obj){ - /*Fixme: em64t: This may be not necessary!*/ - assert(obj); - return read_slot( (REF*)((POINTER_SIZE_INT*)obj + 1) ); -} - -inline Partial_Reveal_Object *next_marked_obj_in_block(Partial_Reveal_Object *cur_obj, Partial_Reveal_Object *block_end) -{ - while(cur_obj < block_end){ - if( obj_is_marked_in_vt(cur_obj)) - return cur_obj; - cur_obj = obj_end(cur_obj); - } + When compressed_ref is true, REF is 32bit. It doesn't work in case of 2. + So we always use native pointer for both. + If case 1 design is changed in future to use 32bit for both, we need reconsider. - return NULL; -} - -#ifdef USE_32BITS_HASHCODE -inline Partial_Reveal_Object* block_get_first_marked_object_extend(Block_Header* block, void** start_pos) -{ - Partial_Reveal_Object* cur_obj = (Partial_Reveal_Object*)block->base; - Partial_Reveal_Object* block_end = (Partial_Reveal_Object*)block->free; - - Partial_Reveal_Object* first_marked_obj = next_marked_obj_in_block(cur_obj, block_end); - if(!first_marked_obj) - return NULL; - - *start_pos = obj_end_extend(first_marked_obj); + Use REF here has another implication. Since it's a random number, can have 1 in LSBs, which might be confused as FWD_BIT or MARK_BIT. + We need ensure that, if we use REF, we never check this prefetch pointer for FWD_BIT or MARK_BIT. - return first_marked_obj; -} -#endif +*/ -inline Partial_Reveal_Object* block_get_first_marked_object(Block_Header* block, void** start_pos) +inline void obj_set_prefetched_next_pointer(Partial_Reveal_Object* p_obj, Partial_Reveal_Object* raw_prefetched_next) { - Partial_Reveal_Object* cur_obj = (Partial_Reveal_Object*)block->base; - Partial_Reveal_Object* block_end = (Partial_Reveal_Object*)block->free; + Partial_Reveal_Object** p_ref = (Partial_Reveal_Object**)p_obj + 1; + *p_ref = raw_prefetched_next; - Partial_Reveal_Object* first_marked_obj = next_marked_obj_in_block(cur_obj, block_end); - if(!first_marked_obj) - return NULL; - - *start_pos = obj_end(first_marked_obj); - - return first_marked_obj; +/* see comments above. + REF* p_ref = (REF*)p_obj + 1; + write_slot(p_ref, raw_prefetched_next); +*/ } -inline Partial_Reveal_Object* block_get_next_marked_object(Block_Header* block, void** start_pos) +inline Partial_Reveal_Object* obj_get_prefetched_next_pointer(Partial_Reveal_Object* p_obj) { - Partial_Reveal_Object* cur_obj = *(Partial_Reveal_Object**)start_pos; - Partial_Reveal_Object* block_end = (Partial_Reveal_Object*)block->free; + assert(p_obj); + Partial_Reveal_Object** p_ref = (Partial_Reveal_Object**)p_obj + 1; + return *p_ref; - Partial_Reveal_Object* next_marked_obj = next_marked_obj_in_block(cur_obj, block_end); - if(!next_marked_obj) - return NULL; - - *start_pos = obj_end(next_marked_obj); - - return next_marked_obj; +/* see comments above. + REF* p_ref = (REF*)p_obj + 1; + return read_slot(p_ref); +*/ } -inline Partial_Reveal_Object *block_get_first_marked_obj_prefetch_next(Block_Header *block, void **start_pos) +inline Partial_Reveal_Object *next_marked_obj_in_block(Partial_Reveal_Object *cur_obj, Partial_Reveal_Object *block_end) { - Partial_Reveal_Object *cur_obj = (Partial_Reveal_Object *)block->base; - Partial_Reveal_Object *block_end = (Partial_Reveal_Object *)block->free; - - Partial_Reveal_Object *first_marked_obj = next_marked_obj_in_block(cur_obj, block_end); - if(!first_marked_obj) - return NULL; - - Partial_Reveal_Object *next_obj = obj_end(first_marked_obj); - *start_pos = next_obj; - - if(next_obj >= block_end) - return first_marked_obj; - - Partial_Reveal_Object *next_marked_obj = next_marked_obj_in_block(next_obj, block_end); - - if(next_marked_obj){ - if(next_marked_obj != next_obj) - obj_set_prefetched_next_pointer(next_obj, next_marked_obj); - } else { - obj_set_prefetched_next_pointer(next_obj, 0); + while(cur_obj < block_end){ + if( obj_is_marked_in_vt(cur_obj)) + return cur_obj; + cur_obj = obj_end(cur_obj); } - return first_marked_obj; -} - -inline Partial_Reveal_Object *block_get_first_marked_obj_after_prefetch(Block_Header *block, void **start_pos) -{ -#ifdef USE_32BITS_HASHCODE - return block_get_first_marked_object_extend(block, start_pos); -#else - return block_get_first_marked_object(block, start_pos); -#endif -} - -inline Partial_Reveal_Object *block_get_next_marked_obj_prefetch_next(Block_Header *block, void **start_pos) -{ - Partial_Reveal_Object *cur_obj = *(Partial_Reveal_Object **)start_pos; - Partial_Reveal_Object *block_end = (Partial_Reveal_Object *)block->free; - - if(cur_obj >= block_end) - return NULL; - Partial_Reveal_Object *cur_marked_obj; - - if(obj_is_marked_in_vt(cur_obj)) - cur_marked_obj = cur_obj; - else - cur_marked_obj = (Partial_Reveal_Object *)obj_get_prefetched_next_pointer(cur_obj); - - if(!cur_marked_obj) - return NULL; - - Partial_Reveal_Object *next_obj = obj_end(cur_marked_obj); - *start_pos = next_obj; - - if(next_obj >= block_end) - return cur_marked_obj; - - Partial_Reveal_Object *next_marked_obj = next_marked_obj_in_block(next_obj, block_end); - - if(next_marked_obj){ - if(next_marked_obj != next_obj) - obj_set_prefetched_next_pointer(next_obj, next_marked_obj); - } else { - obj_set_prefetched_next_pointer(next_obj, 0); - } - - return cur_marked_obj; + return NULL; } -inline Partial_Reveal_Object *block_get_next_marked_obj_after_prefetch(Block_Header *block, void **start_pos) +inline Partial_Reveal_Object* obj_get_fw_in_table(Partial_Reveal_Object *p_obj) { - Partial_Reveal_Object *cur_obj = (Partial_Reveal_Object *)(*start_pos); - Partial_Reveal_Object *block_end = (Partial_Reveal_Object *)block->free; - - if(cur_obj >= block_end) - return NULL; - - Partial_Reveal_Object *cur_marked_obj; - - if(obj_is_marked_in_vt(cur_obj) || obj_is_fw_in_oi(cur_obj)) - cur_marked_obj = cur_obj; - else - cur_marked_obj = obj_get_prefetched_next_pointer(cur_obj); - - if(!cur_marked_obj) - return NULL; - -#ifdef USE_32BITS_HASHCODE - Partial_Reveal_Object *next_obj = obj_end_extend(cur_marked_obj); -#else - Partial_Reveal_Object *next_obj = obj_end(cur_marked_obj); -#endif - - *start_pos = next_obj; - - return cur_marked_obj; -} - -inline REF obj_get_fw_in_table(Partial_Reveal_Object *p_obj) -{ /* only for inter-sector compaction */ unsigned int index = OBJECT_INDEX_TO_OFFSET_TABLE(p_obj); Block_Header *curr_block = GC_BLOCK_HEADER(p_obj); Partial_Reveal_Object* new_addr = (Partial_Reveal_Object *)(((POINTER_SIZE_INT)p_obj) - curr_block->table[index]); - REF new_ref = obj_ptr_to_ref(new_addr); - return new_ref; + return new_addr; } inline void block_clear_table(Block_Header* block) @@ -328,6 +224,19 @@ } #ifdef USE_32BITS_HASHCODE +Partial_Reveal_Object* block_get_first_marked_object_extend(Block_Header* block, void** start_pos); +#endif + +Partial_Reveal_Object* block_get_first_marked_object(Block_Header* block, void** start_pos); +Partial_Reveal_Object* block_get_next_marked_object(Block_Header* block, void** start_pos); +Partial_Reveal_Object *block_get_first_marked_obj_prefetch_next(Block_Header *block, void **start_pos); +Partial_Reveal_Object *block_get_first_marked_obj_after_prefetch(Block_Header *block, void **start_pos); +Partial_Reveal_Object *block_get_next_marked_obj_prefetch_next(Block_Header *block, void **start_pos); +Partial_Reveal_Object *block_get_next_marked_obj_after_prefetch(Block_Header *block, void **start_pos); + + +/* <-- blocked_space hashcode_buf ops */ +#ifdef USE_32BITS_HASHCODE inline Hashcode_Buf* block_set_hashcode_buf(Block_Header *block, Hashcode_Buf* new_hashcode_buf) { Hashcode_Buf* old_hashcode_buf = block->hashcode_buf; @@ -351,10 +260,13 @@ Hashcode_Buf* hashcode_buf = block_get_hashcode_buf(GC_BLOCK_HEADER(p_obj)); return hashcode_buf_lookup(p_obj,hashcode_buf); } -#endif +#endif //#ifdef USE_32BITS_HASHCODE +/* blocked_space hashcode_buf ops --> */ + #endif //#ifndef _BLOCK_H_ + Index: src/common/gc_common.cpp =================================================================== --- src/common/gc_common.cpp (revision 606795) +++ src/common/gc_common.cpp (working copy) @@ -43,8 +43,6 @@ extern POINTER_SIZE_INT INIT_LOS_SIZE; extern Boolean FORCE_FULL_COMPACT; -extern Boolean MINOR_ALGORITHM; -extern Boolean MAJOR_ALGORITHM; extern unsigned int NUM_MARKERS; extern unsigned int NUM_COLLECTORS; @@ -91,7 +89,7 @@ assert(property_name); char *value = get_property(property_name, VM_PROPERTIES); if (NULL == value){ - DIE2("gc.base","Warning: property value "<generate_barrier = TRUE; + } } + + if (is_property_set("gc.concurrent_enumeration", VM_PROPERTIES) == 1){ + USE_CONCURRENT_ENUMERATION= get_boolean_property("gc.concurrent_enumeration"); + if(USE_CONCURRENT_ENUMERATION){ + USE_CONCURRENT_GC = TRUE; + gc->generate_barrier = TRUE; + } + } + + if (is_property_set("gc.concurrent_mark", VM_PROPERTIES) == 1){ + USE_CONCURRENT_MARK= get_boolean_property("gc.concurrent_mark"); + if(USE_CONCURRENT_MARK){ + USE_CONCURRENT_GC = TRUE; + gc->generate_barrier = TRUE; + } + } + + if (is_property_set("gc.concurrent_sweep", VM_PROPERTIES) == 1){ + USE_CONCURRENT_SWEEP= get_boolean_property("gc.concurrent_sweep"); + if(USE_CONCURRENT_SWEEP){ + USE_CONCURRENT_GC = TRUE; + } + } + + char* concurrent_algo = NULL; + if (is_property_set("gc.concurrent_algorithm", VM_PROPERTIES) == 1) { + concurrent_algo = get_property("gc.concurrent_algorithm", VM_PROPERTIES); + } + + gc_decide_concurrent_algorithm(gc, concurrent_algo); + #if defined(ALLOC_ZEROING) && defined(ALLOC_PREFETCH) if(is_property_set("gc.prefetch",VM_PROPERTIES) ==1) { - PREFETCH_ENABLED=get_boolean_property("gc.prefetch"); + PREFETCH_ENABLED=get_boolean_property("gc.prefetch"); } if(is_property_set("gc.prefetch_distance",VM_PROPERTIES)==1) { PREFETCH_DISTANCE = get_size_property("gc.prefetch_distance"); if(!PREFETCH_ENABLED) { - WARN2("gc.prefetch_distance","Warning: Prefetch distance set with Prefetch disabled!"); - } + WARN2("gc.prefetch_distance","Warning: Prefetch distance set with Prefetch disabled!"); + } } if(is_property_set("gc.prefetch_stride",VM_PROPERTIES)==1) { @@ -319,11 +354,34 @@ void gc_assign_free_area_to_mutators(GC* gc) { -#ifndef USE_MARK_SWEEP_GC +#if !defined(USE_MARK_SWEEP_GC) && !defined(USE_UNIQUE_MOVE_COMPACT_GC) gc_gen_assign_free_area_to_mutators((GC_Gen*)gc); #endif } +void gc_init_collector_alloc(GC* gc, Collector* collector) +{ +#ifndef USE_MARK_SWEEP_GC + gc_gen_init_collector_alloc((GC_Gen*)gc, collector); +#else + gc_init_collector_free_chunk_list(collector); +#endif +} + +void gc_reset_collector_alloc(GC* gc, Collector* collector) +{ +#if !defined(USE_MARK_SWEEP_GC) && !defined(USE_UNIQUE_MOVE_COMPACT_GC) + gc_gen_reset_collector_alloc((GC_Gen*)gc, collector); +#endif +} + +void gc_destruct_collector_alloc(GC* gc, Collector* collector) +{ +#ifndef USE_MARK_SWEEP_GC + gc_gen_destruct_collector_alloc((GC_Gen*)gc, collector); +#endif +} + void gc_copy_interior_pointer_table_to_rootset(); /*used for computing collection time and mutator time*/ @@ -333,6 +391,27 @@ int64 get_collection_end_time() { return collection_end_time; } +void gc_prepare_rootset(GC* gc) +{ + if(!USE_CONCURRENT_GC){ + gc_metadata_verify(gc, TRUE); +#ifndef BUILD_IN_REFERENT + gc_finref_metadata_verify((GC*)gc, TRUE); +#endif + } + + /* Stop the threads and collect the roots. */ + lock(gc->enumerate_rootset_lock); + INFO2("gc.process", "GC: stop the threads and enumerate rootset ...\n"); + gc_clear_rootset(gc); + gc_reset_rootset(gc); + vm_enumerate_root_set_all_threads(); + gc_copy_interior_pointer_table_to_rootset(); + gc_set_rootset(gc); + unlock(gc->enumerate_rootset_lock); + +} + void gc_reclaim_heap(GC* gc, unsigned int gc_cause) { INFO2("gc.process", "\nGC: GC start ...\n"); @@ -367,28 +446,33 @@ gc_set_rootset(gc); unlock(gc->enumerate_rootset_lock); - if(USE_CONCURRENT_GC && gc_mark_is_concurrent()){ - gc_finish_concurrent_mark(gc); - } + if(USE_CONCURRENT_GC && gc_sweep_is_concurrent()){ + if(gc_is_concurrent_sweep_phase()) + gc_finish_concurrent_sweep(gc); + }else{ + if(USE_CONCURRENT_GC && gc_is_concurrent_mark_phase()){ + gc_finish_concurrent_mark(gc, TRUE); + } - gc->in_collection = TRUE; - - /* this has to be done after all mutators are suspended */ - gc_reset_mutator_context(gc); - - if(!IGNORE_FINREF ) gc_set_obj_with_fin(gc); + gc->in_collection = TRUE; + + /* this has to be done after all mutators are suspended */ + gc_reset_mutator_context(gc); + + if(!IGNORE_FINREF ) gc_set_obj_with_fin(gc); -#ifndef USE_MARK_SWEEP_GC - gc_gen_reclaim_heap((GC_Gen*)gc, collection_start_time); +#if defined(USE_MARK_SWEEP_GC) + gc_ms_reclaim_heap((GC_MS*)gc); +#elif defined(USE_UNIQUE_MOVE_COMPACT_GC) + gc_mc_reclaim_heap((GC_MC*)gc); #else - gc_ms_reclaim_heap((GC_MS*)gc); + gc_gen_reclaim_heap((GC_Gen*)gc, collection_start_time); #endif + } - gc_reset_interior_pointer_table(); - collection_end_time = time_now(); -#ifndef USE_MARK_SWEEP_GC +#if !defined(USE_MARK_SWEEP_GC)&&!defined(USE_UNIQUE_MOVE_COMPACT_GC) gc_gen_collection_verbose_info((GC_Gen*)gc, collection_end_time - collection_start_time, mutator_time); gc_gen_space_verbose_info((GC_Gen*)gc); #endif @@ -397,11 +481,15 @@ int64 mark_time = 0; if(USE_CONCURRENT_GC && gc_mark_is_concurrent()){ + mark_time = gc_get_concurrent_mark_time(gc); gc_reset_concurrent_mark(gc); - mark_time = gc_get_concurrent_mark_time(gc); + } + + if(USE_CONCURRENT_GC && gc_sweep_is_concurrent()){ + gc_reset_concurrent_sweep(gc); } -#ifndef USE_MARK_SWEEP_GC +#if !defined(USE_MARK_SWEEP_GC)&&!defined(USE_UNIQUE_MOVE_COMPACT_GC) if(USE_CONCURRENT_GC && gc_need_start_concurrent_mark(gc)) gc_start_concurrent_mark(gc); #endif @@ -432,6 +520,10 @@ vm_reclaim_native_objs(); gc->in_collection = FALSE; + + gc_reset_collector_state(gc); + + gc_clear_dirty_set(gc); vm_resume_threads_after(); assert(hythread_is_suspend_enabled()); @@ -441,3 +533,4 @@ } + Index: src/common/gc_common.h =================================================================== --- src/common/gc_common.h (revision 606795) +++ src/common/gc_common.h (working copy) @@ -38,6 +38,10 @@ #include "../gen/gc_for_barrier.h" +/* +#define USE_MARK_SWEEP_GC //define it to only use Mark-Sweep GC (no NOS, no LOS). +*/ +//#define USE_UNIQUE_MOVE_COMPACT_GC //define it to only use Move-Compact GC (no NOS, no LOS). #define GC_GEN_STATS #define null 0 @@ -77,6 +81,7 @@ #define USE_32BITS_HASHCODE +/* define it to use only mark-sweep GC for entire heap management */ //#define USE_MARK_SWEEP_GC typedef void (*TaskType)(void*); @@ -84,17 +89,15 @@ enum Collection_Algorithm{ COLLECTION_ALGOR_NIL, - /*minor nongen collection*/ + MINOR_GEN_FORWARD_POOL, MINOR_NONGEN_FORWARD_POOL, - /* minor gen collection */ - MINOR_GEN_FORWARD_POOL, + MINOR_GEN_SEMISPACE_POOL, + MINOR_NONGEN_SEMISPACE_POOL, - /* major collection */ MAJOR_COMPACT_SLIDE, MAJOR_COMPACT_MOVE, - MAJOR_MARK_SWEEP - + MAJOR_MARK_SWEEP }; /* Possible combinations: @@ -110,6 +113,7 @@ /* Two main kinds: generational GC and mark-sweep GC; this is decided at compiling time */ GEN_GC = 0x1, MARK_SWEEP_GC = 0x2, + MOVE_COMPACT_NO_LOS = 0x4, /* Mask of bits standing for two basic kinds */ GC_BASIC_KIND_MASK = ~(unsigned int)0x7, @@ -124,7 +128,8 @@ /* Sub-kinds of mark-sweep GC use the 12~15th LSB */ MS_COLLECTION = 0x1002, /* 0x1000 & MARK_SWEEP_GC */ - MS_COMPACT_COLLECTION = 0x2002 /* 0x2000 & MARK_SWEEP_GC */ + MS_COMPACT_COLLECTION = 0x2002, /* 0x2000 & MARK_SWEEP_GC */ + MC_COLLECTION = 0x1004 }; extern Boolean IS_FALLBACK_COMPACTION; /* only for mark/fw bits debugging purpose */ @@ -133,7 +138,8 @@ GC_CAUSE_NIL, GC_CAUSE_NOS_IS_FULL, GC_CAUSE_LOS_IS_FULL, - GC_CAUSE_SSPACE_IS_FULL, + GC_CAUSE_COS_IS_FULL, + GC_CAUSE_WSPACE_IS_FULL, GC_CAUSE_RUNTIME_FORCE_GC }; @@ -375,6 +381,20 @@ return TRUE; } +extern volatile Boolean obj_alloced_live; +inline Boolean is_obj_alloced_live() +{ return obj_alloced_live; } + +inline void gc_enable_alloc_obj_live() +{ + obj_alloced_live = TRUE; +} + +inline void gc_disenable_alloc_obj_live() +{ + obj_alloced_live = FALSE; +} + /* all GCs inherit this GC structure */ struct Marker; struct Mutator; @@ -433,8 +453,9 @@ SpinLock concurrent_mark_lock; SpinLock enumerate_rootset_lock; - + SpinLock concurrent_sweep_lock; + /* system info */ unsigned int _system_alloc_unit; unsigned int _machine_page_size_bytes; @@ -456,6 +477,8 @@ return (addr >= gc_heap_base(gc) && addr < gc_heap_ceiling(gc)); } +Boolean obj_belongs_to_gc_heap(Partial_Reveal_Object* p_obj); + /* gc must match exactly that kind if returning TRUE */ inline Boolean gc_match_kind(GC *gc, unsigned int kind) { @@ -472,11 +495,16 @@ return (Boolean)(gc->collect_kind & multi_kinds); } +inline void gc_reset_collector_state(GC* gc) +{ gc->num_active_collectors = 0;} + inline unsigned int gc_get_processor_num(GC* gc) { return gc->_num_processors; } void gc_parse_options(GC* gc); void gc_reclaim_heap(GC* gc, unsigned int gc_cause); +void gc_prepare_rootset(GC* gc); + int64 get_collection_end_time(); /* generational GC related */ @@ -498,6 +526,10 @@ #endif /* STATIC_NOS_MAPPING */ +void gc_init_collector_alloc(GC* gc, Collector* collector); +void gc_reset_collector_alloc(GC* gc, Collector* collector); +void gc_destruct_collector_alloc(GC* gc, Collector* collector); + FORCE_INLINE Boolean addr_belongs_to_nos(void* addr) { return addr >= nos_boundary; } @@ -513,5 +545,5 @@ inline Boolean obj_is_moved(Partial_Reveal_Object* p_obj) { return ((p_obj >= los_boundary) || (*p_global_lspace_move_obj)); } -extern Boolean VTABLE_TRACING; +extern Boolean TRACE_JLC_VIA_VTABLE; #endif //_GC_COMMON_H_ Index: src/common/gc_concurrent.cpp =================================================================== --- src/common/gc_concurrent.cpp (revision 606795) +++ src/common/gc_concurrent.cpp (working copy) @@ -14,24 +14,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #include "gc_common.h" #include "gc_metadata.h" #include "../thread/mutator.h" #include "../thread/marker.h" +#include "../thread/collector.h" #include "../finalizer_weakref/finalizer_weakref.h" #include "../gen/gen.h" #include "../mark_sweep/gc_ms.h" #include "interior_pointer.h" #include "collection_scheduler.h" #include "gc_concurrent.h" +#include "../gen/gc_for_barrier.h" -Boolean USE_CONCURRENT_GC = FALSE; +Boolean USE_CONCURRENT_GC = FALSE; +Boolean USE_CONCURRENT_ENUMERATION = FALSE; +Boolean USE_CONCURRENT_MARK = FALSE; +Boolean USE_CONCURRENT_SWEEP = FALSE; -volatile Boolean concurrent_mark_phase = FALSE; -volatile Boolean mark_is_concurrent = FALSE; +volatile Boolean concurrent_mark_phase = FALSE; +volatile Boolean mark_is_concurrent = FALSE; +volatile Boolean concurrent_sweep_phase = FALSE; +volatile Boolean sweep_is_concurrent = FALSE; +volatile Boolean gc_sweeping_global_normal_chunk = FALSE; +unsigned int CONCURRENT_ALGO = 0; + static void gc_check_concurrent_mark(GC* gc) { if(!is_mark_finished(gc)){ @@ -39,7 +48,13 @@ #ifndef USE_MARK_SWEEP_GC gc_gen_start_concurrent_mark((GC_Gen*)gc); #else - gc_ms_start_concurrent_mark((GC_MS*)gc, MAX_NUM_MARKERS); + if(gc_concurrent_match_algorithm(OTF_REM_OBJ_SNAPSHOT_ALGO)){ + gc_ms_start_concurrent_mark((GC_MS*)gc, MIN_NUM_MARKERS); + }else if(gc_concurrent_match_algorithm(OTF_REM_NEW_TARGET_ALGO)){ + gc_ms_start_concurrent_mark((GC_MS*)gc, MIN_NUM_MARKERS); + }else if(gc_concurrent_match_algorithm(MOSTLY_CONCURRENT_ALGO)){ + //ignore. + } #endif unlock(gc->concurrent_mark_lock); } @@ -48,12 +63,14 @@ static void gc_wait_concurrent_mark_finish(GC* gc) { wait_mark_finish(gc); + gc_set_barrier_function(WRITE_BARRIER_REM_NIL); gc_set_concurrent_status(gc,GC_CONCURRENT_STATUS_NIL); - gc_reset_snaptshot(gc); } void gc_start_concurrent_mark(GC* gc) { + int disable_count; + if(!try_lock(gc->concurrent_mark_lock) || gc_mark_is_concurrent()) return; /*prepare rootset*/ @@ -61,6 +78,7 @@ lock(gc->enumerate_rootset_lock); gc_metadata_verify(gc, TRUE); gc_reset_rootset(gc); + disable_count = hythread_reset_suspend_disable(); vm_enumerate_root_set_all_threads(); gc_copy_interior_pointer_table_to_rootset(); gc_set_rootset(gc); @@ -79,21 +97,77 @@ #ifndef USE_MARK_SWEEP_GC gc_gen_start_concurrent_mark((GC_Gen*)gc); #else - gc_ms_start_concurrent_mark((GC_MS*)gc, MIN_NUM_MARKERS); + if(gc_concurrent_match_algorithm(OTF_REM_OBJ_SNAPSHOT_ALGO)){ + gc_set_barrier_function(WRITE_BARRIER_REM_OBJ_SNAPSHOT); + gc_ms_start_concurrent_mark((GC_MS*)gc, MIN_NUM_MARKERS); + }else if(gc_concurrent_match_algorithm(MOSTLY_CONCURRENT_ALGO)){ + gc_set_barrier_function(WRITE_BARRIER_REM_SOURCE_OBJ); + gc_ms_start_most_concurrent_mark((GC_MS*)gc, MIN_NUM_MARKERS); + }else if(gc_concurrent_match_algorithm(OTF_REM_NEW_TARGET_ALGO)){ + gc_set_barrier_function(WRITE_BARRIER_REM_OLD_VAR); + gc_ms_start_concurrent_mark((GC_MS*)gc, MIN_NUM_MARKERS); + } #endif if(TRUE){ unlock(gc->enumerate_rootset_lock); vm_resume_threads_after(); + assert(hythread_is_suspend_enabled()); + hythread_set_suspend_disable(disable_count); } unlock(gc->concurrent_mark_lock); } -void gc_finish_concurrent_mark(GC* gc) +void wspace_mark_scan_mostly_concurrent_reset(); +void wspace_mark_scan_mostly_concurrent_terminate(); + +void gc_finish_concurrent_mark(GC* gc, Boolean is_STW) { gc_check_concurrent_mark(gc); + + if(gc_concurrent_match_algorithm(MOSTLY_CONCURRENT_ALGO)) + wspace_mark_scan_mostly_concurrent_terminate(); + gc_wait_concurrent_mark_finish(gc); + + if(gc_concurrent_match_algorithm(MOSTLY_CONCURRENT_ALGO)){ + /*If gc use mostly concurrent algorithm, there's a final marking pause. + Suspend the mutators once again and finish the marking phase.*/ + int disable_count; + if(!is_STW){ + /*suspend the mutators.*/ + lock(gc->enumerate_rootset_lock); + gc_metadata_verify(gc, TRUE); + gc_reset_rootset(gc); + disable_count = hythread_reset_suspend_disable(); + vm_enumerate_root_set_all_threads(); + gc_copy_interior_pointer_table_to_rootset(); + gc_set_rootset(gc); + } + + /*prepare dirty object*/ + gc_prepare_dirty_set(gc); + + gc_set_weakref_sets(gc); + + /*start STW mark*/ +#ifndef USE_MARK_SWEEP_GC + assert(0); +#else + gc_ms_start_final_mark_after_concurrent((GC_MS*)gc, MIN_NUM_MARKERS); +#endif + + wspace_mark_scan_mostly_concurrent_reset(); + gc_clear_dirty_set(gc); + if(!is_STW){ + unlock(gc->enumerate_rootset_lock); + vm_resume_threads_after(); + assert(hythread_is_suspend_enabled()); + hythread_set_suspend_disable(disable_count); + } + } + gc_reset_dirty_set(gc); } void gc_reset_concurrent_mark(GC* gc) @@ -112,8 +186,141 @@ if(marker->time_mark > time_mark){ time_mark = marker->time_mark; } + marker->time_mark = 0; } return time_mark; } +void gc_start_concurrent_sweep(GC* gc) +{ + if(!try_lock(gc->concurrent_sweep_lock) || gc_sweep_is_concurrent()) return; + + /*FIXME: enable finref*/ + if(!IGNORE_FINREF ){ + gc_set_obj_with_fin(gc); + Collector* collector = gc->collectors[0]; + collector_identify_finref(collector); +#ifndef BUILD_IN_REFERENT + }else{ + gc_set_weakref_sets(gc); + gc_update_weakref_ignore_finref(gc); +#endif + } + + gc_set_concurrent_status(gc, GC_CONCURRENT_SWEEP_PHASE); + + gc_set_weakref_sets(gc); + + /*Note: We assumed that adding entry to weakroot_pool is happened in STW rootset enumeration. + So, when this assumption changed, we should modified the below function.*/ + gc_identify_dead_weak_roots(gc); + + /*start concurrent mark*/ +#ifndef USE_MARK_SWEEP_GC + assert(0); +#else + gc_ms_start_concurrent_sweep((GC_MS*)gc, MIN_NUM_MARKERS); +#endif + + unlock(gc->concurrent_sweep_lock); +} + +void gc_reset_concurrent_sweep(GC* gc) +{ + gc->num_active_collectors = 0; + gc_sweep_unset_concurrent(); +} + +void gc_wait_concurrent_sweep_finish(GC* gc) +{ + wait_collection_finish(gc); + gc_set_concurrent_status(gc,GC_CONCURRENT_STATUS_NIL); +} + +void gc_finish_concurrent_sweep(GC * gc) +{ + gc_wait_concurrent_sweep_finish(gc); +} + +void gc_check_concurrent_phase(GC * gc) +{ + /*Note: we do not finish concurrent mark here if we do not want to start concurrent sweep.*/ + if(gc_is_concurrent_mark_phase(gc) && is_mark_finished(gc) && USE_CONCURRENT_SWEEP){ + /*Although all conditions above are satisfied, we can not guarantee concurrent marking is finished. + Because, sometimes, the concurrent marking has not started yet. We check the concurrent mark lock + here to guarantee this occasional case.*/ + if(try_lock(gc->concurrent_mark_lock)){ + unlock(gc->concurrent_mark_lock); + gc_finish_concurrent_mark(gc, FALSE); + } + } + + if(gc_is_concurrent_sweep_phase(gc) && is_collector_finished(gc)){ + //The reason is same as concurrent mark above. + if(try_lock(gc->concurrent_sweep_lock)){ + unlock(gc->concurrent_sweep_lock); + gc_finish_concurrent_sweep(gc); + } + } +} + + +void gc_reset_after_concurrent_collection(GC* gc) +{ + /*FIXME: enable concurrent GEN mode.*/ + gc_reset_interior_pointer_table(); + if(gc_is_gen_mode()) gc_prepare_mutator_remset(gc); + + /* Clear rootset pools here rather than in each collection algorithm */ + gc_clear_rootset(gc); + + if(!IGNORE_FINREF ){ + INFO2("gc.process", "GC: finref process after collection ...\n"); + gc_put_finref_to_vm(gc); + gc_reset_finref_metadata(gc); + gc_activate_finref_threads((GC*)gc); +#ifndef BUILD_IN_REFERENT + } else { + gc_clear_weakref_pools(gc); +#endif + } + +#ifdef USE_MARK_SWEEP_GC + gc_ms_update_space_statistics((GC_MS*)gc); +#endif + + gc_clear_dirty_set(gc); + + vm_reclaim_native_objs(); + gc->in_collection = FALSE; + + gc_reset_collector_state(gc); + + if(USE_CONCURRENT_GC && gc_mark_is_concurrent()){ + gc_reset_concurrent_mark(gc); + } + + if(USE_CONCURRENT_GC && gc_sweep_is_concurrent()){ + gc_reset_concurrent_sweep(gc); + } +} + +void gc_decide_concurrent_algorithm(GC* gc, char* concurrent_algo) +{ + if(!concurrent_algo){ + CONCURRENT_ALGO = OTF_REM_OBJ_SNAPSHOT_ALGO; + }else{ + string_to_upper(concurrent_algo); + + if(!strcmp(concurrent_algo, "OTF_OBJ")){ + CONCURRENT_ALGO = OTF_REM_OBJ_SNAPSHOT_ALGO; + + }else if(!strcmp(concurrent_algo, "MOSTLY_CON")){ + CONCURRENT_ALGO = MOSTLY_CONCURRENT_ALGO; + }else if(!strcmp(concurrent_algo, "OTF_SLOT")){ + CONCURRENT_ALGO = OTF_REM_NEW_TARGET_ALGO; + } + } +} + Index: src/common/gc_concurrent.h =================================================================== --- src/common/gc_concurrent.h (revision 606795) +++ src/common/gc_concurrent.h (working copy) @@ -22,12 +22,47 @@ enum GC_CONCURRENT_STATUS{ GC_CONCURRENT_STATUS_NIL = 0x00, GC_CONCURRENT_MARK_PHASE = 0x01, + GC_CONCURRENT_MARK_FINAL_PAUSE_PHASE = 0x11, // for mostly concurrent only. + GC_CONCURRENT_SWEEP_PHASE = 0x02 }; +enum HANDSHAKE_SINGAL{ + HANDSHAKE_NIL = 0x00, + + /*mutator to collector*/ + ENABLE_COLLECTOR_SWEEP_LOCAL_CHUNKS = 0x01, + DISABLE_COLLECTOR_SWEEP_LOCAL_CHUNKS = 0x02, + + + ENABLE_COLLECTOR_SWEEP_GLOBAL_CHUNKS = 0x03, + DISABLE_COLLECTOR_SWEEP_GLOBAL_CHUNKS = 0x04 +// /*collector to mutator*/ +// ENABLE_MUTATOR_ALLOC_BARRIER = 0x03, +// DISABLE_MUTATOR_ALLOC_BARRIER = 0x04 +}; + +extern Boolean USE_CONCURRENT_GC; +extern Boolean USE_CONCURRENT_ENUMERATION; +extern Boolean USE_CONCURRENT_MARK; +extern Boolean USE_CONCURRENT_SWEEP; + extern volatile Boolean concurrent_mark_phase; extern volatile Boolean mark_is_concurrent; -extern Boolean USE_CONCURRENT_GC; +extern volatile Boolean concurrent_sweep_phase; +extern volatile Boolean sweep_is_concurrent; +extern unsigned int CONCURRENT_ALGO; +enum CONCURRENT_MARK_ALGORITHM{ + OTF_REM_OBJ_SNAPSHOT_ALGO = 0x01, + OTF_REM_NEW_TARGET_ALGO = 0x02, + MOSTLY_CONCURRENT_ALGO = 0x03 +}; + +inline Boolean gc_concurrent_match_algorithm(unsigned int concurrent_algo) +{ + return CONCURRENT_ALGO == concurrent_algo; +} + inline Boolean gc_mark_is_concurrent() { return mark_is_concurrent; @@ -35,11 +70,15 @@ inline void gc_mark_set_concurrent() { + if(gc_concurrent_match_algorithm(OTF_REM_OBJ_SNAPSHOT_ALGO) + ||gc_concurrent_match_algorithm(OTF_REM_NEW_TARGET_ALGO)) + gc_enable_alloc_obj_live(); mark_is_concurrent = TRUE; } inline void gc_mark_unset_concurrent() { + gc_disenable_alloc_obj_live(); mark_is_concurrent = FALSE; } @@ -53,21 +92,77 @@ return gc->gc_concurrent_status == GC_CONCURRENT_MARK_PHASE; } +inline Boolean gc_sweep_is_concurrent() +{ + return sweep_is_concurrent; +} + +inline void gc_sweep_set_concurrent() +{ + sweep_is_concurrent = TRUE; +} + +inline void gc_sweep_unset_concurrent() +{ + sweep_is_concurrent = FALSE; +} + +inline Boolean gc_is_concurrent_sweep_phase() +{ + return concurrent_sweep_phase; +} + +inline Boolean gc_is_concurrent_sweep_phase(GC* gc) +{ + return gc->gc_concurrent_status == GC_CONCURRENT_SWEEP_PHASE; +} + inline void gc_set_concurrent_status(GC*gc, unsigned int status) { + /*Reset status*/ + concurrent_mark_phase = FALSE; + concurrent_sweep_phase = FALSE; + gc->gc_concurrent_status = status; - if(status == GC_CONCURRENT_MARK_PHASE){ - concurrent_mark_phase = TRUE; - gc_mark_set_concurrent(); - }else{ - concurrent_mark_phase = FALSE; + switch(status){ + case GC_CONCURRENT_MARK_PHASE: + concurrent_mark_phase = TRUE; + gc_mark_set_concurrent(); + break; + case GC_CONCURRENT_SWEEP_PHASE: + concurrent_sweep_phase = TRUE; + gc_sweep_set_concurrent(); + break; + default: + assert(!concurrent_mark_phase && !concurrent_sweep_phase); } + + return; } void gc_reset_concurrent_mark(GC* gc); void gc_start_concurrent_mark(GC* gc); -void gc_finish_concurrent_mark(GC* gc); +void gc_finish_concurrent_mark(GC* gc, Boolean is_STW); int64 gc_get_concurrent_mark_time(GC* gc); +void gc_start_concurrent_sweep(GC* gc); +void gc_finish_concurrent_sweep(GC * gc); +void gc_reset_after_concurrent_collection(GC* gc); +void gc_check_concurrent_phase(GC * gc); + +void gc_decide_concurrent_algorithm(GC* gc, char* concurrent_algo); + +void gc_reset_concurrent_sweep(GC* gc); + +extern volatile Boolean gc_sweeping_global_normal_chunk; + +inline Boolean gc_is_sweeping_global_normal_chunk() +{ return gc_sweeping_global_normal_chunk; } + +inline void gc_set_sweeping_global_normal_chunk() +{ gc_sweeping_global_normal_chunk = TRUE; } + +inline void gc_unset_sweeping_global_normal_chunk() +{ gc_sweeping_global_normal_chunk = FALSE; } #endif Index: src/common/gc_for_class.h =================================================================== --- src/common/gc_for_class.h (revision 606795) +++ src/common/gc_for_class.h (working copy) @@ -25,9 +25,6 @@ #include "open/types.h" #include "gc_platform.h" -#ifndef FORCE_INLINE -#define FORCE_INLINE inline -#endif /* CONST_MARK_BIT is used in mark_scan in vt, no matter MARK_BIT_FLIPPING used or not. MARK_BIT_FLIPPING is used in oi for marking and forwarding in non-gen nursery forwarding (the marking is for those objects not in nos.) @@ -158,23 +155,23 @@ extern POINTER_SIZE_INT vtable_base; #ifdef COMPRESS_VTABLE -FORCE_INLINE VT compress_vt(Partial_Reveal_VTable* vt) +FORCE_INLINE VT encode_vt(Partial_Reveal_VTable* vt) { assert(vt); return (VT)((POINTER_SIZE_INT)vt - vtable_base); } -FORCE_INLINE Partial_Reveal_VTable* uncompress_vt(VT vt) +FORCE_INLINE Partial_Reveal_VTable* decode_vt(VT vt) { assert(vt); return (Partial_Reveal_VTable*)((POINTER_SIZE_INT)vt + vtable_base); } #else/*ifdef COMPRESS_VTABLE*/ -FORCE_INLINE VT compress_vt(Partial_Reveal_VTable* vt) +FORCE_INLINE VT encode_vt(Partial_Reveal_VTable* vt) { return (VT)vt; } -FORCE_INLINE Partial_Reveal_VTable* uncompress_vt(VT vt) +FORCE_INLINE Partial_Reveal_VTable* decode_vt(VT vt) { return (Partial_Reveal_VTable*) vt; } #endif @@ -226,13 +223,13 @@ FORCE_INLINE GC_VTable_Info *obj_get_gcvt_raw(Partial_Reveal_Object *obj) { - Partial_Reveal_VTable *vtable = uncompress_vt(obj_get_vt(obj)); + Partial_Reveal_VTable *vtable = decode_vt(obj_get_vt(obj)); return vtable_get_gcvt_raw(vtable); } FORCE_INLINE GC_VTable_Info *obj_get_gcvt(Partial_Reveal_Object *obj) { - Partial_Reveal_VTable* vtable = uncompress_vt(obj_get_vt(obj) ); + Partial_Reveal_VTable* vtable = decode_vt(obj_get_vt(obj) ); return vtable_get_gcvt(vtable); } @@ -244,7 +241,7 @@ FORCE_INLINE Boolean object_has_ref_field_before_scan(Partial_Reveal_Object *obj) { - Partial_Reveal_VTable *vt = uncompress_vt(obj_get_vt_raw(obj)); + Partial_Reveal_VTable *vt = decode_vt(obj_get_vt_raw(obj)); GC_VTable_Info *gcvt = vtable_get_gcvt_raw(vt); return (Boolean)((POINTER_SIZE_INT)gcvt & GC_CLASS_FLAG_REFS); } @@ -321,3 +318,4 @@ + Index: src/common/gc_for_vm.cpp =================================================================== --- src/common/gc_for_vm.cpp (revision 606795) +++ src/common/gc_for_vm.cpp (working copy) @@ -63,7 +63,6 @@ #endif vm_helper_register_magic_helper(VM_RT_NEW_RESOLVED_USING_VTABLE_AND_SIZE, "org/apache/harmony/drlvm/gc_gen/GCHelper", "alloc"); vm_helper_register_magic_helper(VM_RT_NEW_VECTOR_USING_VTABLE, "org/apache/harmony/drlvm/gc_gen/GCHelper", "allocArray"); - vm_helper_register_magic_helper(VM_RT_GC_HEAP_WRITE_REF, "org/apache/harmony/drlvm/gc_gen/GCHelper", "write_barrier_slot_rem"); } int gc_init() @@ -73,10 +72,12 @@ vm_gc_lock_init(); -#ifndef USE_MARK_SWEEP_GC - unsigned int gc_struct_size = sizeof(GC_Gen); +#if defined(USE_MARK_SWEEP_GC) +unsigned int gc_struct_size = sizeof(GC_MS); +#elif defined(USE_UNIQUE_MOVE_COMPACT_GC) +unsigned int gc_struct_size = sizeof(GC_MC); #else - unsigned int gc_struct_size = sizeof(GC_MS); +unsigned int gc_struct_size = sizeof(GC_Gen); #endif GC* gc = (GC*)STD_MALLOC(gc_struct_size); assert(gc); @@ -85,10 +86,16 @@ gc_parse_options(gc); -// iberezhniuk: compile-time switching is now used in both VM and GC +#ifdef BUILD_IN_REFERENT + if( ! IGNORE_FINREF){ + INFO2(" gc.init" , "finref must be ignored, since BUILD_IN_REFERENT is defined." ); + IGNORE_FINREF = TRUE; + } +#endif + +/* VT pointer compression is a compile-time option, reference compression and vtable compression are orthogonal */ #ifdef COMPRESS_VTABLE assert(vm_vtable_pointers_are_compressed()); - // ppervov: reference compression and vtable compression are orthogonal vtable_base = vm_get_vtable_base(); #endif @@ -98,10 +105,12 @@ gc_metadata_initialize(gc); /* root set and mark stack */ -#ifndef USE_MARK_SWEEP_GC +#if defined(USE_MARK_SWEEP_GC) + gc_ms_initialize((GC_MS*)gc, min_heap_size_bytes, max_heap_size_bytes); +#elif defined(USE_UNIQUE_MOVE_COMPACT_GC) + gc_mc_initialize((GC_MC*)gc, min_heap_size_bytes, max_heap_size_bytes); +#else gc_gen_initialize((GC_Gen*)gc, min_heap_size_bytes, max_heap_size_bytes); -#else - gc_ms_initialize((GC_MS*)gc, min_heap_size_bytes, max_heap_size_bytes); #endif set_native_finalizer_thread_flag(!IGNORE_FINREF); @@ -132,11 +141,13 @@ INFO2("gc.process", "GC: call GC wrapup ...."); GC* gc = p_global_gc; -#ifndef USE_MARK_SWEEP_GC +#if defined(USE_MARK_SWEEP_GC) + gc_ms_destruct((GC_MS*)gc); +#elif defined(USE_UNIQUE_MOVE_COMPACT_GC) + gc_mc_destruct((GC_MC*)gc); +#else gc_gen_wrapup_verbose((GC_Gen*)gc); gc_gen_destruct((GC_Gen*)gc); -#else - gc_ms_destruct((GC_MS*)gc); #endif gc_metadata_destruct(gc); /* root set and mark stack */ @@ -205,7 +216,7 @@ Boolean gc_supports_class_unloading() { - return VTABLE_TRACING; + return TRACE_JLC_VIA_VTABLE; } void gc_add_weak_root_set_entry(Managed_Object_Handle *ref, Boolean is_pinned, Boolean is_short_weak) @@ -253,29 +264,35 @@ int64 gc_free_memory() { -#ifndef USE_MARK_SWEEP_GC +#if defined(USE_MARK_SWEEP_GC) + return (int64)gc_ms_free_memory_size((GC_MS*)p_global_gc); +#elif defined(USE_UNIQUE_MOVE_COMPACT_GC) + return (int64)gc_mc_free_memory_size((GC_MC*)p_global_gc); +#else return (int64)gc_gen_free_memory_size((GC_Gen*)p_global_gc); -#else - return (int64)gc_ms_free_memory_size((GC_MS*)p_global_gc); #endif } /* java heap size.*/ int64 gc_total_memory() { -#ifndef USE_MARK_SWEEP_GC +#if defined(USE_MARK_SWEEP_GC) + return (int64)((POINTER_SIZE_INT)gc_ms_total_memory_size((GC_MS*)p_global_gc)); +#elif defined(USE_UNIQUE_MOVE_COMPACT_GC) + return (int64)((POINTER_SIZE_INT)gc_mc_total_memory_size((GC_MC*)p_global_gc)); +#else return (int64)((POINTER_SIZE_INT)gc_gen_total_memory_size((GC_Gen*)p_global_gc)); -#else - return (int64)((POINTER_SIZE_INT)gc_ms_total_memory_size((GC_MS*)p_global_gc)); #endif } int64 gc_max_memory() { -#ifndef USE_MARK_SWEEP_GC - return (int64)((POINTER_SIZE_INT)gc_gen_total_memory_size((GC_Gen*)p_global_gc)); +#if defined(USE_MARK_SWEEP_GC) + return (int64)((POINTER_SIZE_INT)gc_ms_total_memory_size((GC_MS*)p_global_gc)); +#elif defined(USE_UNIQUE_MOVE_COMPACT_GC) + return (int64)((POINTER_SIZE_INT)gc_mc_total_memory_size((GC_MC*)p_global_gc)); #else - return (int64)((POINTER_SIZE_INT)gc_ms_total_memory_size((GC_MS*)p_global_gc)); + return (int64)((POINTER_SIZE_INT)gc_gen_total_memory_size((GC_Gen*)p_global_gc)); #endif } @@ -343,7 +360,7 @@ #else //USE_32BITS_HASHCODE int32 gc_get_hashcode(Managed_Object_Handle p_object) { -#ifdef USE_MARK_SWEEP_GC +#if defined(USE_MARK_SWEEP_GC) || defined(USE_UNIQUE_MOVE_COMPACT_GC) return (int32)0;//p_object; #endif @@ -403,10 +420,12 @@ // data structures in not consistent for heap iteration if (!JVMTI_HEAP_ITERATION) return; -#ifndef USE_MARK_SWEEP_GC - gc_gen_iterate_heap((GC_Gen *)p_global_gc); +#if defined(USE_MARK_SWEEP_GC) + gc_ms_iterate_heap((GC_MS*)p_global_gc); +#elif defined(USE_UNIQUE_MOVE_COMPACT_GC) + gc_mc_iterate_heap((GC_MC*)p_global_gc); #else - gc_ms_iterate_heap((GC_MS*)p_global_gc); + gc_gen_iterate_heap((GC_Gen *)p_global_gc); #endif } @@ -419,3 +438,9 @@ mutator_need_block = FALSE; return old_flag; } + +Boolean obj_belongs_to_gc_heap(Partial_Reveal_Object* p_obj) +{ + return address_belongs_to_gc_heap(p_obj, p_global_gc); +} + Index: src/common/gc_metadata.cpp =================================================================== --- src/common/gc_metadata.cpp (revision 606795) +++ src/common/gc_metadata.cpp (working copy) @@ -76,7 +76,7 @@ gc_metadata.mutator_remset_pool = sync_pool_create(); gc_metadata.collector_remset_pool = sync_pool_create(); gc_metadata.collector_repset_pool = sync_pool_create(); - gc_metadata.dirty_obj_snaptshot_pool = sync_pool_create(); + gc_metadata.gc_dirty_set_pool = sync_pool_create(); gc_metadata.weakroot_pool = sync_pool_create(); #ifdef USE_32BITS_HASHCODE gc_metadata.collector_hashcode_pool = sync_pool_create(); @@ -99,7 +99,7 @@ sync_pool_destruct(metadata->mutator_remset_pool); sync_pool_destruct(metadata->collector_remset_pool); sync_pool_destruct(metadata->collector_repset_pool); - sync_pool_destruct(metadata->dirty_obj_snaptshot_pool); + sync_pool_destruct(metadata->gc_dirty_set_pool); sync_pool_destruct(metadata->weakroot_pool); #ifdef USE_32BITS_HASHCODE sync_pool_destruct(metadata->collector_hashcode_pool); @@ -188,10 +188,17 @@ //if(obj_is_moved(p_obj)) /*Fixme: los_boundery ruined the modularity of gc_common.h*/ if(p_obj < los_boundary){ - write_slot(p_ref, obj_get_fw_in_oi(p_obj)); + p_obj = obj_get_fw_in_oi(p_obj); }else{ - *p_ref = obj_get_fw_in_table(p_obj); + p_obj = obj_get_fw_in_table(p_obj); } + + write_slot(p_ref, p_obj); + + }else if(gc_match_kind(gc, MC_COLLECTION)){ + p_obj = obj_get_fw_in_table(p_obj); + write_slot(p_ref, p_obj); + }else{ if(obj_is_fw_in_oi(p_obj)){ /* Condition obj_is_moved(p_obj) is for preventing mistaking previous mark bit of large obj as fw bit when fallback happens. @@ -415,52 +422,51 @@ Boolean obj_is_mark_black_in_table(Partial_Reveal_Object* p_obj); #endif -void gc_reset_snaptshot(GC* gc) +void gc_reset_dirty_set(GC* gc) { GC_Metadata* metadata = gc->metadata; - /*reset mutator local snapshot block*/ Mutator *mutator = gc->mutator_list; while (mutator) { - Vector_Block* local_snapshot = mutator->dirty_obj_snapshot; - assert(local_snapshot); - if(!vector_block_is_empty(local_snapshot)){ + Vector_Block* local_dirty_set = mutator->dirty_set; + assert(local_dirty_set); + if(!vector_block_is_empty(local_dirty_set)){ #ifdef _DEBUG - POINTER_SIZE_INT* iter = vector_block_iterator_init(local_snapshot); - while(!vector_block_iterator_end(local_snapshot,iter)){ + POINTER_SIZE_INT* iter = vector_block_iterator_init(local_dirty_set); + while(!vector_block_iterator_end(local_dirty_set,iter)){ Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*) *iter; - iter = vector_block_iterator_advance(local_snapshot, iter); + iter = vector_block_iterator_advance(local_dirty_set, iter); #ifdef USE_MARK_SWEEP_GC assert(obj_is_mark_black_in_table(p_obj)); #endif } #endif - vector_block_clear(mutator->dirty_obj_snapshot); + vector_block_clear(mutator->dirty_set); } mutator = mutator->next; } - /*reset global snapshot pool*/ - Pool* global_snapshot = metadata->dirty_obj_snaptshot_pool; + /*reset global dirty set pool*/ + Pool* global_dirty_set_pool = metadata->gc_dirty_set_pool; - if(!pool_is_empty(global_snapshot)){ - Vector_Block* snapshot_block = pool_get_entry(global_snapshot); - while(snapshot_block != NULL){ - if(!vector_block_is_empty(snapshot_block)){ + if(!pool_is_empty(global_dirty_set_pool)){ + Vector_Block* dirty_set = pool_get_entry(global_dirty_set_pool); + while(dirty_set != NULL){ + if(!vector_block_is_empty(dirty_set)){ #ifdef _DEBUG - POINTER_SIZE_INT* iter = vector_block_iterator_init(snapshot_block); - while(!vector_block_iterator_end(snapshot_block,iter)){ + POINTER_SIZE_INT* iter = vector_block_iterator_init(dirty_set); + while(!vector_block_iterator_end(dirty_set,iter)){ Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*) *iter; - iter = vector_block_iterator_advance(snapshot_block, iter); + iter = vector_block_iterator_advance(dirty_set, iter); #ifdef USE_MARK_SWEEP_GC assert(obj_is_mark_black_in_table(p_obj)); #endif } #endif } - vector_block_clear(snapshot_block); - pool_put_entry(metadata->free_set_pool,snapshot_block); - snapshot_block = pool_get_entry(global_snapshot); + vector_block_clear(dirty_set); + pool_put_entry(metadata->free_set_pool,dirty_set); + dirty_set = pool_get_entry(global_dirty_set_pool); } } @@ -468,8 +474,38 @@ } +void gc_prepare_dirty_set(GC* gc) +{ + GC_Metadata* metadata = gc->metadata; + Pool* gc_dirty_set_pool = metadata->gc_dirty_set_pool; + lock(gc->mutator_list_lock); + + Mutator *mutator = gc->mutator_list; + while (mutator) { + //FIXME: temproray solution for mostly concurrent. + lock(mutator->dirty_set_lock); + pool_put_entry(gc_dirty_set_pool, mutator->dirty_set); + mutator->dirty_set = free_set_pool_get_entry(metadata); + unlock(mutator->dirty_set_lock); + mutator = mutator->next; + } + unlock(gc->mutator_list_lock); +} +void gc_clear_dirty_set(GC* gc) +{ + gc_prepare_dirty_set(gc); + GC_Metadata* metadata = gc->metadata; + Vector_Block* dirty_set = pool_get_entry(metadata->gc_dirty_set_pool); + while(dirty_set){ + vector_block_clear(dirty_set); + pool_put_entry(metadata->free_set_pool, dirty_set); + dirty_set = pool_get_entry(metadata->gc_dirty_set_pool); + } +} + + Index: src/common/gc_metadata.h =================================================================== --- src/common/gc_metadata.h (revision 606795) +++ src/common/gc_metadata.h (working copy) @@ -49,7 +49,7 @@ Pool* collector_hashcode_pool; #endif - Pool* dirty_obj_snaptshot_pool; + Pool* gc_dirty_set_pool; }GC_Metadata; @@ -64,13 +64,16 @@ void gc_clear_rootset(GC* gc); void gc_fix_rootset(Collector* collector, Boolean double_fix); void gc_clear_remset(GC* gc); -void gc_reset_snaptshot(GC* gc); +void gc_reset_dirty_set(GC* gc); void gc_identify_dead_weak_roots(GC *gc); void gc_update_weak_roots(GC *gc, Boolean double_fix); void gc_clear_remset(GC* gc); +void gc_clear_dirty_set(GC* gc); +void gc_prepare_dirty_set(GC* gc); + inline void gc_task_pool_clear(Pool* task_pool) { Vector_Block* task = pool_get_entry(task_pool); @@ -136,21 +139,20 @@ assert(mutator->rem_set); } -inline void mutator_snapshotset_add_entry(Mutator* mutator, Partial_Reveal_Object* p_obj) +inline void mutator_dirtyset_add_entry(Mutator* mutator, Partial_Reveal_Object* p_obj) { - Vector_Block* dirty_obj_snapshot = mutator->dirty_obj_snapshot; - vector_block_add_entry(dirty_obj_snapshot, (POINTER_SIZE_INT)p_obj); + Vector_Block* dirty_set = mutator->dirty_set; + vector_block_add_entry(dirty_set, (POINTER_SIZE_INT)p_obj); - if( !vector_block_is_full(dirty_obj_snapshot) ) return; + if( !vector_block_is_full(dirty_set) ) return; - vector_block_set_full(dirty_obj_snapshot); + vector_block_set_full(dirty_set); - if(vector_block_set_exclusive(dirty_obj_snapshot)){ - //?vector_block_set_full(dirty_obj_snapshot); //ynhe - pool_put_entry(gc_metadata.dirty_obj_snaptshot_pool, dirty_obj_snapshot); + if(vector_block_set_exclusive(dirty_set)){ + pool_put_entry(gc_metadata.gc_dirty_set_pool, dirty_set); } - mutator->dirty_obj_snapshot = free_set_pool_get_entry(&gc_metadata); + mutator->dirty_set = free_set_pool_get_entry(&gc_metadata); } inline void collector_repset_add_entry(Collector* collector, Partial_Reveal_Object** p_ref) Index: src/common/gc_platform.h =================================================================== --- src/common/gc_platform.h (revision 606795) +++ src/common/gc_platform.h (working copy) @@ -55,14 +55,12 @@ #ifdef _WINDOWS_ #define FORCE_INLINE __forceinline +#elif defined (__linux__) +#define FORCE_INLINE inline __attribute__((always_inline)) #else +#define FORCE_INLINE inline +#endif /* _WINDOWS_ */ -#ifdef __linux__ -#define FORCE_INLINE inline __attribute__((always_inline)) -#endif - -#endif - #define ABS_DIFF(x, y) (((x)>(y))?((x)-(y)):((y)-(x))) #define USEC_PER_SEC INT64_C(1000000) @@ -118,6 +116,11 @@ (hythread_entrypoint_t)func, data); } +inline int vm_thread_is_suspend_enable() +{ + return hythread_is_suspend_enabled(); +} + inline void *atomic_casptr(volatile void **mem, void *with, const void *cmp) { return apr_atomic_casptr(mem, with, cmp); } Index: src/common/gc_space.h =================================================================== --- src/common/gc_space.h (revision 606795) +++ src/common/gc_space.h (working copy) @@ -128,117 +128,15 @@ inline unsigned int blocked_space_free_mem_size(Blocked_Space *space){ return GC_BLOCK_SIZE_BYTES * (space->ceiling_block_idx - space->free_block_idx + 1); } inline Boolean blocked_space_used_mem_size(Blocked_Space *space){ return GC_BLOCK_SIZE_BYTES * (space->free_block_idx - space->first_block_idx); } -inline void space_init_blocks(Blocked_Space* space) -{ - Block* blocks = (Block*)space->heap_start; - Block_Header* last_block = (Block_Header*)blocks; - unsigned int start_idx = space->first_block_idx; - for(unsigned int i=0; i < space->num_managed_blocks; i++){ - Block_Header* block = (Block_Header*)&(blocks[i]); - block_init(block); - block->block_idx = i + start_idx; - last_block->next = block; - last_block = block; - } - last_block->next = NULL; - space->blocks = blocks; - - return; -} +void space_init_blocks(Blocked_Space* space); +void space_desturct_blocks(Blocked_Space* space); -inline void space_desturct_blocks(Blocked_Space* space) -{ - Block* blocks = (Block*)space->heap_start; - unsigned int i=0; - for(; i < space->num_managed_blocks; i++){ - Block_Header* block = (Block_Header*)&(blocks[i]); - block_destruct(block); - } -} +void blocked_space_shrink(Blocked_Space* space, unsigned int changed_size); +void blocked_space_extend(Blocked_Space* space, unsigned int changed_size); -inline void blocked_space_shrink(Blocked_Space* space, unsigned int changed_size) -{ - unsigned int block_dec_count = changed_size >> GC_BLOCK_SHIFT_COUNT; - void* new_base = (void*)&(space->blocks[space->num_managed_blocks - block_dec_count]); - - void* decommit_base = (void*)round_down_to_size((POINTER_SIZE_INT)new_base, SPACE_ALLOC_UNIT); - - assert( ((Block_Header*)decommit_base)->block_idx >= space->free_block_idx); - - void* old_end = (void*)&space->blocks[space->num_managed_blocks]; - POINTER_SIZE_INT decommit_size = (POINTER_SIZE_INT)old_end - (POINTER_SIZE_INT)decommit_base; - assert(decommit_size && !(decommit_size%GC_BLOCK_SIZE_BYTES)); - - Boolean result = vm_decommit_mem(decommit_base, decommit_size); - assert(result == TRUE); - - space->committed_heap_size = (POINTER_SIZE_INT)decommit_base - (POINTER_SIZE_INT)space->heap_start; - space->num_managed_blocks = (unsigned int)(space->committed_heap_size >> GC_BLOCK_SHIFT_COUNT); - - Block_Header* new_last_block = (Block_Header*)&space->blocks[space->num_managed_blocks - 1]; - space->ceiling_block_idx = new_last_block->block_idx; - new_last_block->next = NULL; -} +void blocked_space_block_iterator_init(Blocked_Space *space); +void blocked_space_block_iterator_init_free(Blocked_Space *space); +Block_Header *blocked_space_block_iterator_get(Blocked_Space *space); +Block_Header *blocked_space_block_iterator_next(Blocked_Space *space); -inline void blocked_space_extend(Blocked_Space* space, unsigned int changed_size) -{ - unsigned int block_inc_count = changed_size >> GC_BLOCK_SHIFT_COUNT; - - void* old_base = (void*)&space->blocks[space->num_managed_blocks]; - void* commit_base = (void*)round_down_to_size((POINTER_SIZE_INT)old_base, SPACE_ALLOC_UNIT); - unsigned int block_diff_count = (unsigned int)(((POINTER_SIZE_INT)old_base - (POINTER_SIZE_INT)commit_base) >> GC_BLOCK_SHIFT_COUNT); - block_inc_count += block_diff_count; - - POINTER_SIZE_INT commit_size = block_inc_count << GC_BLOCK_SHIFT_COUNT; - void* result = vm_commit_mem(commit_base, commit_size); - assert(result == commit_base); - - void* new_end = (void*)((POINTER_SIZE_INT)commit_base + commit_size); - space->committed_heap_size = (POINTER_SIZE_INT)new_end - (POINTER_SIZE_INT)space->heap_start; - /*Fixme: For_Heap_Adjust, but need fix if static mapping.*/ - space->heap_end = new_end; - /* init the grown blocks */ - Block_Header* block = (Block_Header*)commit_base; - Block_Header* last_block = (Block_Header*)((Block*)block -1); - unsigned int start_idx = last_block->block_idx + 1; - unsigned int i; - for(i=0; block < new_end; i++){ - block_init(block); - block->block_idx = start_idx + i; - last_block->next = block; - last_block = block; - block = (Block_Header*)((Block*)block + 1); - } - last_block->next = NULL; - space->ceiling_block_idx = last_block->block_idx; - space->num_managed_blocks = (unsigned int)(space->committed_heap_size >> GC_BLOCK_SHIFT_COUNT); -} - -inline void blocked_space_block_iterator_init(Blocked_Space *space) -{ space->block_iterator = (Block_Header*)space->blocks; } - -inline void blocked_space_block_iterator_init_free(Blocked_Space *space) -{ space->block_iterator = (Block_Header*)&space->blocks[space->free_block_idx - space->first_block_idx]; } - -inline Block_Header *blocked_space_block_iterator_get(Blocked_Space *space) -{ return (Block_Header*)space->block_iterator; } - -inline Block_Header *blocked_space_block_iterator_next(Blocked_Space *space) -{ - Block_Header *cur_block = (Block_Header*)space->block_iterator; - - while(cur_block != NULL){ - Block_Header *next_block = cur_block->next; - - Block_Header *temp = (Block_Header*)atomic_casptr((volatile void **)&space->block_iterator, next_block, cur_block); - if(temp != cur_block){ - cur_block = (Block_Header*)space->block_iterator; - continue; - } - return cur_block; - } - /* run out space blocks */ - return NULL; -} - #endif //#ifndef _GC_SPACE_H_ Index: src/common/hashcode.cpp =================================================================== --- src/common/hashcode.cpp (revision 0) +++ src/common/hashcode.cpp (revision 0) @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hashcode.h" +#include "gc_metadata.h" + +/* <-- processing of hashcode in different GC algorithms */ +Obj_Info_Type slide_compact_process_hashcode(Partial_Reveal_Object* p_obj, void* dest_addr, + unsigned int* p_obj_size, Collector* collector, + Hashcode_Buf* old_buf, Hashcode_Buf* new_buf) +{ + Obj_Info_Type obj_info = get_obj_info(p_obj); + POINTER_SIZE_INT hashcode = 0; + + switch(obj_info & HASHCODE_MASK){ + case HASHCODE_SET_UNALLOCATED: + if((POINTER_SIZE_INT)dest_addr != (POINTER_SIZE_INT)p_obj){ + *p_obj_size += GC_OBJECT_ALIGNMENT; + obj_info = obj_info | HASHCODE_ATTACHED_BIT; + *(int*) &hashcode = hashcode_gen(p_obj); + POINTER_SIZE_INT obj_end_pos = (POINTER_SIZE_INT)dest_addr + vm_object_size(p_obj); + collector_hashcodeset_add_entry(collector, (Partial_Reveal_Object**)obj_end_pos); + collector_hashcodeset_add_entry(collector, (Partial_Reveal_Object**)hashcode); + } + break; + + case HASHCODE_SET_ATTACHED: + obj_sethash_in_vt(p_obj); + break; + + case HASHCODE_SET_BUFFERED: + *(int*) &hashcode = hashcode_buf_lookup(p_obj, old_buf); + if((POINTER_SIZE_INT)dest_addr != (POINTER_SIZE_INT)p_obj){ + *p_obj_size += GC_OBJECT_ALIGNMENT; + obj_info = obj_info & ~HASHCODE_BUFFERED_BIT; + obj_info = obj_info | HASHCODE_ATTACHED_BIT; + POINTER_SIZE_INT obj_end_pos = (POINTER_SIZE_INT)dest_addr + vm_object_size(p_obj); + collector_hashcodeset_add_entry(collector, (Partial_Reveal_Object**)obj_end_pos); + collector_hashcodeset_add_entry(collector, (Partial_Reveal_Object**)hashcode); + }else{ + hashcode_buf_add((Partial_Reveal_Object*)dest_addr, *(int*) &hashcode, new_buf); + } + break; + + case HASHCODE_UNSET: + break; + + default: + assert(0); + + } + return obj_info; +} + +void move_compact_process_hashcode(Partial_Reveal_Object* p_obj,Hashcode_Buf* old_buf, + Hashcode_Buf* new_buf) +{ + if(hashcode_is_set(p_obj) && !hashcode_is_attached(p_obj)){ + int hashcode; + if(hashcode_is_buffered(p_obj)){ + /*already buffered objects;*/ + hashcode = hashcode_buf_lookup(p_obj, old_buf); + hashcode_buf_add(p_obj, hashcode, new_buf); + }else{ + /*objects need buffering.*/ + hashcode = hashcode_gen(p_obj); + hashcode_buf_add(p_obj, hashcode, new_buf); + Obj_Info_Type oi = get_obj_info_raw(p_obj); + set_obj_info(p_obj, oi | HASHCODE_BUFFERED_BIT); + } + } +} + +/* processing of hashcode in different GC algorithms --> */ + + + Index: src/common/hashcode.h =================================================================== --- src/common/hashcode.h (revision 606795) +++ src/common/hashcode.h (working copy) @@ -287,103 +287,43 @@ return; } -void collector_hashcodeset_add_entry(Collector* collector, Partial_Reveal_Object** p_ref); +int obj_lookup_hashcode_in_buf(Partial_Reveal_Object *p_obj); -inline Obj_Info_Type slide_compact_process_hashcode(Partial_Reveal_Object* p_obj, void* dest_addr, - unsigned int* p_obj_size, Collector* collector, - Hashcode_Buf* old_buf, Hashcode_Buf* new_buf) +inline int hashcode_lookup(Partial_Reveal_Object* p_obj,Obj_Info_Type obj_info) { - Obj_Info_Type obj_info = get_obj_info(p_obj); - POINTER_SIZE_INT hashcode = 0; - - switch(obj_info & HASHCODE_MASK){ - case HASHCODE_SET_UNALLOCATED: - if((POINTER_SIZE_INT)dest_addr != (POINTER_SIZE_INT)p_obj){ - *p_obj_size += GC_OBJECT_ALIGNMENT; - obj_info = obj_info | HASHCODE_ATTACHED_BIT; - *(int*) &hashcode = hashcode_gen(p_obj); - POINTER_SIZE_INT obj_end_pos = (POINTER_SIZE_INT)dest_addr + vm_object_size(p_obj); - collector_hashcodeset_add_entry(collector, (Partial_Reveal_Object**)obj_end_pos); - collector_hashcodeset_add_entry(collector, (Partial_Reveal_Object**)hashcode); - } - break; - - case HASHCODE_SET_ATTACHED: - obj_sethash_in_vt(p_obj); - break; - - case HASHCODE_SET_BUFFERED: - *(int*) &hashcode = hashcode_buf_lookup(p_obj, old_buf); - if((POINTER_SIZE_INT)dest_addr != (POINTER_SIZE_INT)p_obj){ - *p_obj_size += GC_OBJECT_ALIGNMENT; - obj_info = obj_info & ~HASHCODE_BUFFERED_BIT; - obj_info = obj_info | HASHCODE_ATTACHED_BIT; - POINTER_SIZE_INT obj_end_pos = (POINTER_SIZE_INT)dest_addr + vm_object_size(p_obj); - collector_hashcodeset_add_entry(collector, (Partial_Reveal_Object**)obj_end_pos); - collector_hashcodeset_add_entry(collector, (Partial_Reveal_Object**)hashcode); - }else{ - hashcode_buf_add((Partial_Reveal_Object*)dest_addr, *(int*) &hashcode, new_buf); - } - break; - - case HASHCODE_UNSET: - break; - - default: - assert(0); - + int hash; + if(hashcode_is_attached(p_obj)){ + int offset = vm_object_size(p_obj); + unsigned char* pos = (unsigned char *)p_obj; + hash = *(int*) (pos + offset); + }else if(hashcode_is_buffered(p_obj)){ + hash = obj_lookup_hashcode_in_buf(p_obj); } - return obj_info; + return hash; } -inline void move_compact_process_hashcode(Partial_Reveal_Object* p_obj,Hashcode_Buf* old_buf, - Hashcode_Buf* new_buf) +inline void precompute_hashcode_extend_size(Partial_Reveal_Object* p_obj, void* targ_addr, + unsigned int * obj_size_precompute) { - if(hashcode_is_set(p_obj) && !hashcode_is_attached(p_obj)){ - int hashcode; - if(hashcode_is_buffered(p_obj)){ - /*already buffered objects;*/ - hashcode = hashcode_buf_lookup(p_obj, old_buf); - hashcode_buf_add(p_obj, hashcode, new_buf); - }else{ - /*objects need buffering.*/ - hashcode = hashcode_gen(p_obj); - hashcode_buf_add(p_obj, hashcode, new_buf); - Obj_Info_Type oi = get_obj_info_raw(p_obj); - set_obj_info(p_obj, oi | HASHCODE_BUFFERED_BIT); - } + if(hashcode_is_set(p_obj) && !hashcode_is_attached(p_obj)){ + if((POINTER_SIZE_INT)targ_addr != (POINTER_SIZE_INT)p_obj) + *obj_size_precompute += GC_OBJECT_ALIGNMENT; } } -inline Obj_Info_Type trace_forward_process_hashcode(Partial_Reveal_Object* p_obj, Partial_Reveal_Object* p_old_obj, +inline Obj_Info_Type forward_obj_attach_hashcode(Partial_Reveal_Object* p_targ_obj, Partial_Reveal_Object* p_obj, Obj_Info_Type oi, unsigned int p_obj_size) { oi |= HASHCODE_ATTACHED_BIT; - *(int *)(((char*)p_obj) + p_obj_size - GC_OBJECT_ALIGNMENT) = hashcode_gen(p_old_obj); - assert(vm_object_size(p_obj) != 0); + *(int *)(((char*)p_targ_obj) + p_obj_size) = hashcode_gen(p_obj); return oi; } -inline void precompute_hashcode_extend_size(Partial_Reveal_Object* p_obj, void* dest_addr, - unsigned int * obj_size_precompute) -{ - if(hashcode_is_set(p_obj) && !hashcode_is_attached(p_obj)){ - if((POINTER_SIZE_INT)dest_addr != (POINTER_SIZE_INT)p_obj) - *obj_size_precompute += GC_OBJECT_ALIGNMENT; - } -} +Obj_Info_Type slide_compact_process_hashcode(Partial_Reveal_Object* p_obj, void* dest_addr, + unsigned int* p_obj_size, Collector* collector, + Hashcode_Buf* old_buf, Hashcode_Buf* new_buf); -inline int obj_lookup_hashcode_in_buf(Partial_Reveal_Object *p_obj); -inline int hashcode_lookup(Partial_Reveal_Object* p_obj,Obj_Info_Type obj_info) -{ - int hash; - if(hashcode_is_attached(p_obj)){ - int offset = vm_object_size(p_obj); - unsigned char* pos = (unsigned char *)p_obj; - hash = *(int*) (pos + offset); - }else if(hashcode_is_buffered(p_obj)){ - hash = obj_lookup_hashcode_in_buf(p_obj); - } - return hash; -} +void move_compact_process_hashcode(Partial_Reveal_Object* p_obj,Hashcode_Buf* old_buf, + Hashcode_Buf* new_buf); + #endif //_HASHCODE_H_ Index: src/common/mark_scan_pool.cpp =================================================================== --- src/common/mark_scan_pool.cpp (revision 606795) +++ src/common/mark_scan_pool.cpp (working copy) @@ -47,8 +47,8 @@ vm_notify_obj_alive( (void *)p_obj); assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); - Partial_Reveal_VTable *vtable = uncompress_vt(obj_get_vt(p_obj)); - if(VTABLE_TRACING) + Partial_Reveal_VTable *vtable = decode_vt(obj_get_vt(p_obj)); + if(TRACE_JLC_VIA_VTABLE) if(vtable->vtmark == VT_UNMARKED) { vtable->vtmark = VT_MARKED; if(obj_mark_in_vt(vtable->jlC)) Index: src/common/object_status.h =================================================================== --- src/common/object_status.h (revision 606795) +++ src/common/object_status.h (working copy) @@ -20,7 +20,7 @@ #include "../gen/gen.h" #include "../mark_sweep/gc_ms.h" -#include "../mark_sweep/sspace_mark_sweep.h" +#include "../mark_sweep/wspace_mark_sweep.h" inline Boolean obj_is_dead_in_gen_minor_gc(Partial_Reveal_Object *p_obj) @@ -59,6 +59,12 @@ } #endif +#ifdef USE_UNIQUE_MOVE_COMPACT_GC +inline Boolean obj_is_dead_in_move_compact_no_los_gc(Partial_Reveal_Object *p_obj) +{ + return !obj_is_marked_in_vt(p_obj); +} +#endif inline Boolean gc_obj_is_dead(GC *gc, Partial_Reveal_Object *p_obj) { assert(p_obj); @@ -67,6 +73,10 @@ return obj_is_dead_in_mark_sweep_gc(p_obj); #endif +#ifdef USE_UNIQUE_MOVE_COMPACT_GC + return obj_is_dead_in_move_compact_no_los_gc(p_obj); +#endif + if(gc_match_kind(gc, MINOR_COLLECTION)){ if(gc_is_gen_mode()) return obj_is_dead_in_gen_minor_gc(p_obj); @@ -89,10 +99,14 @@ /* assert(!gc_obj_is_dead(gc, p_obj)); commented out for weakroot */ #ifdef USE_MARK_SWEEP_GC - Sspace *sspace = gc_ms_get_sspace((GC_MS*)gc); - return sspace->move_object; + Wspace *wspace = gc_ms_get_wspace((GC_MS*)gc); + return wspace->move_object; #endif +#ifdef USE_UNIQUE_MOVE_COMPACT_GC + Cspace *cspace = gc_mc_get_cspace((GC_MC*)gc); + return cspace->move_object; +#endif if(gc_is_gen_mode() && gc_match_kind(gc, MINOR_COLLECTION)) return fspace_obj_to_be_forwarded(p_obj); Index: src/common/space_tuner.cpp =================================================================== --- src/common/space_tuner.cpp (revision 606795) +++ src/common/space_tuner.cpp (working copy) @@ -198,7 +198,7 @@ Collector *collector = gc->collectors[i]; non_los_live_obj_size += collector->non_los_live_obj_size; los_live_obj_size += collector->los_live_obj_size; - for(unsigned int j = NORMAL_SIZE_SEGMENT_NUM; j--;){ + for(unsigned int j = 0; j < NORMAL_SIZE_SEGMENT_NUM; j++) { segment_live_size[j] += collector->segment_live_size[j]; } memset(collector->segment_live_size, 0, sizeof(POINTER_SIZE_INT) * NORMAL_SIZE_SEGMENT_NUM); @@ -206,14 +206,12 @@ //POINTER_SIZE_INT additional_non_los_size = ((collector_num * 2) << GC_BLOCK_SHIFT_COUNT) + (non_los_live_obj_size >> GC_BLOCK_SHIFT_COUNT) * (GC_OBJ_SIZE_THRESHOLD/4); double additional_non_los_size = 0; - for(unsigned int i = NORMAL_SIZE_SEGMENT_NUM; i--;){ + for(unsigned int i = 0; i < NORMAL_SIZE_SEGMENT_NUM; i++) { additional_non_los_size += (double)segment_live_size[i] * SEGMENT_INDEX_TO_SIZE(i) / non_los_live_obj_size; } additional_non_los_size *= 1.2; // in case of some cases worse than average one POINTER_SIZE_INT non_los_live_block = non_los_live_obj_size / (GC_BLOCK_BODY_SIZE_BYTES-(POINTER_SIZE_INT)additional_non_los_size); - additional_non_los_size *= non_los_live_block + 1; - additional_non_los_size += collector_num << (GC_BLOCK_SHIFT_COUNT + 1); - non_los_live_obj_size = round_up_to_size(non_los_live_obj_size + (POINTER_SIZE_INT)additional_non_los_size, GC_BLOCK_SIZE_BYTES); + non_los_live_obj_size = (non_los_live_block << GC_BLOCK_SHIFT_COUNT); if(non_los_live_obj_size > non_los_size) non_los_live_obj_size = non_los_size; @@ -539,3 +537,4 @@ return; } + Index: src/common/weak_roots.cpp =================================================================== --- src/common/weak_roots.cpp (revision 606795) +++ src/common/weak_roots.cpp (working copy) @@ -70,7 +70,9 @@ if(IS_MOVE_COMPACT){ assert(space_of_addr(gc, p_obj)->move_object); - *p_ref = ref_to_obj_ptr(obj_get_fw_in_table(p_obj)); + *p_ref = obj_get_fw_in_table(p_obj); + } else if(gc_match_kind(gc, MC_COLLECTION)){ + *p_ref = obj_get_fw_in_table(p_obj); } else if(gc_match_kind(gc, MS_COMPACT_COLLECTION) || gc_get_mos((GC_Gen*)gc)->collect_algorithm==MAJOR_MARK_SWEEP){ if(obj_is_fw_in_oi(p_obj)){ p_obj = obj_get_fw_in_oi(p_obj); Index: src/finalizer_weakref/finalizer_weakref.cpp =================================================================== --- src/finalizer_weakref/finalizer_weakref.cpp (revision 606795) +++ src/finalizer_weakref/finalizer_weakref.cpp (working copy) @@ -31,6 +31,7 @@ #include "../common/space_tuner.h" #include "../common/compressed_ref.h" #include "../common/object_status.h" +#include "../common/gc_concurrent.h" Boolean IGNORE_FINREF = FALSE; Boolean DURING_RESURRECTION = FALSE; @@ -115,6 +116,7 @@ extern void trace_obj_in_ms_fallback_marking(Collector *collector, void *p_ref); extern void trace_obj_in_space_tune_marking(Collector *collector, void *p_obj); extern void trace_obj_in_ms_marking(Collector *collector, void *p_obj); +extern void trace_obj_in_ms_concurrent_mark(Collector *collector, void *p_obj); typedef void (* Trace_Object_Func)(Collector *collector, void *p_ref_or_obj); @@ -163,7 +165,10 @@ } else { assert(gc_match_kind(gc, MARK_SWEEP_GC)); p_ref_or_obj = p_obj; - trace_object = trace_obj_in_ms_marking; + if(!gc_mark_is_concurrent()) + trace_object = trace_obj_in_ms_marking; + else + trace_object = trace_obj_in_ms_concurrent_mark; } collector->trace_stack = free_task_pool_get_entry(metadata); @@ -662,16 +667,18 @@ if(address_belongs_to_gc_heap(p_ref, gc) && (p_ref >= los_boundary)){ unsigned int offset = get_gc_referent_offset(); Partial_Reveal_Object *p_old_ref = (Partial_Reveal_Object*)((POINTER_SIZE_INT)p_ref - offset); - Partial_Reveal_Object *p_new_ref = ref_to_obj_ptr(obj_get_fw_in_table(p_old_ref)); + Partial_Reveal_Object *p_new_ref = obj_get_fw_in_table(p_old_ref); p_ref = (REF*)((POINTER_SIZE_INT)p_new_ref + offset); } Partial_Reveal_Object *p_obj = read_slot(p_ref); assert(space_of_addr(gc, p_obj)->move_object); if(p_obj < los_boundary) - write_slot(p_ref, obj_get_fw_in_oi(p_obj)); + p_obj = obj_get_fw_in_oi(p_obj); else - *p_ref = obj_get_fw_in_table(p_obj); + p_obj = obj_get_fw_in_table(p_obj); + + write_slot(p_ref, p_obj); } /* In two cases mark-sweep needs fixing repointed refs: @@ -806,3 +813,4 @@ + Index: src/finalizer_weakref/finalizer_weakref.h =================================================================== --- src/finalizer_weakref/finalizer_weakref.h (revision 606795) +++ src/finalizer_weakref/finalizer_weakref.h (working copy) @@ -64,6 +64,7 @@ REF *p_referent_field = obj_get_referent_field(p_obj); REF p_referent = *p_referent_field; if (!p_referent) return; + if(DURING_RESURRECTION){ write_slot(p_referent_field, NULL); return; @@ -87,6 +88,19 @@ } } +inline void scan_weak_reference_direct(Collector *collector, Partial_Reveal_Object *p_obj, Scan_Slot_Func scan_slot) +{ + WeakReferenceType type = special_reference_type(p_obj); + if(type == NOT_REFERENCE) + return; + REF *p_referent_field = obj_get_referent_field(p_obj); + REF p_referent = *p_referent_field; + if (!p_referent) return; + + scan_slot(collector, p_referent_field); +} + + extern void gc_update_weakref_ignore_finref(GC *gc); extern void collector_identify_finref(Collector *collector); extern void fallback_finref_cleanup(GC *gc); Index: src/finalizer_weakref/finalizer_weakref_metadata.cpp =================================================================== --- src/finalizer_weakref/finalizer_weakref_metadata.cpp (revision 606795) +++ src/finalizer_weakref/finalizer_weakref_metadata.cpp (working copy) @@ -188,7 +188,7 @@ Mutator *mutator = gc->mutator_list; while(mutator){ pool_put_entry(obj_with_fin_pool, mutator->obj_with_fin); - mutator->obj_with_fin = NULL; + mutator->obj_with_fin = finref_get_free_block(gc); mutator = mutator->next; } } @@ -216,15 +216,27 @@ for(unsigned int i = 0; i < num_active_collectors; i++) { Collector *collector = gc->collectors[i]; - pool_put_entry(metadata->softref_pool, collector->softref_set); - pool_put_entry(metadata->weakref_pool, collector->weakref_set); - pool_put_entry(metadata->phanref_pool, collector->phanref_set); + if(!vector_block_is_empty(collector->softref_set)) + pool_put_entry(metadata->softref_pool, collector->softref_set); + else + pool_put_entry(metadata->free_pool, collector->softref_set); + + if(!vector_block_is_empty(collector->weakref_set)) + pool_put_entry(metadata->weakref_pool, collector->weakref_set); + else + pool_put_entry(metadata->free_pool, collector->weakref_set); + + if(!vector_block_is_empty(collector->phanref_set)) + pool_put_entry(metadata->phanref_pool, collector->phanref_set); + else + pool_put_entry(metadata->free_pool, collector->phanref_set); + collector->softref_set = NULL; collector->weakref_set= NULL; collector->phanref_set= NULL; } - if(gc_mark_is_concurrent()){ + if(gc_mark_is_concurrent() && !gc_sweep_is_concurrent()){ unsigned int num_active_markers = gc->num_active_markers; for(unsigned int i = 0; i < num_active_markers; i++) { @@ -274,7 +286,7 @@ metadata->obj_with_fin_pool = finalizable_obj_pool; metadata->finalizable_obj_pool = obj_with_fin_pool; - gc_reset_obj_with_fin(gc); + //gc_reset_obj_with_fin(gc); } @@ -429,3 +441,4 @@ return TRUE; } + Index: src/gen/gc_for_barrier.cpp =================================================================== --- src/gen/gc_for_barrier.cpp (revision 606795) +++ src/gen/gc_for_barrier.cpp (working copy) @@ -22,16 +22,16 @@ #include "../gen/gen.h" #include "../thread/mutator.h" #include "gc_for_barrier.h" -#include "../mark_sweep/sspace_mark_sweep.h" +#include "../mark_sweep/wspace_mark_sweep.h" #include "../common/gc_concurrent.h" /* All the write barrier interfaces need cleanup */ -Boolean gen_mode; +volatile unsigned int write_barrier_function; /* The implementations are only temporary */ -static void gc_slot_write_barrier(Managed_Object_Handle *p_slot, +static void write_barrier_rem_source_slot(Managed_Object_Handle *p_slot, Managed_Object_Handle p_target) { if(p_target >= nos_boundary && p_slot < nos_boundary){ @@ -44,6 +44,19 @@ return; } +static void write_barrier_rem_source_obj(Managed_Object_Handle p_obj_holding_ref) +{ + if(obj_need_remember((Partial_Reveal_Object*)p_obj_holding_ref)){ + Mutator *mutator = (Mutator *)gc_get_tls(); + + //FIXME: Release lock. + lock(mutator->dirty_set_lock); + obj_dirty_in_table((Partial_Reveal_Object *) p_obj_holding_ref); + mutator_dirtyset_add_entry(mutator, (Partial_Reveal_Object*)p_obj_holding_ref); + unlock(mutator->dirty_set_lock); + } +} + static void gc_object_write_barrier(Managed_Object_Handle p_object) { @@ -82,27 +95,13 @@ return; } -void gc_heap_wrote_object (Managed_Object_Handle p_obj_written) -{ - /*concurrent mark: since object clone and array copy do not modify object slot, - we treat it as an new object. It has already been marked when dest object was created.*/ - if( !gc_is_gen_mode() ) return; - if( object_has_ref_field((Partial_Reveal_Object*)p_obj_written)){ - /* for array copy and object clone */ - gc_object_write_barrier(p_obj_written); - } -} -/* The following routines were supposed to be the only way to alter any value in gc heap. */ -void gc_heap_write_ref (Managed_Object_Handle p_obj_holding_ref, unsigned offset, Managed_Object_Handle p_target) -{ assert(0); } - /*This function is for concurrent mark.*/ -static void gc_dirty_object_write_barrier(Managed_Object_Handle p_obj_holding_ref) +static void write_barrier_rem_obj_snapshot(Managed_Object_Handle p_obj_holding_ref) { Mutator *mutator = (Mutator *)gc_get_tls(); REF* p_obj_slot; - if(obj_need_take_snaptshot((Partial_Reveal_Object*)p_obj_holding_ref)){ + if(obj_need_take_snapshot((Partial_Reveal_Object*)p_obj_holding_ref)){ if (object_is_array((Partial_Reveal_Object*)p_obj_holding_ref)) { Partial_Reveal_Object* array = (Partial_Reveal_Object*)p_obj_holding_ref; assert(!obj_is_primitive_array(array)); @@ -114,7 +113,7 @@ p_obj_slot = (REF*)vector_get_element_address_ref((Vector_Handle) array, i); obj_to_snapshot = (Partial_Reveal_Object*)read_slot(p_obj_slot); if (obj_to_snapshot != NULL) - mutator_snapshotset_add_entry(mutator, obj_to_snapshot); + mutator_dirtyset_add_entry(mutator, obj_to_snapshot); } }else{ /* scan non-array object */ @@ -123,12 +122,12 @@ int *ref_iterator = object_ref_iterator_init(p_obj); Partial_Reveal_Object* obj_to_snapshot; - + for(unsigned int i=0; idirty_set_lock); + + obj_dirty_in_table((Partial_Reveal_Object *) p_obj_written); + mutator_dirtyset_add_entry(mutator, (Partial_Reveal_Object*)p_obj_written); + + unlock(mutator->dirty_set_lock); + } + + if( !gc_is_gen_mode() ) return; + if( object_has_ref_field((Partial_Reveal_Object*)p_obj_written)){ + /* for array copy and object clone */ + gc_object_write_barrier(p_obj_written); + } +} + /* FIXME:: this is not the right interface for write barrier */ void gc_heap_slot_write_ref (Managed_Object_Handle p_obj_holding_ref,Managed_Object_Handle *p_slot, Managed_Object_Handle p_target) -{ - if(!gc_is_concurrent_mark_phase()){ - *p_slot = p_target; - - if( !gc_is_gen_mode() ) return; - gc_slot_write_barrier(p_slot, p_target); - }else{ - gc_dirty_object_write_barrier(p_obj_holding_ref); - *p_slot = p_target; +{ + switch(write_barrier_function){ + case WRITE_BARRIER_REM_NIL: + *p_slot = p_target; + break; + case WRITE_BARRIER_REM_SOURCE_REF: + *p_slot = p_target; + write_barrier_rem_source_slot(p_slot, p_target); + break; + case WRITE_BARRIER_REM_SOURCE_OBJ: + *p_slot = p_target; + write_barrier_rem_source_obj(p_obj_holding_ref); + break; + case WRITE_BARRIER_REM_OBJ_SNAPSHOT: + write_barrier_rem_obj_snapshot(p_obj_holding_ref); + *p_slot = p_target; + break; + case WRITE_BARRIER_REM_OLD_VAR: + write_barrier_rem_slot_oldvar(p_slot); + *p_slot = p_target; + break; + default: + assert(0); + return; } } /* this is used for global object update, e.g., strings. */ void gc_heap_write_global_slot(Managed_Object_Handle *p_slot,Managed_Object_Handle p_target) { - /*concurrent mark: global object is enumerated, so the old object has been already marked.*/ + /*Concurrent Mark & Generational Mode: + Global objects are roots. After root set enumeration, this objects will be touched by GC. No barrier here. + */ *p_slot = p_target; - - /* Since globals are roots, no barrier here */ } Index: src/gen/gc_for_barrier.h =================================================================== --- src/gen/gc_for_barrier.h (revision 606795) +++ src/gen/gc_for_barrier.h (working copy) @@ -24,6 +24,22 @@ #include "../jni/java_support.h" +extern volatile unsigned int write_barrier_function; + +enum Write_Barrier_Function{ + WRITE_BARRIER_REM_NIL = 0x00, + WRITE_BARRIER_REM_SOURCE_OBJ = 0x01, + WRITE_BARRIER_REM_SOURCE_REF = 0x02, + WRITE_BARRIER_REM_OLD_VAR = 0x03, + WRITE_BARRIER_REM_NEW_VAR = 0x04, + WRITE_BARRIER_REM_OBJ_SNAPSHOT = 0x05 +}; + +inline void gc_set_barrier_function(unsigned int wb_function) +{ + write_barrier_function = wb_function; +} + extern Boolean gen_mode; inline Boolean gc_is_gen_mode() @@ -32,20 +48,24 @@ inline void gc_enable_gen_mode() { gen_mode = TRUE; + gc_set_barrier_function(WRITE_BARRIER_REM_SOURCE_REF); HelperClass_set_GenMode(TRUE); } inline void gc_disable_gen_mode() { gen_mode = FALSE; + gc_set_barrier_function(WRITE_BARRIER_REM_NIL); HelperClass_set_GenMode(FALSE); } inline void gc_set_gen_mode(Boolean status) { gen_mode = status; + if(gen_mode) + gc_set_barrier_function(WRITE_BARRIER_REM_SOURCE_REF); HelperClass_set_GenMode(status); } - #endif /* _GC_FOR_BARRIER_H_ */ + Index: src/gen/gen.cpp =================================================================== --- src/gen/gen.cpp (revision 606795) +++ src/gen/gen.cpp (working copy) @@ -42,13 +42,16 @@ POINTER_SIZE_INT MIN_NOS_SIZE = 0; POINTER_SIZE_INT MAX_NOS_SIZE = 0; -static unsigned int MINOR_ALGO = 0; +/* should clean up */ +unsigned int MINOR_ALGO = 0; static unsigned int MAJOR_ALGO = 0; Boolean GEN_NONGEN_SWITCH = FALSE; Boolean JVMTI_HEAP_ITERATION = true; +Boolean gen_mode; + #ifndef STATIC_NOS_MAPPING void* nos_boundary; #endif @@ -288,16 +291,23 @@ void gc_set_mos(GC_Gen *gc, Space *mos){ gc->mos = mos; } void gc_set_los(GC_Gen *gc, Space *los){ gc->los = los; } +Space_Alloc_Func nos_alloc; Space_Alloc_Func mos_alloc; -//void* mos_alloc(unsigned size, Allocator *allocator){return mspace_alloc(size, allocator);} -void* nos_alloc(unsigned size, Allocator *allocator){return fspace_alloc(size, allocator);} Space_Alloc_Func los_alloc; -//void* los_alloc(unsigned size, Allocator *allocator){return lspace_alloc(size, allocator);} + void* los_try_alloc(POINTER_SIZE_INT size, GC* gc){ return lspace_try_alloc((Lspace*)((GC_Gen*)gc)->los, size); } void gc_nos_initialize(GC_Gen *gc, void *start, POINTER_SIZE_INT nos_size, POINTER_SIZE_INT commit_size) { - Space *nos = (Space*)fspace_initialize((GC*)gc, start, nos_size, commit_size); + Space *nos; + if(MINOR_ALGO == MINOR_NONGEN_SEMISPACE_POOL){ + nos = (Space*)sspace_initialize((GC*)gc, start, nos_size, commit_size); + nos_alloc = sspace_alloc; + }else{ + nos = (Space*)fspace_initialize((GC*)gc, start, nos_size, commit_size); + nos_alloc = fspace_alloc; + } + gc_set_nos(gc, nos); nos->collect_algorithm = MINOR_ALGO; } @@ -309,8 +319,8 @@ { Space *mos; if(MAJOR_ALGO == MAJOR_MARK_SWEEP){ - mos = (Space*)sspace_initialize((GC*)gc, start, mos_size, commit_size); - mos_alloc = sspace_alloc; + mos = (Space*)wspace_initialize((GC*)gc, start, mos_size, commit_size); + mos_alloc = wspace_alloc; } else { mos = (Space*)mspace_initialize((GC*)gc, start, mos_size, commit_size); mos_alloc = mspace_alloc; @@ -322,7 +332,7 @@ void gc_mos_destruct(GC_Gen *gc) { if(MAJOR_ALGO == MAJOR_MARK_SWEEP) - sspace_destruct((Sspace*)gc->mos); + wspace_destruct((Wspace*)gc->mos); else mspace_destruct((Mspace*)gc->mos); } @@ -333,7 +343,7 @@ if(MAJOR_ALGO == MAJOR_MARK_SWEEP){ assert(los_size == 0); los = NULL; - los_alloc = sspace_alloc; + los_alloc = wspace_alloc; } else { los = (Space*)lspace_initialize((GC*)gc, start, los_size); los_alloc = lspace_alloc; @@ -350,7 +360,7 @@ Boolean FORCE_FULL_COMPACT = FALSE; Boolean IGNORE_VTABLE_TRACING = FALSE; -Boolean VTABLE_TRACING = FALSE; +Boolean TRACE_JLC_VIA_VTABLE = FALSE; unsigned int gc_next_collection_kind(GC_Gen* gc) { @@ -365,19 +375,20 @@ { /* this is for debugging. */ gc->last_collect_kind = gc->collect_kind; - +#if defined(USE_MARK_SWEEP_GC) + gc->collect_kind = MS_COLLECTION; +#elif defined(USE_UNIQUE_MOVE_COMPACT_GC) + gc->collect_kind = MC_COLLECTION; +#else if(gc->force_major_collect || cause== GC_CAUSE_LOS_IS_FULL || FORCE_FULL_COMPACT) gc->collect_kind = NORMAL_MAJOR_COLLECTION; else gc->collect_kind = MINOR_COLLECTION; if(IGNORE_VTABLE_TRACING || (gc->collect_kind == MINOR_COLLECTION)) - VTABLE_TRACING = FALSE; + TRACE_JLC_VIA_VTABLE = FALSE; else - VTABLE_TRACING = TRUE; - -#ifdef USE_MARK_SWEEP_GC - gc->collect_kind = MS_COLLECTION; + TRACE_JLC_VIA_VTABLE = TRUE; #endif return; } @@ -399,7 +410,11 @@ MINOR_ALGO = MINOR_GEN_FORWARD_POOL; gc_enable_gen_mode(); - }else{ + }else if(!strcmp(minor_algo, "MINOR_NONGEN_SEMISPACE_POOL")){ + MINOR_ALGO = MINOR_NONGEN_SEMISPACE_POOL; + gc_disable_gen_mode(); + + }else { WARN2("gc.base","\nWarning: GC algorithm setting incorrect. Will use default value.\n"); } @@ -430,6 +445,18 @@ } +static Boolean nos_alloc_block(Space* space, Allocator* allocator) +{ + Boolean result; + if(MINOR_ALGO == MINOR_NONGEN_SEMISPACE_POOL) + result = sspace_alloc_block((Sspace*)space, allocator); + else + result = fspace_alloc_block((Fspace*)space, allocator); + + return result; +} + +/* assign a free area to the mutator who triggers the collection */ void gc_gen_assign_free_area_to_mutators(GC_Gen* gc) { if(gc->cause == GC_CAUSE_LOS_IS_FULL){ @@ -438,14 +465,15 @@ los->failure_size = 0; }else{ - Blocked_Space* nos = (Blocked_Space*)gc->nos; - if(nos->num_managed_blocks == 0) return; + /* it is possible that NOS has no free block, + because MOS takes all space after fallback or LOS extension. + Allocator should be cleared. */ + Allocator *allocator = (Allocator *)gc_get_tls(); + Boolean ok = nos_alloc_block(gc->nos, allocator); + /* we don't care about the return value. If no block available, that means, + first allocation after mutator resumption will probably trigger OOME. */ + } - Mutator *mutator = (Mutator *)gc_get_tls(); - allocator_init_free_block((Allocator*)mutator, (Block_Header*)nos->blocks); - nos->free_block_idx++; - } - return; } @@ -554,12 +582,17 @@ } static inline void nos_collection(Space *nos) -{ fspace_collection((Fspace*)nos); } +{ + if(MINOR_ALGO == MINOR_NONGEN_SEMISPACE_POOL) + sspace_collection((Sspace*)nos); + else + fspace_collection((Fspace*)nos); +} static inline void mos_collection(Space *mos) { if(MAJOR_ALGO == MAJOR_MARK_SWEEP) - sspace_collection((Sspace*)mos); + wspace_collection((Wspace*)mos); else mspace_collection((Mspace*)mos); } @@ -623,18 +656,27 @@ } } } - + static void nos_reset_after_collection(Space *nos) { - fspace_reset_after_collection((Fspace*)nos); + if(MINOR_ALGO == MINOR_NONGEN_SEMISPACE_POOL) + sspace_reset_after_collection((Sspace*)nos); + else + fspace_reset_after_collection((Fspace*)nos); } +static void nos_prepare_for_collection(Space *nos) +{ + if(MINOR_ALGO == MINOR_NONGEN_SEMISPACE_POOL) + sspace_prepare_for_collection((Sspace*)nos); +} + static void mos_reset_after_collection(Space *mos) { if(MAJOR_ALGO != MAJOR_MARK_SWEEP) mspace_reset_after_collection((Mspace*)mos); else - sspace_reset_after_collection((Sspace*)mos); + wspace_reset_after_collection((Wspace*)mos); } Boolean IS_FALLBACK_COMPACTION = FALSE; /* only for debugging, don't use it. */ @@ -662,7 +704,9 @@ gc_gen_stats_reset_before_collection(gc); gc_gen_collector_stats_reset(gc); #endif - + + nos_prepare_for_collection(nos); + if(gc_match_kind((GC*)gc, MINOR_COLLECTION)){ INFO2("gc.process", "GC: start minor collection ...\n"); @@ -747,6 +791,7 @@ if(MAJOR_ALGO != MAJOR_MARK_SWEEP) los->move_object = TRUE; + mos_collection(mos); /* collect both mos and nos */ los_collection(los); if(MAJOR_ALGO != MAJOR_MARK_SWEEP) @@ -778,7 +823,7 @@ assert(MAJOR_ALGO == MAJOR_MARK_SWEEP || !los->move_object); - if(MAJOR_ALGO != MAJOR_MARK_SWEEP){ + if(MAJOR_ALGO != MAJOR_MARK_SWEEP && MINOR_ALGO != MINOR_NONGEN_SEMISPACE_POOL){ gc_gen_adjust_heap_size(gc); int64 pause_time = time_now() - gc_start_time; @@ -863,14 +908,6 @@ } } -void gc_gen_hook_for_collector_init(Collector *collector) -{ - if(MAJOR_ALGO == MAJOR_MARK_SWEEP){ - allocator_init_local_chunks((Allocator*)collector); - collector_init_free_chunk_list(collector); - } -} - void gc_gen_collection_verbose_info(GC_Gen *gc, int64 pause_time, int64 mutator_time) { @@ -962,4 +999,40 @@ #endif } +/* init collector alloc_space */ +void gc_gen_init_collector_alloc(GC_Gen* gc, Collector* collector) +{ + if(MAJOR_ALGO == MAJOR_MARK_SWEEP){ + allocator_init_local_chunks((Allocator*)collector); + gc_init_collector_free_chunk_list(collector); + } + Allocator* allocator = (Allocator*)collector; + + if( MINOR_ALGO == MINOR_NONGEN_SEMISPACE_POOL || MINOR_ALGO == MINOR_GEN_SEMISPACE_POOL){ + allocator->alloc_space = gc->nos; + /* init backup allocator */ + unsigned int size = sizeof(Allocator); + allocator = (Allocator*)STD_MALLOC(size); //assign its alloc_space below. + memset(allocator, 0, size); + collector->backup_allocator = allocator; + } + + allocator->alloc_space = gc->mos; +} + +void gc_gen_reset_collector_alloc(GC_Gen* gc, Collector* collector) +{ + alloc_context_reset((Allocator*)collector); + if( MINOR_ALGO == MINOR_NONGEN_SEMISPACE_POOL || MINOR_ALGO == MINOR_GEN_SEMISPACE_POOL){ + alloc_context_reset(collector->backup_allocator); + } +} + +void gc_gen_destruct_collector_alloc(GC_Gen* gc, Collector* collector) +{ + if( MINOR_ALGO == MINOR_NONGEN_SEMISPACE_POOL || MINOR_ALGO == MINOR_GEN_SEMISPACE_POOL){ + STD_FREE(collector->backup_allocator); + } +} + Index: src/gen/gen.h =================================================================== --- src/gen/gen.h (revision 606795) +++ src/gen/gen.h (working copy) @@ -25,10 +25,15 @@ #include "../common/gc_common.h" #include "../thread/gc_thread.h" + #include "../trace_forward/fspace.h" +#include "../semi_space/sspace.h" + #include "../mark_compact/mspace.h" #include "../los/lspace.h" -#include "../mark_sweep/sspace.h" + +#include "../mark_sweep/wspace.h" + #include "../finalizer_weakref/finalizer_weakref_metadata.h" #ifdef GC_GEN_STATS @@ -105,6 +110,7 @@ SpinLock concurrent_mark_lock; SpinLock enumerate_rootset_lock; + SpinLock concurrent_sweep_lock; /* system info */ @@ -165,7 +171,7 @@ } extern Space_Alloc_Func mos_alloc; -void* nos_alloc(unsigned size, Allocator *allocator); +extern Space_Alloc_Func nos_alloc; extern Space_Alloc_Func los_alloc; void* los_try_alloc(POINTER_SIZE_INT size, GC* gc); @@ -186,6 +192,9 @@ void gc_gen_reclaim_heap(GC_Gen* gc, int64 gc_start_time); void gc_gen_assign_free_area_to_mutators(GC_Gen* gc); +void gc_gen_init_collector_alloc(GC_Gen* gc, Collector* collector); +void gc_gen_reset_collector_alloc(GC_Gen* gc, Collector* collector); +void gc_gen_destruct_collector_alloc(GC_Gen* gc, Collector* collector); void gc_gen_adjust_heap_size(GC_Gen* gc, int64 pause_time); @@ -204,3 +213,4 @@ + Index: src/gen/gen_adapt.cpp =================================================================== --- src/gen/gen_adapt.cpp (revision 606795) +++ src/gen/gen_adapt.cpp (working copy) @@ -376,15 +376,15 @@ }else if ( new_nos_size >= curr_nos_size ){ INFO2("gc.process", "GC: gc_gen space adjustment after GC["<num_collections<<"] ..."); POINTER_SIZE_INT adapt_size = new_nos_size - curr_nos_size; - INFO2("gc.space", "GC: Space Adapt: nos ---> mos (" + INFO2("gc.space", "GC: Space Adapt: mos ---> nos (" <num_collections<<"] ..."); POINTER_SIZE_INT adapt_size = curr_nos_size - new_nos_size; - INFO2("gc.space", "GC: Space Adapt: mos ---> nos (" + INFO2("gc.space", "GC: Space Adapt: nos ---> mos (" < nos (" <num_collections<<"] ...\n"); POINTER_SIZE_INT adapt_size = curr_nos_size - new_nos_size; INFO2("gc.space", "GC: Space Adapt: nos ---> mos (" <free)); } -JNIEXPORT jint JNICALL Java_org_apache_harmony_drlvm_gc_1gen_GCHelper_getPrefetchDist(JNIEnv *e, jclass c) +JNIEXPORT jint JNICALL Java_org_apache_harmony_drlvm_gc_1gen_GCHelper_getTlaCeilingOffset(JNIEnv *, jclass) { - return 0; + return (jint)((POINTER_SIZE_INT) &(((Allocator*)0)->ceiling)); } -JNIEXPORT jint JNICALL Java_org_apache_harmony_drlvm_gc_1gen_GCHelper_getPrefetchStride(JNIEnv *e, jclass c) +JNIEXPORT jint JNICALL Java_org_apache_harmony_drlvm_gc_1gen_GCHelper_getTlaEndOffset(JNIEnv *, jclass) { - return 0; + return (jint)((POINTER_SIZE_INT) &(((Allocator*)0)->end)); } -JNIEXPORT jboolean JNICALL Java_org_apache_harmony_drlvm_gc_1gen_GCHelper_isPrefetchEnabled(JNIEnv *, jclass) { - return (jboolean) JNI_FALSE; -} -#endif /* _IPF_ */ - - -JNIEXPORT jint JNICALL Java_org_apache_harmony_drlvm_gc_1gen_GCHelper_getTlaFreeOffset(JNIEnv *, jclass) { - Allocator allocator; - return (jint) ((POINTER_SIZE_INT)&allocator.free - (POINTER_SIZE_INT)&allocator); -} - -JNIEXPORT jint JNICALL Java_org_apache_harmony_drlvm_gc_1gen_GCHelper_getTlaCeilingOffset(JNIEnv *, jclass) { - Allocator allocator; - return (jint) ((POINTER_SIZE_INT)&allocator.ceiling - (POINTER_SIZE_INT)&allocator); -} - -JNIEXPORT jint JNICALL Java_org_apache_harmony_drlvm_gc_1gen_GCHelper_getTlaEndOffset(JNIEnv *, jclass) { - Allocator allocator; - return (jint) ((POINTER_SIZE_INT)&allocator.end - (POINTER_SIZE_INT)&allocator); -} - -JNIEXPORT jint JNICALL Java_org_apache_harmony_drlvm_gc_1gen_GCHelper_getGCObjectAlignment(JNIEnv *, jclass) { +JNIEXPORT jint JNICALL Java_org_apache_harmony_drlvm_gc_1gen_GCHelper_getGCObjectAlignment(JNIEnv *, jclass) +{ return (jint) GC_OBJECT_ALIGNMENT; } -JNIEXPORT jint JNICALL Java_org_apache_harmony_drlvm_gc_1gen_GCHelper_getLargeObjectSize(JNIEnv *, jclass) { +JNIEXPORT jint JNICALL Java_org_apache_harmony_drlvm_gc_1gen_GCHelper_getLargeObjectSize(JNIEnv *, jclass) +{ return (jint) GC_OBJ_SIZE_THRESHOLD; } Index: src/los/lspace_alloc_collect.cpp =================================================================== --- src/los/lspace_alloc_collect.cpp (revision 606795) +++ src/los/lspace_alloc_collect.cpp (working copy) @@ -206,9 +206,7 @@ Lspace* lspace = (Lspace*)gc_get_los((GC_Gen*)allocator->gc); Free_Area_Pool* pool = lspace->free_pool; - if(gc_need_start_concurrent_mark(allocator->gc)) - gc_start_concurrent_mark(allocator->gc); - + gc_try_schedule_collection(allocator->gc, GC_CAUSE_NIL); while( try_count < 2 ){ if(p_result = lspace_try_alloc(lspace, alloc_size)) @@ -273,7 +271,7 @@ if( obj_info != 0 ) { collector_remset_add_entry(collector, (Partial_Reveal_Object **)dest_addr); - collector_remset_add_entry(collector, (Partial_Reveal_Object **)obj_info); + collector_remset_add_entry(collector, (Partial_Reveal_Object **)(POINTER_SIZE_INT)obj_info); } obj_set_fw_in_oi(p_obj, dest_addr); @@ -482,3 +480,4 @@ } + Index: src/mark_compact/fallback_mark_scan.cpp =================================================================== --- src/mark_compact/fallback_mark_scan.cpp (revision 606795) +++ src/mark_compact/fallback_mark_scan.cpp (working copy) @@ -41,8 +41,8 @@ assert(p_obj); assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); - Partial_Reveal_VTable *vtable = uncompress_vt(obj_get_vt(p_obj)); - if(VTABLE_TRACING) + Partial_Reveal_VTable *vtable = decode_vt(obj_get_vt(p_obj)); + if(TRACE_JLC_VIA_VTABLE) if(!(vtable->vtmark & VT_FALLBACK_MARKED)) { vtable->vtmark |= VT_FALLBACK_MARKED; //we need different marking for fallback compaction collector_tracestack_push(collector, &(vtable->jlC)); @@ -203,24 +203,42 @@ } #ifdef USE_32BITS_HASHCODE + +/* for semispace NOS, actually only the fromspace needs cleaning of oi. */ void fallback_clear_fwd_obj_oi(Collector* collector) { GC* gc = collector->gc; + Blocked_Space* space = (Blocked_Space*)((GC_Gen*)gc)->nos; + assert(gc_match_kind(gc, FALLBACK_COLLECTION)); unsigned int num_active_collectors = gc->num_active_collectors; atomic_cas32( &num_finished_collectors, 0, num_active_collectors); - Block_Header* curr_block = fspace_get_next_block(); + Block_Header* curr_block = blocked_space_block_iterator_next(space); while(curr_block){ Partial_Reveal_Object* curr_obj = (Partial_Reveal_Object*) curr_block->base; while(curr_obj < curr_block->free){ + unsigned int obj_size = vm_object_size(curr_obj); + /* forwarded object is dead object (after fallback marking),but we need know its size to iterate live object */ if(obj_is_fw_in_oi(curr_obj)){ - set_obj_info(curr_obj, (Obj_Info_Type)0); + if(obj_is_sethash_in_vt(curr_obj)){ + /* this only happens in semispace GC, where an object with attached hashcode is forwarded. + This object should be in survivor_area, forwarded from fromspace in last minor collection. + We restore its hashbits correctly in oi. */ + set_obj_info(curr_obj, (Obj_Info_Type)HASHCODE_SET_ATTACHED); + }else{ + set_obj_info(curr_obj, (Obj_Info_Type)0); + } } - curr_obj = (Partial_Reveal_Object*)((POINTER_SIZE_INT)curr_obj + vm_object_size(curr_obj)); + /* if it's not forwared, it may still have hashcode attached if its in survivor_area. + It's not forwarded because fallback happens before it's forwarded. */ + if(hashcode_is_attached(curr_obj)) + obj_size += GC_OBJECT_ALIGNMENT; + + curr_obj = (Partial_Reveal_Object*)((POINTER_SIZE_INT)curr_obj + obj_size); } - curr_block = fspace_get_next_block(); + curr_block = blocked_space_block_iterator_next(space); } atomic_inc32(&num_finished_collectors); while(num_finished_collectors < num_active_collectors) ; @@ -228,7 +246,9 @@ void fallback_clear_fwd_obj_oi_init(Collector* collector) { - fspace_block_iterate_init((Fspace*)((GC_Gen*)collector->gc)->nos); + Blocked_Space* space = (Blocked_Space*)((GC_Gen*)collector->gc)->nos; + blocked_space_block_iterator_init(space); + } #endif @@ -238,3 +258,4 @@ + Index: src/mark_compact/mspace.cpp =================================================================== --- src/mark_compact/mspace.cpp (revision 606795) +++ src/mark_compact/mspace.cpp (working copy) @@ -21,16 +21,6 @@ #include "mspace.h" -#include "../common/gc_space.h" - -static void mspace_destruct_blocks(Mspace* mspace) -{ -#ifdef USE_32BITS_HASHCODE - space_desturct_blocks((Blocked_Space*)mspace); -#endif - return; -} - struct GC_Gen; extern void gc_set_mos(GC_Gen* gc, Space* space); extern Space* gc_get_nos(GC_Gen* gc); @@ -95,8 +85,10 @@ void mspace_destruct(Mspace* mspace) { - //FIXME:: when map the to-half, the decommission start address should change - mspace_destruct_blocks(mspace); + //FIXME:: when map the to-half, the decommission start address should change +#ifdef USE_32BITS_HASHCODE + space_desturct_blocks((Blocked_Space*)mspace); +#endif STD_FREE(mspace); } @@ -181,3 +173,4 @@ return mspace->expected_threshold_ratio; } + Index: src/mark_compact/mspace_alloc.cpp =================================================================== --- src/mark_compact/mspace_alloc.cpp (revision 606795) +++ src/mark_compact/mspace_alloc.cpp (working copy) @@ -43,20 +43,16 @@ return TRUE; } - /* Mspace is out, a collection should be triggered. It can be caused by mutator allocation - And it can be caused by collector allocation during nos forwarding. */ + /* Mspace is out. If it's caused by mutator, a collection should be triggered. + If it's caused by collector, a fallback should be triggered. */ return FALSE; } -struct GC_Gen; -Space* gc_get_mos(GC_Gen* gc); void* mspace_alloc(unsigned int size, Allocator* allocator) { void *p_return = NULL; - - Mspace* mspace = (Mspace*)gc_get_mos((GC_Gen*)allocator->gc); - + /* All chunks of data requested need to be multiples of GC_OBJECT_ALIGNMENT */ assert((size % GC_OBJECT_ALIGNMENT) == 0); assert( size <= GC_OBJ_SIZE_THRESHOLD ); @@ -66,7 +62,8 @@ if(p_return) return p_return; /* grab a new block */ - Boolean ok = mspace_alloc_block(mspace, allocator); + Mspace* mspace = (Mspace*)allocator->alloc_space;; + Boolean ok = mspace_alloc_block(mspace, allocator); if(!ok) return NULL; p_return = thread_local_alloc(size, allocator); @@ -76,3 +73,4 @@ } + Index: src/mark_compact/mspace_extend_compact.cpp =================================================================== --- src/mark_compact/mspace_extend_compact.cpp (revision 606795) +++ src/mark_compact/mspace_extend_compact.cpp (working copy) @@ -15,10 +15,6 @@ * limitations under the License. */ -/** - * @author Chunrong Lai, 2006/12/25 - */ - #include "mspace_collect_compact.h" #include "../trace_forward/fspace.h" #include "../los/lspace.h" @@ -45,11 +41,11 @@ first_block_to_move = nos_start_block; } -static POINTER_SIZE_INT fspace_shrink(Fspace *fspace) +static POINTER_SIZE_INT nspace_shrink(Fspace *nspace) { - void *committed_nos_end = (void *)((POINTER_SIZE_INT)space_heap_start((Space *)fspace) + fspace->committed_heap_size); + void *committed_nos_end = (void *)((POINTER_SIZE_INT)space_heap_start((Space *)nspace) + nspace->committed_heap_size); - POINTER_SIZE_INT nos_used_size = (POINTER_SIZE_INT)nos_first_free_block - (POINTER_SIZE_INT)fspace->heap_start; + POINTER_SIZE_INT nos_used_size = (POINTER_SIZE_INT)nos_first_free_block - (POINTER_SIZE_INT)nspace->heap_start; POINTER_SIZE_INT nos_free_size = (POINTER_SIZE_INT)committed_nos_end - (POINTER_SIZE_INT)nos_first_free_block; POINTER_SIZE_INT decommit_size = (nos_used_size <= nos_free_size) ? nos_used_size : nos_free_size; assert(decommit_size); @@ -64,26 +60,26 @@ Boolean result = vm_decommit_mem(decommit_base, decommit_size); assert(result == TRUE); - fspace->committed_heap_size = (POINTER_SIZE_INT)decommit_base - (POINTER_SIZE_INT)fspace->heap_start; - fspace->num_managed_blocks = (unsigned int)(fspace->committed_heap_size >> GC_BLOCK_SHIFT_COUNT); + nspace->committed_heap_size = (POINTER_SIZE_INT)decommit_base - (POINTER_SIZE_INT)nspace->heap_start; + nspace->num_managed_blocks = (unsigned int)(nspace->committed_heap_size >> GC_BLOCK_SHIFT_COUNT); - Block_Header *new_last_block = (Block_Header *)&fspace->blocks[fspace->num_managed_blocks - 1]; - fspace->ceiling_block_idx = new_last_block->block_idx; + Block_Header *new_last_block = (Block_Header *)&nspace->blocks[nspace->num_managed_blocks - 1]; + nspace->ceiling_block_idx = new_last_block->block_idx; new_last_block->next = NULL; return decommit_size; } -static void link_mspace_extended_blocks(Mspace *mspace, Fspace *fspace) +static void link_mspace_extended_blocks(Mspace *mspace, Fspace *nspace) { Block_Header *old_last_mos_block = (Block_Header *)(mos_first_new_block -1); old_last_mos_block->next = (Block_Header *)mos_first_new_block; void *new_committed_mos_end = (void *)((POINTER_SIZE_INT)space_heap_start((Space *)mspace) + mspace->committed_heap_size); Block_Header *new_last_mos_block = (Block_Header *)((Block *)new_committed_mos_end -1); - new_last_mos_block->next = (Block_Header *)space_heap_start((Space *)fspace); + new_last_mos_block->next = (Block_Header *)space_heap_start((Space *)nspace); } -static Block *mspace_extend_without_link(Mspace *mspace, Fspace *fspace, unsigned int commit_size) +static Block *mspace_extend_without_link(Mspace *mspace, Fspace *nspace, unsigned int commit_size) { assert(commit_size && !(commit_size % GC_BLOCK_SIZE_BYTES)); @@ -267,11 +263,14 @@ static volatile unsigned int num_space_changing_collectors = 0; #ifndef STATIC_NOS_MAPPING + +/* FIXME:: this is a sequential process, the atomic parallel constructs should be removed. + Better to call this function in the sequential region of last phase. */ void mspace_extend_compact(Collector *collector) { GC_Gen *gc_gen = (GC_Gen *)collector->gc; - Mspace *mspace = (Mspace *)gc_gen->mos; - Fspace *fspace = (Fspace *)gc_gen->nos; + Blocked_Space *mspace = (Blocked_Space *)gc_gen->mos; + Blocked_Space *nspace = (Blocked_Space *)gc_gen->nos; /*For_LOS adaptive: when doing EXTEND_COLLECTION, mspace->survive_ratio should not be updated in gc_decide_next_collect( )*/ gc_gen->collect_kind |= EXTEND_COLLECTION; @@ -281,18 +280,20 @@ atomic_cas32( &num_space_changing_collectors, 0, num_active_collectors + 1); old_num = atomic_inc32(&num_space_changing_collectors); if( ++old_num == num_active_collectors ){ - Block *old_nos_boundary = fspace->blocks; + if(NOS_SIZE) /* when NOS_SIZE is speficied, it can't be shrunk. */ + WARN2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: MOS is overflowed, have to reduce NOS size."); + Block *old_nos_boundary = nspace->blocks; nos_boundary = &mspace->blocks[mspace->free_block_idx - mspace->first_block_idx]; - if(fspace->num_managed_blocks != 0) + if(nspace->num_managed_blocks != 0) /* FIXME:: why can it be 0 here?? */ assert(nos_boundary > old_nos_boundary); POINTER_SIZE_INT mem_change_size = ((Block *)nos_boundary - old_nos_boundary) << GC_BLOCK_SHIFT_COUNT; - fspace->heap_start = nos_boundary; - fspace->blocks = (Block *)nos_boundary; - fspace->committed_heap_size -= mem_change_size; - fspace->num_managed_blocks = (unsigned int)(fspace->committed_heap_size >> GC_BLOCK_SHIFT_COUNT); - fspace->num_total_blocks = fspace->num_managed_blocks; - fspace->first_block_idx = mspace->free_block_idx; - fspace->free_block_idx = fspace->first_block_idx; + nspace->heap_start = nos_boundary; + nspace->blocks = (Block *)nos_boundary; + nspace->committed_heap_size -= mem_change_size; + nspace->num_managed_blocks = (unsigned int)(nspace->committed_heap_size >> GC_BLOCK_SHIFT_COUNT); + nspace->num_total_blocks = nspace->num_managed_blocks; + nspace->first_block_idx = mspace->free_block_idx; + nspace->free_block_idx = nspace->first_block_idx; mspace->heap_end = nos_boundary; mspace->committed_heap_size += mem_change_size; @@ -315,7 +316,7 @@ { GC_Gen *gc_gen = (GC_Gen *)collector->gc; Mspace *mspace = gc_gen->mos; - Fspace *fspace = gc_gen->nos; + Fspace *nspace = gc_gen->nos; Lspace *lspace = gc_gen->los; /*For_LOS adaptive: when doing EXTEND_COLLECTION, mspace->survive_ratio should not be updated in gc_decide_next_collect( )*/ @@ -324,7 +325,7 @@ unsigned int num_active_collectors = gc_gen->num_active_collectors; unsigned int old_num; - Block *nos_first_block = fspace->blocks; + Block *nos_first_block = nspace->blocks; nos_first_free_block = &mspace->blocks[mspace->free_block_idx - mspace->first_block_idx]; assert(nos_first_free_block > nos_first_block); @@ -333,8 +334,8 @@ atomic_cas32( &num_space_changing_collectors, 0, num_active_collectors + 1); old_num = atomic_inc32(&num_space_changing_collectors); if( old_num == 0 ){ - unsigned int mem_changed_size = fspace_shrink(fspace); - mos_first_new_block = mspace_extend_without_link(mspace, fspace, mem_changed_size); + unsigned int mem_changed_size = nspace_shrink(nspace); + mos_first_new_block = mspace_extend_without_link(mspace, nspace, mem_changed_size); set_first_and_end_block_to_move(collector, mem_changed_size); //mspace_block_iter_init_for_extension(mspace, (Block_Header *)first_block_to_move); @@ -353,7 +354,7 @@ /* init the iterator: prepare for refixing */ lspace_refix_repointed_refs(collector, lspace, (void *)first_block_to_move, (void *)nos_first_free_block, (first_block_to_move - mos_first_new_block) << GC_BLOCK_SHIFT_COUNT); gc_refix_rootset(collector, (void *)first_block_to_move, (void *)nos_first_free_block, (first_block_to_move - mos_first_new_block) << GC_BLOCK_SHIFT_COUNT); - link_mspace_extended_blocks(mspace, fspace); + link_mspace_extended_blocks(mspace, nspace); mspace_block_iter_init_for_extension(mspace, (Block_Header *)first_block_to_move); num_refixing_collectors++; } Index: src/mark_compact/mspace_slide_compact.cpp =================================================================== --- src/mark_compact/mspace_slide_compact.cpp (revision 606795) +++ src/mark_compact/mspace_slide_compact.cpp (working copy) @@ -20,7 +20,6 @@ */ #include "mspace_collect_compact.h" -#include "../trace_forward/fspace.h" #include "../los/lspace.h" #include "../finalizer_weakref/finalizer_weakref.h" @@ -28,7 +27,6 @@ #include "../gen/gen_stats.h" #endif - struct GC_Gen; Space* gc_get_nos(GC_Gen* gc); Space* gc_get_mos(GC_Gen* gc); @@ -116,7 +114,7 @@ if( obj_info != 0 ) { collector_remset_add_entry(collector, (Partial_Reveal_Object **)dest_addr); - collector_remset_add_entry(collector, (Partial_Reveal_Object **)obj_info); + collector_remset_add_entry(collector, (Partial_Reveal_Object **)(POINTER_SIZE_INT)obj_info); } obj_set_fw_in_oi(p_obj, dest_addr); @@ -376,7 +374,7 @@ assert(obj_is_marked_in_vt(p_obj)); #ifdef USE_32BITS_HASHCODE obj_clear_dual_bits_in_vt(p_obj); - #else +#else obj_unmark_in_vt(p_obj); #endif @@ -411,7 +409,6 @@ { GC* gc = collector->gc; Mspace* mspace = (Mspace*)gc_get_mos((GC_Gen*)gc); - Fspace* fspace = (Fspace*)gc_get_nos((GC_Gen*)gc); Lspace* lspace = (Lspace*)gc_get_los((GC_Gen*)gc); unsigned int num_active_collectors = gc->num_active_collectors; @@ -421,7 +418,7 @@ *have references that are going to be repointed. */ - TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: pass1: mark live objects in heap ..."); + TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: pass1: marking..."); unsigned int old_num = atomic_cas32( &num_marking_collectors, 0, num_active_collectors+1); @@ -435,6 +432,7 @@ /* last collector's world here */ if( ++old_num == num_active_collectors ){ + if(!IGNORE_FINREF ) collector_identify_finref(collector); #ifndef BUILD_IN_REFERENT @@ -461,24 +459,24 @@ } while(num_marking_collectors != num_active_collectors + 1); - TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: finish pass1"); + TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: finish pass1 and start pass2: relocating mos&nos..."); /* Pass 2: ************************************************** assign target addresses for all to-be-moved objects */ + atomic_cas32( &num_repointing_collectors, 0, num_active_collectors+1); #ifdef USE_32BITS_HASHCODE if(gc_match_kind(gc, FALLBACK_COLLECTION)) fallback_clear_fwd_obj_oi(collector); #endif - TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: pass2: computer target addresses for to-be-moved objects in mos and nos ..."); mspace_compute_object_target(collector, mspace); old_num = atomic_inc32(&num_repointing_collectors); /*last collector's world here*/ if( ++old_num == num_active_collectors ){ if(lspace->move_object) { - TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: pass2: computer target addresses for to-be-moved objects in los ..."); + TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: pass2: relocating los ..."); lspace_compute_object_target(collector, lspace); } gc->collect_result = gc_collection_result(gc); @@ -492,12 +490,11 @@ } while(num_repointing_collectors != num_active_collectors + 1); if(!gc->collect_result) return; - TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: finish pass2"); + TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: finish pass2 and start pass3: repointing..."); /* Pass 3: ************************************************** *update all references whose objects are to be moved */ - TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: pass3: update all references ..."); old_num = atomic_cas32( &num_fixing_collectors, 0, num_active_collectors+1); mspace_fix_repointed_refs(collector, mspace); old_num = atomic_inc32(&num_fixing_collectors); @@ -520,13 +517,11 @@ } while(num_fixing_collectors != num_active_collectors + 1); - TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: finish pass3"); + TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: finish pass3 and start pass4: moving..."); /* Pass 4: ************************************************** move objects */ - TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: pass4: move objects to target address ..."); - atomic_cas32( &num_moving_collectors, 0, num_active_collectors); mspace_sliding_compact(collector, mspace); @@ -534,12 +529,11 @@ atomic_inc32(&num_moving_collectors); while(num_moving_collectors != num_active_collectors); - TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: finish pass4"); + TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: finish pass4 and start pass 5: restoring obj_info..."); /* Pass 5: ************************************************** restore obj_info */ - TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: pass5: restore obj_info ..."); atomic_cas32( &num_restoring_collectors, 0, num_active_collectors+1); collector_restore_obj_info(collector); @@ -557,7 +551,8 @@ while(num_restoring_collectors != num_active_collectors + 1); /* Dealing with out of memory in mspace */ - if(mspace->free_block_idx > fspace->first_block_idx){ + void* mspace_border = &mspace->blocks[mspace->free_block_idx - mspace->first_block_idx]; + if( mspace_border > nos_boundary){ atomic_cas32( &num_extending_collectors, 0, num_active_collectors); mspace_extend_compact(collector); @@ -566,16 +561,7 @@ while(num_extending_collectors != num_active_collectors); } - TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: finish pass5 ..."); + TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: finish pass5 and done."); - if( (POINTER_SIZE_INT)collector->thread_handle != 0 ){ - TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"] finished"); - return; - } - - /* Leftover: ************************************************** - */ - - TRACE2("gc.process", "GC: collector[0] finished"); return; } Index: src/mark_compact/space_tune_mark_scan.cpp =================================================================== --- src/mark_compact/space_tune_mark_scan.cpp (revision 606795) +++ src/mark_compact/space_tune_mark_scan.cpp (working copy) @@ -55,8 +55,8 @@ vm_notify_obj_alive( (void *)p_obj); assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); - Partial_Reveal_VTable *vtable = uncompress_vt(obj_get_vt(p_obj)); - if(VTABLE_TRACING) + Partial_Reveal_VTable *vtable = decode_vt(obj_get_vt(p_obj)); + if(TRACE_JLC_VIA_VTABLE) if(vtable->vtmark == VT_UNMARKED) { vtable->vtmark = VT_MARKED; if(obj_mark_in_vt(vtable->jlC)) Index: src/mark_sweep/gc_ms.cpp =================================================================== --- src/mark_sweep/gc_ms.cpp (revision 606795) +++ src/mark_sweep/gc_ms.cpp (working copy) @@ -20,6 +20,7 @@ #ifdef USE_MARK_SWEEP_GC #include "gc_ms.h" +#include "wspace_mark_sweep.h" #include "../finalizer_weakref/finalizer_weakref.h" #include "../common/compressed_ref.h" #include "../thread/marker.h" @@ -38,14 +39,14 @@ assert(max_heap_size <= max_heap_size_bytes); assert(max_heap_size >= min_heap_size_bytes); - void *sspace_base; - sspace_base = vm_reserve_mem(0, max_heap_size); - sspace_initialize((GC*)gc_ms, sspace_base, max_heap_size, max_heap_size); + void *wspace_base; + wspace_base = vm_reserve_mem(0, max_heap_size); + wspace_initialize((GC*)gc_ms, wspace_base, max_heap_size, max_heap_size); - HEAP_NULL = (POINTER_SIZE_INT)sspace_base; + HEAP_NULL = (POINTER_SIZE_INT)wspace_base; - gc_ms->heap_start = sspace_base; - gc_ms->heap_end = (void*)((POINTER_SIZE_INT)sspace_base + max_heap_size); + gc_ms->heap_start = wspace_base; + gc_ms->heap_end = (void*)((POINTER_SIZE_INT)wspace_base + max_heap_size); gc_ms->reserved_heap_size = max_heap_size; gc_ms->committed_heap_size = max_heap_size; gc_ms->num_collections = 0; @@ -54,40 +55,72 @@ void gc_ms_destruct(GC_MS *gc_ms) { - Sspace *sspace = gc_ms->sspace; - void *sspace_start = sspace->heap_start; - sspace_destruct(sspace); - gc_ms->sspace = NULL; - vm_unmap_mem(sspace_start, space_committed_size((Space*)sspace)); + Wspace *wspace = gc_ms->wspace; + void *wspace_start = wspace->heap_start; + wspace_destruct(wspace); + gc_ms->wspace = NULL; + vm_unmap_mem(wspace_start, space_committed_size((Space*)wspace)); } void gc_ms_reclaim_heap(GC_MS *gc) { if(verify_live_heap) gc_verify_heap((GC*)gc, TRUE); - Sspace *sspace = gc_ms_get_sspace(gc); + Wspace *wspace = gc_ms_get_wspace(gc); - sspace_collection(sspace); + wspace_collection(wspace); - sspace_reset_after_collection(sspace); + wspace_reset_after_collection(wspace); if(verify_live_heap) gc_verify_heap((GC*)gc, FALSE); } -void sspace_mark_scan_concurrent(Marker* marker); +void wspace_mark_scan_concurrent(Marker* marker); void gc_ms_start_concurrent_mark(GC_MS* gc, unsigned int num_markers) { if(gc->num_active_markers == 0) pool_iterator_init(gc->metadata->gc_rootset_pool); - marker_execute_task_concurrent((GC*)gc,(TaskType)sspace_mark_scan_concurrent,(Space*)gc->sspace, num_markers); + marker_execute_task_concurrent((GC*)gc,(TaskType)wspace_mark_scan_concurrent,(Space*)gc->wspace, num_markers); } +void wspace_mark_scan_mostly_concurrent(Marker* marker); +void gc_ms_start_most_concurrent_mark(GC_MS* gc, unsigned int num_markers) +{ + if(gc->num_active_markers == 0) + pool_iterator_init(gc->metadata->gc_rootset_pool); + + marker_execute_task_concurrent((GC*)gc,(TaskType)wspace_mark_scan_mostly_concurrent,(Space*)gc->wspace, num_markers); +} + +void gc_ms_start_final_mark_after_concurrent(GC_MS* gc, unsigned int num_markers) +{ + pool_iterator_init(gc->metadata->gc_rootset_pool); + + marker_execute_task((GC*)gc,(TaskType)wspace_mark_scan_mostly_concurrent,(Space*)gc->wspace); +} + + + +void wspace_sweep_concurrent(Collector* collector); +void gc_ms_start_concurrent_sweep(GC_MS* gc, unsigned int num_collectors) +{ + ops_color_flip(); + //FIXME: Need barrier here. + //apr_memory_rw_barrier(); + gc_disenable_alloc_obj_live(); + wspace_init_pfc_pool_iterator(gc->wspace); + + collector_execute_task_concurrent((GC*)gc, (TaskType)wspace_sweep_concurrent, (Space*)gc->wspace, num_collectors); + + collector_release_weakref_sets((GC*)gc, num_collectors); +} + void gc_ms_start_concurrent_mark(GC_MS* gc) { pool_iterator_init(gc->metadata->gc_rootset_pool); - marker_execute_task_concurrent((GC*)gc,(TaskType)sspace_mark_scan_concurrent,(Space*)gc->sspace); + marker_execute_task_concurrent((GC*)gc,(TaskType)wspace_mark_scan_concurrent,(Space*)gc->wspace); } void gc_ms_update_space_statistics(GC_MS* gc) @@ -95,7 +128,7 @@ POINTER_SIZE_INT num_live_obj = 0; POINTER_SIZE_INT size_live_obj = 0; - Space_Statistics* sspace_stat = gc->sspace->space_statistic; + Space_Statistics* wspace_stat = gc->wspace->space_statistic; unsigned int num_collectors = gc->num_active_collectors; Collector** collectors = gc->collectors; @@ -106,10 +139,10 @@ size_live_obj += collector->live_obj_size; } - sspace_stat->num_live_obj = num_live_obj; - sspace_stat->size_live_obj = size_live_obj; - sspace_stat->last_size_free_space = sspace_stat->size_free_space; - sspace_stat->size_free_space = gc->committed_heap_size - size_live_obj;/*TODO:inaccurate value.*/ + wspace_stat->num_live_obj = num_live_obj; + wspace_stat->size_live_obj = size_live_obj; + wspace_stat->last_size_free_space = wspace_stat->size_free_space; + wspace_stat->size_free_space = gc->committed_heap_size - size_live_obj;/*TODO:inaccurate value.*/ } void gc_ms_iterate_heap(GC_MS *gc) Index: src/mark_sweep/gc_ms.h =================================================================== --- src/mark_sweep/gc_ms.h (revision 606795) +++ src/mark_sweep/gc_ms.h (working copy) @@ -20,7 +20,7 @@ #ifdef USE_MARK_SWEEP_GC -#include "sspace.h" +#include "wspace.h" /* heap size limit is not interesting. only for manual tuning purpose */ @@ -77,35 +77,37 @@ SpinLock concurrent_mark_lock; SpinLock enumerate_rootset_lock; + SpinLock concurrent_sweep_lock; + /* system info */ unsigned int _system_alloc_unit; unsigned int _machine_page_size_bytes; unsigned int _num_processors; /* END of GC --> */ - Sspace *sspace; + Wspace *wspace; } GC_MS; ////////////////////////////////////////////////////////////////////////////////////////// inline void *gc_ms_fast_alloc(unsigned size, Allocator *allocator) -{ return sspace_thread_local_alloc(size, allocator); } +{ return wspace_thread_local_alloc(size, allocator); } inline void *gc_ms_alloc(unsigned size, Allocator *allocator) -{ return sspace_alloc(size, allocator); } +{ return wspace_alloc(size, allocator); } -inline Sspace *gc_ms_get_sspace(GC_MS *gc) -{ return gc->sspace; } +inline Wspace *gc_ms_get_wspace(GC_MS *gc) +{ return gc->wspace; } -inline void gc_ms_set_sspace(GC_MS *gc, Sspace *sspace) -{ gc->sspace = sspace; } +inline void gc_ms_set_wspace(GC_MS *gc, Wspace *wspace) +{ gc->wspace = wspace; } inline POINTER_SIZE_INT gc_ms_free_memory_size(GC_MS *gc) -{ return sspace_free_memory_size(gc_ms_get_sspace(gc)); } +{ return wspace_free_memory_size(gc_ms_get_wspace(gc)); } inline POINTER_SIZE_INT gc_ms_total_memory_size(GC_MS *gc) -{ return space_committed_size((Space*)gc_ms_get_sspace(gc)); } +{ return space_committed_size((Space*)gc_ms_get_wspace(gc)); } ///////////////////////////////////////////////////////////////////////////////////////// @@ -117,7 +119,12 @@ void gc_ms_start_concurrent_mark(GC_MS* gc); void gc_ms_start_concurrent_mark(GC_MS* gc, unsigned int num_markers); void gc_ms_update_space_statistics(GC_MS* gc); +void gc_ms_start_concurrent_sweep(GC_MS* gc, unsigned int num_collectors); +void gc_ms_start_most_concurrent_mark(GC_MS* gc, unsigned int num_markers); +void gc_ms_start_final_mark_after_concurrent(GC_MS* gc, unsigned int num_markers); + + #endif // USE_MARK_SWEEP_GC #endif // _GC_MS_H_ Index: src/mark_sweep/sspace.cpp =================================================================== --- src/mark_sweep/sspace.cpp (revision 606795) +++ src/mark_sweep/sspace.cpp (working copy) @@ -1,258 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sspace.h" -#include "sspace_chunk.h" -#include "sspace_verify.h" -#include "gc_ms.h" -#include "../gen/gen.h" - -struct GC_Gen; - -Sspace *sspace_initialize(GC *gc, void *start, POINTER_SIZE_INT sspace_size, POINTER_SIZE_INT commit_size) -{ - /* With sspace in the heap, the heap must be composed of a single sspace or a sspace and a NOS. - * In either case, the reserved size and committed size of sspace must be the same. - * Because sspace has only mark-sweep collection, it is not possible to shrink sspace. - * So there is no need to use dynamic space resizing. - */ - assert(sspace_size == commit_size); - - Sspace *sspace = (Sspace*)STD_MALLOC(sizeof(Sspace)); - assert(sspace); - memset(sspace, 0, sizeof(Sspace)); - - sspace->reserved_heap_size = sspace_size; - - void *reserved_base = start; - - /* commit sspace mem */ - if(!large_page_hint) - vm_commit_mem(reserved_base, commit_size); - memset(reserved_base, 0, commit_size); - sspace->committed_heap_size = commit_size; - - sspace->heap_start = reserved_base; - sspace->heap_end = (void *)((POINTER_SIZE_INT)reserved_base + sspace_size); - - sspace->num_collections = 0; - sspace->time_collections = 0; - sspace->survive_ratio = 0.2f; - - sspace->move_object = FALSE; - sspace->gc = gc; - - sspace_init_chunks(sspace); - - sspace->space_statistic = (Space_Statistics*)STD_MALLOC(sizeof(Space_Statistics)); - assert(sspace->space_statistic); - memset(sspace->space_statistic, 0, sizeof(Space_Statistics)); - -#ifdef USE_MARK_SWEEP_GC - gc_ms_set_sspace((GC_MS*)gc, sspace); -#else - gc_set_mos((GC_Gen*)gc, (Space*)sspace); -#endif - -#ifdef SSPACE_VERIFY - sspace_verify_init(gc); -#endif - return sspace; -} - -static void sspace_destruct_chunks(Sspace *sspace) { return; } - -void sspace_destruct(Sspace *sspace) -{ - //FIXME:: when map the to-half, the decommission start address should change - sspace_destruct_chunks(sspace); - STD_FREE(sspace); -} - -void sspace_reset_after_collection(Sspace *sspace) -{ - sspace->move_object = FALSE; - sspace->need_compact = FALSE; - sspace->need_fix = FALSE; -} - -void allocator_init_local_chunks(Allocator *allocator) -{ - Sspace *sspace = gc_get_sspace(allocator->gc); - Size_Segment **size_segs = sspace->size_segments; - - /* Alloc mem for size segments (Chunk_Header**) */ - unsigned int seg_size = sizeof(Chunk_Header**) * SIZE_SEGMENT_NUM; - Chunk_Header ***local_chunks = (Chunk_Header***)STD_MALLOC(seg_size); - memset(local_chunks, 0, seg_size); - - /* Alloc mem for local chunk pointers */ - unsigned int chunk_ptr_size = 0; - for(unsigned int i = SIZE_SEGMENT_NUM; i--;){ - if(size_segs[i]->local_alloc){ - chunk_ptr_size += size_segs[i]->chunk_num; - } - } - chunk_ptr_size *= sizeof(Chunk_Header*); - Chunk_Header **chunk_ptrs = (Chunk_Header**)STD_MALLOC(chunk_ptr_size); - memset(chunk_ptrs, 0, chunk_ptr_size); - - for(unsigned int i = 0; i < SIZE_SEGMENT_NUM; ++i){ - if(size_segs[i]->local_alloc){ - local_chunks[i] = chunk_ptrs; - chunk_ptrs += size_segs[i]->chunk_num; - } - } - - allocator->local_chunks = local_chunks; -} - -void allocactor_destruct_local_chunks(Allocator *allocator) -{ - Sspace *sspace = gc_get_sspace(allocator->gc); - Size_Segment **size_segs = sspace->size_segments; - Chunk_Header ***local_chunks = allocator->local_chunks; - Chunk_Header **chunk_ptrs = NULL; - unsigned int chunk_ptr_num = 0; - - /* Find local chunk pointers' head and their number */ - for(unsigned int i = 0; i < SIZE_SEGMENT_NUM; ++i){ - if(size_segs[i]->local_alloc){ - chunk_ptr_num += size_segs[i]->chunk_num; - assert(local_chunks[i]); - if(!chunk_ptrs) - chunk_ptrs = local_chunks[i]; - } - } - - /* Put local pfc to the according pools */ - for(unsigned int i = 0; i < chunk_ptr_num; ++i){ - if(chunk_ptrs[i]) - sspace_put_pfc(sspace, chunk_ptrs[i]); - } - - /* Free mem for local chunk pointers */ - STD_FREE(chunk_ptrs); - - /* Free mem for size segments (Chunk_Header**) */ - STD_FREE(local_chunks); -} - -static void allocator_clear_local_chunks(Allocator *allocator) -{ - Sspace *sspace = gc_get_sspace(allocator->gc); - Size_Segment **size_segs = sspace->size_segments; - Chunk_Header ***local_chunks = allocator->local_chunks; - - for(unsigned int i = SIZE_SEGMENT_NUM; i--;){ - if(!size_segs[i]->local_alloc){ - assert(!local_chunks[i]); - continue; - } - Chunk_Header **chunks = local_chunks[i]; - assert(chunks); - for(unsigned int j = size_segs[i]->chunk_num; j--;){ - if(chunks[j]) - sspace_put_pfc(sspace, chunks[j]); - chunks[j] = NULL; - } - } -} - -static void gc_clear_mutator_local_chunks(GC *gc) -{ -#ifdef USE_MARK_SWEEP_GC - /* release local chunks of each mutator in unique mark-sweep GC */ - Mutator *mutator = gc->mutator_list; - while(mutator){ - allocator_clear_local_chunks((Allocator*)mutator); - mutator = mutator->next; - } -#endif -} - -void gc_clear_collector_local_chunks(GC *gc) -{ - if(!gc_match_kind(gc, MAJOR_COLLECTION)) return; - /* release local chunks of each collector in gen GC */ - for(unsigned int i = gc->num_collectors; i--;){ - allocator_clear_local_chunks((Allocator*)gc->collectors[i]); - } -} - -#ifdef USE_MARK_SWEEP_GC -void sspace_set_space_statistic(Sspace *sspace) -{ - GC_MS *gc = (GC_MS*)sspace->gc; - - for(unsigned int i = 0; i < gc->num_collectors; ++i){ - sspace->surviving_obj_num += gc->collectors[i]->live_obj_num; - sspace->surviving_obj_size += gc->collectors[i]->live_obj_size; - } -} -#endif - -extern void sspace_decide_compaction_need(Sspace *sspace); -extern void mark_sweep_sspace(Collector *collector); - -void sspace_collection(Sspace *sspace) -{ - GC *gc = sspace->gc; - sspace->num_collections++; - - gc_clear_mutator_local_chunks(gc); - gc_clear_collector_local_chunks(gc); - -#ifdef SSPACE_ALLOC_INFO - sspace_alloc_info_summary(); -#endif -#ifdef SSPACE_CHUNK_INFO - sspace_chunks_info(sspace, FALSE); -#endif - - sspace_decide_compaction_need(sspace); - if(sspace->need_compact && gc_match_kind(gc, MARK_SWEEP_GC)){ - assert(gc_match_kind(gc, MS_COLLECTION)); - gc->collect_kind = MS_COMPACT_COLLECTION; - } - if(sspace->need_compact || gc_match_kind(gc, MAJOR_COLLECTION)) - sspace->need_fix = TRUE; - - //printf("\n\n>>>>>>>>%s>>>>>>>>>>>>\n\n", sspace->need_compact ? "COMPACT" : "NO COMPACT"); -#ifdef SSPACE_VERIFY - sspace_verify_before_collection(gc); - sspace_verify_vtable_mark(gc); -#endif - -#ifdef SSPACE_TIME - sspace_gc_time(gc, TRUE); -#endif - - pool_iterator_init(gc->metadata->gc_rootset_pool); - sspace_clear_chunk_list(gc); - - collector_execute_task(gc, (TaskType)mark_sweep_sspace, (Space*)sspace); - -#ifdef SSPACE_TIME - sspace_gc_time(gc, FALSE); -#endif - -#ifdef SSPACE_CHUNK_INFO - sspace_chunks_info(sspace, FALSE); -#endif - -} Index: src/mark_sweep/sspace.h =================================================================== --- src/mark_sweep/sspace.h (revision 606795) +++ src/mark_sweep/sspace.h (working copy) @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _SWEEP_SPACE_H_ -#define _SWEEP_SPACE_H_ - -#include "../thread/gc_thread.h" -#include "../thread/collector_alloc.h" -#include "../thread/mutator.h" -#include "../common/gc_common.h" - -/* - * The sweep space accomodates objects collected by mark-sweep - */ - -struct Size_Segment; -struct Free_Chunk_List; - -typedef struct Sspace { - /* <-- first couple of fields are overloadded as Space */ - void *heap_start; - void *heap_end; - POINTER_SIZE_INT reserved_heap_size; - POINTER_SIZE_INT committed_heap_size; - unsigned int num_collections; - int64 time_collections; - float survive_ratio; - unsigned int collect_algorithm; - GC *gc; - Boolean move_object; - - Space_Statistics* space_statistic; - - /* Size allocted since last minor collection. */ - volatile POINTER_SIZE_INT last_alloced_size; - /* Size allocted since last major collection. */ - volatile POINTER_SIZE_INT accumu_alloced_size; - /* Total size allocated since VM starts. */ - volatile POINTER_SIZE_INT total_alloced_size; - - /* Size survived from last collection. */ - POINTER_SIZE_INT last_surviving_size; - /* Size survived after a certain period. */ - POINTER_SIZE_INT period_surviving_size; - - /* END of Space --> */ - - Boolean need_compact; - Boolean need_fix; /* There are repointed ref needing fixing */ - Size_Segment **size_segments; - Pool ***pfc_pools; - Free_Chunk_List *aligned_free_chunk_lists; - Free_Chunk_List *unaligned_free_chunk_lists; - Free_Chunk_List *hyper_free_chunk_list; - POINTER_SIZE_INT surviving_obj_num; - POINTER_SIZE_INT surviving_obj_size; -} Sspace; - -#ifdef USE_MARK_SWEEP_GC -void sspace_set_space_statistic(Sspace *sspace); -#endif - -Sspace *sspace_initialize(GC *gc, void *start, POINTER_SIZE_INT sspace_size, POINTER_SIZE_INT commit_size); -void sspace_destruct(Sspace *sspace); -void sspace_reset_after_collection(Sspace *sspace); - -void *sspace_thread_local_alloc(unsigned size, Allocator *allocator); -void *sspace_alloc(unsigned size, Allocator *allocator); - -void sspace_collection(Sspace *sspace); - -void allocator_init_local_chunks(Allocator *allocator); -void allocactor_destruct_local_chunks(Allocator *allocator); -void collector_init_free_chunk_list(Collector *collector); - -POINTER_SIZE_INT sspace_free_memory_size(Sspace *sspace); - - -#ifndef USE_MARK_SWEEP_GC -#define gc_get_sspace(gc) ((Sspace*)gc_get_mos((GC_Gen*)(gc))) -#else -#define gc_get_sspace(gc) (gc_ms_get_sspace((GC_MS*)(gc))); -#endif - -#endif // _SWEEP_SPACE_H_ Index: src/mark_sweep/sspace_alloc.cpp =================================================================== --- src/mark_sweep/sspace_alloc.cpp (revision 606795) +++ src/mark_sweep/sspace_alloc.cpp (working copy) @@ -1,225 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sspace.h" -#include "sspace_chunk.h" -//#include "sspace_mark_sweep.h" -#include "sspace_alloc.h" -#include "gc_ms.h" -#include "../gen/gen.h" - - -/* Only used in pfc_set_slot_index() */ -inline unsigned int first_free_index_in_color_word(POINTER_SIZE_INT word, POINTER_SIZE_INT alloc_color) -{ - for(unsigned int index = 0; index < BITS_PER_WORD; index += COLOR_BITS_PER_OBJ) - if(!(word & (alloc_color << index))) - return index; - - assert(0); /* There must be a free obj in this table word */ - return MAX_SLOT_INDEX; -} - -/* Given an index word in table, set pfc's slot_index - * The value of argument alloc_color can be cur_alloc_color or cur_mark_color. - * It depends on in which phase this func is called. - * In sweeping phase, sspace has been marked but alloc and mark colors have not been flipped, - * so we have to use cur_mark_color as alloc_color. - * In compaction phase, two colors have been flipped, so we use cur_alloc_color. - */ -void pfc_set_slot_index(Chunk_Header *chunk, unsigned int first_free_word_index, POINTER_SIZE_INT alloc_color) -{ - unsigned int index_in_word = first_free_index_in_color_word(chunk->table[first_free_word_index], alloc_color); - assert(index_in_word != MAX_SLOT_INDEX); - chunk->slot_index = composed_slot_index(first_free_word_index, index_in_word); -} - -/* From the table's beginning search the first free slot, and set it to pfc's slot_index */ -void pfc_reset_slot_index(Chunk_Header *chunk) -{ - POINTER_SIZE_INT *table = chunk->table; - - unsigned int index_word_num = (chunk->slot_num + SLOT_NUM_PER_WORD_IN_TABLE - 1) / SLOT_NUM_PER_WORD_IN_TABLE; - for(unsigned int i=0; i LARGE_OBJ_THRESHOLD) return NULL; - - Sspace *sspace = gc_get_sspace(allocator->gc); - - /* Flexible alloc mechanism: - Size_Segment *size_seg = sspace_get_size_seg(sspace, size); - unsigned int seg_index = size_seg->seg_index; - */ - unsigned int seg_index = (size-GC_OBJECT_ALIGNMENT) / MEDIUM_OBJ_THRESHOLD; - assert(seg_index <= 2); - Size_Segment *size_seg = sspace->size_segments[seg_index]; - assert(size_seg->local_alloc); - - size = (unsigned int)NORMAL_SIZE_ROUNDUP(size, size_seg); - unsigned int index = NORMAL_SIZE_TO_INDEX(size, size_seg); - - Chunk_Header **chunks = allocator->local_chunks[seg_index]; - Chunk_Header *chunk = chunks[index]; - if(!chunk){ - chunk = sspace_get_pfc(sspace, seg_index, index); - //if(!chunk) chunk = sspace_steal_pfc(sspace, seg_index, index); - if(!chunk) return NULL; - chunk->status |= CHUNK_IN_USE; - chunks[index] = chunk; - } - void *p_obj = alloc_in_chunk(chunks[index]); - assert(p_obj); - -#ifdef SSPACE_ALLOC_INFO - sspace_alloc_info(size); -#endif -#ifdef SSPACE_VERIFY - sspace_verify_alloc(p_obj, size); -#endif - - return p_obj; -} - -static void *sspace_alloc_normal_obj(Sspace *sspace, unsigned size, Allocator *allocator) -{ - Size_Segment *size_seg = sspace_get_size_seg(sspace, size); - unsigned int seg_index = size_seg->seg_index; - - size = (unsigned int)NORMAL_SIZE_ROUNDUP(size, size_seg); - unsigned int index = NORMAL_SIZE_TO_INDEX(size, size_seg); - - Chunk_Header *chunk = NULL; - void *p_obj = NULL; - - if(size_seg->local_alloc){ - Chunk_Header **chunks = allocator->local_chunks[seg_index]; - chunk = chunks[index]; - if(!chunk){ - chunk = sspace_get_pfc(sspace, seg_index, index); - if(!chunk){ - chunk = (Chunk_Header*)sspace_get_normal_free_chunk(sspace); - if(chunk) normal_chunk_init(chunk, size); - } - //if(!chunk) chunk = sspace_steal_pfc(sspace, seg_index, index); - if(!chunk) return NULL; - chunk->status |= CHUNK_IN_USE; - chunks[index] = chunk; - } - p_obj = alloc_in_chunk(chunks[index]); - } else { - if(gc_need_start_concurrent_mark(allocator->gc)) - gc_start_concurrent_mark(allocator->gc); - chunk = sspace_get_pfc(sspace, seg_index, index); - if(!chunk){ - chunk = (Chunk_Header*)sspace_get_normal_free_chunk(sspace); - if(chunk) normal_chunk_init(chunk, size); - } - //if(!chunk) chunk = sspace_steal_pfc(sspace, seg_index, index); - if(!chunk) return NULL; - p_obj = alloc_in_chunk(chunk); - if(chunk) - sspace_put_pfc(sspace, chunk); - } - - return p_obj; -} - -static void *sspace_alloc_super_obj(Sspace *sspace, unsigned size, Allocator *allocator) -{ - assert(size > SUPER_OBJ_THRESHOLD); - - if(gc_need_start_concurrent_mark(allocator->gc)) - gc_start_concurrent_mark(allocator->gc); - - unsigned int chunk_size = SUPER_SIZE_ROUNDUP(size); - assert(chunk_size > SUPER_OBJ_THRESHOLD); - assert(!(chunk_size & CHUNK_GRANULARITY_LOW_MASK)); - - Chunk_Header *chunk; - if(chunk_size <= HYPER_OBJ_THRESHOLD) - chunk = (Chunk_Header*)sspace_get_abnormal_free_chunk(sspace, chunk_size); - else - chunk = (Chunk_Header*)sspace_get_hyper_free_chunk(sspace, chunk_size, FALSE); - - if(!chunk) return NULL; - abnormal_chunk_init(chunk, chunk_size, size); - chunk->table[0] = cur_alloc_color; - set_super_obj_mask(chunk->base); - assert(get_obj_info_raw((Partial_Reveal_Object*)chunk->base) & SUPER_OBJ_MASK); - //printf("Obj: %x size: %x\t", (POINTER_SIZE_INT)chunk->base, size); - return chunk->base; -} - -static void *sspace_try_alloc(unsigned size, Allocator *allocator) -{ - Sspace *sspace = gc_get_sspace(allocator->gc); - void *p_obj = NULL; - - if(size <= SUPER_OBJ_THRESHOLD) - p_obj = sspace_alloc_normal_obj(sspace, size, allocator); - else - p_obj = sspace_alloc_super_obj(sspace, size, allocator); - -#ifdef SSPACE_ALLOC_INFO - if(p_obj) sspace_alloc_info(size); -#endif -#ifdef SSPACE_VERIFY - if(p_obj) sspace_verify_alloc(p_obj, size); -#endif - - if(p_obj && gc_is_concurrent_mark_phase()) obj_mark_black_in_table((Partial_Reveal_Object*)p_obj,size); - return p_obj; -} - -/* FIXME:: the collection should be seperated from the alloation */ -void *sspace_alloc(unsigned size, Allocator *allocator) -{ - void *p_obj = NULL; - - /* First, try to allocate object from TLB (thread local chunk) */ - p_obj = sspace_try_alloc(size, allocator); - if(p_obj) return p_obj; - - if(allocator->gc->in_collection) return NULL; - - vm_gc_lock_enum(); - /* after holding lock, try if other thread collected already */ - p_obj = sspace_try_alloc(size, allocator); - if(p_obj){ - vm_gc_unlock_enum(); - return p_obj; - } - gc_reclaim_heap(allocator->gc, GC_CAUSE_SSPACE_IS_FULL); - vm_gc_unlock_enum(); - -#ifdef SSPACE_CHUNK_INFO - printf("Failure size: %x\n", size); -#endif - - p_obj = sspace_try_alloc(size, allocator); - - return p_obj; -} Index: src/mark_sweep/sspace_alloc.h =================================================================== --- src/mark_sweep/sspace_alloc.h (revision 606795) +++ src/mark_sweep/sspace_alloc.h (working copy) @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _SSPACE_ALLOC_H_ -#define _SSPACE_ALLOC_H_ - -#include "sspace_chunk.h" -#include "sspace_mark_sweep.h" -#include "../common/gc_concurrent.h" -#include "../common/collection_scheduler.h" - -extern POINTER_SIZE_INT cur_alloc_color; -extern POINTER_SIZE_INT cur_alloc_mask; -extern POINTER_SIZE_INT cur_mark_mask; - - -inline Boolean slot_is_alloc_in_table(POINTER_SIZE_INT *table, unsigned int slot_index) -{ - unsigned int color_bits_index = slot_index * COLOR_BITS_PER_OBJ; - unsigned int word_index = color_bits_index / BITS_PER_WORD; - unsigned int index_in_word = color_bits_index % BITS_PER_WORD; - - return (Boolean)(table[word_index] & (cur_alloc_color << index_in_word)); -} - -inline unsigned int composed_slot_index(unsigned int word_index, unsigned int index_in_word) -{ - unsigned int color_bits_index = word_index*BITS_PER_WORD + index_in_word; - return color_bits_index/COLOR_BITS_PER_OBJ; -} - -inline unsigned int next_free_index_in_color_word(POINTER_SIZE_INT word, unsigned int index) -{ - while(index < BITS_PER_WORD){ - if(!(word & (cur_alloc_color << index))) - return index; - index += COLOR_BITS_PER_OBJ; - } - return MAX_SLOT_INDEX; -} - -inline unsigned int next_alloc_index_in_color_word(POINTER_SIZE_INT word, unsigned int index) -{ - while(index < BITS_PER_WORD){ - if(word & (cur_alloc_color << index)) - return index; - index += COLOR_BITS_PER_OBJ; - } - return MAX_SLOT_INDEX; -} - -inline unsigned int next_free_slot_index_in_table(POINTER_SIZE_INT *table, unsigned int slot_index, unsigned int slot_num) -{ - assert(slot_is_alloc_in_table(table, slot_index)); - ++slot_index; - - unsigned int max_word_index = ((slot_num-1) * COLOR_BITS_PER_OBJ) / BITS_PER_WORD; - - unsigned int color_bits_index = slot_index * COLOR_BITS_PER_OBJ; - unsigned int word_index = color_bits_index / BITS_PER_WORD; - unsigned int index_in_word = color_bits_index % BITS_PER_WORD; - - for(; word_index <= max_word_index; ++word_index, index_in_word = 0){ - if(table[word_index] == cur_alloc_mask) - continue; - index_in_word = next_free_index_in_color_word(table[word_index], index_in_word); - if(index_in_word != MAX_SLOT_INDEX){ - assert(index_in_word < BITS_PER_WORD); - return composed_slot_index(word_index, index_in_word); - } - } - - return MAX_SLOT_INDEX; -} - -/* Only used in sspace compaction after sweeping now */ -inline unsigned int next_alloc_slot_index_in_table(POINTER_SIZE_INT *table, unsigned int slot_index, unsigned int slot_num) -{ - unsigned int max_word_index = ((slot_num-1) * COLOR_BITS_PER_OBJ) / BITS_PER_WORD; - - unsigned int color_bits_index = slot_index * COLOR_BITS_PER_OBJ; - unsigned int word_index = color_bits_index / BITS_PER_WORD; - unsigned int index_in_word = color_bits_index % BITS_PER_WORD; - - for(; word_index <= max_word_index; ++word_index, index_in_word = 0){ - if(!table[word_index]) - continue; - index_in_word = next_alloc_index_in_color_word(table[word_index], index_in_word); - if(index_in_word != MAX_SLOT_INDEX){ - assert(index_in_word < BITS_PER_WORD); - return composed_slot_index(word_index, index_in_word); - } - } - - return MAX_SLOT_INDEX; -} - -inline Partial_Reveal_Object *next_alloc_slot_in_chunk(Chunk_Header *chunk) -{ - POINTER_SIZE_INT *table = chunk->table; - - unsigned int slot_index = next_alloc_slot_index_in_table(table, chunk->slot_index, chunk->slot_num); - assert((slot_index == MAX_SLOT_INDEX) - || (slot_index < chunk->slot_num) && slot_is_alloc_in_table(table, slot_index)); - if(slot_index == MAX_SLOT_INDEX) - return NULL; - Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)slot_index_to_addr(chunk, slot_index); - chunk->slot_index = slot_index + 1; - return p_obj; -} - -inline void clear_free_slot_in_table(POINTER_SIZE_INT *table, unsigned int ceiling_slot_index) -{ - assert(ceiling_slot_index && ceiling_slot_index != MAX_SLOT_INDEX); - unsigned int index_word_num = ceiling_slot_index / SLOT_NUM_PER_WORD_IN_TABLE; - memset(table, 0, BYTES_PER_WORD*index_word_num); - unsigned int bits_need_clear = ceiling_slot_index % SLOT_NUM_PER_WORD_IN_TABLE; - if(!bits_need_clear) return; - POINTER_SIZE_INT bit_mask = ~(((POINTER_SIZE_INT)1 << (bits_need_clear*COLOR_BITS_PER_OBJ)) - 1); - table[index_word_num] &= bit_mask; -} - -inline void alloc_slot_in_table(POINTER_SIZE_INT *table, unsigned int slot_index) -{ - assert(!slot_is_alloc_in_table(table, slot_index)); - - unsigned int color_bits_index = slot_index * COLOR_BITS_PER_OBJ; - unsigned int word_index = color_bits_index / BITS_PER_WORD; - unsigned int index_in_word = color_bits_index % BITS_PER_WORD; - - table[word_index] |= cur_alloc_color << index_in_word; -} - -/* We don't enable fresh chunk alloc for now, - * because we observed perf down for the extra conditional statement when no many fresh chunks. - */ -//#define ENABLE_FRESH_CHUNK_ALLOC - -/* 1. No need of synchronization. This is an allocator local chunk. - * 2. If this chunk runs out of space, clear the chunk pointer. - * So it is important to give a parameter which is a local chunk pointer of a allocator while invoking this func. - */ -inline void *alloc_in_chunk(Chunk_Header* &chunk) -{ - POINTER_SIZE_INT *table = chunk->table; - unsigned int slot_index = chunk->slot_index; - - assert(chunk->alloc_num < chunk->slot_num); - ++chunk->alloc_num; - assert(chunk->base); - void *p_obj = (void*)((POINTER_SIZE_INT)chunk->base + ((POINTER_SIZE_INT)chunk->slot_size * slot_index)); - alloc_slot_in_table(table, slot_index); - if(chunk->status & CHUNK_NEED_ZEROING) - memset(p_obj, 0, chunk->slot_size); -#ifdef SSPACE_VERIFY - sspace_verify_free_area((POINTER_SIZE_INT*)p_obj, chunk->slot_size); -#endif - - if(p_obj && gc_is_concurrent_mark_phase()) - obj_mark_black_in_table((Partial_Reveal_Object*)p_obj, chunk->slot_size); - -#ifdef ENABLE_FRESH_CHUNK_ALLOC - if(chunk->status & CHUNK_FRESH){ - ++slot_index; - chunk->slot_index = (slot_index < chunk->slot_num) ? slot_index : MAX_SLOT_INDEX; - } else -#endif - chunk->slot_index = next_free_slot_index_in_table(table, slot_index, chunk->slot_num); - if(chunk->slot_index == MAX_SLOT_INDEX){ - chunk->status = CHUNK_USED | CHUNK_NORMAL; - chunk = NULL; - } - - assert(!chunk || chunk->slot_index < chunk->slot_num); - return p_obj; -} - -inline void set_super_obj_mask(void *large_obj) -{ ((Partial_Reveal_Object*)large_obj)->obj_info |= SUPER_OBJ_MASK; } - -#endif // _SSPACE_ALLOC_H_ Index: src/mark_sweep/sspace_chunk.cpp =================================================================== --- src/mark_sweep/sspace_chunk.cpp (revision 606795) +++ src/mark_sweep/sspace_chunk.cpp (working copy) @@ -1,687 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sspace_chunk.h" -#include "sspace_verify.h" - -#define NUM_ALIGNED_FREE_CHUNK_BUCKET (HYPER_OBJ_THRESHOLD >> NORMAL_CHUNK_SHIFT_COUNT) -#define NUM_UNALIGNED_FREE_CHUNK_BUCKET (HYPER_OBJ_THRESHOLD >> CHUNK_GRANULARITY_BITS) - -/* PFC stands for partially free chunk */ -static Size_Segment *size_segments[SIZE_SEGMENT_NUM]; -static Pool **pfc_pools[SIZE_SEGMENT_NUM]; -static Boolean *pfc_steal_flags[SIZE_SEGMENT_NUM]; - -static Free_Chunk_List aligned_free_chunk_lists[NUM_ALIGNED_FREE_CHUNK_BUCKET]; -static Free_Chunk_List unaligned_free_chunk_lists[NUM_UNALIGNED_FREE_CHUNK_BUCKET]; -static Free_Chunk_List hyper_free_chunk_list; - - -static void init_size_segment(Size_Segment *seg, unsigned int size_min, unsigned int size_max, unsigned int gran_shift_bits, Boolean local_alloc) -{ - seg->size_min = size_min; - seg->size_max = size_max; - seg->local_alloc = local_alloc; - seg->chunk_num = (seg->size_max - seg->size_min) >> gran_shift_bits; - seg->gran_shift_bits = gran_shift_bits; - seg->granularity = (POINTER_SIZE_INT)(1 << gran_shift_bits); - seg->gran_low_mask = seg->granularity - 1; - seg->gran_high_mask = ~seg->gran_low_mask; -} - -void sspace_init_chunks(Sspace *sspace) -{ - unsigned int i, j; - - /* Init size segments */ - Size_Segment *size_seg_start = (Size_Segment*)STD_MALLOC(sizeof(Size_Segment) * SIZE_SEGMENT_NUM); - for(i = SIZE_SEGMENT_NUM; i--;){ - size_segments[i] = size_seg_start + i; - size_segments[i]->seg_index = i; - } - init_size_segment(size_segments[0], 0, MEDIUM_OBJ_THRESHOLD, SMALL_GRANULARITY_BITS, SMALL_IS_LOCAL_ALLOC); - init_size_segment(size_segments[1], MEDIUM_OBJ_THRESHOLD, LARGE_OBJ_THRESHOLD, MEDIUM_GRANULARITY_BITS, MEDIUM_IS_LOCAL_ALLOC); - init_size_segment(size_segments[2], LARGE_OBJ_THRESHOLD, SUPER_OBJ_THRESHOLD, LARGE_GRANULARITY_BITS, LARGE_IS_LOCAL_ALLOC); - - /* Init partially free chunk pools */ - for(i = SIZE_SEGMENT_NUM; i--;){ - pfc_pools[i] = (Pool**)STD_MALLOC(sizeof(Pool*) * size_segments[i]->chunk_num); - pfc_steal_flags[i] = (Boolean*)STD_MALLOC(sizeof(Boolean) * size_segments[i]->chunk_num); - for(j=size_segments[i]->chunk_num; j--;){ - pfc_pools[i][j] = sync_pool_create(); - pfc_steal_flags[i][j] = FALSE; - } - } - - /* Init aligned free chunk lists */ - for(i = NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) - free_chunk_list_init(&aligned_free_chunk_lists[i]); - - /* Init nonaligned free chunk lists */ - for(i = NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) - free_chunk_list_init(&unaligned_free_chunk_lists[i]); - - /* Init super free chunk lists */ - free_chunk_list_init(&hyper_free_chunk_list); - - sspace->size_segments = size_segments; - sspace->pfc_pools = pfc_pools; - sspace->aligned_free_chunk_lists = aligned_free_chunk_lists; - sspace->unaligned_free_chunk_lists = unaligned_free_chunk_lists; - sspace->hyper_free_chunk_list = &hyper_free_chunk_list; - - /* Init the first free chunk: from heap start to heap end */ - Free_Chunk *free_chunk = (Free_Chunk*)sspace->heap_start; - free_chunk->adj_next = (Chunk_Header_Basic*)sspace->heap_end; - POINTER_SIZE_INT chunk_size = sspace->reserved_heap_size; - assert(chunk_size > CHUNK_GRANULARITY && !(chunk_size % CHUNK_GRANULARITY)); - sspace_put_free_chunk(sspace, free_chunk); -} - -static void pfc_pool_set_steal_flag(Pool *pool, unsigned int steal_threshold, Boolean &steal_flag) -{ - Chunk_Header *chunk = (Chunk_Header*)pool_get_entry(pool); - while(chunk){ - steal_threshold--; - if(!steal_threshold) - break; - chunk = chunk->next; - } - steal_flag = steal_threshold ? FALSE : TRUE; -} - -void sspace_clear_chunk_list(GC *gc) -{ - unsigned int i, j; - unsigned int collector_num = gc->num_collectors; - unsigned int steal_threshold = collector_num << PFC_STEAL_THRESHOLD; - - for(i = SIZE_SEGMENT_NUM; i--;){ - for(j = size_segments[i]->chunk_num; j--;){ - Pool *pool = pfc_pools[i][j]; - pfc_pool_set_steal_flag(pool, steal_threshold, pfc_steal_flags[i][j]); - pool_empty(pool); - } - } - - for(i=NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) - free_chunk_list_clear(&aligned_free_chunk_lists[i]); - - for(i=NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) - free_chunk_list_clear(&unaligned_free_chunk_lists[i]); - - free_chunk_list_clear(&hyper_free_chunk_list); -} - -/* Simply put the free chunk to the according list - * Don't merge continuous free chunks - * The merging job is executed in merging phase - */ -static void list_put_free_chunk(Free_Chunk_List *list, Free_Chunk *chunk) -{ - chunk->status = CHUNK_FREE; - chunk->prev = NULL; - - lock(list->lock); - chunk->next = list->head; - if(list->head) - list->head->prev = chunk; - list->head = chunk; - if(!list->tail) - list->tail = chunk; - assert(list->chunk_num < ~((unsigned int)0)); - ++list->chunk_num; - unlock(list->lock); -} -/* The difference between this and the above normal one is this func needn't hold the list lock. - * This is for the calling in partitioning chunk functions. - * Please refer to the comments of sspace_get_hyper_free_chunk(). - */ -static void list_put_hyper_free_chunk(Free_Chunk_List *list, Free_Chunk *chunk) -{ - chunk->status = CHUNK_FREE; - chunk->prev = NULL; - - /* lock(list->lock); - * the list lock must have been held like in getting a free chunk and partitioning it - * or needn't be held like in sspace initialization and the merging phase - */ - chunk->next = list->head; - if(list->head) - list->head->prev = chunk; - list->head = chunk; - if(!list->tail) - list->tail = chunk; - assert(list->chunk_num < ~((unsigned int)0)); - ++list->chunk_num; - //unlock(list->lock); -} - -static Free_Chunk *free_list_get_head(Free_Chunk_List *list) -{ - lock(list->lock); - Free_Chunk *chunk = list->head; - if(chunk){ - list->head = chunk->next; - if(list->head) - list->head->prev = NULL; - else - list->tail = NULL; - assert(list->chunk_num); - --list->chunk_num; - assert(chunk->status == CHUNK_FREE); - } - unlock(list->lock); - return chunk; -} - -void sspace_put_free_chunk(Sspace *sspace, Free_Chunk *chunk) -{ - POINTER_SIZE_INT chunk_size = CHUNK_SIZE(chunk); - assert(!(chunk_size % CHUNK_GRANULARITY)); - - if(chunk_size > HYPER_OBJ_THRESHOLD) - list_put_hyper_free_chunk(sspace->hyper_free_chunk_list, chunk); - else if(!((POINTER_SIZE_INT)chunk & NORMAL_CHUNK_LOW_MASK) && !(chunk_size & NORMAL_CHUNK_LOW_MASK)) - list_put_free_chunk(&sspace->aligned_free_chunk_lists[ALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)], chunk); - else - list_put_free_chunk(&sspace->unaligned_free_chunk_lists[UNALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)], chunk); -} - -static inline Free_Chunk *partition_normal_free_chunk(Sspace *sspace, Free_Chunk *chunk) -{ - assert(CHUNK_SIZE(chunk) > NORMAL_CHUNK_SIZE_BYTES); - - Chunk_Header_Basic *adj_next = chunk->adj_next; - Free_Chunk *normal_chunk = (Free_Chunk*)(((POINTER_SIZE_INT)chunk + NORMAL_CHUNK_SIZE_BYTES-1) & NORMAL_CHUNK_HIGH_MASK); - - if(chunk != normal_chunk){ - assert(chunk < normal_chunk); - chunk->adj_next = (Chunk_Header_Basic*)normal_chunk; - sspace_put_free_chunk(sspace, chunk); - } - normal_chunk->adj_next = (Chunk_Header_Basic*)((POINTER_SIZE_INT)normal_chunk + NORMAL_CHUNK_SIZE_BYTES); - if(normal_chunk->adj_next != adj_next){ - assert(normal_chunk->adj_next < adj_next); - Free_Chunk *back_chunk = (Free_Chunk*)normal_chunk->adj_next; - back_chunk->adj_next = adj_next; - sspace_put_free_chunk(sspace, back_chunk); - } - - normal_chunk->status = CHUNK_FREE; - return normal_chunk; -} - -/* Partition the free chunk to two free chunks: - * the first one's size is chunk_size - * the second will be inserted into free chunk list according to its size - */ -static inline Free_Chunk *partition_abnormal_free_chunk(Sspace *sspace,Free_Chunk *chunk, unsigned int chunk_size) -{ - assert(CHUNK_SIZE(chunk) > chunk_size); - - Free_Chunk *new_chunk = (Free_Chunk*)((POINTER_SIZE_INT)chunk->adj_next - chunk_size); - assert(chunk < new_chunk); - - new_chunk->adj_next = chunk->adj_next; - chunk->adj_next = (Chunk_Header_Basic*)new_chunk; - sspace_put_free_chunk(sspace, chunk); - new_chunk->status = CHUNK_FREE; - return new_chunk; -} - -Free_Chunk *sspace_get_normal_free_chunk(Sspace *sspace) -{ - Free_Chunk_List *aligned_lists = sspace->aligned_free_chunk_lists; - Free_Chunk_List *unaligned_lists = sspace->unaligned_free_chunk_lists; - Free_Chunk_List *list = NULL; - Free_Chunk *chunk = NULL; - - /* Search in aligned chunk lists first */ - unsigned int index = 0; - while(index < NUM_ALIGNED_FREE_CHUNK_BUCKET){ - list = &aligned_lists[index]; - if(list->head) - chunk = free_list_get_head(list); - if(chunk){ - if(CHUNK_SIZE(chunk) > NORMAL_CHUNK_SIZE_BYTES) - chunk = partition_normal_free_chunk(sspace, chunk); - //zeroing_free_chunk(chunk); - return chunk; - } - index++; - } - assert(!chunk); - - /* Search in unaligned chunk lists with larger chunk. - (NORMAL_CHUNK_SIZE_BYTES + (NORMAL_CHUNK_SIZE_BYTES-CHUNK_GRANULARITY)) - is the smallest size which can guarantee the chunk includes a normal chunk. - */ - index = UNALIGNED_CHUNK_SIZE_TO_INDEX((NORMAL_CHUNK_SIZE_BYTES<<1) - CHUNK_GRANULARITY); - while(index < NUM_UNALIGNED_FREE_CHUNK_BUCKET){ - list = &unaligned_lists[index]; - if(list->head) - chunk = free_list_get_head(list); - if(chunk){ - chunk = partition_normal_free_chunk(sspace, chunk); - assert(!((POINTER_SIZE_INT)chunk & NORMAL_CHUNK_LOW_MASK)); - //zeroing_free_chunk(chunk); - return chunk; - } - index++; - } - assert(!chunk); - - /* search in the hyper free chunk list */ - chunk = sspace_get_hyper_free_chunk(sspace, NORMAL_CHUNK_SIZE_BYTES, TRUE); - assert(!((POINTER_SIZE_INT)chunk & NORMAL_CHUNK_LOW_MASK)); - - return chunk; -} - -Free_Chunk *sspace_get_abnormal_free_chunk(Sspace *sspace, unsigned int chunk_size) -{ - assert(chunk_size > CHUNK_GRANULARITY); - assert(!(chunk_size % CHUNK_GRANULARITY)); - assert(chunk_size <= HYPER_OBJ_THRESHOLD); - - Free_Chunk_List *unaligned_lists = sspace->unaligned_free_chunk_lists; - Free_Chunk_List *list = NULL; - Free_Chunk *chunk = NULL; - unsigned int index = 0; - - /* Search in the list with chunk size of multiple chunk_size */ - unsigned int search_size = chunk_size; - while(search_size <= HYPER_OBJ_THRESHOLD){ - index = UNALIGNED_CHUNK_SIZE_TO_INDEX(search_size); - list = &unaligned_lists[index]; - if(list->head) - chunk = free_list_get_head(list); - if(chunk){ - if(search_size > chunk_size) - chunk = partition_abnormal_free_chunk(sspace, chunk, chunk_size); - zeroing_free_chunk(chunk); - return chunk; - } - search_size += chunk_size; - } - assert(!chunk); - - /* search in the hyper free chunk list */ - chunk = sspace_get_hyper_free_chunk(sspace, chunk_size, FALSE); - if(chunk) return chunk; - - /* Search again in abnormal chunk lists */ - index = UNALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size); - while(index < NUM_UNALIGNED_FREE_CHUNK_BUCKET){ - list = &unaligned_lists[index]; - if(list->head) - chunk = free_list_get_head(list); - if(chunk){ - if(index > UNALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)) - chunk = partition_abnormal_free_chunk(sspace, chunk, chunk_size); - zeroing_free_chunk(chunk); - return chunk; - } - ++index; - } - - return chunk; -} - -Free_Chunk *sspace_get_hyper_free_chunk(Sspace *sspace, unsigned int chunk_size, Boolean is_normal_chunk) -{ - assert(chunk_size >= CHUNK_GRANULARITY); - assert(!(chunk_size % CHUNK_GRANULARITY)); - - Free_Chunk_List *list = sspace->hyper_free_chunk_list; - lock(list->lock); - - Free_Chunk *prev_chunk = NULL; - Free_Chunk *chunk = list->head; - while(chunk){ - if(CHUNK_SIZE(chunk) >= chunk_size){ - Free_Chunk *next_chunk = chunk->next; - if(prev_chunk) - prev_chunk->next = next_chunk; - else - list->head = next_chunk; - if(next_chunk) - next_chunk->prev = prev_chunk; - else - list->tail = prev_chunk; - break; - } - prev_chunk = chunk; - chunk = chunk->next; - } - - /* unlock(list->lock); - * We move this unlock to the end of this func for the following reason. - * A case might occur that two allocator are asking for a hyper chunk at the same time, - * and there is only one chunk in the list and it can satify the requirements of both of them. - * If allocator 1 gets the list lock first, it will get the unique chunk and releases the lock here. - * And then allocator 2 holds the list lock after allocator 1 releases it, - * it will found there is no hyper chunk in the list and return NULL. - * In fact the unique hyper chunk is large enough. - * If allocator 1 chops down one piece and put back the rest into the list, allocator 2 will be satisfied. - * So we will get a wrong info here if we release the lock here, which makes us invoke GC much earlier than needed. - */ - - if(chunk){ - if(is_normal_chunk) - chunk = partition_normal_free_chunk(sspace, chunk); - else if(CHUNK_SIZE(chunk) > chunk_size) - chunk = partition_abnormal_free_chunk(sspace, chunk, chunk_size); - if(!is_normal_chunk) - zeroing_free_chunk(chunk); - } - - unlock(list->lock); - - return chunk; -} - -void sspace_collect_free_chunks_to_list(Sspace *sspace, Free_Chunk_List *list) -{ - unsigned int i; - - for(i = NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) - move_free_chunks_between_lists(list, &sspace->aligned_free_chunk_lists[i]); - - for(i = NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) - move_free_chunks_between_lists(list, &sspace->unaligned_free_chunk_lists[i]); - - move_free_chunks_between_lists(list, sspace->hyper_free_chunk_list); - - Free_Chunk *chunk = list->head; - while(chunk){ - chunk->status = CHUNK_FREE | CHUNK_TO_MERGE; - chunk = chunk->next; - } -} - - -typedef struct PFC_Pool_Iterator { - volatile unsigned int seg_index; - volatile unsigned int chunk_index; - SpinLock lock; -} PFC_Pool_Iterator; - -static PFC_Pool_Iterator pfc_pool_iterator; - -void sspace_init_pfc_pool_iterator(Sspace *sspace) -{ - assert(pfc_pool_iterator.lock == FREE_LOCK); - pfc_pool_iterator.seg_index = 0; - pfc_pool_iterator.chunk_index = 0; -} - -Pool *sspace_grab_next_pfc_pool(Sspace *sspace) -{ - Pool *pfc_pool = NULL; - - lock(pfc_pool_iterator.lock); - for(; pfc_pool_iterator.seg_index < SIZE_SEGMENT_NUM; ++pfc_pool_iterator.seg_index){ - for(; pfc_pool_iterator.chunk_index < size_segments[pfc_pool_iterator.seg_index]->chunk_num; ++pfc_pool_iterator.chunk_index){ - pfc_pool = pfc_pools[pfc_pool_iterator.seg_index][pfc_pool_iterator.chunk_index]; - ++pfc_pool_iterator.chunk_index; - unlock(pfc_pool_iterator.lock); - return pfc_pool; - } - pfc_pool_iterator.chunk_index = 0; - } - unlock(pfc_pool_iterator.lock); - - return NULL; -} - -#define min_value(x, y) (((x) < (y)) ? (x) : (y)) - -Chunk_Header *sspace_steal_pfc(Sspace *sspace, unsigned int seg_index, unsigned int index) -{ - Size_Segment *size_seg = sspace->size_segments[seg_index]; - Chunk_Header *pfc = NULL; - unsigned int max_index = min_value(index + PFC_STEAL_NUM + 1, size_seg->chunk_num); - ++index; - for(; index < max_index; ++index){ - if(!pfc_steal_flags[seg_index][index]) continue; - pfc = sspace_get_pfc(sspace, seg_index, index); - if(pfc) return pfc; - } - return NULL; -} - -static POINTER_SIZE_INT free_mem_in_pfc_pools(Sspace *sspace, Boolean show_chunk_info) -{ - Size_Segment **size_segs = sspace->size_segments; - Pool ***pfc_pools = sspace->pfc_pools; - POINTER_SIZE_INT free_mem_size = 0; - - for(unsigned int i = 0; i < SIZE_SEGMENT_NUM; ++i){ - for(unsigned int j = 0; j < size_segs[i]->chunk_num; ++j){ - Pool *pfc_pool = pfc_pools[i][j]; - if(pool_is_empty(pfc_pool)) - continue; - pool_iterator_init(pfc_pool); - Chunk_Header *chunk = (Chunk_Header*)pool_iterator_next(pfc_pool); - assert(chunk); - unsigned int slot_num = chunk->slot_num; - unsigned int chunk_num = 0; - unsigned int alloc_num = 0; - while(chunk){ - assert(chunk->slot_num == slot_num); - ++chunk_num; - alloc_num += chunk->alloc_num; - chunk = (Chunk_Header*)pool_iterator_next(pfc_pool); - } - unsigned int total_slot_num = slot_num * chunk_num; - assert(alloc_num < total_slot_num); -#ifdef SSPACE_CHUNK_INFO - if(show_chunk_info) - printf("Size: %x\tchunk num: %d\tLive Ratio: %f\n", NORMAL_INDEX_TO_SIZE(j, size_segs[i]), chunk_num, (float)alloc_num/total_slot_num); -#endif - free_mem_size += NORMAL_INDEX_TO_SIZE(j, size_segs[i]) * (total_slot_num-alloc_num); - assert(free_mem_size < sspace->committed_heap_size); - } - } - - return free_mem_size; -} - -static POINTER_SIZE_INT free_mem_in_free_lists(Sspace *sspace, Free_Chunk_List *lists, unsigned int list_num, Boolean show_chunk_info) -{ - POINTER_SIZE_INT free_mem_size = 0; - - for(unsigned int index = 0; index < list_num; ++index){ - Free_Chunk *chunk = lists[index].head; - if(!chunk) continue; - POINTER_SIZE_INT chunk_size = CHUNK_SIZE(chunk); - assert(chunk_size <= HYPER_OBJ_THRESHOLD); - unsigned int chunk_num = 0; - while(chunk){ - assert(CHUNK_SIZE(chunk) == chunk_size); - ++chunk_num; - chunk = chunk->next; - } - free_mem_size += chunk_size * chunk_num; - assert(free_mem_size < sspace->committed_heap_size); -#ifdef SSPACE_CHUNK_INFO - if(show_chunk_info) - printf("Free Size: %x\tnum: %d\n", chunk_size, chunk_num); -#endif - } - - return free_mem_size; -} - -static POINTER_SIZE_INT free_mem_in_hyper_free_list(Sspace *sspace, Boolean show_chunk_info) -{ - POINTER_SIZE_INT free_mem_size = 0; - - Free_Chunk_List *list = sspace->hyper_free_chunk_list; - Free_Chunk *chunk = list->head; - while(chunk){ -#ifdef SSPACE_CHUNK_INFO - if(show_chunk_info) - printf("Size: %x\n", CHUNK_SIZE(chunk)); -#endif - free_mem_size += CHUNK_SIZE(chunk); - assert(free_mem_size < sspace->committed_heap_size); - chunk = chunk->next; - } - - return free_mem_size; -} - -POINTER_SIZE_INT free_mem_in_sspace(Sspace *sspace, Boolean show_chunk_info) -{ - POINTER_SIZE_INT free_mem_size = 0; - -#ifdef SSPACE_CHUNK_INFO - if(show_chunk_info) - printf("\n\nPFC INFO:\n\n"); -#endif - free_mem_size += free_mem_in_pfc_pools(sspace, show_chunk_info); - -#ifdef SSPACE_CHUNK_INFO - if(show_chunk_info) - printf("\n\nALIGNED FREE CHUNK INFO:\n\n"); -#endif - free_mem_size += free_mem_in_free_lists(sspace, aligned_free_chunk_lists, NUM_ALIGNED_FREE_CHUNK_BUCKET, show_chunk_info); - -#ifdef SSPACE_CHUNK_INFO - if(show_chunk_info) - printf("\n\nUNALIGNED FREE CHUNK INFO:\n\n"); -#endif - free_mem_size += free_mem_in_free_lists(sspace, unaligned_free_chunk_lists, NUM_UNALIGNED_FREE_CHUNK_BUCKET, show_chunk_info); - -#ifdef SSPACE_CHUNK_INFO - if(show_chunk_info) - printf("\n\nSUPER FREE CHUNK INFO:\n\n"); -#endif - free_mem_size += free_mem_in_hyper_free_list(sspace, show_chunk_info); - - return free_mem_size; -} - - -#ifdef SSPACE_CHUNK_INFO -void sspace_chunks_info(Sspace *sspace, Boolean show_info) -{ - if(!show_info) return; - - POINTER_SIZE_INT free_mem_size = free_mem_in_sspace(sspace, TRUE); - - float free_mem_ratio = (float)free_mem_size / sspace->committed_heap_size; - printf("\n\nFree mem ratio: %f\n\n", free_mem_ratio); -} -#endif - - -/* Because this computation doesn't use lock, its result is not accurate. And it is enough. */ -POINTER_SIZE_INT sspace_free_memory_size(Sspace *sspace) -{ - POINTER_SIZE_INT free_size = 0; - - vm_gc_lock_enum(); - /* - for(unsigned int i=NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) - free_size += NORMAL_CHUNK_SIZE_BYTES * (i+1) * sspace->aligned_free_chunk_lists[i].chunk_num; - - for(unsigned int i=NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) - free_size += CHUNK_GRANULARITY * (i+1) * sspace->unaligned_free_chunk_lists[i].chunk_num; - - Free_Chunk *hyper_chunk = sspace->hyper_free_chunk_list->head; - while(hyper_chunk){ - free_size += CHUNK_SIZE(hyper_chunk); - hyper_chunk = hyper_chunk->next; - } - */ - free_size = free_mem_in_sspace(sspace, FALSE); - vm_gc_unlock_enum(); - - return free_size; -} - - -#ifdef SSPACE_ALLOC_INFO - -#define MEDIUM_THRESHOLD 256 -#define LARGE_THRESHOLD (1024) -#define SUPER_THRESHOLD (6*KB) -#define HYPER_THRESHOLD (64*KB) - -#define SMALL_OBJ_ARRAY_NUM (MEDIUM_THRESHOLD >> 2) -#define MEDIUM_OBJ_ARRAY_NUM (LARGE_THRESHOLD >> 4) -#define LARGE_OBJ_ARRAY_NUM (SUPER_THRESHOLD >> 6) -#define SUPER_OBJ_ARRAY_NUM (HYPER_THRESHOLD >> 10) - -volatile unsigned int small_obj_num[SMALL_OBJ_ARRAY_NUM]; -volatile unsigned int medium_obj_num[MEDIUM_OBJ_ARRAY_NUM]; -volatile unsigned int large_obj_num[LARGE_OBJ_ARRAY_NUM]; -volatile unsigned int super_obj_num[SUPER_OBJ_ARRAY_NUM]; -volatile unsigned int hyper_obj_num; - -void sspace_alloc_info(unsigned int size) -{ - if(size <= MEDIUM_THRESHOLD) - atomic_inc32(&small_obj_num[(size>>2)-1]); - else if(size <= LARGE_THRESHOLD) - atomic_inc32(&medium_obj_num[(size>>4)-1]); - else if(size <= SUPER_THRESHOLD) - atomic_inc32(&large_obj_num[(size>>6)-1]); - else if(size <= HYPER_THRESHOLD) - atomic_inc32(&super_obj_num[(size>>10)-1]); - else - atomic_inc32(&hyper_obj_num); -} - -void sspace_alloc_info_summary(void) -{ - unsigned int i; - - printf("\n\nNORMAL OBJ\n\n"); - for(i = 0; i < SMALL_OBJ_ARRAY_NUM; i++){ - printf("Size: %x\tnum: %d\n", (i+1)<<2, small_obj_num[i]); - small_obj_num[i] = 0; - } - - i = ((MEDIUM_THRESHOLD + (1<<4))>>4) - 1; - for(; i < MEDIUM_OBJ_ARRAY_NUM; i++){ - printf("Size: %x\tnum: %d\n", (i+1)<<4, medium_obj_num[i]); - medium_obj_num[i] = 0; - } - - i = ((LARGE_THRESHOLD + (1<<6))>>6) - 1; - for(; i < LARGE_OBJ_ARRAY_NUM; i++){ - printf("Size: %x\tnum: %d\n", (i+1)<<6, large_obj_num[i]); - large_obj_num[i] = 0; - } - - i = ((SUPER_THRESHOLD + (1<<10))>>10) - 1; - for(; i < SUPER_OBJ_ARRAY_NUM; i++){ - printf("Size: %x\tnum: %d\n", (i+1)<<10, super_obj_num[i]); - super_obj_num[i] = 0; - } - - printf("\n\nHYPER OBJ\n\n"); - printf("num: %d\n", hyper_obj_num); - hyper_obj_num = 0; -} - -#endif Index: src/mark_sweep/sspace_chunk.h =================================================================== --- src/mark_sweep/sspace_chunk.h (revision 606795) +++ src/mark_sweep/sspace_chunk.h (working copy) @@ -1,359 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _SSPACE_CHUNK_H_ -#define _SSPACE_CHUNK_H_ - -#include "sspace.h" - -enum Chunk_Status { - CHUNK_NIL = 0, - CHUNK_FREE = 0x1, - CHUNK_FRESH = 0x2, - CHUNK_NORMAL = 0x10, - CHUNK_ABNORMAL = 0x20, - CHUNK_NEED_ZEROING = 0x100, - CHUNK_TO_MERGE = 0x200, - CHUNK_IN_USE = 0x400, /* just keep info for now, not used */ - CHUNK_USED = 0x800 /* just keep info for now, not used */ -}; - -typedef volatile POINTER_SIZE_INT Chunk_Status_t; - -typedef struct Chunk_Header_Basic { - Chunk_Header_Basic *next; - Chunk_Header_Basic *prev; - Chunk_Status_t status; - Chunk_Header_Basic *adj_next; // adjacent next chunk - Chunk_Header_Basic *adj_prev; // adjacent previous chunk, for merging continuous free chunks -} Chunk_Header_Basic; - -typedef struct Chunk_Header { - /* Beginning of Chunk_Header_Basic */ - Chunk_Header *next; /* pointing to the next pfc in the pfc pool */ - Chunk_Header *prev; /* pointing to the prev pfc in the pfc pool */ - Chunk_Status_t status; - Chunk_Header_Basic *adj_next; // adjacent next chunk - Chunk_Header_Basic *adj_prev; // adjacent previous chunk, for merging continuous free chunks - /* End of Chunk_Header_Basic */ - void *base; - unsigned int slot_size; - unsigned int slot_num; - unsigned int slot_index; /* the index of which is the first free slot in this chunk */ - unsigned int alloc_num; /* the index of which is the first free slot in this chunk */ - POINTER_SIZE_INT table[1]; -} Chunk_Header; - - -#define NORMAL_CHUNK_SHIFT_COUNT 16 -#define NORMAL_CHUNK_SIZE_BYTES (1 << NORMAL_CHUNK_SHIFT_COUNT) -#define NORMAL_CHUNK_LOW_MASK ((POINTER_SIZE_INT)(NORMAL_CHUNK_SIZE_BYTES - 1)) -#define NORMAL_CHUNK_HIGH_MASK (~NORMAL_CHUNK_LOW_MASK) -#define NORMAL_CHUNK_HEADER(addr) ((Chunk_Header*)((POINTER_SIZE_INT)(addr) & NORMAL_CHUNK_HIGH_MASK)) -#define ABNORMAL_CHUNK_HEADER(addr) ((Chunk_Header*)((POINTER_SIZE_INT)addr & CHUNK_GRANULARITY_HIGH_MASK)) - -#define MAX_SLOT_INDEX 0xFFffFFff -#define COLOR_BITS_PER_OBJ 4 // should be powers of 2 -#define SLOT_NUM_PER_WORD_IN_TABLE (BITS_PER_WORD /COLOR_BITS_PER_OBJ) - -/* Two equations: - * 1. CHUNK_HEADER_VARS_SIZE_BYTES + NORMAL_CHUNK_TABLE_SIZE_BYTES + slot_size*NORMAL_CHUNK_SLOT_NUM = NORMAL_CHUNK_SIZE_BYTES - * 2. (BITS_PER_BYTE * NORMAL_CHUNK_TABLE_SIZE_BYTES)/COLOR_BITS_PER_OBJ >= NORMAL_CHUNK_SLOT_NUM - * ===> - * NORMAL_CHUNK_SLOT_NUM <= BITS_PER_BYTE*(NORMAL_CHUNK_SIZE_BYTES - CHUNK_HEADER_VARS_SIZE_BYTES) / (BITS_PER_BYTE*slot_size + COLOR_BITS_PER_OBJ) - * ===> - * NORMAL_CHUNK_SLOT_NUM = BITS_PER_BYTE*(NORMAL_CHUNK_SIZE_BYTES - CHUNK_HEADER_VARS_SIZE_BYTES) / (BITS_PER_BYTE*slot_size + COLOR_BITS_PER_OBJ) - */ - -#define CHUNK_HEADER_VARS_SIZE_BYTES ((POINTER_SIZE_INT)&(((Chunk_Header*)0)->table)) -#define NORMAL_CHUNK_SLOT_AREA_SIZE_BITS (BITS_PER_BYTE * (NORMAL_CHUNK_SIZE_BYTES - CHUNK_HEADER_VARS_SIZE_BYTES)) -#define SIZE_BITS_PER_SLOT(chunk) (BITS_PER_BYTE * chunk->slot_size + COLOR_BITS_PER_OBJ) - -#define NORMAL_CHUNK_SLOT_NUM(chunk) (NORMAL_CHUNK_SLOT_AREA_SIZE_BITS / SIZE_BITS_PER_SLOT(chunk)) -#define NORMAL_CHUNK_TABLE_SIZE_BYTES(chunk) (((NORMAL_CHUNK_SLOT_NUM(chunk) + SLOT_NUM_PER_WORD_IN_TABLE-1) / SLOT_NUM_PER_WORD_IN_TABLE) * BYTES_PER_WORD) -#define NORMAL_CHUNK_HEADER_SIZE_BYTES(chunk) (CHUNK_HEADER_VARS_SIZE_BYTES + NORMAL_CHUNK_TABLE_SIZE_BYTES(chunk)) - -#define NORMAL_CHUNK_BASE(chunk) ((void*)((POINTER_SIZE_INT)(chunk) + NORMAL_CHUNK_HEADER_SIZE_BYTES(chunk))) -#define ABNORMAL_CHUNK_BASE(chunk) ((void*)((POINTER_SIZE_INT)(chunk) + sizeof(Chunk_Header))) - -#define CHUNK_END(chunk) ((chunk)->adj_next) -#define CHUNK_SIZE(chunk) ((POINTER_SIZE_INT)chunk->adj_next - (POINTER_SIZE_INT)chunk) - - -inline void *slot_index_to_addr(Chunk_Header *chunk, unsigned int index) -{ return (void*)((POINTER_SIZE_INT)chunk->base + chunk->slot_size * index); } - -inline unsigned int slot_addr_to_index(Chunk_Header *chunk, void *addr) -{ return (unsigned int)(((POINTER_SIZE_INT)addr - (POINTER_SIZE_INT)chunk->base) / chunk->slot_size); } - -typedef struct Free_Chunk { - /* Beginning of Chunk_Header_Basic */ - Free_Chunk *next; /* pointing to the next free Free_Chunk */ - Free_Chunk *prev; /* pointing to the prev free Free_Chunk */ - Chunk_Status_t status; - Chunk_Header_Basic *adj_next; // adjacent next chunk - Chunk_Header_Basic *adj_prev; // adjacent previous chunk, for merging continuous free chunks - /* End of Chunk_Header_Basic */ -} Free_Chunk; - -typedef struct Free_Chunk_List { - Free_Chunk *head; /* get new free chunk from head */ - Free_Chunk *tail; /* put free chunk to tail */ - unsigned int chunk_num; - SpinLock lock; -} Free_Chunk_List; - -/* -typedef union Chunk{ - Chunk_Header header; - Free_Chunk free_chunk; - unsigned char raw_bytes[NORMAL_CHUNK_SIZE_BYTES]; -} Chunk; -*/ - -inline void free_chunk_list_init(Free_Chunk_List *list) -{ - list->head = NULL; - list->tail = NULL; - list->chunk_num = 0; - list->lock = FREE_LOCK; -} - -inline void free_chunk_list_clear(Free_Chunk_List *list) -{ - list->head = NULL; - list->tail = NULL; - list->chunk_num = 0; - assert(list->lock == FREE_LOCK); -} - -inline void free_list_detach_chunk(Free_Chunk_List *list, Free_Chunk *chunk) -{ - if(chunk->prev) - chunk->prev->next = chunk->next; - else // chunk is the head - list->head = chunk->next; - if(chunk->next) - chunk->next->prev = chunk->prev; -} - -inline void move_free_chunks_between_lists(Free_Chunk_List *to_list, Free_Chunk_List *from_list) -{ - if(to_list->tail){ - to_list->head->prev = from_list->tail; - } else { - to_list->tail = from_list->tail; - } - if(from_list->head){ - from_list->tail->next = to_list->head; - to_list->head = from_list->head; - } - from_list->head = NULL; - from_list->tail = NULL; -} - -/* Padding the last index word in table to facilitate allocation */ -inline void chunk_pad_last_index_word(Chunk_Header *chunk, POINTER_SIZE_INT alloc_mask) -{ - unsigned int ceiling_index_in_last_word = (chunk->slot_num * COLOR_BITS_PER_OBJ) % BITS_PER_WORD; - if(!ceiling_index_in_last_word) - return; - POINTER_SIZE_INT padding_mask = ~((1 << ceiling_index_in_last_word) - 1); - padding_mask &= alloc_mask; - unsigned int last_word_index = (chunk->slot_num-1) / SLOT_NUM_PER_WORD_IN_TABLE; - chunk->table[last_word_index] |= padding_mask; -} - -/* Depadding the last index word in table to facilitate allocation */ -inline void chunk_depad_last_index_word(Chunk_Header *chunk) -{ - unsigned int ceiling_index_in_last_word = (chunk->slot_num * COLOR_BITS_PER_OBJ) % BITS_PER_WORD; - if(!ceiling_index_in_last_word) - return; - POINTER_SIZE_INT depadding_mask = (1 << ceiling_index_in_last_word) - 1; - unsigned int last_word_index = (chunk->slot_num-1) / SLOT_NUM_PER_WORD_IN_TABLE; - chunk->table[last_word_index] &= depadding_mask; -} - -extern POINTER_SIZE_INT cur_alloc_mask; -/* Used for allocating a fixed-size chunk from free area lists */ -inline void normal_chunk_init(Chunk_Header *chunk, unsigned int slot_size) -{ - assert(chunk->status == CHUNK_FREE); - assert(CHUNK_SIZE(chunk) == NORMAL_CHUNK_SIZE_BYTES); - - chunk->next = NULL; - chunk->status = CHUNK_FRESH | CHUNK_NORMAL | CHUNK_NEED_ZEROING; - chunk->slot_size = slot_size; - chunk->slot_num = NORMAL_CHUNK_SLOT_NUM(chunk); - chunk->slot_index = 0; - chunk->alloc_num = 0; - chunk->base = NORMAL_CHUNK_BASE(chunk); - memset(chunk->table, 0, NORMAL_CHUNK_TABLE_SIZE_BYTES(chunk));//memset table - chunk_pad_last_index_word(chunk, cur_alloc_mask); -} - -/* Used for allocating a chunk for large object from free area lists */ -inline void abnormal_chunk_init(Chunk_Header *chunk, unsigned int chunk_size, unsigned int obj_size) -{ - assert(chunk->status == CHUNK_FREE); - assert(CHUNK_SIZE(chunk) == chunk_size); - - chunk->next = NULL; - chunk->status = CHUNK_ABNORMAL; - chunk->slot_size = obj_size; - chunk->slot_num = 1; - chunk->slot_index = 0; - chunk->base = ABNORMAL_CHUNK_BASE(chunk); -} - - -#ifdef POINTER64 - #define GC_OBJECT_ALIGNMENT_BITS 3 -#else - #define GC_OBJECT_ALIGNMENT_BITS 2 -#endif - -#define MEDIUM_OBJ_THRESHOLD (128) -#define LARGE_OBJ_THRESHOLD (256) -#define SUPER_OBJ_THRESHOLD (1024) -#define HYPER_OBJ_THRESHOLD (128*KB) - -#define SMALL_GRANULARITY_BITS (GC_OBJECT_ALIGNMENT_BITS) -#define MEDIUM_GRANULARITY_BITS (SMALL_GRANULARITY_BITS + 1) -#define LARGE_GRANULARITY_BITS 7 -#define CHUNK_GRANULARITY_BITS 10 - -#define CHUNK_GRANULARITY (1 << CHUNK_GRANULARITY_BITS) -#define CHUNK_GRANULARITY_LOW_MASK ((POINTER_SIZE_INT)(CHUNK_GRANULARITY-1)) -#define CHUNK_GRANULARITY_HIGH_MASK (~CHUNK_GRANULARITY_LOW_MASK) - -#define SMALL_IS_LOCAL_ALLOC TRUE -#define MEDIUM_IS_LOCAL_ALLOC TRUE -#define LARGE_IS_LOCAL_ALLOC FALSE - -#define NORMAL_SIZE_ROUNDUP(size, seg) (((size) + seg->granularity-1) & seg->gran_high_mask) -#define SUPER_OBJ_TOTAL_SIZE(size) (sizeof(Chunk_Header) + (size)) -#define SUPER_SIZE_ROUNDUP(size) ((SUPER_OBJ_TOTAL_SIZE(size) + CHUNK_GRANULARITY-1) & CHUNK_GRANULARITY_HIGH_MASK) - -#define NORMAL_SIZE_TO_INDEX(size, seg) ((((size)-(seg)->size_min) >> (seg)->gran_shift_bits) - 1) -#define ALIGNED_CHUNK_SIZE_TO_INDEX(size) (((size) >> NORMAL_CHUNK_SHIFT_COUNT) - 1) -#define UNALIGNED_CHUNK_SIZE_TO_INDEX(size) (((size) >> CHUNK_GRANULARITY_BITS) - 1) - -#define NORMAL_INDEX_TO_SIZE(index, seg) ((((index) + 1) << (seg)->gran_shift_bits) + (seg)->size_min) -#define ALIGNED_CHUNK_INDEX_TO_SIZE(index) (((index) + 1) << NORMAL_CHUNK_SHIFT_COUNT) -#define UNALIGNED_CHUNK_INDEX_TO_SIZE(index) (((index) + 1) << CHUNK_GRANULARITY_BITS) - -#define SUPER_OBJ_MASK ((Obj_Info_Type)0x20) /* the 4th bit in obj info */ - -#define PFC_STEAL_NUM 3 -#define PFC_STEAL_THRESHOLD 3 - - -#define SIZE_SEGMENT_NUM 3 - -typedef struct Size_Segment { - unsigned int size_min; - unsigned int size_max; - unsigned int seg_index; - Boolean local_alloc; - unsigned int chunk_num; - unsigned int gran_shift_bits; - POINTER_SIZE_INT granularity; - POINTER_SIZE_INT gran_low_mask; - POINTER_SIZE_INT gran_high_mask; -} Size_Segment; - -inline Size_Segment *sspace_get_size_seg(Sspace *sspace, unsigned int size) -{ - Size_Segment **size_segs = sspace->size_segments; - - unsigned int seg_index = 0; - for(; seg_index < SIZE_SEGMENT_NUM; ++seg_index) - if(size <= size_segs[seg_index]->size_max) break; - assert(seg_index < SIZE_SEGMENT_NUM); - assert(size_segs[seg_index]->seg_index == seg_index); - return size_segs[seg_index]; -} - -inline Chunk_Header *sspace_get_pfc(Sspace *sspace, unsigned int seg_index, unsigned int index) -{ - Pool *pfc_pool = sspace->pfc_pools[seg_index][index]; - Chunk_Header *chunk = (Chunk_Header*)pool_get_entry(pfc_pool); - assert(!chunk || chunk->status == (CHUNK_NORMAL | CHUNK_NEED_ZEROING)); - return chunk; -} - -inline void sspace_put_pfc(Sspace *sspace, Chunk_Header *chunk) -{ - unsigned int size = chunk->slot_size; - assert(chunk && (size <= SUPER_OBJ_THRESHOLD)); - assert(chunk->base && chunk->alloc_num); - assert(chunk->alloc_num < chunk->slot_num); - assert(chunk->slot_index < chunk->slot_num); - - Size_Segment **size_segs = sspace->size_segments; - chunk->status = CHUNK_NORMAL | CHUNK_NEED_ZEROING; - - for(unsigned int i = 0; i < SIZE_SEGMENT_NUM; ++i){ - if(size > size_segs[i]->size_max) continue; - assert(!(size & size_segs[i]->gran_low_mask)); - assert(size > size_segs[i]->size_min); - unsigned int index = NORMAL_SIZE_TO_INDEX(size, size_segs[i]); - Pool *pfc_pool = sspace->pfc_pools[i][index]; - pool_put_entry(pfc_pool, chunk); - return; - } -} - -inline void sspace_rebuild_chunk_chain(Sspace *sspace) -{ - Chunk_Header_Basic *sspace_ceiling = (Chunk_Header_Basic*)space_heap_end((Space*)sspace); - Chunk_Header_Basic *prev_chunk = (Chunk_Header_Basic*)space_heap_start((Space*)sspace); - Chunk_Header_Basic *chunk = prev_chunk->adj_next; - prev_chunk->adj_prev = NULL; - - while(chunk < sspace_ceiling){ - chunk->adj_prev = prev_chunk; - prev_chunk = chunk; - chunk = chunk->adj_next; - } -} - - -extern void sspace_init_chunks(Sspace *sspace); -extern void sspace_clear_chunk_list(GC *gc); - -extern void sspace_put_free_chunk(Sspace *sspace, Free_Chunk *chunk); -extern Free_Chunk *sspace_get_normal_free_chunk(Sspace *sspace); -extern Free_Chunk *sspace_get_abnormal_free_chunk(Sspace *sspace, unsigned int chunk_size); -extern Free_Chunk *sspace_get_hyper_free_chunk(Sspace *sspace, unsigned int chunk_size, Boolean is_normal_chunk); - -extern void sspace_init_pfc_pool_iterator(Sspace *sspace); -extern Pool *sspace_grab_next_pfc_pool(Sspace *sspace); - -extern Chunk_Header *sspace_steal_pfc(Sspace *sspace, unsigned int index); - -extern POINTER_SIZE_INT free_mem_in_sspace(Sspace *sspace, Boolean show_chunk_info); - -extern void zeroing_free_chunk(Free_Chunk *chunk); - -extern void gc_clear_collector_local_chunks(GC *gc); - -extern void sspace_collect_free_chunks_to_list(Sspace *sspace, Free_Chunk_List *list); - -#endif //#ifndef _SSPACE_CHUNK_H_ Index: src/mark_sweep/sspace_compact.cpp =================================================================== --- src/mark_sweep/sspace_compact.cpp (revision 606795) +++ src/mark_sweep/sspace_compact.cpp (working copy) @@ -1,231 +1 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "sspace_chunk.h" -#include "sspace_alloc.h" -#include "sspace_mark_sweep.h" -#include "sspace_verify.h" - - -#define PFC_SORT_NUM 8 - -void sspace_decide_compaction_need(Sspace *sspace) -{ - POINTER_SIZE_INT free_mem_size = free_mem_in_sspace(sspace, FALSE); - float free_mem_ratio = (float)free_mem_size / sspace->committed_heap_size; - -#ifdef USE_MARK_SWEEP_GC - if(!gc_mark_is_concurrent() && (free_mem_ratio > SSPACE_COMPACT_RATIO) && (sspace->gc->cause != GC_CAUSE_RUNTIME_FORCE_GC)){ -#else - if(gc_match_kind(sspace->gc, MAJOR_COLLECTION)){ -#endif - sspace->need_compact = sspace->move_object = TRUE; - } else { - sspace->need_compact = sspace->move_object = FALSE; - } -} - -static inline void sorted_chunk_bucket_add_entry(Chunk_Header **head, Chunk_Header **tail, Chunk_Header *chunk) -{ - chunk->prev = NULL; /* Field adj_prev is used as prev */ - - if(!*head){ - assert(!*tail); - chunk->next = NULL; - *head = *tail = chunk; - return; - } - - assert(*tail); - chunk->next = *head; - (*head)->prev = chunk; - *head = chunk; -} - -/* One assumption: pfc_pool is not empty */ -static Boolean pfc_pool_roughly_sort(Pool *pfc_pool, Chunk_Header **least_free_chunk, Chunk_Header **most_free_chunk) -{ - Chunk_Header *bucket_head[PFC_SORT_NUM]; /* Sorted chunk buckets' heads */ - Chunk_Header *bucket_tail[PFC_SORT_NUM]; /* Sorted chunk buckets' tails */ - unsigned int slot_num; - unsigned int chunk_num = 0; - unsigned int slot_alloc_num = 0; - - /* Init buckets' heads and tails */ - memset(bucket_head, 0, sizeof(Chunk_Header*) * PFC_SORT_NUM); - memset(bucket_tail, 0, sizeof(Chunk_Header*) * PFC_SORT_NUM); - - /* Roughly sort chunks in pfc_pool */ - pool_iterator_init(pfc_pool); - Chunk_Header *chunk = (Chunk_Header*)pool_iterator_next(pfc_pool); - if(chunk) slot_num = chunk->slot_num; - while(chunk){ - ++chunk_num; - assert(chunk->alloc_num); - slot_alloc_num += chunk->alloc_num; - Chunk_Header *next_chunk = chunk->next; - unsigned int bucket_index = (chunk->alloc_num*PFC_SORT_NUM-1) / slot_num; - assert(bucket_index < PFC_SORT_NUM); - sorted_chunk_bucket_add_entry(&bucket_head[bucket_index], &bucket_tail[bucket_index], chunk); - chunk = next_chunk; - } - - /* Empty the pfc pool because some chunks in this pool will be free after compaction */ - pool_empty(pfc_pool); - - /* If we can't get a free chunk after compaction, there is no need to compact. - * This condition includes that the chunk num in pfc pool is equal to 1, in which case there is also no need to compact - */ - if(slot_num*(chunk_num-1) <= slot_alloc_num){ - for(unsigned int i = 0; i < PFC_SORT_NUM; i++){ - Chunk_Header *chunk = bucket_head[i]; - while(chunk){ - Chunk_Header *next_chunk = chunk->next; - pool_put_entry(pfc_pool, chunk); - chunk = next_chunk; - } - } - return FALSE; - } - - /* Link the sorted chunk buckets into one single ordered bidirectional list */ - Chunk_Header *head = NULL; - Chunk_Header *tail = NULL; - for(unsigned int i = PFC_SORT_NUM; i--;){ - assert((head && tail) || (!head && !tail)); - assert((bucket_head[i] && bucket_tail[i]) || (!bucket_head[i] && !bucket_tail[i])); - if(!bucket_head[i]) continue; - if(!tail){ - head = bucket_head[i]; - tail = bucket_tail[i]; - } else { - tail->next = bucket_head[i]; - bucket_head[i]->prev = tail; - tail = bucket_tail[i]; - } - } - - assert(head && tail); - *least_free_chunk = head; - *most_free_chunk = tail; - - return TRUE; -} - -static inline Chunk_Header *get_least_free_chunk(Chunk_Header **least_free_chunk, Chunk_Header **most_free_chunk) -{ - if(!*least_free_chunk){ - assert(!*most_free_chunk); - return NULL; - } - Chunk_Header *result = *least_free_chunk; - *least_free_chunk = (*least_free_chunk)->next; - if(*least_free_chunk) - (*least_free_chunk)->prev = NULL; - else - *most_free_chunk = NULL; - return result; -} -static inline Chunk_Header *get_most_free_chunk(Chunk_Header **least_free_chunk, Chunk_Header **most_free_chunk) -{ - if(!*most_free_chunk){ - assert(!*least_free_chunk); - return NULL; - } - Chunk_Header *result = *most_free_chunk; - *most_free_chunk = (*most_free_chunk)->prev; - if(*most_free_chunk) - (*most_free_chunk)->next = NULL; - else - *least_free_chunk = NULL; - assert(!result->next); - return result; -} - -static inline void move_obj_between_chunks(Chunk_Header **dest_ptr, Chunk_Header *src) -{ - Chunk_Header *dest = *dest_ptr; - assert(dest->slot_size == src->slot_size); - - unsigned int slot_size = dest->slot_size; - unsigned int alloc_num = src->alloc_num; - assert(alloc_num); - - while(alloc_num && dest){ - Partial_Reveal_Object *p_obj = next_alloc_slot_in_chunk(src); - void *target = alloc_in_chunk(dest); - assert(p_obj && target); - memcpy(target, p_obj, slot_size); -#ifdef SSPACE_VERIFY - sspace_modify_mark_in_compact(target, p_obj, slot_size); -#endif - obj_set_fw_in_oi(p_obj, target); - --alloc_num; - } - - /* dest might be set to NULL, so we use *dest_ptr here */ - assert((*dest_ptr)->alloc_num <= (*dest_ptr)->slot_num); - src->alloc_num = alloc_num; - if(!dest){ - assert((*dest_ptr)->alloc_num == (*dest_ptr)->slot_num); - *dest_ptr = NULL; - clear_free_slot_in_table(src->table, src->slot_index); - } -} - -void sspace_compact(Collector *collector, Sspace *sspace) -{ - Chunk_Header *least_free_chunk, *most_free_chunk; - Pool *pfc_pool = sspace_grab_next_pfc_pool(sspace); - - for(; pfc_pool; pfc_pool = sspace_grab_next_pfc_pool(sspace)){ - if(pool_is_empty(pfc_pool)) continue; - Boolean pfc_pool_need_compact = pfc_pool_roughly_sort(pfc_pool, &least_free_chunk, &most_free_chunk); - if(!pfc_pool_need_compact) continue; - - Chunk_Header *dest = get_least_free_chunk(&least_free_chunk, &most_free_chunk); - Chunk_Header *src = get_most_free_chunk(&least_free_chunk, &most_free_chunk); - Boolean src_is_new = TRUE; - while(dest && src){ - if(src_is_new) - src->slot_index = 0; - chunk_depad_last_index_word(src); - move_obj_between_chunks(&dest, src); - if(!dest) - dest = get_least_free_chunk(&least_free_chunk, &most_free_chunk); - if(!src->alloc_num){ - collector_add_free_chunk(collector, (Free_Chunk*)src); - src = get_most_free_chunk(&least_free_chunk, &most_free_chunk); - src_is_new = TRUE; - } else { - src_is_new = FALSE; - } - } - - /* Rebuild the pfc_pool */ - if(dest) - sspace_put_pfc(sspace, dest); - if(src){ - chunk_pad_last_index_word(src, cur_alloc_mask); - pfc_reset_slot_index(src); - sspace_put_pfc(sspace, src); - } - } -} - - - Index: src/mark_sweep/sspace_fallback_mark.cpp =================================================================== --- src/mark_sweep/sspace_fallback_mark.cpp (revision 606795) +++ src/mark_sweep/sspace_fallback_mark.cpp (working copy) @@ -1,216 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sspace_mark_sweep.h" -#include "../finalizer_weakref/finalizer_weakref.h" - -static Sspace *sspace_in_fallback_marking; - - -static FORCE_INLINE Boolean obj_mark_black(Partial_Reveal_Object *obj) -{ - if(obj_belongs_to_space(obj, (Space*)sspace_in_fallback_marking)){ - Boolean marked_by_self = obj_mark_black_in_table(obj); - -#ifndef USE_MARK_SWEEP_GC - /* When fallback happens, some objects in MOS have their fw bit set, which is actually their mark bit in the last minor gc. - * If we don't clear it, some objects that didn't be moved will be mistaken for being moved in the coming fixing phase. - */ - if(marked_by_self){ - Obj_Info_Type oi = obj->obj_info; - Obj_Info_Type new_oi = oi & DUAL_MARKBITS_MASK; - while(new_oi != oi){ - Obj_Info_Type temp = atomic_casptrsz((volatile Obj_Info_Type*)get_obj_info_addr(obj), new_oi, oi); - if(temp == oi) break; - oi = obj->obj_info; - new_oi = oi & DUAL_MARKBITS_MASK; - } - } -#endif - return marked_by_self; - } else { - return obj_mark_in_vt(obj); - } -} - -static FORCE_INLINE void scan_slot(Collector *collector, REF *p_ref) -{ - if( read_slot(p_ref) == NULL) return; - - collector_tracestack_push(collector, p_ref); -} - -static FORCE_INLINE void scan_object(Collector *collector, REF *p_ref) -{ - Partial_Reveal_Object *p_obj = read_slot(p_ref); - assert(p_obj); - assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); - - if(obj_belongs_to_nos(p_obj) && obj_is_fw_in_oi(p_obj)){ - assert(obj_get_vt(p_obj) == obj_get_vt(obj_get_fw_in_oi(p_obj))); - p_obj = obj_get_fw_in_oi(p_obj); - assert(p_obj); - write_slot(p_ref, p_obj); - } - - if(!obj_mark_black(p_obj)) - return; - - if(!object_has_ref_field(p_obj)) return; - - if(object_is_array(p_obj)){ /* scan array object */ - Partial_Reveal_Array *array = (Partial_Reveal_Array*)p_obj; - unsigned int array_length = array->array_len; - - REF *p_ref = (REF *)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); - for (unsigned int i = 0; i < array_length; i++) - scan_slot(collector, p_ref+i); - - return; - } - - /* scan non-array object */ - unsigned int num_refs = object_ref_field_num(p_obj); - int *ref_iterator = object_ref_iterator_init(p_obj); - - for(unsigned int i=0; itrace_stack; - while(!vector_stack_is_empty(trace_stack)){ - p_ref = (REF*)vector_stack_pop(trace_stack); - scan_object(collector, p_ref); - trace_stack = collector->trace_stack; - } -} - -/* NOTE:: This is another marking version: marking in color bitmap table. - Originally, we have to mark the object before put it into markstack, to - guarantee there is only one occurrance of an object in markstack. This is to - guarantee there is only one occurrance of a repointed ref slot in repset (they - are put to the set when the object is scanned). If the same object is put to - markstack twice, they will be scanned twice and their ref slots will be recorded twice. - Problem occurs when the ref slot is updated first time with new position, - the second time the value in the ref slot is not the old position as expected. - It needs to read the original obj header for forwarding pointer. With the new value, - it will read something nonsense since the obj is not moved yet. - This can be worked around if we want. - To do this we have to use atomic instruction for marking, which is undesirable. - So we abondoned this design. We no longer use the repset to remember repointed slots. -*/ - -/* for marking phase termination detection */ -static volatile unsigned int num_finished_collectors = 0; - -void sspace_fallback_mark_scan(Collector *collector, Sspace *sspace) -{ - GC *gc = collector->gc; - GC_Metadata *metadata = gc->metadata; - sspace_in_fallback_marking = sspace; - - /* reset the num_finished_collectors to be 0 by one collector. This is necessary for the barrier later. */ - unsigned int num_active_collectors = gc->num_active_collectors; - atomic_cas32(&num_finished_collectors, 0, num_active_collectors); - - collector->trace_stack = free_task_pool_get_entry(metadata); - - Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool); - - /* first step: copy all root objects to mark tasks. - FIXME:: can be done sequentially before coming here to eliminate atomic ops */ - while(root_set){ - POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); - while(!vector_block_iterator_end(root_set,iter)){ - REF *p_ref = (REF*)*iter; - iter = vector_block_iterator_advance(root_set,iter); - - /* root ref can't be NULL, (remset may have NULL ref entry, but this function is only for MAJOR_COLLECTION */ - assert(read_slot(p_ref) != NULL); - /* we have to mark the object before putting it into marktask, because - it is possible to have two slots containing a same object. They will - be scanned twice and their ref slots will be recorded twice. Problem - occurs after the ref slot is updated first time with new position - and the second time the value is the ref slot is the old position as expected. - This can be worked around if we want. - */ - collector_tracestack_push(collector, p_ref); - } - root_set = pool_iterator_next(metadata->gc_rootset_pool); - } - /* put back the last trace_stack task */ - pool_put_entry(metadata->mark_task_pool, collector->trace_stack); - - /* second step: iterate over the mark tasks and scan objects */ - /* get a task buf for the mark stack */ - collector->trace_stack = free_task_pool_get_entry(metadata); - -retry: - Vector_Block *mark_task = pool_get_entry(metadata->mark_task_pool); - - while(mark_task){ - POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task); - while(!vector_block_iterator_end(mark_task, iter)){ - REF *p_ref = (REF*)*iter; - iter = vector_block_iterator_advance(mark_task, iter); - - /* FIXME:: we should not let mark_task empty during working, , other may want to steal it. - degenerate my stack into mark_task, and grab another mark_task */ - trace_object(collector, p_ref); - } - /* run out one task, put back to the pool and grab another task */ - vector_stack_clear(mark_task); - pool_put_entry(metadata->free_task_pool, mark_task); - mark_task = pool_get_entry(metadata->mark_task_pool); - } - - /* termination detection. This is also a barrier. - NOTE:: We can simply spin waiting for num_finished_collectors, because each - generated new task would surely be processed by its generating collector eventually. - So code below is only for load balance optimization. */ - atomic_inc32(&num_finished_collectors); - while(num_finished_collectors != num_active_collectors){ - if(!pool_is_empty(metadata->mark_task_pool)){ - atomic_dec32(&num_finished_collectors); - goto retry; - } - } - - /* put back the last mark stack to the free pool */ - mark_task = (Vector_Block*)collector->trace_stack; - vector_stack_clear(mark_task); - pool_put_entry(metadata->free_task_pool, mark_task); - collector->trace_stack = NULL; - - return; -} - -void trace_obj_in_ms_fallback_marking(Collector *collector, void *p_ref) -{ - trace_object(collector, (REF*)p_ref); -} Index: src/mark_sweep/sspace_mark.cpp =================================================================== --- src/mark_sweep/sspace_mark.cpp (revision 606795) +++ src/mark_sweep/sspace_mark.cpp (working copy) @@ -1,245 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sspace_mark_sweep.h" -#include "../finalizer_weakref/finalizer_weakref.h" - -static Sspace *sspace_in_marking; -static FORCE_INLINE Boolean obj_mark_gray(Partial_Reveal_Object *obj) -{ - if(obj_belongs_to_space(obj, (Space*)sspace_in_marking)) - return obj_mark_gray_in_table(obj); - else - return obj_mark_in_vt(obj); -} - -static FORCE_INLINE Boolean obj_mark_black(Partial_Reveal_Object *obj) -{ - if(obj_belongs_to_space(obj, (Space*)sspace_in_marking)){ - Boolean marked_by_self = obj_mark_black_in_table(obj); - -#ifndef USE_MARK_SWEEP_GC - /* When fallback happens, some objects in MOS have their fw bit set, which is actually their mark bit in the last minor gc. - * If we don't clear it, some objects that didn't be moved will be mistaken for being moved in the coming fixing phase. - */ - if(marked_by_self){ - Obj_Info_Type oi = obj->obj_info; - Obj_Info_Type new_oi = oi & DUAL_MARKBITS_MASK; - while(new_oi != oi){ - Obj_Info_Type temp = atomic_casptrsz((volatile Obj_Info_Type*)get_obj_info_addr(obj), new_oi, oi); - if(temp == oi) break; - oi = obj->obj_info; - new_oi = oi & DUAL_MARKBITS_MASK; - } - } -#endif - return marked_by_self; - } else { - return obj_mark_in_vt(obj); - } -} - - -/* The caller must be in places where alloc color and mark color haven't been flipped */ -Boolean obj_is_marked_in_table(Partial_Reveal_Object *obj) -{ - unsigned int index_in_word; - volatile POINTER_SIZE_INT *p_color_word = get_color_word_in_table(obj, index_in_word); - assert(p_color_word); - - POINTER_SIZE_INT color_word = *p_color_word; - POINTER_SIZE_INT mark_color = cur_mark_gray_color << index_in_word; - - return (Boolean)(color_word & mark_color); -} - -static FORCE_INLINE void scan_slot(Collector *collector, REF *p_ref) -{ - Partial_Reveal_Object *p_obj = read_slot(p_ref); - if( p_obj == NULL) return; - - assert(address_belongs_to_gc_heap(p_obj, collector->gc)); - if(obj_mark_gray(p_obj)){ - assert(p_obj); - collector_tracestack_push(collector, p_obj); - } -} - -static FORCE_INLINE void scan_object(Collector *collector, Partial_Reveal_Object *p_obj) -{ - assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); - - Partial_Reveal_VTable *vtable = uncompress_vt(obj_get_vt(p_obj)); - if(VTABLE_TRACING) - if(vtable->vtmark == VT_UNMARKED) { - vtable->vtmark = VT_MARKED; - if(obj_mark_black(vtable->jlC)) - collector_tracestack_push(collector, vtable->jlC); - } - - if(!object_has_ref_field(p_obj)) return; - - REF *p_ref; - - if(object_is_array(p_obj)){ /* scan array object */ - Partial_Reveal_Array *array = (Partial_Reveal_Array*)p_obj; - unsigned int array_length = array->array_len; - - p_ref = (REF *)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); - for (unsigned int i = 0; i < array_length; i++) - scan_slot(collector, p_ref+i); - - return; - } - - /* scan non-array object */ - unsigned int num_refs = object_ref_field_num(p_obj); - int *ref_iterator = object_ref_iterator_init(p_obj); - - for(unsigned int i=0; itrace_stack; - while(!vector_stack_is_empty(trace_stack)){ - p_obj = (Partial_Reveal_Object*)vector_stack_pop(trace_stack); - scan_object(collector, p_obj); - obj_mark_black(p_obj); - trace_stack = collector->trace_stack; - } -} - -/* NOTE:: This is another marking version: marking in color bitmap table. - Originally, we have to mark the object before put it into markstack, to - guarantee there is only one occurrance of an object in markstack. This is to - guarantee there is only one occurrance of a repointed ref slot in repset (they - are put to the set when the object is scanned). If the same object is put to - markstack twice, they will be scanned twice and their ref slots will be recorded twice. - Problem occurs when the ref slot is updated first time with new position, - the second time the value in the ref slot is not the old position as expected. - It needs to read the original obj header for forwarding pointer. With the new value, - it will read something nonsense since the obj is not moved yet. - This can be worked around if we want. - To do this we have to use atomic instruction for marking, which is undesirable. - So we abondoned this design. We no longer use the repset to remember repointed slots. -*/ - -/* for marking phase termination detection */ -static volatile unsigned int num_finished_collectors = 0; - -void sspace_mark_scan(Collector *collector, Sspace *sspace) -{ - GC *gc = collector->gc; - GC_Metadata *metadata = gc->metadata; - sspace_in_marking = sspace; - - /* reset the num_finished_collectors to be 0 by one collector. This is necessary for the barrier later. */ - unsigned int num_active_collectors = gc->num_active_collectors; - atomic_cas32(&num_finished_collectors, 0, num_active_collectors); - - collector->trace_stack = free_task_pool_get_entry(metadata); - - Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool); - - /* first step: copy all root objects to mark tasks. - FIXME:: can be done sequentially before coming here to eliminate atomic ops */ - while(root_set){ - POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); - while(!vector_block_iterator_end(root_set,iter)){ - REF *p_ref = (REF*)*iter; - iter = vector_block_iterator_advance(root_set,iter); - - Partial_Reveal_Object *p_obj = read_slot(p_ref); - /* root ref can't be NULL, (remset may have NULL ref entry, but this function is only for MAJOR_COLLECTION */ - assert(p_obj != NULL); - /* we have to mark the object before putting it into marktask, because - it is possible to have two slots containing a same object. They will - be scanned twice and their ref slots will be recorded twice. Problem - occurs after the ref slot is updated first time with new position - and the second time the value is the ref slot is the old position as expected. - This can be worked around if we want. - */ - assert(address_belongs_to_gc_heap(p_obj, gc)); - if(obj_mark_gray(p_obj)) - collector_tracestack_push(collector, p_obj); - } - root_set = pool_iterator_next(metadata->gc_rootset_pool); - } - /* put back the last trace_stack task */ - pool_put_entry(metadata->mark_task_pool, collector->trace_stack); - - /* second step: iterate over the mark tasks and scan objects */ - /* get a task buf for the mark stack */ - collector->trace_stack = free_task_pool_get_entry(metadata); - -retry: - Vector_Block *mark_task = pool_get_entry(metadata->mark_task_pool); - - while(mark_task){ - POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task); - while(!vector_block_iterator_end(mark_task, iter)){ - Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)*iter; - iter = vector_block_iterator_advance(mark_task, iter); - - /* FIXME:: we should not let mark_task empty during working, , other may want to steal it. - degenerate my stack into mark_task, and grab another mark_task */ - trace_object(collector, p_obj); - } - /* run out one task, put back to the pool and grab another task */ - vector_stack_clear(mark_task); - pool_put_entry(metadata->free_task_pool, mark_task); - mark_task = pool_get_entry(metadata->mark_task_pool); - } - - /* termination detection. This is also a barrier. - NOTE:: We can simply spin waiting for num_finished_collectors, because each - generated new task would surely be processed by its generating collector eventually. - So code below is only for load balance optimization. */ - atomic_inc32(&num_finished_collectors); - while(num_finished_collectors != num_active_collectors){ - if(!pool_is_empty(metadata->mark_task_pool)){ - atomic_dec32(&num_finished_collectors); - goto retry; - } - } - - /* put back the last mark stack to the free pool */ - mark_task = (Vector_Block*)collector->trace_stack; - vector_stack_clear(mark_task); - pool_put_entry(metadata->free_task_pool, mark_task); - collector->trace_stack = NULL; - - return; -} - -void trace_obj_in_ms_marking(Collector *collector, void *p_obj) -{ - obj_mark_gray((Partial_Reveal_Object*)p_obj); - trace_object(collector, (Partial_Reveal_Object*)p_obj); -} Index: src/mark_sweep/sspace_mark_concurrent.cpp =================================================================== --- src/mark_sweep/sspace_mark_concurrent.cpp (revision 606795) +++ src/mark_sweep/sspace_mark_concurrent.cpp (working copy) @@ -1,235 +1 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "sspace_mark_sweep.h" -#include "../finalizer_weakref/finalizer_weakref.h" -#include "../thread/marker.h" - -Boolean obj_is_marked_in_table(Partial_Reveal_Object *obj); - -static FORCE_INLINE void scan_slot(Collector* marker, REF *p_ref) -{ - Partial_Reveal_Object *p_obj = read_slot(p_ref); - if( p_obj == NULL) return; - - assert(address_belongs_to_gc_heap(p_obj, marker->gc)); - if(obj_mark_gray_in_table(p_obj)){ - assert(p_obj); - collector_tracestack_push((Collector*)marker, p_obj); - } -} - -static FORCE_INLINE void scan_object(Marker* marker, Partial_Reveal_Object *p_obj) -{ - assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); - - if(obj_is_dirty_in_table(p_obj)){ - assert(obj_is_mark_black_in_table(p_obj)); - return; - } - - if(!object_has_ref_field(p_obj)) return; - - REF *p_ref; - - if(object_is_array(p_obj)){ /* scan array object */ - Partial_Reveal_Array *array = (Partial_Reveal_Array*)p_obj; - unsigned int array_length = array->array_len; - - p_ref = (REF *)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); - for (unsigned int i = 0; i < array_length; i++) - scan_slot((Collector*)marker, p_ref+i); - - return; - } - - /* scan non-array object */ - unsigned int num_refs = object_ref_field_num(p_obj); - int *ref_iterator = object_ref_iterator_init(p_obj); - - for(unsigned int i=0; itrace_stack; - while(!vector_stack_is_empty(trace_stack)){ - p_obj = (Partial_Reveal_Object*)vector_stack_pop(trace_stack); - scan_object(marker, p_obj); - obj_mark_black_in_table(p_obj); - trace_stack = marker->trace_stack; - } -} - -static Boolean concurrent_mark_need_terminating(GC* gc) -{ - GC_Metadata *metadata = gc->metadata; - return gc_local_snapshot_is_empty(gc) && pool_is_empty(metadata->dirty_obj_snaptshot_pool); -} - -/* for marking phase termination detection */ -static volatile unsigned int num_active_markers = 0; - -void sspace_mark_scan_concurrent(Marker* marker) -{ - int64 time_mark_start = time_now(); - GC *gc = marker->gc; - GC_Metadata *metadata = gc->metadata; - - /* reset the num_finished_collectors to be 0 by one collector. This is necessary for the barrier later. */ - atomic_inc32(&num_active_markers); - - marker->trace_stack = free_task_pool_get_entry(metadata); - - Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool); - - /* first step: copy all root objects to mark tasks. - FIXME:: can be done sequentially before coming here to eliminate atomic ops */ - while(root_set){ - POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); - while(!vector_block_iterator_end(root_set,iter)){ - REF *p_ref = (REF *)*iter; - iter = vector_block_iterator_advance(root_set,iter); - - Partial_Reveal_Object *p_obj = read_slot(p_ref); - /* root ref can't be NULL, (remset may have NULL ref entry, but this function is only for MAJOR_COLLECTION */ - assert(p_obj!=NULL); - /* we have to mark the object before putting it into marktask, because - it is possible to have two slots containing a same object. They will - be scanned twice and their ref slots will be recorded twice. Problem - occurs after the ref slot is updated first time with new position - and the second time the value is the ref slot is the old position as expected. - This can be worked around if we want. - */ - assert(address_belongs_to_gc_heap(p_obj, gc)); - if(obj_mark_gray_in_table(p_obj)) - collector_tracestack_push((Collector*)marker, p_obj); - } - root_set = pool_iterator_next(metadata->gc_rootset_pool); - } - /* put back the last trace_stack task */ - pool_put_entry(metadata->mark_task_pool, marker->trace_stack); - - marker_notify_mark_root_done(marker); - - /*second step: mark dirty object snapshot pool*/ - marker->trace_stack = free_task_pool_get_entry(metadata); - -retry: - - Vector_Block* snapshot_set = pool_get_entry(metadata->dirty_obj_snaptshot_pool); - - while(snapshot_set){ - POINTER_SIZE_INT* iter = vector_block_iterator_init(snapshot_set); - while(!vector_block_iterator_end(snapshot_set,iter)){ - Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)*iter; - iter = vector_block_iterator_advance(snapshot_set,iter); - - assert(p_obj!=NULL); //ynhe, restrict? - if(obj_mark_gray_in_table(p_obj)) - collector_tracestack_push((Collector*)marker, p_obj); - } - vector_block_clear(snapshot_set); - pool_put_entry(metadata->free_set_pool, snapshot_set); - snapshot_set = pool_get_entry(metadata->dirty_obj_snaptshot_pool); - } - - /* put back the last trace_stack task */ - pool_put_entry(metadata->mark_task_pool, marker->trace_stack); - - /* third step: iterate over the mark tasks and scan objects */ - /* get a task buf for the mark stack */ - marker->trace_stack = free_task_pool_get_entry(metadata); - - - Vector_Block *mark_task = pool_get_entry(metadata->mark_task_pool); - - while(mark_task){ - POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task); - while(!vector_block_iterator_end(mark_task,iter)){ - Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)*iter; - iter = vector_block_iterator_advance(mark_task,iter); - trace_object(marker, p_obj); - } - /* run out one task, put back to the pool and grab another task */ - vector_stack_clear(mark_task); - pool_put_entry(metadata->free_task_pool, mark_task); - mark_task = pool_get_entry(metadata->mark_task_pool); - } - - /* termination condition: - 1.all thread finished current job. - 2.local snapshot vectors are empty. - 3.global snapshot pool is empty. - */ - atomic_dec32(&num_active_markers); - while(num_active_markers != 0 || !concurrent_mark_need_terminating(gc)){ - if(!pool_is_empty(metadata->mark_task_pool) || !pool_is_empty(metadata->dirty_obj_snaptshot_pool)){ - atomic_inc32(&num_active_markers); - goto retry; - }else{ - /*grab a block from mutator and begin tracing*/ - POINTER_SIZE_INT thread_num = (POINTER_SIZE_INT)marker->thread_handle; - Vector_Block* local_snapshot_set = gc_get_local_snapshot(gc, (unsigned int)(thread_num + 1)); - /*1. If local_snapshot_set has been set full bit, the block is full and will no longer be put into global snapshot pool; - so it should be checked again to see if there're remaining entrys unscanned in it. In this case, the - share bit in local_snapshot_set should not be clear, beacause of rescanning exclusively. - 2. If local_snapshot_set has not been set full bit, the block is used by mutator and has the chance to be put into - global snapshot pool. In this case, we simply cleared the share bit in local_snapshot_set. - */ - if(local_snapshot_set != NULL){ - atomic_inc32(&num_active_markers); - while(!vector_block_is_empty(local_snapshot_set) || !vector_block_not_full_set_unshared(local_snapshot_set)){ - Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*) vector_block_get_entry(local_snapshot_set); - if(obj_mark_gray_in_table(p_obj)) - collector_tracestack_push((Collector*)marker, p_obj); - } - goto retry; - } - } - } - - /* put back the last mark stack to the free pool */ - mark_task = (Vector_Block*)marker->trace_stack; - vector_stack_clear(mark_task); - pool_put_entry(metadata->free_task_pool, mark_task); - marker->trace_stack = NULL; - assert(pool_is_empty(metadata->dirty_obj_snaptshot_pool)); - - int64 time_mark = time_now() - time_mark_start; - marker->time_mark = time_mark; - - return; -} - -void trace_obj_in_ms_concurrent_mark(Marker *marker, void *p_obj) -{ - obj_mark_gray_in_table((Partial_Reveal_Object*)p_obj); - trace_object(marker, (Partial_Reveal_Object *)p_obj); -} - Index: src/mark_sweep/sspace_mark_sweep.cpp =================================================================== --- src/mark_sweep/sspace_mark_sweep.cpp (revision 606795) +++ src/mark_sweep/sspace_mark_sweep.cpp (working copy) @@ -1,390 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sspace_alloc.h" -#include "sspace_mark_sweep.h" -#include "sspace_verify.h" -#include "gc_ms.h" -#include "../gen/gen.h" -#include "../thread/collector.h" -#include "../finalizer_weakref/finalizer_weakref.h" -#include "../common/fix_repointed_refs.h" -#include "../common/gc_concurrent.h" - -POINTER_SIZE_INT cur_alloc_mask = (~MARK_MASK_IN_TABLE) & FLIP_COLOR_MASK_IN_TABLE; -POINTER_SIZE_INT cur_mark_mask = MARK_MASK_IN_TABLE; -POINTER_SIZE_INT cur_alloc_color = OBJ_COLOR_WHITE; -POINTER_SIZE_INT cur_mark_gray_color = OBJ_COLOR_GRAY; -POINTER_SIZE_INT cur_mark_black_color = OBJ_COLOR_BLACK; - -static Chunk_Header_Basic *volatile next_chunk_for_fixing; - - -/******************** General interfaces for Mark-Sweep-Compact ***********************/ - -static void ops_color_flip(void) -{ - POINTER_SIZE_INT temp = cur_alloc_color; - cur_alloc_color = cur_mark_black_color; - cur_mark_black_color = temp; - cur_alloc_mask = (~cur_alloc_mask) & FLIP_COLOR_MASK_IN_TABLE; - cur_mark_mask = (~cur_mark_mask) & FLIP_COLOR_MASK_IN_TABLE; -} - -void collector_init_free_chunk_list(Collector *collector) -{ - Free_Chunk_List *list = (Free_Chunk_List*)STD_MALLOC(sizeof(Free_Chunk_List)); - free_chunk_list_init(list); - collector->free_chunk_list = list; -} - -/* Argument need_construct stands for whether or not the dual-directon list needs constructing */ -Chunk_Header_Basic *sspace_grab_next_chunk(Sspace *sspace, Chunk_Header_Basic *volatile *shared_next_chunk, Boolean need_construct) -{ - Chunk_Header_Basic *cur_chunk = *shared_next_chunk; - - Chunk_Header_Basic *sspace_ceiling = (Chunk_Header_Basic*)space_heap_end((Space*)sspace); - while(cur_chunk < sspace_ceiling){ - Chunk_Header_Basic *next_chunk = CHUNK_END(cur_chunk); - - Chunk_Header_Basic *temp = (Chunk_Header_Basic*)atomic_casptr((volatile void**)shared_next_chunk, next_chunk, cur_chunk); - if(temp == cur_chunk){ - if(need_construct && next_chunk < sspace_ceiling) - next_chunk->adj_prev = cur_chunk; - return cur_chunk; - } - cur_chunk = *shared_next_chunk; - } - - return NULL; -} - - -/******************** Interfaces for Forwarding ***********************/ - -static void nos_init_block_for_forwarding(GC_Gen *gc_gen) -{ blocked_space_block_iterator_init((Blocked_Space*)gc_get_nos(gc_gen)); } - -static inline void block_forward_live_objects(Collector *collector, Sspace *sspace, Block_Header *cur_block) -{ - Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)cur_block->base; - Partial_Reveal_Object *block_end = (Partial_Reveal_Object*)cur_block->free; - - for(; p_obj < block_end; p_obj = (Partial_Reveal_Object*)((POINTER_SIZE_INT)p_obj + vm_object_size(p_obj))){ - if(!obj_is_marked_in_vt(p_obj)) continue; - - obj_clear_dual_bits_in_vt(p_obj); - Partial_Reveal_Object *p_target_obj = collector_forward_object(collector, p_obj); /* Could be implemented with a optimized function */ - if(!p_target_obj){ - assert(collector->result == FALSE); - printf("Out of mem in forwarding nos!\n"); - exit(0); - } - } -} - -static void collector_forward_nos_to_sspace(Collector *collector, Sspace *sspace) -{ - Blocked_Space *nos = (Blocked_Space*)gc_get_nos((GC_Gen*)collector->gc); - Block_Header *cur_block = blocked_space_block_iterator_next(nos); - - /* We must iterate over all nos blocks to forward live objects in them */ - while(cur_block){ - block_forward_live_objects(collector, sspace, cur_block); - cur_block = blocked_space_block_iterator_next(nos); - } -} - - -/******************** Interfaces for Ref Fixing ***********************/ - -static void sspace_init_chunk_for_ref_fixing(Sspace *sspace) -{ - next_chunk_for_fixing = (Chunk_Header_Basic*)space_heap_start((Space*)sspace); - next_chunk_for_fixing->adj_prev = NULL; -} - -static inline void slot_double_fix(REF *p_ref) -{ - Partial_Reveal_Object *p_obj = read_slot(p_ref); - if(!p_obj) return; - - if(obj_is_fw_in_oi(p_obj)){ - p_obj = obj_get_fw_in_oi(p_obj); - assert(p_obj); - if(obj_is_fw_in_oi(p_obj)){ - p_obj = obj_get_fw_in_oi(p_obj); - assert(p_obj); - } - write_slot(p_ref, p_obj); - } -} - -static inline void object_double_fix_ref_slots(Partial_Reveal_Object *p_obj) -{ - if(!object_has_ref_field(p_obj)) return; - - /* scan array object */ - if(object_is_array(p_obj)){ - Partial_Reveal_Array *array = (Partial_Reveal_Array*)p_obj; - assert(!obj_is_primitive_array(p_obj)); - - int32 array_length = array->array_len; - REF *p_refs = (REF*)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); - for(int i = 0; i < array_length; i++){ - slot_double_fix(p_refs + i); - } - return; - } - - /* scan non-array object */ - unsigned int num_refs = object_ref_field_num(p_obj); - int* ref_iterator = object_ref_iterator_init(p_obj); - - for(unsigned int i=0; islot_index = 0; - chunk_depad_last_index_word(chunk); - - unsigned int alloc_num = chunk->alloc_num; - assert(alloc_num); - - /* After compaction, many chunks are filled with objects. - * For these chunks, we needn't find the allocated slot one by one by calling next_alloc_slot_in_chunk. - * That is a little time consuming. - * We'd like to fix those objects by incrementing their addr to find the next. - */ - if(alloc_num == chunk->slot_num){ /* Filled with objects */ - unsigned int slot_size = chunk->slot_size; - Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)slot_index_to_addr(chunk, 0); - for(unsigned int i = alloc_num; i--;){ - if(double_fix) - object_double_fix_ref_slots(p_obj); - else - object_fix_ref_slots(p_obj); -#ifdef SSPACE_VERIFY - sspace_verify_fix_in_compact(); -#endif - p_obj = (Partial_Reveal_Object*)((POINTER_SIZE_INT)p_obj + slot_size); - } - } else { /* Chunk is not full */ - while(alloc_num){ - Partial_Reveal_Object *p_obj = next_alloc_slot_in_chunk(chunk); - assert(p_obj); - if(double_fix) - object_double_fix_ref_slots(p_obj); - else - object_fix_ref_slots(p_obj); -#ifdef SSPACE_VERIFY - sspace_verify_fix_in_compact(); -#endif - --alloc_num; - } - } - - if(chunk->alloc_num != chunk->slot_num){ - chunk_pad_last_index_word(chunk, cur_alloc_mask); - pfc_reset_slot_index(chunk); - } -} - -static void abnormal_chunk_fix_repointed_refs(Chunk_Header *chunk, Boolean double_fix) -{ - if(double_fix) - object_double_fix_ref_slots((Partial_Reveal_Object*)chunk->base); - else - object_fix_ref_slots((Partial_Reveal_Object*)chunk->base); -#ifdef SSPACE_VERIFY - sspace_verify_fix_in_compact(); -#endif -} - -static void sspace_fix_repointed_refs(Collector *collector, Sspace *sspace, Boolean double_fix) -{ - Chunk_Header_Basic *chunk = sspace_grab_next_chunk(sspace, &next_chunk_for_fixing, TRUE); - - while(chunk){ - if(chunk->status & CHUNK_NORMAL) - normal_chunk_fix_repointed_refs((Chunk_Header*)chunk, double_fix); - else if(chunk->status & CHUNK_ABNORMAL) - abnormal_chunk_fix_repointed_refs((Chunk_Header*)chunk, double_fix); - - chunk = sspace_grab_next_chunk(sspace, &next_chunk_for_fixing, TRUE); - } -} - - -/******************** Main body of Mark-Sweep-Compact ***********************/ - -static volatile unsigned int num_marking_collectors = 0; -static volatile unsigned int num_sweeping_collectors = 0; -static volatile unsigned int num_compacting_collectors = 0; -static volatile unsigned int num_forwarding_collectors = 0; -static volatile unsigned int num_fixing_collectors = 0; - -void mark_sweep_sspace(Collector *collector) -{ - GC *gc = collector->gc; - Sspace *sspace = gc_get_sspace(gc); - Space *nos = NULL; - if(gc_match_kind(gc, MAJOR_COLLECTION)) - nos = gc_get_nos((GC_Gen*)gc); - - unsigned int num_active_collectors = gc->num_active_collectors; - - /* Pass 1: ************************************************** - Mark all live objects in heap ****************************/ - atomic_cas32(&num_marking_collectors, 0, num_active_collectors+1); - - if(!gc_mark_is_concurrent()){ - if(gc_match_kind(gc, FALLBACK_COLLECTION)) - sspace_fallback_mark_scan(collector, sspace); - else - sspace_mark_scan(collector, sspace); - } - - unsigned int old_num = atomic_inc32(&num_marking_collectors); - if( ++old_num == num_active_collectors ){ - /* last collector's world here */ -#ifdef SSPACE_TIME - sspace_mark_time(FALSE); -#endif - if(!IGNORE_FINREF ) - collector_identify_finref(collector); -#ifndef BUILD_IN_REFERENT - else { - gc_set_weakref_sets(gc); - gc_update_weakref_ignore_finref(gc); - } -#endif - gc_identify_dead_weak_roots(gc); - gc_init_chunk_for_sweep(gc, sspace); - /* let other collectors go */ - num_marking_collectors++; - } - while(num_marking_collectors != num_active_collectors + 1); - - /* Pass 2: ************************************************** - Sweep dead objects ***************************************/ - atomic_cas32( &num_sweeping_collectors, 0, num_active_collectors+1); - - sspace_sweep(collector, sspace); - - old_num = atomic_inc32(&num_sweeping_collectors); - if( ++old_num == num_active_collectors ){ -#ifdef SSPACE_TIME - sspace_sweep_time(FALSE, sspace->need_compact); -#endif - ops_color_flip(); -#ifdef SSPACE_VERIFY - sspace_verify_after_sweep(gc); -#endif - - if(gc_match_kind(gc, MAJOR_COLLECTION)){ - sspace_merge_free_chunks(gc, sspace); - nos_init_block_for_forwarding((GC_Gen*)gc); - } - if(sspace->need_compact) - sspace_init_pfc_pool_iterator(sspace); - if(sspace->need_fix) - sspace_init_chunk_for_ref_fixing(sspace); - /* let other collectors go */ - num_sweeping_collectors++; - } - while(num_sweeping_collectors != num_active_collectors + 1); - - /* Optional Pass: ******************************************* - Forward live obj in nos to mos (sspace) ******************/ - if(gc_match_kind(gc, MAJOR_COLLECTION)){ - atomic_cas32( &num_forwarding_collectors, 0, num_active_collectors+1); - - collector_forward_nos_to_sspace(collector, sspace); - - old_num = atomic_inc32(&num_forwarding_collectors); - if( ++old_num == num_active_collectors ){ - gc_clear_collector_local_chunks(gc); - num_forwarding_collectors++; - } - - while(num_forwarding_collectors != num_active_collectors + 1); - } - - /* Optional Pass: ******************************************* - Compact pfcs with the same size **************************/ - if(sspace->need_compact){ - atomic_cas32(&num_compacting_collectors, 0, num_active_collectors+1); - - sspace_compact(collector, sspace); - - /* If we need forward nos to mos, i.e. in major collection, an extra fixing phase after compaction is needed. */ - old_num = atomic_inc32(&num_compacting_collectors); - if( ++old_num == num_active_collectors ){ - if(gc_match_kind(gc, MAJOR_COLLECTION)) - sspace_remerge_free_chunks(gc, sspace); - /* let other collectors go */ - num_compacting_collectors++; - } - while(num_compacting_collectors != num_active_collectors + 1); - } - - /* Optional Pass: ******************************************* - Fix repointed refs ***************************************/ - if(sspace->need_fix){ - atomic_cas32( &num_fixing_collectors, 0, num_active_collectors); - - /* When we forwarded nos AND compacted sspace, - * we need double fix object slots, - * because some objects are forwarded from nos to mos and compacted into another chunk afterwards. - */ - Boolean double_fix = gc_match_kind(gc, MAJOR_COLLECTION) && sspace->need_compact; - sspace_fix_repointed_refs(collector, sspace, double_fix); - - atomic_inc32(&num_fixing_collectors); - while(num_fixing_collectors != num_active_collectors); - } - - if( collector->thread_handle != 0 ) - return; - - /* Leftover: *************************************************/ - - if(sspace->need_fix){ - Boolean double_fix = gc_match_kind(gc, MAJOR_COLLECTION) && sspace->need_compact; - gc_fix_rootset(collector, double_fix); -#ifdef SSPACE_TIME - sspace_fix_time(FALSE); -#endif - } - - if(!gc_match_kind(gc, MAJOR_COLLECTION)) - sspace_merge_free_chunks(gc, sspace); - -#ifdef USE_MARK_SWEEP_GC - sspace_set_space_statistic(sspace); -#endif - -#ifdef SSPACE_VERIFY - sspace_verify_after_collection(gc); -#endif -} Index: src/mark_sweep/sspace_mark_sweep.h =================================================================== --- src/mark_sweep/sspace_mark_sweep.h (revision 606795) +++ src/mark_sweep/sspace_mark_sweep.h (working copy) @@ -1,362 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _SSPACE_MARK_SWEEP_H_ -#define _SSPACE_MARK_SWEEP_H_ - -#include "sspace_chunk.h" -#include "sspace_verify.h" - -#define PFC_REUSABLE_RATIO 0.1 -#define SSPACE_COMPACT_RATIO 0.06 - -inline Boolean chunk_is_reusable(Chunk_Header *chunk) -{ return (float)(chunk->slot_num-chunk->alloc_num)/chunk->slot_num > PFC_REUSABLE_RATIO; } - -#define OBJ_ALLOC_BIT_IN_TABLE 0x01 -#define OBJ_BLACK_BIT_IN_TABLE 0x02 -#define OBJ_GRAY_BIT_IN_TABLE 0x04 -#define OBJ_COLOR_BIT_IN_TABLE 0x06 -#define OBJ_DIRTY_BIT_IN_TABLE 0x08 - -enum Obj_Color { - OBJ_COLOR_BLUE = 0x0, - OBJ_COLOR_WHITE = OBJ_ALLOC_BIT_IN_TABLE, - OBJ_COLOR_GRAY = OBJ_GRAY_BIT_IN_TABLE, - OBJ_COLOR_BLACK = OBJ_BLACK_BIT_IN_TABLE, - OBJ_COLOR_MASK = OBJ_COLOR_BIT_IN_TABLE -}; - -#ifdef POINTER64 - //#define BLACK_MASK_IN_TABLE ((POINTER_SIZE_INT)0xAAAAAAAAAAAAAAAA) - #define MARK_MASK_IN_TABLE ((POINTER_SIZE_INT)0x2222222222222222) - #define FLIP_COLOR_MASK_IN_TABLE ((POINTER_SIZE_INT)0x3333333333333333) - //#define DIRY_MASK_IN_TABLE ((POINTER_SIZE_INT)0x4444444444444444) -#else - #define MARK_MASK_IN_TABLE ((POINTER_SIZE_INT)0x22222222) - #define FLIP_COLOR_MASK_IN_TABLE ((POINTER_SIZE_INT)0x33333333) - //#define DIRY_MASK_IN_TABLE ((POINTER_SIZE_INT)0x44444444) -#endif - -extern POINTER_SIZE_INT cur_alloc_color; -extern POINTER_SIZE_INT cur_mark_gray_color; -extern POINTER_SIZE_INT cur_mark_black_color; -extern POINTER_SIZE_INT cur_alloc_mask; -extern POINTER_SIZE_INT cur_mark_mask; - -inline Boolean is_super_obj(Partial_Reveal_Object *obj) -{ - //return get_obj_info_raw(obj) & SUPER_OBJ_MASK;/* - if(vm_object_size(obj) > SUPER_OBJ_THRESHOLD){ - return TRUE; - } else { - return FALSE; - } -} - -inline POINTER_SIZE_INT *get_color_word_in_table(Partial_Reveal_Object *obj, unsigned int &index_in_word) -{ - Chunk_Header *chunk; - unsigned int index; - - if(is_super_obj(obj)){ - chunk = ABNORMAL_CHUNK_HEADER(obj); - index = 0; - } else { - chunk = NORMAL_CHUNK_HEADER(obj); - index = slot_addr_to_index(chunk, obj); - } - unsigned int word_index = index / SLOT_NUM_PER_WORD_IN_TABLE; - index_in_word = COLOR_BITS_PER_OBJ * (index % SLOT_NUM_PER_WORD_IN_TABLE); - - return &chunk->table[word_index]; -} -inline POINTER_SIZE_INT *get_color_word_in_table(Partial_Reveal_Object *obj, unsigned int &index_in_word, unsigned int size) -{ - Chunk_Header *chunk; - unsigned int index; - - if(size > SUPER_OBJ_THRESHOLD){ - chunk = ABNORMAL_CHUNK_HEADER(obj); - index = 0; - } else { - chunk = NORMAL_CHUNK_HEADER(obj); - index = slot_addr_to_index(chunk, obj); - } - unsigned int word_index = index >> 3; - index_in_word = COLOR_BITS_PER_OBJ * (index & (((unsigned int)(SLOT_NUM_PER_WORD_IN_TABLE-1)))); - - return &chunk->table[word_index]; -} - -#if 0 -/* Accurate marking: TRUE stands for being marked by this collector, and FALSE for another collector */ -inline Boolean obj_mark_in_table(Partial_Reveal_Object *obj) -{ - volatile POINTER_SIZE_INT *p_color_word; - unsigned int index_in_word; - p_color_word = get_color_word_in_table(obj, index_in_word); - assert(p_color_word); - - POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); - POINTER_SIZE_INT mark_color = cur_mark_color << index_in_word; - - POINTER_SIZE_INT old_word = *p_color_word; - POINTER_SIZE_INT new_word = (old_word & color_bits_mask) | mark_color; - while(new_word != old_word) { - POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); - if(temp == old_word){ -#ifdef SSPACE_VERIFY -#ifndef SSPACE_VERIFY_FINREF - assert(obj_is_marked_in_vt(obj)); -#endif - obj_unmark_in_vt(obj); - sspace_record_mark(obj, vm_object_size(obj)); -#endif - return TRUE; - } - old_word = *p_color_word; - new_word = (old_word & color_bits_mask) | mark_color; - } - - return FALSE; -} - -#endif - -inline Boolean obj_is_mark_gray_in_table(Partial_Reveal_Object *obj) -{ - POINTER_SIZE_INT *p_color_word; - unsigned int index_in_word; - p_color_word = get_color_word_in_table(obj, index_in_word); - POINTER_SIZE_INT current_word = *p_color_word; - POINTER_SIZE_INT mark_gray_color = cur_mark_gray_color << index_in_word; - POINTER_SIZE_INT mark_black_color = cur_mark_black_color<< index_in_word; - - - if(current_word & mark_gray_color && !(current_word & mark_black_color)) - return TRUE; - else - return FALSE; -} - -inline Boolean obj_is_mark_black_in_table(Partial_Reveal_Object *obj) -{ - POINTER_SIZE_INT *p_color_word; - unsigned int index_in_word; - p_color_word = get_color_word_in_table(obj, index_in_word); - POINTER_SIZE_INT current_word = *p_color_word; - POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; - - if(current_word & mark_black_color) - return TRUE; - else - return FALSE; - -} - -inline Boolean obj_is_mark_black_in_table(Partial_Reveal_Object *obj, unsigned int size) -{ - POINTER_SIZE_INT *p_color_word; - unsigned int index_in_word; - p_color_word = get_color_word_in_table(obj, index_in_word, size); - POINTER_SIZE_INT current_word = *p_color_word; - POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; - - if(current_word & mark_black_color) - return TRUE; - else - return FALSE; - -} - - -inline Boolean obj_mark_gray_in_table(Partial_Reveal_Object *obj) -{ - volatile POINTER_SIZE_INT *p_color_word; - unsigned int index_in_word; - p_color_word = get_color_word_in_table(obj, index_in_word); - assert(p_color_word); - - //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); - POINTER_SIZE_INT mark_color = cur_mark_gray_color << index_in_word; - - POINTER_SIZE_INT old_word = *p_color_word; - if(old_word & mark_color) return FALSE; /*already marked gray or black.*/ - - //POINTER_SIZE_INT new_word = (old_word & color_bits_mask) | mark_color; - POINTER_SIZE_INT new_word = old_word | mark_color; - while(new_word != old_word) { - POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); - if(temp == old_word){ - return TRUE; /*returning true does not mean it's marked by this thread. */ - } - old_word = *p_color_word; - if(old_word & mark_color) return FALSE; /*already marked gray or black.*/ - - //new_word = (old_word & color_bits_mask) | mark_color; - new_word = old_word | mark_color; - } - - return FALSE; -} - -inline Boolean obj_mark_black_in_table(Partial_Reveal_Object *obj, unsigned int size) -{ - //assert(obj_is_mark_in_table(obj)); - volatile POINTER_SIZE_INT *p_color_word; - unsigned int index_in_word; - p_color_word = get_color_word_in_table(obj, index_in_word, size); - assert(p_color_word); - - //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); - POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; - - POINTER_SIZE_INT old_word = *p_color_word; - if(old_word & mark_black_color) return FALSE; /*already marked black*/ - - POINTER_SIZE_INT new_word = old_word | mark_black_color; - while(new_word != old_word) { - POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); - if(temp == old_word){ - return TRUE; /*returning true does not mean it's marked by this thread. */ - } - old_word = *p_color_word; - if(old_word & mark_black_color) return FALSE; /*already marked black*/ - - new_word = old_word | mark_black_color; - } - - return FALSE; - -} - -inline Boolean obj_mark_black_in_table(Partial_Reveal_Object *obj) -{ - // assert(obj_is_mark_in_table(obj)); - volatile POINTER_SIZE_INT *p_color_word; - unsigned int index_in_word; - p_color_word = get_color_word_in_table(obj, index_in_word); - assert(p_color_word); - - POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); - POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; - - POINTER_SIZE_INT old_word = *p_color_word; - if(obj_is_mark_black_in_table(obj)) return FALSE; /*already marked black*/ - - POINTER_SIZE_INT new_word = old_word | mark_black_color; - while(new_word != old_word) { - POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); - if(temp == old_word){ - return TRUE; /*returning true does not mean it's marked by this thread. */ - } - old_word = *p_color_word; - if(obj_is_mark_black_in_table(obj)) return FALSE; /*already marked black*/ - - new_word = old_word | mark_black_color; - } - - return FALSE; -} - -inline Boolean obj_dirty_in_table(Partial_Reveal_Object *obj) -{ - volatile POINTER_SIZE_INT *p_color_word; - unsigned int index_in_word; - p_color_word = get_color_word_in_table(obj, index_in_word); - assert(p_color_word); - - POINTER_SIZE_INT obj_dirty_bit_in_word = OBJ_DIRTY_BIT_IN_TABLE<< index_in_word; - - POINTER_SIZE_INT old_word = *p_color_word; - if(old_word & obj_dirty_bit_in_word) return FALSE; - - POINTER_SIZE_INT new_word = old_word | obj_dirty_bit_in_word; - while(new_word != old_word) { - POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); - if(temp == old_word){ - return TRUE; /*returning true does not mean it's marked by this thread. */ - } - old_word = *p_color_word; - if(old_word & obj_dirty_bit_in_word) return FALSE; - - new_word = old_word | obj_dirty_bit_in_word; - } - - return FALSE; -} - -inline Boolean obj_is_dirty_in_table(Partial_Reveal_Object *obj) -{ - POINTER_SIZE_INT *p_color_word; - unsigned int index_in_word; - p_color_word = get_color_word_in_table(obj, index_in_word); - POINTER_SIZE_INT current_word = *p_color_word; - POINTER_SIZE_INT obj_dirty_bit_in_word = OBJ_DIRTY_BIT_IN_TABLE<< index_in_word; - - - if(current_word & obj_dirty_bit_in_word) - return TRUE; - else - return FALSE; -} - -inline Boolean obj_is_alloc_in_color_table(Partial_Reveal_Object *obj) -{ - POINTER_SIZE_INT *p_color_word; - unsigned int index_in_word; - p_color_word = get_color_word_in_table(obj, index_in_word); - POINTER_SIZE_INT current_word = *p_color_word; - POINTER_SIZE_INT obj_alloc_color_bit_in_word = cur_alloc_color << index_in_word; - - return (Boolean)(current_word & obj_alloc_color_bit_in_word); -} - -inline Boolean obj_need_take_snaptshot(Partial_Reveal_Object *obj) -{ - return !obj_is_mark_black_in_table(obj) && !obj_is_dirty_in_table(obj); -} - -inline void collector_add_free_chunk(Collector *collector, Free_Chunk *chunk) -{ - Free_Chunk_List *list = collector->free_chunk_list; - - chunk->status = CHUNK_FREE | CHUNK_TO_MERGE; - chunk->next = list->head; - chunk->prev = NULL; - if(list->head) - list->head->prev = chunk; - else - list->tail = chunk; - list->head = chunk; -} - - -extern void sspace_mark_scan(Collector *collector, Sspace *sspace); -extern void sspace_fallback_mark_scan(Collector *collector, Sspace *sspace); -extern void gc_init_chunk_for_sweep(GC *gc, Sspace *sspace); -extern void sspace_sweep(Collector *collector, Sspace *sspace); -extern void sspace_compact(Collector *collector, Sspace *sspace); -extern void sspace_merge_free_chunks(GC *gc, Sspace *sspace); -extern void sspace_remerge_free_chunks(GC *gc, Sspace *sspace); -extern Chunk_Header_Basic *sspace_grab_next_chunk(Sspace *sspace, Chunk_Header_Basic *volatile *shared_next_chunk, Boolean need_construct); - -extern void pfc_set_slot_index(Chunk_Header *chunk, unsigned int first_free_word_index, POINTER_SIZE_INT alloc_color); -extern void pfc_reset_slot_index(Chunk_Header *chunk); - -#endif // _SSPACE_MARK_SWEEP_H_ Index: src/mark_sweep/sspace_sweep.cpp =================================================================== --- src/mark_sweep/sspace_sweep.cpp (revision 606795) +++ src/mark_sweep/sspace_sweep.cpp (working copy) @@ -1,273 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sspace_chunk.h" -#include "sspace_mark_sweep.h" - - -static Chunk_Header_Basic *volatile next_chunk_for_sweep; - - -void gc_init_chunk_for_sweep(GC *gc, Sspace *sspace) -{ - next_chunk_for_sweep = (Chunk_Header_Basic*)space_heap_start((Space*)sspace); - next_chunk_for_sweep->adj_prev = NULL; - - unsigned int i = gc->num_collectors; - while(i--){ - Free_Chunk_List *list = gc->collectors[i]->free_chunk_list; - assert(!list->head); - assert(!list->tail); - assert(list->lock == FREE_LOCK); - } -} - -static unsigned int word_set_bit_num(POINTER_SIZE_INT word) -{ - unsigned int count = 0; - - while(word){ - word &= word - 1; - ++count; - } - return count; -} - -void zeroing_free_chunk(Free_Chunk *chunk) -{ - assert(chunk->status == CHUNK_FREE); - - void *start = (void*)((POINTER_SIZE_INT)chunk + sizeof(Free_Chunk)); - POINTER_SIZE_INT size = CHUNK_SIZE(chunk) - sizeof(Free_Chunk); - memset(start, 0, size); -} - -/* Zeroing should be optimized to do it while sweeping index word */ -static void zeroing_free_areas_in_pfc(Chunk_Header *chunk, unsigned int live_num) -{ - assert(live_num); - - assert(chunk->status & CHUNK_NORMAL); - unsigned int slot_num = chunk->slot_num; - unsigned int slot_size = chunk->slot_size; - POINTER_SIZE_INT chunk_base = (POINTER_SIZE_INT)chunk->base; - POINTER_SIZE_INT *table = chunk->table; - - POINTER_SIZE_INT base = (POINTER_SIZE_INT)NULL; - assert(slot_num >= live_num); - unsigned int free_slot_num = slot_num - live_num; - unsigned int cur_free_slot_num = 0; - unsigned int slot_index = chunk->slot_index; - unsigned int word_index = slot_index / SLOT_NUM_PER_WORD_IN_TABLE; - assert(live_num >= slot_index); - live_num -= slot_index; - POINTER_SIZE_INT index_word = table[word_index]; - POINTER_SIZE_INT mark_color = cur_mark_black_color << (COLOR_BITS_PER_OBJ * (slot_index % SLOT_NUM_PER_WORD_IN_TABLE)); - for(; slot_index < slot_num; ++slot_index){ - assert(!(index_word & ~cur_mark_mask)); - if(index_word & mark_color){ - if(cur_free_slot_num){ - memset((void*)base, 0, slot_size*cur_free_slot_num); - assert(free_slot_num >= cur_free_slot_num); - free_slot_num -= cur_free_slot_num; - cur_free_slot_num = 0; - if(!free_slot_num) break; - } - assert(live_num); - --live_num; - } else { - if(cur_free_slot_num){ - ++cur_free_slot_num; - } else { - base = chunk_base + slot_size * slot_index; - cur_free_slot_num = 1; - if(!live_num) break; - } - } - mark_color <<= COLOR_BITS_PER_OBJ; - if(!mark_color){ - mark_color = cur_mark_black_color; - ++word_index; - index_word = table[word_index]; - while(index_word == cur_mark_mask && cur_free_slot_num == 0 && slot_index < slot_num){ - slot_index += SLOT_NUM_PER_WORD_IN_TABLE; - ++word_index; - index_word = table[word_index]; - assert(live_num >= SLOT_NUM_PER_WORD_IN_TABLE); - live_num -= SLOT_NUM_PER_WORD_IN_TABLE; - } - while(index_word == 0 && cur_free_slot_num > 0 && slot_index < slot_num){ - slot_index += SLOT_NUM_PER_WORD_IN_TABLE; - ++word_index; - index_word = table[word_index]; - cur_free_slot_num += SLOT_NUM_PER_WORD_IN_TABLE; - } - } - } - assert((cur_free_slot_num>0 && live_num==0) || (cur_free_slot_num==0 && live_num>0)); - if(cur_free_slot_num) - memset((void*)base, 0, slot_size*free_slot_num); -} - -static void collector_sweep_normal_chunk(Collector *collector, Sspace *sspace, Chunk_Header *chunk) -{ - unsigned int slot_num = chunk->slot_num; - unsigned int live_num = 0; - unsigned int first_free_word_index = MAX_SLOT_INDEX; - POINTER_SIZE_INT *table = chunk->table; - - unsigned int index_word_num = (slot_num + SLOT_NUM_PER_WORD_IN_TABLE - 1) / SLOT_NUM_PER_WORD_IN_TABLE; - for(unsigned int i=0; ialloc_num = live_num; - collector->live_obj_size += live_num * chunk->slot_size; - collector->live_obj_num += live_num; - - if(!live_num){ /* all objects in this chunk are dead */ - collector_add_free_chunk(collector, (Free_Chunk*)chunk); - } else if(chunk_is_reusable(chunk)){ /* most objects in this chunk are swept, add chunk to pfc list*/ - chunk->alloc_num = live_num; - chunk_pad_last_index_word((Chunk_Header*)chunk, cur_mark_mask); - sspace_put_pfc(sspace, chunk); - } - /* the rest: chunks with free rate < PFC_REUSABLE_RATIO. we don't use them */ -} - -static inline void collector_sweep_abnormal_chunk(Collector *collector, Sspace *sspace, Chunk_Header *chunk) -{ - assert(chunk->status == CHUNK_ABNORMAL); - POINTER_SIZE_INT *table = chunk->table; - table[0] &= cur_mark_mask; - if(!table[0]){ - collector_add_free_chunk(collector, (Free_Chunk*)chunk); - } - else { - collector->live_obj_size += CHUNK_SIZE(chunk); - collector->live_obj_num++; - } -} - -void sspace_sweep(Collector *collector, Sspace *sspace) -{ - Chunk_Header_Basic *chunk; - collector->live_obj_size = 0; - collector->live_obj_num = 0; - - chunk = sspace_grab_next_chunk(sspace, &next_chunk_for_sweep, TRUE); - while(chunk){ - /* chunk is free before GC */ - if(chunk->status == CHUNK_FREE){ - collector_add_free_chunk(collector, (Free_Chunk*)chunk); - } else if(chunk->status & CHUNK_NORMAL){ /* chunk is used as a normal sized obj chunk */ - collector_sweep_normal_chunk(collector, sspace, (Chunk_Header*)chunk); - } else { /* chunk is used as a super obj chunk */ - collector_sweep_abnormal_chunk(collector, sspace, (Chunk_Header*)chunk); - } - - chunk = sspace_grab_next_chunk(sspace, &next_chunk_for_sweep, TRUE); - } -} - -/************ For merging free chunks in sspace ************/ - -static void merge_free_chunks_in_list(Sspace *sspace, Free_Chunk_List *list) -{ - Free_Chunk *sspace_ceiling = (Free_Chunk*)space_heap_end((Space*)sspace); - Free_Chunk *chunk = list->head; - - while(chunk){ - assert(chunk->status == (CHUNK_FREE | CHUNK_TO_MERGE)); - /* Remove current chunk from the chunk list */ - list->head = chunk->next; - if(list->head) - list->head->prev = NULL; - /* Check if the prev adjacent chunks are free */ - Free_Chunk *prev_chunk = (Free_Chunk*)chunk->adj_prev; - while(prev_chunk && prev_chunk->status == (CHUNK_FREE | CHUNK_TO_MERGE)){ - assert(prev_chunk < chunk); - /* Remove prev_chunk from list */ - free_list_detach_chunk(list, prev_chunk); - prev_chunk->adj_next = chunk->adj_next; - chunk = prev_chunk; - prev_chunk = (Free_Chunk*)chunk->adj_prev; - } - /* Check if the back adjcent chunks are free */ - Free_Chunk *back_chunk = (Free_Chunk*)chunk->adj_next; - while(back_chunk < sspace_ceiling && back_chunk->status == (CHUNK_FREE | CHUNK_TO_MERGE)){ - assert(chunk < back_chunk); - /* Remove back_chunk from list */ - free_list_detach_chunk(list, back_chunk); - back_chunk = (Free_Chunk*)back_chunk->adj_next; - chunk->adj_next = (Chunk_Header_Basic*)back_chunk; - } - if(back_chunk < sspace_ceiling) - back_chunk->adj_prev = (Chunk_Header_Basic*)chunk; - - /* put the free chunk to the according free chunk list */ - sspace_put_free_chunk(sspace, chunk); - - chunk = list->head; - } -} - -void sspace_merge_free_chunks(GC *gc, Sspace *sspace) -{ - Free_Chunk_List free_chunk_list; - free_chunk_list.head = NULL; - free_chunk_list.tail = NULL; - - /* Collect free chunks from collectors to one list */ - for(unsigned int i=0; inum_collectors; ++i){ - Free_Chunk_List *list = gc->collectors[i]->free_chunk_list; - move_free_chunks_between_lists(&free_chunk_list, list); - } - - merge_free_chunks_in_list(sspace, &free_chunk_list); -} - -void sspace_remerge_free_chunks(GC *gc, Sspace *sspace) -{ - Free_Chunk_List free_chunk_list; - free_chunk_list.head = NULL; - free_chunk_list.tail = NULL; - - /* If a new chunk is partitioned from a bigger one in the forwarding phase, - * its adj_prev has not been set yet. - * And the adj_prev field of the chunk next to it will be wrong either. - * So a rebuilding operation is needed here. - */ - sspace_rebuild_chunk_chain(sspace); - - /* Collect free chunks from sspace free chunk lists to one list */ - sspace_collect_free_chunks_to_list(sspace, &free_chunk_list); - - /* Collect free chunks from collectors to one list */ - for(unsigned int i=0; inum_collectors; ++i){ - Free_Chunk_List *list = gc->collectors[i]->free_chunk_list; - move_free_chunks_between_lists(&free_chunk_list, list); - } - - merge_free_chunks_in_list(sspace, &free_chunk_list); -} Index: src/mark_sweep/sspace_verify.cpp =================================================================== --- src/mark_sweep/sspace_verify.cpp (revision 606795) +++ src/mark_sweep/sspace_verify.cpp (working copy) @@ -1,676 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sspace_verify.h" -#include "sspace_chunk.h" -#include "sspace_mark_sweep.h" -#include "../utils/vector_block.h" -#include "gc_ms.h" -#include "../gen/gen.h" -#include "../finalizer_weakref/finalizer_weakref.h" - -#ifdef SSPACE_VERIFY - -#define VERIFY_CARD_SIZE_BYTES_SHIFT 12 -#define VERIFY_CARD_SIZE_BYTES (1 << VERIFY_CARD_SIZE_BYTES_SHIFT) -#define VERIFY_CARD_LOW_MASK (VERIFY_CARD_SIZE_BYTES - 1) -#define VERIFY_CARD_HIGH_MASK (~VERIFY_CARD_LOW_MASK) - -#define VERIFY_MAX_OBJ_SIZE_BYTES (1 << (32-VERIFY_CARD_SIZE_BYTES_SHIFT)) - -typedef struct Verify_Card { - SpinLock lock; - Vector_Block *block; -} Verify_Card; - -typedef unsigned int Obj_Addr; - -static GC *gc_in_verify = NULL; -static Verify_Card *alloc_verify_cards = NULL; -static Verify_Card *mark_verify_cards = NULL; -static POINTER_SIZE_INT card_num = 0; -static volatile POINTER_SIZE_INT alloc_obj_num = 0; -static volatile POINTER_SIZE_INT live_obj_in_mark = 0; -static volatile POINTER_SIZE_INT live_obj_in_fix = 0; - -void sspace_verify_init(GC *gc) -{ - gc_in_verify = gc; - - Sspace *sspace = gc_get_sspace(gc); - POINTER_SIZE_INT space_size = space_committed_size((Space*)sspace); - card_num = space_size >> VERIFY_CARD_SIZE_BYTES_SHIFT; - POINTER_SIZE_INT cards_size = sizeof(Verify_Card) * card_num; - - alloc_verify_cards = (Verify_Card*)STD_MALLOC(cards_size); - memset(alloc_verify_cards, 0, cards_size); - - mark_verify_cards = (Verify_Card*)STD_MALLOC(cards_size); - memset(mark_verify_cards, 0, cards_size); -} - -static Obj_Addr compose_obj_addr(unsigned int offset, unsigned int size) -{ - assert(size < VERIFY_MAX_OBJ_SIZE_BYTES); - return offset | (size << VERIFY_CARD_SIZE_BYTES_SHIFT); -} - -static void *decompose_obj_addr(Obj_Addr obj_addr, POINTER_SIZE_INT card_index, unsigned int & size) -{ - assert(card_index < card_num); - POINTER_SIZE_INT card_offset = obj_addr & VERIFY_CARD_LOW_MASK; - POINTER_SIZE_INT heap_offset = VERIFY_CARD_SIZE_BYTES * card_index + card_offset; - size = (obj_addr & VERIFY_CARD_HIGH_MASK) >> VERIFY_CARD_SIZE_BYTES_SHIFT; - assert(size < VERIFY_MAX_OBJ_SIZE_BYTES); - return (void*)(heap_offset + (POINTER_SIZE_INT)gc_heap_base(gc_in_verify)); -} - -static Boolean obj_addr_overlapped(Obj_Addr addr1, Obj_Addr addr2) -{ - unsigned int offset1 = addr1 & VERIFY_CARD_LOW_MASK; - unsigned int size1 = (addr1 & VERIFY_CARD_HIGH_MASK) >> VERIFY_CARD_SIZE_BYTES_SHIFT; - unsigned int ceiling1 = offset1 + size1; - unsigned int offset2 = addr2 & VERIFY_CARD_LOW_MASK; - unsigned int size2 = (addr2 & VERIFY_CARD_HIGH_MASK) >> VERIFY_CARD_SIZE_BYTES_SHIFT; - unsigned int ceiling2 = offset2 + size2; - - unsigned int reason = 0; - if(offset1 == offset2) - reason = 1; - if((offset1 < offset2) && (ceiling1 > offset2)) - reason = 2; - if((offset2 < offset1) && (ceiling2 > offset1)) - reason = 3; - if(!reason) - return FALSE; - printf("\nreason: %d\nold offset: %x size: %d\nnew offset: %x size: %d", reason, (POINTER_SIZE_INT)offset1, size1, (POINTER_SIZE_INT)offset2, size2); - return TRUE; -} - -static Vector_Block *create_vector_block(unsigned int size) -{ - Vector_Block *block = (Vector_Block*)STD_MALLOC(size); - vector_block_init(block, size); - return block; -} - -static void verify_card_get_block(Verify_Card *card) -{ - lock(card->lock); - if(card->block){ - unlock(card->lock); - return; - } - card->block = create_vector_block(VECTOR_BLOCK_DATA_SIZE_BYTES); - unlock(card->lock); -} - -void sspace_verify_alloc(void *addr, unsigned int size) -{ - assert(address_belongs_to_gc_heap(addr, gc_in_verify)); - atomic_inc32(&alloc_obj_num); - - unsigned int heap_offset = (unsigned int)((POINTER_SIZE_INT)addr - (POINTER_SIZE_INT)gc_heap_base(gc_in_verify)); - unsigned int card_offset = heap_offset & VERIFY_CARD_LOW_MASK; - Verify_Card *card = &alloc_verify_cards[heap_offset >> VERIFY_CARD_SIZE_BYTES_SHIFT]; - - verify_card_get_block(card); - Vector_Block *block = card->block; - - Obj_Addr obj_addr = compose_obj_addr(card_offset, size); - - lock(card->lock); - Obj_Addr *p_addr = block->head; - while(p_addr < block->tail){ - assert(!obj_addr_overlapped(obj_addr, *p_addr)); - p_addr++; - } - vector_block_add_entry(block, obj_addr); - unlock(card->lock); -} - -/* size is rounded up size */ -static Boolean obj_position_is_correct(void *addr, unsigned int size) -{ - Chunk_Header *chunk = NULL; - - if(size <= SUPER_OBJ_THRESHOLD) - chunk = NORMAL_CHUNK_HEADER(addr); - else - chunk = ABNORMAL_CHUNK_HEADER(addr); - if(chunk->slot_size != size) return FALSE; - if(((POINTER_SIZE_INT)addr - (POINTER_SIZE_INT)chunk->base) % size != 0) return FALSE; - return TRUE; -} - -static void sspace_verify_weakref(Partial_Reveal_Object *p_obj) -{ - WeakReferenceType type = special_reference_type(p_obj); - if(type == NOT_REFERENCE) return; - - REF *p_referent_field = obj_get_referent_field(p_obj); - Partial_Reveal_Object *p_referent = read_slot(p_referent_field); - if (!p_referent) return; - - unsigned int size = vm_object_size(p_referent); - if(size <= SUPER_OBJ_THRESHOLD){ - Sspace *sspace = gc_get_sspace(gc_in_verify); - Size_Segment *size_seg = sspace_get_size_seg(sspace, size); - size = NORMAL_SIZE_ROUNDUP(size, size_seg); - } - - assert(obj_position_is_correct(p_referent, size)); -} - -static void mark_card_add_entry(void *addr, unsigned int size) -{ - assert(address_belongs_to_gc_heap(addr, gc_in_verify)); - - unsigned int heap_offset = (unsigned int)((POINTER_SIZE_INT)addr - (POINTER_SIZE_INT)gc_heap_base(gc_in_verify)); - unsigned int card_offset = heap_offset & VERIFY_CARD_LOW_MASK; - Verify_Card *card = &mark_verify_cards[heap_offset >> VERIFY_CARD_SIZE_BYTES_SHIFT]; - - verify_card_get_block(card); - Vector_Block *block = card->block; - - if(size <= SUPER_OBJ_THRESHOLD){ - Sspace *sspace = gc_get_sspace(gc_in_verify); - Size_Segment *size_seg = sspace_get_size_seg(sspace, size); - size = NORMAL_SIZE_ROUNDUP(size, size_seg); - } - - assert(obj_position_is_correct(addr, size)); - Obj_Addr obj_addr = compose_obj_addr(card_offset, size); - - lock(card->lock); - Obj_Addr *p_addr = block->head; - while(p_addr < block->tail){ - assert(!obj_addr_overlapped(obj_addr, *p_addr)); - p_addr++; - } - vector_block_add_entry(block, obj_addr); - unlock(card->lock); - -} - -/* size is real size of obj */ -void sspace_record_mark(void *addr, unsigned int size) -{ - atomic_inc32(&live_obj_in_mark); - mark_card_add_entry(addr, size); -} - -static void verify_mark(void *addr, unsigned int size, Boolean destructively) -{ - assert(address_belongs_to_gc_heap(addr, gc_in_verify)); - - unsigned int heap_offset = (unsigned int)((POINTER_SIZE_INT)addr - (POINTER_SIZE_INT)gc_heap_base(gc_in_verify)); - unsigned int card_offset = heap_offset & VERIFY_CARD_LOW_MASK; - Verify_Card *card = &mark_verify_cards[heap_offset >> VERIFY_CARD_SIZE_BYTES_SHIFT]; - - Vector_Block *block = card->block; - assert(block); - - Obj_Addr obj_addr = compose_obj_addr(card_offset, size); - - Obj_Addr *p_addr = block->head; - while(p_addr < block->tail){ - if(obj_addr == *p_addr){ - if(destructively) - *p_addr = 0; - break; - } - p_addr++; - } - assert(p_addr < block->tail); -} - -void sspace_modify_mark_in_compact(void *new_addr, void *old_addr, unsigned int size) -{ - /* Verify the old addr and remove it in the according mark card */ - verify_mark(old_addr, size, TRUE); - /* Add new_addr into mark card */ - mark_card_add_entry(new_addr, size); -} - -void sspace_verify_fix_in_compact(void) -{ - atomic_inc32(&live_obj_in_fix); -} - -static void check_and_clear_mark_cards(void) -{ - for(POINTER_SIZE_INT i=0; ihead; - while(p_addr < block->tail){ - if(*p_addr){ - unsigned int size = 0; - void *addr = NULL; - addr = decompose_obj_addr(*p_addr, i, size); - printf("Extra mark obj: %x size: %d\n", (POINTER_SIZE_INT)addr, size); - } - p_addr++; - } - vector_block_clear(block); - } -} - -static void clear_alloc_cards(void) -{ - for(POINTER_SIZE_INT i=0; iblock) - vector_block_clear(card->block); - } -} - -static void summarize_sweep_verify(GC *gc) -{ - POINTER_SIZE_INT live_obj_num = 0; - for(unsigned int i=0; inum_collectors; ++i){ - live_obj_num += gc->collectors[i]->live_obj_num; - } - printf("Live obj in sweeping: %d\n", live_obj_num); -} - -void sspace_verify_free_area(POINTER_SIZE_INT *start, POINTER_SIZE_INT size) -{ - POINTER_SIZE_INT *p_value = start; - - assert(!(size % BYTES_PER_WORD)); - size /= BYTES_PER_WORD; - while(size--) - assert(!*p_value++); -} - -static POINTER_SIZE_INT sspace_live_obj_num(Sspace *sspace, Boolean gc_finished) -{ - Chunk_Header *chunk = (Chunk_Header*)space_heap_start((Space*)sspace); - Chunk_Header *sspace_ceiling = (Chunk_Header*)space_heap_end((Space*)sspace); - POINTER_SIZE_INT live_num = 0; - - for(; chunk < sspace_ceiling; chunk = (Chunk_Header*)CHUNK_END(chunk)){ - /* chunk is free before GC */ - if(chunk->status & CHUNK_FREE){ - assert((gc_finished && chunk->status==CHUNK_FREE) - || (!gc_finished && chunk->status==(CHUNK_FREE|CHUNK_TO_MERGE))); - continue; - } - if(chunk->status & CHUNK_ABNORMAL){ - assert(chunk->status == CHUNK_ABNORMAL); - assert(chunk->slot_size > SUPER_OBJ_THRESHOLD); - Partial_Reveal_Object *obj = (Partial_Reveal_Object*)chunk->base; - assert(chunk->slot_size == vm_object_size(obj)); - assert(get_obj_info_raw(obj) & SUPER_OBJ_MASK); - } - /* chunk is used as a normal or abnormal one in which there are live objects */ - unsigned int slot_num = chunk->slot_num; - POINTER_SIZE_INT *table = chunk->table; - POINTER_SIZE_INT live_num_in_chunk = 0; - - unsigned int word_index = 0; - for(unsigned int i=0; islot_size, gc_finished); - if(gc_finished){ - sspace_verify_alloc(p_obj, chunk->slot_size); - sspace_verify_weakref((Partial_Reveal_Object*)p_obj); - } - } - } - live_num += live_num_in_chunk; - } - - return live_num; -} - -static void allocator_verify_local_chunks(Allocator *allocator) -{ - Sspace *sspace = gc_get_sspace(allocator->gc); - Size_Segment **size_segs = sspace->size_segments; - Chunk_Header ***local_chunks = allocator->local_chunks; - - for(unsigned int i = SIZE_SEGMENT_NUM; i--;){ - if(!size_segs[i]->local_alloc){ - assert(!local_chunks[i]); - continue; - } - Chunk_Header **chunks = local_chunks[i]; - assert(chunks); - for(unsigned int j = size_segs[i]->chunk_num; j--;){ - assert(!chunks[j]); - } - } -} - -static void gc_verify_allocator_local_chunks(GC *gc) -{ - if(gc_match_kind(gc, MARK_SWEEP_GC)){ - Mutator *mutator = gc->mutator_list; - while(mutator){ - allocator_verify_local_chunks((Allocator*)mutator); - mutator = mutator->next; - } - } - - if(gc_match_kind(gc, MAJOR_COLLECTION)) - for(unsigned int i = gc->num_collectors; i--;){ - allocator_verify_local_chunks((Allocator*)gc->collectors[i]); - } -} - -void sspace_verify_before_collection(GC *gc) -{ - printf("Allocated obj: %d\n", alloc_obj_num); - alloc_obj_num = 0; -} - -void sspace_verify_after_sweep(GC *gc) -{ - printf("Live obj in marking: %d\n", live_obj_in_mark); - live_obj_in_mark = 0; - - summarize_sweep_verify(gc); - - Sspace *sspace = gc_get_sspace(gc); - POINTER_SIZE_INT total_live_obj = sspace_live_obj_num(sspace, FALSE); - printf("Live obj after sweep: %d\n", total_live_obj); -} - -void sspace_verify_after_collection(GC *gc) -{ - printf("Live obj in fixing: %d\n", live_obj_in_fix); - live_obj_in_fix = 0; - - clear_alloc_cards(); - - Sspace *sspace = gc_get_sspace(gc); - POINTER_SIZE_INT total_live_obj = sspace_live_obj_num(sspace, TRUE); - printf("Live obj after collection: %d\n", total_live_obj); - check_and_clear_mark_cards(); - gc_verify_allocator_local_chunks(gc); -} - -/* -void sspace_verify_super_obj(GC *gc) -{ - Sspace *sspace = gc_get_sspace(gc); - Chunk_Header *chunk = (Chunk_Header*)space_heap_start((Space*)sspace); - Chunk_Header *sspace_ceiling = (Chunk_Header*)space_heap_end((Space*)sspace); - - for(; chunk < sspace_ceiling; chunk = (Chunk_Header*)CHUNK_END(chunk)){ - if(chunk->status & CHUNK_ABNORMAL){ - assert(chunk->status == CHUNK_ABNORMAL); - assert(chunk->slot_size > SUPER_OBJ_THRESHOLD); - Partial_Reveal_Object *obj = (Partial_Reveal_Object*)chunk->base; - assert(chunk->slot_size == vm_object_size(obj)); - assert(get_obj_info_raw(obj) & SUPER_OBJ_MASK); - } - } -} -*/ - - -/* sspace verify marking with vtable marking in advance */ - -Sspace *sspace_in_verifier; -static Pool *trace_pool = NULL; -static Vector_Block *trace_stack = NULL; -POINTER_SIZE_INT live_obj_in_verify_marking = 0; - -static Boolean obj_mark_in_vtable(GC *gc, Partial_Reveal_Object *obj) -{ - assert(address_belongs_to_gc_heap(obj, gc)); - assert((vm_object_size(obj) <= SUPER_OBJ_THRESHOLD) || (get_obj_info_raw(obj) & SUPER_OBJ_MASK)); - Boolean marked = obj_mark_in_vt(obj); -#ifdef SSPACE_VERIFY - if(marked) live_obj_in_verify_marking++; -#endif - return marked; -} - -static void tracestack_push(void *p_obj) -{ - vector_stack_push(trace_stack, (POINTER_SIZE_INT)p_obj); - - if( !vector_stack_is_full(trace_stack)) return; - - pool_put_entry(trace_pool, trace_stack); - trace_stack = free_task_pool_get_entry(&gc_metadata); - assert(trace_stack); -} - -static FORCE_INLINE void scan_slot(GC *gc, REF *p_ref) -{ - Partial_Reveal_Object *p_obj = read_slot(p_ref); - if( p_obj == NULL) return; - - if(obj_belongs_to_space(p_obj, (Space*)sspace_in_verifier) && obj_mark_in_vtable(gc, p_obj)) - tracestack_push(p_obj); - - return; -} - -static FORCE_INLINE void scan_object(GC *gc, Partial_Reveal_Object *p_obj) -{ - if(!object_has_ref_field(p_obj) ) return; - - REF *p_ref; - - if (object_is_array(p_obj)) { /* scan array object */ - - Partial_Reveal_Array *array = (Partial_Reveal_Array*)p_obj; - unsigned int array_length = array->array_len; - - p_ref = (REF*)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); - - for (unsigned int i = 0; i < array_length; i++) { - scan_slot(gc, p_ref+i); - } - } else { /* scan non-array object */ - - unsigned int num_refs = object_ref_field_num(p_obj); - - int *ref_iterator = object_ref_iterator_init(p_obj); - - for(unsigned int i=0; imetadata; - Pool *rootset_pool = metadata->gc_rootset_pool; - - trace_stack = free_task_pool_get_entry(metadata); - trace_pool = sync_pool_create(); - - pool_iterator_init(rootset_pool); - Vector_Block *root_set = pool_iterator_next(rootset_pool); - - while(root_set){ - POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); - while(!vector_block_iterator_end(root_set, iter)){ - REF *p_ref = (REF*)*iter; - iter = vector_block_iterator_advance(root_set, iter); - - Partial_Reveal_Object *p_obj = read_slot(p_ref); - assert(p_obj!=NULL); - if(obj_belongs_to_space(p_obj, (Space*)sspace_in_verifier) && obj_mark_in_vtable(gc, p_obj)) - tracestack_push(p_obj); - } - root_set = pool_iterator_next(metadata->gc_rootset_pool); - } - /* put back the last trace_stack task */ - pool_put_entry(trace_pool, trace_stack); - - /* second step: iterate over the mark tasks and scan objects */ - /* get a task buf for the mark stack */ - trace_stack = free_task_pool_get_entry(metadata); - - Vector_Block *mark_task = pool_get_entry(trace_pool); - - while(mark_task){ - POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task); - while(!vector_block_iterator_end(mark_task, iter)){ - Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)*iter; - iter = vector_block_iterator_advance(mark_task, iter); - - trace_object(gc, p_obj); - } - /* run out one task, put back to the pool and grab another task */ - vector_stack_clear(mark_task); - pool_put_entry(metadata->free_task_pool, mark_task); - mark_task = pool_get_entry(trace_pool); - } - - /* put back the last mark stack to the free pool */ - vector_stack_clear(trace_stack); - pool_put_entry(metadata->free_task_pool, trace_stack); - trace_stack = NULL; - sync_pool_destruct(trace_pool); - trace_pool = NULL; - printf("Live obj in vtable marking: %d\n", live_obj_in_verify_marking); - live_obj_in_verify_marking = 0; -} - - -#endif - - - -#ifdef SSPACE_TIME - -inline uint64 tsc() -{ - __asm _emit 0x0F; - __asm _emit 0x31 -} - -#define CPU_HZ 3000000 // per ms - -static uint64 gc_start_time; -static uint64 mark_start_time; -static uint64 sweep_start_time; -static uint64 compact_start_time; -static uint64 fix_start_time; -static uint64 merge_start_time; - -void sspace_gc_time(GC *gc, Boolean before_gc) -{ - if(before_gc){ - gc_start_time = tsc(); - mark_start_time = gc_start_time; - } else { - uint64 end_time = tsc(); - assert(end_time > gc_start_time); - printf("\n\nGC %d time: %dms\n\n", gc->num_collections, (end_time-gc_start_time) / CPU_HZ); - } -} - -void sspace_mark_time(Boolean before_mark) -{ - assert(before_mark == FALSE); - if(before_mark){ - mark_start_time = tsc(); - } else { - uint64 end_time = tsc(); - assert(end_time > mark_start_time); - printf("\nMark time: %dms\n", (end_time-mark_start_time) / CPU_HZ); - sweep_start_time = end_time; - } -} - -void sspace_sweep_time(Boolean before_sweep, Boolean sspace_need_compact) -{ - assert(before_sweep == FALSE); - if(before_sweep){ - sweep_start_time = tsc(); - } else { - uint64 end_time = tsc(); - assert(end_time > sweep_start_time); - printf("\nSweep time: %dms\n", (end_time-sweep_start_time) / CPU_HZ); - if(sspace_need_compact) - compact_start_time = end_time; - else - merge_start_time = end_time; - } -} - -void sspace_compact_time(Boolean before_compact) -{ - assert(before_compact == FALSE); - if(before_compact){ - compact_start_time = tsc(); - } else { - uint64 end_time = tsc(); - assert(end_time > compact_start_time); - printf("\nCompact time: %dms\n", (end_time-compact_start_time) / CPU_HZ); - fix_start_time = end_time; - } -} - -void sspace_fix_time(Boolean before_fix) -{ - assert(before_fix == FALSE); - if(before_fix){ - fix_start_time = tsc(); - } else { - uint64 end_time = tsc(); - assert(end_time > fix_start_time); - printf("\nFix time: %dms\n", (end_time-fix_start_time) / CPU_HZ); - merge_start_time = end_time; - } -} - -void sspace_merge_time(Boolean before_merge) -{ - assert(before_merge == FALSE); - if(before_merge){ - merge_start_time = tsc(); - } else { - uint64 end_time = tsc(); - assert(end_time > merge_start_time); - printf("\nMerge time: %dms\n\n", (end_time-merge_start_time) / CPU_HZ); - } -} - -#endif Index: src/mark_sweep/sspace_verify.h =================================================================== --- src/mark_sweep/sspace_verify.h (revision 606795) +++ src/mark_sweep/sspace_verify.h (working copy) @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _SSPACE_VERIFY_H_ -#define _SSPACE_VERIFY_H_ - -#include "../common/gc_common.h" - -//#define SSPACE_VERIFY -//#define SSPACE_VERIFY_FINREF -//#define SSPACE_CHUNK_INFO -//#define SSPACE_ALLOC_INFO -//#define SSPACE_TIME - -struct Sspace; - -void sspace_verify_init(GC *gc); -void sspace_verify_alloc(void *addr, unsigned int size); -void sspace_verify_vtable_mark(GC *gc); -void sspace_record_mark(void *addr, unsigned int size); -void sspace_modify_mark_in_compact(void *new_addr, void *old_addr, unsigned int size); -void sspace_verify_fix_in_compact(void); -void sspace_verify_free_area(POINTER_SIZE_INT *start, POINTER_SIZE_INT size); -void sspace_verify_before_collection(GC *gc); -void sspace_verify_after_sweep(GC *gc); -void sspace_verify_after_collection(GC *gc); - -void sspace_chunks_info(Sspace *sspace, Boolean show_info); -void sspace_alloc_info(unsigned int size); -void sspace_alloc_info_summary(void); - -void sspace_gc_time(GC *gc, Boolean before_gc); -void sspace_mark_time(Boolean before_mark); -void sspace_sweep_time(Boolean before_sweep, Boolean sspace_need_compact); -void sspace_compact_time(Boolean before_compact); -void sspace_fix_time(Boolean before_fix); -void sspace_merge_time(Boolean before_merge); - -#endif // _SSPACE_VERIFY_H_ Index: src/mark_sweep/wspace.cpp =================================================================== --- src/mark_sweep/wspace.cpp (revision 0) +++ src/mark_sweep/wspace.cpp (revision 0) @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wspace.h" +#include "wspace_chunk.h" +#include "wspace_verify.h" +#include "gc_ms.h" +#include "../gen/gen.h" + +struct GC_Gen; + +Wspace *wspace_initialize(GC *gc, void *start, POINTER_SIZE_INT wspace_size, POINTER_SIZE_INT commit_size) +{ + /* With wspace in the heap, the heap must be composed of a single wspace or a wspace and a NOS. + * In either case, the reserved size and committed size of wspace must be the same. + * Because wspace has only mark-sweep collection, it is not possible to shrink wspace. + * So there is no need to use dynamic space resizing. + */ + assert(wspace_size == commit_size); + + Wspace *wspace = (Wspace*)STD_MALLOC(sizeof(Wspace)); + assert(wspace); + memset(wspace, 0, sizeof(Wspace)); + + wspace->reserved_heap_size = wspace_size; + + void *reserved_base = start; + + /* commit wspace mem */ + if(!large_page_hint) + vm_commit_mem(reserved_base, commit_size); + memset(reserved_base, 0, commit_size); + wspace->committed_heap_size = commit_size; + + wspace->heap_start = reserved_base; + wspace->heap_end = (void *)((POINTER_SIZE_INT)reserved_base + wspace_size); + + wspace->num_collections = 0; + wspace->time_collections = 0; + wspace->survive_ratio = 0.2f; + + wspace->move_object = FALSE; + wspace->gc = gc; + + wspace_init_chunks(wspace); + + wspace->space_statistic = (Space_Statistics*)STD_MALLOC(sizeof(Space_Statistics)); + assert(wspace->space_statistic); + memset(wspace->space_statistic, 0, sizeof(Space_Statistics)); + +#ifdef USE_MARK_SWEEP_GC + gc_ms_set_wspace((GC_MS*)gc, wspace); +#else + gc_set_mos((GC_Gen*)gc, (Space*)wspace); +#endif + +#ifdef SSPACE_VERIFY + wspace_verify_init(gc); +#endif + return wspace; +} + +static void wspace_destruct_chunks(Wspace *wspace) { return; } + +void wspace_destruct(Wspace *wspace) +{ + //FIXME:: when map the to-half, the decommission start address should change + wspace_destruct_chunks(wspace); + STD_FREE(wspace); +} + +void wspace_reset_after_collection(Wspace *wspace) +{ + wspace->move_object = FALSE; + wspace->need_compact = FALSE; + wspace->need_fix = FALSE; +} + +void allocator_init_local_chunks(Allocator *allocator) +{ + Wspace *wspace = gc_get_wspace(allocator->gc); + Size_Segment **size_segs = wspace->size_segments; + + /* Alloc mem for size segments (Chunk_Header**) */ + unsigned int seg_size = sizeof(Chunk_Header**) * SIZE_SEGMENT_NUM; + Chunk_Header ***local_chunks = (Chunk_Header***)STD_MALLOC(seg_size); + memset(local_chunks, 0, seg_size); + + /* Alloc mem for local chunk pointers */ + unsigned int chunk_ptr_size = 0; + for(unsigned int i = SIZE_SEGMENT_NUM; i--;){ + if(size_segs[i]->local_alloc){ + chunk_ptr_size += size_segs[i]->chunk_num; + } + } + chunk_ptr_size *= sizeof(Chunk_Header*); + Chunk_Header **chunk_ptrs = (Chunk_Header**)STD_MALLOC(chunk_ptr_size); + memset(chunk_ptrs, 0, chunk_ptr_size); + + for(unsigned int i = 0; i < SIZE_SEGMENT_NUM; ++i){ + if(size_segs[i]->local_alloc){ + local_chunks[i] = chunk_ptrs; + chunk_ptrs += size_segs[i]->chunk_num; + } + } + + allocator->local_chunks = local_chunks; +} + +void allocactor_destruct_local_chunks(Allocator *allocator) +{ + Wspace *wspace = gc_get_wspace(allocator->gc); + Size_Segment **size_segs = wspace->size_segments; + Chunk_Header ***local_chunks = allocator->local_chunks; + Chunk_Header **chunk_ptrs = NULL; + unsigned int chunk_ptr_num = 0; + + /* Find local chunk pointers' head and their number */ + for(unsigned int i = 0; i < SIZE_SEGMENT_NUM; ++i){ + if(size_segs[i]->local_alloc){ + chunk_ptr_num += size_segs[i]->chunk_num; + assert(local_chunks[i]); + if(!chunk_ptrs) + chunk_ptrs = local_chunks[i]; + } + } + + /* Put local pfc to the according pools */ + for(unsigned int i = 0; i < chunk_ptr_num; ++i){ + if(chunk_ptrs[i]) + wspace_put_pfc(wspace, chunk_ptrs[i]); + } + + /* Free mem for local chunk pointers */ + STD_FREE(chunk_ptrs); + + /* Free mem for size segments (Chunk_Header**) */ + STD_FREE(local_chunks); +} + +static void allocator_clear_local_chunks(Allocator *allocator) +{ + Wspace *wspace = gc_get_wspace(allocator->gc); + Size_Segment **size_segs = wspace->size_segments; + Chunk_Header ***local_chunks = allocator->local_chunks; + + for(unsigned int i = SIZE_SEGMENT_NUM; i--;){ + if(!size_segs[i]->local_alloc){ + assert(!local_chunks[i]); + continue; + } + Chunk_Header **chunks = local_chunks[i]; + assert(chunks); + for(unsigned int j = size_segs[i]->chunk_num; j--;){ + if(chunks[j]) + wspace_put_pfc(wspace, chunks[j]); + chunks[j] = NULL; + } + } +} + +static void gc_clear_mutator_local_chunks(GC *gc) +{ +#ifdef USE_MARK_SWEEP_GC + /* release local chunks of each mutator in unique mark-sweep GC */ + Mutator *mutator = gc->mutator_list; + while(mutator){ + allocator_clear_local_chunks((Allocator*)mutator); + mutator = mutator->next; + } +#endif +} + +void gc_clear_collector_local_chunks(GC *gc) +{ + if(!gc_match_kind(gc, MAJOR_COLLECTION)) return; + /* release local chunks of each collector in gen GC */ + for(unsigned int i = gc->num_collectors; i--;){ + allocator_clear_local_chunks((Allocator*)gc->collectors[i]); + } +} + +#ifdef USE_MARK_SWEEP_GC +void wspace_set_space_statistic(Wspace *wspace) +{ + GC_MS *gc = (GC_MS*)wspace->gc; + + for(unsigned int i = 0; i < gc->num_collectors; ++i){ + wspace->surviving_obj_num += gc->collectors[i]->live_obj_num; + wspace->surviving_obj_size += gc->collectors[i]->live_obj_size; + } +} +#endif + +extern void wspace_decide_compaction_need(Wspace *wspace); +extern void mark_sweep_wspace(Collector *collector); + +void wspace_collection(Wspace *wspace) +{ + GC *gc = wspace->gc; + wspace->num_collections++; + + gc_clear_mutator_local_chunks(gc); + gc_clear_collector_local_chunks(gc); + +#ifdef SSPACE_ALLOC_INFO + wspace_alloc_info_summary(); +#endif +#ifdef SSPACE_CHUNK_INFO + wspace_chunks_info(wspace, FALSE); +#endif + wspace_clear_used_chunk_pool(wspace); + + + wspace_decide_compaction_need(wspace); + if(wspace->need_compact && gc_match_kind(gc, MARK_SWEEP_GC)){ + assert(gc_match_kind(gc, MS_COLLECTION)); + gc->collect_kind = MS_COMPACT_COLLECTION; + } + if(wspace->need_compact || gc_match_kind(gc, MAJOR_COLLECTION)) + wspace->need_fix = TRUE; + + //printf("\n\n>>>>>>>>%s>>>>>>>>>>>>\n\n", wspace->need_compact ? "COMPACT" : "NO COMPACT"); +#ifdef SSPACE_VERIFY + wspace_verify_before_collection(gc); + wspace_verify_vtable_mark(gc); +#endif + +#ifdef SSPACE_TIME + wspace_gc_time(gc, TRUE); +#endif + + pool_iterator_init(gc->metadata->gc_rootset_pool); + wspace_clear_chunk_list(wspace); + + collector_execute_task(gc, (TaskType)mark_sweep_wspace, (Space*)wspace); + +#ifdef SSPACE_TIME + wspace_gc_time(gc, FALSE); +#endif + +#ifdef SSPACE_CHUNK_INFO + wspace_chunks_info(wspace, FALSE); +#endif + +} Index: src/mark_sweep/wspace.h =================================================================== --- src/mark_sweep/wspace.h (revision 0) +++ src/mark_sweep/wspace.h (revision 0) @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _SWEEP_SPACE_H_ +#define _SWEEP_SPACE_H_ + +#include "../thread/gc_thread.h" +#include "../thread/collector_alloc.h" +#include "../thread/mutator.h" +#include "../common/gc_common.h" +#include "../common/gc_concurrent.h" + +/* + * The sweep space accomodates objects collected by mark-sweep + */ + +struct Size_Segment; +struct Free_Chunk_List; + +typedef struct Wspace { + /* <-- first couple of fields are overloadded as Space */ + void *heap_start; + void *heap_end; + POINTER_SIZE_INT reserved_heap_size; + POINTER_SIZE_INT committed_heap_size; + unsigned int num_collections; + int64 time_collections; + float survive_ratio; + unsigned int collect_algorithm; + GC *gc; + Boolean move_object; + + Space_Statistics* space_statistic; + + /* Size allocted since last minor collection. */ + volatile POINTER_SIZE_INT last_alloced_size; + /* Size allocted since last major collection. */ + volatile POINTER_SIZE_INT accumu_alloced_size; + /* Total size allocated since VM starts. */ + volatile POINTER_SIZE_INT total_alloced_size; + + /* Size survived from last collection. */ + POINTER_SIZE_INT last_surviving_size; + /* Size survived after a certain period. */ + POINTER_SIZE_INT period_surviving_size; + + /* END of Space --> */ + + Boolean need_compact; + Boolean need_fix; /* There are repointed ref needing fixing */ + Size_Segment **size_segments; + Pool ***pfc_pools; + Pool ***pfc_pools_backup; + Pool* used_chunk_pool; + Pool* unreusable_normal_chunk_pool; + Pool* live_abnormal_chunk_pool; + Free_Chunk_List *aligned_free_chunk_lists; + Free_Chunk_List *unaligned_free_chunk_lists; + Free_Chunk_List *hyper_free_chunk_list; + POINTER_SIZE_INT surviving_obj_num; + POINTER_SIZE_INT surviving_obj_size; +} Wspace; + +#ifdef USE_MARK_SWEEP_GC +void wspace_set_space_statistic(Wspace *wspace); +#endif + +Wspace *wspace_initialize(GC *gc, void *start, POINTER_SIZE_INT wspace_size, POINTER_SIZE_INT commit_size); +void wspace_destruct(Wspace *wspace); +void wspace_reset_after_collection(Wspace *wspace); + +void *wspace_thread_local_alloc(unsigned size, Allocator *allocator); +void *wspace_alloc(unsigned size, Allocator *allocator); + +void wspace_collection(Wspace *wspace); + +void allocator_init_local_chunks(Allocator *allocator); +void allocactor_destruct_local_chunks(Allocator *allocator); +void gc_init_collector_free_chunk_list(Collector *collector); + +POINTER_SIZE_INT wspace_free_memory_size(Wspace *wspace); + + +#ifndef USE_MARK_SWEEP_GC +#define gc_get_wspace(gc) ((Wspace*)gc_get_mos((GC_Gen*)(gc))) +#else +#define gc_get_wspace(gc) (gc_ms_get_wspace((GC_MS*)(gc))) +#endif + +#endif // _SWEEP_SPACE_H_ Index: src/mark_sweep/wspace_alloc.cpp =================================================================== --- src/mark_sweep/wspace_alloc.cpp (revision 0) +++ src/mark_sweep/wspace_alloc.cpp (revision 0) @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wspace.h" +#include "wspace_chunk.h" +//#include "wspace_mark_sweep.h" +#include "wspace_alloc.h" +#include "gc_ms.h" +#include "../gen/gen.h" +#include "../common/gc_concurrent.h" + + +/* Only used in pfc_set_slot_index() */ +inline unsigned int first_free_index_in_color_word(POINTER_SIZE_INT word, POINTER_SIZE_INT alloc_color) +{ + for(unsigned int index = 0; index < BITS_PER_WORD; index += COLOR_BITS_PER_OBJ) + if(!(word & (alloc_color << index))) + return index; + + assert(0); /* There must be a free obj in this table word */ + return MAX_SLOT_INDEX; +} + +/* Given an index word in table, set pfc's slot_index + * The value of argument alloc_color can be cur_alloc_color or cur_mark_color. + * It depends on in which phase this func is called. + * In sweeping phase, wspace has been marked but alloc and mark colors have not been flipped, + * so we have to use cur_mark_color as alloc_color. + * In compaction phase, two colors have been flipped, so we use cur_alloc_color. + */ +void pfc_set_slot_index(Chunk_Header *chunk, unsigned int first_free_word_index, POINTER_SIZE_INT alloc_color) +{ + unsigned int index_in_word = first_free_index_in_color_word(chunk->table[first_free_word_index], alloc_color); + assert(index_in_word != MAX_SLOT_INDEX); + chunk->slot_index = composed_slot_index(first_free_word_index, index_in_word); +} + +/* From the table's beginning search the first free slot, and set it to pfc's slot_index */ +void pfc_reset_slot_index(Chunk_Header *chunk) +{ + POINTER_SIZE_INT *table = chunk->table; + + unsigned int index_word_num = (chunk->slot_num + SLOT_NUM_PER_WORD_IN_TABLE - 1) / SLOT_NUM_PER_WORD_IN_TABLE; + for(unsigned int i=0; i LARGE_OBJ_THRESHOLD) return NULL; + + Wspace *wspace = gc_get_wspace(allocator->gc); + + /* Flexible alloc mechanism: + Size_Segment *size_seg = wspace_get_size_seg(wspace, size); + unsigned int seg_index = size_seg->seg_index; + */ + unsigned int seg_index = (size-GC_OBJECT_ALIGNMENT) / MEDIUM_OBJ_THRESHOLD; + assert(seg_index <= 2); + Size_Segment *size_seg = wspace->size_segments[seg_index]; + assert(size_seg->local_alloc); + + size = (unsigned int)NORMAL_SIZE_ROUNDUP(size, size_seg); + unsigned int index = NORMAL_SIZE_TO_INDEX(size, size_seg); + + Chunk_Header **chunks = allocator->local_chunks[seg_index]; + Chunk_Header *chunk = chunks[index]; + if(!chunk){ + mutator_post_signal((Mutator*) allocator,DISABLE_COLLECTOR_SWEEP_LOCAL_CHUNKS); + + chunk = wspace_get_pfc(wspace, seg_index, index); + //if(!chunk) chunk = wspace_steal_pfc(wspace, seg_index, index); + if(!chunk) return NULL; + chunk->status |= CHUNK_IN_USE; + chunks[index] = chunk; + + mutator_post_signal((Mutator*) allocator,ENABLE_COLLECTOR_SWEEP_LOCAL_CHUNKS); + } + void *p_obj = alloc_in_chunk(chunks[index]); + + if(chunk->slot_index == MAX_SLOT_INDEX){ + chunk->status = CHUNK_USED | CHUNK_NORMAL; + /*register to used chunk list.*/ + wspace_register_used_chunk(wspace,chunk); + chunks[index] = NULL; + chunk = NULL; + } + + assert(!chunk || chunk->slot_index <= chunk->alloc_num); + assert(!chunk || chunk->slot_index < chunk->slot_num); + assert(p_obj); + +#ifdef SSPACE_ALLOC_INFO + wspace_alloc_info(size); +#endif +#ifdef SSPACE_VERIFY + wspace_verify_alloc(p_obj, size); +#endif + + return p_obj; +} + +static void *wspace_alloc_normal_obj(Wspace *wspace, unsigned size, Allocator *allocator) +{ + Size_Segment *size_seg = wspace_get_size_seg(wspace, size); + unsigned int seg_index = size_seg->seg_index; + + size = (unsigned int)NORMAL_SIZE_ROUNDUP(size, size_seg); + unsigned int index = NORMAL_SIZE_TO_INDEX(size, size_seg); + + Chunk_Header *chunk = NULL; + void *p_obj = NULL; + + if(size_seg->local_alloc){ + Chunk_Header **chunks = allocator->local_chunks[seg_index]; + chunk = chunks[index]; + if(!chunk){ + mutator_post_signal((Mutator*) allocator,DISABLE_COLLECTOR_SWEEP_LOCAL_CHUNKS); + chunk = wspace_get_pfc(wspace, seg_index, index); + if(!chunk){ + chunk = (Chunk_Header*)wspace_get_normal_free_chunk(wspace); + if(chunk) normal_chunk_init(chunk, size); + } + //if(!chunk) chunk = wspace_steal_pfc(wspace, seg_index, index); + if(!chunk) return NULL; + chunk->status |= CHUNK_IN_USE; + chunks[index] = chunk; + mutator_post_signal((Mutator*) allocator,ENABLE_COLLECTOR_SWEEP_LOCAL_CHUNKS); + } + + p_obj = alloc_in_chunk(chunks[index]); + + if(chunk->slot_index == MAX_SLOT_INDEX){ + chunk->status = CHUNK_USED | CHUNK_NORMAL; + /*register to used chunk list.*/ + wspace_register_used_chunk(wspace,chunk); + chunks[index] = NULL; + } + + } else { + gc_try_schedule_collection(allocator->gc, GC_CAUSE_NIL); + + mutator_post_signal((Mutator*) allocator,DISABLE_COLLECTOR_SWEEP_GLOBAL_CHUNKS); + + if(USE_CONCURRENT_SWEEP){ + while(gc_is_sweeping_global_normal_chunk()){ + mutator_post_signal((Mutator*) allocator,ENABLE_COLLECTOR_SWEEP_GLOBAL_CHUNKS); + } + } + + chunk = wspace_get_pfc(wspace, seg_index, index); + if(!chunk){ + chunk = (Chunk_Header*)wspace_get_normal_free_chunk(wspace); + if(chunk) normal_chunk_init(chunk, size); + } + //if(!chunk) chunk = wspace_steal_pfc(wspace, seg_index, index); + if(!chunk) return NULL; + p_obj = alloc_in_chunk(chunk); + + if(chunk->slot_index == MAX_SLOT_INDEX){ + chunk->status = CHUNK_USED | CHUNK_NORMAL; + /*register to used chunk list.*/ + wspace_register_used_chunk(wspace,chunk); + chunk = NULL; + } + + if(chunk){ + wspace_put_pfc(wspace, chunk); + } + + mutator_post_signal((Mutator*) allocator,ENABLE_COLLECTOR_SWEEP_GLOBAL_CHUNKS); + } + + return p_obj; +} + +static void *wspace_alloc_super_obj(Wspace *wspace, unsigned size, Allocator *allocator) +{ + assert(size > SUPER_OBJ_THRESHOLD); + + gc_try_schedule_collection(allocator->gc, GC_CAUSE_NIL); + + unsigned int chunk_size = SUPER_SIZE_ROUNDUP(size); + assert(chunk_size > SUPER_OBJ_THRESHOLD); + assert(!(chunk_size & CHUNK_GRANULARITY_LOW_MASK)); + + Chunk_Header *chunk; + if(chunk_size <= HYPER_OBJ_THRESHOLD) + chunk = (Chunk_Header*)wspace_get_abnormal_free_chunk(wspace, chunk_size); + else + chunk = (Chunk_Header*)wspace_get_hyper_free_chunk(wspace, chunk_size, FALSE); + + if(!chunk) return NULL; + abnormal_chunk_init(chunk, chunk_size, size); + if(is_obj_alloced_live()){ + chunk->table[0] |= cur_mark_black_color ; + } + //FIXME: Need barrier here. + //apr_memory_rw_barrier(); + + chunk->table[0] |= cur_alloc_color; + set_super_obj_mask(chunk->base); + assert(chunk->status == CHUNK_ABNORMAL); + chunk->status = CHUNK_ABNORMAL| CHUNK_USED; + wspace_register_used_chunk(wspace, chunk); + assert(get_obj_info_raw((Partial_Reveal_Object*)chunk->base) & SUPER_OBJ_MASK); + return chunk->base; +} + +static void *wspace_try_alloc(unsigned size, Allocator *allocator) +{ + Wspace *wspace = gc_get_wspace(allocator->gc); + void *p_obj = NULL; + + if(size <= SUPER_OBJ_THRESHOLD) + p_obj = wspace_alloc_normal_obj(wspace, size, allocator); + else + p_obj = wspace_alloc_super_obj(wspace, size, allocator); + +#ifdef SSPACE_ALLOC_INFO + if(p_obj) wspace_alloc_info(size); +#endif +#ifdef SSPACE_VERIFY + if(p_obj) wspace_verify_alloc(p_obj, size); +#endif + +#ifdef WSPACE_CONCURRENT_GC_STATS + if(p_obj && gc_is_concurrent_mark_phase()) ((Partial_Reveal_Object*)p_obj)->obj_info |= NEW_OBJ_MASK; +#endif + + return p_obj; +} + +/* FIXME:: the collection should be seperated from the alloation */ +void *wspace_alloc(unsigned size, Allocator *allocator) +{ + void *p_obj = NULL; + + /* First, try to allocate object from TLB (thread local chunk) */ + p_obj = wspace_try_alloc(size, allocator); + if(p_obj) return p_obj; + + if(allocator->gc->in_collection) return NULL; + + vm_gc_lock_enum(); + /* after holding lock, try if other thread collected already */ + p_obj = wspace_try_alloc(size, allocator); + if(p_obj){ + vm_gc_unlock_enum(); + return p_obj; + } + gc_reclaim_heap(allocator->gc, GC_CAUSE_WSPACE_IS_FULL); + vm_gc_unlock_enum(); + +#ifdef SSPACE_CHUNK_INFO + printf("Failure size: %x\n", size); +#endif + + p_obj = wspace_try_alloc(size, allocator); + + return p_obj; +} Index: src/mark_sweep/wspace_alloc.h =================================================================== --- src/mark_sweep/wspace_alloc.h (revision 0) +++ src/mark_sweep/wspace_alloc.h (revision 0) @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _SSPACE_ALLOC_H_ +#define _SSPACE_ALLOC_H_ + +#include "wspace_chunk.h" +#include "wspace_mark_sweep.h" +#include "../common/gc_concurrent.h" +#include "../common/collection_scheduler.h" + +extern POINTER_SIZE_INT cur_alloc_color; +extern POINTER_SIZE_INT cur_alloc_mask; +extern POINTER_SIZE_INT cur_mark_mask; + + +inline Boolean slot_is_alloc_in_table(POINTER_SIZE_INT *table, unsigned int slot_index) +{ + unsigned int color_bits_index = slot_index * COLOR_BITS_PER_OBJ; + unsigned int word_index = color_bits_index / BITS_PER_WORD; + unsigned int index_in_word = color_bits_index % BITS_PER_WORD; + + return (Boolean)(table[word_index] & (cur_alloc_color << index_in_word)); +} + +inline unsigned int composed_slot_index(unsigned int word_index, unsigned int index_in_word) +{ + unsigned int color_bits_index = word_index*BITS_PER_WORD + index_in_word; + return color_bits_index/COLOR_BITS_PER_OBJ; +} + +inline unsigned int next_free_index_in_color_word(POINTER_SIZE_INT word, unsigned int index) +{ + while(index < BITS_PER_WORD){ + if(!(word & (cur_alloc_color << index))) + return index; + index += COLOR_BITS_PER_OBJ; + } + return MAX_SLOT_INDEX; +} + +inline unsigned int next_alloc_index_in_color_word(POINTER_SIZE_INT word, unsigned int index) +{ + while(index < BITS_PER_WORD){ + if(word & (cur_alloc_color << index)) + return index; + index += COLOR_BITS_PER_OBJ; + } + return MAX_SLOT_INDEX; +} + +inline unsigned int next_free_slot_index_in_table(POINTER_SIZE_INT *table, unsigned int slot_index, unsigned int slot_num) +{ + assert(slot_is_alloc_in_table(table, slot_index)); + ++slot_index; + + unsigned int max_word_index = ((slot_num-1) * COLOR_BITS_PER_OBJ) / BITS_PER_WORD; + + unsigned int color_bits_index = slot_index * COLOR_BITS_PER_OBJ; + unsigned int word_index = color_bits_index / BITS_PER_WORD; + unsigned int index_in_word = color_bits_index % BITS_PER_WORD; + + for(; word_index <= max_word_index; ++word_index, index_in_word = 0){ + if((table[word_index] & cur_alloc_mask) == cur_alloc_mask) + continue; + index_in_word = next_free_index_in_color_word(table[word_index], index_in_word); + if(index_in_word != MAX_SLOT_INDEX){ + assert(index_in_word < BITS_PER_WORD); + return composed_slot_index(word_index, index_in_word); + } + } + + return MAX_SLOT_INDEX; +} + +/* Only used in wspace compaction after sweeping now */ +inline unsigned int next_alloc_slot_index_in_table(POINTER_SIZE_INT *table, unsigned int slot_index, unsigned int slot_num) +{ + unsigned int max_word_index = ((slot_num-1) * COLOR_BITS_PER_OBJ) / BITS_PER_WORD; + + unsigned int color_bits_index = slot_index * COLOR_BITS_PER_OBJ; + unsigned int word_index = color_bits_index / BITS_PER_WORD; + unsigned int index_in_word = color_bits_index % BITS_PER_WORD; + + for(; word_index <= max_word_index; ++word_index, index_in_word = 0){ + if(!table[word_index]) + continue; + index_in_word = next_alloc_index_in_color_word(table[word_index], index_in_word); + if(index_in_word != MAX_SLOT_INDEX){ + assert(index_in_word < BITS_PER_WORD); + return composed_slot_index(word_index, index_in_word); + } + } + + return MAX_SLOT_INDEX; +} + +inline Partial_Reveal_Object *next_alloc_slot_in_chunk(Chunk_Header *chunk) +{ + POINTER_SIZE_INT *table = chunk->table; + + unsigned int slot_index = next_alloc_slot_index_in_table(table, chunk->slot_index, chunk->slot_num); + assert((slot_index == MAX_SLOT_INDEX) + || (slot_index < chunk->slot_num) && slot_is_alloc_in_table(table, slot_index)); + if(slot_index == MAX_SLOT_INDEX) + return NULL; + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)slot_index_to_addr(chunk, slot_index); + chunk->slot_index = slot_index + 1; + return p_obj; +} + +inline void clear_free_slot_in_table(POINTER_SIZE_INT *table, unsigned int ceiling_slot_index) +{ + assert(ceiling_slot_index && ceiling_slot_index != MAX_SLOT_INDEX); + unsigned int index_word_num = ceiling_slot_index / SLOT_NUM_PER_WORD_IN_TABLE; + memset(table, 0, BYTES_PER_WORD*index_word_num); + unsigned int bits_need_clear = ceiling_slot_index % SLOT_NUM_PER_WORD_IN_TABLE; + if(!bits_need_clear) return; + POINTER_SIZE_INT bit_mask = ~(((POINTER_SIZE_INT)1 << (bits_need_clear*COLOR_BITS_PER_OBJ)) - 1); + table[index_word_num] &= bit_mask; +} + +inline void alloc_slot_in_table(POINTER_SIZE_INT *table, unsigned int slot_index) +{ + assert(!slot_is_alloc_in_table(table, slot_index)); + + unsigned int color_bits_index = slot_index * COLOR_BITS_PER_OBJ; + unsigned int word_index = color_bits_index / BITS_PER_WORD; + unsigned int index_in_word = color_bits_index % BITS_PER_WORD; + + volatile POINTER_SIZE_INT *p_color_word = &table[word_index]; + assert(p_color_word); + + POINTER_SIZE_INT mark_alloc_color = cur_alloc_color << index_in_word; + + POINTER_SIZE_INT old_word = *p_color_word; + + POINTER_SIZE_INT new_word = old_word | mark_alloc_color; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + assert(!slot_is_alloc_in_table(table, slot_index)); + + new_word = old_word | mark_alloc_color; + } + return; + +} + +/* We don't enable fresh chunk alloc for now, + * because we observed perf down for the extra conditional statement when no many fresh chunks. + */ +//#define ENABLE_FRESH_CHUNK_ALLOC + +/* 1. No need of synchronization. This is an allocator local chunk. + * 2. If this chunk runs out of space, clear the chunk pointer. + * So it is important to give a parameter which is a local chunk pointer of a allocator while invoking this func. + */ +inline void *alloc_in_chunk(Chunk_Header* &chunk) +{ + POINTER_SIZE_INT *table = chunk->table; + unsigned int slot_index = chunk->slot_index; + + assert(chunk->alloc_num < chunk->slot_num); + ++chunk->alloc_num; + assert(chunk->base); + void *p_obj = (void*)((POINTER_SIZE_INT)chunk->base + ((POINTER_SIZE_INT)chunk->slot_size * slot_index)); + + /*mark black is placed here because of race condition between ops color flip. */ + if(p_obj && is_obj_alloced_live()) + obj_mark_black_in_table((Partial_Reveal_Object*)p_obj, chunk->slot_size); + + //FIXME: Need barrier here. + //apr_memory_rw_barrier(); + + alloc_slot_in_table(table, slot_index); + if(chunk->status & CHUNK_NEED_ZEROING) + memset(p_obj, 0, chunk->slot_size); +#ifdef SSPACE_VERIFY + wspace_verify_free_area((POINTER_SIZE_INT*)p_obj, chunk->slot_size); +#endif + +#ifdef ENABLE_FRESH_CHUNK_ALLOC + if(chunk->status & CHUNK_FRESH){ + ++slot_index; + chunk->slot_index = (slot_index < chunk->slot_num) ? slot_index : MAX_SLOT_INDEX; + } else +#endif + + chunk->slot_index = next_free_slot_index_in_table(table, slot_index, chunk->slot_num); + + if(chunk->alloc_num == chunk->slot_num) chunk->slot_index = MAX_SLOT_INDEX; + return p_obj; +} + +inline void set_super_obj_mask(void *large_obj) +{ ((Partial_Reveal_Object*)large_obj)->obj_info |= SUPER_OBJ_MASK; } + +#endif // _SSPACE_ALLOC_H_ Index: src/mark_sweep/wspace_chunk.cpp =================================================================== --- src/mark_sweep/wspace_chunk.cpp (revision 0) +++ src/mark_sweep/wspace_chunk.cpp (revision 0) @@ -0,0 +1,913 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "gc_ms.h" +#include "wspace.h" +#include "wspace_chunk.h" +#include "wspace_verify.h" + +/* PFC stands for partially free chunk */ +static Size_Segment *size_segments[SIZE_SEGMENT_NUM]; +static Pool **pfc_pools[SIZE_SEGMENT_NUM]; +static Pool **pfc_pools_backup[SIZE_SEGMENT_NUM]; +static Boolean *pfc_steal_flags[SIZE_SEGMENT_NUM]; + +static Free_Chunk_List aligned_free_chunk_lists[NUM_ALIGNED_FREE_CHUNK_BUCKET]; +static Free_Chunk_List unaligned_free_chunk_lists[NUM_UNALIGNED_FREE_CHUNK_BUCKET]; +static Free_Chunk_List hyper_free_chunk_list; + + +static void init_size_segment(Size_Segment *seg, unsigned int size_min, unsigned int size_max, unsigned int gran_shift_bits, Boolean local_alloc) +{ + seg->size_min = size_min; + seg->size_max = size_max; + seg->local_alloc = local_alloc; + seg->chunk_num = (seg->size_max - seg->size_min) >> gran_shift_bits; + seg->gran_shift_bits = gran_shift_bits; + seg->granularity = (POINTER_SIZE_INT)(1 << gran_shift_bits); + seg->gran_low_mask = seg->granularity - 1; + seg->gran_high_mask = ~seg->gran_low_mask; +} + +void wspace_init_chunks(Wspace *wspace) +{ + unsigned int i, j; + + /* Init size segments */ + Size_Segment *size_seg_start = (Size_Segment*)STD_MALLOC(sizeof(Size_Segment) * SIZE_SEGMENT_NUM); + for(i = SIZE_SEGMENT_NUM; i--;){ + size_segments[i] = size_seg_start + i; + size_segments[i]->seg_index = i; + } + init_size_segment(size_segments[0], 0, MEDIUM_OBJ_THRESHOLD, SMALL_GRANULARITY_BITS, SMALL_IS_LOCAL_ALLOC); + init_size_segment(size_segments[1], MEDIUM_OBJ_THRESHOLD, LARGE_OBJ_THRESHOLD, MEDIUM_GRANULARITY_BITS, MEDIUM_IS_LOCAL_ALLOC); + init_size_segment(size_segments[2], LARGE_OBJ_THRESHOLD, SUPER_OBJ_THRESHOLD, LARGE_GRANULARITY_BITS, LARGE_IS_LOCAL_ALLOC); + + /* Init partially free chunk pools */ + for(i = SIZE_SEGMENT_NUM; i--;){ + pfc_pools[i] = (Pool**)STD_MALLOC(sizeof(Pool*) * size_segments[i]->chunk_num); + pfc_pools_backup[i] = (Pool**)STD_MALLOC(sizeof(Pool*) * size_segments[i]->chunk_num); + pfc_steal_flags[i] = (Boolean*)STD_MALLOC(sizeof(Boolean) * size_segments[i]->chunk_num); + for(j=size_segments[i]->chunk_num; j--;){ + pfc_pools[i][j] = sync_pool_create(); + pfc_pools_backup[i][j] = sync_pool_create(); + pfc_steal_flags[i][j] = FALSE; + } + } + + /* Init aligned free chunk lists */ + for(i = NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) + free_chunk_list_init(&aligned_free_chunk_lists[i]); + + /* Init nonaligned free chunk lists */ + for(i = NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) + free_chunk_list_init(&unaligned_free_chunk_lists[i]); + + /* Init super free chunk lists */ + free_chunk_list_init(&hyper_free_chunk_list); + + wspace->size_segments = size_segments; + wspace->pfc_pools = pfc_pools; + wspace->pfc_pools_backup = pfc_pools_backup; + wspace->used_chunk_pool = sync_pool_create(); + wspace->unreusable_normal_chunk_pool = sync_pool_create(); + wspace->live_abnormal_chunk_pool = sync_pool_create(); + wspace->aligned_free_chunk_lists = aligned_free_chunk_lists; + wspace->unaligned_free_chunk_lists = unaligned_free_chunk_lists; + wspace->hyper_free_chunk_list = &hyper_free_chunk_list; + + /* Init the first free chunk: from heap start to heap end */ + Free_Chunk *free_chunk = (Free_Chunk*)wspace->heap_start; + free_chunk->adj_next = (Chunk_Header_Basic*)wspace->heap_end; + POINTER_SIZE_INT chunk_size = wspace->reserved_heap_size; + assert(chunk_size > CHUNK_GRANULARITY && !(chunk_size % CHUNK_GRANULARITY)); + wspace_put_free_chunk(wspace, free_chunk); +} + +static void pfc_pool_set_steal_flag(Pool *pool, unsigned int steal_threshold, Boolean &steal_flag) +{ + Chunk_Header *chunk = (Chunk_Header*)pool_get_entry(pool); + while(chunk){ + steal_threshold--; + if(!steal_threshold) + break; + chunk = chunk->next; + } + steal_flag = steal_threshold ? FALSE : TRUE; +} + +void wspace_clear_chunk_list(Wspace* wspace) +{ + unsigned int i, j; + GC* gc = wspace->gc; + unsigned int collector_num = gc->num_collectors; + unsigned int steal_threshold = collector_num << PFC_STEAL_THRESHOLD; + + Pool*** pfc_pools = wspace->pfc_pools; + for(i = SIZE_SEGMENT_NUM; i--;){ + for(j = size_segments[i]->chunk_num; j--;){ + Pool *pool = pfc_pools[i][j]; + pfc_pool_set_steal_flag(pool, steal_threshold, pfc_steal_flags[i][j]); + pool_empty(pool); + } + } + + Pool*** pfc_pools_backup = wspace->pfc_pools_backup; + for(i = SIZE_SEGMENT_NUM; i--;){ + for(j = size_segments[i]->chunk_num; j--;){ + Pool *pool = pfc_pools_backup[i][j]; + pfc_pool_set_steal_flag(pool, steal_threshold, pfc_steal_flags[i][j]); + pool_empty(pool); + } + } + + for(i=NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) + free_chunk_list_clear(&aligned_free_chunk_lists[i]); + + for(i=NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) + free_chunk_list_clear(&unaligned_free_chunk_lists[i]); + + free_chunk_list_clear(&hyper_free_chunk_list); +} + +/* Simply put the free chunk to the according list + * Don't merge continuous free chunks + * The merging job is executed in merging phase + */ +static void list_put_free_chunk(Free_Chunk_List *list, Free_Chunk *chunk) +{ + chunk->status = CHUNK_FREE; + chunk->prev = NULL; + + lock(list->lock); + chunk->next = list->head; + if(list->head) + list->head->prev = chunk; + list->head = chunk; + if(!list->tail) + list->tail = chunk; + assert(list->chunk_num < ~((unsigned int)0)); + ++list->chunk_num; + unlock(list->lock); +} + +static void list_put_free_chunk_to_head(Free_Chunk_List *list, Free_Chunk *chunk) +{ + + chunk->status = CHUNK_FREE; + chunk->prev = NULL; + chunk->next = NULL; + + lock(list->lock); + chunk->next = list->head; + if(list->head) + list->head->prev = chunk; + list->head = chunk; + if(!list->tail) + list->tail = chunk; + assert(list->chunk_num < ~((unsigned int)0)); + ++list->chunk_num; + unlock(list->lock); +} + +static void list_put_free_chunk_to_tail(Free_Chunk_List *list, Free_Chunk *chunk) +{ + //modified for concurrent sweep. + //chunk->status = CHUNK_FREE; + chunk->prev = NULL; + chunk->next = NULL; + + lock(list->lock); + chunk->prev = list->tail; + if(list->head) + list->tail->next = chunk; + list->tail = chunk; + if(!list->head) + list->head = chunk; + assert(list->chunk_num < ~((unsigned int)0)); + ++list->chunk_num; + unlock(list->lock); +} + + + +/* The difference between this and the above normal one is this func needn't hold the list lock. + * This is for the calling in partitioning chunk functions. + * Please refer to the comments of wspace_get_hyper_free_chunk(). + */ +static void list_put_hyper_free_chunk(Free_Chunk_List *list, Free_Chunk *chunk) +{ + chunk->status = CHUNK_FREE; + chunk->prev = NULL; + + /* lock(list->lock); + * the list lock must have been held like in getting a free chunk and partitioning it + * or needn't be held like in wspace initialization and the merging phase + */ + chunk->next = list->head; + if(list->head) + list->head->prev = chunk; + list->head = chunk; + if(!list->tail) + list->tail = chunk; + assert(list->chunk_num < ~((unsigned int)0)); + ++list->chunk_num; + //unlock(list->lock); +} + +static void list_put_hyper_free_chunk_to_head(Free_Chunk_List *list, Free_Chunk *chunk) +{ + chunk->status = CHUNK_FREE; + chunk->prev = NULL; + + chunk->next = list->head; + if(list->head) + list->head->prev = chunk; + list->head = chunk; + if(!list->tail) + list->tail = chunk; + assert(list->chunk_num < ~((unsigned int)0)); + ++list->chunk_num; +} + +static void list_put_hyper_free_chunk_to_tail(Free_Chunk_List *list, Free_Chunk *chunk) +{ + //modified for concurrent sweep. + //chunk->status = CHUNK_FREE; + chunk->next = NULL; + + chunk->prev = list->tail; + if(list->tail) + list->tail->next = chunk; + list->tail = chunk; + if(!list->head) + list->head = chunk; + assert(list->chunk_num < ~((unsigned int)0)); + ++list->chunk_num; +} + + + +static Free_Chunk *free_list_get_head(Free_Chunk_List *list) +{ + lock(list->lock); + Free_Chunk *chunk = list->head; + if(chunk){ + list->head = chunk->next; + if(list->head) + list->head->prev = NULL; + else + list->tail = NULL; + assert(list->chunk_num); + --list->chunk_num; + //assert(chunk->status == CHUNK_FREE); + assert(chunk->status & CHUNK_FREE); + } + unlock(list->lock); + return chunk; +} + +void wspace_put_free_chunk(Wspace *wspace, Free_Chunk *chunk) +{ + POINTER_SIZE_INT chunk_size = CHUNK_SIZE(chunk); + assert(!(chunk_size % CHUNK_GRANULARITY)); + + if(chunk_size > HYPER_OBJ_THRESHOLD) + list_put_hyper_free_chunk(wspace->hyper_free_chunk_list, chunk); + else if(!((POINTER_SIZE_INT)chunk & NORMAL_CHUNK_LOW_MASK) && !(chunk_size & NORMAL_CHUNK_LOW_MASK)) + list_put_free_chunk(&wspace->aligned_free_chunk_lists[ALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)], chunk); + else + list_put_free_chunk(&wspace->unaligned_free_chunk_lists[UNALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)], chunk); +} + +void wspace_put_free_chunk_to_head(Wspace *wspace, Free_Chunk *chunk) +{ + POINTER_SIZE_INT chunk_size = CHUNK_SIZE(chunk); + assert(!(chunk_size % CHUNK_GRANULARITY)); + + if(chunk_size > HYPER_OBJ_THRESHOLD){ + lock(wspace->hyper_free_chunk_list->lock); + list_put_hyper_free_chunk_to_head(wspace->hyper_free_chunk_list, chunk); + unlock(wspace->hyper_free_chunk_list->lock); + }else if(!((POINTER_SIZE_INT)chunk & NORMAL_CHUNK_LOW_MASK) && !(chunk_size & NORMAL_CHUNK_LOW_MASK)) + list_put_free_chunk_to_head(&wspace->aligned_free_chunk_lists[ALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)], chunk); + else + list_put_free_chunk_to_head(&wspace->unaligned_free_chunk_lists[UNALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)], chunk); + +} + +void wspace_put_free_chunk_to_tail(Wspace *wspace, Free_Chunk *chunk) +{ + POINTER_SIZE_INT chunk_size = CHUNK_SIZE(chunk); + assert(!(chunk_size % CHUNK_GRANULARITY)); + + if(chunk_size > HYPER_OBJ_THRESHOLD){ + lock(wspace->hyper_free_chunk_list->lock); + list_put_hyper_free_chunk_to_tail(wspace->hyper_free_chunk_list, chunk); + unlock(wspace->hyper_free_chunk_list->lock); + }else if(!((POINTER_SIZE_INT)chunk & NORMAL_CHUNK_LOW_MASK) && !(chunk_size & NORMAL_CHUNK_LOW_MASK)) + list_put_free_chunk_to_tail(&wspace->aligned_free_chunk_lists[ALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)], chunk); + else + list_put_free_chunk_to_tail(&wspace->unaligned_free_chunk_lists[UNALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)], chunk); + +} + +static inline Free_Chunk *partition_normal_free_chunk(Wspace *wspace, Free_Chunk *chunk) +{ + assert(CHUNK_SIZE(chunk) > NORMAL_CHUNK_SIZE_BYTES); + + Chunk_Header_Basic *adj_next = chunk->adj_next; + Free_Chunk *normal_chunk = (Free_Chunk*)(((POINTER_SIZE_INT)chunk + NORMAL_CHUNK_SIZE_BYTES-1) & NORMAL_CHUNK_HIGH_MASK); + + if(chunk != normal_chunk){ + assert(chunk < normal_chunk); + chunk->adj_next = (Chunk_Header_Basic*)normal_chunk; + wspace_put_free_chunk(wspace, chunk); + } + normal_chunk->adj_next = (Chunk_Header_Basic*)((POINTER_SIZE_INT)normal_chunk + NORMAL_CHUNK_SIZE_BYTES); + if(normal_chunk->adj_next != adj_next){ + assert(normal_chunk->adj_next < adj_next); + Free_Chunk *back_chunk = (Free_Chunk*)normal_chunk->adj_next; + back_chunk->adj_next = adj_next; + wspace_put_free_chunk(wspace, back_chunk); + } + + normal_chunk->status = CHUNK_FREE; + return normal_chunk; +} + +/* Partition the free chunk to two free chunks: + * the first one's size is chunk_size + * the second will be inserted into free chunk list according to its size + */ +static inline Free_Chunk *partition_abnormal_free_chunk(Wspace *wspace,Free_Chunk *chunk, unsigned int chunk_size) +{ + assert(CHUNK_SIZE(chunk) > chunk_size); + + Free_Chunk *new_chunk = (Free_Chunk*)((POINTER_SIZE_INT)chunk->adj_next - chunk_size); + assert(chunk < new_chunk); + + new_chunk->adj_next = chunk->adj_next; + chunk->adj_next = (Chunk_Header_Basic*)new_chunk; + wspace_put_free_chunk(wspace, chunk); + new_chunk->status = CHUNK_FREE; + return new_chunk; +} + +Free_Chunk *wspace_get_normal_free_chunk(Wspace *wspace) +{ + Free_Chunk_List *aligned_lists = wspace->aligned_free_chunk_lists; + Free_Chunk_List *unaligned_lists = wspace->unaligned_free_chunk_lists; + Free_Chunk_List *list = NULL; + Free_Chunk *chunk = NULL; + + /* Search in aligned chunk lists first */ + unsigned int index = 0; + while(index < NUM_ALIGNED_FREE_CHUNK_BUCKET){ + list = &aligned_lists[index]; + if(list->head) + chunk = free_list_get_head(list); + if(chunk){ + if(CHUNK_SIZE(chunk) > NORMAL_CHUNK_SIZE_BYTES) + chunk = partition_normal_free_chunk(wspace, chunk); + //zeroing_free_chunk(chunk); + return chunk; + } + index++; + } + assert(!chunk); + + /* Search in unaligned chunk lists with larger chunk. + (NORMAL_CHUNK_SIZE_BYTES + (NORMAL_CHUNK_SIZE_BYTES-CHUNK_GRANULARITY)) + is the smallest size which can guarantee the chunk includes a normal chunk. + */ + index = UNALIGNED_CHUNK_SIZE_TO_INDEX((NORMAL_CHUNK_SIZE_BYTES<<1) - CHUNK_GRANULARITY); + while(index < NUM_UNALIGNED_FREE_CHUNK_BUCKET){ + list = &unaligned_lists[index]; + if(list->head) + chunk = free_list_get_head(list); + if(chunk){ + chunk = partition_normal_free_chunk(wspace, chunk); + assert(!((POINTER_SIZE_INT)chunk & NORMAL_CHUNK_LOW_MASK)); + //zeroing_free_chunk(chunk); + return chunk; + } + index++; + } + assert(!chunk); + + /* search in the hyper free chunk list */ + chunk = wspace_get_hyper_free_chunk(wspace, NORMAL_CHUNK_SIZE_BYTES, TRUE); + assert(!((POINTER_SIZE_INT)chunk & NORMAL_CHUNK_LOW_MASK)); + + return chunk; +} + +Free_Chunk *wspace_get_abnormal_free_chunk(Wspace *wspace, unsigned int chunk_size) +{ + assert(chunk_size > CHUNK_GRANULARITY); + assert(!(chunk_size % CHUNK_GRANULARITY)); + assert(chunk_size <= HYPER_OBJ_THRESHOLD); + + Free_Chunk_List *unaligned_lists = wspace->unaligned_free_chunk_lists; + Free_Chunk_List *list = NULL; + Free_Chunk *chunk = NULL; + unsigned int index = 0; + + /* Search in the list with chunk size of multiple chunk_size */ + unsigned int search_size = chunk_size; + while(search_size <= HYPER_OBJ_THRESHOLD){ + index = UNALIGNED_CHUNK_SIZE_TO_INDEX(search_size); + list = &unaligned_lists[index]; + if(list->head) + chunk = free_list_get_head(list); + if(chunk){ + if(search_size > chunk_size) + chunk = partition_abnormal_free_chunk(wspace, chunk, chunk_size); + zeroing_free_chunk(chunk); + return chunk; + } + search_size += chunk_size; + } + assert(!chunk); + + /* search in the hyper free chunk list */ + chunk = wspace_get_hyper_free_chunk(wspace, chunk_size, FALSE); + if(chunk) return chunk; + + /* Search again in abnormal chunk lists */ + index = UNALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size); + while(index < NUM_UNALIGNED_FREE_CHUNK_BUCKET){ + list = &unaligned_lists[index]; + if(list->head) + chunk = free_list_get_head(list); + if(chunk){ + if(index > UNALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)) + chunk = partition_abnormal_free_chunk(wspace, chunk, chunk_size); + zeroing_free_chunk(chunk); + return chunk; + } + ++index; + } + + return chunk; +} + +Free_Chunk *wspace_get_hyper_free_chunk(Wspace *wspace, unsigned int chunk_size, Boolean is_normal_chunk) +{ + assert(chunk_size >= CHUNK_GRANULARITY); + assert(!(chunk_size % CHUNK_GRANULARITY)); + + Free_Chunk_List *list = wspace->hyper_free_chunk_list; + lock(list->lock); + + Free_Chunk *prev_chunk = NULL; + Free_Chunk *chunk = list->head; + while(chunk){ + if(CHUNK_SIZE(chunk) >= chunk_size){ + Free_Chunk *next_chunk = chunk->next; + if(prev_chunk) + prev_chunk->next = next_chunk; + else + list->head = next_chunk; + if(next_chunk) + next_chunk->prev = prev_chunk; + else + list->tail = prev_chunk; + break; + } + prev_chunk = chunk; + chunk = chunk->next; + } + + /* unlock(list->lock); + * We move this unlock to the end of this func for the following reason. + * A case might occur that two allocator are asking for a hyper chunk at the same time, + * and there is only one chunk in the list and it can satify the requirements of both of them. + * If allocator 1 gets the list lock first, it will get the unique chunk and releases the lock here. + * And then allocator 2 holds the list lock after allocator 1 releases it, + * it will found there is no hyper chunk in the list and return NULL. + * In fact the unique hyper chunk is large enough. + * If allocator 1 chops down one piece and put back the rest into the list, allocator 2 will be satisfied. + * So we will get a wrong info here if we release the lock here, which makes us invoke GC much earlier than needed. + */ + + if(chunk){ + if(is_normal_chunk) + chunk = partition_normal_free_chunk(wspace, chunk); + else if(CHUNK_SIZE(chunk) > chunk_size) + chunk = partition_abnormal_free_chunk(wspace, chunk, chunk_size); + if(!is_normal_chunk) + zeroing_free_chunk(chunk); + } + + unlock(list->lock); + + return chunk; +} + +void wspace_collect_free_chunks_to_list(Wspace *wspace, Free_Chunk_List *list) +{ + unsigned int i; + + for(i = NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) + move_free_chunks_between_lists(list, &wspace->aligned_free_chunk_lists[i]); + + for(i = NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) + move_free_chunks_between_lists(list, &wspace->unaligned_free_chunk_lists[i]); + + move_free_chunks_between_lists(list, wspace->hyper_free_chunk_list); + + Free_Chunk *chunk = list->head; + while(chunk){ + chunk->status = CHUNK_FREE | CHUNK_TO_MERGE; + chunk = chunk->next; + } +} + + +typedef struct PFC_Pool_Iterator { + volatile unsigned int seg_index; + volatile unsigned int chunk_index; + SpinLock lock; +} PFC_Pool_Iterator; + +static PFC_Pool_Iterator pfc_pool_iterator; + +void wspace_init_pfc_pool_iterator(Wspace *wspace) +{ + assert(pfc_pool_iterator.lock == FREE_LOCK); + pfc_pool_iterator.seg_index = 0; + pfc_pool_iterator.chunk_index = 0; +} + +void wspace_exchange_pfc_pool(Wspace *wspace) +{ + Pool*** empty_pfc_pools = wspace->pfc_pools; + wspace->pfc_pools = wspace->pfc_pools_backup; + wspace->pfc_pools_backup = empty_pfc_pools; +} + +Pool *wspace_grab_next_pfc_pool(Wspace *wspace) +{ + Pool*** pfc_pools = wspace->pfc_pools; + Pool *pfc_pool = NULL; + + lock(pfc_pool_iterator.lock); + for(; pfc_pool_iterator.seg_index < SIZE_SEGMENT_NUM; ++pfc_pool_iterator.seg_index){ + for(; pfc_pool_iterator.chunk_index < size_segments[pfc_pool_iterator.seg_index]->chunk_num; ++pfc_pool_iterator.chunk_index){ + pfc_pool = pfc_pools[pfc_pool_iterator.seg_index][pfc_pool_iterator.chunk_index]; + ++pfc_pool_iterator.chunk_index; + unlock(pfc_pool_iterator.lock); + return pfc_pool; + } + pfc_pool_iterator.chunk_index = 0; + } + unlock(pfc_pool_iterator.lock); + + return NULL; +} + +#define min_value(x, y) (((x) < (y)) ? (x) : (y)) + +Chunk_Header *wspace_steal_pfc(Wspace *wspace, unsigned int seg_index, unsigned int index) +{ + Size_Segment *size_seg = wspace->size_segments[seg_index]; + Chunk_Header *pfc = NULL; + unsigned int max_index = min_value(index + PFC_STEAL_NUM + 1, size_seg->chunk_num); + ++index; + for(; index < max_index; ++index){ + if(!pfc_steal_flags[seg_index][index]) continue; + pfc = wspace_get_pfc(wspace, seg_index, index); + if(pfc) return pfc; + } + return NULL; +} + +static POINTER_SIZE_INT free_mem_in_pfc_pools(Wspace *wspace, Boolean show_chunk_info) +{ + Size_Segment **size_segs = wspace->size_segments; + Pool ***pfc_pools = wspace->pfc_pools; + POINTER_SIZE_INT free_mem_size = 0; + + for(unsigned int i = 0; i < SIZE_SEGMENT_NUM; ++i){ + for(unsigned int j = 0; j < size_segs[i]->chunk_num; ++j){ + Pool *pfc_pool = pfc_pools[i][j]; + if(pool_is_empty(pfc_pool)) + continue; + pool_iterator_init(pfc_pool); + Chunk_Header *chunk = (Chunk_Header*)pool_iterator_next(pfc_pool); + assert(chunk); + unsigned int slot_num = chunk->slot_num; + unsigned int chunk_num = 0; + unsigned int alloc_num = 0; + while(chunk){ + assert(chunk->slot_num == slot_num); + ++chunk_num; + alloc_num += chunk->alloc_num; + chunk = (Chunk_Header*)pool_iterator_next(pfc_pool); + } + unsigned int total_slot_num = slot_num * chunk_num; + assert(alloc_num < total_slot_num); +#ifdef SSPACE_CHUNK_INFO + if(show_chunk_info) + printf("Size: %x\tchunk num: %d\tLive Ratio: %f\n", NORMAL_INDEX_TO_SIZE(j, size_segs[i]), chunk_num, (float)alloc_num/total_slot_num); +#endif + free_mem_size += NORMAL_INDEX_TO_SIZE(j, size_segs[i]) * (total_slot_num-alloc_num); + assert(free_mem_size < wspace->committed_heap_size); + } + } + + return free_mem_size; +} + +static POINTER_SIZE_INT free_mem_in_free_lists(Wspace *wspace, Free_Chunk_List *lists, unsigned int list_num, Boolean show_chunk_info) +{ + POINTER_SIZE_INT free_mem_size = 0; + + for(unsigned int index = 0; index < list_num; ++index){ + Free_Chunk *chunk = lists[index].head; + if(!chunk) continue; + POINTER_SIZE_INT chunk_size = CHUNK_SIZE(chunk); + assert(chunk_size <= HYPER_OBJ_THRESHOLD); + unsigned int chunk_num = 0; + while(chunk){ + assert(CHUNK_SIZE(chunk) == chunk_size); + ++chunk_num; + chunk = chunk->next; + } + free_mem_size += chunk_size * chunk_num; + assert(free_mem_size < wspace->committed_heap_size); +#ifdef SSPACE_CHUNK_INFO + if(show_chunk_info) + printf("Free Size: %x\tnum: %d\n", chunk_size, chunk_num); +#endif + } + + return free_mem_size; +} + +static POINTER_SIZE_INT free_mem_in_hyper_free_list(Wspace *wspace, Boolean show_chunk_info) +{ + POINTER_SIZE_INT free_mem_size = 0; + + Free_Chunk_List *list = wspace->hyper_free_chunk_list; + Free_Chunk *chunk = list->head; + while(chunk){ +#ifdef SSPACE_CHUNK_INFO + if(show_chunk_info) + printf("Size: %x\n", CHUNK_SIZE(chunk)); +#endif + free_mem_size += CHUNK_SIZE(chunk); + assert(free_mem_size <= wspace->committed_heap_size); + chunk = chunk->next; + } + + return free_mem_size; +} + +POINTER_SIZE_INT free_mem_in_wspace(Wspace *wspace, Boolean show_chunk_info) +{ + POINTER_SIZE_INT free_mem_size = 0; + +#ifdef SSPACE_CHUNK_INFO + if(show_chunk_info) + printf("\n\nPFC INFO:\n\n"); +#endif + free_mem_size += free_mem_in_pfc_pools(wspace, show_chunk_info); + +#ifdef SSPACE_CHUNK_INFO + if(show_chunk_info) + printf("\n\nALIGNED FREE CHUNK INFO:\n\n"); +#endif + free_mem_size += free_mem_in_free_lists(wspace, aligned_free_chunk_lists, NUM_ALIGNED_FREE_CHUNK_BUCKET, show_chunk_info); + +#ifdef SSPACE_CHUNK_INFO + if(show_chunk_info) + printf("\n\nUNALIGNED FREE CHUNK INFO:\n\n"); +#endif + free_mem_size += free_mem_in_free_lists(wspace, unaligned_free_chunk_lists, NUM_UNALIGNED_FREE_CHUNK_BUCKET, show_chunk_info); + +#ifdef SSPACE_CHUNK_INFO + if(show_chunk_info) + printf("\n\nSUPER FREE CHUNK INFO:\n\n"); +#endif + free_mem_size += free_mem_in_hyper_free_list(wspace, show_chunk_info); + + return free_mem_size; +} + + +#ifdef SSPACE_CHUNK_INFO +void wspace_chunks_info(Wspace *wspace, Boolean show_info) +{ + if(!show_info) return; + + POINTER_SIZE_INT free_mem_size = free_mem_in_wspace(wspace, TRUE); + + float free_mem_ratio = (float)free_mem_size / wspace->committed_heap_size; + printf("\n\nFree mem ratio: %f\n\n", free_mem_ratio); +} +#endif + + +/* Because this computation doesn't use lock, its result is not accurate. And it is enough. */ +POINTER_SIZE_INT wspace_free_memory_size(Wspace *wspace) +{ + POINTER_SIZE_INT free_size = 0; + + vm_gc_lock_enum(); + /* + for(unsigned int i=NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) + free_size += NORMAL_CHUNK_SIZE_BYTES * (i+1) * wspace->aligned_free_chunk_lists[i].chunk_num; + + for(unsigned int i=NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) + free_size += CHUNK_GRANULARITY * (i+1) * wspace->unaligned_free_chunk_lists[i].chunk_num; + + Free_Chunk *hyper_chunk = wspace->hyper_free_chunk_list->head; + while(hyper_chunk){ + free_size += CHUNK_SIZE(hyper_chunk); + hyper_chunk = hyper_chunk->next; + } + */ + free_size = free_mem_in_wspace(wspace, FALSE); + vm_gc_unlock_enum(); + + return free_size; +} + + +#ifdef SSPACE_ALLOC_INFO + +#define MEDIUM_THRESHOLD 256 +#define LARGE_THRESHOLD (1024) +#define SUPER_THRESHOLD (6*KB) +#define HYPER_THRESHOLD (64*KB) + +#define SMALL_OBJ_ARRAY_NUM (MEDIUM_THRESHOLD >> 2) +#define MEDIUM_OBJ_ARRAY_NUM (LARGE_THRESHOLD >> 4) +#define LARGE_OBJ_ARRAY_NUM (SUPER_THRESHOLD >> 6) +#define SUPER_OBJ_ARRAY_NUM (HYPER_THRESHOLD >> 10) + +volatile unsigned int small_obj_num[SMALL_OBJ_ARRAY_NUM]; +volatile unsigned int medium_obj_num[MEDIUM_OBJ_ARRAY_NUM]; +volatile unsigned int large_obj_num[LARGE_OBJ_ARRAY_NUM]; +volatile unsigned int super_obj_num[SUPER_OBJ_ARRAY_NUM]; +volatile unsigned int hyper_obj_num; + +void wspace_alloc_info(unsigned int size) +{ + if(size <= MEDIUM_THRESHOLD) + atomic_inc32(&small_obj_num[(size>>2)-1]); + else if(size <= LARGE_THRESHOLD) + atomic_inc32(&medium_obj_num[(size>>4)-1]); + else if(size <= SUPER_THRESHOLD) + atomic_inc32(&large_obj_num[(size>>6)-1]); + else if(size <= HYPER_THRESHOLD) + atomic_inc32(&super_obj_num[(size>>10)-1]); + else + atomic_inc32(&hyper_obj_num); +} + +void wspace_alloc_info_summary(void) +{ + unsigned int i; + + printf("\n\nNORMAL OBJ\n\n"); + for(i = 0; i < SMALL_OBJ_ARRAY_NUM; i++){ + printf("Size: %x\tnum: %d\n", (i+1)<<2, small_obj_num[i]); + small_obj_num[i] = 0; + } + + i = ((MEDIUM_THRESHOLD + (1<<4))>>4) - 1; + for(; i < MEDIUM_OBJ_ARRAY_NUM; i++){ + printf("Size: %x\tnum: %d\n", (i+1)<<4, medium_obj_num[i]); + medium_obj_num[i] = 0; + } + + i = ((LARGE_THRESHOLD + (1<<6))>>6) - 1; + for(; i < LARGE_OBJ_ARRAY_NUM; i++){ + printf("Size: %x\tnum: %d\n", (i+1)<<6, large_obj_num[i]); + large_obj_num[i] = 0; + } + + i = ((SUPER_THRESHOLD + (1<<10))>>10) - 1; + for(; i < SUPER_OBJ_ARRAY_NUM; i++){ + printf("Size: %x\tnum: %d\n", (i+1)<<10, super_obj_num[i]); + super_obj_num[i] = 0; + } + + printf("\n\nHYPER OBJ\n\n"); + printf("num: %d\n", hyper_obj_num); + hyper_obj_num = 0; +} + +#endif + +#ifdef SSPACE_USE_FASTDIV + +static int total_malloc_bytes = 0; +static char *cur_free_ptr = NULL; +static int cur_free_bytes = 0; + +void *malloc_wrapper(int size) +{ + massert(size > 0); + if(!cur_free_ptr) { + cur_free_bytes = INIT_ALLOC_SIZE; + cur_free_ptr = (char*) STD_MALLOC(cur_free_bytes); + } + + massert(cur_free_bytes >= size); + + total_malloc_bytes += size; + cur_free_bytes -= size; + + void * ret = cur_free_ptr; + cur_free_ptr += size; + return ret; +} + +void free_wrapper(int size) +{ + massert(size > 0); + massert(cur_free_ptr); + massert(total_malloc_bytes >= size); + cur_free_bytes += size; + total_malloc_bytes -= size; + cur_free_ptr -= size; +} + +unsigned int *shift_table; +unsigned short *compact_table[MAX_SLOT_SIZE_AFTER_SHIFTING]; +unsigned int mask[MAX_SLOT_SIZE_AFTER_SHIFTING]; +static int already_inited = 0; +void fastdiv_init() +{ + if(already_inited) return; + already_inited = 1; + + int i; + int shift_table_size = (MAX_SLOT_SIZE + 1) * sizeof shift_table[0]; + shift_table = (unsigned int *)malloc_wrapper(shift_table_size); + memset(shift_table, 0x00, shift_table_size) ; + for(i = MAX_SLOT_SIZE + 1;i--;) { + shift_table[i] = 0; + int v = i; + while(v && !(v & 1)) { + v >>= 1; + shift_table[i]++; + } + } + + memset(compact_table, 0x00, sizeof compact_table); + memset(mask, 0x00, sizeof mask); + for(i = 1;i < 32;i += 2) { + int cur = 1; + unsigned short *p = NULL; + while(1) { + p = (unsigned short*)malloc_wrapper(cur * sizeof p[0]); + memset(p, 0xff, cur * sizeof p[0]); + int j; + for(j = 0; j <= MAX_ADDR_OFFSET;j += i) { + int pos = j & (cur - 1); + if(p[pos] == 0xffff) { + p[pos] = j / i; + }else { + break; + } + } + if(j <= MAX_ADDR_OFFSET) { + free_wrapper(cur * sizeof p[0]); + cur <<= 1; + p = NULL; + }else { + break; + } + } + massert(p); + mask[i] = cur - 1; + while(cur && p[cur - 1] == 0xffff) { + free_wrapper(sizeof p[0]); + cur--; + } + compact_table[i] = p; + } +} + +#endif + + Index: src/mark_sweep/wspace_chunk.h =================================================================== --- src/mark_sweep/wspace_chunk.h (revision 0) +++ src/mark_sweep/wspace_chunk.h (revision 0) @@ -0,0 +1,548 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _SSPACE_CHUNK_H_ +#define _SSPACE_CHUNK_H_ + +#include "wspace.h" + +//#define SSPACE_USE_FASTDIV +#ifdef SSPACE_USE_FASTDIV +#if 0 +#define massert(x) do { if(!(x)) { printf("%s:%s, , Assertion %s failed\n", __FILE__, __LINE__, #x); exit(1); }} while(0) +#else +#define massert(x) +#endif +#define MAX_SLOT_SIZE 1024 +#define MAX_SLOT_SIZE_AFTER_SHIFTING 31 +#define MAX_ADDR_OFFSET (16 * 1024) +#define INIT_ALLOC_SIZE (128 * 1024) + +extern unsigned int *shift_table; +extern unsigned short *compact_table[MAX_SLOT_SIZE_AFTER_SHIFTING]; +extern unsigned int mask[MAX_SLOT_SIZE_AFTER_SHIFTING]; +extern void fastdiv_init(void); + +inline int fastdiv_div(int x, int y) +{ + massert(x % y == 0); + massert(0 <= y && y <= 1024); + massert(y % 4 == 0); + massert(y <= 128 || y % 8 == 0); + massert(y <= 256 || y % 128 == 0); + massert(x <= (1 << 16)); + + int s = shift_table[y]; + massert(s >= 2); + x >>= s; + y >>= s; + + massert(x >= 0 && x <= 16 * (1 << 10)); + massert(y <= 32 && y % 2); + + return (int)compact_table[y][x & mask[y]]; +} +#else +#define fastdiv_div(x,y) ((x) / (y)) +#define fastdiv_init() ((void)0) +#endif + +enum Chunk_Status { + CHUNK_NIL = 0, + CHUNK_FREE = 0x1, + CHUNK_FRESH = 0x2, + CHUNK_NORMAL = 0x10, + CHUNK_ABNORMAL = 0x20, + CHUNK_NEED_ZEROING = 0x100, + CHUNK_TO_MERGE = 0x200, + CHUNK_IN_USE = 0x400, /* just keep info for now, not used */ + CHUNK_USED = 0x800, /* just keep info for now, not used */ + CHUNK_MERGED = 0x1000 +}; + +typedef volatile POINTER_SIZE_INT Chunk_Status_t; + +typedef struct Chunk_Header_Basic { + Chunk_Header_Basic *next; + Chunk_Header_Basic *prev; + Chunk_Status_t status; + Chunk_Header_Basic *adj_next; // adjacent next chunk + Chunk_Header_Basic *adj_prev; // adjacent previous chunk, for merging continuous free chunks +} Chunk_Header_Basic; + +typedef struct Chunk_Header { + /* Beginning of Chunk_Header_Basic */ + Chunk_Header *next; /* pointing to the next pfc in the pfc pool */ + Chunk_Header *prev; /* pointing to the prev pfc in the pfc pool */ + Chunk_Status_t status; + Chunk_Header_Basic *adj_next; // adjacent next chunk + Chunk_Header_Basic *adj_prev; // adjacent previous chunk, for merging continuous free chunks + /* End of Chunk_Header_Basic */ + void *base; + unsigned int slot_size; + unsigned int slot_num; + unsigned int slot_index; /* the index of which is the first free slot in this chunk */ + unsigned int alloc_num; /* the index of which is the first free slot in this chunk */ + POINTER_SIZE_INT table[1]; +} Chunk_Header; + + +#define NORMAL_CHUNK_SHIFT_COUNT 16 +#define NORMAL_CHUNK_SIZE_BYTES (1 << NORMAL_CHUNK_SHIFT_COUNT) +#define NORMAL_CHUNK_LOW_MASK ((POINTER_SIZE_INT)(NORMAL_CHUNK_SIZE_BYTES - 1)) +#define NORMAL_CHUNK_HIGH_MASK (~NORMAL_CHUNK_LOW_MASK) +#define NORMAL_CHUNK_HEADER(addr) ((Chunk_Header*)((POINTER_SIZE_INT)(addr) & NORMAL_CHUNK_HIGH_MASK)) +#define ABNORMAL_CHUNK_HEADER(addr) ((Chunk_Header*)((POINTER_SIZE_INT)addr & CHUNK_GRANULARITY_HIGH_MASK)) + +#define MAX_SLOT_INDEX 0xFFffFFff +#define COLOR_BITS_PER_OBJ 4 // should be powers of 2 +#define SLOT_NUM_PER_WORD_IN_TABLE (BITS_PER_WORD /COLOR_BITS_PER_OBJ) + +/* Two equations: + * 1. CHUNK_HEADER_VARS_SIZE_BYTES + NORMAL_CHUNK_TABLE_SIZE_BYTES + slot_size*NORMAL_CHUNK_SLOT_NUM = NORMAL_CHUNK_SIZE_BYTES + * 2. (BITS_PER_BYTE * NORMAL_CHUNK_TABLE_SIZE_BYTES)/COLOR_BITS_PER_OBJ >= NORMAL_CHUNK_SLOT_NUM + * ===> + * NORMAL_CHUNK_SLOT_NUM <= BITS_PER_BYTE*(NORMAL_CHUNK_SIZE_BYTES - CHUNK_HEADER_VARS_SIZE_BYTES) / (BITS_PER_BYTE*slot_size + COLOR_BITS_PER_OBJ) + * ===> + * NORMAL_CHUNK_SLOT_NUM = BITS_PER_BYTE*(NORMAL_CHUNK_SIZE_BYTES - CHUNK_HEADER_VARS_SIZE_BYTES) / (BITS_PER_BYTE*slot_size + COLOR_BITS_PER_OBJ) + */ + +#define CHUNK_HEADER_VARS_SIZE_BYTES ((POINTER_SIZE_INT)&(((Chunk_Header*)0)->table)) +#define NORMAL_CHUNK_SLOT_AREA_SIZE_BITS (BITS_PER_BYTE * (NORMAL_CHUNK_SIZE_BYTES - CHUNK_HEADER_VARS_SIZE_BYTES)) +#define SIZE_BITS_PER_SLOT(chunk) (BITS_PER_BYTE * chunk->slot_size + COLOR_BITS_PER_OBJ) + +#define NORMAL_CHUNK_SLOT_NUM(chunk) (NORMAL_CHUNK_SLOT_AREA_SIZE_BITS / SIZE_BITS_PER_SLOT(chunk)) +#define NORMAL_CHUNK_TABLE_SIZE_BYTES(chunk) (((NORMAL_CHUNK_SLOT_NUM(chunk) + SLOT_NUM_PER_WORD_IN_TABLE-1) / SLOT_NUM_PER_WORD_IN_TABLE) * BYTES_PER_WORD) +#define NORMAL_CHUNK_HEADER_SIZE_BYTES(chunk) (CHUNK_HEADER_VARS_SIZE_BYTES + NORMAL_CHUNK_TABLE_SIZE_BYTES(chunk)) + +#define NORMAL_CHUNK_BASE(chunk) ((void*)((POINTER_SIZE_INT)(chunk) + NORMAL_CHUNK_HEADER_SIZE_BYTES(chunk))) +#define ABNORMAL_CHUNK_BASE(chunk) ((void*)((POINTER_SIZE_INT)(chunk) + sizeof(Chunk_Header))) + +#define CHUNK_END(chunk) ((chunk)->adj_next) +#define CHUNK_SIZE(chunk) ((POINTER_SIZE_INT)chunk->adj_next - (POINTER_SIZE_INT)chunk) + +#define NUM_ALIGNED_FREE_CHUNK_BUCKET (HYPER_OBJ_THRESHOLD >> NORMAL_CHUNK_SHIFT_COUNT) +#define NUM_UNALIGNED_FREE_CHUNK_BUCKET (HYPER_OBJ_THRESHOLD >> CHUNK_GRANULARITY_BITS) + +inline void *slot_index_to_addr(Chunk_Header *chunk, unsigned int index) +{ return (void*)((POINTER_SIZE_INT)chunk->base + chunk->slot_size * index); } + +inline unsigned int slot_addr_to_index(Chunk_Header *chunk, void *addr) +{ return (unsigned int)fastdiv_div(((POINTER_SIZE_INT)addr - (POINTER_SIZE_INT)chunk->base) , chunk->slot_size); } + +typedef struct Free_Chunk { + /* Beginning of Chunk_Header_Basic */ + Free_Chunk *next; /* pointing to the next free Free_Chunk */ + Free_Chunk *prev; /* pointing to the prev free Free_Chunk */ + Chunk_Status_t status; + Chunk_Header_Basic *adj_next; // adjacent next chunk + Chunk_Header_Basic *adj_prev; // adjacent previous chunk, for merging continuous free chunks + /* End of Chunk_Header_Basic */ +} Free_Chunk; + +typedef struct Free_Chunk_List { + Free_Chunk *head; /* get new free chunk from head */ + Free_Chunk *tail; /* put free chunk to tail */ + unsigned int chunk_num; + SpinLock lock; +} Free_Chunk_List; + +/* +typedef union Chunk{ + Chunk_Header header; + Free_Chunk free_chunk; + unsigned char raw_bytes[NORMAL_CHUNK_SIZE_BYTES]; +} Chunk; +*/ + +inline Boolean is_free_chunk_merged(Free_Chunk* free_chunk) +{ + assert(free_chunk->status & CHUNK_FREE); + return (Boolean)(free_chunk->status & CHUNK_MERGED); +} + +inline void free_chunk_list_init(Free_Chunk_List *list) +{ + list->head = NULL; + list->tail = NULL; + list->chunk_num = 0; + list->lock = FREE_LOCK; +} + +inline void free_chunk_list_clear(Free_Chunk_List *list) +{ + list->head = NULL; + list->tail = NULL; + list->chunk_num = 0; + assert(list->lock == FREE_LOCK); +} + +inline void free_list_detach_chunk(Free_Chunk_List *list, Free_Chunk *chunk) +{ + if(chunk->prev) + chunk->prev->next = chunk->next; + else // chunk is the head + list->head = chunk->next; + + if(chunk->next) + chunk->next->prev = chunk->prev; + else + list->tail = chunk->prev; +} + +inline void move_free_chunks_between_lists(Free_Chunk_List *to_list, Free_Chunk_List *from_list) +{ + if(to_list->tail){ + to_list->head->prev = from_list->tail; + } else { + to_list->tail = from_list->tail; + } + if(from_list->head){ + from_list->tail->next = to_list->head; + to_list->head = from_list->head; + } + from_list->head = NULL; + from_list->tail = NULL; +} + +/* Padding the last index word in table to facilitate allocation */ +inline void chunk_pad_last_index_word(Chunk_Header *chunk, POINTER_SIZE_INT alloc_mask) +{ + unsigned int ceiling_index_in_last_word = (chunk->slot_num * COLOR_BITS_PER_OBJ) % BITS_PER_WORD; + if(!ceiling_index_in_last_word) + return; + POINTER_SIZE_INT padding_mask = ~((1 << ceiling_index_in_last_word) - 1); + padding_mask &= alloc_mask; + unsigned int last_word_index = (chunk->slot_num-1) / SLOT_NUM_PER_WORD_IN_TABLE; + chunk->table[last_word_index] |= padding_mask; +} + +inline void chunk_pad_last_index_word_concurrent(Chunk_Header *chunk, POINTER_SIZE_INT alloc_mask) +{ + unsigned int ceiling_index_in_last_word = (chunk->slot_num * COLOR_BITS_PER_OBJ) % BITS_PER_WORD; + if(!ceiling_index_in_last_word) + return; + POINTER_SIZE_INT padding_mask = ~((1 << ceiling_index_in_last_word) - 1); + padding_mask &= alloc_mask; + unsigned int last_word_index = (chunk->slot_num-1) / SLOT_NUM_PER_WORD_IN_TABLE; + POINTER_SIZE_INT old_word = chunk->table[last_word_index]; + POINTER_SIZE_INT new_word = old_word | padding_mask; + while (old_word == new_word){ + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void **) &chunk->table[last_word_index],(void*)new_word ,(void*)old_word ); + if(temp == old_word) + return; + old_word = chunk->table[last_word_index]; + new_word = old_word | padding_mask; + } +} + + + +/* Depadding the last index word in table to facilitate allocation */ +inline void chunk_depad_last_index_word(Chunk_Header *chunk) +{ + unsigned int ceiling_index_in_last_word = (chunk->slot_num * COLOR_BITS_PER_OBJ) % BITS_PER_WORD; + if(!ceiling_index_in_last_word) + return; + POINTER_SIZE_INT depadding_mask = (1 << ceiling_index_in_last_word) - 1; + unsigned int last_word_index = (chunk->slot_num-1) / SLOT_NUM_PER_WORD_IN_TABLE; + chunk->table[last_word_index] &= depadding_mask; +} + +extern POINTER_SIZE_INT cur_alloc_mask; +/* Used for allocating a fixed-size chunk from free area lists */ +inline void normal_chunk_init(Chunk_Header *chunk, unsigned int slot_size) +{ + //Modified this assertion for concurrent sweep + //assert(chunk->status == CHUNK_FREE); + assert(chunk->status & CHUNK_FREE); + assert(CHUNK_SIZE(chunk) == NORMAL_CHUNK_SIZE_BYTES); + + chunk->next = NULL; + chunk->status = CHUNK_FRESH | CHUNK_NORMAL | CHUNK_NEED_ZEROING; + chunk->slot_size = slot_size; + chunk->slot_num = NORMAL_CHUNK_SLOT_NUM(chunk); + chunk->slot_index = 0; + chunk->alloc_num = 0; + chunk->base = NORMAL_CHUNK_BASE(chunk); + memset(chunk->table, 0, NORMAL_CHUNK_TABLE_SIZE_BYTES(chunk));//memset table + //chunk_pad_last_index_word(chunk, cur_alloc_mask); + fastdiv_init(); +} + +/* Used for allocating a chunk for large object from free area lists */ +inline void abnormal_chunk_init(Chunk_Header *chunk, unsigned int chunk_size, unsigned int obj_size) +{ + //Modified this assertion for concurrent sweep + //assert(chunk->status == CHUNK_FREE); + assert(chunk->status & CHUNK_FREE); + assert(CHUNK_SIZE(chunk) == chunk_size); + + chunk->next = NULL; + chunk->status = CHUNK_ABNORMAL; + chunk->slot_size = obj_size; + chunk->slot_num = 1; + chunk->slot_index = 0; + chunk->base = ABNORMAL_CHUNK_BASE(chunk); +} + + +#ifdef POINTER64 + #define GC_OBJECT_ALIGNMENT_BITS 3 +#else + #define GC_OBJECT_ALIGNMENT_BITS 2 +#endif + +#define MEDIUM_OBJ_THRESHOLD (128) +#define LARGE_OBJ_THRESHOLD (256) +#define SUPER_OBJ_THRESHOLD (1024) +#define HYPER_OBJ_THRESHOLD (128*KB) + +#define SMALL_GRANULARITY_BITS (GC_OBJECT_ALIGNMENT_BITS) +#define MEDIUM_GRANULARITY_BITS (SMALL_GRANULARITY_BITS + 1) +#define LARGE_GRANULARITY_BITS 7 +#define CHUNK_GRANULARITY_BITS 10 + +#define CHUNK_GRANULARITY (1 << CHUNK_GRANULARITY_BITS) +#define CHUNK_GRANULARITY_LOW_MASK ((POINTER_SIZE_INT)(CHUNK_GRANULARITY-1)) +#define CHUNK_GRANULARITY_HIGH_MASK (~CHUNK_GRANULARITY_LOW_MASK) + +#define SMALL_IS_LOCAL_ALLOC TRUE +#define MEDIUM_IS_LOCAL_ALLOC TRUE +#define LARGE_IS_LOCAL_ALLOC FALSE + +#define NORMAL_SIZE_ROUNDUP(size, seg) (((size) + seg->granularity-1) & seg->gran_high_mask) +#define SUPER_OBJ_TOTAL_SIZE(size) (sizeof(Chunk_Header) + (size)) +#define SUPER_SIZE_ROUNDUP(size) ((SUPER_OBJ_TOTAL_SIZE(size) + CHUNK_GRANULARITY-1) & CHUNK_GRANULARITY_HIGH_MASK) + +#define NORMAL_SIZE_TO_INDEX(size, seg) ((((size)-(seg)->size_min) >> (seg)->gran_shift_bits) - 1) +#define ALIGNED_CHUNK_SIZE_TO_INDEX(size) (((size) >> NORMAL_CHUNK_SHIFT_COUNT) - 1) +#define UNALIGNED_CHUNK_SIZE_TO_INDEX(size) (((size) >> CHUNK_GRANULARITY_BITS) - 1) + +#define NORMAL_INDEX_TO_SIZE(index, seg) ((((index) + 1) << (seg)->gran_shift_bits) + (seg)->size_min) +#define ALIGNED_CHUNK_INDEX_TO_SIZE(index) (((index) + 1) << NORMAL_CHUNK_SHIFT_COUNT) +#define UNALIGNED_CHUNK_INDEX_TO_SIZE(index) (((index) + 1) << CHUNK_GRANULARITY_BITS) + +#define SUPER_OBJ_MASK ((Obj_Info_Type)0x20) /* the 4th bit in obj info */ + +//#define WSPACE_CONCURRENT_GC_STATS + +#ifdef WSPACE_CONCURRENT_GC_STATS +#define NEW_OBJ_MASK ((Obj_Info_Type)0x40) +#endif + +#define PFC_STEAL_NUM 3 +#define PFC_STEAL_THRESHOLD 3 + + +#define SIZE_SEGMENT_NUM 3 + +typedef struct Size_Segment { + unsigned int size_min; + unsigned int size_max; + unsigned int seg_index; + Boolean local_alloc; + unsigned int chunk_num; + unsigned int gran_shift_bits; + POINTER_SIZE_INT granularity; + POINTER_SIZE_INT gran_low_mask; + POINTER_SIZE_INT gran_high_mask; +} Size_Segment; + +inline Size_Segment *wspace_get_size_seg(Wspace *wspace, unsigned int size) +{ + Size_Segment **size_segs = wspace->size_segments; + + unsigned int seg_index = 0; + for(; seg_index < SIZE_SEGMENT_NUM; ++seg_index) + if(size <= size_segs[seg_index]->size_max) break; + assert(seg_index < SIZE_SEGMENT_NUM); + assert(size_segs[seg_index]->seg_index == seg_index); + return size_segs[seg_index]; +} + +inline Chunk_Header *wspace_get_pfc(Wspace *wspace, unsigned int seg_index, unsigned int index) +{ + /*1. Search PFC pool*/ + Pool *pfc_pool = wspace->pfc_pools[seg_index][index]; + Chunk_Header *chunk = (Chunk_Header*)pool_get_entry(pfc_pool); + + /*2. If in concurrent sweeping phase, search PFC backup pool*/ + if(!chunk && gc_is_concurrent_sweep_phase()){ + pfc_pool = wspace->pfc_pools_backup[seg_index][index]; + chunk = (Chunk_Header*)pool_get_entry(pfc_pool); + } + assert(!chunk || chunk->status == (CHUNK_NORMAL | CHUNK_NEED_ZEROING)); + return chunk; +} + +inline void wspace_put_pfc(Wspace *wspace, Chunk_Header *chunk) +{ + unsigned int size = chunk->slot_size; + assert(chunk && (size <= SUPER_OBJ_THRESHOLD)); + assert(chunk->base && chunk->alloc_num); + assert(chunk->alloc_num < chunk->slot_num); + assert(chunk->slot_index < chunk->slot_num); + + Size_Segment **size_segs = wspace->size_segments; + chunk->status = CHUNK_NORMAL | CHUNK_NEED_ZEROING; + + for(unsigned int i = 0; i < SIZE_SEGMENT_NUM; ++i){ + if(size > size_segs[i]->size_max) continue; + assert(!(size & size_segs[i]->gran_low_mask)); + assert(size > size_segs[i]->size_min); + unsigned int index = NORMAL_SIZE_TO_INDEX(size, size_segs[i]); + Pool *pfc_pool; + pfc_pool = wspace->pfc_pools[i][index]; + pool_put_entry(pfc_pool, chunk); + return; + } +} + +inline void wspace_put_pfc_backup(Wspace *wspace, Chunk_Header *chunk) +{ + unsigned int size = chunk->slot_size; + assert(chunk && (size <= SUPER_OBJ_THRESHOLD)); + assert(chunk->base && chunk->alloc_num); + assert(chunk->alloc_num < chunk->slot_num); + assert(chunk->slot_index < chunk->slot_num); + + Size_Segment **size_segs = wspace->size_segments; + chunk->status = CHUNK_NORMAL | CHUNK_NEED_ZEROING; + + for(unsigned int i = 0; i < SIZE_SEGMENT_NUM; ++i){ + if(size > size_segs[i]->size_max) continue; + assert(!(size & size_segs[i]->gran_low_mask)); + assert(size > size_segs[i]->size_min); + unsigned int index = NORMAL_SIZE_TO_INDEX(size, size_segs[i]); + //FIXME: concurrent sweep + Pool *pfc_pool; + pfc_pool = wspace->pfc_pools_backup[i][index]; + pool_put_entry(pfc_pool, chunk); + return; + } +} + + +/*Function for concurrent sweeping. In concurrent sweeping, PFC chunks are put back to pfc_pools_backup + instead of pfc_pools. */ +inline void wspace_backup_pfc(Wspace *wspace, Chunk_Header *chunk) +{ + unsigned int size = chunk->slot_size; + assert(chunk->base && chunk->alloc_num); + assert(chunk && (size <= SUPER_OBJ_THRESHOLD)); + assert(chunk->slot_index < chunk->slot_num); + + Size_Segment **size_segs = wspace->size_segments; + chunk->status = CHUNK_NORMAL | CHUNK_NEED_ZEROING; + + for(unsigned int i = 0; i < SIZE_SEGMENT_NUM; ++i){ + if(size > size_segs[i]->size_max) continue; + assert(!(size & size_segs[i]->gran_low_mask)); + assert(size > size_segs[i]->size_min); + unsigned int index = NORMAL_SIZE_TO_INDEX(size, size_segs[i]); + Pool *pfc_pool = wspace->pfc_pools_backup[i][index]; + pool_put_entry(pfc_pool, chunk); + return; + } +} + +inline void wspace_rebuild_chunk_chain(Wspace *wspace) +{ + Chunk_Header_Basic *wspace_ceiling = (Chunk_Header_Basic*)space_heap_end((Space*)wspace); + Chunk_Header_Basic *prev_chunk = (Chunk_Header_Basic*)space_heap_start((Space*)wspace); + Chunk_Header_Basic *chunk = prev_chunk->adj_next; + prev_chunk->adj_prev = NULL; + + while(chunk < wspace_ceiling){ + chunk->adj_prev = prev_chunk; + prev_chunk = chunk; + chunk = chunk->adj_next; + } +} + +inline Chunk_Header_Basic* chunk_pool_get_chunk(Pool* chunk_pool) +{ + return (Chunk_Header_Basic*)pool_get_entry(chunk_pool); +} + +inline void chunk_pool_put_chunk(Pool* chunk_pool, Chunk_Header* chunk) +{ + pool_put_entry(chunk_pool,chunk); + return; +} + +inline void wspace_register_used_chunk(Wspace* wspace, Chunk_Header* chunk) +{ + pool_put_entry(wspace->used_chunk_pool, chunk); + return; +} + +inline void wspace_clear_used_chunk_pool(Wspace* wspace) +{ + pool_empty(wspace->used_chunk_pool); + return; +} + +inline void wspace_register_unreusable_normal_chunk(Wspace* wspace, Chunk_Header* chunk) +{ + pool_put_entry(wspace->unreusable_normal_chunk_pool, chunk); + return; +} + +inline Chunk_Header* wspace_get_unreusable_normal_chunk(Wspace* wspace) +{ + return (Chunk_Header*)pool_get_entry(wspace->unreusable_normal_chunk_pool); +} + +inline void wspace_register_live_abnormal_chunk(Wspace* wspace, Chunk_Header* chunk) +{ + pool_put_entry(wspace->live_abnormal_chunk_pool, chunk); + return; +} + +inline Chunk_Header* wspace_get_live_abnormal_chunk(Wspace* wspace) +{ + return (Chunk_Header*)pool_get_entry(wspace->live_abnormal_chunk_pool); +} + +extern void wspace_init_chunks(Wspace *wspace); +extern void wspace_clear_chunk_list(Wspace* wspace); + +extern void wspace_put_free_chunk(Wspace *wspace, Free_Chunk *chunk); +void wspace_put_free_chunk_to_head(Wspace *wspace, Free_Chunk *chunk); +void wspace_put_free_chunk_to_tail(Wspace *wspace, Free_Chunk *chunk); + +extern Free_Chunk *wspace_get_normal_free_chunk(Wspace *wspace); +extern Free_Chunk *wspace_get_abnormal_free_chunk(Wspace *wspace, unsigned int chunk_size); +extern Free_Chunk *wspace_get_hyper_free_chunk(Wspace *wspace, unsigned int chunk_size, Boolean is_normal_chunk); + +extern void wspace_init_pfc_pool_iterator(Wspace *wspace); +extern Pool *wspace_grab_next_pfc_pool(Wspace *wspace); +void wspace_exchange_pfc_pool(Wspace *wspace); + +extern Chunk_Header *wspace_steal_pfc(Wspace *wspace, unsigned int index); + +extern POINTER_SIZE_INT free_mem_in_wspace(Wspace *wspace, Boolean show_chunk_info); + +extern void zeroing_free_chunk(Free_Chunk *chunk); + +extern void gc_clear_collector_local_chunks(GC *gc); + +extern void wspace_collect_free_chunks_to_list(Wspace *wspace, Free_Chunk_List *list); + +#endif //#ifndef _SSPACE_CHUNK_H_ Index: src/mark_sweep/wspace_compact.cpp =================================================================== --- src/mark_sweep/wspace_compact.cpp (revision 0) +++ src/mark_sweep/wspace_compact.cpp (revision 0) @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wspace_chunk.h" +#include "wspace_alloc.h" +#include "wspace_mark_sweep.h" +#include "wspace_verify.h" + + +#define PFC_SORT_NUM 8 + +void wspace_decide_compaction_need(Wspace *wspace) +{ + POINTER_SIZE_INT free_mem_size = free_mem_in_wspace(wspace, FALSE); + float free_mem_ratio = (float)free_mem_size / wspace->committed_heap_size; + +#ifdef USE_MARK_SWEEP_GC + if(!gc_mark_is_concurrent() && (free_mem_ratio > SSPACE_COMPACT_RATIO) && (wspace->gc->cause != GC_CAUSE_RUNTIME_FORCE_GC)){ +#else + if(gc_match_kind(wspace->gc, MAJOR_COLLECTION)){ +#endif + wspace->need_compact = wspace->move_object = TRUE; + } else { + wspace->need_compact = wspace->move_object = FALSE; + } +} + +static inline void sorted_chunk_bucket_add_entry(Chunk_Header **head, Chunk_Header **tail, Chunk_Header *chunk) +{ + chunk->prev = NULL; /* Field adj_prev is used as prev */ + + if(!*head){ + assert(!*tail); + chunk->next = NULL; + *head = *tail = chunk; + return; + } + + assert(*tail); + chunk->next = *head; + (*head)->prev = chunk; + *head = chunk; +} + +/* One assumption: pfc_pool is not empty */ +static Boolean pfc_pool_roughly_sort(Pool *pfc_pool, Chunk_Header **least_free_chunk, Chunk_Header **most_free_chunk) +{ + Chunk_Header *bucket_head[PFC_SORT_NUM]; /* Sorted chunk buckets' heads */ + Chunk_Header *bucket_tail[PFC_SORT_NUM]; /* Sorted chunk buckets' tails */ + unsigned int slot_num; + unsigned int chunk_num = 0; + unsigned int slot_alloc_num = 0; + + /* Init buckets' heads and tails */ + memset(bucket_head, 0, sizeof(Chunk_Header*) * PFC_SORT_NUM); + memset(bucket_tail, 0, sizeof(Chunk_Header*) * PFC_SORT_NUM); + + /* Roughly sort chunks in pfc_pool */ + pool_iterator_init(pfc_pool); + Chunk_Header *chunk = (Chunk_Header*)pool_iterator_next(pfc_pool); + if(chunk) slot_num = chunk->slot_num; + while(chunk){ + ++chunk_num; + assert(chunk->alloc_num); + slot_alloc_num += chunk->alloc_num; + Chunk_Header *next_chunk = chunk->next; + unsigned int bucket_index = (chunk->alloc_num*PFC_SORT_NUM-1) / slot_num; + assert(bucket_index < PFC_SORT_NUM); + sorted_chunk_bucket_add_entry(&bucket_head[bucket_index], &bucket_tail[bucket_index], chunk); + chunk = next_chunk; + } + + /* Empty the pfc pool because some chunks in this pool will be free after compaction */ + pool_empty(pfc_pool); + + /* If we can't get a free chunk after compaction, there is no need to compact. + * This condition includes that the chunk num in pfc pool is equal to 1, in which case there is also no need to compact + */ + if(slot_num*(chunk_num-1) <= slot_alloc_num){ + for(unsigned int i = 0; i < PFC_SORT_NUM; i++){ + Chunk_Header *chunk = bucket_head[i]; + while(chunk){ + Chunk_Header *next_chunk = chunk->next; + pool_put_entry(pfc_pool, chunk); + chunk = next_chunk; + } + } + return FALSE; + } + + /* Link the sorted chunk buckets into one single ordered bidirectional list */ + Chunk_Header *head = NULL; + Chunk_Header *tail = NULL; + for(unsigned int i = PFC_SORT_NUM; i--;){ + assert((head && tail) || (!head && !tail)); + assert((bucket_head[i] && bucket_tail[i]) || (!bucket_head[i] && !bucket_tail[i])); + if(!bucket_head[i]) continue; + if(!tail){ + head = bucket_head[i]; + tail = bucket_tail[i]; + } else { + tail->next = bucket_head[i]; + bucket_head[i]->prev = tail; + tail = bucket_tail[i]; + } + } + + assert(head && tail); + *least_free_chunk = head; + *most_free_chunk = tail; + + return TRUE; +} + +static inline Chunk_Header *get_least_free_chunk(Chunk_Header **least_free_chunk, Chunk_Header **most_free_chunk) +{ + if(!*least_free_chunk){ + assert(!*most_free_chunk); + return NULL; + } + Chunk_Header *result = *least_free_chunk; + *least_free_chunk = (*least_free_chunk)->next; + if(*least_free_chunk) + (*least_free_chunk)->prev = NULL; + else + *most_free_chunk = NULL; + return result; +} +static inline Chunk_Header *get_most_free_chunk(Chunk_Header **least_free_chunk, Chunk_Header **most_free_chunk) +{ + if(!*most_free_chunk){ + assert(!*least_free_chunk); + return NULL; + } + Chunk_Header *result = *most_free_chunk; + *most_free_chunk = (*most_free_chunk)->prev; + if(*most_free_chunk) + (*most_free_chunk)->next = NULL; + else + *least_free_chunk = NULL; + assert(!result->next); + return result; +} + +static inline void move_obj_between_chunks(Chunk_Header **dest_ptr, Chunk_Header *src) +{ + Chunk_Header *dest = *dest_ptr; + assert(dest->slot_size == src->slot_size); + + unsigned int slot_size = dest->slot_size; + unsigned int alloc_num = src->alloc_num; + assert(alloc_num); + + while(alloc_num && dest){ + Partial_Reveal_Object *p_obj = next_alloc_slot_in_chunk(src); + void *target = alloc_in_chunk(dest); + + if(dest->slot_index == MAX_SLOT_INDEX){ + dest->status = CHUNK_USED | CHUNK_NORMAL; + dest = NULL; + } + + assert(p_obj && target); + memcpy(target, p_obj, slot_size); +#ifdef SSPACE_VERIFY + wspace_modify_mark_in_compact(target, p_obj, slot_size); +#endif + obj_set_fw_in_oi(p_obj, target); + --alloc_num; + } + + /* dest might be set to NULL, so we use *dest_ptr here */ + assert((*dest_ptr)->alloc_num <= (*dest_ptr)->slot_num); + src->alloc_num = alloc_num; + if(!dest){ + assert((*dest_ptr)->alloc_num == (*dest_ptr)->slot_num); + *dest_ptr = NULL; + clear_free_slot_in_table(src->table, src->slot_index); + } +} + +void wspace_compact(Collector *collector, Wspace *wspace) +{ + Chunk_Header *least_free_chunk, *most_free_chunk; + Pool *pfc_pool = wspace_grab_next_pfc_pool(wspace); + + for(; pfc_pool; pfc_pool = wspace_grab_next_pfc_pool(wspace)){ + if(pool_is_empty(pfc_pool)) continue; + Boolean pfc_pool_need_compact = pfc_pool_roughly_sort(pfc_pool, &least_free_chunk, &most_free_chunk); + if(!pfc_pool_need_compact) continue; + + Chunk_Header *dest = get_least_free_chunk(&least_free_chunk, &most_free_chunk); + Chunk_Header *src = get_most_free_chunk(&least_free_chunk, &most_free_chunk); + Boolean src_is_new = TRUE; + while(dest && src){ + if(src_is_new) + src->slot_index = 0; + chunk_depad_last_index_word(src); + move_obj_between_chunks(&dest, src); + if(!dest) + dest = get_least_free_chunk(&least_free_chunk, &most_free_chunk); + if(!src->alloc_num){ + collector_add_free_chunk(collector, (Free_Chunk*)src); + src = get_most_free_chunk(&least_free_chunk, &most_free_chunk); + src_is_new = TRUE; + } else { + src_is_new = FALSE; + } + } + + /* Rebuild the pfc_pool */ + if(dest) + wspace_put_pfc(wspace, dest); + if(src){ + //chunk_pad_last_index_word(src, cur_alloc_mask); + pfc_reset_slot_index(src); + wspace_put_pfc(wspace, src); + } + } +} + + + Index: src/mark_sweep/wspace_concurrent_gc_stats.cpp =================================================================== --- src/mark_sweep/wspace_concurrent_gc_stats.cpp (revision 0) +++ src/mark_sweep/wspace_concurrent_gc_stats.cpp (revision 0) @@ -0,0 +1,201 @@ +#ifdef WSPACE_CONCURRENT_GC_STATS + +#include "wspace_alloc.h" +#include "wspace_mark_sweep.h" +#include "wspace_verify.h" +#include "gc_ms.h" +#include "../gen/gen.h" +#include "../thread/collector.h" +#include "../finalizer_weakref/finalizer_weakref.h" +#include "../common/fix_repointed_refs.h" +#include "../common/gc_concurrent.h" + + + +static unsigned int total_object_num = 0; +static unsigned int total_live_obj_num = 0; +static unsigned int total_float_obj_num = 0; +static unsigned int total_live_new_obj_num = 0; +static unsigned int total_float_new_obj_num = 0; + +static Chunk_Header_Basic *volatile next_chunk_for_scanning; + + +static void wspace_init_chunk_for_scanning(Wspace *wspace) +{ + next_chunk_for_scanning = (Chunk_Header_Basic*)space_heap_start((Space*)wspace); +} + + +static void normal_chunk_scanning(Chunk_Header *chunk) +{ + chunk->slot_index = 0; + chunk_depad_last_index_word(chunk); + + unsigned int alloc_num = chunk->alloc_num; + assert(alloc_num); + + if(alloc_num == chunk->slot_num){ /* Filled with objects */ + unsigned int slot_size = chunk->slot_size; + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)slot_index_to_addr(chunk, 0); + for(unsigned int i = alloc_num; i--;){ + total_object_num ++; + if(obj_is_marked_in_vt(p_obj)){ + total_live_obj_num ++; + if(p_obj->obj_info & NEW_OBJ_MASK){ + p_obj->obj_info &= ~NEW_OBJ_MASK; + total_live_new_obj_num ++; + } + }else{ + total_float_obj_num ++; + if(p_obj->obj_info & NEW_OBJ_MASK){ + p_obj->obj_info &= ~NEW_OBJ_MASK; + total_float_new_obj_num ++; + } + } + p_obj = (Partial_Reveal_Object*)((POINTER_SIZE_INT)p_obj + slot_size); + } + } else { /* Chunk is not full */ + while(alloc_num){ + Partial_Reveal_Object *p_obj = next_alloc_slot_in_chunk(chunk); + + total_object_num ++; + if(obj_is_marked_in_vt(p_obj)){ + total_live_obj_num ++; + if(p_obj->obj_info & NEW_OBJ_MASK){ + p_obj->obj_info &= ~NEW_OBJ_MASK; + total_live_new_obj_num ++; + } + }else{ + total_float_obj_num ++; + if(p_obj->obj_info & NEW_OBJ_MASK){ + p_obj->obj_info &= ~NEW_OBJ_MASK; + total_float_new_obj_num ++; + } + } + + assert(p_obj); + --alloc_num; + } + } + + if(chunk->alloc_num != chunk->slot_num){ + //chunk_pad_last_index_word(chunk, cur_alloc_mask); + pfc_reset_slot_index(chunk); + } + +} + +static void abnormal_chunk_scanning(Chunk_Header *chunk) +{ + Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*)chunk->base; + total_object_num ++; + if(obj_is_marked_in_vt(p_obj)){ + total_live_obj_num ++; + if(p_obj->obj_info & NEW_OBJ_MASK){ + p_obj->obj_info &= ~NEW_OBJ_MASK; + total_live_new_obj_num ++; + } + }else{ + total_float_obj_num ++; + if(p_obj->obj_info & NEW_OBJ_MASK){ + p_obj->obj_info &= ~NEW_OBJ_MASK; + total_float_new_obj_num ++; + } + } +} + +void wspace_scan_heap(GC* gc) +{ + Wspace *wspace = gc_get_wspace(gc); + total_object_num = 0; + total_live_obj_num = 0; + total_float_obj_num = 0; + total_live_new_obj_num = 0; + total_float_new_obj_num = 0; + + wspace_init_chunk_for_scanning(wspace); + + Chunk_Header_Basic *chunk = wspace_grab_next_chunk(wspace, &next_chunk_for_scanning, FALSE); + + while(chunk){ + if(chunk->status & CHUNK_NORMAL) + normal_chunk_scanning((Chunk_Header*)chunk); + else if(chunk->status & CHUNK_ABNORMAL) + abnormal_chunk_scanning((Chunk_Header*)chunk); + + chunk = wspace_grab_next_chunk(wspace, &next_chunk_for_scanning, FALSE); + } + + printf("total_object_num: %d \n", total_object_num); + printf("total_live_obj_num: %d \n", total_live_obj_num); + printf("total_float_obj_num: %d \n", total_float_obj_num); + printf("total_live_new_obj_num: %d \n", total_live_new_obj_num); + printf("total_float_new_obj_num: %d \n", total_float_new_obj_num); +} + + + +static void normal_chunk_clear(Chunk_Header *chunk) +{ + chunk->slot_index = 0; + chunk_depad_last_index_word(chunk); + + unsigned int alloc_num = chunk->alloc_num; + assert(alloc_num); + + if(alloc_num == chunk->slot_num){ /* Filled with objects */ + unsigned int slot_size = chunk->slot_size; + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)slot_index_to_addr(chunk, 0); + for(unsigned int i = alloc_num; i--;){ + if(p_obj->obj_info & NEW_OBJ_MASK){ + p_obj->obj_info &= ~NEW_OBJ_MASK; + } + } + } else { /* Chunk is not full */ + while(alloc_num){ + Partial_Reveal_Object *p_obj = next_alloc_slot_in_chunk(chunk); + if(p_obj->obj_info & NEW_OBJ_MASK){ + p_obj->obj_info &= ~NEW_OBJ_MASK; + total_live_new_obj_num ++; + } + assert(p_obj); + --alloc_num; + } + } + + if(chunk->alloc_num != chunk->slot_num){ + //chunk_pad_last_index_word(chunk, cur_alloc_mask); + pfc_reset_slot_index(chunk); + } + +} + +static void abnormal_chunk_clear(Chunk_Header *chunk) +{ + Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*)chunk->base; + if(p_obj->obj_info & NEW_OBJ_MASK){ + p_obj->obj_info &= ~NEW_OBJ_MASK; + } +} + +void wspace_clear_heap(GC* gc) +{ + Wspace *wspace = gc_get_wspace(gc); + + wspace_init_chunk_for_scanning(wspace); + + Chunk_Header_Basic *chunk = wspace_grab_next_chunk(wspace, &next_chunk_for_scanning, FALSE); + + while(chunk){ + if(chunk->status & CHUNK_NORMAL) + normal_chunk_clear((Chunk_Header*)chunk); + else if(chunk->status & CHUNK_ABNORMAL) + abnormal_chunk_clear((Chunk_Header*)chunk); + + chunk = wspace_grab_next_chunk(wspace, &next_chunk_for_scanning, FALSE); + } + +} +#endif + Index: src/mark_sweep/wspace_fallback_mark.cpp =================================================================== --- src/mark_sweep/wspace_fallback_mark.cpp (revision 0) +++ src/mark_sweep/wspace_fallback_mark.cpp (revision 0) @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wspace_mark_sweep.h" +#include "../finalizer_weakref/finalizer_weakref.h" + +static Wspace *wspace_in_fallback_marking; + + +static FORCE_INLINE Boolean obj_mark_black(Partial_Reveal_Object *obj) +{ + if(obj_belongs_to_space(obj, (Space*)wspace_in_fallback_marking)){ + Boolean marked_by_self = obj_mark_black_in_table(obj); + +#ifndef USE_MARK_SWEEP_GC + /* When fallback happens, some objects in MOS have their fw bit set, which is actually their mark bit in the last minor gc. + * If we don't clear it, some objects that didn't be moved will be mistaken for being moved in the coming fixing phase. + */ + if(marked_by_self){ + Obj_Info_Type oi = obj->obj_info; + Obj_Info_Type new_oi = oi & DUAL_MARKBITS_MASK; + while(new_oi != oi){ + Obj_Info_Type temp = atomic_casptrsz((volatile Obj_Info_Type*)get_obj_info_addr(obj), new_oi, oi); + if(temp == oi) break; + oi = obj->obj_info; + new_oi = oi & DUAL_MARKBITS_MASK; + } + } +#endif + return marked_by_self; + } else { + return obj_mark_in_vt(obj); + } +} + +static FORCE_INLINE void scan_slot(Collector *collector, REF *p_ref) +{ + if( read_slot(p_ref) == NULL) return; + + collector_tracestack_push(collector, p_ref); +} + +static FORCE_INLINE void scan_object(Collector *collector, REF *p_ref) +{ + Partial_Reveal_Object *p_obj = read_slot(p_ref); + assert(p_obj); + assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); + + if(obj_belongs_to_nos(p_obj) && obj_is_fw_in_oi(p_obj)){ + assert(obj_get_vt(p_obj) == obj_get_vt(obj_get_fw_in_oi(p_obj))); + p_obj = obj_get_fw_in_oi(p_obj); + assert(p_obj); + write_slot(p_ref, p_obj); + } + + if(!obj_mark_black(p_obj)) + return; + + if(!object_has_ref_field(p_obj)) return; + + if(object_is_array(p_obj)){ /* scan array object */ + Partial_Reveal_Array *array = (Partial_Reveal_Array*)p_obj; + unsigned int array_length = array->array_len; + + REF *p_ref = (REF *)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); + for (unsigned int i = 0; i < array_length; i++) + scan_slot(collector, p_ref+i); + + return; + } + + /* scan non-array object */ + unsigned int num_refs = object_ref_field_num(p_obj); + int *ref_iterator = object_ref_iterator_init(p_obj); + + for(unsigned int i=0; itrace_stack; + while(!vector_stack_is_empty(trace_stack)){ + p_ref = (REF*)vector_stack_pop(trace_stack); + scan_object(collector, p_ref); + trace_stack = collector->trace_stack; + } +} + +/* NOTE:: This is another marking version: marking in color bitmap table. + Originally, we have to mark the object before put it into markstack, to + guarantee there is only one occurrance of an object in markstack. This is to + guarantee there is only one occurrance of a repointed ref slot in repset (they + are put to the set when the object is scanned). If the same object is put to + markstack twice, they will be scanned twice and their ref slots will be recorded twice. + Problem occurs when the ref slot is updated first time with new position, + the second time the value in the ref slot is not the old position as expected. + It needs to read the original obj header for forwarding pointer. With the new value, + it will read something nonsense since the obj is not moved yet. + This can be worked around if we want. + To do this we have to use atomic instruction for marking, which is undesirable. + So we abondoned this design. We no longer use the repset to remember repointed slots. +*/ + +/* for marking phase termination detection */ +static volatile unsigned int num_finished_collectors = 0; + +void wspace_fallback_mark_scan(Collector *collector, Wspace *wspace) +{ + GC *gc = collector->gc; + GC_Metadata *metadata = gc->metadata; + wspace_in_fallback_marking = wspace; + + /* reset the num_finished_collectors to be 0 by one collector. This is necessary for the barrier later. */ + unsigned int num_active_collectors = gc->num_active_collectors; + atomic_cas32(&num_finished_collectors, 0, num_active_collectors); + + collector->trace_stack = free_task_pool_get_entry(metadata); + + Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool); + + /* first step: copy all root objects to mark tasks. + FIXME:: can be done sequentially before coming here to eliminate atomic ops */ + while(root_set){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); + while(!vector_block_iterator_end(root_set,iter)){ + REF *p_ref = (REF*)*iter; + iter = vector_block_iterator_advance(root_set,iter); + + /* root ref can't be NULL, (remset may have NULL ref entry, but this function is only for MAJOR_COLLECTION */ + assert(read_slot(p_ref) != NULL); + /* we have to mark the object before putting it into marktask, because + it is possible to have two slots containing a same object. They will + be scanned twice and their ref slots will be recorded twice. Problem + occurs after the ref slot is updated first time with new position + and the second time the value is the ref slot is the old position as expected. + This can be worked around if we want. + */ + collector_tracestack_push(collector, p_ref); + } + root_set = pool_iterator_next(metadata->gc_rootset_pool); + } + /* put back the last trace_stack task */ + pool_put_entry(metadata->mark_task_pool, collector->trace_stack); + + /* second step: iterate over the mark tasks and scan objects */ + /* get a task buf for the mark stack */ + collector->trace_stack = free_task_pool_get_entry(metadata); + +retry: + Vector_Block *mark_task = pool_get_entry(metadata->mark_task_pool); + + while(mark_task){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task); + while(!vector_block_iterator_end(mark_task, iter)){ + REF *p_ref = (REF*)*iter; + iter = vector_block_iterator_advance(mark_task, iter); + + /* FIXME:: we should not let mark_task empty during working, , other may want to steal it. + degenerate my stack into mark_task, and grab another mark_task */ + trace_object(collector, p_ref); + } + /* run out one task, put back to the pool and grab another task */ + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + mark_task = pool_get_entry(metadata->mark_task_pool); + } + + /* termination detection. This is also a barrier. + NOTE:: We can simply spin waiting for num_finished_collectors, because each + generated new task would surely be processed by its generating collector eventually. + So code below is only for load balance optimization. */ + atomic_inc32(&num_finished_collectors); + while(num_finished_collectors != num_active_collectors){ + if(!pool_is_empty(metadata->mark_task_pool)){ + atomic_dec32(&num_finished_collectors); + goto retry; + } + } + + /* put back the last mark stack to the free pool */ + mark_task = (Vector_Block*)collector->trace_stack; + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + collector->trace_stack = NULL; + + return; +} + +void trace_obj_in_ms_fallback_marking(Collector *collector, void *p_ref) +{ + trace_object(collector, (REF*)p_ref); +} Index: src/mark_sweep/wspace_mark.cpp =================================================================== --- src/mark_sweep/wspace_mark.cpp (revision 0) +++ src/mark_sweep/wspace_mark.cpp (revision 0) @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wspace_mark_sweep.h" +#include "../finalizer_weakref/finalizer_weakref.h" + +static Wspace *wspace_in_marking; +static FORCE_INLINE Boolean obj_mark_gray(Partial_Reveal_Object *obj) +{ + if(obj_belongs_to_space(obj, (Space*)wspace_in_marking)) + return obj_mark_gray_in_table(obj); + else + return obj_mark_in_vt(obj); +} + +static FORCE_INLINE Boolean obj_mark_black(Partial_Reveal_Object *obj) +{ + if(obj_belongs_to_space(obj, (Space*)wspace_in_marking)){ + Boolean marked_by_self = obj_mark_black_in_table(obj); + +#ifndef USE_MARK_SWEEP_GC + /* When fallback happens, some objects in MOS have their fw bit set, which is actually their mark bit in the last minor gc. + * If we don't clear it, some objects that didn't be moved will be mistaken for being moved in the coming fixing phase. + */ + if(marked_by_self){ + Obj_Info_Type oi = obj->obj_info; + Obj_Info_Type new_oi = oi & DUAL_MARKBITS_MASK; + while(new_oi != oi){ + Obj_Info_Type temp = atomic_casptrsz((volatile Obj_Info_Type*)get_obj_info_addr(obj), new_oi, oi); + if(temp == oi) break; + oi = obj->obj_info; + new_oi = oi & DUAL_MARKBITS_MASK; + } + } +#endif + return marked_by_self; + } else { + return obj_mark_in_vt(obj); + } +} + + +/* The caller must be in places where alloc color and mark color haven't been flipped */ +Boolean obj_is_marked_in_table(Partial_Reveal_Object *obj) +{ + unsigned int index_in_word; + volatile POINTER_SIZE_INT *p_color_word = get_color_word_in_table(obj, index_in_word); + assert(p_color_word); + + POINTER_SIZE_INT color_word = *p_color_word; + POINTER_SIZE_INT mark_color = cur_mark_gray_color << index_in_word; + + return (Boolean)(color_word & mark_color); +} + +static FORCE_INLINE void scan_slot(Collector *collector, REF *p_ref) +{ + Partial_Reveal_Object *p_obj = read_slot(p_ref); + if( p_obj == NULL) return; + + assert(address_belongs_to_gc_heap(p_obj, collector->gc)); + if(obj_mark_gray(p_obj)){ + assert(p_obj); + collector_tracestack_push(collector, p_obj); + } +} + +static FORCE_INLINE void scan_object(Collector *collector, Partial_Reveal_Object *p_obj) +{ + assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); + + Partial_Reveal_VTable *vtable = decode_vt(obj_get_vt(p_obj)); + if(TRACE_JLC_VIA_VTABLE) + if(vtable->vtmark == VT_UNMARKED) { + vtable->vtmark = VT_MARKED; + if(obj_mark_black(vtable->jlC)) + collector_tracestack_push(collector, vtable->jlC); + } + + if(!object_has_ref_field(p_obj)) return; + + REF *p_ref; + + if(object_is_array(p_obj)){ /* scan array object */ + Partial_Reveal_Array *array = (Partial_Reveal_Array*)p_obj; + unsigned int array_length = array->array_len; + + p_ref = (REF *)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); + for (unsigned int i = 0; i < array_length; i++) + scan_slot(collector, p_ref+i); + + return; + } + + /* scan non-array object */ + unsigned int num_refs = object_ref_field_num(p_obj); + int *ref_iterator = object_ref_iterator_init(p_obj); + + for(unsigned int i=0; itrace_stack; + while(!vector_stack_is_empty(trace_stack)){ + p_obj = (Partial_Reveal_Object*)vector_stack_pop(trace_stack); + scan_object(collector, p_obj); + obj_mark_black(p_obj); + trace_stack = collector->trace_stack; + } +} + +/* NOTE:: This is another marking version: marking in color bitmap table. + Originally, we have to mark the object before put it into markstack, to + guarantee there is only one occurrance of an object in markstack. This is to + guarantee there is only one occurrance of a repointed ref slot in repset (they + are put to the set when the object is scanned). If the same object is put to + markstack twice, they will be scanned twice and their ref slots will be recorded twice. + Problem occurs when the ref slot is updated first time with new position, + the second time the value in the ref slot is not the old position as expected. + It needs to read the original obj header for forwarding pointer. With the new value, + it will read something nonsense since the obj is not moved yet. + This can be worked around if we want. + To do this we have to use atomic instruction for marking, which is undesirable. + So we abondoned this design. We no longer use the repset to remember repointed slots. +*/ + +/* for marking phase termination detection */ +static volatile unsigned int num_finished_collectors = 0; + +void wspace_mark_scan(Collector *collector, Wspace *wspace) +{ + GC *gc = collector->gc; + GC_Metadata *metadata = gc->metadata; + wspace_in_marking = wspace; + + /* reset the num_finished_collectors to be 0 by one collector. This is necessary for the barrier later. */ + unsigned int num_active_collectors = gc->num_active_collectors; + atomic_cas32(&num_finished_collectors, 0, num_active_collectors); + + collector->trace_stack = free_task_pool_get_entry(metadata); + + Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool); + + /* first step: copy all root objects to mark tasks. + FIXME:: can be done sequentially before coming here to eliminate atomic ops */ + while(root_set){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); + while(!vector_block_iterator_end(root_set,iter)){ + REF *p_ref = (REF*)*iter; + iter = vector_block_iterator_advance(root_set,iter); + + Partial_Reveal_Object *p_obj = read_slot(p_ref); + /* root ref can't be NULL, (remset may have NULL ref entry, but this function is only for MAJOR_COLLECTION */ + assert(p_obj != NULL); + /* we have to mark the object before putting it into marktask, because + it is possible to have two slots containing a same object. They will + be scanned twice and their ref slots will be recorded twice. Problem + occurs after the ref slot is updated first time with new position + and the second time the value is the ref slot is the old position as expected. + This can be worked around if we want. + */ + assert(address_belongs_to_gc_heap(p_obj, gc)); + if(obj_mark_gray(p_obj)) + collector_tracestack_push(collector, p_obj); + } + root_set = pool_iterator_next(metadata->gc_rootset_pool); + } + /* put back the last trace_stack task */ + pool_put_entry(metadata->mark_task_pool, collector->trace_stack); + + /* second step: iterate over the mark tasks and scan objects */ + /* get a task buf for the mark stack */ + collector->trace_stack = free_task_pool_get_entry(metadata); + +retry: + Vector_Block *mark_task = pool_get_entry(metadata->mark_task_pool); + + while(mark_task){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task); + while(!vector_block_iterator_end(mark_task, iter)){ + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)*iter; + iter = vector_block_iterator_advance(mark_task, iter); + + /* FIXME:: we should not let mark_task empty during working, , other may want to steal it. + degenerate my stack into mark_task, and grab another mark_task */ + trace_object(collector, p_obj); + } + /* run out one task, put back to the pool and grab another task */ + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + mark_task = pool_get_entry(metadata->mark_task_pool); + } + + /* termination detection. This is also a barrier. + NOTE:: We can simply spin waiting for num_finished_collectors, because each + generated new task would surely be processed by its generating collector eventually. + So code below is only for load balance optimization. */ + atomic_inc32(&num_finished_collectors); + while(num_finished_collectors != num_active_collectors){ + if(!pool_is_empty(metadata->mark_task_pool)){ + atomic_dec32(&num_finished_collectors); + goto retry; + } + } + + /* put back the last mark stack to the free pool */ + mark_task = (Vector_Block*)collector->trace_stack; + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + collector->trace_stack = NULL; + + return; +} + +void trace_obj_in_ms_marking(Collector *collector, void *p_obj) +{ + obj_mark_gray((Partial_Reveal_Object*)p_obj); + trace_object(collector, (Partial_Reveal_Object*)p_obj); +} Index: src/mark_sweep/wspace_mark_mostly_concurrent.cpp =================================================================== --- src/mark_sweep/wspace_mark_mostly_concurrent.cpp (revision 0) +++ src/mark_sweep/wspace_mark_mostly_concurrent.cpp (revision 0) @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wspace_mark_sweep.h" +#include "../finalizer_weakref/finalizer_weakref.h" +#include "../thread/marker.h" + +volatile Boolean need_terminate_concurrent_mark; + +Boolean obj_is_marked_in_table(Partial_Reveal_Object *obj); + +static FORCE_INLINE void scan_slot(Collector* marker, REF *p_ref) +{ + Partial_Reveal_Object *p_obj = read_slot(p_ref); + if( p_obj == NULL) return; + + assert(address_belongs_to_gc_heap(p_obj, marker->gc)); + if(obj_mark_gray_in_table(p_obj)){ + assert(p_obj); + collector_tracestack_push((Collector*)marker, p_obj); + } +} + +static FORCE_INLINE void scan_object(Marker* marker, Partial_Reveal_Object *p_obj) +{ + assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); + if(obj_is_dirty_in_table(p_obj)){ + return; + } + + if(!object_has_ref_field(p_obj)) return; + + REF *p_ref; + + if(object_is_array(p_obj)){ /* scan array object */ + Partial_Reveal_Array *array = (Partial_Reveal_Array*)p_obj; + unsigned int array_length = array->array_len; + + p_ref = (REF *)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); + for (unsigned int i = 0; i < array_length; i++) + scan_slot((Collector*)marker, p_ref+i); + + return; + } + + /* scan non-array object */ + unsigned int num_refs = object_ref_field_num(p_obj); + int *ref_iterator = object_ref_iterator_init(p_obj); + + for(unsigned int i=0; itrace_stack; + while(!vector_stack_is_empty(trace_stack)){ + p_obj = (Partial_Reveal_Object*)vector_stack_pop(trace_stack); + scan_object(marker, p_obj); + obj_mark_black_in_table(p_obj); + trace_stack = marker->trace_stack; + } +} + +/* for marking phase termination detection */ +void wspace_mark_scan_mostly_concurrent_reset() +{ need_terminate_concurrent_mark = FALSE; } + +void wspace_mark_scan_mostly_concurrent_terminate() +{ need_terminate_concurrent_mark = TRUE; } + +static Boolean concurrent_mark_need_terminating(GC* gc) +{ + if(need_terminate_concurrent_mark) return TRUE; + + GC_Metadata *metadata = gc->metadata; + return pool_is_empty(metadata->gc_dirty_set_pool); +} + +static volatile unsigned int num_active_markers = 0; + +void wspace_mark_scan_mostly_concurrent(Marker* marker) +{ + int64 time_mark_start = time_now(); + GC *gc = marker->gc; + GC_Metadata *metadata = gc->metadata; + + /* reset the num_finished_collectors to be 0 by one collector. This is necessary for the barrier later. */ + unsigned int current_thread_id = atomic_inc32(&num_active_markers); + + unsigned int num_dirtyset_slot = 0; + + marker->trace_stack = free_task_pool_get_entry(metadata); + + Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool); + + /* first step: copy all root objects to mark tasks.*/ + while(root_set){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); + while(!vector_block_iterator_end(root_set,iter)){ + REF *p_ref = (REF *)*iter; + iter = vector_block_iterator_advance(root_set,iter); + + Partial_Reveal_Object *p_obj = read_slot(p_ref); + /* root ref can't be NULL, (remset may have NULL ref entry, but this function is only for MAJOR_COLLECTION */ + assert(p_obj!=NULL); + assert(address_belongs_to_gc_heap(p_obj, gc)); + if(obj_mark_gray_in_table(p_obj)) + collector_tracestack_push((Collector*)marker, p_obj); + } + root_set = pool_iterator_next(metadata->gc_rootset_pool); + } + /* put back the last trace_stack task */ + pool_put_entry(metadata->mark_task_pool, marker->trace_stack); + + marker_notify_mark_root_done(marker); + + marker->trace_stack = free_task_pool_get_entry(metadata); + +retry: + + /*second step: mark dirty pool*/ + + Vector_Block* dirty_set = pool_get_entry(metadata->gc_dirty_set_pool); + + while(dirty_set){ + POINTER_SIZE_INT* iter = vector_block_iterator_init(dirty_set); + while(!vector_block_iterator_end(dirty_set,iter)){ + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)*iter; + iter = vector_block_iterator_advance(dirty_set,iter); + + assert(p_obj!=NULL); //FIXME: restrict condition? + + obj_clear_dirty_in_table(p_obj); + obj_clear_mark_in_table(p_obj); + + if(obj_mark_gray_in_table(p_obj)) + collector_tracestack_push((Collector*)marker, p_obj); + + num_dirtyset_slot ++; + } + vector_block_clear(dirty_set); + pool_put_entry(metadata->free_set_pool, dirty_set); + dirty_set = pool_get_entry(metadata->gc_dirty_set_pool); + } + + /* put back the last trace_stack task */ + pool_put_entry(metadata->mark_task_pool, marker->trace_stack); + + /* third step: iterate over the mark tasks and scan objects */ + marker->trace_stack = free_task_pool_get_entry(metadata); + + + Vector_Block *mark_task = pool_get_entry(metadata->mark_task_pool); + + while(mark_task){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task); + while(!vector_block_iterator_end(mark_task,iter)){ + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)*iter; + iter = vector_block_iterator_advance(mark_task,iter); + trace_object(marker, p_obj); + } + /* run out one task, put back to the pool and grab another task */ + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + mark_task = pool_get_entry(metadata->mark_task_pool); + } + + if(current_thread_id == 0){ + gc_prepare_dirty_set(marker->gc); + } + + /* conditions to terminate mark: + 1.All thread finished current job. + 2.Flag is set to terminate concurrent mark. + */ + atomic_dec32(&num_active_markers); + while(num_active_markers != 0 || !concurrent_mark_need_terminating(gc)){ + if(!pool_is_empty(metadata->mark_task_pool) || !pool_is_empty(metadata->gc_dirty_set_pool)){ + atomic_inc32(&num_active_markers); + goto retry; + } + } + + /* put back the last mark stack to the free pool */ + mark_task = (Vector_Block*)marker->trace_stack; + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + marker->trace_stack = NULL; + + int64 time_mark = time_now() - time_mark_start; + marker->time_mark += time_mark; + marker->num_dirty_slots_traced += num_dirtyset_slot; + return; +} + +void trace_obj_in_ms_mostly_concurrent_mark(Collector *collector, void *p_obj) +{ + obj_mark_gray_in_table((Partial_Reveal_Object*)p_obj); + trace_object((Marker*)collector, (Partial_Reveal_Object *)p_obj); +} + Index: src/mark_sweep/wspace_mark_otf_concurrent.cpp =================================================================== --- src/mark_sweep/wspace_mark_otf_concurrent.cpp (revision 0) +++ src/mark_sweep/wspace_mark_otf_concurrent.cpp (revision 0) @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "wspace_mark_sweep.h" +#include "../finalizer_weakref/finalizer_weakref.h" +#include "../thread/marker.h" + +Boolean obj_is_marked_in_table(Partial_Reveal_Object *obj); + +static FORCE_INLINE void scan_slot(Collector* marker, REF *p_ref) +{ + Partial_Reveal_Object *p_obj = read_slot(p_ref); + if( p_obj == NULL) return; + + assert(address_belongs_to_gc_heap(p_obj, marker->gc)); + if(obj_mark_gray_in_table(p_obj)){ + assert(p_obj); + collector_tracestack_push((Collector*)marker, p_obj); + } +} + +static FORCE_INLINE void scan_object(Marker* marker, Partial_Reveal_Object *p_obj) +{ + assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); + + if(obj_is_dirty_in_table(p_obj)){ + assert(obj_is_mark_black_in_table(p_obj)); + return; + } + + if(!object_has_ref_field(p_obj)) return; + + REF *p_ref; + + if(object_is_array(p_obj)){ /* scan array object */ + Partial_Reveal_Array *array = (Partial_Reveal_Array*)p_obj; + unsigned int array_length = array->array_len; + + p_ref = (REF *)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); + for (unsigned int i = 0; i < array_length; i++) + scan_slot((Collector*)marker, p_ref+i); + + return; + } + + /* scan non-array object */ + unsigned int num_refs = object_ref_field_num(p_obj); + int *ref_iterator = object_ref_iterator_init(p_obj); + + for(unsigned int i=0; itrace_stack; + while(!vector_stack_is_empty(trace_stack)){ + p_obj = (Partial_Reveal_Object*)vector_stack_pop(trace_stack); + scan_object(marker, p_obj); + obj_mark_black_in_table(p_obj); + trace_stack = marker->trace_stack; + } +} + +static Boolean concurrent_mark_need_terminating(GC* gc) +{ + GC_Metadata *metadata = gc->metadata; + return gc_local_dirtyset_is_empty(gc) && pool_is_empty(metadata->gc_dirty_set_pool); +} + +/* for marking phase termination detection */ +static volatile unsigned int num_active_markers = 0; + +void wspace_mark_scan_concurrent(Marker* marker) +{ + int64 time_mark_start = time_now(); + GC *gc = marker->gc; + GC_Metadata *metadata = gc->metadata; + + /* reset the num_finished_collectors to be 0 by one collector. This is necessary for the barrier later. */ + atomic_inc32(&num_active_markers); + + marker->trace_stack = free_task_pool_get_entry(metadata); + + Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool); + + /* first step: copy all root objects to mark tasks.*/ + while(root_set){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); + while(!vector_block_iterator_end(root_set,iter)){ + REF *p_ref = (REF *)*iter; + iter = vector_block_iterator_advance(root_set,iter); + + Partial_Reveal_Object *p_obj = read_slot(p_ref); + /* root ref can't be NULL, (remset may have NULL ref entry, but this function is only for MAJOR_COLLECTION */ + assert(p_obj!=NULL); + assert(address_belongs_to_gc_heap(p_obj, gc)); + if(obj_mark_gray_in_table(p_obj)) + collector_tracestack_push((Collector*)marker, p_obj); + } + root_set = pool_iterator_next(metadata->gc_rootset_pool); + } + /* put back the last trace_stack task */ + pool_put_entry(metadata->mark_task_pool, marker->trace_stack); + + marker_notify_mark_root_done(marker); + + marker->trace_stack = free_task_pool_get_entry(metadata); + +retry: + + /*second step: mark dirty object snapshot pool*/ + Vector_Block* dirty_set = pool_get_entry(metadata->gc_dirty_set_pool); + + while(dirty_set){ + POINTER_SIZE_INT* iter = vector_block_iterator_init(dirty_set); + while(!vector_block_iterator_end(dirty_set,iter)){ + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)*iter; + iter = vector_block_iterator_advance(dirty_set,iter); + + assert(p_obj!=NULL); //FIXME: restrict? + if(obj_mark_gray_in_table(p_obj)) + collector_tracestack_push((Collector*)marker, p_obj); + } + vector_block_clear(dirty_set); + pool_put_entry(metadata->free_set_pool, dirty_set); + dirty_set = pool_get_entry(metadata->gc_dirty_set_pool); + } + + /* put back the last trace_stack task */ + pool_put_entry(metadata->mark_task_pool, marker->trace_stack); + + /* third step: iterate over the mark tasks and scan objects */ + /* get a task buf for the mark stack */ + marker->trace_stack = free_task_pool_get_entry(metadata); + + + Vector_Block *mark_task = pool_get_entry(metadata->mark_task_pool); + + while(mark_task){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task); + while(!vector_block_iterator_end(mark_task,iter)){ + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)*iter; + iter = vector_block_iterator_advance(mark_task,iter); + trace_object(marker, p_obj); + } + /* run out one task, put back to the pool and grab another task */ + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + mark_task = pool_get_entry(metadata->mark_task_pool); + } + + /* termination condition: + 1.all thread finished current job. + 2.local snapshot vectors are empty. + 3.global snapshot pool is empty. + */ + atomic_dec32(&num_active_markers); + while(num_active_markers != 0 || !concurrent_mark_need_terminating(gc)){ + if(!pool_is_empty(metadata->mark_task_pool) || !pool_is_empty(metadata->gc_dirty_set_pool)){ + atomic_inc32(&num_active_markers); + goto retry; + }else{ + /*grab a block from mutator and begin tracing*/ + POINTER_SIZE_INT thread_num = (POINTER_SIZE_INT)marker->thread_handle; + Vector_Block* local_dirty_set = gc_get_local_dirty_set(gc, (unsigned int)(thread_num + 1)); + /*1. If local_dirty_set has been set full bit, the block is full and will no longer be put into global snapshot pool; + so it should be checked again to see if there're remaining entries unscanned in it. In this case, the + share bit in local_dirty_set should not be cleared, beacause of rescanning exclusively. + 2. If local_dirty_set has not been set full bit, the block is used by mutator and has the chance to be put into + global snapshot pool. In this case, we simply clear the share bit in local_dirty_set. + */ + if(local_dirty_set != NULL){ + atomic_inc32(&num_active_markers); + while(!vector_block_is_empty(local_dirty_set) || !vector_block_not_full_set_unshared(local_dirty_set)){ + Partial_Reveal_Object* p_obj = (Partial_Reveal_Object*) vector_block_get_entry(local_dirty_set); + if(obj_mark_gray_in_table(p_obj)) + collector_tracestack_push((Collector*)marker, p_obj); + } + goto retry; + } + } + } + + /* put back the last mark stack to the free pool */ + mark_task = (Vector_Block*)marker->trace_stack; + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + marker->trace_stack = NULL; + assert(pool_is_empty(metadata->gc_dirty_set_pool)); + + int64 time_mark = time_now() - time_mark_start; + marker->time_mark = time_mark; + + return; +} + +void trace_obj_in_ms_concurrent_mark(Collector *collector, void *p_obj) +{ + obj_mark_gray_in_table((Partial_Reveal_Object*)p_obj); + trace_object((Marker*)collector, (Partial_Reveal_Object *)p_obj); +} + Index: src/mark_sweep/wspace_mark_sweep.cpp =================================================================== --- src/mark_sweep/wspace_mark_sweep.cpp (revision 0) +++ src/mark_sweep/wspace_mark_sweep.cpp (revision 0) @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wspace_alloc.h" +#include "wspace_mark_sweep.h" +#include "wspace_verify.h" +#include "gc_ms.h" +#include "../gen/gen.h" +#include "../thread/collector.h" +#include "../finalizer_weakref/finalizer_weakref.h" +#include "../common/fix_repointed_refs.h" +#include "../common/gc_concurrent.h" + +POINTER_SIZE_INT cur_alloc_mask = (~MARK_MASK_IN_TABLE) & FLIP_COLOR_MASK_IN_TABLE; +POINTER_SIZE_INT cur_mark_mask = MARK_MASK_IN_TABLE; +POINTER_SIZE_INT cur_alloc_color = OBJ_COLOR_WHITE; +POINTER_SIZE_INT cur_mark_gray_color = OBJ_COLOR_GRAY; +POINTER_SIZE_INT cur_mark_black_color = OBJ_COLOR_BLACK; + +static Chunk_Header_Basic *volatile next_chunk_for_fixing; + + +/******************** General interfaces for Mark-Sweep-Compact ***********************/ +void gc_init_collector_free_chunk_list(Collector *collector) +{ + Free_Chunk_List *list = (Free_Chunk_List*)STD_MALLOC(sizeof(Free_Chunk_List)); + free_chunk_list_init(list); + collector->free_chunk_list = list; +} + +/* Argument need_construct stands for whether or not the dual-directon list needs constructing */ +Chunk_Header_Basic *wspace_grab_next_chunk(Wspace *wspace, Chunk_Header_Basic *volatile *shared_next_chunk, Boolean need_construct) +{ + Chunk_Header_Basic *cur_chunk = *shared_next_chunk; + + Chunk_Header_Basic *wspace_ceiling = (Chunk_Header_Basic*)space_heap_end((Space*)wspace); + while(cur_chunk < wspace_ceiling){ + Chunk_Header_Basic *next_chunk = CHUNK_END(cur_chunk); + + Chunk_Header_Basic *temp = (Chunk_Header_Basic*)atomic_casptr((volatile void**)shared_next_chunk, next_chunk, cur_chunk); + if(temp == cur_chunk){ + if(need_construct && next_chunk < wspace_ceiling) + next_chunk->adj_prev = cur_chunk; + return cur_chunk; + } + cur_chunk = *shared_next_chunk; + } + + return NULL; +} + + +/******************** Interfaces for Forwarding ***********************/ + +static void nos_init_block_for_forwarding(GC_Gen *gc_gen) +{ blocked_space_block_iterator_init((Blocked_Space*)gc_get_nos(gc_gen)); } + +static inline void block_forward_live_objects(Collector *collector, Wspace *wspace, Block_Header *cur_block) +{ + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)cur_block->base; + Partial_Reveal_Object *block_end = (Partial_Reveal_Object*)cur_block->free; + + for(; p_obj < block_end; p_obj = (Partial_Reveal_Object*)((POINTER_SIZE_INT)p_obj + vm_object_size(p_obj))){ + if(!obj_is_marked_in_vt(p_obj)) continue; + + obj_clear_dual_bits_in_vt(p_obj); + Partial_Reveal_Object *p_target_obj = collector_forward_object(collector, p_obj); /* Could be implemented with a optimized function */ + if(!p_target_obj){ + assert(collector->result == FALSE); + printf("Out of mem in forwarding nos!\n"); + exit(0); + } + } +} + +static void collector_forward_nos_to_wspace(Collector *collector, Wspace *wspace) +{ + Blocked_Space *nos = (Blocked_Space*)gc_get_nos((GC_Gen*)collector->gc); + Block_Header *cur_block = blocked_space_block_iterator_next(nos); + + /* We must iterate over all nos blocks to forward live objects in them */ + while(cur_block){ + block_forward_live_objects(collector, wspace, cur_block); + cur_block = blocked_space_block_iterator_next(nos); + } +} + + +/******************** Interfaces for Ref Fixing ***********************/ + +static void wspace_init_chunk_for_ref_fixing(Wspace *wspace) +{ + next_chunk_for_fixing = (Chunk_Header_Basic*)space_heap_start((Space*)wspace); + next_chunk_for_fixing->adj_prev = NULL; +} + +static inline void slot_double_fix(REF *p_ref) +{ + Partial_Reveal_Object *p_obj = read_slot(p_ref); + if(!p_obj) return; + + if(obj_is_fw_in_oi(p_obj)){ + p_obj = obj_get_fw_in_oi(p_obj); + assert(p_obj); + if(obj_is_fw_in_oi(p_obj)){ + p_obj = obj_get_fw_in_oi(p_obj); + assert(p_obj); + } + write_slot(p_ref, p_obj); + } +} + +static inline void object_double_fix_ref_slots(Partial_Reveal_Object *p_obj) +{ + if(!object_has_ref_field(p_obj)) return; + + /* scan array object */ + if(object_is_array(p_obj)){ + Partial_Reveal_Array *array = (Partial_Reveal_Array*)p_obj; + assert(!obj_is_primitive_array(p_obj)); + + int32 array_length = array->array_len; + REF *p_refs = (REF*)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); + for(int i = 0; i < array_length; i++){ + slot_double_fix(p_refs + i); + } + return; + } + + /* scan non-array object */ + unsigned int num_refs = object_ref_field_num(p_obj); + int* ref_iterator = object_ref_iterator_init(p_obj); + + for(unsigned int i=0; islot_index = 0; + chunk_depad_last_index_word(chunk); + + unsigned int alloc_num = chunk->alloc_num; + assert(alloc_num); + + /* After compaction, many chunks are filled with objects. + * For these chunks, we needn't find the allocated slot one by one by calling next_alloc_slot_in_chunk. + * That is a little time consuming. + * We'd like to fix those objects by incrementing their addr to find the next. + */ + if(alloc_num == chunk->slot_num){ /* Filled with objects */ + unsigned int slot_size = chunk->slot_size; + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)slot_index_to_addr(chunk, 0); + for(unsigned int i = alloc_num; i--;){ + if(double_fix) + object_double_fix_ref_slots(p_obj); + else + object_fix_ref_slots(p_obj); +#ifdef SSPACE_VERIFY + wspace_verify_fix_in_compact(); +#endif + p_obj = (Partial_Reveal_Object*)((POINTER_SIZE_INT)p_obj + slot_size); + } + } else { /* Chunk is not full */ + while(alloc_num){ + Partial_Reveal_Object *p_obj = next_alloc_slot_in_chunk(chunk); + assert(p_obj); + if(double_fix) + object_double_fix_ref_slots(p_obj); + else + object_fix_ref_slots(p_obj); +#ifdef SSPACE_VERIFY + wspace_verify_fix_in_compact(); +#endif + --alloc_num; + } + } + + if(chunk->alloc_num != chunk->slot_num){ + //chunk_pad_last_index_word(chunk, cur_alloc_mask); + pfc_reset_slot_index(chunk); + } +} + +static void abnormal_chunk_fix_repointed_refs(Chunk_Header *chunk, Boolean double_fix) +{ + if(double_fix) + object_double_fix_ref_slots((Partial_Reveal_Object*)chunk->base); + else + object_fix_ref_slots((Partial_Reveal_Object*)chunk->base); +#ifdef SSPACE_VERIFY + wspace_verify_fix_in_compact(); +#endif +} + +static void wspace_fix_repointed_refs(Collector *collector, Wspace *wspace, Boolean double_fix) +{ + Chunk_Header_Basic *chunk = wspace_grab_next_chunk(wspace, &next_chunk_for_fixing, TRUE); + + while(chunk){ + if(chunk->status & CHUNK_NORMAL) + normal_chunk_fix_repointed_refs((Chunk_Header*)chunk, double_fix); + else if(chunk->status & CHUNK_ABNORMAL) + abnormal_chunk_fix_repointed_refs((Chunk_Header*)chunk, double_fix); + + chunk = wspace_grab_next_chunk(wspace, &next_chunk_for_fixing, TRUE); + } +} + + +/******************** Main body of Mark-Sweep-Compact ***********************/ + +static volatile unsigned int num_marking_collectors = 0; +static volatile unsigned int num_sweeping_collectors = 0; +static volatile unsigned int num_compacting_collectors = 0; +static volatile unsigned int num_forwarding_collectors = 0; +static volatile unsigned int num_fixing_collectors = 0; + +void mark_sweep_wspace(Collector *collector) +{ + GC *gc = collector->gc; + Wspace *wspace = gc_get_wspace(gc); + Space *nos = NULL; + if(gc_match_kind(gc, MAJOR_COLLECTION)) + nos = gc_get_nos((GC_Gen*)gc); + + unsigned int num_active_collectors = gc->num_active_collectors; + + /* Pass 1: ************************************************** + Mark all live objects in heap ****************************/ + atomic_cas32(&num_marking_collectors, 0, num_active_collectors+1); + + if(!gc_mark_is_concurrent()){ + if(gc_match_kind(gc, FALLBACK_COLLECTION)) + wspace_fallback_mark_scan(collector, wspace); + else + wspace_mark_scan(collector, wspace); + } + + unsigned int old_num = atomic_inc32(&num_marking_collectors); + if( ++old_num == num_active_collectors ){ + /* last collector's world here */ +#ifdef SSPACE_TIME + wspace_mark_time(FALSE); +#endif + if(!IGNORE_FINREF ) + collector_identify_finref(collector); +#ifndef BUILD_IN_REFERENT + else { + gc_set_weakref_sets(gc); + gc_update_weakref_ignore_finref(gc); + } +#endif + gc_identify_dead_weak_roots(gc); + gc_init_chunk_for_sweep(gc, wspace); + /* let other collectors go */ + num_marking_collectors++; + } + while(num_marking_collectors != num_active_collectors + 1); + + /* Pass 2: ************************************************** + Sweep dead objects ***************************************/ + atomic_cas32( &num_sweeping_collectors, 0, num_active_collectors+1); + + wspace_sweep(collector, wspace); + + old_num = atomic_inc32(&num_sweeping_collectors); + if( ++old_num == num_active_collectors ){ +#ifdef SSPACE_TIME + wspace_sweep_time(FALSE, wspace->need_compact); +#endif + ops_color_flip(); +#ifdef SSPACE_VERIFY + wspace_verify_after_sweep(gc); +#endif + + if(gc_match_kind(gc, MAJOR_COLLECTION)){ + wspace_merge_free_chunks(gc, wspace); + nos_init_block_for_forwarding((GC_Gen*)gc); + } + if(wspace->need_compact) + wspace_init_pfc_pool_iterator(wspace); + if(wspace->need_fix) + wspace_init_chunk_for_ref_fixing(wspace); + /* let other collectors go */ + num_sweeping_collectors++; + } + while(num_sweeping_collectors != num_active_collectors + 1); + + /* Optional Pass: ******************************************* + Forward live obj in nos to mos (wspace) ******************/ + if(gc_match_kind(gc, MAJOR_COLLECTION)){ + atomic_cas32( &num_forwarding_collectors, 0, num_active_collectors+1); + + collector_forward_nos_to_wspace(collector, wspace); + + old_num = atomic_inc32(&num_forwarding_collectors); + if( ++old_num == num_active_collectors ){ + gc_clear_collector_local_chunks(gc); + num_forwarding_collectors++; + } + + while(num_forwarding_collectors != num_active_collectors + 1); + } + + /* Optional Pass: ******************************************* + Compact pfcs with the same size **************************/ + if(wspace->need_compact){ + atomic_cas32(&num_compacting_collectors, 0, num_active_collectors+1); + + wspace_compact(collector, wspace); + + /* If we need forward nos to mos, i.e. in major collection, an extra fixing phase after compaction is needed. */ + old_num = atomic_inc32(&num_compacting_collectors); + if( ++old_num == num_active_collectors ){ + if(gc_match_kind(gc, MAJOR_COLLECTION)) + wspace_remerge_free_chunks(gc, wspace); + /* let other collectors go */ + num_compacting_collectors++; + } + while(num_compacting_collectors != num_active_collectors + 1); + } + + /* Optional Pass: ******************************************* + Fix repointed refs ***************************************/ + if(wspace->need_fix){ + atomic_cas32( &num_fixing_collectors, 0, num_active_collectors); + + /* When we forwarded nos AND compacted wspace, + * we need double fix object slots, + * because some objects are forwarded from nos to mos and compacted into another chunk afterwards. + */ + Boolean double_fix = gc_match_kind(gc, MAJOR_COLLECTION) && wspace->need_compact; + wspace_fix_repointed_refs(collector, wspace, double_fix); + + atomic_inc32(&num_fixing_collectors); + while(num_fixing_collectors != num_active_collectors); + } + + if( collector->thread_handle != 0 ) + return; + + /* Leftover: *************************************************/ + + if(wspace->need_fix){ + Boolean double_fix = gc_match_kind(gc, MAJOR_COLLECTION) && wspace->need_compact; + gc_fix_rootset(collector, double_fix); +#ifdef SSPACE_TIME + wspace_fix_time(FALSE); +#endif + } + + if(!gc_match_kind(gc, MAJOR_COLLECTION)) + wspace_merge_free_chunks(gc, wspace); + +#ifdef USE_MARK_SWEEP_GC + wspace_set_space_statistic(wspace); +#endif + +#ifdef SSPACE_VERIFY + wspace_verify_after_collection(gc); +#endif +} Index: src/mark_sweep/wspace_mark_sweep.h =================================================================== --- src/mark_sweep/wspace_mark_sweep.h (revision 0) +++ src/mark_sweep/wspace_mark_sweep.h (revision 0) @@ -0,0 +1,453 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _SSPACE_MARK_SWEEP_H_ +#define _SSPACE_MARK_SWEEP_H_ + +#include "wspace_chunk.h" +#include "wspace_verify.h" + +#define PFC_REUSABLE_RATIO 0.1 +#define SSPACE_COMPACT_RATIO 0.06 + +inline Boolean chunk_is_reusable(Chunk_Header *chunk) +{ return (float)(chunk->slot_num-chunk->alloc_num)/chunk->slot_num > PFC_REUSABLE_RATIO; } + +#define OBJ_ALLOC_BIT_IN_TABLE 0x01 +#define OBJ_BLACK_BIT_IN_TABLE 0x02 +#define OBJ_GRAY_BIT_IN_TABLE 0x04 +#define OBJ_COLOR_BIT_IN_TABLE 0x06 +#define OBJ_DIRTY_BIT_IN_TABLE 0x08 + +enum Obj_Color { + OBJ_COLOR_BLUE = 0x0, + OBJ_COLOR_WHITE = OBJ_ALLOC_BIT_IN_TABLE, + OBJ_COLOR_GRAY = OBJ_GRAY_BIT_IN_TABLE, + OBJ_COLOR_BLACK = OBJ_BLACK_BIT_IN_TABLE, + OBJ_COLOR_MASK = OBJ_COLOR_BIT_IN_TABLE +}; + +#ifdef POINTER64 + //#define BLACK_MASK_IN_TABLE ((POINTER_SIZE_INT)0xAAAAAAAAAAAAAAAA) + #define MARK_MASK_IN_TABLE ((POINTER_SIZE_INT)0x2222222222222222) + #define FLIP_COLOR_MASK_IN_TABLE ((POINTER_SIZE_INT)0x3333333333333333) + //#define DIRY_MASK_IN_TABLE ((POINTER_SIZE_INT)0x4444444444444444) +#else + #define MARK_MASK_IN_TABLE ((POINTER_SIZE_INT)0x22222222) + #define FLIP_COLOR_MASK_IN_TABLE ((POINTER_SIZE_INT)0x33333333) + //#define DIRY_MASK_IN_TABLE ((POINTER_SIZE_INT)0x44444444) +#endif + +extern POINTER_SIZE_INT cur_alloc_color; +extern POINTER_SIZE_INT cur_mark_gray_color; +extern POINTER_SIZE_INT cur_mark_black_color; +extern POINTER_SIZE_INT cur_alloc_mask; +extern POINTER_SIZE_INT cur_mark_mask; + +inline Boolean is_super_obj(Partial_Reveal_Object *obj) +{ + //return get_obj_info_raw(obj) & SUPER_OBJ_MASK;/* + if(vm_object_size(obj) > SUPER_OBJ_THRESHOLD){ + return TRUE; + } else { + return FALSE; + } +} + +FORCE_INLINE POINTER_SIZE_INT *get_color_word_in_table(Partial_Reveal_Object *obj, unsigned int &index_in_word) +{ + Chunk_Header *chunk; + unsigned int index; + + if(is_super_obj(obj)){ + chunk = ABNORMAL_CHUNK_HEADER(obj); + index = 0; + } else { + chunk = NORMAL_CHUNK_HEADER(obj); + index = slot_addr_to_index(chunk, obj); + } + unsigned int word_index = index / SLOT_NUM_PER_WORD_IN_TABLE; + index_in_word = COLOR_BITS_PER_OBJ * (index % SLOT_NUM_PER_WORD_IN_TABLE); + //unsigned int word_index = index >> 3; + //index_in_word = COLOR_BITS_PER_OBJ * (index & (((unsigned int)(SLOT_NUM_PER_WORD_IN_TABLE-1)))); + + return &chunk->table[word_index]; +} +FORCE_INLINE POINTER_SIZE_INT *get_color_word_in_table(Partial_Reveal_Object *obj, unsigned int &index_in_word, unsigned int size) +{ + Chunk_Header *chunk; + unsigned int index; + + if(size > SUPER_OBJ_THRESHOLD){ + chunk = ABNORMAL_CHUNK_HEADER(obj); + index = 0; + } else { + chunk = NORMAL_CHUNK_HEADER(obj); + index = slot_addr_to_index(chunk, obj); + } + unsigned int word_index = index >> 3; + index_in_word = COLOR_BITS_PER_OBJ * (index & (((unsigned int)(SLOT_NUM_PER_WORD_IN_TABLE-1)))); + + return &chunk->table[word_index]; +} + +#if 0 +/* Accurate marking: TRUE stands for being marked by this collector, and FALSE for another collector */ +inline Boolean obj_mark_in_table(Partial_Reveal_Object *obj) +{ + volatile POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + assert(p_color_word); + + POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); + POINTER_SIZE_INT mark_color = cur_mark_color << index_in_word; + + POINTER_SIZE_INT old_word = *p_color_word; + POINTER_SIZE_INT new_word = (old_word & color_bits_mask) | mark_color; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ +#ifdef SSPACE_VERIFY +#ifndef SSPACE_VERIFY_FINREF + assert(obj_is_marked_in_vt(obj)); +#endif + obj_unmark_in_vt(obj); + wspace_record_mark(obj, vm_object_size(obj)); +#endif + return TRUE; + } + old_word = *p_color_word; + new_word = (old_word & color_bits_mask) | mark_color; + } + + return FALSE; +} + +#endif + +FORCE_INLINE Boolean obj_is_mark_gray_in_table(Partial_Reveal_Object *obj) +{ + POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + POINTER_SIZE_INT current_word = *p_color_word; + POINTER_SIZE_INT mark_gray_color = cur_mark_gray_color << index_in_word; + + if(current_word & mark_gray_color) + return TRUE; + else + return FALSE; +} + +FORCE_INLINE Boolean obj_is_mark_black_in_table(Partial_Reveal_Object *obj) +{ + POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + POINTER_SIZE_INT current_word = *p_color_word; + POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; + + if(current_word & mark_black_color) + return TRUE; + else + return FALSE; + +} + +FORCE_INLINE Boolean obj_is_mark_black_in_table(Partial_Reveal_Object *obj, unsigned int size) +{ + POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word, size); + POINTER_SIZE_INT current_word = *p_color_word; + POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; + + if(current_word & mark_black_color) + return TRUE; + else + return FALSE; + +} + + +FORCE_INLINE Boolean obj_mark_gray_in_table(Partial_Reveal_Object *obj) +{ + volatile POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + assert(p_color_word); + + //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); + POINTER_SIZE_INT mark_color = cur_mark_gray_color << index_in_word; + + POINTER_SIZE_INT old_word = *p_color_word; + if(old_word & mark_color) return FALSE; /*already marked gray or black.*/ + + //POINTER_SIZE_INT new_word = (old_word & color_bits_mask) | mark_color; + POINTER_SIZE_INT new_word = old_word | mark_color; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return TRUE; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + if(old_word & mark_color) return FALSE; /*already marked gray or black.*/ + + //new_word = (old_word & color_bits_mask) | mark_color; + new_word = old_word | mark_color; + } + + return FALSE; +} + +FORCE_INLINE Boolean obj_mark_black_in_table(Partial_Reveal_Object *obj, unsigned int size) +{ + //assert(obj_is_mark_in_table(obj)); + volatile POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word, size); + assert(p_color_word); + + //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); + POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; + + POINTER_SIZE_INT old_word = *p_color_word; + if(old_word & mark_black_color) return FALSE; /*already marked black*/ + + POINTER_SIZE_INT new_word = old_word | mark_black_color; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return TRUE; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + if(old_word & mark_black_color) return FALSE; /*already marked black*/ + + new_word = old_word | mark_black_color; + } + + return FALSE; + +} + +FORCE_INLINE Boolean obj_mark_black_in_table(Partial_Reveal_Object *obj) +{ + // assert(obj_is_mark_in_table(obj)); + volatile POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + assert(p_color_word); + + //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); + POINTER_SIZE_INT mark_black_color = cur_mark_black_color << index_in_word; + + POINTER_SIZE_INT old_word = *p_color_word; + if(obj_is_mark_black_in_table(obj)) return FALSE; /*already marked black*/ + + POINTER_SIZE_INT new_word = old_word | mark_black_color; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return TRUE; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + if(obj_is_mark_black_in_table(obj)) return FALSE; /*already marked black*/ + + new_word = old_word | mark_black_color; + } + + return FALSE; +} + +FORCE_INLINE Boolean obj_dirty_in_table(Partial_Reveal_Object *obj) +{ + volatile POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + assert(p_color_word); + + POINTER_SIZE_INT obj_dirty_bit_in_word = OBJ_DIRTY_BIT_IN_TABLE<< index_in_word; + + POINTER_SIZE_INT old_word = *p_color_word; + if(old_word & obj_dirty_bit_in_word) return FALSE; + + POINTER_SIZE_INT new_word = old_word | obj_dirty_bit_in_word; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return TRUE; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + if(old_word & obj_dirty_bit_in_word) return FALSE; + + new_word = old_word | obj_dirty_bit_in_word; + } + + return FALSE; +} + +FORCE_INLINE Boolean obj_is_dirty_in_table(Partial_Reveal_Object *obj) +{ + POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + POINTER_SIZE_INT current_word = *p_color_word; + POINTER_SIZE_INT obj_dirty_bit_in_word = OBJ_DIRTY_BIT_IN_TABLE<< index_in_word; + + + if(current_word & obj_dirty_bit_in_word) + return TRUE; + else + return FALSE; +} + +FORCE_INLINE Boolean obj_clear_mark_in_table(Partial_Reveal_Object *obj) +{ + volatile POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + assert(p_color_word); + + //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); + POINTER_SIZE_INT mark_color = (cur_mark_black_color|cur_mark_gray_color) << index_in_word; + POINTER_SIZE_INT clear_mask = ~mark_color; + + POINTER_SIZE_INT old_word = *p_color_word; + + POINTER_SIZE_INT new_word = old_word & clear_mask; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return TRUE; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + //if(old_word & clear_mask) return FALSE; /*already marked black*/ + + new_word = old_word & clear_mask; + } + + return FALSE; + +} + +FORCE_INLINE Boolean obj_clear_dirty_in_table(Partial_Reveal_Object *obj) +{ + volatile POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + assert(p_color_word); + + //POINTER_SIZE_INT color_bits_mask = ~(OBJ_COLOR_MASK << index_in_word); + POINTER_SIZE_INT mark_color = OBJ_DIRTY_BIT_IN_TABLE << index_in_word; + POINTER_SIZE_INT clear_mask = ~mark_color; + + POINTER_SIZE_INT old_word = *p_color_word; + + POINTER_SIZE_INT new_word = old_word & clear_mask; + while(new_word != old_word) { + POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void**)p_color_word, (void*)new_word, (void*)old_word); + if(temp == old_word){ + return TRUE; /*returning true does not mean it's marked by this thread. */ + } + old_word = *p_color_word; + //if(old_word & clear_mask) return FALSE; /*already marked black*/ + + new_word = old_word & clear_mask; + } + + return FALSE; + +} + +FORCE_INLINE Boolean obj_is_alloc_in_color_table(Partial_Reveal_Object *obj) +{ + POINTER_SIZE_INT *p_color_word; + unsigned int index_in_word; + p_color_word = get_color_word_in_table(obj, index_in_word); + POINTER_SIZE_INT current_word = *p_color_word; + POINTER_SIZE_INT obj_alloc_color_bit_in_word = cur_alloc_color << index_in_word; + + return (Boolean)(current_word & obj_alloc_color_bit_in_word); +} + +FORCE_INLINE Boolean obj_need_take_snapshot(Partial_Reveal_Object *obj) +{ + return !obj_is_mark_black_in_table(obj) && !obj_is_dirty_in_table(obj); +} + +FORCE_INLINE Boolean obj_need_remember(Partial_Reveal_Object *obj) +{ + return (obj_is_mark_gray_in_table(obj) || obj_is_mark_black_in_table(obj)) && !obj_is_dirty_in_table(obj); +} + +FORCE_INLINE Boolean obj_need_remember_oldvar(Partial_Reveal_Object *obj) +{ + return !obj_is_mark_gray_in_table(obj) && !obj_is_mark_black_in_table(obj); +} + +inline void collector_add_free_chunk(Collector *collector, Free_Chunk *chunk) +{ + Free_Chunk_List *list = collector->free_chunk_list; + + chunk->status = CHUNK_FREE | CHUNK_TO_MERGE; + chunk->next = list->head; + chunk->prev = NULL; + if(list->head) + list->head->prev = chunk; + else + list->tail = chunk; + list->head = chunk; +} + + +inline unsigned int word_set_bit_num(POINTER_SIZE_INT word) +{ + unsigned int count = 0; + + while(word){ + word &= word - 1; + ++count; + } + return count; +} + +inline void ops_color_flip(void) +{ + POINTER_SIZE_INT temp = cur_alloc_color; + cur_alloc_color = cur_mark_black_color; + //FIXME: Need barrier here. + //apr_memory_rw_barrier(); + cur_mark_black_color = temp; + cur_alloc_mask = (~cur_alloc_mask) & FLIP_COLOR_MASK_IN_TABLE; + cur_mark_mask = (~cur_mark_mask) & FLIP_COLOR_MASK_IN_TABLE; + //printf("color flip\n"); +} + +extern void wspace_mark_scan(Collector *collector, Wspace *wspace); +extern void wspace_fallback_mark_scan(Collector *collector, Wspace *wspace); +extern void gc_init_chunk_for_sweep(GC *gc, Wspace *wspace); +extern void wspace_sweep(Collector *collector, Wspace *wspace); +extern void wspace_compact(Collector *collector, Wspace *wspace); +extern void wspace_merge_free_chunks(GC *gc, Wspace *wspace); +extern void wspace_remerge_free_chunks(GC *gc, Wspace *wspace); +extern Chunk_Header_Basic *wspace_grab_next_chunk(Wspace *wspace, Chunk_Header_Basic *volatile *shared_next_chunk, Boolean need_construct); + +extern void pfc_set_slot_index(Chunk_Header *chunk, unsigned int first_free_word_index, POINTER_SIZE_INT alloc_color); +extern void pfc_reset_slot_index(Chunk_Header *chunk); + +#endif // _SSPACE_MARK_SWEEP_H_ Index: src/mark_sweep/wspace_sweep.cpp =================================================================== --- src/mark_sweep/wspace_sweep.cpp (revision 0) +++ src/mark_sweep/wspace_sweep.cpp (revision 0) @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wspace_chunk.h" +#include "wspace_mark_sweep.h" + + +static Chunk_Header_Basic *volatile next_chunk_for_sweep; + + +void gc_init_chunk_for_sweep(GC *gc, Wspace *wspace) +{ + next_chunk_for_sweep = (Chunk_Header_Basic*)space_heap_start((Space*)wspace); + next_chunk_for_sweep->adj_prev = NULL; + + unsigned int i = gc->num_collectors; + while(i--){ + Free_Chunk_List *list = gc->collectors[i]->free_chunk_list; + assert(!list->head); + assert(!list->tail); + assert(list->lock == FREE_LOCK); + } +} + +void zeroing_free_chunk(Free_Chunk *chunk) +{ + //Modified this assertion for concurrent sweep + //assert(chunk->status == CHUNK_FREE); + assert(chunk->status & CHUNK_FREE); + + void *start = (void*)((POINTER_SIZE_INT)chunk + sizeof(Free_Chunk)); + POINTER_SIZE_INT size = CHUNK_SIZE(chunk) - sizeof(Free_Chunk); + memset(start, 0, size); +} + +/* Zeroing should be optimized to do it while sweeping index word */ +static void zeroing_free_areas_in_pfc(Chunk_Header *chunk, unsigned int live_num) +{ + assert(live_num); + + assert(chunk->status & CHUNK_NORMAL); + unsigned int slot_num = chunk->slot_num; + unsigned int slot_size = chunk->slot_size; + POINTER_SIZE_INT chunk_base = (POINTER_SIZE_INT)chunk->base; + POINTER_SIZE_INT *table = chunk->table; + + POINTER_SIZE_INT base = (POINTER_SIZE_INT)NULL; + assert(slot_num >= live_num); + unsigned int free_slot_num = slot_num - live_num; + unsigned int cur_free_slot_num = 0; + unsigned int slot_index = chunk->slot_index; + unsigned int word_index = slot_index / SLOT_NUM_PER_WORD_IN_TABLE; + assert(live_num >= slot_index); + live_num -= slot_index; + POINTER_SIZE_INT index_word = table[word_index]; + POINTER_SIZE_INT mark_color = cur_mark_black_color << (COLOR_BITS_PER_OBJ * (slot_index % SLOT_NUM_PER_WORD_IN_TABLE)); + for(; slot_index < slot_num; ++slot_index){ + assert(!(index_word & ~cur_mark_mask)); + if(index_word & mark_color){ + if(cur_free_slot_num){ + memset((void*)base, 0, slot_size*cur_free_slot_num); + assert(free_slot_num >= cur_free_slot_num); + free_slot_num -= cur_free_slot_num; + cur_free_slot_num = 0; + if(!free_slot_num) break; + } + assert(live_num); + --live_num; + } else { + if(cur_free_slot_num){ + ++cur_free_slot_num; + } else { + base = chunk_base + slot_size * slot_index; + cur_free_slot_num = 1; + if(!live_num) break; + } + } + mark_color <<= COLOR_BITS_PER_OBJ; + if(!mark_color){ + mark_color = cur_mark_black_color; + ++word_index; + index_word = table[word_index]; + while(index_word == cur_mark_mask && cur_free_slot_num == 0 && slot_index < slot_num){ + slot_index += SLOT_NUM_PER_WORD_IN_TABLE; + ++word_index; + index_word = table[word_index]; + assert(live_num >= SLOT_NUM_PER_WORD_IN_TABLE); + live_num -= SLOT_NUM_PER_WORD_IN_TABLE; + } + while(index_word == 0 && cur_free_slot_num > 0 && slot_index < slot_num){ + slot_index += SLOT_NUM_PER_WORD_IN_TABLE; + ++word_index; + index_word = table[word_index]; + cur_free_slot_num += SLOT_NUM_PER_WORD_IN_TABLE; + } + } + } + assert((cur_free_slot_num>0 && live_num==0) || (cur_free_slot_num==0 && live_num>0)); + if(cur_free_slot_num) + memset((void*)base, 0, slot_size*free_slot_num); +} + +static void collector_sweep_normal_chunk(Collector *collector, Wspace *wspace, Chunk_Header *chunk) +{ + unsigned int slot_num = chunk->slot_num; + unsigned int live_num = 0; + unsigned int first_free_word_index = MAX_SLOT_INDEX; + POINTER_SIZE_INT *table = chunk->table; + + unsigned int index_word_num = (slot_num + SLOT_NUM_PER_WORD_IN_TABLE - 1) / SLOT_NUM_PER_WORD_IN_TABLE; + for(unsigned int i=0; ialloc_num = live_num; + collector->live_obj_size += live_num * chunk->slot_size; + collector->live_obj_num += live_num; + + if(!live_num){ /* all objects in this chunk are dead */ + collector_add_free_chunk(collector, (Free_Chunk*)chunk); + } else if(chunk_is_reusable(chunk)){ /* most objects in this chunk are swept, add chunk to pfc list*/ + chunk->alloc_num = live_num; + //chunk_pad_last_index_word((Chunk_Header*)chunk, cur_mark_mask); + wspace_put_pfc(wspace, chunk); + assert(chunk->next != chunk); + }else{ /* the rest: chunks with free rate < PFC_REUSABLE_RATIO. we don't use them */ + chunk->alloc_num = live_num; + chunk->status = CHUNK_USED | CHUNK_NORMAL; + wspace_register_used_chunk(wspace,chunk); + } +} + +static inline void collector_sweep_abnormal_chunk(Collector *collector, Wspace *wspace, Chunk_Header *chunk) +{ + assert(chunk->status & CHUNK_ABNORMAL); + POINTER_SIZE_INT *table = chunk->table; + table[0] &= cur_mark_mask; + if(!table[0]){ + collector_add_free_chunk(collector, (Free_Chunk*)chunk); + } + else { + chunk->status = CHUNK_ABNORMAL| CHUNK_USED; + wspace_register_used_chunk(wspace,chunk); + collector->live_obj_size += CHUNK_SIZE(chunk); + collector->live_obj_num++; + } +} + +void wspace_sweep(Collector *collector, Wspace *wspace) +{ + Chunk_Header_Basic *chunk; + collector->live_obj_size = 0; + collector->live_obj_num = 0; + + chunk = wspace_grab_next_chunk(wspace, &next_chunk_for_sweep, TRUE); + while(chunk){ + /* chunk is free before GC */ + if(chunk->status == CHUNK_FREE){ + collector_add_free_chunk(collector, (Free_Chunk*)chunk); + } else if(chunk->status & CHUNK_NORMAL){ /* chunk is used as a normal sized obj chunk */ + collector_sweep_normal_chunk(collector, wspace, (Chunk_Header*)chunk); + } else { /* chunk is used as a super obj chunk */ + collector_sweep_abnormal_chunk(collector, wspace, (Chunk_Header*)chunk); + } + + chunk = wspace_grab_next_chunk(wspace, &next_chunk_for_sweep, TRUE); + } +} + +/************ For merging free chunks in wspace ************/ + +static void merge_free_chunks_in_list(Wspace *wspace, Free_Chunk_List *list) +{ + Free_Chunk *wspace_ceiling = (Free_Chunk*)space_heap_end((Space*)wspace); + Free_Chunk *chunk = list->head; + + while(chunk){ + assert(chunk->status == (CHUNK_FREE | CHUNK_TO_MERGE)); + /* Remove current chunk from the chunk list */ + list->head = chunk->next; + if(list->head) + list->head->prev = NULL; + /* Check if the prev adjacent chunks are free */ + Free_Chunk *prev_chunk = (Free_Chunk*)chunk->adj_prev; + while(prev_chunk && prev_chunk->status == (CHUNK_FREE | CHUNK_TO_MERGE)){ + assert(prev_chunk < chunk); + /* Remove prev_chunk from list */ + free_list_detach_chunk(list, prev_chunk); + prev_chunk->adj_next = chunk->adj_next; + chunk = prev_chunk; + prev_chunk = (Free_Chunk*)chunk->adj_prev; + } + /* Check if the back adjcent chunks are free */ + Free_Chunk *back_chunk = (Free_Chunk*)chunk->adj_next; + while(back_chunk < wspace_ceiling && back_chunk->status == (CHUNK_FREE | CHUNK_TO_MERGE)){ + assert(chunk < back_chunk); + /* Remove back_chunk from list */ + free_list_detach_chunk(list, back_chunk); + back_chunk = (Free_Chunk*)back_chunk->adj_next; + chunk->adj_next = (Chunk_Header_Basic*)back_chunk; + } + if(back_chunk < wspace_ceiling) + back_chunk->adj_prev = (Chunk_Header_Basic*)chunk; + + /* put the free chunk to the according free chunk list */ + wspace_put_free_chunk(wspace, chunk); + + chunk = list->head; + } +} + +void wspace_merge_free_chunks(GC *gc, Wspace *wspace) +{ + Free_Chunk_List free_chunk_list; + free_chunk_list.head = NULL; + free_chunk_list.tail = NULL; + + /* Collect free chunks from collectors to one list */ + for(unsigned int i=0; inum_collectors; ++i){ + Free_Chunk_List *list = gc->collectors[i]->free_chunk_list; + move_free_chunks_between_lists(&free_chunk_list, list); + } + + merge_free_chunks_in_list(wspace, &free_chunk_list); +} + +void wspace_remerge_free_chunks(GC *gc, Wspace *wspace) +{ + Free_Chunk_List free_chunk_list; + free_chunk_list.head = NULL; + free_chunk_list.tail = NULL; + + /* If a new chunk is partitioned from a bigger one in the forwarding phase, + * its adj_prev has not been set yet. + * And the adj_prev field of the chunk next to it will be wrong either. + * So a rebuilding operation is needed here. + */ + wspace_rebuild_chunk_chain(wspace); + + /* Collect free chunks from wspace free chunk lists to one list */ + wspace_collect_free_chunks_to_list(wspace, &free_chunk_list); + + /* Collect free chunks from collectors to one list */ + for(unsigned int i=0; inum_collectors; ++i){ + Free_Chunk_List *list = gc->collectors[i]->free_chunk_list; + move_free_chunks_between_lists(&free_chunk_list, list); + } + + merge_free_chunks_in_list(wspace, &free_chunk_list); +} Index: src/mark_sweep/wspace_sweep_concurrent.cpp =================================================================== --- src/mark_sweep/wspace_sweep_concurrent.cpp (revision 0) +++ src/mark_sweep/wspace_sweep_concurrent.cpp (revision 0) @@ -0,0 +1,382 @@ +#include "wspace.h" +#include "wspace_chunk.h" +#include "wspace_mark_sweep.h" +#include "gc_ms.h" +#include "../gen/gen.h" + +static void collector_sweep_normal_chunk_concurrent(Collector *collector, Wspace *wspace, Chunk_Header *chunk) +{ + unsigned int slot_num = chunk->slot_num; + unsigned int live_num = 0; + unsigned int first_free_word_index = MAX_SLOT_INDEX; + POINTER_SIZE_INT *table = chunk->table; + + unsigned int index_word_num = (slot_num + SLOT_NUM_PER_WORD_IN_TABLE - 1) / SLOT_NUM_PER_WORD_IN_TABLE; + for(unsigned int i=0; ilive_obj_size += live_num * chunk->slot_size; + collector->live_obj_num += live_num; + + if(!live_num){ /* all objects in this chunk are dead */ + collector_add_free_chunk(collector, (Free_Chunk*)chunk); + } else if(!chunk_is_reusable(chunk)){ /* most objects in this chunk are swept, add chunk to pfc list*/ + chunk->alloc_num = live_num; + wspace_register_unreusable_normal_chunk(wspace, chunk); + } else { /* most objects in this chunk are swept, add chunk to pfc list*/ + chunk->alloc_num = live_num; + wspace_put_pfc_backup(wspace, chunk); + } +} + +static inline void collector_sweep_abnormal_chunk_concurrent(Collector *collector, Wspace *wspace, Chunk_Header *chunk) +{ + assert(chunk->status == (CHUNK_ABNORMAL | CHUNK_USED)); + POINTER_SIZE_INT *table = chunk->table; + table[0] &= cur_alloc_mask; + if(!table[0]){ + collector_add_free_chunk(collector, (Free_Chunk*)chunk); + } + else { + wspace_register_live_abnormal_chunk(wspace, chunk); + collector->live_obj_size += CHUNK_SIZE(chunk); + collector->live_obj_num++; + } +} + +static void wspace_sweep_chunk_concurrent(Wspace* wspace, Collector* collector, Chunk_Header_Basic* chunk) +{ + if(chunk->status & CHUNK_NORMAL){ /* chunk is used as a normal sized obj chunk */ + assert(chunk->status == (CHUNK_NORMAL | CHUNK_USED)); + collector_sweep_normal_chunk_concurrent(collector, wspace, (Chunk_Header*)chunk); + } else { /* chunk is used as a super obj chunk */ + assert(chunk->status == (CHUNK_ABNORMAL | CHUNK_USED)); + collector_sweep_abnormal_chunk_concurrent(collector, wspace, (Chunk_Header*)chunk); + } +} + +static Free_Chunk_List* wspace_get_free_chunk_list(Wspace* wspace) +{ + GC* gc = wspace->gc; + Free_Chunk_List* free_chunk_list = (Free_Chunk_List*) STD_MALLOC(sizeof(Free_Chunk_List)); + assert(free_chunk_list); + memset(free_chunk_list, 0, sizeof(Free_Chunk_List)); + + /* Collect free chunks from collectors to one list */ + for(unsigned int i=0; inum_collectors; ++i){ + Free_Chunk_List *list = gc->collectors[i]->free_chunk_list; + move_free_chunks_between_lists(free_chunk_list, list); + } + + return free_chunk_list; +} + +Boolean wspace_get_free_chunk_concurrent(Wspace *wspace, Free_Chunk* chunk) +{ + POINTER_SIZE_INT chunk_size = CHUNK_SIZE(chunk); + assert(!(chunk_size % CHUNK_GRANULARITY)); + + Free_Chunk_List* free_list = NULL; + + /*Find list*/ + if(chunk_size > HYPER_OBJ_THRESHOLD) + free_list = wspace->hyper_free_chunk_list; + else if(!((POINTER_SIZE_INT)chunk & NORMAL_CHUNK_LOW_MASK) && !(chunk_size & NORMAL_CHUNK_LOW_MASK)) + free_list = &wspace->aligned_free_chunk_lists[ALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)]; + else + free_list = &wspace->unaligned_free_chunk_lists[UNALIGNED_CHUNK_SIZE_TO_INDEX(chunk_size)]; + + /*Lock this free list*/ + lock(free_list->lock); + + /*Search free list for chunk*/ + Free_Chunk* chunk_iter = free_list->head; + while((POINTER_SIZE_INT)chunk_iter){ + if((POINTER_SIZE_INT)chunk_iter == (POINTER_SIZE_INT)chunk){ + /*Find chunk and delete from list.*/ + free_list_detach_chunk(free_list, chunk); + unlock(free_list->lock); + return TRUE; + } + chunk_iter = chunk_iter->next; + } + + unlock(free_list->lock); + + return FALSE; +} + +void wspace_merge_adj_free_chunks(Wspace* wspace,Free_Chunk* chunk) +{ + Free_Chunk *wspace_ceiling = (Free_Chunk*)space_heap_end((Space*)wspace); + + /* Check if the back adjcent chunks are free */ + Free_Chunk *back_chunk = (Free_Chunk*)chunk->adj_next; + while(back_chunk < wspace_ceiling && (back_chunk->status & CHUNK_FREE)){ + assert(chunk < back_chunk); + /* Remove back_chunk from list */ + if(wspace_get_free_chunk_concurrent(wspace,back_chunk)){ + back_chunk = (Free_Chunk*)back_chunk->adj_next; + chunk->adj_next = (Chunk_Header_Basic*)back_chunk; + }else{ + break; + } + } + + chunk->status = CHUNK_FREE | CHUNK_MERGED; + /* put the free chunk to the according free chunk list */ + wspace_put_free_chunk_to_tail(wspace, chunk); + +} + +static void wspace_merge_list_concurrent(Wspace* wspace, Free_Chunk_List* free_list) +{ + lock(free_list->lock); + Free_Chunk* chunk = free_list->head; + + while(chunk && !is_free_chunk_merged(chunk)){ + free_list_detach_chunk(free_list, chunk); + unlock(free_list->lock); + + wspace_merge_adj_free_chunks(wspace, chunk); + + lock(free_list->lock); + chunk = free_list->head; + } + + unlock(free_list->lock); +} + +static void wspace_merge_free_chunks_concurrent(Wspace* wspace, Free_Chunk_List* free_list) +{ + Free_Chunk *chunk = free_list->head; + + /*merge free list*/ + wspace_merge_list_concurrent(wspace, free_list); + + /*check free pool*/ + unsigned int i; + + for(i = NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) + wspace_merge_list_concurrent(wspace, &wspace->aligned_free_chunk_lists[i]); + + for(i = NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) + wspace_merge_list_concurrent(wspace, &wspace->unaligned_free_chunk_lists[i]); + + wspace_merge_list_concurrent(wspace, wspace->hyper_free_chunk_list); +} + +static void wspace_reset_free_list_chunks(Wspace* wspace, Free_Chunk_List* free_list) +{ + lock(free_list->lock); + Free_Chunk* chunk = free_list->head; + + while(chunk ){ + assert(chunk->status & CHUNK_FREE); + chunk->status = CHUNK_FREE; + chunk = chunk->next; + } + + unlock(free_list->lock); +} + + +static void wspace_reset_free_chunks_status(Wspace* wspace) +{ + unsigned int i; + + for(i = NUM_ALIGNED_FREE_CHUNK_BUCKET; i--;) + wspace_reset_free_list_chunks(wspace, &wspace->aligned_free_chunk_lists[i]); + + for(i = NUM_UNALIGNED_FREE_CHUNK_BUCKET; i--;) + wspace_reset_free_list_chunks(wspace, &wspace->unaligned_free_chunk_lists[i]); + + wspace_reset_free_list_chunks(wspace, wspace->hyper_free_chunk_list); + +} + +static void allocator_sweep_local_chunks(Allocator *allocator) +{ + Wspace *wspace = gc_get_wspace(allocator->gc); + Size_Segment **size_segs = wspace->size_segments; + Chunk_Header ***local_chunks = allocator->local_chunks; + + for(unsigned int i = SIZE_SEGMENT_NUM; i--;){ + if(!size_segs[i]->local_alloc){ + assert(!local_chunks[i]); + continue; + } + Chunk_Header **chunks = local_chunks[i]; + assert(chunks); + for(unsigned int j = size_segs[i]->chunk_num; j--;){ + if(chunks[j]){ + unsigned int slot_num = chunks[j]->slot_num; + POINTER_SIZE_INT *table = chunks[j]->table; + + unsigned int index_word_num = (slot_num + SLOT_NUM_PER_WORD_IN_TABLE - 1) / SLOT_NUM_PER_WORD_IN_TABLE; + for(unsigned int i=0; imutator_list_lock); // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + + /* release local chunks of each mutator in unique mark-sweep GC */ + Mutator *mutator = gc->mutator_list; + while(mutator){ + wait_mutator_signal(mutator, DISABLE_COLLECTOR_SWEEP_LOCAL_CHUNKS); + allocator_sweep_local_chunks((Allocator*)mutator); + mutator = mutator->next; + } + + unlock(gc->mutator_list_lock); +#endif +} + +static void gc_check_mutator_local_chunks(GC *gc, unsigned int handshake_signal) +{ + lock(gc->mutator_list_lock); // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + + /* release local chunks of each mutator in unique mark-sweep GC */ + Mutator *mutator = gc->mutator_list; + while(mutator){ + wait_mutator_signal(mutator, handshake_signal); + mutator = mutator->next; + } + + unlock(gc->mutator_list_lock); +} + + +static volatile unsigned int num_sweeping_collectors = 0; + +/*Concurrent Sweep: + The mark bit and alloc bit is exchanged before entering this function. + This function is to clear the mark bit and merge the free chunks concurrently. + */ +void wspace_sweep_concurrent(Collector* collector) +{ + GC *gc = collector->gc; + Wspace *wspace = gc_get_wspace(gc); + + unsigned int num_active_collectors = gc->num_active_collectors; + + atomic_cas32(&num_sweeping_collectors, 0, num_active_collectors+1); + + Pool* used_chunk_pool = wspace->used_chunk_pool; + + Chunk_Header_Basic* chunk_to_sweep; + + /*1. Grab chunks from used list, sweep the chunk and push back to PFC backup list & free list.*/ + chunk_to_sweep = chunk_pool_get_chunk(used_chunk_pool); + while(chunk_to_sweep != NULL){ + wspace_sweep_chunk_concurrent(wspace, collector, chunk_to_sweep); + chunk_to_sweep = chunk_pool_get_chunk(used_chunk_pool); + } + + /*2. Grab chunks from PFC list, sweep the chunk and push back to PFC backup list & free list.*/ + Pool* pfc_pool = wspace_grab_next_pfc_pool(wspace); + while(pfc_pool != NULL){ + if(!pool_is_empty(pfc_pool)){ + /*sweep the chunks in pfc_pool. push back to pfc backup list*/ + chunk_to_sweep = chunk_pool_get_chunk(pfc_pool); + while(chunk_to_sweep != NULL){ + assert(chunk_to_sweep->status == (CHUNK_NORMAL | CHUNK_NEED_ZEROING)); + chunk_to_sweep->status = CHUNK_NORMAL | CHUNK_USED; + wspace_sweep_chunk_concurrent(wspace, collector, chunk_to_sweep); + chunk_to_sweep = chunk_pool_get_chunk(pfc_pool); + } + } + /*grab more pfc pools*/ + pfc_pool = wspace_grab_next_pfc_pool(wspace); + } + + unsigned int old_num = atomic_inc32(&num_sweeping_collectors); + if( ++old_num == num_active_collectors ){ + + /*3. Check the local chunk of mutator*/ + gc_sweep_mutator_local_chunks(wspace->gc); + + /*4. Sweep gloabl alloc normal chunks again*/ + gc_set_sweeping_global_normal_chunk(); + gc_check_mutator_local_chunks(wspace->gc, DISABLE_COLLECTOR_SWEEP_GLOBAL_CHUNKS); + wspace_init_pfc_pool_iterator(wspace); + Pool* pfc_pool = wspace_grab_next_pfc_pool(wspace); + while(pfc_pool != NULL){ + if(!pool_is_empty(pfc_pool)){ + chunk_to_sweep = chunk_pool_get_chunk(pfc_pool); + while(chunk_to_sweep != NULL){ + assert(chunk_to_sweep->status == (CHUNK_NORMAL | CHUNK_NEED_ZEROING)); + chunk_to_sweep->status = CHUNK_NORMAL | CHUNK_USED; + wspace_sweep_chunk_concurrent(wspace, collector, chunk_to_sweep); + chunk_to_sweep = chunk_pool_get_chunk(pfc_pool); + } + } + /*grab more pfc pools*/ + pfc_pool = wspace_grab_next_pfc_pool(wspace); + } + gc_unset_sweeping_global_normal_chunk(); + + /*4. Check the used list again.*/ + chunk_to_sweep = chunk_pool_get_chunk(used_chunk_pool); + while(chunk_to_sweep != NULL){ + wspace_sweep_chunk_concurrent(wspace, collector, chunk_to_sweep); + chunk_to_sweep = chunk_pool_get_chunk(used_chunk_pool); + } + + /*5. Switch the PFC backup list to PFC list.*/ + wspace_exchange_pfc_pool(wspace); + + /*6. Put back live abnormal chunk and normal unreusable chunk*/ + Chunk_Header* used_abnormal_chunk = wspace_get_live_abnormal_chunk(wspace); + while(used_abnormal_chunk){ + used_abnormal_chunk->status = CHUNK_USED | CHUNK_ABNORMAL; + wspace_register_used_chunk(wspace,used_abnormal_chunk); + used_abnormal_chunk = wspace_get_live_abnormal_chunk(wspace); + } + pool_empty(wspace->live_abnormal_chunk_pool); + + Chunk_Header* unreusable_normal_chunk = wspace_get_unreusable_normal_chunk(wspace); + while(unreusable_normal_chunk){ + unreusable_normal_chunk->status = CHUNK_USED | CHUNK_NORMAL; + wspace_register_used_chunk(wspace,unreusable_normal_chunk); + unreusable_normal_chunk = wspace_get_unreusable_normal_chunk(wspace); + } + pool_empty(wspace->unreusable_normal_chunk_pool); + + + /*7. Merge free chunks*/ + Free_Chunk_List* free_chunk_list = wspace_get_free_chunk_list(wspace); + wspace_merge_free_chunks_concurrent(wspace, free_chunk_list); + wspace_reset_free_chunks_status(wspace); + + /* let other collectors go */ + num_sweeping_collectors++; + } + while(num_sweeping_collectors != num_active_collectors + 1); +} + Index: src/mark_sweep/wspace_verify.cpp =================================================================== --- src/mark_sweep/wspace_verify.cpp (revision 0) +++ src/mark_sweep/wspace_verify.cpp (revision 0) @@ -0,0 +1,676 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wspace_verify.h" +#include "wspace_chunk.h" +#include "wspace_mark_sweep.h" +#include "../utils/vector_block.h" +#include "gc_ms.h" +#include "../gen/gen.h" +#include "../finalizer_weakref/finalizer_weakref.h" + +#ifdef SSPACE_VERIFY + +#define VERIFY_CARD_SIZE_BYTES_SHIFT 12 +#define VERIFY_CARD_SIZE_BYTES (1 << VERIFY_CARD_SIZE_BYTES_SHIFT) +#define VERIFY_CARD_LOW_MASK (VERIFY_CARD_SIZE_BYTES - 1) +#define VERIFY_CARD_HIGH_MASK (~VERIFY_CARD_LOW_MASK) + +#define VERIFY_MAX_OBJ_SIZE_BYTES (1 << (32-VERIFY_CARD_SIZE_BYTES_SHIFT)) + +typedef struct Verify_Card { + SpinLock lock; + Vector_Block *block; +} Verify_Card; + +typedef unsigned int Obj_Addr; + +static GC *gc_in_verify = NULL; +static Verify_Card *alloc_verify_cards = NULL; +static Verify_Card *mark_verify_cards = NULL; +static POINTER_SIZE_INT card_num = 0; +static volatile POINTER_SIZE_INT alloc_obj_num = 0; +static volatile POINTER_SIZE_INT live_obj_in_mark = 0; +static volatile POINTER_SIZE_INT live_obj_in_fix = 0; + +void wspace_verify_init(GC *gc) +{ + gc_in_verify = gc; + + Wspace *wspace = gc_get_wspace(gc); + POINTER_SIZE_INT space_size = space_committed_size((Space*)wspace); + card_num = space_size >> VERIFY_CARD_SIZE_BYTES_SHIFT; + POINTER_SIZE_INT cards_size = sizeof(Verify_Card) * card_num; + + alloc_verify_cards = (Verify_Card*)STD_MALLOC(cards_size); + memset(alloc_verify_cards, 0, cards_size); + + mark_verify_cards = (Verify_Card*)STD_MALLOC(cards_size); + memset(mark_verify_cards, 0, cards_size); +} + +static Obj_Addr compose_obj_addr(unsigned int offset, unsigned int size) +{ + assert(size < VERIFY_MAX_OBJ_SIZE_BYTES); + return offset | (size << VERIFY_CARD_SIZE_BYTES_SHIFT); +} + +static void *decompose_obj_addr(Obj_Addr obj_addr, POINTER_SIZE_INT card_index, unsigned int & size) +{ + assert(card_index < card_num); + POINTER_SIZE_INT card_offset = obj_addr & VERIFY_CARD_LOW_MASK; + POINTER_SIZE_INT heap_offset = VERIFY_CARD_SIZE_BYTES * card_index + card_offset; + size = (obj_addr & VERIFY_CARD_HIGH_MASK) >> VERIFY_CARD_SIZE_BYTES_SHIFT; + assert(size < VERIFY_MAX_OBJ_SIZE_BYTES); + return (void*)(heap_offset + (POINTER_SIZE_INT)gc_heap_base(gc_in_verify)); +} + +static Boolean obj_addr_overlapped(Obj_Addr addr1, Obj_Addr addr2) +{ + unsigned int offset1 = addr1 & VERIFY_CARD_LOW_MASK; + unsigned int size1 = (addr1 & VERIFY_CARD_HIGH_MASK) >> VERIFY_CARD_SIZE_BYTES_SHIFT; + unsigned int ceiling1 = offset1 + size1; + unsigned int offset2 = addr2 & VERIFY_CARD_LOW_MASK; + unsigned int size2 = (addr2 & VERIFY_CARD_HIGH_MASK) >> VERIFY_CARD_SIZE_BYTES_SHIFT; + unsigned int ceiling2 = offset2 + size2; + + unsigned int reason = 0; + if(offset1 == offset2) + reason = 1; + if((offset1 < offset2) && (ceiling1 > offset2)) + reason = 2; + if((offset2 < offset1) && (ceiling2 > offset1)) + reason = 3; + if(!reason) + return FALSE; + printf("\nreason: %d\nold offset: %x size: %d\nnew offset: %x size: %d", reason, (POINTER_SIZE_INT)offset1, size1, (POINTER_SIZE_INT)offset2, size2); + return TRUE; +} + +static Vector_Block *create_vector_block(unsigned int size) +{ + Vector_Block *block = (Vector_Block*)STD_MALLOC(size); + vector_block_init(block, size); + return block; +} + +static void verify_card_get_block(Verify_Card *card) +{ + lock(card->lock); + if(card->block){ + unlock(card->lock); + return; + } + card->block = create_vector_block(VECTOR_BLOCK_DATA_SIZE_BYTES); + unlock(card->lock); +} + +void wspace_verify_alloc(void *addr, unsigned int size) +{ + assert(address_belongs_to_gc_heap(addr, gc_in_verify)); + atomic_inc32(&alloc_obj_num); + + unsigned int heap_offset = (unsigned int)((POINTER_SIZE_INT)addr - (POINTER_SIZE_INT)gc_heap_base(gc_in_verify)); + unsigned int card_offset = heap_offset & VERIFY_CARD_LOW_MASK; + Verify_Card *card = &alloc_verify_cards[heap_offset >> VERIFY_CARD_SIZE_BYTES_SHIFT]; + + verify_card_get_block(card); + Vector_Block *block = card->block; + + Obj_Addr obj_addr = compose_obj_addr(card_offset, size); + + lock(card->lock); + Obj_Addr *p_addr = block->head; + while(p_addr < block->tail){ + assert(!obj_addr_overlapped(obj_addr, *p_addr)); + p_addr++; + } + vector_block_add_entry(block, obj_addr); + unlock(card->lock); +} + +/* size is rounded up size */ +static Boolean obj_position_is_correct(void *addr, unsigned int size) +{ + Chunk_Header *chunk = NULL; + + if(size <= SUPER_OBJ_THRESHOLD) + chunk = NORMAL_CHUNK_HEADER(addr); + else + chunk = ABNORMAL_CHUNK_HEADER(addr); + if(chunk->slot_size != size) return FALSE; + if(((POINTER_SIZE_INT)addr - (POINTER_SIZE_INT)chunk->base) % size != 0) return FALSE; + return TRUE; +} + +static void wspace_verify_weakref(Partial_Reveal_Object *p_obj) +{ + WeakReferenceType type = special_reference_type(p_obj); + if(type == NOT_REFERENCE) return; + + REF *p_referent_field = obj_get_referent_field(p_obj); + Partial_Reveal_Object *p_referent = read_slot(p_referent_field); + if (!p_referent) return; + + unsigned int size = vm_object_size(p_referent); + if(size <= SUPER_OBJ_THRESHOLD){ + Wspace *wspace = gc_get_wspace(gc_in_verify); + Size_Segment *size_seg = wspace_get_size_seg(wspace, size); + size = NORMAL_SIZE_ROUNDUP(size, size_seg); + } + + assert(obj_position_is_correct(p_referent, size)); +} + +static void mark_card_add_entry(void *addr, unsigned int size) +{ + assert(address_belongs_to_gc_heap(addr, gc_in_verify)); + + unsigned int heap_offset = (unsigned int)((POINTER_SIZE_INT)addr - (POINTER_SIZE_INT)gc_heap_base(gc_in_verify)); + unsigned int card_offset = heap_offset & VERIFY_CARD_LOW_MASK; + Verify_Card *card = &mark_verify_cards[heap_offset >> VERIFY_CARD_SIZE_BYTES_SHIFT]; + + verify_card_get_block(card); + Vector_Block *block = card->block; + + if(size <= SUPER_OBJ_THRESHOLD){ + Wspace *wspace = gc_get_wspace(gc_in_verify); + Size_Segment *size_seg = wspace_get_size_seg(wspace, size); + size = NORMAL_SIZE_ROUNDUP(size, size_seg); + } + + assert(obj_position_is_correct(addr, size)); + Obj_Addr obj_addr = compose_obj_addr(card_offset, size); + + lock(card->lock); + Obj_Addr *p_addr = block->head; + while(p_addr < block->tail){ + assert(!obj_addr_overlapped(obj_addr, *p_addr)); + p_addr++; + } + vector_block_add_entry(block, obj_addr); + unlock(card->lock); + +} + +/* size is real size of obj */ +void wspace_record_mark(void *addr, unsigned int size) +{ + atomic_inc32(&live_obj_in_mark); + mark_card_add_entry(addr, size); +} + +static void verify_mark(void *addr, unsigned int size, Boolean destructively) +{ + assert(address_belongs_to_gc_heap(addr, gc_in_verify)); + + unsigned int heap_offset = (unsigned int)((POINTER_SIZE_INT)addr - (POINTER_SIZE_INT)gc_heap_base(gc_in_verify)); + unsigned int card_offset = heap_offset & VERIFY_CARD_LOW_MASK; + Verify_Card *card = &mark_verify_cards[heap_offset >> VERIFY_CARD_SIZE_BYTES_SHIFT]; + + Vector_Block *block = card->block; + assert(block); + + Obj_Addr obj_addr = compose_obj_addr(card_offset, size); + + Obj_Addr *p_addr = block->head; + while(p_addr < block->tail){ + if(obj_addr == *p_addr){ + if(destructively) + *p_addr = 0; + break; + } + p_addr++; + } + assert(p_addr < block->tail); +} + +void wspace_modify_mark_in_compact(void *new_addr, void *old_addr, unsigned int size) +{ + /* Verify the old addr and remove it in the according mark card */ + verify_mark(old_addr, size, TRUE); + /* Add new_addr into mark card */ + mark_card_add_entry(new_addr, size); +} + +void wspace_verify_fix_in_compact(void) +{ + atomic_inc32(&live_obj_in_fix); +} + +static void check_and_clear_mark_cards(void) +{ + for(POINTER_SIZE_INT i=0; ihead; + while(p_addr < block->tail){ + if(*p_addr){ + unsigned int size = 0; + void *addr = NULL; + addr = decompose_obj_addr(*p_addr, i, size); + printf("Extra mark obj: %x size: %d\n", (POINTER_SIZE_INT)addr, size); + } + p_addr++; + } + vector_block_clear(block); + } +} + +static void clear_alloc_cards(void) +{ + for(POINTER_SIZE_INT i=0; iblock) + vector_block_clear(card->block); + } +} + +static void summarize_sweep_verify(GC *gc) +{ + POINTER_SIZE_INT live_obj_num = 0; + for(unsigned int i=0; inum_collectors; ++i){ + live_obj_num += gc->collectors[i]->live_obj_num; + } + printf("Live obj in sweeping: %d\n", live_obj_num); +} + +void wspace_verify_free_area(POINTER_SIZE_INT *start, POINTER_SIZE_INT size) +{ + POINTER_SIZE_INT *p_value = start; + + assert(!(size % BYTES_PER_WORD)); + size /= BYTES_PER_WORD; + while(size--) + assert(!*p_value++); +} + +static POINTER_SIZE_INT wspace_live_obj_num(Wspace *wspace, Boolean gc_finished) +{ + Chunk_Header *chunk = (Chunk_Header*)space_heap_start((Space*)wspace); + Chunk_Header *wspace_ceiling = (Chunk_Header*)space_heap_end((Space*)wspace); + POINTER_SIZE_INT live_num = 0; + + for(; chunk < wspace_ceiling; chunk = (Chunk_Header*)CHUNK_END(chunk)){ + /* chunk is free before GC */ + if(chunk->status & CHUNK_FREE){ + assert((gc_finished && chunk->status==CHUNK_FREE) + || (!gc_finished && chunk->status==(CHUNK_FREE|CHUNK_TO_MERGE))); + continue; + } + if(chunk->status & CHUNK_ABNORMAL){ + assert(chunk->status == CHUNK_ABNORMAL); + assert(chunk->slot_size > SUPER_OBJ_THRESHOLD); + Partial_Reveal_Object *obj = (Partial_Reveal_Object*)chunk->base; + assert(chunk->slot_size == vm_object_size(obj)); + assert(get_obj_info_raw(obj) & SUPER_OBJ_MASK); + } + /* chunk is used as a normal or abnormal one in which there are live objects */ + unsigned int slot_num = chunk->slot_num; + POINTER_SIZE_INT *table = chunk->table; + POINTER_SIZE_INT live_num_in_chunk = 0; + + unsigned int word_index = 0; + for(unsigned int i=0; islot_size, gc_finished); + if(gc_finished){ + wspace_verify_alloc(p_obj, chunk->slot_size); + wspace_verify_weakref((Partial_Reveal_Object*)p_obj); + } + } + } + live_num += live_num_in_chunk; + } + + return live_num; +} + +static void allocator_verify_local_chunks(Allocator *allocator) +{ + Wspace *wspace = gc_get_wspace(allocator->gc); + Size_Segment **size_segs = wspace->size_segments; + Chunk_Header ***local_chunks = allocator->local_chunks; + + for(unsigned int i = SIZE_SEGMENT_NUM; i--;){ + if(!size_segs[i]->local_alloc){ + assert(!local_chunks[i]); + continue; + } + Chunk_Header **chunks = local_chunks[i]; + assert(chunks); + for(unsigned int j = size_segs[i]->chunk_num; j--;){ + assert(!chunks[j]); + } + } +} + +static void gc_verify_allocator_local_chunks(GC *gc) +{ + if(gc_match_kind(gc, MARK_SWEEP_GC)){ + Mutator *mutator = gc->mutator_list; + while(mutator){ + allocator_verify_local_chunks((Allocator*)mutator); + mutator = mutator->next; + } + } + + if(gc_match_kind(gc, MAJOR_COLLECTION)) + for(unsigned int i = gc->num_collectors; i--;){ + allocator_verify_local_chunks((Allocator*)gc->collectors[i]); + } +} + +void wspace_verify_before_collection(GC *gc) +{ + printf("Allocated obj: %d\n", alloc_obj_num); + alloc_obj_num = 0; +} + +void wspace_verify_after_sweep(GC *gc) +{ + printf("Live obj in marking: %d\n", live_obj_in_mark); + live_obj_in_mark = 0; + + summarize_sweep_verify(gc); + + Wspace *wspace = gc_get_wspace(gc); + POINTER_SIZE_INT total_live_obj = wspace_live_obj_num(wspace, FALSE); + printf("Live obj after sweep: %d\n", total_live_obj); +} + +void wspace_verify_after_collection(GC *gc) +{ + printf("Live obj in fixing: %d\n", live_obj_in_fix); + live_obj_in_fix = 0; + + clear_alloc_cards(); + + Wspace *wspace = gc_get_wspace(gc); + POINTER_SIZE_INT total_live_obj = wspace_live_obj_num(wspace, TRUE); + printf("Live obj after collection: %d\n", total_live_obj); + check_and_clear_mark_cards(); + gc_verify_allocator_local_chunks(gc); +} + +/* +void wspace_verify_super_obj(GC *gc) +{ + Wspace *wspace = gc_get_wspace(gc); + Chunk_Header *chunk = (Chunk_Header*)space_heap_start((Space*)wspace); + Chunk_Header *wspace_ceiling = (Chunk_Header*)space_heap_end((Space*)wspace); + + for(; chunk < wspace_ceiling; chunk = (Chunk_Header*)CHUNK_END(chunk)){ + if(chunk->status & CHUNK_ABNORMAL){ + assert(chunk->status == CHUNK_ABNORMAL); + assert(chunk->slot_size > SUPER_OBJ_THRESHOLD); + Partial_Reveal_Object *obj = (Partial_Reveal_Object*)chunk->base; + assert(chunk->slot_size == vm_object_size(obj)); + assert(get_obj_info_raw(obj) & SUPER_OBJ_MASK); + } + } +} +*/ + + +/* wspace verify marking with vtable marking in advance */ + +Wspace *wspace_in_verifier; +static Pool *trace_pool = NULL; +static Vector_Block *trace_stack = NULL; +POINTER_SIZE_INT live_obj_in_verify_marking = 0; + +static Boolean obj_mark_in_vtable(GC *gc, Partial_Reveal_Object *obj) +{ + assert(address_belongs_to_gc_heap(obj, gc)); + assert((vm_object_size(obj) <= SUPER_OBJ_THRESHOLD) || (get_obj_info_raw(obj) & SUPER_OBJ_MASK)); + Boolean marked = obj_mark_in_vt(obj); +#ifdef SSPACE_VERIFY + if(marked) live_obj_in_verify_marking++; +#endif + return marked; +} + +static void tracestack_push(void *p_obj) +{ + vector_stack_push(trace_stack, (POINTER_SIZE_INT)p_obj); + + if( !vector_stack_is_full(trace_stack)) return; + + pool_put_entry(trace_pool, trace_stack); + trace_stack = free_task_pool_get_entry(&gc_metadata); + assert(trace_stack); +} + +static FORCE_INLINE void scan_slot(GC *gc, REF *p_ref) +{ + Partial_Reveal_Object *p_obj = read_slot(p_ref); + if( p_obj == NULL) return; + + if(obj_belongs_to_space(p_obj, (Space*)wspace_in_verifier) && obj_mark_in_vtable(gc, p_obj)) + tracestack_push(p_obj); + + return; +} + +static FORCE_INLINE void scan_object(GC *gc, Partial_Reveal_Object *p_obj) +{ + if(!object_has_ref_field(p_obj) ) return; + + REF *p_ref; + + if (object_is_array(p_obj)) { /* scan array object */ + + Partial_Reveal_Array *array = (Partial_Reveal_Array*)p_obj; + unsigned int array_length = array->array_len; + + p_ref = (REF*)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array)); + + for (unsigned int i = 0; i < array_length; i++) { + scan_slot(gc, p_ref+i); + } + } else { /* scan non-array object */ + + unsigned int num_refs = object_ref_field_num(p_obj); + + int *ref_iterator = object_ref_iterator_init(p_obj); + + for(unsigned int i=0; imetadata; + Pool *rootset_pool = metadata->gc_rootset_pool; + + trace_stack = free_task_pool_get_entry(metadata); + trace_pool = sync_pool_create(); + + pool_iterator_init(rootset_pool); + Vector_Block *root_set = pool_iterator_next(rootset_pool); + + while(root_set){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set); + while(!vector_block_iterator_end(root_set, iter)){ + REF *p_ref = (REF*)*iter; + iter = vector_block_iterator_advance(root_set, iter); + + Partial_Reveal_Object *p_obj = read_slot(p_ref); + assert(p_obj!=NULL); + if(obj_belongs_to_space(p_obj, (Space*)wspace_in_verifier) && obj_mark_in_vtable(gc, p_obj)) + tracestack_push(p_obj); + } + root_set = pool_iterator_next(metadata->gc_rootset_pool); + } + /* put back the last trace_stack task */ + pool_put_entry(trace_pool, trace_stack); + + /* second step: iterate over the mark tasks and scan objects */ + /* get a task buf for the mark stack */ + trace_stack = free_task_pool_get_entry(metadata); + + Vector_Block *mark_task = pool_get_entry(trace_pool); + + while(mark_task){ + POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task); + while(!vector_block_iterator_end(mark_task, iter)){ + Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)*iter; + iter = vector_block_iterator_advance(mark_task, iter); + + trace_object(gc, p_obj); + } + /* run out one task, put back to the pool and grab another task */ + vector_stack_clear(mark_task); + pool_put_entry(metadata->free_task_pool, mark_task); + mark_task = pool_get_entry(trace_pool); + } + + /* put back the last mark stack to the free pool */ + vector_stack_clear(trace_stack); + pool_put_entry(metadata->free_task_pool, trace_stack); + trace_stack = NULL; + sync_pool_destruct(trace_pool); + trace_pool = NULL; + printf("Live obj in vtable marking: %d\n", live_obj_in_verify_marking); + live_obj_in_verify_marking = 0; +} + + +#endif + + + +#ifdef SSPACE_TIME + +inline uint64 tsc() +{ + __asm _emit 0x0F; + __asm _emit 0x31 +} + +#define CPU_HZ 3000000 // per ms + +static uint64 gc_start_time; +static uint64 mark_start_time; +static uint64 sweep_start_time; +static uint64 compact_start_time; +static uint64 fix_start_time; +static uint64 merge_start_time; + +void wspace_gc_time(GC *gc, Boolean before_gc) +{ + if(before_gc){ + gc_start_time = tsc(); + mark_start_time = gc_start_time; + } else { + uint64 end_time = tsc(); + assert(end_time > gc_start_time); + printf("\n\nGC %d time: %dms\n\n", gc->num_collections, (end_time-gc_start_time) / CPU_HZ); + } +} + +void wspace_mark_time(Boolean before_mark) +{ + assert(before_mark == FALSE); + if(before_mark){ + mark_start_time = tsc(); + } else { + uint64 end_time = tsc(); + assert(end_time > mark_start_time); + printf("\nMark time: %dms\n", (end_time-mark_start_time) / CPU_HZ); + sweep_start_time = end_time; + } +} + +void wspace_sweep_time(Boolean before_sweep, Boolean wspace_need_compact) +{ + assert(before_sweep == FALSE); + if(before_sweep){ + sweep_start_time = tsc(); + } else { + uint64 end_time = tsc(); + assert(end_time > sweep_start_time); + printf("\nSweep time: %dms\n", (end_time-sweep_start_time) / CPU_HZ); + if(wspace_need_compact) + compact_start_time = end_time; + else + merge_start_time = end_time; + } +} + +void wspace_compact_time(Boolean before_compact) +{ + assert(before_compact == FALSE); + if(before_compact){ + compact_start_time = tsc(); + } else { + uint64 end_time = tsc(); + assert(end_time > compact_start_time); + printf("\nCompact time: %dms\n", (end_time-compact_start_time) / CPU_HZ); + fix_start_time = end_time; + } +} + +void wspace_fix_time(Boolean before_fix) +{ + assert(before_fix == FALSE); + if(before_fix){ + fix_start_time = tsc(); + } else { + uint64 end_time = tsc(); + assert(end_time > fix_start_time); + printf("\nFix time: %dms\n", (end_time-fix_start_time) / CPU_HZ); + merge_start_time = end_time; + } +} + +void wspace_merge_time(Boolean before_merge) +{ + assert(before_merge == FALSE); + if(before_merge){ + merge_start_time = tsc(); + } else { + uint64 end_time = tsc(); + assert(end_time > merge_start_time); + printf("\nMerge time: %dms\n\n", (end_time-merge_start_time) / CPU_HZ); + } +} + +#endif Index: src/mark_sweep/wspace_verify.h =================================================================== --- src/mark_sweep/wspace_verify.h (revision 0) +++ src/mark_sweep/wspace_verify.h (revision 0) @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _SSPACE_VERIFY_H_ +#define _SSPACE_VERIFY_H_ + +#include "../common/gc_common.h" + +//#define SSPACE_VERIFY +//#define SSPACE_VERIFY_FINREF +//#define SSPACE_CHUNK_INFO +//#define SSPACE_ALLOC_INFO +//#define SSPACE_TIME + +struct Wspace; + +void wspace_verify_init(GC *gc); +void wspace_verify_alloc(void *addr, unsigned int size); +void wspace_verify_vtable_mark(GC *gc); +void wspace_record_mark(void *addr, unsigned int size); +void wspace_modify_mark_in_compact(void *new_addr, void *old_addr, unsigned int size); +void wspace_verify_fix_in_compact(void); +void wspace_verify_free_area(POINTER_SIZE_INT *start, POINTER_SIZE_INT size); +void wspace_verify_before_collection(GC *gc); +void wspace_verify_after_sweep(GC *gc); +void wspace_verify_after_collection(GC *gc); + +void wspace_chunks_info(Wspace *wspace, Boolean show_info); +void wspace_alloc_info(unsigned int size); +void wspace_alloc_info_summary(void); + +void wspace_gc_time(GC *gc, Boolean before_gc); +void wspace_mark_time(Boolean before_mark); +void wspace_sweep_time(Boolean before_sweep, Boolean wspace_need_compact); +void wspace_compact_time(Boolean before_compact); +void wspace_fix_time(Boolean before_fix); +void wspace_merge_time(Boolean before_merge); + +#endif // _SSPACE_VERIFY_H_ Index: src/semi_space/sspace.h =================================================================== --- src/semi_space/sspace.h (revision 0) +++ src/semi_space/sspace.h (revision 0) @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _SEMI_SPACE_H_ +#define _SEMI_SPACE_H_ + +#include "../thread/gc_thread.h" + +typedef struct Sspace{ + /* <-- first couple of fields are overloadded as Space */ + void* heap_start; + void* heap_end; + POINTER_SIZE_INT reserved_heap_size; + POINTER_SIZE_INT committed_heap_size; + unsigned int num_collections; + int64 time_collections; + float survive_ratio; + unsigned int collect_algorithm; + GC* gc; + Boolean move_object; + + Space_Statistics* space_statistic; + + /* Size allocted since last minor collection. */ + volatile uint64 last_alloced_size; + /* Size allocted since last major collection. */ + uint64 accumu_alloced_size; + /* Total size allocated since VM starts. */ + uint64 total_alloced_size; + + /* Size survived from last collection. */ + uint64 last_surviving_size; + /* Size survived after a certain period. */ + uint64 period_surviving_size; + + /* END of Space --> */ + + Block* blocks; /* short-cut for mpsace blockheader access, not mandatory */ + + /* FIXME:: the block indices should be replaced with block header addresses */ + unsigned int first_block_idx; /* always pointing to sspace bottom */ + unsigned int ceiling_block_idx; /* tospace ceiling */ + volatile unsigned int free_block_idx; /* tospace cur free block */ + + unsigned int num_used_blocks; + unsigned int num_managed_blocks; + unsigned int num_total_blocks; + + volatile Block_Header* block_iterator; + /* END of Blocked_Space --> */ + + Block_Header* cur_free_block; + unsigned int tospace_first_idx; + void* survivor_area_top; + void* survivor_area_bottom; + +}Sspace; + +Sspace *sspace_initialize(GC* gc, void* start, POINTER_SIZE_INT sspace_size, POINTER_SIZE_INT commit_size); +void sspace_destruct(Sspace *sspace); + +void* sspace_alloc(unsigned size, Allocator *allocator); +Boolean sspace_alloc_block(Sspace* sspace, Allocator* allocator); + +void sspace_collection(Sspace* sspace); +void sspace_prepare_for_collection(Sspace* sspace); +void sspace_reset_after_collection(Sspace* sspace); + +void* semispace_alloc(unsigned int size, Allocator* allocator); + +void nongen_ss_pool(Collector* collector); +void gen_ss_pool(Collector* collector); + +FORCE_INLINE Boolean sspace_has_free_block(Sspace* sspace) +{ + return (sspace->cur_free_block != NULL); +} + +FORCE_INLINE Boolean obj_belongs_to_survivor_area(Sspace* sspace, Partial_Reveal_Object* p_obj) +{ + return (p_obj >= sspace->survivor_area_bottom && + p_obj < sspace->survivor_area_top); +} + +/* treat semispace alloc as thread local alloc. If it fails or p_obj is old, forward it to MOS */ +FORCE_INLINE void* semispace_forward_obj(Partial_Reveal_Object* p_obj, unsigned int size, Allocator* allocator) +{ + void* p_targ_obj = NULL; + Sspace* sspace = (Sspace*)allocator->alloc_space; + + if( !obj_belongs_to_survivor_area(sspace, p_obj) ) + p_targ_obj = semispace_alloc(size, allocator); + + return p_targ_obj; +} + +#endif // _FROM_SPACE_H_ Index: src/thread/collector.cpp =================================================================== --- src/thread/collector.cpp (revision 606795) +++ src/thread/collector.cpp (working copy) @@ -25,7 +25,7 @@ #include "../mark_compact/mspace.h" #include "../finalizer_weakref/finalizer_weakref.h" #include "../common/space_tuner.h" -#include "../mark_sweep/sspace.h" +#include "../mark_sweep/wspace.h" unsigned int MINOR_COLLECTORS = 0; unsigned int MAJOR_COLLECTORS = 0; @@ -84,14 +84,6 @@ */ GC_Metadata* metadata = collector->gc->metadata; - -/* TO_REMOVE - - assert(collector->rep_set==NULL); - if( !gc_is_gen_mode() || !gc_match_kind(collector->gc, MINOR_COLLECTION)){ - collector->rep_set = free_set_pool_get_entry(metadata); - } -*/ if(gc_is_gen_mode() && gc_match_kind(collector->gc, MINOR_COLLECTION) && NOS_PARTIAL_FORWARD){ assert(collector->rem_set==NULL); @@ -102,7 +94,7 @@ collector_reset_weakref_sets(collector); #endif -#ifndef USE_MARK_SWEEP_GC +#if !defined(USE_MARK_SWEEP_GC) && !defined(USE_UNIQUE_MOVE_COMPACT_GC) /*For LOS_Shrink and LOS_Extend*/ if(gc_has_space_tuner(collector->gc) && collector->gc->tuner->kind != TRANS_NOTHING){ collector->non_los_live_obj_size = 0; @@ -157,7 +149,7 @@ return; } -static void wait_collection_finish(GC* gc) +void wait_collection_finish(GC* gc) { unsigned int num_active_collectors = gc->num_active_collectors; for(unsigned int i=0; icollector_is_active = TRUE; /* waken up and check for new task */ TaskType task_func = collector->task_func; @@ -186,9 +179,13 @@ task_func(collector); - alloc_context_reset((Allocator*)collector); - + //conducted after collection to return last TLB in hand + #if !defined(USE_MARK_SWEEP_GC) && !defined(USE_UNIQUE_MOVE_COMPACT_GC) + gc_reset_collector_alloc(collector->gc, collector); + #endif collector_notify_work_done(collector); + + collector->collector_is_active = FALSE; } return 0; @@ -221,7 +218,6 @@ while(old_live_collector_num == live_collector_num) vm_thread_yield(); /* give collector time to die */ - delete collector->trace_stack; return; } @@ -232,14 +228,14 @@ void collector_init_stats(Collector* collector) { -#ifndef USE_MARK_SWEEP_GC +#if !defined(USE_MARK_SWEEP_GC) && !defined(USE_UNIQUE_MOVE_COMPACT_GC) gc_gen_collector_stats_initialize(collector); #endif } void collector_destruct_stats(Collector* collector) { -#ifndef USE_MARK_SWEEP_GC +#if !defined(USE_MARK_SWEEP_GC) && !defined(USE_UNIQUE_MOVE_COMPACT_GC) gc_gen_collector_stats_destruct(collector); #endif } @@ -256,8 +252,7 @@ #ifdef GC_GEN_STATS collector_destruct_stats(collector); #endif - STD_FREE(collector); - + gc_destruct_collector_alloc(gc, collector); } assert(live_collector_num == 0); @@ -288,14 +283,11 @@ /* FIXME:: thread_handle is for temporary control */ collector->thread_handle = (VmThreadHandle)(POINTER_SIZE_INT)i; collector->gc = gc; - collector_init_thread(collector); + //init collector allocator (mainly for semi-space which has two target spaces) + gc_init_collector_alloc(gc, collector); + //init thread scheduling related stuff, creating collector thread + collector_init_thread(collector); -#ifdef USE_MARK_SWEEP_GC - collector_init_free_chunk_list(collector); -#else - gc_gen_hook_for_collector_init(collector); -#endif - #ifdef GC_GEN_STATS collector_init_stats(collector); #endif @@ -316,3 +308,41 @@ return; } + +void collector_execute_task_concurrent(GC* gc, TaskType task_func, Space* space, unsigned int num_collectors) +{ + assign_collector_with_task(gc, task_func, space); + + return; +} + +void collector_release_weakref_sets(GC* gc, unsigned int num_collectors) +{ + Finref_Metadata *metadata = gc->finref_metadata; + unsigned int num_active_collectors = gc->num_active_collectors; + unsigned int i = 0; + for(; icollectors[i]; + pool_put_entry(metadata->free_pool, collector->softref_set); + pool_put_entry(metadata->free_pool, collector->weakref_set); + pool_put_entry(metadata->free_pool, collector->phanref_set); + collector->softref_set = NULL; + collector->weakref_set = NULL; + collector->phanref_set = NULL; + } +} + +Boolean is_collector_finished(GC* gc) +{ + unsigned int num_active_collectors = gc->num_active_collectors; + unsigned int i = 0; + for(; icollectors[i]; + if(collector->collector_is_active){ + return FALSE; + } + } + return TRUE; + +} + Index: src/thread/collector.h =================================================================== --- src/thread/collector.h (revision 606795) +++ src/thread/collector.h (working copy) @@ -43,14 +43,18 @@ void *ceiling; void *end; void *alloc_block; - Chunk_Header ***local_chunks; + Chunk_Header ***local_chunks; /* this is for MARK-SWEEP GC */ Space* alloc_space; GC* gc; VmThreadHandle thread_handle; /* This thread; */ + unsigned int handshake_signal; /*Handshake is used in concurrent GC.*/ /* End of Allocator --> */ /* FIXME:: for testing */ Space* collect_space; + + /* backup allocator in case there are two target copy spaces, such as semispace GC */ + Allocator* backup_allocator; Vector_Block *trace_stack; @@ -77,11 +81,14 @@ void(*task_func)(void*) ; /* current task */ + /* following three fields are to support LOS extension: to estimate MOS size */ POINTER_SIZE_INT non_los_live_obj_size; POINTER_SIZE_INT los_live_obj_size; POINTER_SIZE_INT segment_live_size[NORMAL_SIZE_SEGMENT_NUM]; unsigned int result; + Boolean collector_is_active; + /*for collect statistics info*/ #ifdef GC_GEN_STATS void* stats; @@ -95,6 +102,8 @@ void collector_reset(GC* gc); void collector_execute_task(GC* gc, TaskType task_func, Space* space); +void collector_execute_task_concurrent(GC* gc, TaskType task_func, Space* space, unsigned int num_collectors); +void collector_release_weakref_sets(GC* gc, unsigned int num_collectors); void collector_restore_obj_info(Collector* collector); #ifdef USE_32BITS_HASHCODE @@ -105,6 +114,9 @@ void gc_gen_hook_for_collector_init(Collector *collector); #endif +Boolean is_collector_finished(GC* gc); +void wait_collection_finish(GC* gc); + inline Boolean gc_collection_result(GC* gc) { Boolean result = TRUE; Index: src/thread/collector_alloc.h =================================================================== --- src/thread/collector_alloc.h (revision 606795) +++ src/thread/collector_alloc.h (working copy) @@ -23,13 +23,19 @@ #define _COLLECTOR_ALLOC_H_ #include "gc_thread.h" + #ifdef USE_32BITS_HASHCODE #include "../common/hashcode.h" #endif +#include "../semi_space/sspace.h" + extern Space_Alloc_Func mos_alloc; -/* NOS forward obj to MOS in MINOR_COLLECTION */ +//FIXME: MINOR_ALGO is static +extern unsigned int MINOR_ALGO; + +/* NOS forward obj to other space in MINOR_COLLECTION */ FORCE_INLINE Partial_Reveal_Object* collector_forward_object(Collector* collector, Partial_Reveal_Object* p_obj) { Obj_Info_Type oi = get_obj_info_raw(p_obj); @@ -44,18 +50,36 @@ #ifdef USE_32BITS_HASHCODE Boolean obj_is_set_hashcode = hashcode_is_set(p_obj); - if(obj_is_set_hashcode) size += GC_OBJECT_ALIGNMENT; + Boolean obj_hashcode_attached = FALSE; + if(obj_is_set_hashcode){ + size += GC_OBJECT_ALIGNMENT; + /* the tospace of semispace GC may have objects with hashcode attached*/ + obj_hashcode_attached = hashcode_is_attached(p_obj); + } #endif Partial_Reveal_Object* p_targ_obj = NULL; - if(is_collector_local_alloc){ - p_targ_obj = thread_local_alloc(size, (Allocator*)collector); - if(!p_targ_obj) - p_targ_obj = (Partial_Reveal_Object*)mos_alloc(size, (Allocator*)collector); - } else { - p_targ_obj = (Partial_Reveal_Object*)mos_alloc(size, (Allocator*)collector); + + Allocator* allocator = (Allocator*)collector; + + /* can also use collector->collect_space->collect_algorithm */ + if( MINOR_ALGO == MINOR_NONGEN_SEMISPACE_POOL){ + + p_targ_obj = (Partial_Reveal_Object*)semispace_forward_obj(p_obj, size, allocator); + if( !p_targ_obj ) + allocator = ((Collector*)collector)->backup_allocator; + + }else{ /* other non-ss algorithms. FIXME:: I am going to remove this branch if it has no perf impact. */ + + if(is_collector_local_alloc){ /* try local alloc first if collector supports it. Marksweep doesn't. */ + p_targ_obj = thread_local_alloc(size, allocator); + } } + if(!p_targ_obj){ + p_targ_obj = (Partial_Reveal_Object*)mos_alloc(size, allocator); + } + if(p_targ_obj == NULL){ /* failed to forward an obj */ collector->result = FALSE; @@ -71,37 +95,44 @@ block. The remaining part of the switched block cannot be revivied for next allocation of object that has smaller size than this one. */ assert( obj_is_fw_in_oi(p_obj)); - thread_local_unalloc(size, (Allocator*)collector); + thread_local_unalloc(size, allocator); return NULL; } -assert((((POINTER_SIZE_INT)p_targ_obj) % GC_OBJECT_ALIGNMENT) == 0); + assert((((POINTER_SIZE_INT)p_targ_obj) % GC_OBJECT_ALIGNMENT) == 0); + #ifdef USE_32BITS_HASHCODE - if(obj_is_set_hashcode){ - memcpy(p_targ_obj, p_obj, size-GC_OBJECT_ALIGNMENT); - oi = trace_forward_process_hashcode(p_targ_obj, p_obj ,oi, size); - }else{ - memcpy(p_targ_obj, p_obj, size); + if(obj_is_set_hashcode && !obj_hashcode_attached){ + size -= GC_OBJECT_ALIGNMENT; //restore object size for memcpy from original object + oi = forward_obj_attach_hashcode(p_targ_obj, p_obj ,oi, size); //get oi for following set_obj_info } -#else - memcpy(p_targ_obj, p_obj, size); #endif //USE_32BITS_HASHCODE - /* we need clear the bit to give major collection a clean status. */ - if(gc_is_gen_mode()) + memcpy(p_targ_obj, p_obj, size); //copy once. + + /* restore oi, which currently is the forwarding pointer. + for semispace GC, p_targ_obj is still in NOS, we should clear its oi mark_bits. */ + if( obj_belongs_to_nos(p_targ_obj) || gc_is_gen_mode() ) + /* we need clear the bit to give a clean status (it's possibly unclean due to partial forwarding) */ set_obj_info(p_targ_obj, oi&DUAL_MARKBITS_MASK); - + else{ #ifdef MARK_BIT_FLIPPING - /* we need set MARK_BIT to indicate this object is processed for nongen forwarding */ - else + /* we mark it to make the object look like other original live objects in MOS */ set_obj_info(p_targ_obj, oi|FLIP_MARK_BIT); - -#else +#else + set_obj_info(p_targ_obj, oi); +#endif // MARK_BIT_FLIPPING + } + #ifdef USE_32BITS_HASHCODE - else if(obj_is_set_hashcode) - set_obj_info(p_targ_obj, oi); -#endif -#endif + if(obj_hashcode_attached){ + /* this is tricky. In fallback compaction, we need iterate the heap for live objects, + so we need know the exact object size. The hashbit of original copy is overwritten by forwarding pointer. + We use this bit in VT to indicate the original copy has attached hashcode. + We can't set the bit earlier before the memcopy. */ + obj_sethash_in_vt(p_obj); + } +#endif //USE_32BITS_HASHCODE return p_targ_obj; Index: src/thread/gc_thread.h =================================================================== --- src/thread/gc_thread.h (revision 606795) +++ src/thread/gc_thread.h (working copy) @@ -31,8 +31,8 @@ #define ALLOC_PREFETCH #endif -#ifdef ALLOC_ZEROING -#ifdef ALLOC_PREFETCH +#ifdef ALLOC_ZEROING /* ----------------- */ +#ifdef ALLOC_PREFETCH /* vvvvvvvvvvvvvvvv */ #ifdef _WINDOWS_ #include @@ -45,10 +45,10 @@ extern POINTER_SIZE_INT ZEROING_SIZE; extern POINTER_SIZE_INT PREFETCH_STRIDE; extern Boolean PREFETCH_ENABLED; -#else /* ALLOC_PREFETCH */ +#else /* ALLOC_PREFETCH ^^^^^^^^^^^^^^^^ */ #define ZEROING_SIZE 256 #endif /* !ALLOC_PREFETCH */ -#endif /* ALLOC_ZEROING */ +#endif /* ALLOC_ZEROING ----------------- */ extern POINTER_SIZE_INT tls_gc_offset; @@ -74,6 +74,7 @@ Space* alloc_space; GC *gc; VmThreadHandle thread_handle; /* This thread; */ + unsigned int handshake_signal; /*Handshake is used in concurrent GC.*/ }Allocator; inline void thread_local_unalloc(unsigned int size, Allocator* allocator) @@ -150,7 +151,9 @@ { assert(alloc_block->status == BLOCK_FREE); alloc_block->status = BLOCK_IN_USE; - +#ifdef USE_UNIQUE_MOVE_COMPACT_GC + alloc_block->num_multi_block = 0; +#endif /* set allocation context */ void* new_free = alloc_block->free; allocator->free = new_free; @@ -186,7 +189,8 @@ inline void alloc_context_reset(Allocator* allocator) { Block_Header* block = (Block_Header*)allocator->alloc_block; - /* it can be NULL if GC happens before the mutator resumes, or called by collector */ + /* it can be NULL when GC happens before the mutator resumes (the memory is run out by other mutators), + or the function is called by collector after it finishes collection, before sleeps waiting for new task */ if( block != NULL ){ assert(block->status == BLOCK_IN_USE); block->free = allocator->free; @@ -194,9 +198,9 @@ allocator->alloc_block = NULL; } - allocator->free = NULL; - allocator->ceiling = NULL; - allocator->end = NULL; + allocator->free = NULL; + allocator->ceiling = NULL; + allocator->end = NULL; return; } Index: src/thread/marker.cpp =================================================================== --- src/thread/marker.cpp (revision 606795) +++ src/thread/marker.cpp (working copy) @@ -198,6 +198,7 @@ void assign_marker_with_task(GC* gc, TaskType task_func, Space* space) { + gc->num_active_markers = gc->num_markers; for(unsigned int i=0; inum_markers; i++) { Marker* marker = gc->markers[i]; @@ -216,7 +217,8 @@ gc->num_active_markers += num_markers; for(; i < gc->num_active_markers; i++) { - printf("start mark thread %d \n", i); + //printf("start mark thread %d \n", i); + Marker* marker = gc->markers[i]; marker_reset_thread(marker); @@ -227,16 +229,9 @@ return; } -void marker_execute_task(GC* gc, TaskType task_func, Space* space) -{ - assign_marker_with_task(gc, task_func, space); - wait_mark_finish(gc); - return; -} - void wait_mark_root_finish(GC* gc) { - unsigned int num_marker = gc->num_markers; + unsigned int num_marker = gc->num_active_markers; for(unsigned int i=0; imarkers[i]; @@ -257,6 +252,14 @@ return; } +void marker_execute_task(GC* gc, TaskType task_func, Space* space) +{ + assign_marker_with_task(gc, task_func, space); + wait_mark_root_finish(gc); + wait_mark_finish(gc); + return; +} + void marker_execute_task_concurrent(GC* gc, TaskType task_func, Space* space) { assign_marker_with_task(gc, task_func, space); @@ -274,3 +277,4 @@ return; } + Index: src/thread/marker.h =================================================================== --- src/thread/marker.h (revision 606795) +++ src/thread/marker.h (working copy) @@ -19,7 +19,7 @@ #define _MARKER_H_ #include "../common/gc_space.h" -#include "../mark_sweep/sspace_chunk.h" +#include "../mark_sweep/wspace_chunk.h" typedef struct Marker{ /* <-- first couple of fields are overloaded as Allocator */ @@ -31,6 +31,7 @@ Space* alloc_space; GC* gc; VmThreadHandle thread_handle; /* This thread; */ + unsigned int handshake_signal; /*Handshake is used in concurrent GC.*/ /* End of Allocator --> */ /* FIXME:: for testing */ @@ -66,11 +67,13 @@ POINTER_SIZE_INT segment_live_size[NORMAL_SIZE_SEGMENT_NUM]; unsigned int result; + Boolean marker_is_active; + VmEventHandle markroot_finished_event; - Boolean marker_is_active; int64 time_mark; - Marker* next; + Marker* next; + unsigned int num_dirty_slots_traced; } Marker; typedef Marker* Marker_List; @@ -93,3 +96,4 @@ #endif //_MARKER_H_ + Index: src/thread/mutator.cpp =================================================================== --- src/thread/mutator.cpp (revision 606795) +++ src/thread/mutator.cpp (working copy) @@ -21,7 +21,7 @@ #include "mutator.h" #include "../trace_forward/fspace.h" -#include "../mark_sweep/sspace.h" +#include "../mark_sweep/wspace.h" #include "../finalizer_weakref/finalizer_weakref.h" struct GC_Gen; @@ -38,7 +38,7 @@ mutator->rem_set = free_set_pool_get_entry(gc->metadata); assert(vector_block_is_empty(mutator->rem_set)); } - mutator->dirty_obj_snapshot = free_set_pool_get_entry(gc->metadata); + mutator->dirty_set = free_set_pool_get_entry(gc->metadata); if(!IGNORE_FINREF ) mutator->obj_with_fin = finref_get_free_block(gc); @@ -69,12 +69,13 @@ alloc_context_reset((Allocator*)mutator); -#ifdef USE_MARK_SWEEP_GC - allocactor_destruct_local_chunks((Allocator*)mutator); -#endif lock(gc->mutator_list_lock); // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +#ifdef USE_MARK_SWEEP_GC + allocactor_destruct_local_chunks((Allocator*)mutator); +#endif + volatile Mutator *temp = gc->mutator_list; if (temp == mutator) { /* it is at the head of the list */ gc->mutator_list = temp->next; @@ -99,13 +100,16 @@ mutator->obj_with_fin = NULL; } - if( mutator->dirty_obj_snapshot != NULL){ - if(vector_block_is_empty(mutator->dirty_obj_snapshot)) - pool_put_entry(gc->metadata->free_set_pool, mutator->dirty_obj_snapshot); - else /* FIXME:: this condition may be released. */ - pool_put_entry(gc->metadata->dirty_obj_snaptshot_pool, mutator->dirty_obj_snapshot); + lock(mutator->dirty_set_lock); + if( mutator->dirty_set != NULL){ + if(vector_block_is_empty(mutator->dirty_set)) + pool_put_entry(gc->metadata->free_set_pool, mutator->dirty_set); + else{ /* FIXME:: this condition may be released. */ + pool_put_entry(gc->metadata->gc_dirty_set_pool, mutator->dirty_set); + mutator->dirty_set = NULL; + } } - + unlock(mutator->dirty_set_lock); //gc_set_tls(NULL); return; @@ -132,14 +136,14 @@ return; } -/* -Boolean gc_local_snapshot_is_empty(GC* gc) +Boolean gc_local_dirtyset_is_empty(GC* gc) { lock(gc->mutator_list_lock); Mutator *mutator = gc->mutator_list; while (mutator) { - if(mutator->concurrent_mark_handshake_status != LOCAL_SNAPSHOT_CONTAINER_IS_EMPTY){ + Vector_Block* local_dirty_set = mutator->dirty_set; + if(!vector_block_is_empty(local_dirty_set)){ unlock(gc->mutator_list_lock); return FALSE; } @@ -148,36 +152,18 @@ unlock(gc->mutator_list_lock); return TRUE; -}*/ - -Boolean gc_local_snapshot_is_empty(GC* gc) -{ - lock(gc->mutator_list_lock); - - Mutator *mutator = gc->mutator_list; - while (mutator) { - Vector_Block* local_snapshot_set = mutator->dirty_obj_snapshot; - if(!vector_block_is_empty(local_snapshot_set)){ - unlock(gc->mutator_list_lock); - return FALSE; - } - mutator = mutator->next; - } - - unlock(gc->mutator_list_lock); - return TRUE; } -Vector_Block* gc_get_local_snapshot(GC* gc, unsigned int shared_id) +Vector_Block* gc_get_local_dirty_set(GC* gc, unsigned int shared_id) { lock(gc->mutator_list_lock); Mutator *mutator = gc->mutator_list; while (mutator) { - Vector_Block* local_snapshot = mutator->dirty_obj_snapshot; - if(!vector_block_is_empty(local_snapshot) && vector_block_set_shared(local_snapshot,shared_id)){ + Vector_Block* local_dirty_set = mutator->dirty_set; + if(!vector_block_is_empty(local_dirty_set) && vector_block_set_shared(local_dirty_set,shared_id)){ unlock(gc->mutator_list_lock); - return local_snapshot; + return local_dirty_set; } mutator = mutator->next; } @@ -187,3 +173,4 @@ } + Index: src/thread/mutator.h =================================================================== --- src/thread/mutator.h (revision 606795) +++ src/thread/mutator.h (working copy) @@ -37,12 +37,17 @@ Space* alloc_space; GC* gc; VmThreadHandle thread_handle; /* This thread; */ + volatile unsigned int handshake_signal; /*Handshake is used in concurrent GC.*/ + /* END of Allocator --> */ Vector_Block* rem_set; Vector_Block* obj_with_fin; Mutator* next; /* The gc info area associated with the next active thread. */ - Vector_Block* dirty_obj_snapshot; + Vector_Block* dirty_set; + SpinLock dirty_set_lock; + unsigned int dirty_obj_slot_num; //only ON_THE_FLY + unsigned int dirty_obj_num; //concurrent mark } Mutator; void mutator_initialize(GC* gc, void* tls_gc_info); @@ -52,6 +57,19 @@ void gc_reset_mutator_context(GC* gc); void gc_prepare_mutator_remset(GC* gc); -Boolean gc_local_snapshot_is_empty(GC* gc); -Vector_Block* gc_get_local_snapshot(GC* gc, unsigned int shared_id); +Boolean gc_local_dirtyset_is_empty(GC* gc); +Vector_Block* gc_get_local_dirty_set(GC* gc, unsigned int shared_id); + +inline void mutator_post_signal(Mutator* mutator, unsigned int handshake_signal) +{ + //FIXME: Need barrier here. + //apr_memory_rw_barrier(); + mutator->handshake_signal = handshake_signal; + //apr_memory_rw_barrier(); +} + +inline void wait_mutator_signal(Mutator* mutator, unsigned int handshake_signal) +{ while(mutator->handshake_signal == handshake_signal); } + + #endif /*ifndef _MUTATOR_H_ */ Index: src/thread/mutator_alloc.cpp =================================================================== --- src/thread/mutator_alloc.cpp (revision 606795) +++ src/thread/mutator_alloc.cpp (working copy) @@ -30,6 +30,8 @@ //#define GC_OBJ_SIZE_STATISTIC +volatile Boolean obj_alloced_live = FALSE; + #ifdef GC_OBJ_SIZE_STATISTIC #define GC_OBJ_SIZE_STA_MAX 256*KB unsigned int obj_size_distribution_map[GC_OBJ_SIZE_STA_MAX>>10]; @@ -68,7 +70,7 @@ size = (size & NEXT_TO_HIGH_BIT_CLEAR_MASK); Allocator* allocator = (Allocator*)gc_get_tls(); - Boolean type_has_fin = type_has_finalizer((Partial_Reveal_VTable*)uncompress_vt((VT)ah)); + Boolean type_has_fin = type_has_finalizer((Partial_Reveal_VTable*)decode_vt((VT)ah)); if(type_has_fin && !IGNORE_FINREF && mutator_need_block) vm_heavy_finalizer_block_mutator(); @@ -77,7 +79,11 @@ gc_alloc_statistic_obj_distrubution(size); #endif -#ifndef USE_MARK_SWEEP_GC +#if defined(USE_MARK_SWEEP_GC) + p_obj = (Managed_Object_Handle)gc_ms_alloc(size, allocator); +#elif defined(USE_UNIQUE_MOVE_COMPACT_GC) + p_obj = (Managed_Object_Handle)gc_mc_alloc(size, allocator); +#else if ( size > GC_OBJ_SIZE_THRESHOLD ){ p_obj = (Managed_Object_Handle)los_alloc(size, allocator); #ifdef GC_GEN_STATS @@ -90,8 +96,6 @@ }else{ p_obj = (Managed_Object_Handle)nos_alloc(size, allocator); } -#else - p_obj = (Managed_Object_Handle)gc_ms_alloc(size, allocator); #endif if( p_obj == NULL ) @@ -114,7 +118,7 @@ assert((size % GC_OBJECT_ALIGNMENT) == 0); assert(ah); - if(type_has_finalizer((Partial_Reveal_VTable *) uncompress_vt((VT)ah))) + if(type_has_finalizer((Partial_Reveal_VTable *) decode_vt((VT)ah))) return NULL; #ifdef GC_OBJ_SIZE_STATISTIC @@ -128,10 +132,12 @@ /* Try to allocate an object from the current Thread Local Block */ Managed_Object_Handle p_obj; -#ifndef USE_MARK_SWEEP_GC +#if defined(USE_MARK_SWEEP_GC) + p_obj = (Managed_Object_Handle)gc_ms_fast_alloc(size, allocator); +#elif defined(USE_UNIQUE_MOVE_COMPACT_GC) + p_obj = (Managed_Object_Handle)gc_mc_fast_alloc(size, allocator); +#else p_obj = (Managed_Object_Handle)thread_local_alloc(size, allocator); -#else - p_obj = (Managed_Object_Handle)gc_ms_fast_alloc(size, allocator); #endif if(p_obj == NULL) return NULL; Index: src/trace_forward/fspace.cpp =================================================================== --- src/trace_forward/fspace.cpp (revision 606795) +++ src/trace_forward/fspace.cpp (working copy) @@ -27,14 +27,6 @@ Boolean forward_first_half; void* object_forwarding_boundary=NULL; -static void fspace_destruct_blocks(Fspace* fspace) -{ -#ifdef USE_32BITS_HASHCODE - space_desturct_blocks((Blocked_Space*)fspace); -#endif - return; -} - struct GC_Gen; void gc_set_nos(GC_Gen* gc, Space* space); @@ -101,7 +93,10 @@ void fspace_destruct(Fspace *fspace) { - fspace_destruct_blocks(fspace); +#ifdef USE_32BITS_HASHCODE + space_desturct_blocks((Blocked_Space*)fspace); +#endif + STD_FREE(fspace); } @@ -119,7 +114,7 @@ fspace->free_block_idx = first_idx; fspace->ceiling_block_idx = first_idx + fspace->num_managed_blocks - 1; forward_first_half = TRUE; /* only useful for not-FORWARD_ALL*/ - fspace->num_used_blocks = 0; + fspace->num_used_blocks = 0; }else{ if(forward_first_half){ @@ -164,31 +159,6 @@ return; } -#ifdef USE_32BITS_HASHCODE -Block_Header* fspace_next_block; - -void fspace_block_iterate_init(Fspace* fspace) -{ - fspace_next_block = (Block_Header*) fspace->blocks; -} - -Block_Header* fspace_get_next_block() -{ - Block_Header* curr_block = (Block_Header*) fspace_next_block; - while(fspace_next_block != NULL){ - Block_Header* next_block = curr_block->next; - - Block_Header* temp = (Block_Header*)atomic_casptr((volatile void**)&fspace_next_block, next_block, curr_block); - if(temp != curr_block){ - curr_block = (Block_Header*) fspace_next_block; - continue; - } - return curr_block; - } - return NULL; -} -#endif - void collector_execute_task(GC* gc, TaskType task_func, Space* space); unsigned int mspace_free_block_idx; Index: src/trace_forward/fspace.h =================================================================== --- src/trace_forward/fspace.h (revision 606795) +++ src/trace_forward/fspace.h (working copy) @@ -40,6 +40,7 @@ void fspace_destruct(Fspace *fspace); void* fspace_alloc(unsigned size, Allocator *allocator); +Boolean fspace_alloc_block(Fspace* fspace, Allocator* allocator); void fspace_reset_after_collection(Fspace* fspace); @@ -59,8 +60,4 @@ void fspace_collection(Fspace* fspace); -#ifdef USE_32BITS_HASHCODE -void fspace_block_iterate_init(Fspace* fspace); -Block_Header* fspace_get_next_block(); -#endif #endif // _FROM_SPACE_H_ Index: src/trace_forward/fspace_alloc.cpp =================================================================== --- src/trace_forward/fspace_alloc.cpp (revision 606795) +++ src/trace_forward/fspace_alloc.cpp (working copy) @@ -23,7 +23,7 @@ #include "../common/gc_concurrent.h" #include "../common/collection_scheduler.h" -static Boolean fspace_alloc_block(Fspace* fspace, Allocator* allocator) +Boolean fspace_alloc_block(Fspace* fspace, Allocator* allocator) { alloc_context_reset(allocator); @@ -61,8 +61,8 @@ p_return = thread_local_alloc(size, allocator); if (p_return) return p_return; - if(gc_need_start_concurrent_mark(allocator->gc)) - gc_start_concurrent_mark(allocator->gc); + gc_try_schedule_collection(allocator->gc, GC_CAUSE_NIL); + /* ran out local block, grab a new one*/ Fspace* fspace = (Fspace*)allocator->alloc_space; int attempts = 0; @@ -98,3 +98,4 @@ } + Index: src/trace_forward/fspace_gen_forward_pool.cpp =================================================================== --- src/trace_forward/fspace_gen_forward_pool.cpp (revision 606795) +++ src/trace_forward/fspace_gen_forward_pool.cpp (working copy) @@ -52,8 +52,8 @@ static FORCE_INLINE void scan_object(Collector* collector, Partial_Reveal_Object *p_obj) { assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); - Partial_Reveal_VTable *vtable = uncompress_vt(obj_get_vt(p_obj)); - if(VTABLE_TRACING) + Partial_Reveal_VTable *vtable = decode_vt(obj_get_vt(p_obj)); + if(TRACE_JLC_VIA_VTABLE) if(vtable->vtmark == VT_UNMARKED) { vtable->vtmark = VT_MARKED; trace_forwarded_gen_jlC_from_vtable(collector, vtable->jlC); Index: src/trace_forward/fspace_nongen_forward_pool.cpp =================================================================== --- src/trace_forward/fspace_nongen_forward_pool.cpp (revision 606795) +++ src/trace_forward/fspace_nongen_forward_pool.cpp (working copy) @@ -40,16 +40,21 @@ return; } -void trace_forwarded_nongen_jlC_from_vtable(Collector* collector, Partial_Reveal_Object *p_obj) ; +/* forward declaration */ +static void forward_jlc_instance(Collector* collector, Partial_Reveal_Object *p_obj); + static FORCE_INLINE void scan_object(Collector* collector, Partial_Reveal_Object *p_obj) { assert((((POINTER_SIZE_INT)p_obj) % GC_OBJECT_ALIGNMENT) == 0); - Partial_Reveal_VTable *vtable = uncompress_vt(obj_get_vt(p_obj)); - if(VTABLE_TRACING) + + Partial_Reveal_VTable *vtable = decode_vt(obj_get_vt(p_obj)); + if(TRACE_JLC_VIA_VTABLE){ if(vtable->vtmark == VT_UNMARKED) { vtable->vtmark = VT_MARKED; - trace_forwarded_nongen_jlC_from_vtable(collector, vtable->jlC); + forward_jlc_instance(collector, vtable->jlC); } + } + if (!object_has_ref_field_before_scan(p_obj)) return; REF *p_ref; @@ -142,9 +147,13 @@ return; } -void trace_forwarded_nongen_jlC_from_vtable(Collector* collector, Partial_Reveal_Object *p_obj) -//Forward the vtable->jlc and trace the forwarded object. But do not update the vtable->jlc but leave them for weakroots updating -//We probably do not need this function if we do not perform class unloading in copy-collections. That means all vtable->jlc would be strong roots in this algorithm +/* + Forward the vtable->jlc and trace the forwarded object. + But do not update the vtable->jlc but leave them for weakroots updating + We probably do not need this function if we do not perform class unloading in minor collections. + That means all the weakroots to jlc instances are treated as strong roots. +*/ +static void forward_jlc_instance(Collector* collector, Partial_Reveal_Object *p_obj) { if(!obj_belongs_to_nos(p_obj)){ if(obj_mark_in_oi(p_obj)) @@ -157,7 +166,7 @@ if( p_target_obj == NULL ){ if(collector->result == FALSE ){ vector_stack_clear(collector->trace_stack); - return; + return; /* FIXME: the failure result is not propagated back to GC */ } assert(obj_get_fw_in_oi(p_obj)); return; Index: src/trace_forward/sspace_temp.cpp =================================================================== --- src/trace_forward/sspace_temp.cpp (revision 0) +++ src/trace_forward/sspace_temp.cpp (revision 0) @@ -0,0 +1,14 @@ +/* this file is temporarily existent for developing semi-space algorithm */ +#include "../semi_space/sspace.h" + +Sspace *sspace_initialize(GC* gc, void* start, POINTER_SIZE_INT sspace_size, POINTER_SIZE_INT commit_size){return null;} +void sspace_destruct(Sspace *sspace){ return;} + +void* sspace_alloc(unsigned size, Allocator *allocator){return NULL;} +void* semispace_alloc(unsigned size, Allocator *allocator){return NULL;} + +Boolean sspace_alloc_block(Sspace* sspace, Allocator* allocator){return FALSE;} + +void sspace_collection(Sspace* sspace){return;} +void sspace_reset_after_collection(Sspace* sspace){return;} +void sspace_prepare_for_collection(Sspace* sspace){return;} Index: src/utils/vector_block.h =================================================================== --- src/utils/vector_block.h (revision 606795) +++ src/utils/vector_block.h (working copy) @@ -88,6 +88,20 @@ #endif return value; } +inline void vector_block_add_entry_to_index(Vector_Block* block, unsigned int index, POINTER_SIZE_INT value) +{ +#ifdef _DEBUG + assert(index < VECTOR_BLOCK_ENTRY_NUM); +#endif + block->entries[index] = value; +} +inline POINTER_SIZE_INT vector_block_get_entry_from_index(Vector_Block* block, unsigned int index) +{ +#ifdef _DEBUG + assert(index < VECTOR_BLOCK_ENTRY_NUM); +#endif + return block->entries[index]; +} #define VECTOR_BLOCK_SHARE_BIT 0x01 #define VECTOR_BLOCK_FULL_BIT 0x02 Index: src/verify/verifier_common.cpp =================================================================== --- src/verify/verifier_common.cpp (revision 606795) +++ src/verify/verifier_common.cpp (working copy) @@ -85,10 +85,11 @@ { Partial_Reveal_Object* p_obj = read_slot(p_ref); assert(address_belongs_to_gc_heap(p_obj,heap_verifier->gc)); -#ifndef USE_MARK_SWEEP_GC +#if !defined(USE_MARK_SWEEP_GC) && !defined(USE_UNIQUE_MOVE_COMPACT_GC) GC_Gen* gc = (GC_Gen*)heap_verifier->gc; Space* mspace = gc_get_mos(gc); Space* lspace = gc_get_los(gc); + Space* nos = gc_get_nos(gc); if(p_obj == NULL){ if(gc_match_kind((GC*)gc, MAJOR_COLLECTION) ||(!heap_verifier->gc_is_gen_mode && !NOS_PARTIAL_FORWARD)){ @@ -115,7 +116,7 @@ assert(0); return FALSE; } -#ifndef USE_MARK_SWEEP_GC +#if !defined(USE_MARK_SWEEP_GC) && !defined(USE_UNIQUE_MOVE_COMPACT_GC) }else{ if(heap_verifier->gc_verifier->is_before_fallback_collection){ if(!address_belongs_to_gc_heap(p_obj, heap_verifier->gc)){ @@ -127,8 +128,16 @@ } if(!address_belongs_to_space(p_obj, mspace) && !address_belongs_to_space(p_obj, lspace) && !NOS_PARTIAL_FORWARD){ + if( nos->collect_algorithm == MINOR_NONGEN_SEMISPACE_POOL){ + if( obj_belongs_to_survivor_area((Sspace*)nos, p_obj)) return TRUE; + else{ + printf("\nERROR: obj referenced by rootset is not in survivor_area after semispace GC!\n"); + assert(0); + } + } + + printf("\nERROR: obj referenced by rootset is in NOS after GC!\n"); assert(0); - printf("\nERROR: obj referenced by rootset is in nos after GC!\n"); return FALSE; } } @@ -262,3 +271,4 @@ } + Index: src/verify/verifier_common.h =================================================================== --- src/verify/verifier_common.h (revision 606795) +++ src/verify/verifier_common.h (working copy) @@ -23,12 +23,13 @@ #include "../common/gc_space.h" #include "../gen/gen.h" #include "../mark_sweep/gc_ms.h" +#include "../semi_space/sspace.h" #include "../common/space_tuner.h" #ifdef USE_32BITS_HASHCODE #include "../common/hashcode.h" #endif #ifdef USE_MARK_SWEEP_GC -#include "../mark_sweep/sspace_mark_sweep.h" +#include "../mark_sweep/wspace_mark_sweep.h" #endif #include "../common/gc_concurrent.h" @@ -116,11 +117,9 @@ assert(address_belongs_to_gc_heap(read_slot(p_ref), (GC*)heap_verifier->gc)); Partial_Reveal_Object* UNUSED p_obj = read_slot(p_ref); assert(p_obj); - assert(uncompress_vt(obj_get_vt(p_obj))); - assert(!address_belongs_to_gc_heap(uncompress_vt(obj_get_vt(p_obj)), (GC*)heap_verifier->gc)); + assert(decode_vt(obj_get_vt(p_obj))); + assert(!address_belongs_to_gc_heap(decode_vt(obj_get_vt(p_obj)), (GC*)heap_verifier->gc)); -//ynhe - #ifdef USE_MARK_SWEEP_GC GC_MS* gc = (GC_MS*)heap_verifier->gc; if(!heap_verifier->is_before_gc){ @@ -129,7 +128,6 @@ if(!obj_is_alloc_in_color_table(p_obj)) printf("\nERROR: obj after GC should be set its alloc color!\n"); }else{ - //ynhe if(gc_mark_is_concurrent()) assert(obj_is_mark_black_in_table(p_obj)); } @@ -148,7 +146,9 @@ assert(address_belongs_to_gc_heap(p_obj, (GC*)heap_verifier->gc)); assert(obj_get_vt(p_obj)); - assert(!address_belongs_to_gc_heap(uncompress_vt(obj_get_vt(p_obj)), (GC*)heap_verifier->gc)); + + /*FIXME:: should add more sanity checks, such as vt->gcvt->class... */ + assert(!address_belongs_to_gc_heap(decode_vt(obj_get_vt(p_obj)), (GC*)heap_verifier->gc)); } Index: src/verify/verifier_scanner.cpp =================================================================== --- src/verify/verifier_scanner.cpp (revision 606795) +++ src/verify/verifier_scanner.cpp (working copy) @@ -36,7 +36,7 @@ static FORCE_INLINE void scan_object(Heap_Verifier* heap_verifier, Partial_Reveal_Object *p_obj) { GC_Verifier* gc_verifier = heap_verifier->gc_verifier; -#ifndef USE_MARK_SWEEP_GC +#if !defined(USE_MARK_SWEEP_GC) && !defined(USE_UNIQUE_MOVE_COMPACT_GC) if(gc_verifier->is_before_fallback_collection) { if(obj_belongs_to_nos(p_obj) && obj_is_fw_in_oi(p_obj)){ assert(obj_get_vt(p_obj) == obj_get_vt(obj_get_fw_in_oi(p_obj))); @@ -364,7 +364,7 @@ void verifier_scan_all_objects(Heap_Verifier* heap_verifier) { -#ifndef USE_MARK_SWEEP_GC +#if !defined(USE_MARK_SWEEP_GC) && !defined(USE_UNIQUE_MOVE_COMPACT_GC) GC_Gen* gc = (GC_Gen*)heap_verifier->gc; Space* fspace = gc_get_nos(gc); Space* mspace = gc_get_mos(gc); @@ -412,7 +412,7 @@ void verifier_scan_unreachable_objects(Heap_Verifier* heap_verifier) { -#ifndef USE_MARK_SWEEP_GC +#if !defined(USE_MARK_SWEEP_GC) && !defined(USE_UNIQUE_MOVE_COMPACT_GC) if(heap_verifier->is_before_gc) return; GC_Gen* gc = (GC_Gen*)heap_verifier->gc; Space* mspace = gc_get_mos(gc); @@ -433,3 +433,4 @@ } + Index: src/verify/verify_gc_effect.cpp =================================================================== --- src/verify/verify_gc_effect.cpp (revision 606795) +++ src/verify/verify_gc_effect.cpp (working copy) @@ -18,7 +18,7 @@ #include "verifier_common.h" #include "verify_gc_effect.h" #ifdef USE_MARK_SWEEP_GC -#include "../mark_sweep/sspace_mark_sweep.h" +#include "../mark_sweep/wspace_mark_sweep.h" #endif static POINTER_SIZE_INT hash_obj_distance = 0; @@ -325,7 +325,6 @@ if(!obj_is_alloc_in_color_table(p_obj)) printf("\nERROR: obj after GC should be set its alloc color!\n"); }else{ - //ynhe if(gc_mark_is_concurrent()) assert(obj_is_mark_black_in_table(p_obj)); } @@ -553,3 +552,4 @@ + Index: src/verify/verify_live_heap.cpp =================================================================== --- src/verify/verify_live_heap.cpp (revision 606795) +++ src/verify/verify_live_heap.cpp (working copy) @@ -146,6 +146,29 @@ Heap_Verifier* get_heap_verifier() { return heap_verifier; } +/* this is tempoprarily used to check the rootset sanity after fallback mark-scan */ +void gc_verify_rootset(GC* gc) +{ + Heap_Verifier_Metadata* verifier_metadata = heap_verifier->heap_verifier_metadata; + Pool* root_set_pool = verifier_metadata->root_set_pool; + pool_iterator_init(root_set_pool); + Vector_Block* root_set = pool_iterator_next(root_set_pool); + + while(root_set){ + POINTER_SIZE_INT* iter = vector_block_iterator_init(root_set); + while(!vector_block_iterator_end(root_set,iter)){ + REF* p_ref = (REF* )*iter; + iter = vector_block_iterator_advance(root_set,iter); + Partial_Reveal_Object* p_obj = read_slot(p_ref); + if(obj_belongs_to_nos(p_obj)){ + assert(!obj_is_fw_in_oi(p_obj)); + } + verify_object_header(p_obj, heap_verifier); + } + root_set = pool_iterator_next(root_set_pool); + } + return; +} Index: src/verify/verify_live_heap.h =================================================================== --- src/verify/verify_live_heap.h (revision 606795) +++ src/verify/verify_live_heap.h (working copy) @@ -25,8 +25,9 @@ void gc_init_heap_verification(GC* gc); void gc_terminate_heap_verification(GC* gc); void gc_verify_heap(GC* gc, Boolean is_before_gc); +void gc_verify_rootset(GC* gc); - void event_mutator_allocate_newobj(Partial_Reveal_Object* p_newobj, POINTER_SIZE_INT size, VT vt_raw); void event_gc_collect_kind_changed(GC* gc); + #endif