Index: vm/gc_gen/src/trace_forward/fspace_alloc.cpp =================================================================== --- vm/gc_gen/src/trace_forward/fspace_alloc.cpp (revision 612325) +++ vm/gc_gen/src/trace_forward/fspace_alloc.cpp (working copy) @@ -33,7 +33,7 @@ while(old_free_idx <= fspace->ceiling_block_idx){ unsigned int allocated_idx = atomic_cas32(&fspace->free_block_idx, new_free_idx, old_free_idx); if(allocated_idx != old_free_idx){ /* if failed */ - old_free_idx = fspace->free_block_idx; + old_free_idx = allocated_idx; new_free_idx = old_free_idx+1; continue; } Index: vm/gc_gen/src/common/gc_for_vm.cpp =================================================================== --- vm/gc_gen/src/common/gc_for_vm.cpp (revision 612325) +++ vm/gc_gen/src/common/gc_for_vm.cpp (working copy) @@ -352,7 +352,7 @@ Obj_Info_Type temp = atomic_casptrsz((volatile POINTER_SIZE_INT*)(&obj->obj_info), new_info, info); if (temp == info) break; - info = get_obj_info_raw(obj); + info = temp; new_info = info | hash; } } @@ -388,7 +388,7 @@ Obj_Info_Type temp = atomic_casptrsz((volatile POINTER_SIZE_INT*)(&p_obj->obj_info), new_info, info); if (temp == info) break; - info = get_obj_info_raw(p_obj); + info = temp; new_info = info | HASHCODE_SET_BIT; } hash = hashcode_gen((void*)p_obj); Index: vm/gc_gen/src/common/gc_platform.h =================================================================== --- vm/gc_gen/src/common/gc_platform.h (revision 612325) +++ vm/gc_gen/src/common/gc_platform.h (working copy) @@ -122,24 +122,19 @@ } inline void *atomic_casptr(volatile void **mem, void *with, const void *cmp) -{ return apr_atomic_casptr(mem, with, cmp); } +{ return port_atomic_casptr(mem, with, cmp); } inline POINTER_SIZE_INT atomic_casptrsz(volatile POINTER_SIZE_INT* mem, POINTER_SIZE_INT swap, POINTER_SIZE_INT cmp) { - // we can't use apr_atomic_casptr, which can't work correctly in 64bit machine. -#ifdef POINTER64 - return port_atomic_cas64(mem, swap, cmp); -#else - return apr_atomic_cas32(mem, swap, cmp); -#endif + return (POINTER_SIZE_INT)atomic_casptr((volatile void**)mem, (void*)swap, (const void*)cmp); } inline uint32 atomic_cas32(volatile unsigned int *mem, - apr_uint32_t swap, - apr_uint32_t cmp) -{ return (uint32)apr_atomic_cas32(mem, swap, cmp); } + uint32 swap, + uint32 cmp) +{ return (uint32)port_atomic_cas32(mem, swap, cmp); } inline uint32 atomic_inc32(volatile unsigned int *mem) { return (uint32)apr_atomic_inc32(mem); } @@ -294,7 +289,8 @@ }; #define try_lock(x) (!atomic_cas32(&(x), LOCKED, FREE_LOCK)) -#define lock(x) while( !try_lock(x)){ while( x==LOCKED ){ vm_thread_yield();}} +//#define lock(x) while( !try_lock(x)){ while( x==LOCKED ){ vm_thread_yield();}} +#define lock(x) while( !try_lock(x)){ vm_thread_yield(); } #define unlock(x) do{ x = FREE_LOCK;}while(0) #endif //_GC_PLATFORM_H_ Index: vm/gc_gen/src/common/gc_block.cpp =================================================================== --- vm/gc_gen/src/common/gc_block.cpp (revision 612325) +++ vm/gc_gen/src/common/gc_block.cpp (working copy) @@ -146,7 +146,7 @@ Block_Header *temp = (Block_Header*)atomic_casptr((volatile void **)&space->block_iterator, next_block, cur_block); if(temp != cur_block){ - cur_block = (Block_Header*)space->block_iterator; + cur_block = temp; continue; } return cur_block; Index: vm/gc_gen/src/common/gc_common.h =================================================================== --- vm/gc_gen/src/common/gc_common.h (revision 612325) +++ vm/gc_gen/src/common/gc_common.h (working copy) @@ -369,9 +369,12 @@ if( info & OBJ_DIRTY_BIT ) return FALSE; Obj_Info_Type new_info = info | OBJ_DIRTY_BIT; - while(info != atomic_casptrsz((volatile POINTER_SIZE_INT*)get_obj_info_addr(p_obj), new_info, info)){ - info = get_obj_info_raw(p_obj); - if( info & OBJ_DIRTY_BIT ) return FALSE; + while(TRUE) { + new_info = atomic_casptrsz((volatile POINTER_SIZE_INT*)get_obj_info_addr(p_obj), new_info, info); + if (new_info == info) + break; + if (new_info & OBJ_DIRTY_BIT) return FALSE; + info = new_info; new_info = info |OBJ_DIRTY_BIT; } return TRUE; Index: vm/gc_gen/src/mark_sweep/wspace_mark_sweep.cpp =================================================================== --- vm/gc_gen/src/mark_sweep/wspace_mark_sweep.cpp (revision 612325) +++ vm/gc_gen/src/mark_sweep/wspace_mark_sweep.cpp (working copy) @@ -57,7 +57,7 @@ next_chunk->adj_prev = cur_chunk; return cur_chunk; } - cur_chunk = *shared_next_chunk; + cur_chunk = temp; } return NULL; Index: vm/gc_gen/src/mark_sweep/wspace_alloc.h =================================================================== --- vm/gc_gen/src/mark_sweep/wspace_alloc.h (revision 612325) +++ vm/gc_gen/src/mark_sweep/wspace_alloc.h (working copy) @@ -155,7 +155,7 @@ if(temp == old_word){ return; /*returning true does not mean it's marked by this thread. */ } - old_word = *p_color_word; + old_word = temp; //FIXME: this assertion is too strong here because of concurrent sweeping. assert(!slot_is_alloc_in_table(table, slot_index)); Index: vm/gc_gen/src/mark_sweep/wspace_mark_sweep.h =================================================================== --- vm/gc_gen/src/mark_sweep/wspace_mark_sweep.h (revision 612325) +++ vm/gc_gen/src/mark_sweep/wspace_mark_sweep.h (working copy) @@ -131,7 +131,7 @@ #endif return TRUE; } - old_word = *p_color_word; + old_word = temp; new_word = (old_word & color_bits_mask) | mark_color; } @@ -205,7 +205,7 @@ if(temp == old_word){ return TRUE; /*returning true does not mean it's marked by this thread. */ } - old_word = *p_color_word; + old_word = temp; if(old_word & mark_color) return FALSE; /*already marked gray or black.*/ //new_word = (old_word & color_bits_mask) | mark_color; @@ -235,7 +235,7 @@ if(temp == old_word){ return TRUE; /*returning true does not mean it's marked by this thread. */ } - old_word = *p_color_word; + old_word = temp; if(old_word & mark_black_color) return FALSE; /*already marked black*/ new_word = old_word | mark_black_color; @@ -265,7 +265,7 @@ if(temp == old_word){ return TRUE; /*returning true does not mean it's marked by this thread. */ } - old_word = *p_color_word; + old_word = temp; if(obj_is_mark_black_in_table(obj)) return FALSE; /*already marked black*/ new_word = old_word | mark_black_color; @@ -292,7 +292,7 @@ if(temp == old_word){ return TRUE; /*returning true does not mean it's marked by this thread. */ } - old_word = *p_color_word; + old_word = temp; if(old_word & obj_dirty_bit_in_word) return FALSE; new_word = old_word | obj_dirty_bit_in_word; @@ -335,7 +335,7 @@ if(temp == old_word){ return TRUE; /*returning true does not mean it's marked by this thread. */ } - old_word = *p_color_word; + old_word = temp; //if(old_word & clear_mask) return FALSE; /*already marked black*/ new_word = old_word & clear_mask; @@ -364,7 +364,7 @@ if(temp == old_word){ return TRUE; /*returning true does not mean it's marked by this thread. */ } - old_word = *p_color_word; + old_word = temp; //if(old_word & clear_mask) return FALSE; /*already marked black*/ new_word = old_word & clear_mask; Index: vm/gc_gen/src/mark_sweep/wspace_fallback_mark.cpp =================================================================== --- vm/gc_gen/src/mark_sweep/wspace_fallback_mark.cpp (revision 612325) +++ vm/gc_gen/src/mark_sweep/wspace_fallback_mark.cpp (working copy) @@ -36,7 +36,7 @@ while(new_oi != oi){ Obj_Info_Type temp = atomic_casptrsz((volatile Obj_Info_Type*)get_obj_info_addr(obj), new_oi, oi); if(temp == oi) break; - oi = obj->obj_info; + oi = temp; new_oi = oi & DUAL_MARKBITS_MASK; } } Index: vm/gc_gen/src/mark_sweep/wspace_chunk.h =================================================================== --- vm/gc_gen/src/mark_sweep/wspace_chunk.h (revision 612325) +++ vm/gc_gen/src/mark_sweep/wspace_chunk.h (working copy) @@ -245,7 +245,7 @@ POINTER_SIZE_INT temp = (POINTER_SIZE_INT)atomic_casptr((volatile void **) &chunk->table[last_word_index],(void*)new_word ,(void*)old_word ); if(temp == old_word) return; - old_word = chunk->table[last_word_index]; + old_word = temp; new_word = old_word | padding_mask; } } Index: vm/gc_gen/src/mark_sweep/wspace_sweep_concurrent.cpp =================================================================== --- vm/gc_gen/src/mark_sweep/wspace_sweep_concurrent.cpp (revision 612325) +++ vm/gc_gen/src/mark_sweep/wspace_sweep_concurrent.cpp (working copy) @@ -247,7 +247,7 @@ if(temp == old_word){ break; } - old_word = table[i]; + old_word = temp; new_word = old_word & cur_alloc_mask; } } Index: vm/gc_gen/src/mark_sweep/wspace_mark.cpp =================================================================== --- vm/gc_gen/src/mark_sweep/wspace_mark.cpp (revision 612325) +++ vm/gc_gen/src/mark_sweep/wspace_mark.cpp (working copy) @@ -42,7 +42,7 @@ while(new_oi != oi){ Obj_Info_Type temp = atomic_casptrsz((volatile Obj_Info_Type*)get_obj_info_addr(obj), new_oi, oi); if(temp == oi) break; - oi = obj->obj_info; + oi = temp; new_oi = oi & DUAL_MARKBITS_MASK; } } Index: vm/gc_gen/src/los/lspace_alloc_collect.cpp =================================================================== --- vm/gc_gen/src/los/lspace_alloc_collect.cpp (revision 612325) +++ vm/gc_gen/src/los/lspace_alloc_collect.cpp (working copy) @@ -167,8 +167,11 @@ memset(p_result, 0, alloc_size); uint64 vold = lspace->last_alloced_size; uint64 vnew = vold + alloc_size; - while( vold != port_atomic_cas64(&lspace->last_alloced_size, vnew, vold) ){ - vold = lspace->last_alloced_size; + while( TRUE ) { + vnew = port_atomic_cas64(&lspace->last_alloced_size, vnew, vold); + if (vnew == vold) + break; + vold = vnew; vnew = vold + alloc_size; } return p_result; @@ -186,8 +189,11 @@ memset(p_result, 0, alloc_size); uint64 vold = lspace->last_alloced_size; uint64 vnew = vold + alloc_size; - while( vold != port_atomic_cas64(&lspace->last_alloced_size, vnew, vold) ){ - vold = lspace->last_alloced_size; + while( TRUE ) { + vnew = port_atomic_cas64(&lspace->last_alloced_size, vnew, vold); + if (vnew == vold) + break; + vold = vnew; vnew = vold + alloc_size; } return p_result; Index: vm/gc_gen/src/utils/vector_block.h =================================================================== --- vm/gc_gen/src/utils/vector_block.h (revision 612325) +++ vm/gc_gen/src/utils/vector_block.h (working copy) @@ -125,7 +125,7 @@ while(TRUE){ POINTER_SIZE_INT old_var = (POINTER_SIZE_INT)atomic_casptr((volatile void **)& block->next ,(void*) new_shared_var,(void*) old_shared_var); if(old_var == old_shared_var) return TRUE; - old_shared_var = (POINTER_SIZE_INT) block->next; + old_shared_var = old_var; new_shared_var = old_shared_var | VECTOR_BLOCK_FULL_BIT; } assert(0); @@ -142,7 +142,7 @@ while(TRUE){ POINTER_SIZE_INT old_var = (POINTER_SIZE_INT)atomic_casptr((volatile void **)& block->next ,(void*) new_shared_var,(void*) old_shared_var); if(old_var == old_shared_var) return TRUE; - old_shared_var = (POINTER_SIZE_INT) block->next; + old_shared_var = old_var; if(old_shared_var != 0) return FALSE; } } @@ -157,7 +157,7 @@ while(TRUE){ POINTER_SIZE_INT old_var = (POINTER_SIZE_INT)atomic_casptr((volatile void **)& block->next ,(void*) new_shared_var,(void*) old_shared_var); if(old_var == old_shared_var) return TRUE; - old_shared_var = (POINTER_SIZE_INT) block->next; + old_shared_var = old_var; if(old_shared_var & VECTOR_BLOCK_FULL_BIT) return FALSE; } assert(0); @@ -173,7 +173,7 @@ while(TRUE){ POINTER_SIZE_INT old_var = (POINTER_SIZE_INT)atomic_casptr((volatile void **)& block->next ,(void*) new_shared_var,(void*) old_shared_var); if(old_var == old_shared_var) return TRUE; - old_shared_var = (POINTER_SIZE_INT) block->next; + old_shared_var = old_var; if(old_shared_var & VECTOR_BLOCK_SHARE_BIT) return FALSE; } } Index: vm/gc_gen/src/utils/bit_ops.h =================================================================== --- vm/gc_gen/src/utils/bit_ops.h (revision 612325) +++ vm/gc_gen/src/utils/bit_ops.h (working copy) @@ -100,7 +100,7 @@ while (true) { POINTER_SIZE_INT temp = atomic_casptrsz(p_word, new_value, old_value); if (temp == old_value) break; - old_value = *p_word; + old_value = temp; new_value = old_value|mask; } return; @@ -122,7 +122,7 @@ while (true) { POINTER_SIZE_INT temp = atomic_casptrsz(p_word, new_value, old_value); if (temp == old_value) break; - old_value = *p_word; + old_value = temp; new_value = old_value & mask; } return; Index: vm/gc_gen/src/utils/sync_stack.h =================================================================== --- vm/gc_gen/src/utils/sync_stack.h (revision 612325) +++ vm/gc_gen/src/utils/sync_stack.h (working copy) @@ -94,7 +94,7 @@ if(temp == entry){ /* got it */ return entry; } - entry = stack->cur; + entry = temp; } return NULL; } @@ -112,7 +112,7 @@ top_entry->next = NULL; return top_entry; } - cur_top = stack->top; + *(POINTER_SIZE_INT*)&cur_top = temp; top_entry = stack_top_get_entry(cur_top); version = stack_top_get_version(cur_top); } @@ -131,7 +131,7 @@ if(temp == *(POINTER_SIZE_INT*)&cur_top){ // got it return TRUE; } - cur_top = stack->top; + *(POINTER_SIZE_INT*)&cur_top = temp; node->next = stack_top_get_entry(cur_top); new_version = stack_top_get_next_version(cur_top); temp = stack_top_contruct(node, new_version); Index: vm/gc_gen/src/mark_compact/mspace_alloc.cpp =================================================================== --- vm/gc_gen/src/mark_compact/mspace_alloc.cpp (revision 612325) +++ vm/gc_gen/src/mark_compact/mspace_alloc.cpp (working copy) @@ -31,7 +31,7 @@ while( old_free_idx <= mspace->ceiling_block_idx ){ unsigned int allocated_idx = atomic_cas32(&mspace->free_block_idx, new_free_idx, old_free_idx); if(allocated_idx != old_free_idx){ - old_free_idx = mspace->free_block_idx; + old_free_idx = allocated_idx; new_free_idx = old_free_idx+1; continue; } Index: vm/gc_gen/src/mark_compact/mspace_collect_compact.cpp =================================================================== --- vm/gc_gen/src/mark_compact/mspace_collect_compact.cpp (revision 612325) +++ vm/gc_gen/src/mark_compact/mspace_collect_compact.cpp (working copy) @@ -174,7 +174,7 @@ Block_Header* temp = (Block_Header*)atomic_casptr((volatile void **)&next_block_for_compact, next_compact_block, cur_compact_block); if(temp != cur_compact_block){ - cur_compact_block = (Block_Header*)next_block_for_compact; + cur_compact_block = temp; continue; } /* got it, set its state to be BLOCK_IN_COMPACT. It must be the first time touched by compactor */ Index: vm/gc_gen/src/mark_compact/mspace_extend_compact.cpp =================================================================== --- vm/gc_gen/src/mark_compact/mspace_extend_compact.cpp (revision 612325) +++ vm/gc_gen/src/mark_compact/mspace_extend_compact.cpp (working copy) @@ -126,7 +126,7 @@ Block_Header *temp = (Block_Header *)atomic_casptr((volatile void **)&mspace->block_iterator, next_block, cur_block); if(temp != cur_block){ - cur_block = (Block_Header*)mspace->block_iterator; + cur_block = temp; continue; } return cur_block; Index: vm/gc_gen/src/semi_space/sspace_forward.cpp =================================================================== --- vm/gc_gen/src/semi_space/sspace_forward.cpp (revision 612325) +++ vm/gc_gen/src/semi_space/sspace_forward.cpp (working copy) @@ -27,7 +27,7 @@ while( old_free_idx <= sspace->ceiling_block_idx ){ unsigned int allocated_idx = atomic_cas32(&sspace->free_block_idx, new_free_idx, old_free_idx); if(allocated_idx != old_free_idx){ - old_free_idx = sspace->free_block_idx; + old_free_idx = allocated_idx; new_free_idx = old_free_idx+1; continue; } Index: vm/gc_gen/src/semi_space/sspace_alloc.cpp =================================================================== --- vm/gc_gen/src/semi_space/sspace_alloc.cpp (revision 612325) +++ vm/gc_gen/src/semi_space/sspace_alloc.cpp (working copy) @@ -28,7 +28,7 @@ Block_Header* new_free_blk = old_free_blk->next; Block_Header* allocated_blk = (Block_Header*)atomic_casptr((volatile void**)&sspace->cur_free_block, new_free_blk, old_free_blk); if(allocated_blk != old_free_blk){ /* if failed */ - old_free_blk = sspace->cur_free_block; + old_free_blk = allocated_blk; continue; } /* ok, got one */ Index: vm/port/include/port_atomic.h =================================================================== --- vm/port/include/port_atomic.h (revision 612325) +++ vm/port/include/port_atomic.h (working copy) @@ -42,15 +42,20 @@ #ifdef __cplusplus extern "C" { #endif + +#ifdef WIN32 +#define INLINE __forceinline +#else // WIN32 +#ifdef __linux__ +#define INLINE inline __attribute__((always_inline)) +#else // __linux__ #ifdef __cplusplus #define INLINE inline -#else -#ifdef WIN32 -#define INLINE __forceinline -#else +#else // __cplusplus #define INLINE static -#endif -#endif +#endif // __cplusplus +#endif // __linux__ +#endif // WIN32 #if defined(_IPF_) || defined(DOXYGEN) @@ -165,24 +170,41 @@ INLINE uint8 port_atomic_cas8(volatile uint8 * data , uint8 value, uint8 comp) { #if defined(_IA32_) || defined(_EM64T_) + uint8 ret; __asm__ __volatile__( "lock cmpxchgb %1, (%2)" - :"=a"(comp) - :"d"(value), "r"(data), "0"(comp) + :"=a"(ret) + :"q"(value), "r"(data), "0"(comp) + : "memory" ); - return comp; + return ret; #else ABORT("Not supported"); #endif } INLINE uint16 port_atomic_cas16(volatile uint16 * data , uint16 value, uint16 comp) { +#if defined(_IA32_) || defined(_EM64T_) uint16 ret; + __asm__ __volatile__( + "lock cmpxchgw %w1, (%2)" + :"=a"(ret) + :"q"(value), "r"(data), "0"(comp) + : "memory" + ); + return ret; +#else + ABORT("Not supported"); +#endif +} + +INLINE uint32 port_atomic_cas32(volatile uint32* data , uint32 value, uint32 comp) { #if defined(_IA32_) || defined(_EM64T_) + uint32 ret; __asm__ __volatile__( - "lock cmpxchgw %w1, %2" + "lock cmpxchgl %1, (%2)" :"=a"(ret) - :"q"(value), "m"(*data), "0"(comp) + :"q"(value), "r"(data), "0"(comp) : "memory" ); return ret; @@ -213,39 +235,41 @@ ); return comp; #elif defined(_EM64T_) // defined(_IA32_) + uint64 ret; __asm__ __volatile__( "lock cmpxchgq %1, (%2)" - :"=a"(comp) /* outputs */ - :"d"(value), "r"(data), "a"(comp) + :"=a"(ret) /* outputs */ + :"q"(value), "r"(data), "0"(comp) + : "memory" ); - return comp; + return ret; #else ABORT("Not supported"); #endif } -INLINE void * port_atomic_compare_exchange_pointer(volatile void ** data, void * value, const void * comp) { +INLINE void * port_atomic_casptr(volatile void ** data, void * value, const void * comp) { #if defined(_IA32_) - //return (void *) port_atomic_compare_exchange32((uint32 *)data, (uint32)value, (uint32)comp); - uint32 Exchange = (uint32)value; - uint32 Comperand = (uint32)comp; + //return (void *) port_atomic_cas32((uint32 *)data, (uint32)value, (uint32)comp); + uint32 ret; __asm__ __volatile__( "lock cmpxchgl %1, (%2)" - :"=a"(Comperand) - :"d"(Exchange), "r"(data), "a"(Comperand) + :"=a"(ret) + :"q"(value), "r"(data), "0"(comp) + : "memory" ); - return (void*)Comperand; + return (void*)ret; #elif defined(_EM64T_) // defined(_IA32_) //return (void *) port_atomic_cas64((uint64 *)data, (uint64)value, (uint64)comp); - uint64 Exchange = (uint64)value; - uint64 Comperand = (uint64)comp; - __asm__( - "lock cmpxchgq %1, (%2)" - :"=a"(Comperand) - :"d"(Exchange), "r"(data), "a"(Comperand) + uint64 ret; + __asm__ __volatile__( + "lock cmpxchgq %q1, (%2)" + :"=a"(ret) + :"q"(value), "r"(data), "0"(comp) + : "memory" ); - return (void *)Comperand; + return (void*)ret; #else // defined(_EM64T_) ABORT("Not supported"); Index: vm/port/src/atomic/linux_ipf/port_atomic.c =================================================================== --- vm/port/src/atomic/linux_ipf/port_atomic.c (revision 612325) +++ vm/port/src/atomic/linux_ipf/port_atomic.c (working copy) @@ -27,7 +27,7 @@ extern "C" { #endif -APR_DECLARE(void *) port_atomic_compare_exchange_pointer(volatile void ** data, void * value, const void * comp) { +APR_DECLARE(void *) port_atomic_casptr(volatile void ** data, void * value, const void * comp) { return (void *) port_atomic_cas64((uint64 *)data, (uint64)value, (uint64)comp); }