From nobody Mon Sep 17 00:00:00 2001 From: Ilya Berezhniuk Date: Sun, 17 Feb 2008 03:21:37 +0300 Subject: [PATCH] VM memory barriers was moved to PORT as inline --- make/vm/port.xml | 1 vm/port/include/port_atomic.h | 31 ++-- vm/port/include/port_barriers.h | 141 ++++++++++++++++++++ vm/port/include/port_general.h | 20 +++ vm/port/src/barriers/linux_ipf/port_barriers.asm | 39 ++++++ vm/vmcore/include/atomics.h | 3 vm/vmcore/src/class_support/String_Pool.cpp | 8 + vm/vmcore/src/class_support/method.cpp | 8 + .../org_apache_harmony_util_concurrent_Atomics.cpp | 13 +- vm/vmcore/src/thread/linux/atomics.cpp | 54 -------- vm/vmcore/src/thread/win/atomics.cpp | 60 --------- 11 files changed, 227 insertions(+), 151 deletions(-) create mode 100755 vm/port/include/port_barriers.h create mode 100755 vm/port/src/barriers/linux_ipf/port_barriers.asm delete mode 100644 vm/vmcore/src/thread/linux/atomics.cpp delete mode 100644 vm/vmcore/src/thread/win/atomics.cpp 02d2b27040c99982e9b213136a572fdffba602f0 diff --git a/make/vm/port.xml b/make/vm/port.xml index 8998b75..ae7fdcb 100644 --- a/make/vm/port.xml +++ b/make/vm/port.xml @@ -64,6 +64,7 @@ + diff --git a/vm/port/include/port_atomic.h b/vm/port/include/port_atomic.h index 5d5eda1..17a4fb7 100644 --- a/vm/port/include/port_atomic.h +++ b/vm/port/include/port_atomic.h @@ -42,18 +42,9 @@ #include "port_general.h" #ifdef __cplusplus extern "C" { #endif -#ifdef __cplusplus -#define INLINE inline -#else -#ifdef WIN32 -#define INLINE __forceinline -#else -#define INLINE static -#endif -#endif - #if defined(_IPF_) || defined(DOXYGEN) +#if defined(_IPF_) || defined(DOXYGEN) /** * The atomic compare and exchange operation on uint8. @@ -98,7 +89,7 @@ APR_DECLARE(uint64) port_atomic_cas64(vo #elif defined(WIN32) && !defined(_WIN64) -INLINE uint8 port_atomic_cas8(volatile uint8 * data , uint8 value, uint8 comp) { +PORT_INLINE uint8 port_atomic_cas8(volatile uint8 * data , uint8 value, uint8 comp) { __asm { mov al, comp mov dl, value @@ -109,7 +100,7 @@ INLINE uint8 port_atomic_cas8(volatile u return comp; } -INLINE uint16 port_atomic_cas16(volatile uint16 * data , uint16 value, uint16 comp) { +PORT_INLINE uint16 port_atomic_cas16(volatile uint16 * data , uint16 value, uint16 comp) { __asm { mov ax, comp mov dx, value @@ -120,7 +111,7 @@ INLINE uint16 port_atomic_cas16(volatile return comp; } -INLINE uint64 port_atomic_cas64(volatile uint64 * data , uint64 value, uint64 comp) { +PORT_INLINE uint64 port_atomic_cas64(volatile uint64 * data , uint64 value, uint64 comp) { __asm { lea esi, comp mov eax, [esi] @@ -145,16 +136,16 @@ #elif defined(_EM64T_) && defined (_WIN6 #pragma intrinsic(_InterlockedCompareExchange16) #pragma intrinsic(_InterlockedCompareExchange64) -INLINE uint8 port_atomic_cas8(volatile uint8 * data, +PORT_INLINE uint8 port_atomic_cas8(volatile uint8 * data, uint8 value, uint8 comp); -INLINE uint16 port_atomic_cas16(volatile uint16 * data, +PORT_INLINE uint16 port_atomic_cas16(volatile uint16 * data, uint16 value, uint16 comp) { return _InterlockedCompareExchange16((volatile SHORT *)data, value, comp); } -INLINE uint64 port_atomic_cas64(volatile uint64 * data, +PORT_INLINE uint64 port_atomic_cas64(volatile uint64 * data, uint64 value, uint64 comp) { return _InterlockedCompareExchange64((volatile LONG64 *)data, value, comp); @@ -163,7 +154,7 @@ INLINE uint64 port_atomic_cas64(volatile #elif defined (PLATFORM_POSIX) -INLINE uint8 port_atomic_cas8(volatile uint8 * data , uint8 value, uint8 comp) { +PORT_INLINE uint8 port_atomic_cas8(volatile uint8 * data , uint8 value, uint8 comp) { #if defined(_IA32_) || defined(_EM64T_) __asm__ __volatile__( "lock cmpxchgb %1, (%2)" @@ -176,7 +167,7 @@ #else #endif } -INLINE uint16 port_atomic_cas16(volatile uint16 * data , uint16 value, uint16 comp) { +PORT_INLINE uint16 port_atomic_cas16(volatile uint16 * data , uint16 value, uint16 comp) { uint16 ret; #if defined(_IA32_) || defined(_EM64T_) __asm__ __volatile__( @@ -191,7 +182,7 @@ #else #endif } -INLINE uint64 port_atomic_cas64(volatile uint64 * data , uint64 value, uint64 comp) { +PORT_INLINE uint64 port_atomic_cas64(volatile uint64 * data , uint64 value, uint64 comp) { #if defined(_IA32_) __asm__ __volatile__( "push %%ebx;\n\t" @@ -224,7 +215,7 @@ #else #endif } -INLINE void * port_atomic_compare_exchange_pointer(volatile void ** data, void * value, const void * comp) { +PORT_INLINE void * port_atomic_compare_exchange_pointer(volatile void ** data, void * value, const void * comp) { #if defined(_IA32_) //return (void *) port_atomic_compare_exchange32((uint32 *)data, (uint32)value, (uint32)comp); uint32 Exchange = (uint32)value; diff --git a/vm/port/include/port_barriers.h b/vm/port/include/port_barriers.h new file mode 100755 index 0000000..c263a8b --- /dev/null +++ b/vm/port/include/port_barriers.h @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _PORT_BARRIERS_H_ +#define _PORT_BARRIERS_H_ + +/** +* @file +* Memory barriers +*/ + +#include "open/platform_types.h" +#include "port_general.h" + +/** + * @defgroup port_barriers Memory read/write barriers + */ + +#ifdef __cplusplus +extern "C" { +#endif + + +#if defined(_IPF_) +#if defined (PLATFORM_POSIX) + +void port_rw_barrier(void); +void port_write_barrier(void); + +#else /* POSIX */ +#error "Windows/IPF is not supported!" +#endif /* POSIX */ +#endif /* _IPF_ */ + + +#if defined (PLATFORM_POSIX) + +#if !defined(_IPF_) +PORT_INLINE void port_rw_barrier(void) +{ +#if defined(_EM64T_) + asm volatile ("mfence" : : : "memory"); +#else /* General x86 case */ + /* + * This code must use a lock-prefixed assembly instruction, so that + * we can support P3 processors (SSE2 only). With P4 and SSE3, we + * could use 'mfence'. + * References: + * Java Memory Model cookbook + * - http://gee.cs.oswego.edu/dl/jmm/cookbook.html + * Linux Kernel, mb() function + * - http://lxr.linux.no/source/include/asm-i386/system.h + * This is a GCC inline assembly command. The final bit, "memory", will + * clobber all of the memory registers. + */ + asm volatile ("lock; addl $0,0(%%esp)" : : : "memory"); +#endif +} + +PORT_INLINE void port_write_barrier(void) +{ + /* General x86 and x86_64 case */ + /* + * We could use the same lock-prefixed assembly instruction above, + * but since we have support for P3 processors (SSE2) we'll just + * use 'sfence'. + */ + asm volatile ("sfence" : : : "memory"); +} +#endif /* !defined(_IPF_) */ + +#else /* !defined (POSIX) */ + + +/* MSVC barrier intrinsics setup */ +#if _MSC_VER < 1400 + /* VC++ 2003 */ + void _ReadWriteBarrier(void); + void _WriteBarrier(void); + void _mm_mfence(void); + void _mm_sfence(void); +#else + /* VC++ 2005 */ + #include + #include +#endif + +#if !defined(__INTEL_COMPILER) +# pragma intrinsic (_ReadWriteBarrier) +# pragma intrinsic (_WriteBarrier) +#else +# define _ReadWriteBarrier __memory_barrier +# define _WriteBarrier __memory_barrier +#endif + + +PORT_INLINE void port_rw_barrier(void) +{ +#ifdef _EM64T_ + /* if x86_64/x64/EM64T, then use an mfence to flush memory caches */ + _mm_mfence(); +#else + /* otherwise, we assume this is an x86, so insert an inline assembly + * macro to insert a lock instruction + * + * the lock is what's needed, so the 'add' is setup, essentially, as a no-op + */ + __asm {lock add [esp], 0 } +#endif + _ReadWriteBarrier(); +} + +PORT_INLINE void port_write_barrier(void) +{ + _mm_sfence(); + _WriteBarrier(); +} + + +#endif /* !defined (POSIX) */ + + +#ifdef __cplusplus +} +#endif + +#endif /* _PORT_BARRIERS_H_ */ diff --git a/vm/port/include/port_general.h b/vm/port/include/port_general.h index 21f6c55..12afd99 100644 --- a/vm/port/include/port_general.h +++ b/vm/port/include/port_general.h @@ -32,4 +32,24 @@ # define NULL ((void *)0) #endif /* __cplusplus */ #endif /* NULL */ + +#ifdef __cplusplus +#define PORT_INLINE inline +#else // !__cplusplus + +#ifdef WIN32 +#define PORT_INLINE __inline +#else // !WIN32 + +#ifdef __linux__ +#define PORT_INLINE inline __attribute__((always_inline)) +#else // !__linux__ +#define PORT_INLINE static +#endif // __linux__ + +#endif // WIN32 + +#endif // __cplusplus + + #endif /* _PORT_GENERAL_H_ */ diff --git a/vm/port/src/barriers/linux_ipf/port_barriers.asm b/vm/port/src/barriers/linux_ipf/port_barriers.asm new file mode 100755 index 0000000..fc80cd8 --- /dev/null +++ b/vm/port/src/barriers/linux_ipf/port_barriers.asm @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + .section .text + .proc port_rw_barrier# + .global port_rw_barrier# + .align 32 +port_rw_barrier: + + mf ;; + br.ret.sptk.many b0 ;; + + .endp port_rw_barrier# + + + + .section .text + .proc port_write_barrier# + .global port_write_barrier# + .align 32 +port_write_barrier: + + mf ;; + br.ret.sptk.many b0 ;; + + .endp port_write_barrier# diff --git a/vm/vmcore/include/atomics.h b/vm/vmcore/include/atomics.h index 9e0386c..95e91fc 100644 --- a/vm/vmcore/include/atomics.h +++ b/vm/vmcore/include/atomics.h @@ -50,7 +50,4 @@ JNIEXPORT jboolean compareAndSetLongArra JNIEXPORT jboolean vmSupportsCAS8(); -void MemoryReadWriteBarrier(); - -void MemoryWriteBarrier(); #endif // _ATOMICS_H_ diff --git a/vm/vmcore/src/class_support/String_Pool.cpp b/vm/vmcore/src/class_support/String_Pool.cpp index 71e3300..660f553 100644 --- a/vm/vmcore/src/class_support/String_Pool.cpp +++ b/vm/vmcore/src/class_support/String_Pool.cpp @@ -32,7 +32,7 @@ #include "open/gc.h" #include "platform_lowlevel.h" #include "String_Pool.h" #include "environment.h" -#include "atomics.h" +#include "port_barriers.h" #include "vm_strings.h" #include "vm_stats.h" #include "ini.h" @@ -254,7 +254,7 @@ #endif // We need ordering of writes here as we use the collection without lock. // Entry's next pointer should be updated before we update head reference. cur_entry = new(mem) Entry(s, len, 0); - MemoryWriteBarrier(); + port_write_barrier(); *last_entry = cur_entry; unlock_pool(); @@ -310,9 +310,9 @@ void String_Pool::register_interned_stri Interned_Strings * new_elem = (Interned_Strings *)memory_pool.alloc(sizeof(Interned_Strings)); memset(new_elem, 0, sizeof(Interned_Strings)); current_interned->next = new_elem; - MemoryWriteBarrier(); + port_write_barrier(); current_interned = new_elem; - MemoryWriteBarrier(); + port_write_barrier(); local_current_interned->free_slot++; } else { UNSAFE_REGION_START diff --git a/vm/vmcore/src/class_support/method.cpp b/vm/vmcore/src/class_support/method.cpp index b1343cd..27d5fac 100644 --- a/vm/vmcore/src/class_support/method.cpp +++ b/vm/vmcore/src/class_support/method.cpp @@ -30,7 +30,7 @@ #include "method_lookup.h" #include "nogc.h" #include "open/vm_util.h" #include "jit_intf_cpp.h" -#include "atomics.h" +#include "port_barriers.h" #include "cci.h" #ifdef _IPF_ @@ -412,13 +412,13 @@ CodeChunkInfo *Method::get_chunk_info_mt jit_info->_next = _jits; // Write barrier is needed as we use // unlocked fast access to the collection - MemoryWriteBarrier(); + port_write_barrier(); _jits = jit_info; } else { jit_info->_next = _jits->_next; // Write barrier is needed as we use // unlocked fast access to the collection - MemoryWriteBarrier(); + port_write_barrier(); _jits->_next = jit_info; } unlock(); @@ -459,7 +459,7 @@ void *Method::allocate_JIT_data_block(si jit_info = create_code_chunk_info_mt(); jit_info->set_jit(jit); jit_info->_next = _jits; - MemoryWriteBarrier(); + port_write_barrier(); _jits = jit_info; block->next = jit_info->_data_blocks; jit_info->_data_blocks = block; diff --git a/vm/vmcore/src/kernel_classes/native/org_apache_harmony_util_concurrent_Atomics.cpp b/vm/vmcore/src/kernel_classes/native/org_apache_harmony_util_concurrent_Atomics.cpp index 46afa3f..b9d9e3a 100644 --- a/vm/vmcore/src/kernel_classes/native/org_apache_harmony_util_concurrent_Atomics.cpp +++ b/vm/vmcore/src/kernel_classes/native/org_apache_harmony_util_concurrent_Atomics.cpp @@ -22,6 +22,7 @@ #include "jni.h" #include "jni_direct.h" #include "atomics.h" +#include "port_barriers.h" #include "org_apache_harmony_util_concurrent_Atomics.h" JNIEXPORT jint JNICALL Java_org_apache_harmony_util_concurrent_Atomics_arrayBaseOffset @@ -47,7 +48,7 @@ Java_org_apache_harmony_util_concurrent_ jobject obj, jlong offset, jint value) { SetIntFieldOffset(env, obj, (jint)offset, value); - MemoryReadWriteBarrier(); + port_rw_barrier(); } @@ -55,7 +56,7 @@ JNIEXPORT jint JNICALL Java_org_apache_harmony_util_concurrent_Atomics_getIntVolatile__Ljava_lang_Object_2J(JNIEnv * env, jclass self, jobject obj, jlong offset) { - MemoryReadWriteBarrier(); + port_rw_barrier(); return GetIntFieldOffset(env, obj, (jint)offset); } @@ -64,7 +65,7 @@ Java_org_apache_harmony_util_concurrent_ jobject obj, jlong offset, jlong value) { SetLongFieldOffset(env, obj, (jint)offset, value); - MemoryReadWriteBarrier(); + port_rw_barrier(); } @@ -72,7 +73,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_harmony_util_concurrent_Atomics_getLongVolatile__Ljava_lang_Object_2J(JNIEnv * env, jclass self, jobject obj, jlong offset) { - MemoryReadWriteBarrier(); + port_rw_barrier(); return GetLongFieldOffset(env, obj, (jint)offset); } @@ -81,7 +82,7 @@ Java_org_apache_harmony_util_concurrent_ jobject obj, jlong offset, jobject value) { SetObjectFieldOffset(env, obj, (jint)offset, value); - MemoryReadWriteBarrier(); + port_rw_barrier(); } @@ -89,7 +90,7 @@ JNIEXPORT jobject JNICALL Java_org_apache_harmony_util_concurrent_Atomics_getObjectVolatile__Ljava_lang_Object_2J(JNIEnv * env, jclass self, jobject obj, jlong offset) { - MemoryReadWriteBarrier(); + port_rw_barrier(); return GetObjectFieldOffset(env, obj, (jint)offset); } diff --git a/vm/vmcore/src/thread/linux/atomics.cpp b/vm/vmcore/src/thread/linux/atomics.cpp deleted file mode 100644 index 69281de..0000000 --- a/vm/vmcore/src/thread/linux/atomics.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "atomics.h" - - -void MemoryReadWriteBarrier() { -#if defined(_EM64T_) - asm volatile ("mfence" : : : "memory"); -#elif defined(_IPF_) - asm volatile ("mf" : : : "memory"); -#else // General x86 case - /* - * This code must use a lock-prefixed assembly instruction, so that - * we can support P3 processors (SSE2 only). With P4 and SSE3, we - * could use 'mfence'. - * References: - * Java Memory Model cookbook - * - http://gee.cs.oswego.edu/dl/jmm/cookbook.html - * Linux Kernel, mb() function - * - http://lxr.linux.no/source/include/asm-i386/system.h - * This is a GCC inline assembly command. The final bit, "memory", will - * clobber all of the memory registers. - */ - asm volatile ("lock; addl $0,0(%%esp)" : : : "memory"); -#endif -} - -void MemoryWriteBarrier() { -#if defined(_IPF_) - asm volatile ("mf" : : : "memory"); -#else // General x86 and x86_64 case - /* - * We could use the same lock-prefixed assembly instruction above, - * but since we have support for P3 processors (SSE2) we'll just - * use 'sfence'. - */ - asm volatile ("sfence" : : : "memory"); -#endif -} - diff --git a/vm/vmcore/src/thread/win/atomics.cpp b/vm/vmcore/src/thread/win/atomics.cpp deleted file mode 100644 index 8f93586..0000000 --- a/vm/vmcore/src/thread/win/atomics.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "atomics.h" - -// MSVC barrier intrinsics setup -#if _MSC_VER < 1400 - // VC++ 2003 - extern "C" void _ReadWriteBarrier(); - extern "C" void _WriteBarrier(); - extern "C" void _mm_mfence(void); - extern "C" void _mm_sfence(void); -#else - // VC++ 2005 - #include - #include -#endif - -#if !defined(__INTEL_COMPILER) -# pragma intrinsic (_ReadWriteBarrier) -# pragma intrinsic (_WriteBarrier) -#else -# define _ReadWriteBarrier __memory_barrier -# define _WriteBarrier __memory_barrier -#endif - - -void MemoryReadWriteBarrier() { -#ifdef _EM64T_ - // if x86_64/x64/EM64T, then use an mfence to flush memory caches - _mm_mfence(); -#else - /* otherwise, we assume this is an x86, so insert an inline assembly - * macro to insert a lock instruction - * - * the lock is what's needed, so the 'add' is setup, essentially, as a no-op - */ - __asm {lock add [esp], 0 } -#endif - _ReadWriteBarrier(); -} - -void MemoryWriteBarrier() { - _mm_sfence(); - _WriteBarrier(); -} - -- 1.3.3