From 73b3eb4337703d291f5a99b1ad7e6a8ac9df1c2c Mon Sep 17 00:00:00 2001 From: Tim Armstrong Date: Wed, 8 Apr 2020 09:26:31 -0700 Subject: [PATCH] WIP on generic atomics. Not compiling yet. Change-Id: Ie1d0d101c0f586a9158680eebd8935ce1ef63c28 --- be/src/gutil/atomicops.h | 18 +- be/src/gutil/atomicops_internals_portable.h | 242 ++++++++++++++++++++ 2 files changed, 243 insertions(+), 17 deletions(-) create mode 100644 be/src/gutil/atomicops_internals_portable.h diff --git a/be/src/gutil/atomicops.h b/be/src/gutil/atomicops.h index ad8d7736a..6c28c0682 100644 --- a/be/src/gutil/atomicops.h +++ b/be/src/gutil/atomicops.h @@ -70,23 +70,7 @@ #include "gutil/arm_instruction_set_select.h" // ThreadSanitizer provides own implementation of atomicops. -#if defined(THREAD_SANITIZER) -#include "gutil/atomicops-internals-tsan.h" -#elif defined(__APPLE__) -#include "gutil/atomicops-internals-macosx.h" -#elif defined(__GNUC__) && defined(ARMV6) -#include "gutil/atomicops-internals-arm-v6plus.h" -#elif defined(ARMV3) -#include "gutil/atomicops-internals-arm-generic.h" -#elif defined(__GNUC__) && (defined(__i386) || defined(__x86_64__)) -#include "gutil/atomicops-internals-x86.h" -#elif defined(__GNUC__) && defined(ARCH_POWERPC64) -#include "gutil/atomicops-internals-powerpc.h" -#elif defined(OS_WINDOWS) -#include "gutil/atomicops-internals-windows.h" -#else -#error You need to implement atomic operations for this architecture -#endif +#include "gutil/atomicops_internals_portable.h" // Signed type that can hold a pointer and supports the atomic ops below, as // well as atomic loads and stores. Instances must be naturally-aligned. diff --git a/be/src/gutil/atomicops_internals_portable.h b/be/src/gutil/atomicops_internals_portable.h new file mode 100644 index 000000000..069ec117a --- /dev/null +++ b/be/src/gutil/atomicops_internals_portable.h @@ -0,0 +1,242 @@ +// Copyright (c) 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// This file is an internal atomic implementation, use atomicops.h instead. +// +// This implementation uses C++11 atomics' member functions. The code base is +// currently written assuming atomicity revolves around accesses instead of +// C++11's memory locations. The burden is on the programmer to ensure that all +// memory locations accessed atomically are never accessed non-atomically (tsan +// should help with this). +// +// TODO(jfb) Modify the atomicops.h API and user code to declare atomic +// locations as truly atomic. See the static_assert below. +// +// Of note in this implementation: +// * All NoBarrier variants are implemented as relaxed. +// * All Barrier variants are implemented as sequentially-consistent. +// * Compare exchange's failure ordering is always the same as the success one +// (except for release, which fails as relaxed): using a weaker ordering is +// only valid under certain uses of compare exchange. +// * Acquire store doesn't exist in the C11 memory model, it is instead +// implemented as a relaxed store followed by a sequentially consistent +// fence. +// * Release load doesn't exist in the C11 memory model, it is instead +// implemented as sequentially consistent fence followed by a relaxed load. +// * Atomic increment is expected to return the post-incremented value, whereas +// C11 fetch add returns the previous value. The implementation therefore +// needs to increment twice (which the compiler should be able to detect and +// optimize). + +#ifndef BASE_ATOMICOPS_INTERNALS_PORTABLE_H_ +#define BASE_ATOMICOPS_INTERNALS_PORTABLE_H_ + +#include + +typedef int32_t Atomic32; +typedef int64_t Atomic64; + +namespace base { +namespace subtle { + +typedef int32_t Atomic32; +typedef int64_t Atomic64; + +// This implementation is transitional and maintains the original API for +// atomicops.h. This requires casting memory locations to the atomic types, and +// assumes that the API and the C++11 implementation are layout-compatible, +// which isn't true for all implementations or hardware platforms. The static +// assertion should detect this issue, were it to fire then this header +// shouldn't be used. +// +// TODO(jfb) If this header manages to stay committed then the API should be +// modified, and all call sites updated. +typedef volatile std::atomic* AtomicLocation32; +static_assert(sizeof(*(AtomicLocation32) nullptr) == sizeof(Atomic32), + "incompatible 32-bit atomic layout"); + +inline void MemoryBarrier() { +#if defined(__GLIBCXX__) + // Work around libstdc++ bug 51038 where atomic_thread_fence was declared but + // not defined, leading to the linker complaining about undefined references. + __atomic_thread_fence(std::memory_order_seq_cst); +#else + std::atomic_thread_fence(std::memory_order_seq_cst); +#endif +} + +inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + ((AtomicLocation32)ptr) + ->compare_exchange_strong(old_value, + new_value, + std::memory_order_relaxed, + std::memory_order_relaxed); + return old_value; +} + +inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, + Atomic32 new_value) { + return ((AtomicLocation32)ptr) + ->exchange(new_value, std::memory_order_relaxed); +} + +inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + return increment + + ((AtomicLocation32)ptr) + ->fetch_add(increment, std::memory_order_relaxed); +} + +inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + return increment + ((AtomicLocation32)ptr)->fetch_add(increment); +} + +inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + ((AtomicLocation32)ptr) + ->compare_exchange_strong(old_value, + new_value, + std::memory_order_acquire, + std::memory_order_acquire); + return old_value; +} + +inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + ((AtomicLocation32)ptr) + ->compare_exchange_strong(old_value, + new_value, + std::memory_order_release, + std::memory_order_relaxed); + return old_value; +} + +inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { + ((AtomicLocation32)ptr)->store(value, std::memory_order_relaxed); +} + +inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { + ((AtomicLocation32)ptr)->store(value, std::memory_order_relaxed); + MemoryBarrier(); +} + +inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { + ((AtomicLocation32)ptr)->store(value, std::memory_order_release); +} + +inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { + return ((AtomicLocation32)ptr)->load(std::memory_order_relaxed); +} + +inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { + return ((AtomicLocation32)ptr)->load(std::memory_order_acquire); +} + +inline Atomic32 Release_Load(volatile const Atomic32* ptr) { + MemoryBarrier(); + return ((AtomicLocation32)ptr)->load(std::memory_order_relaxed); +} + +typedef volatile std::atomic* AtomicLocation64; +static_assert(sizeof(*(AtomicLocation64) nullptr) == sizeof(Atomic64), + "incompatible 64-bit atomic layout"); + +inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + ((AtomicLocation64)ptr) + ->compare_exchange_strong(old_value, + new_value, + std::memory_order_relaxed, + std::memory_order_relaxed); + return old_value; +} + +inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, + Atomic64 new_value) { + return ((AtomicLocation64)ptr) + ->exchange(new_value, std::memory_order_relaxed); +} + +inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, + Atomic64 new_value) { + return ((AtomicLocation64)ptr) + ->exchange(new_value, std::memory_order_acquire); +} + +inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, + Atomic64 new_value) { + return ((AtomicLocation64)ptr) + ->exchange(new_value, std::memory_order_release); +} + +inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + return increment + + ((AtomicLocation64)ptr) + ->fetch_add(increment, std::memory_order_relaxed); +} + +inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + return increment + ((AtomicLocation64)ptr)->fetch_add(increment); +} + +inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + ((AtomicLocation64)ptr) + ->compare_exchange_strong(old_value, + new_value, + std::memory_order_acquire, + std::memory_order_acquire); + return old_value; +} + +inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + ((AtomicLocation64)ptr) + ->compare_exchange_strong(old_value, + new_value, + std::memory_order_release, + std::memory_order_relaxed); + return old_value; +} + +inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { + ((AtomicLocation64)ptr)->store(value, std::memory_order_relaxed); +} + +inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { + ((AtomicLocation64)ptr)->store(value, std::memory_order_relaxed); + MemoryBarrier(); +} + +inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { + ((AtomicLocation64)ptr)->store(value, std::memory_order_release); +} + +inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { + return ((AtomicLocation64)ptr)->load(std::memory_order_relaxed); +} + +inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { + return ((AtomicLocation64)ptr)->load(std::memory_order_acquire); +} + +inline Atomic64 Release_Load(volatile const Atomic64* ptr) { + MemoryBarrier(); + return ((AtomicLocation64)ptr)->load(std::memory_order_relaxed); +} + +} // namespace subtle +} // namespace base + +#endif // BASE_ATOMICOPS_INTERNALS_PORTABLE_H_ -- 2.17.1