diff options
Diffstat (limited to 'third_party/libcxx/include/c++/v1/__atomic')
20 files changed, 3046 insertions, 0 deletions
diff --git a/third_party/libcxx/include/c++/v1/__atomic/aliases.h b/third_party/libcxx/include/c++/v1/__atomic/aliases.h new file mode 100644 index 0000000..4fcceba --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/aliases.h @@ -0,0 +1,111 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ALIASES_H +#define _LIBCPP___ATOMIC_ALIASES_H + +#include <__atomic/atomic.h> +#include <__atomic/atomic_lock_free.h> +#include <__atomic/contention_t.h> +#include <__atomic/is_always_lock_free.h> +#include <__config> +#include <__cstddef/ptrdiff_t.h> +#include <__cstddef/size_t.h> +#include <__type_traits/conditional.h> +#include <__type_traits/make_unsigned.h> +#include <cstdint> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +using atomic_bool = atomic<bool>; +using atomic_char = atomic<char>; +using atomic_schar = atomic<signed char>; +using atomic_uchar = atomic<unsigned char>; +using atomic_short = atomic<short>; +using atomic_ushort = atomic<unsigned short>; +using atomic_int = atomic<int>; +using atomic_uint = atomic<unsigned int>; +using atomic_long = atomic<long>; +using atomic_ulong = atomic<unsigned long>; +using atomic_llong = atomic<long long>; +using atomic_ullong = atomic<unsigned long long>; +#if _LIBCPP_HAS_CHAR8_T +using atomic_char8_t = atomic<char8_t>; +#endif +using atomic_char16_t = atomic<char16_t>; +using atomic_char32_t = atomic<char32_t>; +#if _LIBCPP_HAS_WIDE_CHARACTERS +using atomic_wchar_t = atomic<wchar_t>; +#endif + +using atomic_int_least8_t = atomic<int_least8_t>; +using atomic_uint_least8_t = atomic<uint_least8_t>; +using atomic_int_least16_t = atomic<int_least16_t>; +using atomic_uint_least16_t = atomic<uint_least16_t>; +using atomic_int_least32_t = atomic<int_least32_t>; +using atomic_uint_least32_t = atomic<uint_least32_t>; +using atomic_int_least64_t = atomic<int_least64_t>; +using atomic_uint_least64_t = atomic<uint_least64_t>; + +using atomic_int_fast8_t = atomic<int_fast8_t>; +using atomic_uint_fast8_t = atomic<uint_fast8_t>; +using atomic_int_fast16_t = atomic<int_fast16_t>; +using atomic_uint_fast16_t = atomic<uint_fast16_t>; +using atomic_int_fast32_t = atomic<int_fast32_t>; +using atomic_uint_fast32_t = atomic<uint_fast32_t>; +using atomic_int_fast64_t = atomic<int_fast64_t>; +using atomic_uint_fast64_t = atomic<uint_fast64_t>; + +using atomic_int8_t = atomic< int8_t>; +using atomic_uint8_t = atomic<uint8_t>; +using atomic_int16_t = atomic< int16_t>; +using atomic_uint16_t = atomic<uint16_t>; +using atomic_int32_t = atomic< int32_t>; +using atomic_uint32_t = atomic<uint32_t>; +using atomic_int64_t = atomic< int64_t>; +using atomic_uint64_t = atomic<uint64_t>; + +using atomic_intptr_t = atomic<intptr_t>; +using atomic_uintptr_t = atomic<uintptr_t>; +using atomic_size_t = atomic<size_t>; +using atomic_ptrdiff_t = atomic<ptrdiff_t>; +using atomic_intmax_t = atomic<intmax_t>; +using atomic_uintmax_t = atomic<uintmax_t>; + +// C++20 atomic_{signed,unsigned}_lock_free: prefer the contention type most highly, then the largest lock-free type +#if _LIBCPP_STD_VER >= 20 +# if ATOMIC_LLONG_LOCK_FREE == 2 +using __largest_lock_free_type _LIBCPP_NODEBUG = long long; +# elif ATOMIC_INT_LOCK_FREE == 2 +using __largest_lock_free_type _LIBCPP_NODEBUG = int; +# elif ATOMIC_SHORT_LOCK_FREE == 2 +using __largest_lock_free_type _LIBCPP_NODEBUG = short; +# elif ATOMIC_CHAR_LOCK_FREE == 2 +using __largest_lock_free_type _LIBCPP_NODEBUG = char; +# else +# define _LIBCPP_NO_LOCK_FREE_TYPES // There are no lockfree types (this can happen on unusual platforms) +# endif + +# ifndef _LIBCPP_NO_LOCK_FREE_TYPES +using __contention_t_or_largest _LIBCPP_NODEBUG = + __conditional_t<__libcpp_is_always_lock_free<__cxx_contention_t>::__value, + __cxx_contention_t, + __largest_lock_free_type>; + +using atomic_signed_lock_free = atomic<__contention_t_or_largest>; +using atomic_unsigned_lock_free = atomic<make_unsigned_t<__contention_t_or_largest>>; +# endif // !_LIBCPP_NO_LOCK_FREE_TYPES +#endif // C++20 + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_ALIASES_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/atomic.h b/third_party/libcxx/include/c++/v1/__atomic/atomic.h new file mode 100644 index 0000000..3d0ed9c --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/atomic.h @@ -0,0 +1,804 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_H +#define _LIBCPP___ATOMIC_ATOMIC_H + +#include <__atomic/atomic_sync.h> +#include <__atomic/atomic_waitable_traits.h> +#include <__atomic/check_memory_order.h> +#include <__atomic/floating_point_helper.h> +#include <__atomic/is_always_lock_free.h> +#include <__atomic/memory_order.h> +#include <__atomic/support.h> +#include <__config> +#include <__cstddef/ptrdiff_t.h> +#include <__memory/addressof.h> +#include <__type_traits/enable_if.h> +#include <__type_traits/is_floating_point.h> +#include <__type_traits/is_function.h> +#include <__type_traits/is_integral.h> +#include <__type_traits/is_nothrow_constructible.h> +#include <__type_traits/is_same.h> +#include <__type_traits/is_trivially_copyable.h> +#include <__type_traits/remove_pointer.h> +#include <__utility/forward.h> +#include <cstring> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value> +struct __atomic_base // false +{ + mutable __cxx_atomic_impl<_Tp> __a_; + + using value_type = _Tp; + +#if _LIBCPP_STD_VER >= 17 + static constexpr bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value; +#endif + + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const volatile _NOEXCEPT { + return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>)); + } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT { + return static_cast<__atomic_base const volatile*>(this)->is_lock_free(); + } + _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { + std::__cxx_atomic_store(std::addressof(__a_), __d, __m); + } + _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { + std::__cxx_atomic_store(std::addressof(__a_), __d, __m); + } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return std::__cxx_atomic_load(std::addressof(__a_), __m); + } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return std::__cxx_atomic_load(std::addressof(__a_), __m); + } + _LIBCPP_HIDE_FROM_ABI operator _Tp() const volatile _NOEXCEPT { return load(); } + _LIBCPP_HIDE_FROM_ABI operator _Tp() const _NOEXCEPT { return load(); } + _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f); + } + _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f); + } + _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m); + } + +#if _LIBCPP_STD_VER >= 20 + _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT { + std::__atomic_wait(*this, __v, __m); + } + _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT { + std::__atomic_wait(*this, __v, __m); + } + _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { std::__atomic_notify_one(*this); } + _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); } + _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { std::__atomic_notify_all(*this); } + _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); } +#endif // _LIBCPP_STD_VER >= 20 + +#if _LIBCPP_STD_VER >= 20 + _LIBCPP_HIDE_FROM_ABI constexpr __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {} +#else + _LIBCPP_HIDE_FROM_ABI __atomic_base() _NOEXCEPT = default; +#endif + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} + + __atomic_base(const __atomic_base&) = delete; +}; + +// atomic<Integral> + +template <class _Tp> +struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> { + using __base _LIBCPP_NODEBUG = __atomic_base<_Tp, false>; + + using difference_type = typename __base::value_type; + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __atomic_base() _NOEXCEPT = default; + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) volatile _NOEXCEPT { return fetch_add(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) _NOEXCEPT { return fetch_add(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) volatile _NOEXCEPT { return fetch_sub(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) _NOEXCEPT { return fetch_sub(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator++() volatile _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator++() _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--() volatile _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--() _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) _NOEXCEPT { return fetch_add(__op) + __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) _NOEXCEPT { return fetch_sub(__op) - __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) volatile _NOEXCEPT { return fetch_and(__op) & __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) _NOEXCEPT { return fetch_and(__op) & __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) volatile _NOEXCEPT { return fetch_or(__op) | __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) _NOEXCEPT { return fetch_or(__op) | __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) volatile _NOEXCEPT { return fetch_xor(__op) ^ __op; } + _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; } +}; + +#if _LIBCPP_STD_VER >= 20 +// Here we need _IsIntegral because the default template argument is not enough +// e.g __atomic_base<int> is __atomic_base<int, true>, which inherits from +// __atomic_base<int, false> and the caller of the wait function is +// __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work +template <class _Tp, bool _IsIntegral> +struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > { + using __value_type _LIBCPP_NODEBUG = _Tp; + + static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) { + return __a.load(__order); + } + + static _LIBCPP_HIDE_FROM_ABI _Tp + __atomic_load(const volatile __atomic_base<_Tp, _IsIntegral>& __this, memory_order __order) { + return __this.load(__order); + } + + static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_Tp>* + __atomic_contention_address(const __atomic_base<_Tp, _IsIntegral>& __a) { + return std::addressof(__a.__a_); + } + + static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_Tp>* + __atomic_contention_address(const volatile __atomic_base<_Tp, _IsIntegral>& __this) { + return std::addressof(__this.__a_); + } +}; + +#endif // _LIBCPP_STD_VER >= 20 + +template <typename _Tp> +struct __check_atomic_mandates { + using type _LIBCPP_NODEBUG = _Tp; + static_assert(is_trivially_copyable<_Tp>::value, "std::atomic<T> requires that 'T' be a trivially copyable type"); +}; + +template <class _Tp> +struct atomic : public __atomic_base<typename __check_atomic_mandates<_Tp>::type> { + using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>; + +#if _LIBCPP_STD_VER >= 20 + _LIBCPP_HIDE_FROM_ABI atomic() = default; +#else + _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default; +#endif + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} + + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile _NOEXCEPT { + __base::store(__d); + return __d; + } + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) _NOEXCEPT { + __base::store(__d); + return __d; + } + + atomic& operator=(const atomic&) = delete; + atomic& operator=(const atomic&) volatile = delete; +}; + +// atomic<T*> + +template <class _Tp> +struct atomic<_Tp*> : public __atomic_base<_Tp*> { + using __base _LIBCPP_NODEBUG = __atomic_base<_Tp*>; + + using difference_type = ptrdiff_t; + + _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default; + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} + + _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) volatile _NOEXCEPT { + __base::store(__d); + return __d; + } + _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) _NOEXCEPT { + __base::store(__d); + return __d; + } + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + // __atomic_fetch_add accepts function pointers, guard against them. + static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); + return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + // __atomic_fetch_add accepts function pointers, guard against them. + static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); + return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + // __atomic_fetch_sub accepts function pointers, guard against them. + static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); + return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { + // __atomic_fetch_sub accepts function pointers, guard against them. + static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); + return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) volatile _NOEXCEPT { return fetch_add(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) _NOEXCEPT { return fetch_add(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) volatile _NOEXCEPT { return fetch_sub(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) _NOEXCEPT { return fetch_sub(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator++() volatile _NOEXCEPT { return fetch_add(1) + 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator++() _NOEXCEPT { return fetch_add(1) + 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--() volatile _NOEXCEPT { return fetch_sub(1) - 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--() _NOEXCEPT { return fetch_sub(1) - 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT { return fetch_add(__op) + __op; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT { return fetch_sub(__op) - __op; } + + atomic& operator=(const atomic&) = delete; + atomic& operator=(const atomic&) volatile = delete; +}; + +#if _LIBCPP_STD_VER >= 20 +template <class _Tp> +struct __atomic_waitable_traits<atomic<_Tp> > : __atomic_waitable_traits<__atomic_base<_Tp> > {}; + +template <class _Tp> + requires is_floating_point_v<_Tp> +struct atomic<_Tp> : __atomic_base<_Tp> { +private: + template <class _This, class _Operation, class _BuiltinOp> + _LIBCPP_HIDE_FROM_ABI static _Tp + __rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) { + if constexpr (std::__has_rmw_builtin<_Tp>()) { + return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m); + } else { + _Tp __old = __self.load(memory_order_relaxed); + _Tp __new = __operation(__old, __operand); + while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) { +# ifdef _LIBCPP_COMPILER_CLANG_BASED + if constexpr (std::__is_fp80_long_double<_Tp>()) { + // https://llvm.org/PR47978 + // clang bug: __old is not updated on failure for atomic<long double>::compare_exchange_weak + // Note __old = __self.load(memory_order_relaxed) will not work + std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), std::addressof(__old), memory_order_relaxed); + } +# endif + __new = __operation(__old, __operand); + } + return __old; + } + } + + template <class _This> + _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) { + auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) { + return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order); + }; + auto __plus = [](auto __a, auto __b) { return __a + __b; }; + return __rmw_op(std::forward<_This>(__self), __operand, __m, __plus, __builtin_op); + } + + template <class _This> + _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) { + auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) { + return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order); + }; + auto __minus = [](auto __a, auto __b) { return __a - __b; }; + return __rmw_op(std::forward<_This>(__self), __operand, __m, __minus, __builtin_op); + } + +public: + using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>; + using value_type = _Tp; + using difference_type = value_type; + + _LIBCPP_HIDE_FROM_ABI constexpr atomic() noexcept = default; + _LIBCPP_HIDE_FROM_ABI constexpr atomic(_Tp __d) noexcept : __base(__d) {} + + atomic(const atomic&) = delete; + atomic& operator=(const atomic&) = delete; + atomic& operator=(const atomic&) volatile = delete; + + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile noexcept + requires __base::is_always_lock_free + { + __base::store(__d); + return __d; + } + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) noexcept { + __base::store(__d); + return __d; + } + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept + requires __base::is_always_lock_free + { + return __fetch_add(*this, __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept { + return __fetch_add(*this, __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept + requires __base::is_always_lock_free + { + return __fetch_sub(*this, __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept { + return __fetch_sub(*this, __op, __m); + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile noexcept + requires __base::is_always_lock_free + { + return fetch_add(__op) + __op; + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; } + + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile noexcept + requires __base::is_always_lock_free + { + return fetch_sub(__op) - __op; + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; } +}; + +#endif // _LIBCPP_STD_VER >= 20 + +// atomic_is_lock_free + +template <class _Tp> +[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT { + return __o->is_lock_free(); +} + +template <class _Tp> +[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT { + return __o->is_lock_free(); +} + +// atomic_init + +template <class _Tp> +_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void +atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + std::__cxx_atomic_init(std::addressof(__o->__a_), __d); +} + +template <class _Tp> +_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void +atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + std::__cxx_atomic_init(std::addressof(__o->__a_), __d); +} + +// atomic_store + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + __o->store(__d); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + __o->store(__d); +} + +// atomic_store_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { + __o->store(__d, __m); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) { + __o->store(__d, __m); +} + +// atomic_load + +template <class _Tp> +[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT { + return __o->load(); +} + +template <class _Tp> +[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const atomic<_Tp>* __o) _NOEXCEPT { + return __o->load(); +} + +// atomic_load_explicit + +template <class _Tp> +[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp +atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return __o->load(__m); +} + +template <class _Tp> +[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return __o->load(__m); +} + +// atomic_exchange + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->exchange(__d); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->exchange(__d); +} + +// atomic_exchange_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT { + return __o->exchange(__d, __m); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT { + return __o->exchange(__d, __m); +} + +// atomic_compare_exchange_weak + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->compare_exchange_weak(*__e, __d); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak( + atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->compare_exchange_weak(*__e, __d); +} + +// atomic_compare_exchange_strong + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->compare_exchange_strong(*__e, __d); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong( + atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT { + return __o->compare_exchange_strong(*__e, __d); +} + +// atomic_compare_exchange_weak_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit( + volatile atomic<_Tp>* __o, + typename atomic<_Tp>::value_type* __e, + typename atomic<_Tp>::value_type __d, + memory_order __s, + memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return __o->compare_exchange_weak(*__e, __d, __s, __f); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit( + atomic<_Tp>* __o, + typename atomic<_Tp>::value_type* __e, + typename atomic<_Tp>::value_type __d, + memory_order __s, + memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return __o->compare_exchange_weak(*__e, __d, __s, __f); +} + +// atomic_compare_exchange_strong_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit( + volatile atomic<_Tp>* __o, + typename atomic<_Tp>::value_type* __e, + typename atomic<_Tp>::value_type __d, + memory_order __s, + memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return __o->compare_exchange_strong(*__e, __d, __s, __f); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit( + atomic<_Tp>* __o, + typename atomic<_Tp>::value_type* __e, + typename atomic<_Tp>::value_type __d, + memory_order __s, + memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) { + return __o->compare_exchange_strong(*__e, __d, __s, __f); +} + +#if _LIBCPP_STD_VER >= 20 + +// atomic_wait + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +atomic_wait(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT { + return __o->wait(__v); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void atomic_wait(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT { + return __o->wait(__v); +} + +// atomic_wait_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +atomic_wait_explicit(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return __o->wait(__v, __m); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +atomic_wait_explicit(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) { + return __o->wait(__v, __m); +} + +// atomic_notify_one + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT { + __o->notify_one(); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT { + __o->notify_one(); +} + +// atomic_notify_all + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT { + __o->notify_all(); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT { + __o->notify_all(); +} + +#endif // _LIBCPP_STD_VER >= 20 + +// atomic_fetch_add + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { + return __o->fetch_add(__op); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { + return __o->fetch_add(__op); +} + +// atomic_fetch_add_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add_explicit( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_add(__op, __m); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_add(__op, __m); +} + +// atomic_fetch_sub + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { + return __o->fetch_sub(__op); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT { + return __o->fetch_sub(__op); +} + +// atomic_fetch_sub_explicit + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub_explicit( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_sub(__op, __m); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_sub(__op, __m); +} + +// atomic_fetch_and + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_and(__op); +} + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_and(__op); +} + +// atomic_fetch_and_explicit + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and_explicit( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_and(__op, __m); +} + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_and(__op, __m); +} + +// atomic_fetch_or + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_or(__op); +} + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_or(__op); +} + +// atomic_fetch_or_explicit + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_or(__op, __m); +} + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_or(__op, __m); +} + +// atomic_fetch_xor + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_xor(__op); +} + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT { + return __o->fetch_xor(__op); +} + +// atomic_fetch_xor_explicit + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor_explicit( + volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_xor(__op, __m); +} + +template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _Tp +atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT { + return __o->fetch_xor(__op, __m); +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_ATOMIC_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/atomic_flag.h b/third_party/libcxx/include/c++/v1/__atomic/atomic_flag.h new file mode 100644 index 0000000..7c09870 --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/atomic_flag.h @@ -0,0 +1,165 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_FLAG_H +#define _LIBCPP___ATOMIC_ATOMIC_FLAG_H + +#include <__atomic/atomic_sync.h> +#include <__atomic/atomic_waitable_traits.h> +#include <__atomic/memory_order.h> +#include <__atomic/support.h> +#include <__config> +#include <__memory/addressof.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +struct atomic_flag { + __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; + + _LIBCPP_HIDE_FROM_ABI bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT { + return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m); + } + _LIBCPP_HIDE_FROM_ABI bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT { + return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m); + } + + _LIBCPP_HIDE_FROM_ABI bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m); + } + _LIBCPP_HIDE_FROM_ABI bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT { + return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m); + } + _LIBCPP_HIDE_FROM_ABI void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { + __cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m); + } + _LIBCPP_HIDE_FROM_ABI void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT { + __cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m); + } + +#if _LIBCPP_STD_VER >= 20 + _LIBCPP_HIDE_FROM_ABI void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT { + std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m); + } + _LIBCPP_HIDE_FROM_ABI void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT { + std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m); + } + _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { std::__atomic_notify_one(*this); } + _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); } + _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { std::__atomic_notify_all(*this); } + _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); } +#endif + +#if _LIBCPP_STD_VER >= 20 + _LIBCPP_HIDE_FROM_ABI constexpr atomic_flag() _NOEXCEPT : __a_(false) {} +#else + atomic_flag() _NOEXCEPT = default; +#endif + + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION + + atomic_flag(const atomic_flag&) = delete; + atomic_flag& operator=(const atomic_flag&) = delete; + atomic_flag& operator=(const atomic_flag&) volatile = delete; +}; + +#if _LIBCPP_STD_VER >= 20 +template <> +struct __atomic_waitable_traits<atomic_flag> { + using __value_type _LIBCPP_NODEBUG = _LIBCPP_ATOMIC_FLAG_TYPE; + + static _LIBCPP_HIDE_FROM_ABI _LIBCPP_ATOMIC_FLAG_TYPE __atomic_load(const atomic_flag& __a, memory_order __order) { + return std::__cxx_atomic_load(&__a.__a_, __order); + } + + static _LIBCPP_HIDE_FROM_ABI _LIBCPP_ATOMIC_FLAG_TYPE + __atomic_load(const volatile atomic_flag& __a, memory_order __order) { + return std::__cxx_atomic_load(&__a.__a_, __order); + } + + static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE>* + __atomic_contention_address(const atomic_flag& __a) { + return std::addressof(__a.__a_); + } + + static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE>* + __atomic_contention_address(const volatile atomic_flag& __a) { + return std::addressof(__a.__a_); + } +}; +#endif // _LIBCPP_STD_VER >= 20 + +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT { return __o->test(); } + +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const atomic_flag* __o) _NOEXCEPT { return __o->test(); } + +inline _LIBCPP_HIDE_FROM_ABI bool +atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT { + return __o->test(__m); +} + +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT { + return __o->test(__m); +} + +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT { + return __o->test_and_set(); +} + +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT { return __o->test_and_set(); } + +inline _LIBCPP_HIDE_FROM_ABI bool +atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT { + return __o->test_and_set(__m); +} + +inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT { + return __o->test_and_set(__m); +} + +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT { __o->clear(); } + +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear(atomic_flag* __o) _NOEXCEPT { __o->clear(); } + +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT { + __o->clear(__m); +} + +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT { + __o->clear(__m); +} + +#if _LIBCPP_STD_VER >= 20 +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT { + __o->wait(__v); +} + +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT { __o->wait(__v); } + +inline _LIBCPP_HIDE_FROM_ABI void +atomic_flag_wait_explicit(const volatile atomic_flag* __o, bool __v, memory_order __m) _NOEXCEPT { + __o->wait(__v, __m); +} + +inline _LIBCPP_HIDE_FROM_ABI void +atomic_flag_wait_explicit(const atomic_flag* __o, bool __v, memory_order __m) _NOEXCEPT { + __o->wait(__v, __m); +} + +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT { __o->notify_one(); } +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT { __o->notify_one(); } +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT { __o->notify_all(); } +inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT { __o->notify_all(); } +#endif // _LIBCPP_STD_VER >= 20 + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_ATOMIC_FLAG_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/atomic_init.h b/third_party/libcxx/include/c++/v1/__atomic/atomic_init.h new file mode 100644 index 0000000..8e86ba3 --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/atomic_init.h @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_INIT_H +#define _LIBCPP___ATOMIC_ATOMIC_INIT_H + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#define ATOMIC_FLAG_INIT {false} +#define ATOMIC_VAR_INIT(__v) {__v} + +#if _LIBCPP_STD_VER >= 20 && defined(_LIBCPP_COMPILER_CLANG_BASED) && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS) +# pragma clang deprecated(ATOMIC_VAR_INIT) +#endif + +#endif // _LIBCPP___ATOMIC_ATOMIC_INIT_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/atomic_lock_free.h b/third_party/libcxx/include/c++/v1/__atomic/atomic_lock_free.h new file mode 100644 index 0000000..3ae9b88 --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/atomic_lock_free.h @@ -0,0 +1,48 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_LOCK_FREE_H +#define _LIBCPP___ATOMIC_ATOMIC_LOCK_FREE_H + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) +# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE +# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE +# if _LIBCPP_HAS_CHAR8_T +# define ATOMIC_CHAR8_T_LOCK_FREE __CLANG_ATOMIC_CHAR8_T_LOCK_FREE +# endif +# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE +# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE +# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE +# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE +# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE +# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE +# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE +# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE +#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE) +# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE +# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE +# if _LIBCPP_HAS_CHAR8_T +# define ATOMIC_CHAR8_T_LOCK_FREE __GCC_ATOMIC_CHAR8_T_LOCK_FREE +# endif +# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE +# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE +# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE +# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE +# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE +# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE +# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE +# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE +#endif + +#endif // _LIBCPP___ATOMIC_ATOMIC_LOCK_FREE_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/atomic_ref.h b/third_party/libcxx/include/c++/v1/__atomic/atomic_ref.h new file mode 100644 index 0000000..c551f99 --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/atomic_ref.h @@ -0,0 +1,398 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +//===---------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_REF_H +#define _LIBCPP___ATOMIC_ATOMIC_REF_H + +#include <__assert> +#include <__atomic/atomic_sync.h> +#include <__atomic/atomic_waitable_traits.h> +#include <__atomic/check_memory_order.h> +#include <__atomic/floating_point_helper.h> +#include <__atomic/memory_order.h> +#include <__atomic/to_gcc_order.h> +#include <__concepts/arithmetic.h> +#include <__concepts/same_as.h> +#include <__config> +#include <__cstddef/byte.h> +#include <__cstddef/ptrdiff_t.h> +#include <__memory/addressof.h> +#include <__memory/is_sufficiently_aligned.h> +#include <__type_traits/copy_cv.h> +#include <__type_traits/has_unique_object_representation.h> +#include <__type_traits/is_trivially_copyable.h> +#include <cstring> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if _LIBCPP_STD_VER >= 20 + +// These types are required to make __atomic_is_always_lock_free work across GCC and Clang. +// The purpose of this trick is to make sure that we provide an object with the correct alignment +// to __atomic_is_always_lock_free, since that answer depends on the alignment. +template <size_t _Alignment> +struct __alignment_checker_type { + alignas(_Alignment) char __data; +}; + +template <size_t _Alignment> +struct __get_aligner_instance { + static constexpr __alignment_checker_type<_Alignment> __instance{}; +}; + +template <class _Tp> +struct __atomic_ref_base { +private: + _LIBCPP_HIDE_FROM_ABI static _Tp* __clear_padding(_Tp& __val) noexcept { + _Tp* __ptr = std::addressof(__val); +# if __has_builtin(__builtin_clear_padding) + __builtin_clear_padding(__ptr); +# endif + return __ptr; + } + + _LIBCPP_HIDE_FROM_ABI static bool __compare_exchange( + _Tp* __ptr, _Tp* __expected, _Tp* __desired, bool __is_weak, int __success, int __failure) noexcept { + if constexpr ( +# if __has_builtin(__builtin_clear_padding) + has_unique_object_representations_v<_Tp> || floating_point<_Tp> +# else + true // NOLINT(readability-simplify-boolean-expr) +# endif + ) { + return __atomic_compare_exchange(__ptr, __expected, __desired, __is_weak, __success, __failure); + } else { // _Tp has padding bits and __builtin_clear_padding is available + __clear_padding(*__desired); + _Tp __copy = *__expected; + __clear_padding(__copy); + // The algorithm we use here is basically to perform `__atomic_compare_exchange` on the + // values until it has either succeeded, or failed because the value representation of the + // objects involved was different. This is why we loop around __atomic_compare_exchange: + // we basically loop until its failure is caused by the value representation of the objects + // being different, not only their object representation. + while (true) { + _Tp __prev = __copy; + if (__atomic_compare_exchange(__ptr, std::addressof(__copy), __desired, __is_weak, __success, __failure)) { + return true; + } + _Tp __curr = __copy; + if (std::memcmp(__clear_padding(__prev), __clear_padding(__curr), sizeof(_Tp)) != 0) { + // Value representation without padding bits do not compare equal -> + // write the current content of *ptr into *expected + std::memcpy(__expected, std::addressof(__copy), sizeof(_Tp)); + return false; + } + } + } + } + + friend struct __atomic_waitable_traits<__atomic_ref_base<_Tp>>; + + // require types that are 1, 2, 4, 8, or 16 bytes in length to be aligned to at least their size to be potentially + // used lock-free + static constexpr size_t __min_alignment = (sizeof(_Tp) & (sizeof(_Tp) - 1)) || (sizeof(_Tp) > 16) ? 0 : sizeof(_Tp); + +public: + using value_type = _Tp; + + static constexpr size_t required_alignment = alignof(_Tp) > __min_alignment ? alignof(_Tp) : __min_alignment; + + // The __atomic_always_lock_free builtin takes into account the alignment of the pointer if provided, + // so we create a fake pointer with a suitable alignment when querying it. Note that we are guaranteed + // that the pointer is going to be aligned properly at runtime because that is a (checked) precondition + // of atomic_ref's constructor. + static constexpr bool is_always_lock_free = + __atomic_always_lock_free(sizeof(_Tp), std::addressof(__get_aligner_instance<required_alignment>::__instance)); + + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { + return __atomic_is_lock_free(sizeof(_Tp), __ptr_); + } + + _LIBCPP_HIDE_FROM_ABI void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept + _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst, + "atomic_ref: memory order argument to atomic store operation is invalid"); + __atomic_store(__ptr_, __clear_padding(__desired), std::__to_gcc_order(__order)); + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { + store(__desired); + return __desired; + } + + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __order = memory_order::seq_cst) const noexcept + _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__order) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire || + __order == memory_order::seq_cst, + "atomic_ref: memory order argument to atomic load operation is invalid"); + alignas(_Tp) byte __mem[sizeof(_Tp)]; + auto* __ret = reinterpret_cast<_Tp*>(__mem); + __atomic_load(__ptr_, __ret, std::__to_gcc_order(__order)); + return *__ret; + } + + _LIBCPP_HIDE_FROM_ABI operator _Tp() const noexcept { return load(); } + + _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept { + alignas(_Tp) byte __mem[sizeof(_Tp)]; + auto* __ret = reinterpret_cast<_Tp*>(__mem); + __atomic_exchange(__ptr_, __clear_padding(__desired), __ret, std::__to_gcc_order(__order)); + return *__ret; + } + + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + __failure == memory_order::relaxed || __failure == memory_order::consume || + __failure == memory_order::acquire || __failure == memory_order::seq_cst, + "atomic_ref: failure memory order argument to weak atomic compare-and-exchange operation is invalid"); + return __compare_exchange( + __ptr_, + std::addressof(__expected), + std::addressof(__desired), + true, + std::__to_gcc_order(__success), + std::__to_gcc_order(__failure)); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept + _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + __failure == memory_order::relaxed || __failure == memory_order::consume || + __failure == memory_order::acquire || __failure == memory_order::seq_cst, + "atomic_ref: failure memory order argument to strong atomic compare-and-exchange operation is invalid"); + return __compare_exchange( + __ptr_, + std::addressof(__expected), + std::addressof(__desired), + false, + std::__to_gcc_order(__success), + std::__to_gcc_order(__failure)); + } + + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept { + return __compare_exchange( + __ptr_, + std::addressof(__expected), + std::addressof(__desired), + true, + std::__to_gcc_order(__order), + std::__to_gcc_failure_order(__order)); + } + _LIBCPP_HIDE_FROM_ABI bool + compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept { + return __compare_exchange( + __ptr_, + std::addressof(__expected), + std::addressof(__desired), + false, + std::__to_gcc_order(__order), + std::__to_gcc_failure_order(__order)); + } + + _LIBCPP_HIDE_FROM_ABI void wait(_Tp __old, memory_order __order = memory_order::seq_cst) const noexcept + _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__order) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire || + __order == memory_order::seq_cst, + "atomic_ref: memory order argument to atomic wait operation is invalid"); + std::__atomic_wait(*this, __old, __order); + } + _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { std::__atomic_notify_one(*this); } + _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { std::__atomic_notify_all(*this); } +# if _LIBCPP_STD_VER >= 26 + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __copy_cv_t<_Tp, void>* address() const noexcept { return __ptr_; } +# endif + +protected: + using _Aligned_Tp [[__gnu__::__aligned__(required_alignment), __gnu__::__nodebug__]] = _Tp; + _Aligned_Tp* __ptr_; + + _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __ptr_(std::addressof(__obj)) {} +}; + +template <class _Tp> +struct __atomic_waitable_traits<__atomic_ref_base<_Tp>> { + using __value_type _LIBCPP_NODEBUG = _Tp; + + static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_ref_base<_Tp>& __a, memory_order __order) { + return __a.load(__order); + } + static _LIBCPP_HIDE_FROM_ABI const _Tp* __atomic_contention_address(const __atomic_ref_base<_Tp>& __a) { + return __a.__ptr_; + } +}; + +template <class _Tp> +struct atomic_ref : public __atomic_ref_base<_Tp> { + static_assert(is_trivially_copyable_v<_Tp>, "std::atomic_ref<T> requires that 'T' be a trivially copyable type"); + + using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>; + + _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + std::__is_sufficiently_aligned<__base::required_alignment>(std::addressof(__obj)), + "atomic_ref ctor: referenced object must be aligned to required_alignment"); + } + + _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default; + + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); } + + atomic_ref& operator=(const atomic_ref&) = delete; +}; + +template <class _Tp> + requires(std::integral<_Tp> && !std::same_as<bool, _Tp>) +struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> { + using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>; + + using difference_type = __base::value_type; + + _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + std::__is_sufficiently_aligned<__base::required_alignment>(std::addressof(__obj)), + "atomic_ref ctor: referenced object must be aligned to required_alignment"); + } + + _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default; + + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); } + + atomic_ref& operator=(const atomic_ref&) = delete; + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order)); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_sub(this->__ptr_, __arg, std::__to_gcc_order(__order)); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_and(this->__ptr_, __arg, std::__to_gcc_order(__order)); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_or(this->__ptr_, __arg, std::__to_gcc_order(__order)); + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_xor(this->__ptr_, __arg, std::__to_gcc_order(__order)); + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) const noexcept { return fetch_add(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) const noexcept { return fetch_sub(_Tp(1)); } + _LIBCPP_HIDE_FROM_ABI _Tp operator++() const noexcept { return fetch_add(_Tp(1)) + _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator--() const noexcept { return fetch_sub(_Tp(1)) - _Tp(1); } + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; } + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; } + _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __arg) const noexcept { return fetch_and(__arg) & __arg; } + _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __arg) const noexcept { return fetch_or(__arg) | __arg; } + _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __arg) const noexcept { return fetch_xor(__arg) ^ __arg; } +}; + +template <class _Tp> + requires std::floating_point<_Tp> +struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> { + using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>; + + using difference_type = __base::value_type; + + _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) { + _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( + std::__is_sufficiently_aligned<__base::required_alignment>(std::addressof(__obj)), + "atomic_ref ctor: referenced object must be aligned to required_alignment"); + } + + _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default; + + _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); } + + atomic_ref& operator=(const atomic_ref&) = delete; + + _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + if constexpr (std::__has_rmw_builtin<_Tp>()) { + return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order)); + } else { + _Tp __old = this->load(memory_order_relaxed); + _Tp __new = __old + __arg; + while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) { + __new = __old + __arg; + } + return __old; + } + } + _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept { + if constexpr (std::__has_rmw_builtin<_Tp>()) { + return __atomic_fetch_sub(this->__ptr_, __arg, std::__to_gcc_order(__order)); + } else { + _Tp __old = this->load(memory_order_relaxed); + _Tp __new = __old - __arg; + while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) { + __new = __old - __arg; + } + return __old; + } + } + + _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; } + _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; } +}; + +template <class _Tp> +struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> { + using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp*>; + + using difference_type = ptrdiff_t; + + _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp*& __ptr) : __base(__ptr) {} + + _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __desired) const noexcept { return __base::operator=(__desired); } + + atomic_ref& operator=(const atomic_ref&) = delete; + + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_add(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order)); + } + _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept { + return __atomic_fetch_sub(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order)); + } + + _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) const noexcept { return fetch_add(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) const noexcept { return fetch_sub(1); } + _LIBCPP_HIDE_FROM_ABI _Tp* operator++() const noexcept { return fetch_add(1) + 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator--() const noexcept { return fetch_sub(1) - 1; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __arg) const noexcept { return fetch_add(__arg) + __arg; } + _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __arg) const noexcept { return fetch_sub(__arg) - __arg; } +}; + +_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(atomic_ref); + +#endif // _LIBCPP_STD_VER >= 20 + +_LIBCPP_END_NAMESPACE_STD + +_LIBCPP_POP_MACROS + +#endif // _LIBCPP__ATOMIC_ATOMIC_REF_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/atomic_sync.h b/third_party/libcxx/include/c++/v1/__atomic/atomic_sync.h new file mode 100644 index 0000000..12bd1b9 --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/atomic_sync.h @@ -0,0 +1,261 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_H +#define _LIBCPP___ATOMIC_ATOMIC_SYNC_H + +#include <__atomic/atomic_waitable_traits.h> +#include <__atomic/contention_t.h> +#include <__atomic/memory_order.h> +#include <__chrono/duration.h> +#include <__config> +#include <__memory/addressof.h> +#include <__thread/poll_with_backoff.h> +#include <__type_traits/decay.h> +#include <cstring> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if _LIBCPP_STD_VER >= 20 +# if _LIBCPP_HAS_THREADS + +# if !_LIBCPP_AVAILABILITY_HAS_NEW_SYNC + +// old dylib interface kept for backwards compatibility +_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*) _NOEXCEPT; +_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*) _NOEXCEPT; +_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*) _NOEXCEPT; +_LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t) _NOEXCEPT; + +_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*) _NOEXCEPT; +_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*) _NOEXCEPT; +_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t +__libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*) _NOEXCEPT; +_LIBCPP_EXPORTED_FROM_ABI void +__libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t) _NOEXCEPT; +# endif // !_LIBCPP_AVAILABILITY_HAS_NEW_SYNC + +// new dylib interface + +// return the global contention state's current value for the address +_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t +__atomic_monitor_global(void const* __address) _NOEXCEPT; + +// wait on the global contention state to be changed from the given value for the address +_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void +__atomic_wait_global_table(void const* __address, __cxx_contention_t __monitor_value) _NOEXCEPT; + +// notify one waiter waiting on the global contention state for the address +_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_global_table(void const*) _NOEXCEPT; + +// notify all waiters waiting on the global contention state for the address +_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_global_table(void const*) _NOEXCEPT; + +// wait on the address directly with the native platform wait +template <std::size_t _Size> +_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void +__atomic_wait_native(void const* __address, void const* __old_value) _NOEXCEPT; + +// notify one waiter waiting on the address directly with the native platform wait +template <std::size_t _Size> +_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native(const void*) _NOEXCEPT; + +// notify all waiters waiting on the address directly with the native platform wait +template <std::size_t _Size> +_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(const void*) _NOEXCEPT; + +# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC + +template <class _AtomicWaitable, class _Poll> +struct __atomic_wait_backoff_impl { + const _AtomicWaitable& __a_; + _Poll __poll_; + memory_order __order_; + + using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >; + using __value_type _LIBCPP_NODEBUG = typename __waitable_traits::__value_type; + + _LIBCPP_HIDE_FROM_ABI __backoff_results operator()(chrono::nanoseconds __elapsed) const { + if (__elapsed > chrono::microseconds(4)) { + auto __contention_address = const_cast<const void*>( + static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a_))); + + if constexpr (__has_native_atomic_wait<__value_type>) { + auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_); + if (__poll_(__atomic_value)) + return __backoff_results::__poll_success; + std::__atomic_wait_native<sizeof(__value_type)>(__contention_address, std::addressof(__atomic_value)); + } else { + __cxx_contention_t __monitor_val = std::__atomic_monitor_global(__contention_address); + auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_); + if (__poll_(__atomic_value)) + return __backoff_results::__poll_success; + std::__atomic_wait_global_table(__contention_address, __monitor_val); + } + } else { + } // poll + return __backoff_results::__continue_poll; + } +}; + +# else // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC + +template <class _AtomicWaitable, class _Poll> +struct __atomic_wait_backoff_impl { + const _AtomicWaitable& __a_; + _Poll __poll_; + memory_order __order_; + + using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >; + + _LIBCPP_HIDE_FROM_ABI bool + __update_monitor_val_and_poll(__cxx_atomic_contention_t const volatile*, __cxx_contention_t& __monitor_val) const { + // In case the contention type happens to be __cxx_atomic_contention_t, i.e. __cxx_atomic_impl<int64_t>, + // the platform wait is directly monitoring the atomic value itself. + // `__poll_` takes the current value of the atomic as an in-out argument + // to potentially modify it. After it returns, `__monitor` has a value + // which can be safely waited on by `std::__libcpp_atomic_wait` without any + // ABA style issues. + __monitor_val = __waitable_traits::__atomic_load(__a_, __order_); + return __poll_(__monitor_val); + } + + _LIBCPP_HIDE_FROM_ABI bool + __update_monitor_val_and_poll(void const volatile* __contention_address, __cxx_contention_t& __monitor_val) const { + // In case the contention type is anything else, platform wait is monitoring a __cxx_atomic_contention_t + // from the global pool, the monitor comes from __libcpp_atomic_monitor + __monitor_val = std::__libcpp_atomic_monitor(__contention_address); + auto __current_val = __waitable_traits::__atomic_load(__a_, __order_); + return __poll_(__current_val); + } + + _LIBCPP_HIDE_FROM_ABI __backoff_results operator()(chrono::nanoseconds __elapsed) const { + if (__elapsed > chrono::microseconds(4)) { + auto __contention_address = __waitable_traits::__atomic_contention_address(__a_); + __cxx_contention_t __monitor_val; + if (__update_monitor_val_and_poll(__contention_address, __monitor_val)) + return __backoff_results::__poll_success; + std::__libcpp_atomic_wait(__contention_address, __monitor_val); + } else { + } // poll + return __backoff_results::__continue_poll; + } +}; + +# endif // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC + +// The semantics of this function are similar to `atomic`'s +// `.wait(T old, std::memory_order order)`, but instead of having a hardcoded +// predicate (is the loaded value unequal to `old`?), the predicate function is +// specified as an argument. The loaded value is given as an in-out argument to +// the predicate. If the predicate function returns `true`, +// `__atomic_wait_unless` will return. If the predicate function returns +// `false`, it must set the argument to its current understanding of the atomic +// value. The predicate function must not return `false` spuriously. +template <class _AtomicWaitable, class _Poll> +_LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, memory_order __order, _Poll&& __poll) { + static_assert(__atomic_waitable<_AtomicWaitable>); + __atomic_wait_backoff_impl<_AtomicWaitable, __decay_t<_Poll> > __backoff_fn = {__a, __poll, __order}; + std::__libcpp_thread_poll_with_backoff( + /* poll */ + [&]() { + auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order); + return __poll(__current_val); + }, + /* backoff */ __backoff_fn); +} + +# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC + +template <class _AtomicWaitable> +_LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) { + static_assert(__atomic_waitable<_AtomicWaitable>); + using __value_type _LIBCPP_NODEBUG = typename __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__value_type; + using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >; + auto __contention_address = + const_cast<const void*>(static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a))); + if constexpr (__has_native_atomic_wait<__value_type>) { + std::__atomic_notify_one_native<sizeof(__value_type)>(__contention_address); + } else { + std::__atomic_notify_one_global_table(__contention_address); + } +} + +template <class _AtomicWaitable> +_LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) { + static_assert(__atomic_waitable<_AtomicWaitable>); + using __value_type _LIBCPP_NODEBUG = typename __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__value_type; + using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >; + auto __contention_address = + const_cast<const void*>(static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a))); + if constexpr (__has_native_atomic_wait<__value_type>) { + std::__atomic_notify_all_native<sizeof(__value_type)>(__contention_address); + } else { + std::__atomic_notify_all_global_table(__contention_address); + } +} + +# else // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC + +template <class _AtomicWaitable> +_LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) { + static_assert(__atomic_waitable<_AtomicWaitable>); + std::__cxx_atomic_notify_one(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a)); +} + +template <class _AtomicWaitable> +_LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) { + static_assert(__atomic_waitable<_AtomicWaitable>); + std::__cxx_atomic_notify_all(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a)); +} + +# endif + +# else // _LIBCPP_HAS_THREADS + +template <class _AtomicWaitable, class _Poll> +_LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, memory_order __order, _Poll&& __poll) { + std::__libcpp_thread_poll_with_backoff( + /* poll */ + [&]() { + auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order); + return __poll(__current_val); + }, + /* backoff */ __spinning_backoff_policy()); +} + +template <class _AtomicWaitable> +_LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable&) {} + +template <class _AtomicWaitable> +_LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable&) {} + +# endif // _LIBCPP_HAS_THREADS + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { + return std::memcmp(std::addressof(__lhs), std::addressof(__rhs), sizeof(_Tp)) == 0; +} + +template <class _AtomicWaitable, class _Tp> +_LIBCPP_HIDE_FROM_ABI void __atomic_wait(_AtomicWaitable& __a, _Tp __val, memory_order __order) { + static_assert(__atomic_waitable<_AtomicWaitable>); + std::__atomic_wait_unless(__a, __order, [&](_Tp const& __current) { + return !std::__cxx_nonatomic_compare_equal(__current, __val); + }); +} + +#endif // C++20 + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/atomic_sync_timed.h b/third_party/libcxx/include/c++/v1/__atomic/atomic_sync_timed.h new file mode 100644 index 0000000..f3bf780 --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/atomic_sync_timed.h @@ -0,0 +1,136 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H +#define _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H + +#include <__atomic/atomic_waitable_traits.h> +#include <__atomic/contention_t.h> +#include <__atomic/memory_order.h> +#include <__chrono/duration.h> +#include <__config> +#include <__memory/addressof.h> +#include <__thread/poll_with_backoff.h> +#include <__thread/timed_backoff_policy.h> +#include <__type_traits/decay.h> +#include <cstdint> +#include <cstring> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if _LIBCPP_STD_VER >= 20 +# if _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_NEW_SYNC + +_LIBCPP_AVAILABILITY_NEW_SYNC +_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __atomic_monitor_global(void const* __address) _NOEXCEPT; + +// wait on the global contention state to be changed from the given value for the address +_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_global_table_with_timeout( + void const* __address, __cxx_contention_t __monitor_value, uint64_t __timeout_ns) _NOEXCEPT; + +// wait on the address directly with the native platform wait +template <std::size_t _Size> +_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void +__atomic_wait_native_with_timeout(void const* __address, void const* __old_value, uint64_t __timeout_ns) _NOEXCEPT; + +template <class _AtomicWaitable, class _Poll, class _Rep, class _Period> +struct __atomic_wait_timed_backoff_impl { + const _AtomicWaitable& __a_; + _Poll __poll_; + memory_order __order_; + chrono::duration<_Rep, _Period> __rel_time_; + + using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >; + using __value_type _LIBCPP_NODEBUG = typename __waitable_traits::__value_type; + + _LIBCPP_HIDE_FROM_ABI __backoff_results operator()(chrono::nanoseconds __elapsed) const { + if (__elapsed > chrono::microseconds(4)) { + auto __contention_address = const_cast<const void*>( + static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a_))); + + uint64_t __timeout_ns = + static_cast<uint64_t>((chrono::duration_cast<chrono::nanoseconds>(__rel_time_) - __elapsed).count()); + + if constexpr (__has_native_atomic_wait<__value_type>) { + auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_); + if (__poll_(__atomic_value)) + return __backoff_results::__poll_success; + std::__atomic_wait_native_with_timeout<sizeof(__value_type)>( + __contention_address, std::addressof(__atomic_value), __timeout_ns); + } else { + __cxx_contention_t __monitor_val = std::__atomic_monitor_global(__contention_address); + auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_); + if (__poll_(__atomic_value)) + return __backoff_results::__poll_success; + std::__atomic_wait_global_table_with_timeout(__contention_address, __monitor_val, __timeout_ns); + } + } else { + } // poll + return __backoff_results::__continue_poll; + } +}; + +// The semantics of this function are similar to `atomic`'s +// `.wait(T old, std::memory_order order)` with a timeout, but instead of having a hardcoded +// predicate (is the loaded value unequal to `old`?), the predicate function is +// specified as an argument. The loaded value is given as an in-out argument to +// the predicate. If the predicate function returns `true`, +// `__atomic_wait_unless_with_timeout` will return. If the predicate function returns +// `false`, it must set the argument to its current understanding of the atomic +// value. The predicate function must not return `false` spuriously. +template <class _AtomicWaitable, class _Poll, class _Rep, class _Period> +_LIBCPP_HIDE_FROM_ABI bool __atomic_wait_unless_with_timeout( + const _AtomicWaitable& __a, + memory_order __order, + _Poll&& __poll, + chrono::duration<_Rep, _Period> const& __rel_time) { + static_assert(__atomic_waitable<_AtomicWaitable>, ""); + __atomic_wait_timed_backoff_impl<_AtomicWaitable, __decay_t<_Poll>, _Rep, _Period> __backoff_fn = { + __a, __poll, __order, __rel_time}; + auto __poll_result = std::__libcpp_thread_poll_with_backoff( + /* poll */ + [&]() { + auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order); + return __poll(__current_val); + }, + /* backoff */ __backoff_fn, + __rel_time); + + return __poll_result == __poll_with_backoff_results::__poll_success; +} + +# elif _LIBCPP_HAS_THREADS // _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_NEW_SYNC + +template <class _AtomicWaitable, class _Poll, class _Rep, class _Period> +_LIBCPP_HIDE_FROM_ABI bool __atomic_wait_unless_with_timeout( + const _AtomicWaitable& __a, + memory_order __order, + _Poll&& __poll, + chrono::duration<_Rep, _Period> const& __rel_time) { + auto __res = std::__libcpp_thread_poll_with_backoff( + /* poll */ + [&]() { + auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order); + return __poll(__current_val); + }, + /* backoff */ __libcpp_timed_backoff_policy(), + __rel_time); + return __res == __poll_with_backoff_results::__poll_success; +} + +# endif // _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_NEW_SYNC + +#endif // C++20 + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/atomic_waitable_traits.h b/third_party/libcxx/include/c++/v1/__atomic/atomic_waitable_traits.h new file mode 100644 index 0000000..369cd2c --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/atomic_waitable_traits.h @@ -0,0 +1,101 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_WAITABLE_TRAITS_H +#define _LIBCPP___ATOMIC_ATOMIC_WAITABLE_TRAITS_H + +#include <__atomic/contention_t.h> +#include <__atomic/memory_order.h> +#include <__config> +#include <__type_traits/decay.h> +#include <__type_traits/has_unique_object_representation.h> +#include <__type_traits/is_same.h> +#include <__type_traits/is_trivially_copyable.h> +#include <cstring> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if _LIBCPP_STD_VER >= 20 + +// The customisation points to enable the following functions: +// - __atomic_wait +// - __atomic_wait_unless +// - __atomic_notify_one +// - __atomic_notify_all +template <class _Tp, class = void> +struct __atomic_waitable_traits { + using __value_type _LIBCPP_NODEBUG = void; + + template <class _AtomicWaitable> + static void __atomic_load(_AtomicWaitable&&, memory_order) = delete; + + template <class _AtomicWaitable> + static void __atomic_contention_address(_AtomicWaitable&&) = delete; +}; + +template <class _Tp> +concept __atomic_waitable = requires(const _Tp __t, memory_order __order) { + typename __atomic_waitable_traits<__decay_t<_Tp> >::__value_type; + { __atomic_waitable_traits<__decay_t<_Tp> >::__atomic_load(__t, __order) }; + { __atomic_waitable_traits<__decay_t<_Tp> >::__atomic_contention_address(__t) }; +}; + +# ifdef __linux__ +# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(4) +# elif defined(__APPLE__) +# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) \ + _APPLY(4) \ + _APPLY(8) +# elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8 +# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8) +# elif defined(_WIN32) +# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8) +# else +# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(sizeof(__cxx_contention_t)) +# endif // __linux__ + +// concepts defines the types are supported natively by the platform's wait + +# if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE) + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI constexpr bool __has_native_atomic_wait_impl() { + if (alignof(_Tp) % sizeof(_Tp) != 0) + return false; + switch (sizeof(_Tp)) { +# define _LIBCPP_MAKE_CASE(n) \ + case n: \ + return true; + _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_LIBCPP_MAKE_CASE) + default: + return false; +# undef _LIBCPP_MAKE_CASE + }; +} + +template <class _Tp> +concept __has_native_atomic_wait = + has_unique_object_representations_v<_Tp> && is_trivially_copyable_v<_Tp> && + std::__has_native_atomic_wait_impl<_Tp>(); + +# else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE + +template <class _Tp> +concept __has_native_atomic_wait = is_same_v<_Tp, __cxx_contention_t>; + +# endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE + +#endif // C++20 + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_ATOMIC_WAITABLE_TRAITS_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/check_memory_order.h b/third_party/libcxx/include/c++/v1/__atomic/check_memory_order.h new file mode 100644 index 0000000..536f764 --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/check_memory_order.h @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H +#define _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ + _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || __m == memory_order_acquire || __m == memory_order_acq_rel, \ + "memory order argument to atomic operation is invalid") + +#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \ + _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || __m == memory_order_acq_rel, \ + "memory order argument to atomic operation is invalid") + +#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \ + _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || __f == memory_order_acq_rel, \ + "memory order argument to atomic operation is invalid") + +#define _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__m) \ + _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || __m == memory_order_acq_rel, \ + "memory order argument to atomic operation is invalid") + +#endif // _LIBCPP___ATOMIC_CHECK_MEMORY_ORDER_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/contention_t.h b/third_party/libcxx/include/c++/v1/__atomic/contention_t.h new file mode 100644 index 0000000..c6d2570 --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/contention_t.h @@ -0,0 +1,56 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_CONTENTION_T_H +#define _LIBCPP___ATOMIC_CONTENTION_T_H + +#include <__atomic/support.h> +#include <__config> +#include <cstdint> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +// The original definition of `__cxx_contention_t` seemed a bit arbitrary. +// When we enable the _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE ABI, +// use definitions that are based on what the underlying platform supports +// instead. +#if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE) + +# if defined(__linux__) || defined(__Fuchsia__) +using __cxx_contention_t _LIBCPP_NODEBUG = int32_t; +# elif defined(__APPLE__) +using __cxx_contention_t _LIBCPP_NODEBUG = int64_t; +# elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8 +using __cxx_contention_t _LIBCPP_NODEBUG = int64_t; +# elif defined(_AIX) && !defined(__64BIT__) +using __cxx_contention_t _LIBCPP_NODEBUG = int32_t; +# elif defined(_WIN32) +using __cxx_contention_t _LIBCPP_NODEBUG = int64_t; +# else +using __cxx_contention_t _LIBCPP_NODEBUG = int64_t; +# endif // __linux__ || __Fuchsia__ + +#else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE + +# if defined(__linux__) || defined(__Fuchsia__) || (defined(_AIX) && !defined(__64BIT__)) +using __cxx_contention_t _LIBCPP_NODEBUG = int32_t; +# else +using __cxx_contention_t _LIBCPP_NODEBUG = int64_t; +# endif // __linux__ || __Fuchsia__ || (_AIX && !__64BIT__) + +#endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE + +using __cxx_atomic_contention_t _LIBCPP_NODEBUG = __cxx_atomic_impl<__cxx_contention_t>; + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_CONTENTION_T_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/fence.h b/third_party/libcxx/include/c++/v1/__atomic/fence.h new file mode 100644 index 0000000..0a63ced --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/fence.h @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_FENCE_H +#define _LIBCPP___ATOMIC_FENCE_H + +#include <__atomic/memory_order.h> +#include <__atomic/support.h> +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +inline _LIBCPP_HIDE_FROM_ABI void atomic_thread_fence(memory_order __m) _NOEXCEPT { __cxx_atomic_thread_fence(__m); } + +inline _LIBCPP_HIDE_FROM_ABI void atomic_signal_fence(memory_order __m) _NOEXCEPT { __cxx_atomic_signal_fence(__m); } + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_FENCE_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/floating_point_helper.h b/third_party/libcxx/include/c++/v1/__atomic/floating_point_helper.h new file mode 100644 index 0000000..8762ec2 --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/floating_point_helper.h @@ -0,0 +1,55 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_FLOATING_POINT_HELPER_H +#define _LIBCPP___ATOMIC_FLOATING_POINT_HELPER_H + +#include <__config> +#include <__type_traits/is_floating_point.h> +#include <__type_traits/is_same.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if _LIBCPP_STD_VER >= 20 + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI constexpr bool __is_fp80_long_double() { + // Only x87-fp80 long double has 64-bit mantissa + return __LDBL_MANT_DIG__ == 64 && std::is_same_v<_Tp, long double>; +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI constexpr bool __has_rmw_builtin() { + static_assert(std::is_floating_point_v<_Tp>); +# ifndef _LIBCPP_COMPILER_CLANG_BASED + return false; +# else + // The builtin __cxx_atomic_fetch_add errors during compilation for + // long double on platforms with fp80 format. + // For more details, see + // lib/Sema/SemaChecking.cpp function IsAllowedValueType + // LLVM Parser does not allow atomicrmw with x86_fp80 type. + // if (ValType->isSpecificBuiltinType(BuiltinType::LongDouble) && + // &Context.getTargetInfo().getLongDoubleFormat() == + // &llvm::APFloat::x87DoubleExtended()) + // For more info + // https://llvm.org/PR68602 + // https://reviews.llvm.org/D53965 + return !std::__is_fp80_long_double<_Tp>(); +# endif +} + +#endif + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_FLOATING_POINT_HELPER_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/is_always_lock_free.h b/third_party/libcxx/include/c++/v1/__atomic/is_always_lock_free.h new file mode 100644 index 0000000..f928e79 --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/is_always_lock_free.h @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_IS_ALWAYS_LOCK_FREE_H +#define _LIBCPP___ATOMIC_IS_ALWAYS_LOCK_FREE_H + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _Tp> +struct __libcpp_is_always_lock_free { + // __atomic_always_lock_free is available in all Standard modes + static const bool __value = __atomic_always_lock_free(sizeof(_Tp), nullptr); +}; + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_IS_ALWAYS_LOCK_FREE_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/kill_dependency.h b/third_party/libcxx/include/c++/v1/__atomic/kill_dependency.h new file mode 100644 index 0000000..103d52d --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/kill_dependency.h @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_KILL_DEPENDENCY_H +#define _LIBCPP___ATOMIC_KILL_DEPENDENCY_H + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp kill_dependency(_Tp __y) _NOEXCEPT { + return __y; +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_KILL_DEPENDENCY_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/memory_order.h b/third_party/libcxx/include/c++/v1/__atomic/memory_order.h new file mode 100644 index 0000000..3558043 --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/memory_order.h @@ -0,0 +1,65 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_MEMORY_ORDER_H +#define _LIBCPP___ATOMIC_MEMORY_ORDER_H + +#include <__config> +#include <__type_traits/is_same.h> +#include <__type_traits/underlying_type.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +// Figure out what the underlying type for `memory_order` would be if it were +// declared as an unscoped enum (accounting for -fshort-enums). Use this result +// to pin the underlying type in C++20. +enum __legacy_memory_order { __mo_relaxed, __mo_consume, __mo_acquire, __mo_release, __mo_acq_rel, __mo_seq_cst }; + +using __memory_order_underlying_t _LIBCPP_NODEBUG = __underlying_type_t<__legacy_memory_order>; + +#if _LIBCPP_STD_VER >= 20 + +enum class memory_order : __memory_order_underlying_t { + relaxed = __mo_relaxed, + consume = __mo_consume, + acquire = __mo_acquire, + release = __mo_release, + acq_rel = __mo_acq_rel, + seq_cst = __mo_seq_cst +}; + +static_assert(is_same<__underlying_type_t<memory_order>, __memory_order_underlying_t>::value, + "unexpected underlying type for std::memory_order"); + +inline constexpr auto memory_order_relaxed = memory_order::relaxed; +inline constexpr auto memory_order_consume = memory_order::consume; +inline constexpr auto memory_order_acquire = memory_order::acquire; +inline constexpr auto memory_order_release = memory_order::release; +inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; +inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; + +#else + +enum memory_order { + memory_order_relaxed = __mo_relaxed, + memory_order_consume = __mo_consume, + memory_order_acquire = __mo_acquire, + memory_order_release = __mo_release, + memory_order_acq_rel = __mo_acq_rel, + memory_order_seq_cst = __mo_seq_cst, +}; + +#endif // _LIBCPP_STD_VER >= 20 + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_MEMORY_ORDER_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/support.h b/third_party/libcxx/include/c++/v1/__atomic/support.h new file mode 100644 index 0000000..99d0f6a --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/support.h @@ -0,0 +1,121 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_SUPPORT_H +#define _LIBCPP___ATOMIC_SUPPORT_H + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +// +// This file implements base support for atomics on the platform. +// +// The following operations and types must be implemented (where _Atmc +// is __cxx_atomic_base_impl for readability): +// +// clang-format off +// +// template <class _Tp> +// struct __cxx_atomic_base_impl; +// +// #define __cxx_atomic_is_lock_free(__size) +// +// void __cxx_atomic_thread_fence(memory_order __order) noexcept; +// void __cxx_atomic_signal_fence(memory_order __order) noexcept; +// +// template <class _Tp> +// void __cxx_atomic_init(_Atmc<_Tp> volatile* __a, _Tp __val) noexcept; +// template <class _Tp> +// void __cxx_atomic_init(_Atmc<_Tp>* __a, _Tp __val) noexcept; +// +// template <class _Tp> +// void __cxx_atomic_store(_Atmc<_Tp> volatile* __a, _Tp __val, memory_order __order) noexcept; +// template <class _Tp> +// void __cxx_atomic_store(_Atmc<_Tp>* __a, _Tp __val, memory_order __order) noexcept; +// +// template <class _Tp> +// _Tp __cxx_atomic_load(_Atmc<_Tp> const volatile* __a, memory_order __order) noexcept; +// template <class _Tp> +// _Tp __cxx_atomic_load(_Atmc<_Tp> const* __a, memory_order __order) noexcept; +// +// template <class _Tp> +// void __cxx_atomic_load_inplace(_Atmc<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) noexcept; +// template <class _Tp> +// void __cxx_atomic_load_inplace(_Atmc<_Tp> const* __a, _Tp* __dst, memory_order __order) noexcept; +// +// template <class _Tp> +// _Tp __cxx_atomic_exchange(_Atmc<_Tp> volatile* __a, _Tp __value, memory_order __order) noexcept; +// template <class _Tp> +// _Tp __cxx_atomic_exchange(_Atmc<_Tp>* __a, _Tp __value, memory_order __order) noexcept; +// +// template <class _Tp> +// bool __cxx_atomic_compare_exchange_strong(_Atmc<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) noexcept; +// template <class _Tp> +// bool __cxx_atomic_compare_exchange_strong(_Atmc<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) noexcept; +// +// template <class _Tp> +// bool __cxx_atomic_compare_exchange_weak(_Atmc<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) noexcept; +// template <class _Tp> +// bool __cxx_atomic_compare_exchange_weak(_Atmc<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) noexcept; +// +// template <class _Tp> +// _Tp __cxx_atomic_fetch_add(_Atmc<_Tp> volatile* __a, _Tp __delta, memory_order __order) noexcept; +// template <class _Tp> +// _Tp __cxx_atomic_fetch_add(_Atmc<_Tp>* __a, _Tp __delta, memory_order __order) noexcept; +// +// template <class _Tp> +// _Tp* __cxx_atomic_fetch_add(_Atmc<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) noexcept; +// template <class _Tp> +// _Tp* __cxx_atomic_fetch_add(_Atmc<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) noexcept; +// +// template <class _Tp> +// _Tp __cxx_atomic_fetch_sub(_Atmc<_Tp> volatile* __a, _Tp __delta, memory_order __order) noexcept; +// template <class _Tp> +// _Tp __cxx_atomic_fetch_sub(_Atmc<_Tp>* __a, _Tp __delta, memory_order __order) noexcept; +// template <class _Tp> +// _Tp* __cxx_atomic_fetch_sub(_Atmc<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) noexcept; +// template <class _Tp> +// _Tp* __cxx_atomic_fetch_sub(_Atmc<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) noexcept; +// +// template <class _Tp> +// _Tp __cxx_atomic_fetch_and(_Atmc<_Tp> volatile* __a, _Tp __pattern, memory_order __order) noexcept; +// template <class _Tp> +// _Tp __cxx_atomic_fetch_and(_Atmc<_Tp>* __a, _Tp __pattern, memory_order __order) noexcept; +// +// template <class _Tp> +// _Tp __cxx_atomic_fetch_or(_Atmc<_Tp> volatile* __a, _Tp __pattern, memory_order __order) noexcept; +// template <class _Tp> +// _Tp __cxx_atomic_fetch_or(_Atmc<_Tp>* __a, _Tp __pattern, memory_order __order) noexcept; +// template <class _Tp> +// _Tp __cxx_atomic_fetch_xor(_Atmc<_Tp> volatile* __a, _Tp __pattern, memory_order __order) noexcept; +// template <class _Tp> +// _Tp __cxx_atomic_fetch_xor(_Atmc<_Tp>* __a, _Tp __pattern, memory_order __order) noexcept; +// +// clang-format on +// + +#if _LIBCPP_HAS_GCC_ATOMIC_IMP +# include <__atomic/support/gcc.h> +#elif _LIBCPP_HAS_C_ATOMIC_IMP +# include <__atomic/support/c11.h> +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <typename _Tp, typename _Base = __cxx_atomic_base_impl<_Tp> > +struct __cxx_atomic_impl : public _Base { + _LIBCPP_HIDE_FROM_ABI __cxx_atomic_impl() _NOEXCEPT = default; + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT : _Base(__value) {} +}; + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_SUPPORT_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/support/c11.h b/third_party/libcxx/include/c++/v1/__atomic/support/c11.h new file mode 100644 index 0000000..1ad2998 --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/support/c11.h @@ -0,0 +1,264 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_SUPPORT_C11_H +#define _LIBCPP___ATOMIC_SUPPORT_C11_H + +#include <__atomic/memory_order.h> +#include <__config> +#include <__cstddef/ptrdiff_t.h> +#include <__memory/addressof.h> +#include <__type_traits/remove_const.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +// +// This file implements support for C11-style atomics +// + +_LIBCPP_BEGIN_NAMESPACE_STD + +template <typename _Tp> +struct __cxx_atomic_base_impl { + _LIBCPP_HIDE_FROM_ABI +#ifndef _LIBCPP_CXX03_LANG + __cxx_atomic_base_impl() _NOEXCEPT = default; +#else + __cxx_atomic_base_impl() _NOEXCEPT : __a_value() { + } +#endif // _LIBCPP_CXX03_LANG + _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {} + _Atomic(_Tp) __a_value; +}; + +#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) + +_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { + __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); +} + +_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { + __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { + __c11_atomic_init(std::addressof(__a->__a_value), __val); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT { + __c11_atomic_init(std::addressof(__a->__a_value), __val); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { + __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT { + __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { + using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*; + return __c11_atomic_load( + const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { + using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*; + return __c11_atomic_load( + const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT { + using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*; + *__dst = __c11_atomic_load( + const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT { + using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*; + *__dst = __c11_atomic_load( + const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { + return __c11_atomic_exchange( + std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT { + return __c11_atomic_exchange( + std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order)); +} + +_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) { + // Avoid switch statement to make this a constexpr. + return __order == memory_order_release + ? memory_order_relaxed + : (__order == memory_order_acq_rel ? memory_order_acquire : __order); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + __cxx_atomic_base_impl<_Tp> volatile* __a, + _Tp* __expected, + _Tp __value, + memory_order __success, + memory_order __failure) _NOEXCEPT { + return __c11_atomic_compare_exchange_strong( + std::addressof(__a->__a_value), + __expected, + __value, + static_cast<__memory_order_underlying_t>(__success), + static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) + _NOEXCEPT { + return __c11_atomic_compare_exchange_strong( + std::addressof(__a->__a_value), + __expected, + __value, + static_cast<__memory_order_underlying_t>(__success), + static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + __cxx_atomic_base_impl<_Tp> volatile* __a, + _Tp* __expected, + _Tp __value, + memory_order __success, + memory_order __failure) _NOEXCEPT { + return __c11_atomic_compare_exchange_weak( + std::addressof(__a->__a_value), + __expected, + __value, + static_cast<__memory_order_underlying_t>(__success), + static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) + _NOEXCEPT { + return __c11_atomic_compare_exchange_weak( + std::addressof(__a->__a_value), + __expected, + __value, + static_cast<__memory_order_underlying_t>(__success), + static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_add( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_add( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp* +__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_add( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp* +__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_add( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_sub( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_sub( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp* +__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_sub( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp* +__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_sub( + std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_and( + std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_and( + std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_or( + std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_or( + std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); +} + +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_xor( + std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); +} +template <class _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_xor( + std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order)); +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_SUPPORT_C11_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/support/gcc.h b/third_party/libcxx/include/c++/v1/__atomic/support/gcc.h new file mode 100644 index 0000000..73c1b1c --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/support/gcc.h @@ -0,0 +1,265 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_SUPPORT_GCC_H +#define _LIBCPP___ATOMIC_SUPPORT_GCC_H + +#include <__atomic/memory_order.h> +#include <__atomic/to_gcc_order.h> +#include <__config> +#include <__memory/addressof.h> +#include <__type_traits/enable_if.h> +#include <__type_traits/is_assignable.h> +#include <__type_traits/remove_const.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +// +// This file implements support for GCC-style atomics +// + +_LIBCPP_BEGIN_NAMESPACE_STD + +// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because +// the default operator= in an object is not volatile, a byte-by-byte copy +// is required. +template <typename _Tp, typename _Tv, __enable_if_t<is_assignable<_Tp&, _Tv>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { + __a_value = __val; +} +template <typename _Tp, typename _Tv, __enable_if_t<is_assignable<_Tp&, _Tv>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { + volatile char* __to = reinterpret_cast<volatile char*>(std::addressof(__a_value)); + volatile char* __end = __to + sizeof(_Tp); + volatile const char* __from = reinterpret_cast<volatile const char*>(std::addressof(__val)); + while (__to != __end) + *__to++ = *__from++; +} + +template <typename _Tp> +struct __cxx_atomic_base_impl { + _LIBCPP_HIDE_FROM_ABI +#ifndef _LIBCPP_CXX03_LANG + __cxx_atomic_base_impl() _NOEXCEPT = default; +#else + __cxx_atomic_base_impl() _NOEXCEPT : __a_value() { + } +#endif // _LIBCPP_CXX03_LANG + _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {} + _Tp __a_value; +}; + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { + __cxx_atomic_assign_volatile(__a->__a_value, __val); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { + __a->__a_value = __val; +} + +_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) { + __atomic_thread_fence(__to_gcc_order(__order)); +} + +_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) { + __atomic_signal_fence(__to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) { + __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) { + __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { + _Tp __ret; + __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order)); + return __ret; +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_load_inplace(const volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) { + __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_load_inplace(const __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) { + __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { + _Tp __ret; + __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order)); + return __ret; +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) { + _Tp __ret; + __atomic_exchange( + std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order)); + return __ret; +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) { + _Tp __ret; + __atomic_exchange( + std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order)); + return __ret; +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + volatile __cxx_atomic_base_impl<_Tp>* __a, + _Tp* __expected, + _Tp __value, + memory_order __success, + memory_order __failure) { + return __atomic_compare_exchange( + std::addressof(__a->__a_value), + __expected, + std::addressof(__value), + false, + __to_gcc_order(__success), + __to_gcc_failure_order(__failure)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { + return __atomic_compare_exchange( + std::addressof(__a->__a_value), + __expected, + std::addressof(__value), + false, + __to_gcc_order(__success), + __to_gcc_failure_order(__failure)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + volatile __cxx_atomic_base_impl<_Tp>* __a, + _Tp* __expected, + _Tp __value, + memory_order __success, + memory_order __failure) { + return __atomic_compare_exchange( + std::addressof(__a->__a_value), + __expected, + std::addressof(__value), + true, + __to_gcc_order(__success), + __to_gcc_failure_order(__failure)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { + return __atomic_compare_exchange( + std::addressof(__a->__a_value), + __expected, + std::addressof(__value), + true, + __to_gcc_order(__success), + __to_gcc_failure_order(__failure)); +} + +template <typename _Tp> +struct __skip_amt { + enum { value = 1 }; +}; + +template <typename _Tp> +struct __skip_amt<_Tp*> { + enum { value = sizeof(_Tp) }; +}; + +// FIXME: Haven't figured out what the spec says about using arrays with +// atomic_fetch_add. Force a failure rather than creating bad behavior. +template <typename _Tp> +struct __skip_amt<_Tp[]> {}; +template <typename _Tp, int n> +struct __skip_amt<_Tp[n]> {}; + +template <typename _Tp, typename _Td> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) { + return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); +} + +template <typename _Tp, typename _Td> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) { + return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); +} + +template <typename _Tp, typename _Td> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) { + return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); +} + +template <typename _Tp, typename _Td> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) { + return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { + return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { + return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { + return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { + return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { + return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); +} + +template <typename _Tp> +_LIBCPP_HIDE_FROM_ABI _Tp +__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) { + return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order)); +} + +#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_SUPPORT_GCC_H diff --git a/third_party/libcxx/include/c++/v1/__atomic/to_gcc_order.h b/third_party/libcxx/include/c++/v1/__atomic/to_gcc_order.h new file mode 100644 index 0000000..d04c111 --- /dev/null +++ b/third_party/libcxx/include/c++/v1/__atomic/to_gcc_order.h @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_TO_GCC_ORDER_H +#define _LIBCPP___ATOMIC_TO_GCC_ORDER_H + +#include <__atomic/memory_order.h> +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if defined(__ATOMIC_RELAXED) && defined(__ATOMIC_CONSUME) && defined(__ATOMIC_ACQUIRE) && \ + defined(__ATOMIC_RELEASE) && defined(__ATOMIC_ACQ_REL) && defined(__ATOMIC_SEQ_CST) + +_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { + // Avoid switch statement to make this a constexpr. + return __order == memory_order_relaxed + ? __ATOMIC_RELAXED + : (__order == memory_order_acquire + ? __ATOMIC_ACQUIRE + : (__order == memory_order_release + ? __ATOMIC_RELEASE + : (__order == memory_order_seq_cst + ? __ATOMIC_SEQ_CST + : (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL : __ATOMIC_CONSUME)))); +} + +_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { + // Avoid switch statement to make this a constexpr. + return __order == memory_order_relaxed + ? __ATOMIC_RELAXED + : (__order == memory_order_acquire + ? __ATOMIC_ACQUIRE + : (__order == memory_order_release + ? __ATOMIC_RELAXED + : (__order == memory_order_seq_cst + ? __ATOMIC_SEQ_CST + : (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE : __ATOMIC_CONSUME)))); +} + +#endif + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_TO_GCC_ORDER_H |