30#ifndef _GLIBCXX_ATOMIC_FUTEX_H
31#define _GLIBCXX_ATOMIC_FUTEX_H 1
33#pragma GCC system_header
36#if ! (defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1)
42#ifndef _GLIBCXX_ALWAYS_INLINE
43#define _GLIBCXX_ALWAYS_INLINE inline __attribute__((__always_inline__))
46namespace std _GLIBCXX_VISIBILITY(default)
48_GLIBCXX_BEGIN_NAMESPACE_VERSION
50#ifdef _GLIBCXX_HAS_GTHREADS
51#if defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1
52 struct __atomic_futex_unsigned_base
57 _M_futex_wait_until(
unsigned *__addr,
unsigned __val,
bool __has_timeout,
63 _M_futex_wait_until_steady(
unsigned *__addr,
unsigned __val,
67 static void _M_futex_notify_all(
unsigned* __addr);
70 template <
unsigned _Waiter_bit = 0x80000000>
71 class __atomic_futex_unsigned : __atomic_futex_unsigned_base
73 typedef chrono::steady_clock __clock_t;
76 atomic<unsigned> _M_data;
80 __atomic_futex_unsigned(
unsigned __data) : _M_data(__data)
83 _GLIBCXX_ALWAYS_INLINE
unsigned
86 return _M_data.load(__mo) & ~_Waiter_bit;
97 _M_load_and_test_until(
unsigned __assumed,
unsigned __operand,
108 _M_data.fetch_or(_Waiter_bit, memory_order_relaxed);
109 bool __ret = _M_futex_wait_until((
unsigned*)(
void*)&_M_data,
110 __assumed | _Waiter_bit,
111 __has_timeout, __s, __ns);
113 __assumed = _M_load(__mo);
114 if (!__ret || ((__operand == __assumed) == __equal))
127 _M_load_and_test_until_steady(
unsigned __assumed,
unsigned __operand,
138 _M_data.fetch_or(_Waiter_bit, memory_order_relaxed);
139 bool __ret = _M_futex_wait_until_steady((
unsigned*)(
void*)&_M_data,
140 __assumed | _Waiter_bit,
141 __has_timeout, __s, __ns);
143 __assumed = _M_load(__mo);
144 if (!__ret || ((__operand == __assumed) == __equal))
155 _M_load_and_test(
unsigned __assumed,
unsigned __operand,
158 return _M_load_and_test_until(__assumed, __operand, __equal, __mo,
167 template<
typename _Dur>
169 _M_load_and_test_until_impl(
unsigned __assumed,
unsigned __operand,
171 const chrono::time_point<std::chrono::system_clock, _Dur>& __atime)
173 auto __d = __atime.time_since_epoch();
174 if (__d < __d.zero()) [[__unlikely__]]
176 auto __s = chrono::duration_cast<chrono::seconds>(__d);
177 auto __ns = chrono::duration_cast<chrono::nanoseconds>(__d - __s);
178 return _M_load_and_test_until(__assumed, __operand, __equal, __mo,
182 template<
typename _Dur>
184 _M_load_and_test_until_impl(
unsigned __assumed,
unsigned __operand,
186 const chrono::time_point<std::chrono::steady_clock, _Dur>& __atime)
188 auto __d = __atime.time_since_epoch();
189 if (__d < __d.zero()) [[__unlikely__]]
191 auto __s = chrono::duration_cast<chrono::seconds>(__d);
192 auto __ns = chrono::duration_cast<chrono::nanoseconds>(__d - __s);
193 return _M_load_and_test_until_steady(__assumed, __operand, __equal, __mo,
199 _GLIBCXX_ALWAYS_INLINE
unsigned
200 _M_load_when_not_equal(
unsigned __val,
memory_order __mo)
202 unsigned __i = _M_load(__mo);
203 if ((__i & ~_Waiter_bit) != __val)
204 return (__i & ~_Waiter_bit);
206 return _M_load_and_test(__i, __val,
false, __mo);
209 _GLIBCXX_ALWAYS_INLINE
void
212 unsigned __i = _M_load(__mo);
213 if ((__i & ~_Waiter_bit) == __val)
216 _M_load_and_test(__i, __val,
true, __mo);
220 template<
typename _Rep,
typename _Period>
221 _GLIBCXX_ALWAYS_INLINE
bool
222 _M_load_when_equal_for(
unsigned __val,
memory_order __mo,
223 const chrono::duration<_Rep, _Period>& __rtime)
225 using __dur =
typename __clock_t::duration;
226 return _M_load_when_equal_until(__val, __mo,
227 __clock_t::now() + chrono::__detail::ceil<__dur>(__rtime));
231 template<
typename _Clock,
typename _Duration>
232 _GLIBCXX_ALWAYS_INLINE
bool
233 _M_load_when_equal_until(
unsigned __val,
memory_order __mo,
234 const chrono::time_point<_Clock, _Duration>& __atime)
236 typename _Clock::time_point __c_entry = _Clock::now();
238 const __clock_t::time_point __s_entry = __clock_t::now();
239 const auto __delta = __atime - __c_entry;
240 const auto __s_atime = __s_entry +
241 chrono::__detail::ceil<__clock_t::duration>(__delta);
242 if (_M_load_when_equal_until(__val, __mo, __s_atime))
244 __c_entry = _Clock::now();
245 }
while (__c_entry < __atime);
250 template<
typename _Duration>
251 _GLIBCXX_ALWAYS_INLINE
bool
252 _M_load_when_equal_until(
unsigned __val,
memory_order __mo,
253 const chrono::time_point<std::chrono::system_clock, _Duration>& __atime)
255 unsigned __i = _M_load(__mo);
256 if ((__i & ~_Waiter_bit) == __val)
259 __i = _M_load_and_test_until_impl(__i, __val,
true, __mo, __atime);
260 return (__i & ~_Waiter_bit) == __val;
264 template<
typename _Duration>
265 _GLIBCXX_ALWAYS_INLINE
bool
266 _M_load_when_equal_until(
unsigned __val,
memory_order __mo,
267 const chrono::time_point<std::chrono::steady_clock, _Duration>& __atime)
269 unsigned __i = _M_load(__mo);
270 if ((__i & ~_Waiter_bit) == __val)
273 __i = _M_load_and_test_until_impl(__i, __val,
true, __mo, __atime);
274 return (__i & ~_Waiter_bit) == __val;
277 _GLIBCXX_ALWAYS_INLINE
void
280 unsigned* __futex = (
unsigned *)(
void *)&_M_data;
281 if (_M_data.exchange(__val, __mo) & _Waiter_bit)
282 _M_futex_notify_all(__futex);
291 template <
unsigned _Waiter_bit = 0x80000000>
292 class __atomic_futex_unsigned
294 typedef chrono::system_clock __clock_t;
298 condition_variable _M_condvar;
302 __atomic_futex_unsigned(
unsigned __data) : _M_data(__data)
305 _GLIBCXX_ALWAYS_INLINE
unsigned
308 unique_lock<mutex> __lock(_M_mutex);
312 _GLIBCXX_ALWAYS_INLINE
unsigned
313 _M_load_when_not_equal(
unsigned __val,
memory_order __mo)
315 unique_lock<mutex> __lock(_M_mutex);
316 while (_M_data == __val)
317 _M_condvar.wait(__lock);
321 _GLIBCXX_ALWAYS_INLINE
void
324 unique_lock<mutex> __lock(_M_mutex);
325 while (_M_data != __val)
326 _M_condvar.wait(__lock);
329 template<
typename _Rep,
typename _Period>
330 _GLIBCXX_ALWAYS_INLINE
bool
331 _M_load_when_equal_for(
unsigned __val,
memory_order __mo,
332 const chrono::duration<_Rep, _Period>& __rtime)
334 unique_lock<mutex> __lock(_M_mutex);
335 return _M_condvar.wait_for(__lock, __rtime,
336 [&] {
return _M_data == __val;});
339 template<
typename _Clock,
typename _Duration>
340 _GLIBCXX_ALWAYS_INLINE
bool
341 _M_load_when_equal_until(
unsigned __val,
memory_order __mo,
342 const chrono::time_point<_Clock, _Duration>& __atime)
344 unique_lock<mutex> __lock(_M_mutex);
345 return _M_condvar.wait_until(__lock, __atime,
346 [&] {
return _M_data == __val;});
349 _GLIBCXX_ALWAYS_INLINE
void
352 unique_lock<mutex> __lock(_M_mutex);
354 _M_condvar.notify_all();
361_GLIBCXX_END_NAMESPACE_VERSION
duration< int64_t > seconds
seconds
duration< int64_t, nano > nanoseconds
nanoseconds
memory_order
Enumeration for memory_order.
ISO C++ entities toplevel namespace is std.