/* * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* char_compare_and_swap (based on fetch_compare_and_swap) */ #if defined(AO_HAVE_char_fetch_compare_and_swap_full) \ && !defined(AO_HAVE_char_compare_and_swap_full) AO_INLINE int AO_char_compare_and_swap_full(volatile unsigned/**/char *addr, unsigned/**/char old_val, unsigned/**/char new_val) { return AO_char_fetch_compare_and_swap_full(addr, old_val, new_val) == old_val; } # define AO_HAVE_char_compare_and_swap_full #endif #if defined(AO_HAVE_char_fetch_compare_and_swap_acquire) \ && !defined(AO_HAVE_char_compare_and_swap_acquire) AO_INLINE int AO_char_compare_and_swap_acquire(volatile unsigned/**/char *addr, unsigned/**/char old_val, unsigned/**/char new_val) { return AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val) == old_val; } # define AO_HAVE_char_compare_and_swap_acquire #endif #if defined(AO_HAVE_char_fetch_compare_and_swap_release) \ && !defined(AO_HAVE_char_compare_and_swap_release) AO_INLINE int AO_char_compare_and_swap_release(volatile unsigned/**/char *addr, unsigned/**/char old_val, unsigned/**/char new_val) { return AO_char_fetch_compare_and_swap_release(addr, old_val, new_val) == old_val; } # define AO_HAVE_char_compare_and_swap_release #endif #if defined(AO_HAVE_char_fetch_compare_and_swap_write) \ && !defined(AO_HAVE_char_compare_and_swap_write) AO_INLINE int AO_char_compare_and_swap_write(volatile unsigned/**/char *addr, unsigned/**/char old_val, unsigned/**/char new_val) { return AO_char_fetch_compare_and_swap_write(addr, old_val, new_val) == old_val; } # define AO_HAVE_char_compare_and_swap_write #endif #if defined(AO_HAVE_char_fetch_compare_and_swap_read) \ && !defined(AO_HAVE_char_compare_and_swap_read) AO_INLINE int AO_char_compare_and_swap_read(volatile unsigned/**/char *addr, unsigned/**/char old_val, unsigned/**/char new_val) { return AO_char_fetch_compare_and_swap_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_char_compare_and_swap_read #endif #if defined(AO_HAVE_char_fetch_compare_and_swap) \ && !defined(AO_HAVE_char_compare_and_swap) AO_INLINE int AO_char_compare_and_swap(volatile unsigned/**/char *addr, unsigned/**/char old_val, unsigned/**/char new_val) { return AO_char_fetch_compare_and_swap(addr, old_val, new_val) == old_val; } # define AO_HAVE_char_compare_and_swap #endif #if defined(AO_HAVE_char_fetch_compare_and_swap_release_write) \ && !defined(AO_HAVE_char_compare_and_swap_release_write) AO_INLINE int AO_char_compare_and_swap_release_write(volatile unsigned/**/char *addr, unsigned/**/char old_val, unsigned/**/char new_val) { return AO_char_fetch_compare_and_swap_release_write(addr, old_val, new_val) == old_val; } # define AO_HAVE_char_compare_and_swap_release_write #endif #if defined(AO_HAVE_char_fetch_compare_and_swap_acquire_read) \ && !defined(AO_HAVE_char_compare_and_swap_acquire_read) AO_INLINE int AO_char_compare_and_swap_acquire_read(volatile unsigned/**/char *addr, unsigned/**/char old_val, unsigned/**/char new_val) { return AO_char_fetch_compare_and_swap_acquire_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_char_compare_and_swap_acquire_read #endif #if defined(AO_HAVE_char_fetch_compare_and_swap_dd_acquire_read) \ && !defined(AO_HAVE_char_compare_and_swap_dd_acquire_read) AO_INLINE int AO_char_compare_and_swap_dd_acquire_read(volatile unsigned/**/char *addr, unsigned/**/char old_val, unsigned/**/char new_val) { return AO_char_fetch_compare_and_swap_dd_acquire_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_char_compare_and_swap_dd_acquire_read #endif /* char_fetch_and_add */ /* We first try to implement fetch_and_add variants in terms of the */ /* corresponding compare_and_swap variants to minimize adding barriers. */ #if defined(AO_HAVE_char_compare_and_swap_full) \ && !defined(AO_HAVE_char_fetch_and_add_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE unsigned/**/char AO_char_fetch_and_add_full(volatile unsigned/**/char *addr, unsigned/**/char incr) { unsigned/**/char old; do { old = *(unsigned/**/char *)addr; } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old, old + incr))); return old; } # define AO_HAVE_char_fetch_and_add_full #endif #if defined(AO_HAVE_char_compare_and_swap_acquire) \ && !defined(AO_HAVE_char_fetch_and_add_acquire) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE unsigned/**/char AO_char_fetch_and_add_acquire(volatile unsigned/**/char *addr, unsigned/**/char incr) { unsigned/**/char old; do { old = *(unsigned/**/char *)addr; } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_acquire(addr, old, old + incr))); return old; } # define AO_HAVE_char_fetch_and_add_acquire #endif #if defined(AO_HAVE_char_compare_and_swap_release) \ && !defined(AO_HAVE_char_fetch_and_add_release) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE unsigned/**/char AO_char_fetch_and_add_release(volatile unsigned/**/char *addr, unsigned/**/char incr) { unsigned/**/char old; do { old = *(unsigned/**/char *)addr; } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_release(addr, old, old + incr))); return old; } # define AO_HAVE_char_fetch_and_add_release #endif #if defined(AO_HAVE_char_compare_and_swap) \ && !defined(AO_HAVE_char_fetch_and_add) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE unsigned/**/char AO_char_fetch_and_add(volatile unsigned/**/char *addr, unsigned/**/char incr) { unsigned/**/char old; do { old = *(unsigned/**/char *)addr; } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap(addr, old, old + incr))); return old; } # define AO_HAVE_char_fetch_and_add #endif #if defined(AO_HAVE_char_fetch_and_add_full) # if !defined(AO_HAVE_char_fetch_and_add_release) # define AO_char_fetch_and_add_release(addr, val) \ AO_char_fetch_and_add_full(addr, val) # define AO_HAVE_char_fetch_and_add_release # endif # if !defined(AO_HAVE_char_fetch_and_add_acquire) # define AO_char_fetch_and_add_acquire(addr, val) \ AO_char_fetch_and_add_full(addr, val) # define AO_HAVE_char_fetch_and_add_acquire # endif # if !defined(AO_HAVE_char_fetch_and_add_write) # define AO_char_fetch_and_add_write(addr, val) \ AO_char_fetch_and_add_full(addr, val) # define AO_HAVE_char_fetch_and_add_write # endif # if !defined(AO_HAVE_char_fetch_and_add_read) # define AO_char_fetch_and_add_read(addr, val) \ AO_char_fetch_and_add_full(addr, val) # define AO_HAVE_char_fetch_and_add_read # endif #endif /* AO_HAVE_char_fetch_and_add_full */ #if defined(AO_HAVE_char_fetch_and_add) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_char_fetch_and_add_acquire) AO_INLINE unsigned/**/char AO_char_fetch_and_add_acquire(volatile unsigned/**/char *addr, unsigned/**/char incr) { unsigned/**/char result = AO_char_fetch_and_add(addr, incr); AO_nop_full(); return result; } # define AO_HAVE_char_fetch_and_add_acquire #endif #if defined(AO_HAVE_char_fetch_and_add) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_char_fetch_and_add_release) # define AO_char_fetch_and_add_release(addr, incr) \ (AO_nop_full(), AO_char_fetch_and_add(addr, incr)) # define AO_HAVE_char_fetch_and_add_release #endif #if !defined(AO_HAVE_char_fetch_and_add) \ && defined(AO_HAVE_char_fetch_and_add_release) # define AO_char_fetch_and_add(addr, val) \ AO_char_fetch_and_add_release(addr, val) # define AO_HAVE_char_fetch_and_add #endif #if !defined(AO_HAVE_char_fetch_and_add) \ && defined(AO_HAVE_char_fetch_and_add_acquire) # define AO_char_fetch_and_add(addr, val) \ AO_char_fetch_and_add_acquire(addr, val) # define AO_HAVE_char_fetch_and_add #endif #if !defined(AO_HAVE_char_fetch_and_add) \ && defined(AO_HAVE_char_fetch_and_add_write) # define AO_char_fetch_and_add(addr, val) \ AO_char_fetch_and_add_write(addr, val) # define AO_HAVE_char_fetch_and_add #endif #if !defined(AO_HAVE_char_fetch_and_add) \ && defined(AO_HAVE_char_fetch_and_add_read) # define AO_char_fetch_and_add(addr, val) \ AO_char_fetch_and_add_read(addr, val) # define AO_HAVE_char_fetch_and_add #endif #if defined(AO_HAVE_char_fetch_and_add_acquire) \ && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_char_fetch_and_add_full) # define AO_char_fetch_and_add_full(addr, val) \ (AO_nop_full(), AO_char_fetch_and_add_acquire(addr, val)) # define AO_HAVE_char_fetch_and_add_full #endif #if !defined(AO_HAVE_char_fetch_and_add_release_write) \ && defined(AO_HAVE_char_fetch_and_add_write) # define AO_char_fetch_and_add_release_write(addr, val) \ AO_char_fetch_and_add_write(addr, val) # define AO_HAVE_char_fetch_and_add_release_write #endif #if !defined(AO_HAVE_char_fetch_and_add_release_write) \ && defined(AO_HAVE_char_fetch_and_add_release) # define AO_char_fetch_and_add_release_write(addr, val) \ AO_char_fetch_and_add_release(addr, val) # define AO_HAVE_char_fetch_and_add_release_write #endif #if !defined(AO_HAVE_char_fetch_and_add_acquire_read) \ && defined(AO_HAVE_char_fetch_and_add_read) # define AO_char_fetch_and_add_acquire_read(addr, val) \ AO_char_fetch_and_add_read(addr, val) # define AO_HAVE_char_fetch_and_add_acquire_read #endif #if !defined(AO_HAVE_char_fetch_and_add_acquire_read) \ && defined(AO_HAVE_char_fetch_and_add_acquire) # define AO_char_fetch_and_add_acquire_read(addr, val) \ AO_char_fetch_and_add_acquire(addr, val) # define AO_HAVE_char_fetch_and_add_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_char_fetch_and_add_acquire_read) # define AO_char_fetch_and_add_dd_acquire_read(addr, val) \ AO_char_fetch_and_add_acquire_read(addr, val) # define AO_HAVE_char_fetch_and_add_dd_acquire_read # endif #else # if defined(AO_HAVE_char_fetch_and_add) # define AO_char_fetch_and_add_dd_acquire_read(addr, val) \ AO_char_fetch_and_add(addr, val) # define AO_HAVE_char_fetch_and_add_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* char_fetch_and_add1 */ #if defined(AO_HAVE_char_fetch_and_add_full) \ && !defined(AO_HAVE_char_fetch_and_add1_full) # define AO_char_fetch_and_add1_full(addr) \ AO_char_fetch_and_add_full(addr, 1) # define AO_HAVE_char_fetch_and_add1_full #endif #if defined(AO_HAVE_char_fetch_and_add_release) \ && !defined(AO_HAVE_char_fetch_and_add1_release) # define AO_char_fetch_and_add1_release(addr) \ AO_char_fetch_and_add_release(addr, 1) # define AO_HAVE_char_fetch_and_add1_release #endif #if defined(AO_HAVE_char_fetch_and_add_acquire) \ && !defined(AO_HAVE_char_fetch_and_add1_acquire) # define AO_char_fetch_and_add1_acquire(addr) \ AO_char_fetch_and_add_acquire(addr, 1) # define AO_HAVE_char_fetch_and_add1_acquire #endif #if defined(AO_HAVE_char_fetch_and_add_write) \ && !defined(AO_HAVE_char_fetch_and_add1_write) # define AO_char_fetch_and_add1_write(addr) \ AO_char_fetch_and_add_write(addr, 1) # define AO_HAVE_char_fetch_and_add1_write #endif #if defined(AO_HAVE_char_fetch_and_add_read) \ && !defined(AO_HAVE_char_fetch_and_add1_read) # define AO_char_fetch_and_add1_read(addr) \ AO_char_fetch_and_add_read(addr, 1) # define AO_HAVE_char_fetch_and_add1_read #endif #if defined(AO_HAVE_char_fetch_and_add_release_write) \ && !defined(AO_HAVE_char_fetch_and_add1_release_write) # define AO_char_fetch_and_add1_release_write(addr) \ AO_char_fetch_and_add_release_write(addr, 1) # define AO_HAVE_char_fetch_and_add1_release_write #endif #if defined(AO_HAVE_char_fetch_and_add_acquire_read) \ && !defined(AO_HAVE_char_fetch_and_add1_acquire_read) # define AO_char_fetch_and_add1_acquire_read(addr) \ AO_char_fetch_and_add_acquire_read(addr, 1) # define AO_HAVE_char_fetch_and_add1_acquire_read #endif #if defined(AO_HAVE_char_fetch_and_add) \ && !defined(AO_HAVE_char_fetch_and_add1) # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add(addr, 1) # define AO_HAVE_char_fetch_and_add1 #endif #if defined(AO_HAVE_char_fetch_and_add1_full) # if !defined(AO_HAVE_char_fetch_and_add1_release) # define AO_char_fetch_and_add1_release(addr) \ AO_char_fetch_and_add1_full(addr) # define AO_HAVE_char_fetch_and_add1_release # endif # if !defined(AO_HAVE_char_fetch_and_add1_acquire) # define AO_char_fetch_and_add1_acquire(addr) \ AO_char_fetch_and_add1_full(addr) # define AO_HAVE_char_fetch_and_add1_acquire # endif # if !defined(AO_HAVE_char_fetch_and_add1_write) # define AO_char_fetch_and_add1_write(addr) \ AO_char_fetch_and_add1_full(addr) # define AO_HAVE_char_fetch_and_add1_write # endif # if !defined(AO_HAVE_char_fetch_and_add1_read) # define AO_char_fetch_and_add1_read(addr) \ AO_char_fetch_and_add1_full(addr) # define AO_HAVE_char_fetch_and_add1_read # endif #endif /* AO_HAVE_char_fetch_and_add1_full */ #if !defined(AO_HAVE_char_fetch_and_add1) \ && defined(AO_HAVE_char_fetch_and_add1_release) # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_release(addr) # define AO_HAVE_char_fetch_and_add1 #endif #if !defined(AO_HAVE_char_fetch_and_add1) \ && defined(AO_HAVE_char_fetch_and_add1_acquire) # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_acquire(addr) # define AO_HAVE_char_fetch_and_add1 #endif #if !defined(AO_HAVE_char_fetch_and_add1) \ && defined(AO_HAVE_char_fetch_and_add1_write) # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_write(addr) # define AO_HAVE_char_fetch_and_add1 #endif #if !defined(AO_HAVE_char_fetch_and_add1) \ && defined(AO_HAVE_char_fetch_and_add1_read) # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_read(addr) # define AO_HAVE_char_fetch_and_add1 #endif #if defined(AO_HAVE_char_fetch_and_add1_acquire) \ && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_char_fetch_and_add1_full) # define AO_char_fetch_and_add1_full(addr) \ (AO_nop_full(), AO_char_fetch_and_add1_acquire(addr)) # define AO_HAVE_char_fetch_and_add1_full #endif #if !defined(AO_HAVE_char_fetch_and_add1_release_write) \ && defined(AO_HAVE_char_fetch_and_add1_write) # define AO_char_fetch_and_add1_release_write(addr) \ AO_char_fetch_and_add1_write(addr) # define AO_HAVE_char_fetch_and_add1_release_write #endif #if !defined(AO_HAVE_char_fetch_and_add1_release_write) \ && defined(AO_HAVE_char_fetch_and_add1_release) # define AO_char_fetch_and_add1_release_write(addr) \ AO_char_fetch_and_add1_release(addr) # define AO_HAVE_char_fetch_and_add1_release_write #endif #if !defined(AO_HAVE_char_fetch_and_add1_acquire_read) \ && defined(AO_HAVE_char_fetch_and_add1_read) # define AO_char_fetch_and_add1_acquire_read(addr) \ AO_char_fetch_and_add1_read(addr) # define AO_HAVE_char_fetch_and_add1_acquire_read #endif #if !defined(AO_HAVE_char_fetch_and_add1_acquire_read) \ && defined(AO_HAVE_char_fetch_and_add1_acquire) # define AO_char_fetch_and_add1_acquire_read(addr) \ AO_char_fetch_and_add1_acquire(addr) # define AO_HAVE_char_fetch_and_add1_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_char_fetch_and_add1_acquire_read) # define AO_char_fetch_and_add1_dd_acquire_read(addr) \ AO_char_fetch_and_add1_acquire_read(addr) # define AO_HAVE_char_fetch_and_add1_dd_acquire_read # endif #else # if defined(AO_HAVE_char_fetch_and_add1) # define AO_char_fetch_and_add1_dd_acquire_read(addr) \ AO_char_fetch_and_add1(addr) # define AO_HAVE_char_fetch_and_add1_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* char_fetch_and_sub1 */ #if defined(AO_HAVE_char_fetch_and_add_full) \ && !defined(AO_HAVE_char_fetch_and_sub1_full) # define AO_char_fetch_and_sub1_full(addr) \ AO_char_fetch_and_add_full(addr, (unsigned/**/char)(-1)) # define AO_HAVE_char_fetch_and_sub1_full #endif #if defined(AO_HAVE_char_fetch_and_add_release) \ && !defined(AO_HAVE_char_fetch_and_sub1_release) # define AO_char_fetch_and_sub1_release(addr) \ AO_char_fetch_and_add_release(addr, (unsigned/**/char)(-1)) # define AO_HAVE_char_fetch_and_sub1_release #endif #if defined(AO_HAVE_char_fetch_and_add_acquire) \ && !defined(AO_HAVE_char_fetch_and_sub1_acquire) # define AO_char_fetch_and_sub1_acquire(addr) \ AO_char_fetch_and_add_acquire(addr, (unsigned/**/char)(-1)) # define AO_HAVE_char_fetch_and_sub1_acquire #endif #if defined(AO_HAVE_char_fetch_and_add_write) \ && !defined(AO_HAVE_char_fetch_and_sub1_write) # define AO_char_fetch_and_sub1_write(addr) \ AO_char_fetch_and_add_write(addr, (unsigned/**/char)(-1)) # define AO_HAVE_char_fetch_and_sub1_write #endif #if defined(AO_HAVE_char_fetch_and_add_read) \ && !defined(AO_HAVE_char_fetch_and_sub1_read) # define AO_char_fetch_and_sub1_read(addr) \ AO_char_fetch_and_add_read(addr, (unsigned/**/char)(-1)) # define AO_HAVE_char_fetch_and_sub1_read #endif #if defined(AO_HAVE_char_fetch_and_add_release_write) \ && !defined(AO_HAVE_char_fetch_and_sub1_release_write) # define AO_char_fetch_and_sub1_release_write(addr) \ AO_char_fetch_and_add_release_write(addr, (unsigned/**/char)(-1)) # define AO_HAVE_char_fetch_and_sub1_release_write #endif #if defined(AO_HAVE_char_fetch_and_add_acquire_read) \ && !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) # define AO_char_fetch_and_sub1_acquire_read(addr) \ AO_char_fetch_and_add_acquire_read(addr, (unsigned/**/char)(-1)) # define AO_HAVE_char_fetch_and_sub1_acquire_read #endif #if defined(AO_HAVE_char_fetch_and_add) \ && !defined(AO_HAVE_char_fetch_and_sub1) # define AO_char_fetch_and_sub1(addr) \ AO_char_fetch_and_add(addr, (unsigned/**/char)(-1)) # define AO_HAVE_char_fetch_and_sub1 #endif #if defined(AO_HAVE_char_fetch_and_sub1_full) # if !defined(AO_HAVE_char_fetch_and_sub1_release) # define AO_char_fetch_and_sub1_release(addr) \ AO_char_fetch_and_sub1_full(addr) # define AO_HAVE_char_fetch_and_sub1_release # endif # if !defined(AO_HAVE_char_fetch_and_sub1_acquire) # define AO_char_fetch_and_sub1_acquire(addr) \ AO_char_fetch_and_sub1_full(addr) # define AO_HAVE_char_fetch_and_sub1_acquire # endif # if !defined(AO_HAVE_char_fetch_and_sub1_write) # define AO_char_fetch_and_sub1_write(addr) \ AO_char_fetch_and_sub1_full(addr) # define AO_HAVE_char_fetch_and_sub1_write # endif # if !defined(AO_HAVE_char_fetch_and_sub1_read) # define AO_char_fetch_and_sub1_read(addr) \ AO_char_fetch_and_sub1_full(addr) # define AO_HAVE_char_fetch_and_sub1_read # endif #endif /* AO_HAVE_char_fetch_and_sub1_full */ #if !defined(AO_HAVE_char_fetch_and_sub1) \ && defined(AO_HAVE_char_fetch_and_sub1_release) # define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_release(addr) # define AO_HAVE_char_fetch_and_sub1 #endif #if !defined(AO_HAVE_char_fetch_and_sub1) \ && defined(AO_HAVE_char_fetch_and_sub1_acquire) # define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_acquire(addr) # define AO_HAVE_char_fetch_and_sub1 #endif #if !defined(AO_HAVE_char_fetch_and_sub1) \ && defined(AO_HAVE_char_fetch_and_sub1_write) # define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_write(addr) # define AO_HAVE_char_fetch_and_sub1 #endif #if !defined(AO_HAVE_char_fetch_and_sub1) \ && defined(AO_HAVE_char_fetch_and_sub1_read) # define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_read(addr) # define AO_HAVE_char_fetch_and_sub1 #endif #if defined(AO_HAVE_char_fetch_and_sub1_acquire) \ && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_char_fetch_and_sub1_full) # define AO_char_fetch_and_sub1_full(addr) \ (AO_nop_full(), AO_char_fetch_and_sub1_acquire(addr)) # define AO_HAVE_char_fetch_and_sub1_full #endif #if !defined(AO_HAVE_char_fetch_and_sub1_release_write) \ && defined(AO_HAVE_char_fetch_and_sub1_write) # define AO_char_fetch_and_sub1_release_write(addr) \ AO_char_fetch_and_sub1_write(addr) # define AO_HAVE_char_fetch_and_sub1_release_write #endif #if !defined(AO_HAVE_char_fetch_and_sub1_release_write) \ && defined(AO_HAVE_char_fetch_and_sub1_release) # define AO_char_fetch_and_sub1_release_write(addr) \ AO_char_fetch_and_sub1_release(addr) # define AO_HAVE_char_fetch_and_sub1_release_write #endif #if !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) \ && defined(AO_HAVE_char_fetch_and_sub1_read) # define AO_char_fetch_and_sub1_acquire_read(addr) \ AO_char_fetch_and_sub1_read(addr) # define AO_HAVE_char_fetch_and_sub1_acquire_read #endif #if !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) \ && defined(AO_HAVE_char_fetch_and_sub1_acquire) # define AO_char_fetch_and_sub1_acquire_read(addr) \ AO_char_fetch_and_sub1_acquire(addr) # define AO_HAVE_char_fetch_and_sub1_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_char_fetch_and_sub1_acquire_read) # define AO_char_fetch_and_sub1_dd_acquire_read(addr) \ AO_char_fetch_and_sub1_acquire_read(addr) # define AO_HAVE_char_fetch_and_sub1_dd_acquire_read # endif #else # if defined(AO_HAVE_char_fetch_and_sub1) # define AO_char_fetch_and_sub1_dd_acquire_read(addr) \ AO_char_fetch_and_sub1(addr) # define AO_HAVE_char_fetch_and_sub1_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* char_and */ #if defined(AO_HAVE_char_compare_and_swap_full) \ && !defined(AO_HAVE_char_and_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_char_and_full(volatile unsigned/**/char *addr, unsigned/**/char value) { unsigned/**/char old; do { old = *(unsigned/**/char *)addr; } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old, old & value))); } # define AO_HAVE_char_and_full #endif #if defined(AO_HAVE_char_and_full) # if !defined(AO_HAVE_char_and_release) # define AO_char_and_release(addr, val) AO_char_and_full(addr, val) # define AO_HAVE_char_and_release # endif # if !defined(AO_HAVE_char_and_acquire) # define AO_char_and_acquire(addr, val) AO_char_and_full(addr, val) # define AO_HAVE_char_and_acquire # endif # if !defined(AO_HAVE_char_and_write) # define AO_char_and_write(addr, val) AO_char_and_full(addr, val) # define AO_HAVE_char_and_write # endif # if !defined(AO_HAVE_char_and_read) # define AO_char_and_read(addr, val) AO_char_and_full(addr, val) # define AO_HAVE_char_and_read # endif #endif /* AO_HAVE_char_and_full */ #if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_release) # define AO_char_and(addr, val) AO_char_and_release(addr, val) # define AO_HAVE_char_and #endif #if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_acquire) # define AO_char_and(addr, val) AO_char_and_acquire(addr, val) # define AO_HAVE_char_and #endif #if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_write) # define AO_char_and(addr, val) AO_char_and_write(addr, val) # define AO_HAVE_char_and #endif #if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_read) # define AO_char_and(addr, val) AO_char_and_read(addr, val) # define AO_HAVE_char_and #endif #if defined(AO_HAVE_char_and_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_char_and_full) # define AO_char_and_full(addr, val) \ (AO_nop_full(), AO_char_and_acquire(addr, val)) # define AO_HAVE_char_and_full #endif #if !defined(AO_HAVE_char_and_release_write) \ && defined(AO_HAVE_char_and_write) # define AO_char_and_release_write(addr, val) AO_char_and_write(addr, val) # define AO_HAVE_char_and_release_write #endif #if !defined(AO_HAVE_char_and_release_write) \ && defined(AO_HAVE_char_and_release) # define AO_char_and_release_write(addr, val) AO_char_and_release(addr, val) # define AO_HAVE_char_and_release_write #endif #if !defined(AO_HAVE_char_and_acquire_read) \ && defined(AO_HAVE_char_and_read) # define AO_char_and_acquire_read(addr, val) AO_char_and_read(addr, val) # define AO_HAVE_char_and_acquire_read #endif #if !defined(AO_HAVE_char_and_acquire_read) \ && defined(AO_HAVE_char_and_acquire) # define AO_char_and_acquire_read(addr, val) AO_char_and_acquire(addr, val) # define AO_HAVE_char_and_acquire_read #endif /* char_or */ #if defined(AO_HAVE_char_compare_and_swap_full) \ && !defined(AO_HAVE_char_or_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_char_or_full(volatile unsigned/**/char *addr, unsigned/**/char value) { unsigned/**/char old; do { old = *(unsigned/**/char *)addr; } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old, old | value))); } # define AO_HAVE_char_or_full #endif #if defined(AO_HAVE_char_or_full) # if !defined(AO_HAVE_char_or_release) # define AO_char_or_release(addr, val) AO_char_or_full(addr, val) # define AO_HAVE_char_or_release # endif # if !defined(AO_HAVE_char_or_acquire) # define AO_char_or_acquire(addr, val) AO_char_or_full(addr, val) # define AO_HAVE_char_or_acquire # endif # if !defined(AO_HAVE_char_or_write) # define AO_char_or_write(addr, val) AO_char_or_full(addr, val) # define AO_HAVE_char_or_write # endif # if !defined(AO_HAVE_char_or_read) # define AO_char_or_read(addr, val) AO_char_or_full(addr, val) # define AO_HAVE_char_or_read # endif #endif /* AO_HAVE_char_or_full */ #if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_release) # define AO_char_or(addr, val) AO_char_or_release(addr, val) # define AO_HAVE_char_or #endif #if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_acquire) # define AO_char_or(addr, val) AO_char_or_acquire(addr, val) # define AO_HAVE_char_or #endif #if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_write) # define AO_char_or(addr, val) AO_char_or_write(addr, val) # define AO_HAVE_char_or #endif #if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_read) # define AO_char_or(addr, val) AO_char_or_read(addr, val) # define AO_HAVE_char_or #endif #if defined(AO_HAVE_char_or_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_char_or_full) # define AO_char_or_full(addr, val) \ (AO_nop_full(), AO_char_or_acquire(addr, val)) # define AO_HAVE_char_or_full #endif #if !defined(AO_HAVE_char_or_release_write) \ && defined(AO_HAVE_char_or_write) # define AO_char_or_release_write(addr, val) AO_char_or_write(addr, val) # define AO_HAVE_char_or_release_write #endif #if !defined(AO_HAVE_char_or_release_write) \ && defined(AO_HAVE_char_or_release) # define AO_char_or_release_write(addr, val) AO_char_or_release(addr, val) # define AO_HAVE_char_or_release_write #endif #if !defined(AO_HAVE_char_or_acquire_read) && defined(AO_HAVE_char_or_read) # define AO_char_or_acquire_read(addr, val) AO_char_or_read(addr, val) # define AO_HAVE_char_or_acquire_read #endif #if !defined(AO_HAVE_char_or_acquire_read) \ && defined(AO_HAVE_char_or_acquire) # define AO_char_or_acquire_read(addr, val) AO_char_or_acquire(addr, val) # define AO_HAVE_char_or_acquire_read #endif /* char_xor */ #if defined(AO_HAVE_char_compare_and_swap_full) \ && !defined(AO_HAVE_char_xor_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_char_xor_full(volatile unsigned/**/char *addr, unsigned/**/char value) { unsigned/**/char old; do { old = *(unsigned/**/char *)addr; } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old, old ^ value))); } # define AO_HAVE_char_xor_full #endif #if defined(AO_HAVE_char_xor_full) # if !defined(AO_HAVE_char_xor_release) # define AO_char_xor_release(addr, val) AO_char_xor_full(addr, val) # define AO_HAVE_char_xor_release # endif # if !defined(AO_HAVE_char_xor_acquire) # define AO_char_xor_acquire(addr, val) AO_char_xor_full(addr, val) # define AO_HAVE_char_xor_acquire # endif # if !defined(AO_HAVE_char_xor_write) # define AO_char_xor_write(addr, val) AO_char_xor_full(addr, val) # define AO_HAVE_char_xor_write # endif # if !defined(AO_HAVE_char_xor_read) # define AO_char_xor_read(addr, val) AO_char_xor_full(addr, val) # define AO_HAVE_char_xor_read # endif #endif /* AO_HAVE_char_xor_full */ #if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_release) # define AO_char_xor(addr, val) AO_char_xor_release(addr, val) # define AO_HAVE_char_xor #endif #if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_acquire) # define AO_char_xor(addr, val) AO_char_xor_acquire(addr, val) # define AO_HAVE_char_xor #endif #if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_write) # define AO_char_xor(addr, val) AO_char_xor_write(addr, val) # define AO_HAVE_char_xor #endif #if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_read) # define AO_char_xor(addr, val) AO_char_xor_read(addr, val) # define AO_HAVE_char_xor #endif #if defined(AO_HAVE_char_xor_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_char_xor_full) # define AO_char_xor_full(addr, val) \ (AO_nop_full(), AO_char_xor_acquire(addr, val)) # define AO_HAVE_char_xor_full #endif #if !defined(AO_HAVE_char_xor_release_write) \ && defined(AO_HAVE_char_xor_write) # define AO_char_xor_release_write(addr, val) AO_char_xor_write(addr, val) # define AO_HAVE_char_xor_release_write #endif #if !defined(AO_HAVE_char_xor_release_write) \ && defined(AO_HAVE_char_xor_release) # define AO_char_xor_release_write(addr, val) AO_char_xor_release(addr, val) # define AO_HAVE_char_xor_release_write #endif #if !defined(AO_HAVE_char_xor_acquire_read) \ && defined(AO_HAVE_char_xor_read) # define AO_char_xor_acquire_read(addr, val) AO_char_xor_read(addr, val) # define AO_HAVE_char_xor_acquire_read #endif #if !defined(AO_HAVE_char_xor_acquire_read) \ && defined(AO_HAVE_char_xor_acquire) # define AO_char_xor_acquire_read(addr, val) AO_char_xor_acquire(addr, val) # define AO_HAVE_char_xor_acquire_read #endif /* char_and/or/xor_dd_acquire_read are meaningless. */ /* * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* short_compare_and_swap (based on fetch_compare_and_swap) */ #if defined(AO_HAVE_short_fetch_compare_and_swap_full) \ && !defined(AO_HAVE_short_compare_and_swap_full) AO_INLINE int AO_short_compare_and_swap_full(volatile unsigned/**/short *addr, unsigned/**/short old_val, unsigned/**/short new_val) { return AO_short_fetch_compare_and_swap_full(addr, old_val, new_val) == old_val; } # define AO_HAVE_short_compare_and_swap_full #endif #if defined(AO_HAVE_short_fetch_compare_and_swap_acquire) \ && !defined(AO_HAVE_short_compare_and_swap_acquire) AO_INLINE int AO_short_compare_and_swap_acquire(volatile unsigned/**/short *addr, unsigned/**/short old_val, unsigned/**/short new_val) { return AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val) == old_val; } # define AO_HAVE_short_compare_and_swap_acquire #endif #if defined(AO_HAVE_short_fetch_compare_and_swap_release) \ && !defined(AO_HAVE_short_compare_and_swap_release) AO_INLINE int AO_short_compare_and_swap_release(volatile unsigned/**/short *addr, unsigned/**/short old_val, unsigned/**/short new_val) { return AO_short_fetch_compare_and_swap_release(addr, old_val, new_val) == old_val; } # define AO_HAVE_short_compare_and_swap_release #endif #if defined(AO_HAVE_short_fetch_compare_and_swap_write) \ && !defined(AO_HAVE_short_compare_and_swap_write) AO_INLINE int AO_short_compare_and_swap_write(volatile unsigned/**/short *addr, unsigned/**/short old_val, unsigned/**/short new_val) { return AO_short_fetch_compare_and_swap_write(addr, old_val, new_val) == old_val; } # define AO_HAVE_short_compare_and_swap_write #endif #if defined(AO_HAVE_short_fetch_compare_and_swap_read) \ && !defined(AO_HAVE_short_compare_and_swap_read) AO_INLINE int AO_short_compare_and_swap_read(volatile unsigned/**/short *addr, unsigned/**/short old_val, unsigned/**/short new_val) { return AO_short_fetch_compare_and_swap_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_short_compare_and_swap_read #endif #if defined(AO_HAVE_short_fetch_compare_and_swap) \ && !defined(AO_HAVE_short_compare_and_swap) AO_INLINE int AO_short_compare_and_swap(volatile unsigned/**/short *addr, unsigned/**/short old_val, unsigned/**/short new_val) { return AO_short_fetch_compare_and_swap(addr, old_val, new_val) == old_val; } # define AO_HAVE_short_compare_and_swap #endif #if defined(AO_HAVE_short_fetch_compare_and_swap_release_write) \ && !defined(AO_HAVE_short_compare_and_swap_release_write) AO_INLINE int AO_short_compare_and_swap_release_write(volatile unsigned/**/short *addr, unsigned/**/short old_val, unsigned/**/short new_val) { return AO_short_fetch_compare_and_swap_release_write(addr, old_val, new_val) == old_val; } # define AO_HAVE_short_compare_and_swap_release_write #endif #if defined(AO_HAVE_short_fetch_compare_and_swap_acquire_read) \ && !defined(AO_HAVE_short_compare_and_swap_acquire_read) AO_INLINE int AO_short_compare_and_swap_acquire_read(volatile unsigned/**/short *addr, unsigned/**/short old_val, unsigned/**/short new_val) { return AO_short_fetch_compare_and_swap_acquire_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_short_compare_and_swap_acquire_read #endif #if defined(AO_HAVE_short_fetch_compare_and_swap_dd_acquire_read) \ && !defined(AO_HAVE_short_compare_and_swap_dd_acquire_read) AO_INLINE int AO_short_compare_and_swap_dd_acquire_read(volatile unsigned/**/short *addr, unsigned/**/short old_val, unsigned/**/short new_val) { return AO_short_fetch_compare_and_swap_dd_acquire_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_short_compare_and_swap_dd_acquire_read #endif /* short_fetch_and_add */ /* We first try to implement fetch_and_add variants in terms of the */ /* corresponding compare_and_swap variants to minimize adding barriers. */ #if defined(AO_HAVE_short_compare_and_swap_full) \ && !defined(AO_HAVE_short_fetch_and_add_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE unsigned/**/short AO_short_fetch_and_add_full(volatile unsigned/**/short *addr, unsigned/**/short incr) { unsigned/**/short old; do { old = *(unsigned/**/short *)addr; } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old, old + incr))); return old; } # define AO_HAVE_short_fetch_and_add_full #endif #if defined(AO_HAVE_short_compare_and_swap_acquire) \ && !defined(AO_HAVE_short_fetch_and_add_acquire) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE unsigned/**/short AO_short_fetch_and_add_acquire(volatile unsigned/**/short *addr, unsigned/**/short incr) { unsigned/**/short old; do { old = *(unsigned/**/short *)addr; } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_acquire(addr, old, old + incr))); return old; } # define AO_HAVE_short_fetch_and_add_acquire #endif #if defined(AO_HAVE_short_compare_and_swap_release) \ && !defined(AO_HAVE_short_fetch_and_add_release) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE unsigned/**/short AO_short_fetch_and_add_release(volatile unsigned/**/short *addr, unsigned/**/short incr) { unsigned/**/short old; do { old = *(unsigned/**/short *)addr; } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_release(addr, old, old + incr))); return old; } # define AO_HAVE_short_fetch_and_add_release #endif #if defined(AO_HAVE_short_compare_and_swap) \ && !defined(AO_HAVE_short_fetch_and_add) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE unsigned/**/short AO_short_fetch_and_add(volatile unsigned/**/short *addr, unsigned/**/short incr) { unsigned/**/short old; do { old = *(unsigned/**/short *)addr; } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap(addr, old, old + incr))); return old; } # define AO_HAVE_short_fetch_and_add #endif #if defined(AO_HAVE_short_fetch_and_add_full) # if !defined(AO_HAVE_short_fetch_and_add_release) # define AO_short_fetch_and_add_release(addr, val) \ AO_short_fetch_and_add_full(addr, val) # define AO_HAVE_short_fetch_and_add_release # endif # if !defined(AO_HAVE_short_fetch_and_add_acquire) # define AO_short_fetch_and_add_acquire(addr, val) \ AO_short_fetch_and_add_full(addr, val) # define AO_HAVE_short_fetch_and_add_acquire # endif # if !defined(AO_HAVE_short_fetch_and_add_write) # define AO_short_fetch_and_add_write(addr, val) \ AO_short_fetch_and_add_full(addr, val) # define AO_HAVE_short_fetch_and_add_write # endif # if !defined(AO_HAVE_short_fetch_and_add_read) # define AO_short_fetch_and_add_read(addr, val) \ AO_short_fetch_and_add_full(addr, val) # define AO_HAVE_short_fetch_and_add_read # endif #endif /* AO_HAVE_short_fetch_and_add_full */ #if defined(AO_HAVE_short_fetch_and_add) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_short_fetch_and_add_acquire) AO_INLINE unsigned/**/short AO_short_fetch_and_add_acquire(volatile unsigned/**/short *addr, unsigned/**/short incr) { unsigned/**/short result = AO_short_fetch_and_add(addr, incr); AO_nop_full(); return result; } # define AO_HAVE_short_fetch_and_add_acquire #endif #if defined(AO_HAVE_short_fetch_and_add) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_short_fetch_and_add_release) # define AO_short_fetch_and_add_release(addr, incr) \ (AO_nop_full(), AO_short_fetch_and_add(addr, incr)) # define AO_HAVE_short_fetch_and_add_release #endif #if !defined(AO_HAVE_short_fetch_and_add) \ && defined(AO_HAVE_short_fetch_and_add_release) # define AO_short_fetch_and_add(addr, val) \ AO_short_fetch_and_add_release(addr, val) # define AO_HAVE_short_fetch_and_add #endif #if !defined(AO_HAVE_short_fetch_and_add) \ && defined(AO_HAVE_short_fetch_and_add_acquire) # define AO_short_fetch_and_add(addr, val) \ AO_short_fetch_and_add_acquire(addr, val) # define AO_HAVE_short_fetch_and_add #endif #if !defined(AO_HAVE_short_fetch_and_add) \ && defined(AO_HAVE_short_fetch_and_add_write) # define AO_short_fetch_and_add(addr, val) \ AO_short_fetch_and_add_write(addr, val) # define AO_HAVE_short_fetch_and_add #endif #if !defined(AO_HAVE_short_fetch_and_add) \ && defined(AO_HAVE_short_fetch_and_add_read) # define AO_short_fetch_and_add(addr, val) \ AO_short_fetch_and_add_read(addr, val) # define AO_HAVE_short_fetch_and_add #endif #if defined(AO_HAVE_short_fetch_and_add_acquire) \ && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_short_fetch_and_add_full) # define AO_short_fetch_and_add_full(addr, val) \ (AO_nop_full(), AO_short_fetch_and_add_acquire(addr, val)) # define AO_HAVE_short_fetch_and_add_full #endif #if !defined(AO_HAVE_short_fetch_and_add_release_write) \ && defined(AO_HAVE_short_fetch_and_add_write) # define AO_short_fetch_and_add_release_write(addr, val) \ AO_short_fetch_and_add_write(addr, val) # define AO_HAVE_short_fetch_and_add_release_write #endif #if !defined(AO_HAVE_short_fetch_and_add_release_write) \ && defined(AO_HAVE_short_fetch_and_add_release) # define AO_short_fetch_and_add_release_write(addr, val) \ AO_short_fetch_and_add_release(addr, val) # define AO_HAVE_short_fetch_and_add_release_write #endif #if !defined(AO_HAVE_short_fetch_and_add_acquire_read) \ && defined(AO_HAVE_short_fetch_and_add_read) # define AO_short_fetch_and_add_acquire_read(addr, val) \ AO_short_fetch_and_add_read(addr, val) # define AO_HAVE_short_fetch_and_add_acquire_read #endif #if !defined(AO_HAVE_short_fetch_and_add_acquire_read) \ && defined(AO_HAVE_short_fetch_and_add_acquire) # define AO_short_fetch_and_add_acquire_read(addr, val) \ AO_short_fetch_and_add_acquire(addr, val) # define AO_HAVE_short_fetch_and_add_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_short_fetch_and_add_acquire_read) # define AO_short_fetch_and_add_dd_acquire_read(addr, val) \ AO_short_fetch_and_add_acquire_read(addr, val) # define AO_HAVE_short_fetch_and_add_dd_acquire_read # endif #else # if defined(AO_HAVE_short_fetch_and_add) # define AO_short_fetch_and_add_dd_acquire_read(addr, val) \ AO_short_fetch_and_add(addr, val) # define AO_HAVE_short_fetch_and_add_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* short_fetch_and_add1 */ #if defined(AO_HAVE_short_fetch_and_add_full) \ && !defined(AO_HAVE_short_fetch_and_add1_full) # define AO_short_fetch_and_add1_full(addr) \ AO_short_fetch_and_add_full(addr, 1) # define AO_HAVE_short_fetch_and_add1_full #endif #if defined(AO_HAVE_short_fetch_and_add_release) \ && !defined(AO_HAVE_short_fetch_and_add1_release) # define AO_short_fetch_and_add1_release(addr) \ AO_short_fetch_and_add_release(addr, 1) # define AO_HAVE_short_fetch_and_add1_release #endif #if defined(AO_HAVE_short_fetch_and_add_acquire) \ && !defined(AO_HAVE_short_fetch_and_add1_acquire) # define AO_short_fetch_and_add1_acquire(addr) \ AO_short_fetch_and_add_acquire(addr, 1) # define AO_HAVE_short_fetch_and_add1_acquire #endif #if defined(AO_HAVE_short_fetch_and_add_write) \ && !defined(AO_HAVE_short_fetch_and_add1_write) # define AO_short_fetch_and_add1_write(addr) \ AO_short_fetch_and_add_write(addr, 1) # define AO_HAVE_short_fetch_and_add1_write #endif #if defined(AO_HAVE_short_fetch_and_add_read) \ && !defined(AO_HAVE_short_fetch_and_add1_read) # define AO_short_fetch_and_add1_read(addr) \ AO_short_fetch_and_add_read(addr, 1) # define AO_HAVE_short_fetch_and_add1_read #endif #if defined(AO_HAVE_short_fetch_and_add_release_write) \ && !defined(AO_HAVE_short_fetch_and_add1_release_write) # define AO_short_fetch_and_add1_release_write(addr) \ AO_short_fetch_and_add_release_write(addr, 1) # define AO_HAVE_short_fetch_and_add1_release_write #endif #if defined(AO_HAVE_short_fetch_and_add_acquire_read) \ && !defined(AO_HAVE_short_fetch_and_add1_acquire_read) # define AO_short_fetch_and_add1_acquire_read(addr) \ AO_short_fetch_and_add_acquire_read(addr, 1) # define AO_HAVE_short_fetch_and_add1_acquire_read #endif #if defined(AO_HAVE_short_fetch_and_add) \ && !defined(AO_HAVE_short_fetch_and_add1) # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add(addr, 1) # define AO_HAVE_short_fetch_and_add1 #endif #if defined(AO_HAVE_short_fetch_and_add1_full) # if !defined(AO_HAVE_short_fetch_and_add1_release) # define AO_short_fetch_and_add1_release(addr) \ AO_short_fetch_and_add1_full(addr) # define AO_HAVE_short_fetch_and_add1_release # endif # if !defined(AO_HAVE_short_fetch_and_add1_acquire) # define AO_short_fetch_and_add1_acquire(addr) \ AO_short_fetch_and_add1_full(addr) # define AO_HAVE_short_fetch_and_add1_acquire # endif # if !defined(AO_HAVE_short_fetch_and_add1_write) # define AO_short_fetch_and_add1_write(addr) \ AO_short_fetch_and_add1_full(addr) # define AO_HAVE_short_fetch_and_add1_write # endif # if !defined(AO_HAVE_short_fetch_and_add1_read) # define AO_short_fetch_and_add1_read(addr) \ AO_short_fetch_and_add1_full(addr) # define AO_HAVE_short_fetch_and_add1_read # endif #endif /* AO_HAVE_short_fetch_and_add1_full */ #if !defined(AO_HAVE_short_fetch_and_add1) \ && defined(AO_HAVE_short_fetch_and_add1_release) # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_release(addr) # define AO_HAVE_short_fetch_and_add1 #endif #if !defined(AO_HAVE_short_fetch_and_add1) \ && defined(AO_HAVE_short_fetch_and_add1_acquire) # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_acquire(addr) # define AO_HAVE_short_fetch_and_add1 #endif #if !defined(AO_HAVE_short_fetch_and_add1) \ && defined(AO_HAVE_short_fetch_and_add1_write) # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_write(addr) # define AO_HAVE_short_fetch_and_add1 #endif #if !defined(AO_HAVE_short_fetch_and_add1) \ && defined(AO_HAVE_short_fetch_and_add1_read) # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_read(addr) # define AO_HAVE_short_fetch_and_add1 #endif #if defined(AO_HAVE_short_fetch_and_add1_acquire) \ && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_short_fetch_and_add1_full) # define AO_short_fetch_and_add1_full(addr) \ (AO_nop_full(), AO_short_fetch_and_add1_acquire(addr)) # define AO_HAVE_short_fetch_and_add1_full #endif #if !defined(AO_HAVE_short_fetch_and_add1_release_write) \ && defined(AO_HAVE_short_fetch_and_add1_write) # define AO_short_fetch_and_add1_release_write(addr) \ AO_short_fetch_and_add1_write(addr) # define AO_HAVE_short_fetch_and_add1_release_write #endif #if !defined(AO_HAVE_short_fetch_and_add1_release_write) \ && defined(AO_HAVE_short_fetch_and_add1_release) # define AO_short_fetch_and_add1_release_write(addr) \ AO_short_fetch_and_add1_release(addr) # define AO_HAVE_short_fetch_and_add1_release_write #endif #if !defined(AO_HAVE_short_fetch_and_add1_acquire_read) \ && defined(AO_HAVE_short_fetch_and_add1_read) # define AO_short_fetch_and_add1_acquire_read(addr) \ AO_short_fetch_and_add1_read(addr) # define AO_HAVE_short_fetch_and_add1_acquire_read #endif #if !defined(AO_HAVE_short_fetch_and_add1_acquire_read) \ && defined(AO_HAVE_short_fetch_and_add1_acquire) # define AO_short_fetch_and_add1_acquire_read(addr) \ AO_short_fetch_and_add1_acquire(addr) # define AO_HAVE_short_fetch_and_add1_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_short_fetch_and_add1_acquire_read) # define AO_short_fetch_and_add1_dd_acquire_read(addr) \ AO_short_fetch_and_add1_acquire_read(addr) # define AO_HAVE_short_fetch_and_add1_dd_acquire_read # endif #else # if defined(AO_HAVE_short_fetch_and_add1) # define AO_short_fetch_and_add1_dd_acquire_read(addr) \ AO_short_fetch_and_add1(addr) # define AO_HAVE_short_fetch_and_add1_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* short_fetch_and_sub1 */ #if defined(AO_HAVE_short_fetch_and_add_full) \ && !defined(AO_HAVE_short_fetch_and_sub1_full) # define AO_short_fetch_and_sub1_full(addr) \ AO_short_fetch_and_add_full(addr, (unsigned/**/short)(-1)) # define AO_HAVE_short_fetch_and_sub1_full #endif #if defined(AO_HAVE_short_fetch_and_add_release) \ && !defined(AO_HAVE_short_fetch_and_sub1_release) # define AO_short_fetch_and_sub1_release(addr) \ AO_short_fetch_and_add_release(addr, (unsigned/**/short)(-1)) # define AO_HAVE_short_fetch_and_sub1_release #endif #if defined(AO_HAVE_short_fetch_and_add_acquire) \ && !defined(AO_HAVE_short_fetch_and_sub1_acquire) # define AO_short_fetch_and_sub1_acquire(addr) \ AO_short_fetch_and_add_acquire(addr, (unsigned/**/short)(-1)) # define AO_HAVE_short_fetch_and_sub1_acquire #endif #if defined(AO_HAVE_short_fetch_and_add_write) \ && !defined(AO_HAVE_short_fetch_and_sub1_write) # define AO_short_fetch_and_sub1_write(addr) \ AO_short_fetch_and_add_write(addr, (unsigned/**/short)(-1)) # define AO_HAVE_short_fetch_and_sub1_write #endif #if defined(AO_HAVE_short_fetch_and_add_read) \ && !defined(AO_HAVE_short_fetch_and_sub1_read) # define AO_short_fetch_and_sub1_read(addr) \ AO_short_fetch_and_add_read(addr, (unsigned/**/short)(-1)) # define AO_HAVE_short_fetch_and_sub1_read #endif #if defined(AO_HAVE_short_fetch_and_add_release_write) \ && !defined(AO_HAVE_short_fetch_and_sub1_release_write) # define AO_short_fetch_and_sub1_release_write(addr) \ AO_short_fetch_and_add_release_write(addr, (unsigned/**/short)(-1)) # define AO_HAVE_short_fetch_and_sub1_release_write #endif #if defined(AO_HAVE_short_fetch_and_add_acquire_read) \ && !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) # define AO_short_fetch_and_sub1_acquire_read(addr) \ AO_short_fetch_and_add_acquire_read(addr, (unsigned/**/short)(-1)) # define AO_HAVE_short_fetch_and_sub1_acquire_read #endif #if defined(AO_HAVE_short_fetch_and_add) \ && !defined(AO_HAVE_short_fetch_and_sub1) # define AO_short_fetch_and_sub1(addr) \ AO_short_fetch_and_add(addr, (unsigned/**/short)(-1)) # define AO_HAVE_short_fetch_and_sub1 #endif #if defined(AO_HAVE_short_fetch_and_sub1_full) # if !defined(AO_HAVE_short_fetch_and_sub1_release) # define AO_short_fetch_and_sub1_release(addr) \ AO_short_fetch_and_sub1_full(addr) # define AO_HAVE_short_fetch_and_sub1_release # endif # if !defined(AO_HAVE_short_fetch_and_sub1_acquire) # define AO_short_fetch_and_sub1_acquire(addr) \ AO_short_fetch_and_sub1_full(addr) # define AO_HAVE_short_fetch_and_sub1_acquire # endif # if !defined(AO_HAVE_short_fetch_and_sub1_write) # define AO_short_fetch_and_sub1_write(addr) \ AO_short_fetch_and_sub1_full(addr) # define AO_HAVE_short_fetch_and_sub1_write # endif # if !defined(AO_HAVE_short_fetch_and_sub1_read) # define AO_short_fetch_and_sub1_read(addr) \ AO_short_fetch_and_sub1_full(addr) # define AO_HAVE_short_fetch_and_sub1_read # endif #endif /* AO_HAVE_short_fetch_and_sub1_full */ #if !defined(AO_HAVE_short_fetch_and_sub1) \ && defined(AO_HAVE_short_fetch_and_sub1_release) # define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_release(addr) # define AO_HAVE_short_fetch_and_sub1 #endif #if !defined(AO_HAVE_short_fetch_and_sub1) \ && defined(AO_HAVE_short_fetch_and_sub1_acquire) # define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_acquire(addr) # define AO_HAVE_short_fetch_and_sub1 #endif #if !defined(AO_HAVE_short_fetch_and_sub1) \ && defined(AO_HAVE_short_fetch_and_sub1_write) # define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_write(addr) # define AO_HAVE_short_fetch_and_sub1 #endif #if !defined(AO_HAVE_short_fetch_and_sub1) \ && defined(AO_HAVE_short_fetch_and_sub1_read) # define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_read(addr) # define AO_HAVE_short_fetch_and_sub1 #endif #if defined(AO_HAVE_short_fetch_and_sub1_acquire) \ && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_short_fetch_and_sub1_full) # define AO_short_fetch_and_sub1_full(addr) \ (AO_nop_full(), AO_short_fetch_and_sub1_acquire(addr)) # define AO_HAVE_short_fetch_and_sub1_full #endif #if !defined(AO_HAVE_short_fetch_and_sub1_release_write) \ && defined(AO_HAVE_short_fetch_and_sub1_write) # define AO_short_fetch_and_sub1_release_write(addr) \ AO_short_fetch_and_sub1_write(addr) # define AO_HAVE_short_fetch_and_sub1_release_write #endif #if !defined(AO_HAVE_short_fetch_and_sub1_release_write) \ && defined(AO_HAVE_short_fetch_and_sub1_release) # define AO_short_fetch_and_sub1_release_write(addr) \ AO_short_fetch_and_sub1_release(addr) # define AO_HAVE_short_fetch_and_sub1_release_write #endif #if !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) \ && defined(AO_HAVE_short_fetch_and_sub1_read) # define AO_short_fetch_and_sub1_acquire_read(addr) \ AO_short_fetch_and_sub1_read(addr) # define AO_HAVE_short_fetch_and_sub1_acquire_read #endif #if !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) \ && defined(AO_HAVE_short_fetch_and_sub1_acquire) # define AO_short_fetch_and_sub1_acquire_read(addr) \ AO_short_fetch_and_sub1_acquire(addr) # define AO_HAVE_short_fetch_and_sub1_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_short_fetch_and_sub1_acquire_read) # define AO_short_fetch_and_sub1_dd_acquire_read(addr) \ AO_short_fetch_and_sub1_acquire_read(addr) # define AO_HAVE_short_fetch_and_sub1_dd_acquire_read # endif #else # if defined(AO_HAVE_short_fetch_and_sub1) # define AO_short_fetch_and_sub1_dd_acquire_read(addr) \ AO_short_fetch_and_sub1(addr) # define AO_HAVE_short_fetch_and_sub1_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* short_and */ #if defined(AO_HAVE_short_compare_and_swap_full) \ && !defined(AO_HAVE_short_and_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_short_and_full(volatile unsigned/**/short *addr, unsigned/**/short value) { unsigned/**/short old; do { old = *(unsigned/**/short *)addr; } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old, old & value))); } # define AO_HAVE_short_and_full #endif #if defined(AO_HAVE_short_and_full) # if !defined(AO_HAVE_short_and_release) # define AO_short_and_release(addr, val) AO_short_and_full(addr, val) # define AO_HAVE_short_and_release # endif # if !defined(AO_HAVE_short_and_acquire) # define AO_short_and_acquire(addr, val) AO_short_and_full(addr, val) # define AO_HAVE_short_and_acquire # endif # if !defined(AO_HAVE_short_and_write) # define AO_short_and_write(addr, val) AO_short_and_full(addr, val) # define AO_HAVE_short_and_write # endif # if !defined(AO_HAVE_short_and_read) # define AO_short_and_read(addr, val) AO_short_and_full(addr, val) # define AO_HAVE_short_and_read # endif #endif /* AO_HAVE_short_and_full */ #if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_release) # define AO_short_and(addr, val) AO_short_and_release(addr, val) # define AO_HAVE_short_and #endif #if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_acquire) # define AO_short_and(addr, val) AO_short_and_acquire(addr, val) # define AO_HAVE_short_and #endif #if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_write) # define AO_short_and(addr, val) AO_short_and_write(addr, val) # define AO_HAVE_short_and #endif #if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_read) # define AO_short_and(addr, val) AO_short_and_read(addr, val) # define AO_HAVE_short_and #endif #if defined(AO_HAVE_short_and_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_short_and_full) # define AO_short_and_full(addr, val) \ (AO_nop_full(), AO_short_and_acquire(addr, val)) # define AO_HAVE_short_and_full #endif #if !defined(AO_HAVE_short_and_release_write) \ && defined(AO_HAVE_short_and_write) # define AO_short_and_release_write(addr, val) AO_short_and_write(addr, val) # define AO_HAVE_short_and_release_write #endif #if !defined(AO_HAVE_short_and_release_write) \ && defined(AO_HAVE_short_and_release) # define AO_short_and_release_write(addr, val) AO_short_and_release(addr, val) # define AO_HAVE_short_and_release_write #endif #if !defined(AO_HAVE_short_and_acquire_read) \ && defined(AO_HAVE_short_and_read) # define AO_short_and_acquire_read(addr, val) AO_short_and_read(addr, val) # define AO_HAVE_short_and_acquire_read #endif #if !defined(AO_HAVE_short_and_acquire_read) \ && defined(AO_HAVE_short_and_acquire) # define AO_short_and_acquire_read(addr, val) AO_short_and_acquire(addr, val) # define AO_HAVE_short_and_acquire_read #endif /* short_or */ #if defined(AO_HAVE_short_compare_and_swap_full) \ && !defined(AO_HAVE_short_or_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_short_or_full(volatile unsigned/**/short *addr, unsigned/**/short value) { unsigned/**/short old; do { old = *(unsigned/**/short *)addr; } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old, old | value))); } # define AO_HAVE_short_or_full #endif #if defined(AO_HAVE_short_or_full) # if !defined(AO_HAVE_short_or_release) # define AO_short_or_release(addr, val) AO_short_or_full(addr, val) # define AO_HAVE_short_or_release # endif # if !defined(AO_HAVE_short_or_acquire) # define AO_short_or_acquire(addr, val) AO_short_or_full(addr, val) # define AO_HAVE_short_or_acquire # endif # if !defined(AO_HAVE_short_or_write) # define AO_short_or_write(addr, val) AO_short_or_full(addr, val) # define AO_HAVE_short_or_write # endif # if !defined(AO_HAVE_short_or_read) # define AO_short_or_read(addr, val) AO_short_or_full(addr, val) # define AO_HAVE_short_or_read # endif #endif /* AO_HAVE_short_or_full */ #if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_release) # define AO_short_or(addr, val) AO_short_or_release(addr, val) # define AO_HAVE_short_or #endif #if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_acquire) # define AO_short_or(addr, val) AO_short_or_acquire(addr, val) # define AO_HAVE_short_or #endif #if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_write) # define AO_short_or(addr, val) AO_short_or_write(addr, val) # define AO_HAVE_short_or #endif #if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_read) # define AO_short_or(addr, val) AO_short_or_read(addr, val) # define AO_HAVE_short_or #endif #if defined(AO_HAVE_short_or_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_short_or_full) # define AO_short_or_full(addr, val) \ (AO_nop_full(), AO_short_or_acquire(addr, val)) # define AO_HAVE_short_or_full #endif #if !defined(AO_HAVE_short_or_release_write) \ && defined(AO_HAVE_short_or_write) # define AO_short_or_release_write(addr, val) AO_short_or_write(addr, val) # define AO_HAVE_short_or_release_write #endif #if !defined(AO_HAVE_short_or_release_write) \ && defined(AO_HAVE_short_or_release) # define AO_short_or_release_write(addr, val) AO_short_or_release(addr, val) # define AO_HAVE_short_or_release_write #endif #if !defined(AO_HAVE_short_or_acquire_read) && defined(AO_HAVE_short_or_read) # define AO_short_or_acquire_read(addr, val) AO_short_or_read(addr, val) # define AO_HAVE_short_or_acquire_read #endif #if !defined(AO_HAVE_short_or_acquire_read) \ && defined(AO_HAVE_short_or_acquire) # define AO_short_or_acquire_read(addr, val) AO_short_or_acquire(addr, val) # define AO_HAVE_short_or_acquire_read #endif /* short_xor */ #if defined(AO_HAVE_short_compare_and_swap_full) \ && !defined(AO_HAVE_short_xor_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_short_xor_full(volatile unsigned/**/short *addr, unsigned/**/short value) { unsigned/**/short old; do { old = *(unsigned/**/short *)addr; } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old, old ^ value))); } # define AO_HAVE_short_xor_full #endif #if defined(AO_HAVE_short_xor_full) # if !defined(AO_HAVE_short_xor_release) # define AO_short_xor_release(addr, val) AO_short_xor_full(addr, val) # define AO_HAVE_short_xor_release # endif # if !defined(AO_HAVE_short_xor_acquire) # define AO_short_xor_acquire(addr, val) AO_short_xor_full(addr, val) # define AO_HAVE_short_xor_acquire # endif # if !defined(AO_HAVE_short_xor_write) # define AO_short_xor_write(addr, val) AO_short_xor_full(addr, val) # define AO_HAVE_short_xor_write # endif # if !defined(AO_HAVE_short_xor_read) # define AO_short_xor_read(addr, val) AO_short_xor_full(addr, val) # define AO_HAVE_short_xor_read # endif #endif /* AO_HAVE_short_xor_full */ #if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_release) # define AO_short_xor(addr, val) AO_short_xor_release(addr, val) # define AO_HAVE_short_xor #endif #if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_acquire) # define AO_short_xor(addr, val) AO_short_xor_acquire(addr, val) # define AO_HAVE_short_xor #endif #if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_write) # define AO_short_xor(addr, val) AO_short_xor_write(addr, val) # define AO_HAVE_short_xor #endif #if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_read) # define AO_short_xor(addr, val) AO_short_xor_read(addr, val) # define AO_HAVE_short_xor #endif #if defined(AO_HAVE_short_xor_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_short_xor_full) # define AO_short_xor_full(addr, val) \ (AO_nop_full(), AO_short_xor_acquire(addr, val)) # define AO_HAVE_short_xor_full #endif #if !defined(AO_HAVE_short_xor_release_write) \ && defined(AO_HAVE_short_xor_write) # define AO_short_xor_release_write(addr, val) AO_short_xor_write(addr, val) # define AO_HAVE_short_xor_release_write #endif #if !defined(AO_HAVE_short_xor_release_write) \ && defined(AO_HAVE_short_xor_release) # define AO_short_xor_release_write(addr, val) AO_short_xor_release(addr, val) # define AO_HAVE_short_xor_release_write #endif #if !defined(AO_HAVE_short_xor_acquire_read) \ && defined(AO_HAVE_short_xor_read) # define AO_short_xor_acquire_read(addr, val) AO_short_xor_read(addr, val) # define AO_HAVE_short_xor_acquire_read #endif #if !defined(AO_HAVE_short_xor_acquire_read) \ && defined(AO_HAVE_short_xor_acquire) # define AO_short_xor_acquire_read(addr, val) AO_short_xor_acquire(addr, val) # define AO_HAVE_short_xor_acquire_read #endif /* short_and/or/xor_dd_acquire_read are meaningless. */ /* * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* int_compare_and_swap (based on fetch_compare_and_swap) */ #if defined(AO_HAVE_int_fetch_compare_and_swap_full) \ && !defined(AO_HAVE_int_compare_and_swap_full) AO_INLINE int AO_int_compare_and_swap_full(volatile unsigned *addr, unsigned old_val, unsigned new_val) { return AO_int_fetch_compare_and_swap_full(addr, old_val, new_val) == old_val; } # define AO_HAVE_int_compare_and_swap_full #endif #if defined(AO_HAVE_int_fetch_compare_and_swap_acquire) \ && !defined(AO_HAVE_int_compare_and_swap_acquire) AO_INLINE int AO_int_compare_and_swap_acquire(volatile unsigned *addr, unsigned old_val, unsigned new_val) { return AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val) == old_val; } # define AO_HAVE_int_compare_and_swap_acquire #endif #if defined(AO_HAVE_int_fetch_compare_and_swap_release) \ && !defined(AO_HAVE_int_compare_and_swap_release) AO_INLINE int AO_int_compare_and_swap_release(volatile unsigned *addr, unsigned old_val, unsigned new_val) { return AO_int_fetch_compare_and_swap_release(addr, old_val, new_val) == old_val; } # define AO_HAVE_int_compare_and_swap_release #endif #if defined(AO_HAVE_int_fetch_compare_and_swap_write) \ && !defined(AO_HAVE_int_compare_and_swap_write) AO_INLINE int AO_int_compare_and_swap_write(volatile unsigned *addr, unsigned old_val, unsigned new_val) { return AO_int_fetch_compare_and_swap_write(addr, old_val, new_val) == old_val; } # define AO_HAVE_int_compare_and_swap_write #endif #if defined(AO_HAVE_int_fetch_compare_and_swap_read) \ && !defined(AO_HAVE_int_compare_and_swap_read) AO_INLINE int AO_int_compare_and_swap_read(volatile unsigned *addr, unsigned old_val, unsigned new_val) { return AO_int_fetch_compare_and_swap_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_int_compare_and_swap_read #endif #if defined(AO_HAVE_int_fetch_compare_and_swap) \ && !defined(AO_HAVE_int_compare_and_swap) AO_INLINE int AO_int_compare_and_swap(volatile unsigned *addr, unsigned old_val, unsigned new_val) { return AO_int_fetch_compare_and_swap(addr, old_val, new_val) == old_val; } # define AO_HAVE_int_compare_and_swap #endif #if defined(AO_HAVE_int_fetch_compare_and_swap_release_write) \ && !defined(AO_HAVE_int_compare_and_swap_release_write) AO_INLINE int AO_int_compare_and_swap_release_write(volatile unsigned *addr, unsigned old_val, unsigned new_val) { return AO_int_fetch_compare_and_swap_release_write(addr, old_val, new_val) == old_val; } # define AO_HAVE_int_compare_and_swap_release_write #endif #if defined(AO_HAVE_int_fetch_compare_and_swap_acquire_read) \ && !defined(AO_HAVE_int_compare_and_swap_acquire_read) AO_INLINE int AO_int_compare_and_swap_acquire_read(volatile unsigned *addr, unsigned old_val, unsigned new_val) { return AO_int_fetch_compare_and_swap_acquire_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_int_compare_and_swap_acquire_read #endif #if defined(AO_HAVE_int_fetch_compare_and_swap_dd_acquire_read) \ && !defined(AO_HAVE_int_compare_and_swap_dd_acquire_read) AO_INLINE int AO_int_compare_and_swap_dd_acquire_read(volatile unsigned *addr, unsigned old_val, unsigned new_val) { return AO_int_fetch_compare_and_swap_dd_acquire_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_int_compare_and_swap_dd_acquire_read #endif /* int_fetch_and_add */ /* We first try to implement fetch_and_add variants in terms of the */ /* corresponding compare_and_swap variants to minimize adding barriers. */ #if defined(AO_HAVE_int_compare_and_swap_full) \ && !defined(AO_HAVE_int_fetch_and_add_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE unsigned AO_int_fetch_and_add_full(volatile unsigned *addr, unsigned incr) { unsigned old; do { old = *(unsigned *)addr; } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old, old + incr))); return old; } # define AO_HAVE_int_fetch_and_add_full #endif #if defined(AO_HAVE_int_compare_and_swap_acquire) \ && !defined(AO_HAVE_int_fetch_and_add_acquire) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE unsigned AO_int_fetch_and_add_acquire(volatile unsigned *addr, unsigned incr) { unsigned old; do { old = *(unsigned *)addr; } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_acquire(addr, old, old + incr))); return old; } # define AO_HAVE_int_fetch_and_add_acquire #endif #if defined(AO_HAVE_int_compare_and_swap_release) \ && !defined(AO_HAVE_int_fetch_and_add_release) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE unsigned AO_int_fetch_and_add_release(volatile unsigned *addr, unsigned incr) { unsigned old; do { old = *(unsigned *)addr; } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_release(addr, old, old + incr))); return old; } # define AO_HAVE_int_fetch_and_add_release #endif #if defined(AO_HAVE_int_compare_and_swap) \ && !defined(AO_HAVE_int_fetch_and_add) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE unsigned AO_int_fetch_and_add(volatile unsigned *addr, unsigned incr) { unsigned old; do { old = *(unsigned *)addr; } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap(addr, old, old + incr))); return old; } # define AO_HAVE_int_fetch_and_add #endif #if defined(AO_HAVE_int_fetch_and_add_full) # if !defined(AO_HAVE_int_fetch_and_add_release) # define AO_int_fetch_and_add_release(addr, val) \ AO_int_fetch_and_add_full(addr, val) # define AO_HAVE_int_fetch_and_add_release # endif # if !defined(AO_HAVE_int_fetch_and_add_acquire) # define AO_int_fetch_and_add_acquire(addr, val) \ AO_int_fetch_and_add_full(addr, val) # define AO_HAVE_int_fetch_and_add_acquire # endif # if !defined(AO_HAVE_int_fetch_and_add_write) # define AO_int_fetch_and_add_write(addr, val) \ AO_int_fetch_and_add_full(addr, val) # define AO_HAVE_int_fetch_and_add_write # endif # if !defined(AO_HAVE_int_fetch_and_add_read) # define AO_int_fetch_and_add_read(addr, val) \ AO_int_fetch_and_add_full(addr, val) # define AO_HAVE_int_fetch_and_add_read # endif #endif /* AO_HAVE_int_fetch_and_add_full */ #if defined(AO_HAVE_int_fetch_and_add) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_int_fetch_and_add_acquire) AO_INLINE unsigned AO_int_fetch_and_add_acquire(volatile unsigned *addr, unsigned incr) { unsigned result = AO_int_fetch_and_add(addr, incr); AO_nop_full(); return result; } # define AO_HAVE_int_fetch_and_add_acquire #endif #if defined(AO_HAVE_int_fetch_and_add) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_int_fetch_and_add_release) # define AO_int_fetch_and_add_release(addr, incr) \ (AO_nop_full(), AO_int_fetch_and_add(addr, incr)) # define AO_HAVE_int_fetch_and_add_release #endif #if !defined(AO_HAVE_int_fetch_and_add) \ && defined(AO_HAVE_int_fetch_and_add_release) # define AO_int_fetch_and_add(addr, val) \ AO_int_fetch_and_add_release(addr, val) # define AO_HAVE_int_fetch_and_add #endif #if !defined(AO_HAVE_int_fetch_and_add) \ && defined(AO_HAVE_int_fetch_and_add_acquire) # define AO_int_fetch_and_add(addr, val) \ AO_int_fetch_and_add_acquire(addr, val) # define AO_HAVE_int_fetch_and_add #endif #if !defined(AO_HAVE_int_fetch_and_add) \ && defined(AO_HAVE_int_fetch_and_add_write) # define AO_int_fetch_and_add(addr, val) \ AO_int_fetch_and_add_write(addr, val) # define AO_HAVE_int_fetch_and_add #endif #if !defined(AO_HAVE_int_fetch_and_add) \ && defined(AO_HAVE_int_fetch_and_add_read) # define AO_int_fetch_and_add(addr, val) \ AO_int_fetch_and_add_read(addr, val) # define AO_HAVE_int_fetch_and_add #endif #if defined(AO_HAVE_int_fetch_and_add_acquire) \ && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_int_fetch_and_add_full) # define AO_int_fetch_and_add_full(addr, val) \ (AO_nop_full(), AO_int_fetch_and_add_acquire(addr, val)) # define AO_HAVE_int_fetch_and_add_full #endif #if !defined(AO_HAVE_int_fetch_and_add_release_write) \ && defined(AO_HAVE_int_fetch_and_add_write) # define AO_int_fetch_and_add_release_write(addr, val) \ AO_int_fetch_and_add_write(addr, val) # define AO_HAVE_int_fetch_and_add_release_write #endif #if !defined(AO_HAVE_int_fetch_and_add_release_write) \ && defined(AO_HAVE_int_fetch_and_add_release) # define AO_int_fetch_and_add_release_write(addr, val) \ AO_int_fetch_and_add_release(addr, val) # define AO_HAVE_int_fetch_and_add_release_write #endif #if !defined(AO_HAVE_int_fetch_and_add_acquire_read) \ && defined(AO_HAVE_int_fetch_and_add_read) # define AO_int_fetch_and_add_acquire_read(addr, val) \ AO_int_fetch_and_add_read(addr, val) # define AO_HAVE_int_fetch_and_add_acquire_read #endif #if !defined(AO_HAVE_int_fetch_and_add_acquire_read) \ && defined(AO_HAVE_int_fetch_and_add_acquire) # define AO_int_fetch_and_add_acquire_read(addr, val) \ AO_int_fetch_and_add_acquire(addr, val) # define AO_HAVE_int_fetch_and_add_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_int_fetch_and_add_acquire_read) # define AO_int_fetch_and_add_dd_acquire_read(addr, val) \ AO_int_fetch_and_add_acquire_read(addr, val) # define AO_HAVE_int_fetch_and_add_dd_acquire_read # endif #else # if defined(AO_HAVE_int_fetch_and_add) # define AO_int_fetch_and_add_dd_acquire_read(addr, val) \ AO_int_fetch_and_add(addr, val) # define AO_HAVE_int_fetch_and_add_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* int_fetch_and_add1 */ #if defined(AO_HAVE_int_fetch_and_add_full) \ && !defined(AO_HAVE_int_fetch_and_add1_full) # define AO_int_fetch_and_add1_full(addr) \ AO_int_fetch_and_add_full(addr, 1) # define AO_HAVE_int_fetch_and_add1_full #endif #if defined(AO_HAVE_int_fetch_and_add_release) \ && !defined(AO_HAVE_int_fetch_and_add1_release) # define AO_int_fetch_and_add1_release(addr) \ AO_int_fetch_and_add_release(addr, 1) # define AO_HAVE_int_fetch_and_add1_release #endif #if defined(AO_HAVE_int_fetch_and_add_acquire) \ && !defined(AO_HAVE_int_fetch_and_add1_acquire) # define AO_int_fetch_and_add1_acquire(addr) \ AO_int_fetch_and_add_acquire(addr, 1) # define AO_HAVE_int_fetch_and_add1_acquire #endif #if defined(AO_HAVE_int_fetch_and_add_write) \ && !defined(AO_HAVE_int_fetch_and_add1_write) # define AO_int_fetch_and_add1_write(addr) \ AO_int_fetch_and_add_write(addr, 1) # define AO_HAVE_int_fetch_and_add1_write #endif #if defined(AO_HAVE_int_fetch_and_add_read) \ && !defined(AO_HAVE_int_fetch_and_add1_read) # define AO_int_fetch_and_add1_read(addr) \ AO_int_fetch_and_add_read(addr, 1) # define AO_HAVE_int_fetch_and_add1_read #endif #if defined(AO_HAVE_int_fetch_and_add_release_write) \ && !defined(AO_HAVE_int_fetch_and_add1_release_write) # define AO_int_fetch_and_add1_release_write(addr) \ AO_int_fetch_and_add_release_write(addr, 1) # define AO_HAVE_int_fetch_and_add1_release_write #endif #if defined(AO_HAVE_int_fetch_and_add_acquire_read) \ && !defined(AO_HAVE_int_fetch_and_add1_acquire_read) # define AO_int_fetch_and_add1_acquire_read(addr) \ AO_int_fetch_and_add_acquire_read(addr, 1) # define AO_HAVE_int_fetch_and_add1_acquire_read #endif #if defined(AO_HAVE_int_fetch_and_add) \ && !defined(AO_HAVE_int_fetch_and_add1) # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add(addr, 1) # define AO_HAVE_int_fetch_and_add1 #endif #if defined(AO_HAVE_int_fetch_and_add1_full) # if !defined(AO_HAVE_int_fetch_and_add1_release) # define AO_int_fetch_and_add1_release(addr) \ AO_int_fetch_and_add1_full(addr) # define AO_HAVE_int_fetch_and_add1_release # endif # if !defined(AO_HAVE_int_fetch_and_add1_acquire) # define AO_int_fetch_and_add1_acquire(addr) \ AO_int_fetch_and_add1_full(addr) # define AO_HAVE_int_fetch_and_add1_acquire # endif # if !defined(AO_HAVE_int_fetch_and_add1_write) # define AO_int_fetch_and_add1_write(addr) \ AO_int_fetch_and_add1_full(addr) # define AO_HAVE_int_fetch_and_add1_write # endif # if !defined(AO_HAVE_int_fetch_and_add1_read) # define AO_int_fetch_and_add1_read(addr) \ AO_int_fetch_and_add1_full(addr) # define AO_HAVE_int_fetch_and_add1_read # endif #endif /* AO_HAVE_int_fetch_and_add1_full */ #if !defined(AO_HAVE_int_fetch_and_add1) \ && defined(AO_HAVE_int_fetch_and_add1_release) # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_release(addr) # define AO_HAVE_int_fetch_and_add1 #endif #if !defined(AO_HAVE_int_fetch_and_add1) \ && defined(AO_HAVE_int_fetch_and_add1_acquire) # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_acquire(addr) # define AO_HAVE_int_fetch_and_add1 #endif #if !defined(AO_HAVE_int_fetch_and_add1) \ && defined(AO_HAVE_int_fetch_and_add1_write) # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_write(addr) # define AO_HAVE_int_fetch_and_add1 #endif #if !defined(AO_HAVE_int_fetch_and_add1) \ && defined(AO_HAVE_int_fetch_and_add1_read) # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_read(addr) # define AO_HAVE_int_fetch_and_add1 #endif #if defined(AO_HAVE_int_fetch_and_add1_acquire) \ && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_int_fetch_and_add1_full) # define AO_int_fetch_and_add1_full(addr) \ (AO_nop_full(), AO_int_fetch_and_add1_acquire(addr)) # define AO_HAVE_int_fetch_and_add1_full #endif #if !defined(AO_HAVE_int_fetch_and_add1_release_write) \ && defined(AO_HAVE_int_fetch_and_add1_write) # define AO_int_fetch_and_add1_release_write(addr) \ AO_int_fetch_and_add1_write(addr) # define AO_HAVE_int_fetch_and_add1_release_write #endif #if !defined(AO_HAVE_int_fetch_and_add1_release_write) \ && defined(AO_HAVE_int_fetch_and_add1_release) # define AO_int_fetch_and_add1_release_write(addr) \ AO_int_fetch_and_add1_release(addr) # define AO_HAVE_int_fetch_and_add1_release_write #endif #if !defined(AO_HAVE_int_fetch_and_add1_acquire_read) \ && defined(AO_HAVE_int_fetch_and_add1_read) # define AO_int_fetch_and_add1_acquire_read(addr) \ AO_int_fetch_and_add1_read(addr) # define AO_HAVE_int_fetch_and_add1_acquire_read #endif #if !defined(AO_HAVE_int_fetch_and_add1_acquire_read) \ && defined(AO_HAVE_int_fetch_and_add1_acquire) # define AO_int_fetch_and_add1_acquire_read(addr) \ AO_int_fetch_and_add1_acquire(addr) # define AO_HAVE_int_fetch_and_add1_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_int_fetch_and_add1_acquire_read) # define AO_int_fetch_and_add1_dd_acquire_read(addr) \ AO_int_fetch_and_add1_acquire_read(addr) # define AO_HAVE_int_fetch_and_add1_dd_acquire_read # endif #else # if defined(AO_HAVE_int_fetch_and_add1) # define AO_int_fetch_and_add1_dd_acquire_read(addr) \ AO_int_fetch_and_add1(addr) # define AO_HAVE_int_fetch_and_add1_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* int_fetch_and_sub1 */ #if defined(AO_HAVE_int_fetch_and_add_full) \ && !defined(AO_HAVE_int_fetch_and_sub1_full) # define AO_int_fetch_and_sub1_full(addr) \ AO_int_fetch_and_add_full(addr, (unsigned)(-1)) # define AO_HAVE_int_fetch_and_sub1_full #endif #if defined(AO_HAVE_int_fetch_and_add_release) \ && !defined(AO_HAVE_int_fetch_and_sub1_release) # define AO_int_fetch_and_sub1_release(addr) \ AO_int_fetch_and_add_release(addr, (unsigned)(-1)) # define AO_HAVE_int_fetch_and_sub1_release #endif #if defined(AO_HAVE_int_fetch_and_add_acquire) \ && !defined(AO_HAVE_int_fetch_and_sub1_acquire) # define AO_int_fetch_and_sub1_acquire(addr) \ AO_int_fetch_and_add_acquire(addr, (unsigned)(-1)) # define AO_HAVE_int_fetch_and_sub1_acquire #endif #if defined(AO_HAVE_int_fetch_and_add_write) \ && !defined(AO_HAVE_int_fetch_and_sub1_write) # define AO_int_fetch_and_sub1_write(addr) \ AO_int_fetch_and_add_write(addr, (unsigned)(-1)) # define AO_HAVE_int_fetch_and_sub1_write #endif #if defined(AO_HAVE_int_fetch_and_add_read) \ && !defined(AO_HAVE_int_fetch_and_sub1_read) # define AO_int_fetch_and_sub1_read(addr) \ AO_int_fetch_and_add_read(addr, (unsigned)(-1)) # define AO_HAVE_int_fetch_and_sub1_read #endif #if defined(AO_HAVE_int_fetch_and_add_release_write) \ && !defined(AO_HAVE_int_fetch_and_sub1_release_write) # define AO_int_fetch_and_sub1_release_write(addr) \ AO_int_fetch_and_add_release_write(addr, (unsigned)(-1)) # define AO_HAVE_int_fetch_and_sub1_release_write #endif #if defined(AO_HAVE_int_fetch_and_add_acquire_read) \ && !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) # define AO_int_fetch_and_sub1_acquire_read(addr) \ AO_int_fetch_and_add_acquire_read(addr, (unsigned)(-1)) # define AO_HAVE_int_fetch_and_sub1_acquire_read #endif #if defined(AO_HAVE_int_fetch_and_add) \ && !defined(AO_HAVE_int_fetch_and_sub1) # define AO_int_fetch_and_sub1(addr) \ AO_int_fetch_and_add(addr, (unsigned)(-1)) # define AO_HAVE_int_fetch_and_sub1 #endif #if defined(AO_HAVE_int_fetch_and_sub1_full) # if !defined(AO_HAVE_int_fetch_and_sub1_release) # define AO_int_fetch_and_sub1_release(addr) \ AO_int_fetch_and_sub1_full(addr) # define AO_HAVE_int_fetch_and_sub1_release # endif # if !defined(AO_HAVE_int_fetch_and_sub1_acquire) # define AO_int_fetch_and_sub1_acquire(addr) \ AO_int_fetch_and_sub1_full(addr) # define AO_HAVE_int_fetch_and_sub1_acquire # endif # if !defined(AO_HAVE_int_fetch_and_sub1_write) # define AO_int_fetch_and_sub1_write(addr) \ AO_int_fetch_and_sub1_full(addr) # define AO_HAVE_int_fetch_and_sub1_write # endif # if !defined(AO_HAVE_int_fetch_and_sub1_read) # define AO_int_fetch_and_sub1_read(addr) \ AO_int_fetch_and_sub1_full(addr) # define AO_HAVE_int_fetch_and_sub1_read # endif #endif /* AO_HAVE_int_fetch_and_sub1_full */ #if !defined(AO_HAVE_int_fetch_and_sub1) \ && defined(AO_HAVE_int_fetch_and_sub1_release) # define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_release(addr) # define AO_HAVE_int_fetch_and_sub1 #endif #if !defined(AO_HAVE_int_fetch_and_sub1) \ && defined(AO_HAVE_int_fetch_and_sub1_acquire) # define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_acquire(addr) # define AO_HAVE_int_fetch_and_sub1 #endif #if !defined(AO_HAVE_int_fetch_and_sub1) \ && defined(AO_HAVE_int_fetch_and_sub1_write) # define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_write(addr) # define AO_HAVE_int_fetch_and_sub1 #endif #if !defined(AO_HAVE_int_fetch_and_sub1) \ && defined(AO_HAVE_int_fetch_and_sub1_read) # define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_read(addr) # define AO_HAVE_int_fetch_and_sub1 #endif #if defined(AO_HAVE_int_fetch_and_sub1_acquire) \ && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_int_fetch_and_sub1_full) # define AO_int_fetch_and_sub1_full(addr) \ (AO_nop_full(), AO_int_fetch_and_sub1_acquire(addr)) # define AO_HAVE_int_fetch_and_sub1_full #endif #if !defined(AO_HAVE_int_fetch_and_sub1_release_write) \ && defined(AO_HAVE_int_fetch_and_sub1_write) # define AO_int_fetch_and_sub1_release_write(addr) \ AO_int_fetch_and_sub1_write(addr) # define AO_HAVE_int_fetch_and_sub1_release_write #endif #if !defined(AO_HAVE_int_fetch_and_sub1_release_write) \ && defined(AO_HAVE_int_fetch_and_sub1_release) # define AO_int_fetch_and_sub1_release_write(addr) \ AO_int_fetch_and_sub1_release(addr) # define AO_HAVE_int_fetch_and_sub1_release_write #endif #if !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) \ && defined(AO_HAVE_int_fetch_and_sub1_read) # define AO_int_fetch_and_sub1_acquire_read(addr) \ AO_int_fetch_and_sub1_read(addr) # define AO_HAVE_int_fetch_and_sub1_acquire_read #endif #if !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) \ && defined(AO_HAVE_int_fetch_and_sub1_acquire) # define AO_int_fetch_and_sub1_acquire_read(addr) \ AO_int_fetch_and_sub1_acquire(addr) # define AO_HAVE_int_fetch_and_sub1_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_int_fetch_and_sub1_acquire_read) # define AO_int_fetch_and_sub1_dd_acquire_read(addr) \ AO_int_fetch_and_sub1_acquire_read(addr) # define AO_HAVE_int_fetch_and_sub1_dd_acquire_read # endif #else # if defined(AO_HAVE_int_fetch_and_sub1) # define AO_int_fetch_and_sub1_dd_acquire_read(addr) \ AO_int_fetch_and_sub1(addr) # define AO_HAVE_int_fetch_and_sub1_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* int_and */ #if defined(AO_HAVE_int_compare_and_swap_full) \ && !defined(AO_HAVE_int_and_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_int_and_full(volatile unsigned *addr, unsigned value) { unsigned old; do { old = *(unsigned *)addr; } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old, old & value))); } # define AO_HAVE_int_and_full #endif #if defined(AO_HAVE_int_and_full) # if !defined(AO_HAVE_int_and_release) # define AO_int_and_release(addr, val) AO_int_and_full(addr, val) # define AO_HAVE_int_and_release # endif # if !defined(AO_HAVE_int_and_acquire) # define AO_int_and_acquire(addr, val) AO_int_and_full(addr, val) # define AO_HAVE_int_and_acquire # endif # if !defined(AO_HAVE_int_and_write) # define AO_int_and_write(addr, val) AO_int_and_full(addr, val) # define AO_HAVE_int_and_write # endif # if !defined(AO_HAVE_int_and_read) # define AO_int_and_read(addr, val) AO_int_and_full(addr, val) # define AO_HAVE_int_and_read # endif #endif /* AO_HAVE_int_and_full */ #if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_release) # define AO_int_and(addr, val) AO_int_and_release(addr, val) # define AO_HAVE_int_and #endif #if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_acquire) # define AO_int_and(addr, val) AO_int_and_acquire(addr, val) # define AO_HAVE_int_and #endif #if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_write) # define AO_int_and(addr, val) AO_int_and_write(addr, val) # define AO_HAVE_int_and #endif #if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_read) # define AO_int_and(addr, val) AO_int_and_read(addr, val) # define AO_HAVE_int_and #endif #if defined(AO_HAVE_int_and_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_int_and_full) # define AO_int_and_full(addr, val) \ (AO_nop_full(), AO_int_and_acquire(addr, val)) # define AO_HAVE_int_and_full #endif #if !defined(AO_HAVE_int_and_release_write) \ && defined(AO_HAVE_int_and_write) # define AO_int_and_release_write(addr, val) AO_int_and_write(addr, val) # define AO_HAVE_int_and_release_write #endif #if !defined(AO_HAVE_int_and_release_write) \ && defined(AO_HAVE_int_and_release) # define AO_int_and_release_write(addr, val) AO_int_and_release(addr, val) # define AO_HAVE_int_and_release_write #endif #if !defined(AO_HAVE_int_and_acquire_read) \ && defined(AO_HAVE_int_and_read) # define AO_int_and_acquire_read(addr, val) AO_int_and_read(addr, val) # define AO_HAVE_int_and_acquire_read #endif #if !defined(AO_HAVE_int_and_acquire_read) \ && defined(AO_HAVE_int_and_acquire) # define AO_int_and_acquire_read(addr, val) AO_int_and_acquire(addr, val) # define AO_HAVE_int_and_acquire_read #endif /* int_or */ #if defined(AO_HAVE_int_compare_and_swap_full) \ && !defined(AO_HAVE_int_or_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_int_or_full(volatile unsigned *addr, unsigned value) { unsigned old; do { old = *(unsigned *)addr; } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old, old | value))); } # define AO_HAVE_int_or_full #endif #if defined(AO_HAVE_int_or_full) # if !defined(AO_HAVE_int_or_release) # define AO_int_or_release(addr, val) AO_int_or_full(addr, val) # define AO_HAVE_int_or_release # endif # if !defined(AO_HAVE_int_or_acquire) # define AO_int_or_acquire(addr, val) AO_int_or_full(addr, val) # define AO_HAVE_int_or_acquire # endif # if !defined(AO_HAVE_int_or_write) # define AO_int_or_write(addr, val) AO_int_or_full(addr, val) # define AO_HAVE_int_or_write # endif # if !defined(AO_HAVE_int_or_read) # define AO_int_or_read(addr, val) AO_int_or_full(addr, val) # define AO_HAVE_int_or_read # endif #endif /* AO_HAVE_int_or_full */ #if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_release) # define AO_int_or(addr, val) AO_int_or_release(addr, val) # define AO_HAVE_int_or #endif #if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_acquire) # define AO_int_or(addr, val) AO_int_or_acquire(addr, val) # define AO_HAVE_int_or #endif #if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_write) # define AO_int_or(addr, val) AO_int_or_write(addr, val) # define AO_HAVE_int_or #endif #if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_read) # define AO_int_or(addr, val) AO_int_or_read(addr, val) # define AO_HAVE_int_or #endif #if defined(AO_HAVE_int_or_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_int_or_full) # define AO_int_or_full(addr, val) \ (AO_nop_full(), AO_int_or_acquire(addr, val)) # define AO_HAVE_int_or_full #endif #if !defined(AO_HAVE_int_or_release_write) \ && defined(AO_HAVE_int_or_write) # define AO_int_or_release_write(addr, val) AO_int_or_write(addr, val) # define AO_HAVE_int_or_release_write #endif #if !defined(AO_HAVE_int_or_release_write) \ && defined(AO_HAVE_int_or_release) # define AO_int_or_release_write(addr, val) AO_int_or_release(addr, val) # define AO_HAVE_int_or_release_write #endif #if !defined(AO_HAVE_int_or_acquire_read) && defined(AO_HAVE_int_or_read) # define AO_int_or_acquire_read(addr, val) AO_int_or_read(addr, val) # define AO_HAVE_int_or_acquire_read #endif #if !defined(AO_HAVE_int_or_acquire_read) \ && defined(AO_HAVE_int_or_acquire) # define AO_int_or_acquire_read(addr, val) AO_int_or_acquire(addr, val) # define AO_HAVE_int_or_acquire_read #endif /* int_xor */ #if defined(AO_HAVE_int_compare_and_swap_full) \ && !defined(AO_HAVE_int_xor_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_int_xor_full(volatile unsigned *addr, unsigned value) { unsigned old; do { old = *(unsigned *)addr; } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old, old ^ value))); } # define AO_HAVE_int_xor_full #endif #if defined(AO_HAVE_int_xor_full) # if !defined(AO_HAVE_int_xor_release) # define AO_int_xor_release(addr, val) AO_int_xor_full(addr, val) # define AO_HAVE_int_xor_release # endif # if !defined(AO_HAVE_int_xor_acquire) # define AO_int_xor_acquire(addr, val) AO_int_xor_full(addr, val) # define AO_HAVE_int_xor_acquire # endif # if !defined(AO_HAVE_int_xor_write) # define AO_int_xor_write(addr, val) AO_int_xor_full(addr, val) # define AO_HAVE_int_xor_write # endif # if !defined(AO_HAVE_int_xor_read) # define AO_int_xor_read(addr, val) AO_int_xor_full(addr, val) # define AO_HAVE_int_xor_read # endif #endif /* AO_HAVE_int_xor_full */ #if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_release) # define AO_int_xor(addr, val) AO_int_xor_release(addr, val) # define AO_HAVE_int_xor #endif #if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_acquire) # define AO_int_xor(addr, val) AO_int_xor_acquire(addr, val) # define AO_HAVE_int_xor #endif #if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_write) # define AO_int_xor(addr, val) AO_int_xor_write(addr, val) # define AO_HAVE_int_xor #endif #if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_read) # define AO_int_xor(addr, val) AO_int_xor_read(addr, val) # define AO_HAVE_int_xor #endif #if defined(AO_HAVE_int_xor_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_int_xor_full) # define AO_int_xor_full(addr, val) \ (AO_nop_full(), AO_int_xor_acquire(addr, val)) # define AO_HAVE_int_xor_full #endif #if !defined(AO_HAVE_int_xor_release_write) \ && defined(AO_HAVE_int_xor_write) # define AO_int_xor_release_write(addr, val) AO_int_xor_write(addr, val) # define AO_HAVE_int_xor_release_write #endif #if !defined(AO_HAVE_int_xor_release_write) \ && defined(AO_HAVE_int_xor_release) # define AO_int_xor_release_write(addr, val) AO_int_xor_release(addr, val) # define AO_HAVE_int_xor_release_write #endif #if !defined(AO_HAVE_int_xor_acquire_read) \ && defined(AO_HAVE_int_xor_read) # define AO_int_xor_acquire_read(addr, val) AO_int_xor_read(addr, val) # define AO_HAVE_int_xor_acquire_read #endif #if !defined(AO_HAVE_int_xor_acquire_read) \ && defined(AO_HAVE_int_xor_acquire) # define AO_int_xor_acquire_read(addr, val) AO_int_xor_acquire(addr, val) # define AO_HAVE_int_xor_acquire_read #endif /* int_and/or/xor_dd_acquire_read are meaningless. */ /* * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* compare_and_swap (based on fetch_compare_and_swap) */ #if defined(AO_HAVE_fetch_compare_and_swap_full) \ && !defined(AO_HAVE_compare_and_swap_full) AO_INLINE int AO_compare_and_swap_full(volatile AO_t *addr, AO_t old_val, AO_t new_val) { return AO_fetch_compare_and_swap_full(addr, old_val, new_val) == old_val; } # define AO_HAVE_compare_and_swap_full #endif #if defined(AO_HAVE_fetch_compare_and_swap_acquire) \ && !defined(AO_HAVE_compare_and_swap_acquire) AO_INLINE int AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old_val, AO_t new_val) { return AO_fetch_compare_and_swap_acquire(addr, old_val, new_val) == old_val; } # define AO_HAVE_compare_and_swap_acquire #endif #if defined(AO_HAVE_fetch_compare_and_swap_release) \ && !defined(AO_HAVE_compare_and_swap_release) AO_INLINE int AO_compare_and_swap_release(volatile AO_t *addr, AO_t old_val, AO_t new_val) { return AO_fetch_compare_and_swap_release(addr, old_val, new_val) == old_val; } # define AO_HAVE_compare_and_swap_release #endif #if defined(AO_HAVE_fetch_compare_and_swap_write) \ && !defined(AO_HAVE_compare_and_swap_write) AO_INLINE int AO_compare_and_swap_write(volatile AO_t *addr, AO_t old_val, AO_t new_val) { return AO_fetch_compare_and_swap_write(addr, old_val, new_val) == old_val; } # define AO_HAVE_compare_and_swap_write #endif #if defined(AO_HAVE_fetch_compare_and_swap_read) \ && !defined(AO_HAVE_compare_and_swap_read) AO_INLINE int AO_compare_and_swap_read(volatile AO_t *addr, AO_t old_val, AO_t new_val) { return AO_fetch_compare_and_swap_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_compare_and_swap_read #endif #if defined(AO_HAVE_fetch_compare_and_swap) \ && !defined(AO_HAVE_compare_and_swap) AO_INLINE int AO_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val) { return AO_fetch_compare_and_swap(addr, old_val, new_val) == old_val; } # define AO_HAVE_compare_and_swap #endif #if defined(AO_HAVE_fetch_compare_and_swap_release_write) \ && !defined(AO_HAVE_compare_and_swap_release_write) AO_INLINE int AO_compare_and_swap_release_write(volatile AO_t *addr, AO_t old_val, AO_t new_val) { return AO_fetch_compare_and_swap_release_write(addr, old_val, new_val) == old_val; } # define AO_HAVE_compare_and_swap_release_write #endif #if defined(AO_HAVE_fetch_compare_and_swap_acquire_read) \ && !defined(AO_HAVE_compare_and_swap_acquire_read) AO_INLINE int AO_compare_and_swap_acquire_read(volatile AO_t *addr, AO_t old_val, AO_t new_val) { return AO_fetch_compare_and_swap_acquire_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_compare_and_swap_acquire_read #endif #if defined(AO_HAVE_fetch_compare_and_swap_dd_acquire_read) \ && !defined(AO_HAVE_compare_and_swap_dd_acquire_read) AO_INLINE int AO_compare_and_swap_dd_acquire_read(volatile AO_t *addr, AO_t old_val, AO_t new_val) { return AO_fetch_compare_and_swap_dd_acquire_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_compare_and_swap_dd_acquire_read #endif /* fetch_and_add */ /* We first try to implement fetch_and_add variants in terms of the */ /* corresponding compare_and_swap variants to minimize adding barriers. */ #if defined(AO_HAVE_compare_and_swap_full) \ && !defined(AO_HAVE_fetch_and_add_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE AO_t AO_fetch_and_add_full(volatile AO_t *addr, AO_t incr) { AO_t old; do { old = *(AO_t *)addr; } while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old, old + incr))); return old; } # define AO_HAVE_fetch_and_add_full #endif #if defined(AO_HAVE_compare_and_swap_acquire) \ && !defined(AO_HAVE_fetch_and_add_acquire) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE AO_t AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr) { AO_t old; do { old = *(AO_t *)addr; } while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire(addr, old, old + incr))); return old; } # define AO_HAVE_fetch_and_add_acquire #endif #if defined(AO_HAVE_compare_and_swap_release) \ && !defined(AO_HAVE_fetch_and_add_release) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE AO_t AO_fetch_and_add_release(volatile AO_t *addr, AO_t incr) { AO_t old; do { old = *(AO_t *)addr; } while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(addr, old, old + incr))); return old; } # define AO_HAVE_fetch_and_add_release #endif #if defined(AO_HAVE_compare_and_swap) \ && !defined(AO_HAVE_fetch_and_add) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE AO_t AO_fetch_and_add(volatile AO_t *addr, AO_t incr) { AO_t old; do { old = *(AO_t *)addr; } while (AO_EXPECT_FALSE(!AO_compare_and_swap(addr, old, old + incr))); return old; } # define AO_HAVE_fetch_and_add #endif #if defined(AO_HAVE_fetch_and_add_full) # if !defined(AO_HAVE_fetch_and_add_release) # define AO_fetch_and_add_release(addr, val) \ AO_fetch_and_add_full(addr, val) # define AO_HAVE_fetch_and_add_release # endif # if !defined(AO_HAVE_fetch_and_add_acquire) # define AO_fetch_and_add_acquire(addr, val) \ AO_fetch_and_add_full(addr, val) # define AO_HAVE_fetch_and_add_acquire # endif # if !defined(AO_HAVE_fetch_and_add_write) # define AO_fetch_and_add_write(addr, val) \ AO_fetch_and_add_full(addr, val) # define AO_HAVE_fetch_and_add_write # endif # if !defined(AO_HAVE_fetch_and_add_read) # define AO_fetch_and_add_read(addr, val) \ AO_fetch_and_add_full(addr, val) # define AO_HAVE_fetch_and_add_read # endif #endif /* AO_HAVE_fetch_and_add_full */ #if defined(AO_HAVE_fetch_and_add) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_fetch_and_add_acquire) AO_INLINE AO_t AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr) { AO_t result = AO_fetch_and_add(addr, incr); AO_nop_full(); return result; } # define AO_HAVE_fetch_and_add_acquire #endif #if defined(AO_HAVE_fetch_and_add) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_fetch_and_add_release) # define AO_fetch_and_add_release(addr, incr) \ (AO_nop_full(), AO_fetch_and_add(addr, incr)) # define AO_HAVE_fetch_and_add_release #endif #if !defined(AO_HAVE_fetch_and_add) \ && defined(AO_HAVE_fetch_and_add_release) # define AO_fetch_and_add(addr, val) \ AO_fetch_and_add_release(addr, val) # define AO_HAVE_fetch_and_add #endif #if !defined(AO_HAVE_fetch_and_add) \ && defined(AO_HAVE_fetch_and_add_acquire) # define AO_fetch_and_add(addr, val) \ AO_fetch_and_add_acquire(addr, val) # define AO_HAVE_fetch_and_add #endif #if !defined(AO_HAVE_fetch_and_add) \ && defined(AO_HAVE_fetch_and_add_write) # define AO_fetch_and_add(addr, val) \ AO_fetch_and_add_write(addr, val) # define AO_HAVE_fetch_and_add #endif #if !defined(AO_HAVE_fetch_and_add) \ && defined(AO_HAVE_fetch_and_add_read) # define AO_fetch_and_add(addr, val) \ AO_fetch_and_add_read(addr, val) # define AO_HAVE_fetch_and_add #endif #if defined(AO_HAVE_fetch_and_add_acquire) \ && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_fetch_and_add_full) # define AO_fetch_and_add_full(addr, val) \ (AO_nop_full(), AO_fetch_and_add_acquire(addr, val)) # define AO_HAVE_fetch_and_add_full #endif #if !defined(AO_HAVE_fetch_and_add_release_write) \ && defined(AO_HAVE_fetch_and_add_write) # define AO_fetch_and_add_release_write(addr, val) \ AO_fetch_and_add_write(addr, val) # define AO_HAVE_fetch_and_add_release_write #endif #if !defined(AO_HAVE_fetch_and_add_release_write) \ && defined(AO_HAVE_fetch_and_add_release) # define AO_fetch_and_add_release_write(addr, val) \ AO_fetch_and_add_release(addr, val) # define AO_HAVE_fetch_and_add_release_write #endif #if !defined(AO_HAVE_fetch_and_add_acquire_read) \ && defined(AO_HAVE_fetch_and_add_read) # define AO_fetch_and_add_acquire_read(addr, val) \ AO_fetch_and_add_read(addr, val) # define AO_HAVE_fetch_and_add_acquire_read #endif #if !defined(AO_HAVE_fetch_and_add_acquire_read) \ && defined(AO_HAVE_fetch_and_add_acquire) # define AO_fetch_and_add_acquire_read(addr, val) \ AO_fetch_and_add_acquire(addr, val) # define AO_HAVE_fetch_and_add_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_fetch_and_add_acquire_read) # define AO_fetch_and_add_dd_acquire_read(addr, val) \ AO_fetch_and_add_acquire_read(addr, val) # define AO_HAVE_fetch_and_add_dd_acquire_read # endif #else # if defined(AO_HAVE_fetch_and_add) # define AO_fetch_and_add_dd_acquire_read(addr, val) \ AO_fetch_and_add(addr, val) # define AO_HAVE_fetch_and_add_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* fetch_and_add1 */ #if defined(AO_HAVE_fetch_and_add_full) \ && !defined(AO_HAVE_fetch_and_add1_full) # define AO_fetch_and_add1_full(addr) \ AO_fetch_and_add_full(addr, 1) # define AO_HAVE_fetch_and_add1_full #endif #if defined(AO_HAVE_fetch_and_add_release) \ && !defined(AO_HAVE_fetch_and_add1_release) # define AO_fetch_and_add1_release(addr) \ AO_fetch_and_add_release(addr, 1) # define AO_HAVE_fetch_and_add1_release #endif #if defined(AO_HAVE_fetch_and_add_acquire) \ && !defined(AO_HAVE_fetch_and_add1_acquire) # define AO_fetch_and_add1_acquire(addr) \ AO_fetch_and_add_acquire(addr, 1) # define AO_HAVE_fetch_and_add1_acquire #endif #if defined(AO_HAVE_fetch_and_add_write) \ && !defined(AO_HAVE_fetch_and_add1_write) # define AO_fetch_and_add1_write(addr) \ AO_fetch_and_add_write(addr, 1) # define AO_HAVE_fetch_and_add1_write #endif #if defined(AO_HAVE_fetch_and_add_read) \ && !defined(AO_HAVE_fetch_and_add1_read) # define AO_fetch_and_add1_read(addr) \ AO_fetch_and_add_read(addr, 1) # define AO_HAVE_fetch_and_add1_read #endif #if defined(AO_HAVE_fetch_and_add_release_write) \ && !defined(AO_HAVE_fetch_and_add1_release_write) # define AO_fetch_and_add1_release_write(addr) \ AO_fetch_and_add_release_write(addr, 1) # define AO_HAVE_fetch_and_add1_release_write #endif #if defined(AO_HAVE_fetch_and_add_acquire_read) \ && !defined(AO_HAVE_fetch_and_add1_acquire_read) # define AO_fetch_and_add1_acquire_read(addr) \ AO_fetch_and_add_acquire_read(addr, 1) # define AO_HAVE_fetch_and_add1_acquire_read #endif #if defined(AO_HAVE_fetch_and_add) \ && !defined(AO_HAVE_fetch_and_add1) # define AO_fetch_and_add1(addr) AO_fetch_and_add(addr, 1) # define AO_HAVE_fetch_and_add1 #endif #if defined(AO_HAVE_fetch_and_add1_full) # if !defined(AO_HAVE_fetch_and_add1_release) # define AO_fetch_and_add1_release(addr) \ AO_fetch_and_add1_full(addr) # define AO_HAVE_fetch_and_add1_release # endif # if !defined(AO_HAVE_fetch_and_add1_acquire) # define AO_fetch_and_add1_acquire(addr) \ AO_fetch_and_add1_full(addr) # define AO_HAVE_fetch_and_add1_acquire # endif # if !defined(AO_HAVE_fetch_and_add1_write) # define AO_fetch_and_add1_write(addr) \ AO_fetch_and_add1_full(addr) # define AO_HAVE_fetch_and_add1_write # endif # if !defined(AO_HAVE_fetch_and_add1_read) # define AO_fetch_and_add1_read(addr) \ AO_fetch_and_add1_full(addr) # define AO_HAVE_fetch_and_add1_read # endif #endif /* AO_HAVE_fetch_and_add1_full */ #if !defined(AO_HAVE_fetch_and_add1) \ && defined(AO_HAVE_fetch_and_add1_release) # define AO_fetch_and_add1(addr) AO_fetch_and_add1_release(addr) # define AO_HAVE_fetch_and_add1 #endif #if !defined(AO_HAVE_fetch_and_add1) \ && defined(AO_HAVE_fetch_and_add1_acquire) # define AO_fetch_and_add1(addr) AO_fetch_and_add1_acquire(addr) # define AO_HAVE_fetch_and_add1 #endif #if !defined(AO_HAVE_fetch_and_add1) \ && defined(AO_HAVE_fetch_and_add1_write) # define AO_fetch_and_add1(addr) AO_fetch_and_add1_write(addr) # define AO_HAVE_fetch_and_add1 #endif #if !defined(AO_HAVE_fetch_and_add1) \ && defined(AO_HAVE_fetch_and_add1_read) # define AO_fetch_and_add1(addr) AO_fetch_and_add1_read(addr) # define AO_HAVE_fetch_and_add1 #endif #if defined(AO_HAVE_fetch_and_add1_acquire) \ && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_fetch_and_add1_full) # define AO_fetch_and_add1_full(addr) \ (AO_nop_full(), AO_fetch_and_add1_acquire(addr)) # define AO_HAVE_fetch_and_add1_full #endif #if !defined(AO_HAVE_fetch_and_add1_release_write) \ && defined(AO_HAVE_fetch_and_add1_write) # define AO_fetch_and_add1_release_write(addr) \ AO_fetch_and_add1_write(addr) # define AO_HAVE_fetch_and_add1_release_write #endif #if !defined(AO_HAVE_fetch_and_add1_release_write) \ && defined(AO_HAVE_fetch_and_add1_release) # define AO_fetch_and_add1_release_write(addr) \ AO_fetch_and_add1_release(addr) # define AO_HAVE_fetch_and_add1_release_write #endif #if !defined(AO_HAVE_fetch_and_add1_acquire_read) \ && defined(AO_HAVE_fetch_and_add1_read) # define AO_fetch_and_add1_acquire_read(addr) \ AO_fetch_and_add1_read(addr) # define AO_HAVE_fetch_and_add1_acquire_read #endif #if !defined(AO_HAVE_fetch_and_add1_acquire_read) \ && defined(AO_HAVE_fetch_and_add1_acquire) # define AO_fetch_and_add1_acquire_read(addr) \ AO_fetch_and_add1_acquire(addr) # define AO_HAVE_fetch_and_add1_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_fetch_and_add1_acquire_read) # define AO_fetch_and_add1_dd_acquire_read(addr) \ AO_fetch_and_add1_acquire_read(addr) # define AO_HAVE_fetch_and_add1_dd_acquire_read # endif #else # if defined(AO_HAVE_fetch_and_add1) # define AO_fetch_and_add1_dd_acquire_read(addr) \ AO_fetch_and_add1(addr) # define AO_HAVE_fetch_and_add1_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* fetch_and_sub1 */ #if defined(AO_HAVE_fetch_and_add_full) \ && !defined(AO_HAVE_fetch_and_sub1_full) # define AO_fetch_and_sub1_full(addr) \ AO_fetch_and_add_full(addr, (AO_t)(-1)) # define AO_HAVE_fetch_and_sub1_full #endif #if defined(AO_HAVE_fetch_and_add_release) \ && !defined(AO_HAVE_fetch_and_sub1_release) # define AO_fetch_and_sub1_release(addr) \ AO_fetch_and_add_release(addr, (AO_t)(-1)) # define AO_HAVE_fetch_and_sub1_release #endif #if defined(AO_HAVE_fetch_and_add_acquire) \ && !defined(AO_HAVE_fetch_and_sub1_acquire) # define AO_fetch_and_sub1_acquire(addr) \ AO_fetch_and_add_acquire(addr, (AO_t)(-1)) # define AO_HAVE_fetch_and_sub1_acquire #endif #if defined(AO_HAVE_fetch_and_add_write) \ && !defined(AO_HAVE_fetch_and_sub1_write) # define AO_fetch_and_sub1_write(addr) \ AO_fetch_and_add_write(addr, (AO_t)(-1)) # define AO_HAVE_fetch_and_sub1_write #endif #if defined(AO_HAVE_fetch_and_add_read) \ && !defined(AO_HAVE_fetch_and_sub1_read) # define AO_fetch_and_sub1_read(addr) \ AO_fetch_and_add_read(addr, (AO_t)(-1)) # define AO_HAVE_fetch_and_sub1_read #endif #if defined(AO_HAVE_fetch_and_add_release_write) \ && !defined(AO_HAVE_fetch_and_sub1_release_write) # define AO_fetch_and_sub1_release_write(addr) \ AO_fetch_and_add_release_write(addr, (AO_t)(-1)) # define AO_HAVE_fetch_and_sub1_release_write #endif #if defined(AO_HAVE_fetch_and_add_acquire_read) \ && !defined(AO_HAVE_fetch_and_sub1_acquire_read) # define AO_fetch_and_sub1_acquire_read(addr) \ AO_fetch_and_add_acquire_read(addr, (AO_t)(-1)) # define AO_HAVE_fetch_and_sub1_acquire_read #endif #if defined(AO_HAVE_fetch_and_add) \ && !defined(AO_HAVE_fetch_and_sub1) # define AO_fetch_and_sub1(addr) \ AO_fetch_and_add(addr, (AO_t)(-1)) # define AO_HAVE_fetch_and_sub1 #endif #if defined(AO_HAVE_fetch_and_sub1_full) # if !defined(AO_HAVE_fetch_and_sub1_release) # define AO_fetch_and_sub1_release(addr) \ AO_fetch_and_sub1_full(addr) # define AO_HAVE_fetch_and_sub1_release # endif # if !defined(AO_HAVE_fetch_and_sub1_acquire) # define AO_fetch_and_sub1_acquire(addr) \ AO_fetch_and_sub1_full(addr) # define AO_HAVE_fetch_and_sub1_acquire # endif # if !defined(AO_HAVE_fetch_and_sub1_write) # define AO_fetch_and_sub1_write(addr) \ AO_fetch_and_sub1_full(addr) # define AO_HAVE_fetch_and_sub1_write # endif # if !defined(AO_HAVE_fetch_and_sub1_read) # define AO_fetch_and_sub1_read(addr) \ AO_fetch_and_sub1_full(addr) # define AO_HAVE_fetch_and_sub1_read # endif #endif /* AO_HAVE_fetch_and_sub1_full */ #if !defined(AO_HAVE_fetch_and_sub1) \ && defined(AO_HAVE_fetch_and_sub1_release) # define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_release(addr) # define AO_HAVE_fetch_and_sub1 #endif #if !defined(AO_HAVE_fetch_and_sub1) \ && defined(AO_HAVE_fetch_and_sub1_acquire) # define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_acquire(addr) # define AO_HAVE_fetch_and_sub1 #endif #if !defined(AO_HAVE_fetch_and_sub1) \ && defined(AO_HAVE_fetch_and_sub1_write) # define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_write(addr) # define AO_HAVE_fetch_and_sub1 #endif #if !defined(AO_HAVE_fetch_and_sub1) \ && defined(AO_HAVE_fetch_and_sub1_read) # define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_read(addr) # define AO_HAVE_fetch_and_sub1 #endif #if defined(AO_HAVE_fetch_and_sub1_acquire) \ && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_fetch_and_sub1_full) # define AO_fetch_and_sub1_full(addr) \ (AO_nop_full(), AO_fetch_and_sub1_acquire(addr)) # define AO_HAVE_fetch_and_sub1_full #endif #if !defined(AO_HAVE_fetch_and_sub1_release_write) \ && defined(AO_HAVE_fetch_and_sub1_write) # define AO_fetch_and_sub1_release_write(addr) \ AO_fetch_and_sub1_write(addr) # define AO_HAVE_fetch_and_sub1_release_write #endif #if !defined(AO_HAVE_fetch_and_sub1_release_write) \ && defined(AO_HAVE_fetch_and_sub1_release) # define AO_fetch_and_sub1_release_write(addr) \ AO_fetch_and_sub1_release(addr) # define AO_HAVE_fetch_and_sub1_release_write #endif #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \ && defined(AO_HAVE_fetch_and_sub1_read) # define AO_fetch_and_sub1_acquire_read(addr) \ AO_fetch_and_sub1_read(addr) # define AO_HAVE_fetch_and_sub1_acquire_read #endif #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \ && defined(AO_HAVE_fetch_and_sub1_acquire) # define AO_fetch_and_sub1_acquire_read(addr) \ AO_fetch_and_sub1_acquire(addr) # define AO_HAVE_fetch_and_sub1_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_fetch_and_sub1_acquire_read) # define AO_fetch_and_sub1_dd_acquire_read(addr) \ AO_fetch_and_sub1_acquire_read(addr) # define AO_HAVE_fetch_and_sub1_dd_acquire_read # endif #else # if defined(AO_HAVE_fetch_and_sub1) # define AO_fetch_and_sub1_dd_acquire_read(addr) \ AO_fetch_and_sub1(addr) # define AO_HAVE_fetch_and_sub1_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* and */ #if defined(AO_HAVE_compare_and_swap_full) \ && !defined(AO_HAVE_and_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_and_full(volatile AO_t *addr, AO_t value) { AO_t old; do { old = *(AO_t *)addr; } while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old, old & value))); } # define AO_HAVE_and_full #endif #if defined(AO_HAVE_and_full) # if !defined(AO_HAVE_and_release) # define AO_and_release(addr, val) AO_and_full(addr, val) # define AO_HAVE_and_release # endif # if !defined(AO_HAVE_and_acquire) # define AO_and_acquire(addr, val) AO_and_full(addr, val) # define AO_HAVE_and_acquire # endif # if !defined(AO_HAVE_and_write) # define AO_and_write(addr, val) AO_and_full(addr, val) # define AO_HAVE_and_write # endif # if !defined(AO_HAVE_and_read) # define AO_and_read(addr, val) AO_and_full(addr, val) # define AO_HAVE_and_read # endif #endif /* AO_HAVE_and_full */ #if !defined(AO_HAVE_and) && defined(AO_HAVE_and_release) # define AO_and(addr, val) AO_and_release(addr, val) # define AO_HAVE_and #endif #if !defined(AO_HAVE_and) && defined(AO_HAVE_and_acquire) # define AO_and(addr, val) AO_and_acquire(addr, val) # define AO_HAVE_and #endif #if !defined(AO_HAVE_and) && defined(AO_HAVE_and_write) # define AO_and(addr, val) AO_and_write(addr, val) # define AO_HAVE_and #endif #if !defined(AO_HAVE_and) && defined(AO_HAVE_and_read) # define AO_and(addr, val) AO_and_read(addr, val) # define AO_HAVE_and #endif #if defined(AO_HAVE_and_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_and_full) # define AO_and_full(addr, val) \ (AO_nop_full(), AO_and_acquire(addr, val)) # define AO_HAVE_and_full #endif #if !defined(AO_HAVE_and_release_write) \ && defined(AO_HAVE_and_write) # define AO_and_release_write(addr, val) AO_and_write(addr, val) # define AO_HAVE_and_release_write #endif #if !defined(AO_HAVE_and_release_write) \ && defined(AO_HAVE_and_release) # define AO_and_release_write(addr, val) AO_and_release(addr, val) # define AO_HAVE_and_release_write #endif #if !defined(AO_HAVE_and_acquire_read) \ && defined(AO_HAVE_and_read) # define AO_and_acquire_read(addr, val) AO_and_read(addr, val) # define AO_HAVE_and_acquire_read #endif #if !defined(AO_HAVE_and_acquire_read) \ && defined(AO_HAVE_and_acquire) # define AO_and_acquire_read(addr, val) AO_and_acquire(addr, val) # define AO_HAVE_and_acquire_read #endif /* or */ #if defined(AO_HAVE_compare_and_swap_full) \ && !defined(AO_HAVE_or_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_or_full(volatile AO_t *addr, AO_t value) { AO_t old; do { old = *(AO_t *)addr; } while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old, old | value))); } # define AO_HAVE_or_full #endif #if defined(AO_HAVE_or_full) # if !defined(AO_HAVE_or_release) # define AO_or_release(addr, val) AO_or_full(addr, val) # define AO_HAVE_or_release # endif # if !defined(AO_HAVE_or_acquire) # define AO_or_acquire(addr, val) AO_or_full(addr, val) # define AO_HAVE_or_acquire # endif # if !defined(AO_HAVE_or_write) # define AO_or_write(addr, val) AO_or_full(addr, val) # define AO_HAVE_or_write # endif # if !defined(AO_HAVE_or_read) # define AO_or_read(addr, val) AO_or_full(addr, val) # define AO_HAVE_or_read # endif #endif /* AO_HAVE_or_full */ #if !defined(AO_HAVE_or) && defined(AO_HAVE_or_release) # define AO_or(addr, val) AO_or_release(addr, val) # define AO_HAVE_or #endif #if !defined(AO_HAVE_or) && defined(AO_HAVE_or_acquire) # define AO_or(addr, val) AO_or_acquire(addr, val) # define AO_HAVE_or #endif #if !defined(AO_HAVE_or) && defined(AO_HAVE_or_write) # define AO_or(addr, val) AO_or_write(addr, val) # define AO_HAVE_or #endif #if !defined(AO_HAVE_or) && defined(AO_HAVE_or_read) # define AO_or(addr, val) AO_or_read(addr, val) # define AO_HAVE_or #endif #if defined(AO_HAVE_or_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_or_full) # define AO_or_full(addr, val) \ (AO_nop_full(), AO_or_acquire(addr, val)) # define AO_HAVE_or_full #endif #if !defined(AO_HAVE_or_release_write) \ && defined(AO_HAVE_or_write) # define AO_or_release_write(addr, val) AO_or_write(addr, val) # define AO_HAVE_or_release_write #endif #if !defined(AO_HAVE_or_release_write) \ && defined(AO_HAVE_or_release) # define AO_or_release_write(addr, val) AO_or_release(addr, val) # define AO_HAVE_or_release_write #endif #if !defined(AO_HAVE_or_acquire_read) && defined(AO_HAVE_or_read) # define AO_or_acquire_read(addr, val) AO_or_read(addr, val) # define AO_HAVE_or_acquire_read #endif #if !defined(AO_HAVE_or_acquire_read) \ && defined(AO_HAVE_or_acquire) # define AO_or_acquire_read(addr, val) AO_or_acquire(addr, val) # define AO_HAVE_or_acquire_read #endif /* xor */ #if defined(AO_HAVE_compare_and_swap_full) \ && !defined(AO_HAVE_xor_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_xor_full(volatile AO_t *addr, AO_t value) { AO_t old; do { old = *(AO_t *)addr; } while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old, old ^ value))); } # define AO_HAVE_xor_full #endif #if defined(AO_HAVE_xor_full) # if !defined(AO_HAVE_xor_release) # define AO_xor_release(addr, val) AO_xor_full(addr, val) # define AO_HAVE_xor_release # endif # if !defined(AO_HAVE_xor_acquire) # define AO_xor_acquire(addr, val) AO_xor_full(addr, val) # define AO_HAVE_xor_acquire # endif # if !defined(AO_HAVE_xor_write) # define AO_xor_write(addr, val) AO_xor_full(addr, val) # define AO_HAVE_xor_write # endif # if !defined(AO_HAVE_xor_read) # define AO_xor_read(addr, val) AO_xor_full(addr, val) # define AO_HAVE_xor_read # endif #endif /* AO_HAVE_xor_full */ #if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_release) # define AO_xor(addr, val) AO_xor_release(addr, val) # define AO_HAVE_xor #endif #if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_acquire) # define AO_xor(addr, val) AO_xor_acquire(addr, val) # define AO_HAVE_xor #endif #if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_write) # define AO_xor(addr, val) AO_xor_write(addr, val) # define AO_HAVE_xor #endif #if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_read) # define AO_xor(addr, val) AO_xor_read(addr, val) # define AO_HAVE_xor #endif #if defined(AO_HAVE_xor_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_xor_full) # define AO_xor_full(addr, val) \ (AO_nop_full(), AO_xor_acquire(addr, val)) # define AO_HAVE_xor_full #endif #if !defined(AO_HAVE_xor_release_write) \ && defined(AO_HAVE_xor_write) # define AO_xor_release_write(addr, val) AO_xor_write(addr, val) # define AO_HAVE_xor_release_write #endif #if !defined(AO_HAVE_xor_release_write) \ && defined(AO_HAVE_xor_release) # define AO_xor_release_write(addr, val) AO_xor_release(addr, val) # define AO_HAVE_xor_release_write #endif #if !defined(AO_HAVE_xor_acquire_read) \ && defined(AO_HAVE_xor_read) # define AO_xor_acquire_read(addr, val) AO_xor_read(addr, val) # define AO_HAVE_xor_acquire_read #endif #if !defined(AO_HAVE_xor_acquire_read) \ && defined(AO_HAVE_xor_acquire) # define AO_xor_acquire_read(addr, val) AO_xor_acquire(addr, val) # define AO_HAVE_xor_acquire_read #endif /* and/or/xor_dd_acquire_read are meaningless. */