| /* | |
| * Copyright (C) 2015 The Android Open Source Project | |
| * All rights reserved. | |
| * | |
| * Redistribution and use in source and binary forms, with or without | |
| * modification, are permitted provided that the following conditions | |
| * are met: | |
| * * Redistributions of source code must retain the above copyright | |
| * notice, this list of conditions and the following disclaimer. | |
| * * Redistributions in binary form must reproduce the above copyright | |
| * notice, this list of conditions and the following disclaimer in | |
| * the documentation and/or other materials provided with the | |
| * distribution. | |
| * | |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS | |
| * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE | |
| * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, | |
| * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, | |
| * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS | |
| * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED | |
| * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |
| * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT | |
| * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
| * SUCH DAMAGE. | |
| */ | |
| #include <pthread.h> | |
| #include <stdatomic.h> | |
| #include <stdint.h> | |
| #include "private/bionic_futex.h" | |
| int pthread_barrierattr_init(pthread_barrierattr_t* attr) { | |
| *attr = 0; | |
| return 0; | |
| } | |
| int pthread_barrierattr_destroy(pthread_barrierattr_t* attr) { | |
| *attr = 0; | |
| return 0; | |
| } | |
| int pthread_barrierattr_getpshared(const pthread_barrierattr_t* attr, int* pshared) { | |
| *pshared = (*attr & 1) ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE; | |
| return 0; | |
| } | |
| int pthread_barrierattr_setpshared(pthread_barrierattr_t* attr, int pshared) { | |
| if (pshared == PTHREAD_PROCESS_SHARED) { | |
| *attr |= 1; | |
| } else { | |
| *attr &= ~1; | |
| } | |
| return 0; | |
| } | |
| enum BarrierState { | |
| WAIT, | |
| RELEASE, | |
| }; | |
| struct pthread_barrier_internal_t { | |
| // One barrier can be used for unlimited number of cycles. In each cycle, [init_count] | |
| // threads must call pthread_barrier_wait() before any of them successfully return from | |
| // the call. It is undefined behavior if there are more than [init_count] threads call | |
| // pthread_barrier_wait() in one cycle. | |
| uint32_t init_count; | |
| // Barrier state. It is WAIT if waiting for more threads to enter the barrier in this cycle, | |
| // otherwise threads are leaving the barrier. | |
| _Atomic(BarrierState) state; | |
| // Number of threads having entered but not left the barrier in this cycle. | |
| atomic_uint wait_count; | |
| // Whether the barrier is shared across processes. | |
| bool pshared; | |
| uint32_t __reserved[4]; | |
| }; | |
| static_assert(sizeof(pthread_barrier_t) == sizeof(pthread_barrier_internal_t), | |
| "pthread_barrier_t should actually be pthread_barrier_internal_t in implementation." | |
| ); | |
| static_assert(alignof(pthread_barrier_t) >= 4, | |
| "pthread_barrier_t should fulfill the alignment of pthread_barrier_internal_t."); | |
| static inline pthread_barrier_internal_t* __get_internal_barrier(pthread_barrier_t* barrier) { | |
| return reinterpret_cast<pthread_barrier_internal_t*>(barrier); | |
| } | |
| int pthread_barrier_init(pthread_barrier_t* barrier_interface, const pthread_barrierattr_t* attr, | |
| unsigned count) { | |
| pthread_barrier_internal_t* barrier = __get_internal_barrier(barrier_interface); | |
| if (count == 0) { | |
| return EINVAL; | |
| } | |
| barrier->init_count = count; | |
| atomic_init(&barrier->state, WAIT); | |
| atomic_init(&barrier->wait_count, 0); | |
| barrier->pshared = false; | |
| if (attr != nullptr && (*attr & 1)) { | |
| barrier->pshared = true; | |
| } | |
| return 0; | |
| } | |
| // According to POSIX standard, pthread_barrier_wait() synchronizes memory between participating | |
| // threads. It means all memory operations made by participating threads before calling | |
| // pthread_barrier_wait() can be seen by all participating threads after the function call. | |
| // We establish this by making a happens-before relation between all threads entering the barrier | |
| // with the last thread entering the barrier, and a happens-before relation between the last | |
| // thread entering the barrier with all threads leaving the barrier. | |
| int pthread_barrier_wait(pthread_barrier_t* barrier_interface) { | |
| pthread_barrier_internal_t* barrier = __get_internal_barrier(barrier_interface); | |
| // Wait until all threads for the previous cycle have left the barrier. This is needed | |
| // as a participating thread can call pthread_barrier_wait() again before other | |
| // threads have left the barrier. Use acquire operation here to synchronize with | |
| // the last thread leaving the previous cycle, so we can read correct wait_count below. | |
| while(atomic_load_explicit(&barrier->state, memory_order_acquire) == RELEASE) { | |
| __futex_wait_ex(&barrier->state, barrier->pshared, RELEASE, false, nullptr); | |
| } | |
| uint32_t prev_wait_count = atomic_load_explicit(&barrier->wait_count, memory_order_relaxed); | |
| while (true) { | |
| // It happens when there are more than [init_count] threads trying to enter the barrier | |
| // at one cycle. We read the POSIX standard as disallowing this, since additional arriving | |
| // threads are not synchronized with respect to the barrier reset. We also don't know of | |
| // any reasonable cases in which this would be intentional. | |
| if (prev_wait_count >= barrier->init_count) { | |
| return EINVAL; | |
| } | |
| // Use memory_order_acq_rel operation here to synchronize between all threads entering | |
| // the barrier with the last thread entering the barrier. | |
| if (atomic_compare_exchange_weak_explicit(&barrier->wait_count, &prev_wait_count, | |
| prev_wait_count + 1u, memory_order_acq_rel, | |
| memory_order_relaxed)) { | |
| break; | |
| } | |
| } | |
| int result = 0; | |
| if (prev_wait_count + 1 == barrier->init_count) { | |
| result = PTHREAD_BARRIER_SERIAL_THREAD; | |
| if (prev_wait_count != 0) { | |
| // Use release operation here to synchronize between the last thread entering the | |
| // barrier with all threads leaving the barrier. | |
| atomic_store_explicit(&barrier->state, RELEASE, memory_order_release); | |
| __futex_wake_ex(&barrier->state, barrier->pshared, prev_wait_count); | |
| } | |
| } else { | |
| // Use acquire operation here to synchronize between the last thread entering the | |
| // barrier with all threads leaving the barrier. | |
| while (atomic_load_explicit(&barrier->state, memory_order_acquire) == WAIT) { | |
| __futex_wait_ex(&barrier->state, barrier->pshared, WAIT, false, nullptr); | |
| } | |
| } | |
| // Use release operation here to make it not reordered with previous operations. | |
| if (atomic_fetch_sub_explicit(&barrier->wait_count, 1, memory_order_release) == 1) { | |
| // Use release operation here to synchronize with threads entering the barrier for | |
| // the next cycle, or the thread calling pthread_barrier_destroy(). | |
| atomic_store_explicit(&barrier->state, WAIT, memory_order_release); | |
| __futex_wake_ex(&barrier->state, barrier->pshared, barrier->init_count); | |
| } | |
| return result; | |
| } | |
| int pthread_barrier_destroy(pthread_barrier_t* barrier_interface) { | |
| pthread_barrier_internal_t* barrier = __get_internal_barrier(barrier_interface); | |
| if (barrier->init_count == 0) { | |
| return EINVAL; | |
| } | |
| // Use acquire operation here to synchronize with the last thread leaving the barrier. | |
| // So we can read correct wait_count below. | |
| while (atomic_load_explicit(&barrier->state, memory_order_acquire) == RELEASE) { | |
| __futex_wait_ex(&barrier->state, barrier->pshared, RELEASE, false, nullptr); | |
| } | |
| if (atomic_load_explicit(&barrier->wait_count, memory_order_relaxed) != 0) { | |
| return EBUSY; | |
| } | |
| barrier->init_count = 0; | |
| return 0; | |
| } |