Implement pthread barrier.

Bug: 24341262
Change-Id: I5472549e5d7545c1c3f0bef78235f545557b9630
diff --git a/libc/Android.mk b/libc/Android.mk
index 2ea96d4..a7fe15c 100644
--- a/libc/Android.mk
+++ b/libc/Android.mk
@@ -563,6 +563,7 @@
 libc_pthread_src_files := \
     bionic/pthread_atfork.cpp \
     bionic/pthread_attr.cpp \
+    bionic/pthread_barrier.cpp \
     bionic/pthread_cond.cpp \
     bionic/pthread_create.cpp \
     bionic/pthread_detach.cpp \
diff --git a/libc/bionic/pthread_barrier.cpp b/libc/bionic/pthread_barrier.cpp
new file mode 100644
index 0000000..3227daf
--- /dev/null
+++ b/libc/bionic/pthread_barrier.cpp
@@ -0,0 +1,183 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <pthread.h>
+#include <stdatomic.h>
+#include <stdint.h>
+
+#include "private/bionic_futex.h"
+
+int pthread_barrierattr_init(pthread_barrierattr_t* attr) {
+  *attr = 0;
+  return 0;
+}
+
+int pthread_barrierattr_destroy(pthread_barrierattr_t* attr) {
+  *attr = 0;
+  return 0;
+}
+
+int pthread_barrierattr_getpshared(pthread_barrierattr_t* attr, int* pshared) {
+  *pshared = (*attr & 1) ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE;
+  return 0;
+}
+
+int pthread_barrierattr_setpshared(pthread_barrierattr_t* attr, int pshared) {
+  if (pshared == PTHREAD_PROCESS_SHARED) {
+    *attr |= 1;
+  } else {
+    *attr &= ~1;
+  }
+  return 0;
+}
+
+enum BarrierState {
+  WAIT,
+  RELEASE,
+};
+
+struct pthread_barrier_internal_t {
+  // One barrier can be used for unlimited number of cycles. In each cycle, [init_count]
+  // threads must call pthread_barrier_wait() before any of them successfully return from
+  // the call. It is undefined behavior if there are more than [init_count] threads call
+  // pthread_barrier_wait() in one cycle.
+  uint32_t init_count;
+  // Barrier state. It is WAIT if waiting for more threads to enter the barrier in this cycle,
+  // otherwise threads are leaving the barrier.
+  _Atomic(BarrierState) state;
+  // Number of threads having entered but not left the barrier in this cycle.
+  atomic_uint wait_count;
+  // Whether the barrier is shared across processes.
+  bool pshared;
+  uint32_t __reserved[4];
+};
+
+static_assert(sizeof(pthread_barrier_t) == sizeof(pthread_barrier_internal_t),
+              "pthread_barrier_t should actually be pthread_barrier_internal_t in implementation."
+              );
+
+static_assert(alignof(pthread_barrier_t) >= 4,
+              "pthread_barrier_t should fulfill the alignment of pthread_barrier_internal_t.");
+
+static inline pthread_barrier_internal_t* __get_internal_barrier(pthread_barrier_t* barrier) {
+  return reinterpret_cast<pthread_barrier_internal_t*>(barrier);
+}
+
+int pthread_barrier_init(pthread_barrier_t* barrier_interface, const pthread_barrierattr_t* attr,
+                         unsigned count) {
+  pthread_barrier_internal_t* barrier = __get_internal_barrier(barrier_interface);
+  if (count == 0) {
+    return EINVAL;
+  }
+  barrier->init_count = count;
+  atomic_init(&barrier->state, WAIT);
+  atomic_init(&barrier->wait_count, 0);
+  barrier->pshared = false;
+  if (attr != nullptr && (*attr & 1)) {
+    barrier->pshared = true;
+  }
+  return 0;
+}
+
+// According to POSIX standard, pthread_barrier_wait() synchronizes memory between participating
+// threads. It means all memory operations made by participating threads before calling
+// pthread_barrier_wait() can be seen by all participating threads after the function call.
+// We establish this by making a happens-before relation between all threads entering the barrier
+// with the last thread entering the barrier, and a happens-before relation between the last
+// thread entering the barrier with all threads leaving the barrier.
+int pthread_barrier_wait(pthread_barrier_t* barrier_interface) {
+  pthread_barrier_internal_t* barrier = __get_internal_barrier(barrier_interface);
+
+  // Wait until all threads for the previous cycle have left the barrier. This is needed
+  // as a participating thread can call pthread_barrier_wait() again before other
+  // threads have left the barrier. Use acquire operation here to synchronize with
+  // the last thread leaving the previous cycle, so we can read correct wait_count below.
+  while(atomic_load_explicit(&barrier->state, memory_order_acquire) == RELEASE) {
+    __futex_wait_ex(&barrier->state, barrier->pshared, RELEASE, nullptr);
+  }
+
+  uint32_t prev_wait_count = atomic_load_explicit(&barrier->wait_count, memory_order_relaxed);
+  while (true) {
+    // It happens when there are more than [init_count] threads trying to enter the barrier
+    // at one cycle. We read the POSIX standard as disallowing this, since additional arriving
+    // threads are not synchronized with respect to the barrier reset. We also don't know of
+    // any reasonable cases in which this would be intentional.
+    if (prev_wait_count >= barrier->init_count) {
+      return EINVAL;
+    }
+    // Use memory_order_acq_rel operation here to synchronize between all threads entering
+    // the barrier with the last thread entering the barrier.
+    if (atomic_compare_exchange_weak_explicit(&barrier->wait_count, &prev_wait_count,
+                                              prev_wait_count + 1u, memory_order_acq_rel,
+                                              memory_order_relaxed)) {
+      break;
+    }
+  }
+
+  int result = 0;
+  if (prev_wait_count + 1 == barrier->init_count) {
+    result = PTHREAD_BARRIER_SERIAL_THREAD;
+    if (prev_wait_count != 0) {
+      // Use release operation here to synchronize between the last thread entering the
+      // barrier with all threads leaving the barrier.
+      atomic_store_explicit(&barrier->state, RELEASE, memory_order_release);
+      __futex_wake_ex(&barrier->state, barrier->pshared, prev_wait_count);
+    }
+  } else {
+    // Use acquire operation here to synchronize between the last thread entering the
+    // barrier with all threads leaving the barrier.
+    while (atomic_load_explicit(&barrier->state, memory_order_acquire) == WAIT) {
+      __futex_wait_ex(&barrier->state, barrier->pshared, WAIT, nullptr);
+    }
+  }
+  // Use release operation here to make it not reordered with previous operations.
+  if (atomic_fetch_sub_explicit(&barrier->wait_count, 1, memory_order_release) == 1) {
+    // Use release operation here to synchronize with threads entering the barrier for
+    // the next cycle, or the thread calling pthread_barrier_destroy().
+    atomic_store_explicit(&barrier->state, WAIT, memory_order_release);
+    __futex_wake_ex(&barrier->state, barrier->pshared, barrier->init_count);
+  }
+  return result;
+}
+
+int pthread_barrier_destroy(pthread_barrier_t* barrier_interface) {
+  pthread_barrier_internal_t* barrier = __get_internal_barrier(barrier_interface);
+  if (barrier->init_count == 0) {
+    return EINVAL;
+  }
+  // Use acquire operation here to synchronize with the last thread leaving the barrier.
+  // So we can read correct wait_count below.
+  while (atomic_load_explicit(&barrier->state, memory_order_acquire) == RELEASE) {
+    __futex_wait_ex(&barrier->state, barrier->pshared, RELEASE, nullptr);
+  }
+  if (atomic_load_explicit(&barrier->wait_count, memory_order_relaxed) != 0) {
+    return EBUSY;
+  }
+  barrier->init_count = 0;
+  return 0;
+}
diff --git a/libc/include/pthread.h b/libc/include/pthread.h
index 260ae5b..48b2bca 100644
--- a/libc/include/pthread.h
+++ b/libc/include/pthread.h
@@ -96,6 +96,18 @@
 
 #define PTHREAD_ONCE_INIT 0
 
+typedef struct {
+#if defined(__LP64__)
+  int64_t __private[4];
+#else
+  int32_t __private[8];
+#endif
+} pthread_barrier_t;
+
+typedef int pthread_barrierattr_t;
+
+#define PTHREAD_BARRIER_SERIAL_THREAD -1
+
 #if defined(__LP64__)
 #define PTHREAD_STACK_MIN (4 * PAGE_SIZE)
 #else
@@ -130,7 +142,7 @@
 int pthread_attr_setschedpolicy(pthread_attr_t*, int) __nonnull((1));
 int pthread_attr_setscope(pthread_attr_t*, int) __nonnull((1));
 int pthread_attr_setstack(pthread_attr_t*, void*, size_t) __nonnull((1));
-int pthread_attr_setstacksize(pthread_attr_t*, size_t stack_size) __nonnull((1));
+int pthread_attr_setstacksize(pthread_attr_t*, size_t) __nonnull((1));
 
 int pthread_condattr_destroy(pthread_condattr_t*) __nonnull((1));
 int pthread_condattr_getclock(const pthread_condattr_t*, clockid_t*) __nonnull((1, 2));
@@ -205,9 +217,18 @@
 int pthread_rwlock_timedwrlock(pthread_rwlock_t*, const struct timespec*) __nonnull((1, 2));
 int pthread_rwlock_tryrdlock(pthread_rwlock_t*) __nonnull((1));
 int pthread_rwlock_trywrlock(pthread_rwlock_t*) __nonnull((1));
-int pthread_rwlock_unlock(pthread_rwlock_t *rwlock) __nonnull((1));
+int pthread_rwlock_unlock(pthread_rwlock_t *) __nonnull((1));
 int pthread_rwlock_wrlock(pthread_rwlock_t*) __nonnull((1));
 
+int pthread_barrierattr_init(pthread_barrierattr_t* attr) __nonnull((1));
+int pthread_barrierattr_destroy(pthread_barrierattr_t* attr) __nonnull((1));
+int pthread_barrierattr_getpshared(pthread_barrierattr_t* attr, int* pshared) __nonnull((1, 2));
+int pthread_barrierattr_setpshared(pthread_barrierattr_t* attr, int pshared) __nonnull((1));
+
+int pthread_barrier_init(pthread_barrier_t*, const pthread_barrierattr_t*, unsigned) __nonnull((1));
+int pthread_barrier_destroy(pthread_barrier_t*) __nonnull((1));
+int pthread_barrier_wait(pthread_barrier_t*) __nonnull((1));
+
 pthread_t pthread_self(void) __pure2;
 
 int pthread_setname_np(pthread_t, const char*) __nonnull((2));
diff --git a/libc/libc.arm.map b/libc/libc.arm.map
index bc68fc0..489a368 100644
--- a/libc/libc.arm.map
+++ b/libc/libc.arm.map
@@ -1315,6 +1315,13 @@
     preadv;
     preadv64;
     prlimit; # arm mips x86
+    pthread_barrierattr_destroy;
+    pthread_barrierattr_getpshared;
+    pthread_barrierattr_init;
+    pthread_barrierattr_setpshared;
+    pthread_barrier_destroy;
+    pthread_barrier_init;
+    pthread_barrier_wait;
     pwritev;
     pwritev64;
     scandirat;
diff --git a/libc/libc.arm64.map b/libc/libc.arm64.map
index a58b757..6b8b55b 100644
--- a/libc/libc.arm64.map
+++ b/libc/libc.arm64.map
@@ -1160,6 +1160,13 @@
     getgrnam_r;
     preadv;
     preadv64;
+    pthread_barrierattr_destroy;
+    pthread_barrierattr_getpshared;
+    pthread_barrierattr_init;
+    pthread_barrierattr_setpshared;
+    pthread_barrier_destroy;
+    pthread_barrier_init;
+    pthread_barrier_wait;
     pwritev;
     pwritev64;
     scandirat;
diff --git a/libc/libc.map.txt b/libc/libc.map.txt
index 418d4cf..83965bd 100644
--- a/libc/libc.map.txt
+++ b/libc/libc.map.txt
@@ -1342,6 +1342,13 @@
     preadv;
     preadv64;
     prlimit; # arm mips x86
+    pthread_barrierattr_destroy;
+    pthread_barrierattr_getpshared;
+    pthread_barrierattr_init;
+    pthread_barrierattr_setpshared;
+    pthread_barrier_destroy;
+    pthread_barrier_init;
+    pthread_barrier_wait;
     pwritev;
     pwritev64;
     scandirat;
diff --git a/libc/libc.mips.map b/libc/libc.mips.map
index 0c69c19..d74c2c8 100644
--- a/libc/libc.mips.map
+++ b/libc/libc.mips.map
@@ -1278,6 +1278,13 @@
     preadv;
     preadv64;
     prlimit; # arm mips x86
+    pthread_barrierattr_destroy;
+    pthread_barrierattr_getpshared;
+    pthread_barrierattr_init;
+    pthread_barrierattr_setpshared;
+    pthread_barrier_destroy;
+    pthread_barrier_init;
+    pthread_barrier_wait;
     pwritev;
     pwritev64;
     scandirat;
diff --git a/libc/libc.mips64.map b/libc/libc.mips64.map
index a58b757..6b8b55b 100644
--- a/libc/libc.mips64.map
+++ b/libc/libc.mips64.map
@@ -1160,6 +1160,13 @@
     getgrnam_r;
     preadv;
     preadv64;
+    pthread_barrierattr_destroy;
+    pthread_barrierattr_getpshared;
+    pthread_barrierattr_init;
+    pthread_barrierattr_setpshared;
+    pthread_barrier_destroy;
+    pthread_barrier_init;
+    pthread_barrier_wait;
     pwritev;
     pwritev64;
     scandirat;
diff --git a/libc/libc.x86.map b/libc/libc.x86.map
index 544ee13..7dc85cc 100644
--- a/libc/libc.x86.map
+++ b/libc/libc.x86.map
@@ -1276,6 +1276,13 @@
     preadv;
     preadv64;
     prlimit; # arm mips x86
+    pthread_barrierattr_destroy;
+    pthread_barrierattr_getpshared;
+    pthread_barrierattr_init;
+    pthread_barrierattr_setpshared;
+    pthread_barrier_destroy;
+    pthread_barrier_init;
+    pthread_barrier_wait;
     pwritev;
     pwritev64;
     scandirat;
diff --git a/libc/libc.x86_64.map b/libc/libc.x86_64.map
index a58b757..6b8b55b 100644
--- a/libc/libc.x86_64.map
+++ b/libc/libc.x86_64.map
@@ -1160,6 +1160,13 @@
     getgrnam_r;
     preadv;
     preadv64;
+    pthread_barrierattr_destroy;
+    pthread_barrierattr_getpshared;
+    pthread_barrierattr_init;
+    pthread_barrierattr_setpshared;
+    pthread_barrier_destroy;
+    pthread_barrier_init;
+    pthread_barrier_wait;
     pwritev;
     pwritev64;
     scandirat;
diff --git a/tests/pthread_test.cpp b/tests/pthread_test.cpp
index 9f887e3..88b8a5c 100644
--- a/tests/pthread_test.cpp
+++ b/tests/pthread_test.cpp
@@ -721,11 +721,11 @@
   ASSERT_EQ(0, pthread_rwlock_destroy(&l));
 }
 
-static void WaitUntilThreadSleep(std::atomic<pid_t>& pid) {
-  while (pid == 0) {
+static void WaitUntilThreadSleep(std::atomic<pid_t>& tid) {
+  while (tid == 0) {
     usleep(1000);
   }
-  std::string filename = android::base::StringPrintf("/proc/%d/stat", pid.load());
+  std::string filename = android::base::StringPrintf("/proc/%d/stat", tid.load());
   std::regex regex {R"(\s+S\s+)"};
 
   while (true) {
@@ -1653,3 +1653,117 @@
   kill(getpid(), SIGUSR1);
   ASSERT_TRUE(signal_handler_on_altstack_done);
 }
+
+TEST(pthread, pthread_barrierattr_smoke) {
+  pthread_barrierattr_t attr;
+  ASSERT_EQ(0, pthread_barrierattr_init(&attr));
+  int pshared;
+  ASSERT_EQ(0, pthread_barrierattr_getpshared(&attr, &pshared));
+  ASSERT_EQ(PTHREAD_PROCESS_PRIVATE, pshared);
+  ASSERT_EQ(0, pthread_barrierattr_setpshared(&attr, PTHREAD_PROCESS_SHARED));
+  ASSERT_EQ(0, pthread_barrierattr_getpshared(&attr, &pshared));
+  ASSERT_EQ(PTHREAD_PROCESS_SHARED, pshared);
+  ASSERT_EQ(0, pthread_barrierattr_destroy(&attr));
+}
+
+struct BarrierTestHelperArg {
+  std::atomic<pid_t> tid;
+  pthread_barrier_t* barrier;
+  size_t iteration_count;
+};
+
+static void BarrierTestHelper(BarrierTestHelperArg* arg) {
+  arg->tid = gettid();
+  for (size_t i = 0; i < arg->iteration_count; ++i) {
+    ASSERT_EQ(0, pthread_barrier_wait(arg->barrier));
+  }
+}
+
+TEST(pthread, pthread_barrier_smoke) {
+  const size_t BARRIER_ITERATION_COUNT = 10;
+  const size_t BARRIER_THREAD_COUNT = 10;
+  pthread_barrier_t barrier;
+  ASSERT_EQ(0, pthread_barrier_init(&barrier, nullptr, BARRIER_THREAD_COUNT + 1));
+  std::vector<pthread_t> threads(BARRIER_THREAD_COUNT);
+  std::vector<BarrierTestHelperArg> args(threads.size());
+  for (size_t i = 0; i < threads.size(); ++i) {
+    args[i].tid = 0;
+    args[i].barrier = &barrier;
+    args[i].iteration_count = BARRIER_ITERATION_COUNT;
+    ASSERT_EQ(0, pthread_create(&threads[i], nullptr,
+                                reinterpret_cast<void* (*)(void*)>(BarrierTestHelper), &args[i]));
+  }
+  for (size_t iteration = 0; iteration < BARRIER_ITERATION_COUNT; ++iteration) {
+    for (size_t i = 0; i < threads.size(); ++i) {
+      WaitUntilThreadSleep(args[i].tid);
+    }
+    ASSERT_EQ(PTHREAD_BARRIER_SERIAL_THREAD, pthread_barrier_wait(&barrier));
+  }
+  for (size_t i = 0; i < threads.size(); ++i) {
+    ASSERT_EQ(0, pthread_join(threads[i], nullptr));
+  }
+  ASSERT_EQ(0, pthread_barrier_destroy(&barrier));
+}
+
+TEST(pthread, pthread_barrier_destroy) {
+  pthread_barrier_t barrier;
+  ASSERT_EQ(0, pthread_barrier_init(&barrier, nullptr, 2));
+  pthread_t thread;
+  BarrierTestHelperArg arg;
+  arg.tid = 0;
+  arg.barrier = &barrier;
+  arg.iteration_count = 1;
+  ASSERT_EQ(0, pthread_create(&thread, nullptr,
+                              reinterpret_cast<void* (*)(void*)>(BarrierTestHelper), &arg));
+  WaitUntilThreadSleep(arg.tid);
+  ASSERT_EQ(EBUSY, pthread_barrier_destroy(&barrier));
+  ASSERT_EQ(PTHREAD_BARRIER_SERIAL_THREAD, pthread_barrier_wait(&barrier));
+  // Verify if the barrier can be destroyed directly after pthread_barrier_wait().
+  ASSERT_EQ(0, pthread_barrier_destroy(&barrier));
+  ASSERT_EQ(0, pthread_join(thread, nullptr));
+#if defined(__BIONIC__)
+  ASSERT_EQ(EINVAL, pthread_barrier_destroy(&barrier));
+#endif
+}
+
+struct BarrierOrderingTestHelperArg {
+  pthread_barrier_t* barrier;
+  size_t* array;
+  size_t array_length;
+  size_t id;
+};
+
+void BarrierOrderingTestHelper(BarrierOrderingTestHelperArg* arg) {
+  const size_t ITERATION_COUNT = 10000;
+  for (size_t i = 1; i <= ITERATION_COUNT; ++i) {
+    arg->array[arg->id] = i;
+    int ret = pthread_barrier_wait(arg->barrier);
+    ASSERT_TRUE(ret == 0 || ret == PTHREAD_BARRIER_SERIAL_THREAD);
+    for (size_t j = 0; j < arg->array_length; ++j) {
+      ASSERT_EQ(i, arg->array[j]);
+    }
+    ret = pthread_barrier_wait(arg->barrier);
+    ASSERT_TRUE(ret == 0 || ret == PTHREAD_BARRIER_SERIAL_THREAD);
+  }
+}
+
+TEST(pthread, pthread_barrier_check_ordering) {
+  const size_t THREAD_COUNT = 4;
+  pthread_barrier_t barrier;
+  ASSERT_EQ(0, pthread_barrier_init(&barrier, nullptr, THREAD_COUNT));
+  size_t array[THREAD_COUNT];
+  std::vector<pthread_t> threads(THREAD_COUNT);
+  std::vector<BarrierOrderingTestHelperArg> args(THREAD_COUNT);
+  for (size_t i = 0; i < THREAD_COUNT; ++i) {
+    args[i].barrier = &barrier;
+    args[i].array = array;
+    args[i].array_length = THREAD_COUNT;
+    args[i].id = i;
+    ASSERT_EQ(0, pthread_create(&threads[i], nullptr,
+                                reinterpret_cast<void* (*)(void*)>(BarrierOrderingTestHelper),
+                                &args[i]));
+  }
+  for (size_t i = 0; i < THREAD_COUNT; ++i) {
+    ASSERT_EQ(0, pthread_join(threads[i], nullptr));
+  }
+}