commit dba3ec94281c052b943eb24ddee7cc3691caeaaf
author Sylvain Jeaugey <sjeaugey@nvidia.com> Tue Apr 19 10:47:27 2016 -0700
committer Sylvain Jeaugey <sjeaugey@nvidia.com> Tue Apr 19 10:47:27 2016 -0700
tree 8d6fe772e372cf4842db279e16b4cf5480503c35
parent 9de361a1b97b55f1669107e94efc95c1a299234c

Fix random deadlock during ncclCommInitRank.

src/core.cu

diff --git a/src/core.cu b/src/core.cu
index 6adeb36..cec2794 100644
--- a/src/core.cu
+++ b/src/core.cu
@@ -188,7 +188,7 @@
     swapped = __sync_bool_compare_and_swap(&gather->bar, bar_tmp, bar_tmp+1);
   } while(!swapped);
 
-  while (gather->bar != 2*ndev) // Wait for all ranks to arrive at this second barrier
+  while (gather->bar < 2*ndev) // Wait for all ranks to arrive at this second barrier
     sched_yield();
   __sync_synchronize();
 
@@ -203,7 +203,7 @@
     swapped = __sync_bool_compare_and_swap(&gather->bar, bar_tmp, bar_tmp+1);
   } while(!swapped);
 
-  while (gather->bar != 3*ndev) // Wait for all ranks to arrive at this third barrier
+  while (gather->bar < 3*ndev) // Wait for all ranks to arrive at this third barrier
     sched_yield();
   __sync_synchronize();