Add dex2oat watch dog thread for host builds
Change-Id: I90b4b6b3a1aebb82955b4aa84d3f2d599af1d13b
diff --git a/src/dex2oat.cc b/src/dex2oat.cc
index bbc07a6..40218a8 100644
--- a/src/dex2oat.cc
+++ b/src/dex2oat.cc
@@ -473,7 +473,108 @@
return failure_count;
}
+// The primary goal of the watchdog is to prevent stuck build servers
+// during development when fatal aborts lead to a cascade of failures
+// that result in a deadlock.
+class WatchDog {
+
+// WatchDog defines its own CHECK_PTHREAD_CALL to avoid using Log which uses locks
+#undef CHECK_PTHREAD_CALL
+#define CHECK_WATCH_DOG_PTHREAD_CALL(call, args, what) \
+ do { \
+ int rc = call args; \
+ if (rc != 0) { \
+ errno = rc; \
+ std::string message(# call); \
+ message += " failed for "; \
+ message += reason; \
+ Die(message); \
+ } \
+ } while (false)
+
+ public:
+ WatchDog() {
+ if (!kIsWatchDogEnabled) {
+ return;
+ }
+ shutting_down_ = false;
+ const char* reason = "dex2oat watch dog thread startup";
+ CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_init, (&mutex_, NULL), reason);
+ CHECK_WATCH_DOG_PTHREAD_CALL(pthread_cond_init, (&cond_, NULL), reason);
+ CHECK_WATCH_DOG_PTHREAD_CALL(pthread_attr_init, (&attr_), reason);
+ CHECK_WATCH_DOG_PTHREAD_CALL(pthread_create, (&pthread_, &attr_, &CallBack, this), reason);
+ CHECK_WATCH_DOG_PTHREAD_CALL(pthread_attr_destroy, (&attr_), reason);
+ }
+ ~WatchDog() {
+ if (!kIsWatchDogEnabled) {
+ return;
+ }
+ const char* reason = "dex2oat watch dog thread shutdown";
+ CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_lock, (&mutex_), reason);
+ shutting_down_ = true;
+ CHECK_WATCH_DOG_PTHREAD_CALL(pthread_cond_signal, (&cond_), reason);
+ CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_unlock, (&mutex_), reason);
+
+ CHECK_WATCH_DOG_PTHREAD_CALL(pthread_join, (pthread_, NULL), reason);
+
+ CHECK_WATCH_DOG_PTHREAD_CALL(pthread_cond_destroy, (&cond_), reason);
+ CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_destroy, (&mutex_), reason);
+ }
+
+ private:
+ static void* CallBack(void* arg) {
+ WatchDog* self = reinterpret_cast<WatchDog*>(arg);
+ self->Wait();
+ return NULL;
+ }
+
+ static void Die(const std::string& message) {
+ // TODO: Switch to LOG(FATAL) when we can guarantee it won't prevent shutdown in error cases
+ fprintf(stderr, "%s\n", message.c_str());
+ exit(1);
+ }
+
+ void Wait() {
+ int64_t ms = kWatchDogTimeoutSeconds * 1000;
+ int32_t ns = 0;
+ timespec ts;
+ InitTimeSpec(true, CLOCK_REALTIME, ms, ns, &ts);
+ const char* reason = "dex2oat watch dog thread waiting";
+ CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_lock, (&mutex_), reason);
+ while (!shutting_down_) {
+ int rc = TEMP_FAILURE_RETRY(pthread_cond_timedwait(&cond_, &mutex_, &ts));
+ if (rc == ETIMEDOUT) {
+ std::string message(StringPrintf("dex2oat did not finish after %d seconds",
+ kWatchDogTimeoutSeconds));
+ Die(message.c_str());
+ }
+ if (rc != 0) {
+ std::string message(StringPrintf("pthread_cond_timedwait failed: %s",
+ strerror(errno)));
+ Die(message.c_str());
+ }
+ }
+ CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_unlock, (&mutex_), reason);
+ }
+
+ static const bool kIsWatchDogEnabled = !kIsTargetBuild;
+#ifdef ART_USE_LLVM_COMPILER
+ static const unsigned int kWatchDogTimeoutSeconds = 20 * 60; // 15 minutes + buffer
+#else
+ static const unsigned int kWatchDogTimeoutSeconds = 2 * 60; // 1 minute + buffer
+#endif
+
+ bool shutting_down_;
+ // TODO: Switch to Mutex when we can guarantee it won't prevent shutdown in error cases
+ pthread_mutex_t mutex_;
+ pthread_cond_t cond_;
+ pthread_attr_t attr_;
+ pthread_t pthread_;
+};
+
static int dex2oat(int argc, char** argv) {
+ WatchDog watch_dog;
+
InitLogging(argv);
// Skip over argv[0].
diff --git a/src/globals.h b/src/globals.h
index 8577b43..dc9341a 100644
--- a/src/globals.h
+++ b/src/globals.h
@@ -66,6 +66,13 @@
const bool kIsDebugBuild = true;
#endif
+// Whether or not this is a target (vs host) build. Useful in conditionals where ART_TARGET isn't.
+#if defined(ART_TARGET)
+const bool kIsTargetBuild = true;
+#else
+const bool kIsTargetBuild = false;
+#endif
+
} // namespace art
#endif // ART_SRC_GLOBALS_H_
diff --git a/src/mutex.cc b/src/mutex.cc
index 0e33f3c..e2044d6 100644
--- a/src/mutex.cc
+++ b/src/mutex.cc
@@ -27,12 +27,6 @@
#include "thread.h"
#include "utils.h"
-#if defined(__APPLE__)
-#include "AvailabilityMacros.h" // For MAC_OS_X_VERSION_MAX_ALLOWED
-// No clocks to specify on OS/X, fake value to pass to routines that require a clock.
-#define CLOCK_REALTIME 0xebadf00d
-#endif
-
#define CHECK_MUTEX_CALL(call, args) CHECK_PTHREAD_CALL(call, args, name_)
extern int pthread_mutex_lock(pthread_mutex_t* mutex) EXCLUSIVE_LOCK_FUNCTION(mutex);
@@ -100,43 +94,6 @@
}
}
-// Initialize a timespec to either an absolute or relative time.
-static void InitTimeSpec(Thread* self, bool absolute, int clock, int64_t ms, int32_t ns,
- timespec* ts) {
- int64_t endSec;
-
- if (absolute) {
-#if !defined(__APPLE__)
- clock_gettime(clock, ts);
-#else
- UNUSED(clock);
- timeval tv;
- gettimeofday(&tv, NULL);
- ts->tv_sec = tv.tv_sec;
- ts->tv_nsec = tv.tv_usec * 1000;
-#endif
- } else {
- ts->tv_sec = 0;
- ts->tv_nsec = 0;
- }
- endSec = ts->tv_sec + ms / 1000;
- if (UNLIKELY(endSec >= 0x7fffffff)) {
- std::ostringstream ss;
- ScopedObjectAccess soa(self);
- self->Dump(ss);
- LOG(INFO) << "Note: end time exceeds epoch: " << ss.str();
- endSec = 0x7ffffffe;
- }
- ts->tv_sec = endSec;
- ts->tv_nsec = (ts->tv_nsec + (ms % 1000) * 1000000) + ns;
-
- // Catch rollover.
- if (ts->tv_nsec >= 1000000000L) {
- ts->tv_sec++;
- ts->tv_nsec -= 1000000000L;
- }
-}
-
#if ART_USE_FUTEXES
static bool ComputeRelativeTimeSpec(timespec* result_ts, const timespec& lhs, const timespec& rhs) {
const long int one_sec = 1000 * 1000 * 1000; // one second in nanoseconds.
@@ -563,7 +520,7 @@
exclusive_owner_ = SafeGetTid(self);
#else
timespec ts;
- InitTimeSpec(self, true, CLOCK_REALTIME, ms, ns, &ts);
+ InitTimeSpec(true, CLOCK_REALTIME, ms, ns, &ts);
int result = pthread_rwlock_timedwrlock(&rwlock_, &ts);
if (result == ETIMEDOUT) {
return false;
@@ -891,8 +848,8 @@
#endif
guard_.recursion_count_ = 0;
timespec ts;
- InitTimeSpec(self, true, clock, ms, ns, &ts);
- int rc = TIMEDWAIT(&cond_, &guard_.mutex_, &ts);
+ InitTimeSpec(true, clock, ms, ns, &ts);
+ int rc = TEMP_FAILURE_RETRY(TIMEDWAIT(&cond_, &guard_.mutex_, &ts));
if (rc != 0 && rc != ETIMEDOUT) {
errno = rc;
PLOG(FATAL) << "TimedWait failed for " << name_;
diff --git a/src/thread_list.cc b/src/thread_list.cc
index 43a0cee..1a2fd47 100644
--- a/src/thread_list.cc
+++ b/src/thread_list.cc
@@ -207,7 +207,7 @@
// Shouldn't need to wait for longer than 1 millisecond.
const uint64_t threshold = 1;
if (NsToMs(end - start) > threshold) {
- LOG(INFO) << "Warning: waited longer than " << threshold << " ms for thrad suspend"
+ LOG(INFO) << "Warning: waited longer than " << threshold << " ms for thread suspend"
<< std::endl;
}
}
diff --git a/src/thread_pool.cc b/src/thread_pool.cc
index f3319e2..ba53113 100644
--- a/src/thread_pool.cc
+++ b/src/thread_pool.cc
@@ -11,6 +11,7 @@
name_(name),
stack_size_(stack_size) {
const char* reason = "new thread pool worker thread";
+ pthread_attr_t attr;
CHECK_PTHREAD_CALL(pthread_attr_init, (&attr), reason);
CHECK_PTHREAD_CALL(pthread_attr_setstacksize, (&attr, stack_size), reason);
CHECK_PTHREAD_CALL(pthread_create, (&pthread_, &attr, &Callback, this), reason);
diff --git a/src/thread_pool.h b/src/thread_pool.h
index 22e30b7..1d0f85d 100644
--- a/src/thread_pool.h
+++ b/src/thread_pool.h
@@ -47,7 +47,6 @@
const std::string name_;
const size_t stack_size_;
pthread_t pthread_;
- pthread_attr_t attr;
friend class ThreadPool;
DISALLOW_COPY_AND_ASSIGN(ThreadPoolWorker);
diff --git a/src/utils.cc b/src/utils.cc
index cbe07a2..3cf6914 100644
--- a/src/utils.cc
+++ b/src/utils.cc
@@ -180,6 +180,39 @@
#endif
}
+void InitTimeSpec(bool absolute, int clock, int64_t ms, int32_t ns, timespec* ts) {
+ int64_t endSec;
+
+ if (absolute) {
+#if !defined(__APPLE__)
+ clock_gettime(clock, ts);
+#else
+ UNUSED(clock);
+ timeval tv;
+ gettimeofday(&tv, NULL);
+ ts->tv_sec = tv.tv_sec;
+ ts->tv_nsec = tv.tv_usec * 1000;
+#endif
+ } else {
+ ts->tv_sec = 0;
+ ts->tv_nsec = 0;
+ }
+ endSec = ts->tv_sec + ms / 1000;
+ if (UNLIKELY(endSec >= 0x7fffffff)) {
+ std::ostringstream ss;
+ LOG(INFO) << "Note: end time exceeds epoch: " << ss.str();
+ endSec = 0x7ffffffe;
+ }
+ ts->tv_sec = endSec;
+ ts->tv_nsec = (ts->tv_nsec + (ms % 1000) * 1000000) + ns;
+
+ // Catch rollover.
+ if (ts->tv_nsec >= 1000000000L) {
+ ts->tv_sec++;
+ ts->tv_nsec -= 1000000000L;
+ }
+}
+
std::string PrettyDescriptor(const String* java_descriptor) {
if (java_descriptor == NULL) {
return "null";
diff --git a/src/utils.h b/src/utils.h
index 719ce57..d33cc4b 100644
--- a/src/utils.h
+++ b/src/utils.h
@@ -285,6 +285,14 @@
return ns * 1000 * 1000;
}
+#if defined(__APPLE__)
+// No clocks to specify on OS/X, fake value to pass to routines that require a clock.
+#define CLOCK_REALTIME 0xebadf00d
+#endif
+
+// Initialize a timespec to either an absolute or relative time.
+void InitTimeSpec(bool absolute, int clock, int64_t ms, int32_t ns, timespec* ts);
+
// Splits a string using the given separator character into a vector of
// strings. Empty strings will be omitted.
void Split(const std::string& s, char separator, std::vector<std::string>& result);
