commit 2b6b3e5317c2159e97e56173fa559479f757980e
author Evgenii Stepanov <eugenis@google.com> Mon Nov 05 17:03:43 2018 -0800
committer Evgenii Stepanov <eugenis@google.com> Thu Nov 15 13:23:59 2018 -0800
tree 8addc71e8a406db574017764b6cc4ed71b39bd7d
parent 23d3ed8c124f239d26a583f9d096b22f7f6677d1

EnableDebugger: handle the case when hard RLIMIT_CORE is <inf.

ASan and HWASan set rlim_cur = rlim_max = 0 for RLIMIT_CORE.

Bug: 112438058
Test: SANITIZE_TARGET=hwaddress, no RLIMIT_CORE warnings in logcat

Change-Id: I8839705673c371b9b393eca36c7519a1ea73d7be

runtime/native/dalvik_system_ZygoteHooks.cc

diff --git a/runtime/native/dalvik_system_ZygoteHooks.cc b/runtime/native/dalvik_system_ZygoteHooks.cc
index 4d3ad62..8f40dcd 100644
--- a/runtime/native/dalvik_system_ZygoteHooks.cc
+++ b/runtime/native/dalvik_system_ZygoteHooks.cc
@@ -76,12 +76,16 @@
     }
   }
 #endif
-  // We don't want core dumps, though, so set the core dump size to 0.
+  // We don't want core dumps, though, so set the soft limit on core dump size
+  // to 0 without changing the hard limit.
   rlimit rl;
-  rl.rlim_cur = 0;
-  rl.rlim_max = RLIM_INFINITY;
-  if (setrlimit(RLIMIT_CORE, &rl) == -1) {
-    PLOG(ERROR) << "setrlimit(RLIMIT_CORE) failed for pid " << getpid();
+  if (getrlimit(RLIMIT_CORE, &rl) == -1) {
+    PLOG(ERROR) << "getrlimit(RLIMIT_CORE) failed for pid " << getpid();
+  } else {
+    rl.rlim_cur = 0;
+    if (setrlimit(RLIMIT_CORE, &rl) == -1) {
+      PLOG(ERROR) << "setrlimit(RLIMIT_CORE) failed for pid " << getpid();
+    }
   }
 }