[hwasan] Madvise away unused shadow.
Summary:
Whenever a large shadow region is tagged to zero, madvise(DONT_NEED)
as much of it as possible.
This reduces shadow RSS on Android by 45% or so, and total memory use
by 2-4%, probably even more on long running multithreaded programs.
CPU time seems to be in the noise.
Reviewers: kcc, pcc
Subscribers: srhines, kubamracek, llvm-commits
Differential Revision: https://reviews.llvm.org/D56757
git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@351620 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/hwasan/hwasan.cc b/lib/hwasan/hwasan.cc
index 9f2328d..0980c6d 100644
--- a/lib/hwasan/hwasan.cc
+++ b/lib/hwasan/hwasan.cc
@@ -88,6 +88,8 @@
cf.check_printf = false;
cf.intercept_tls_get_addr = true;
cf.exitcode = 99;
+ // 8 shadow pages ~512kB, small enough to cover common stack sizes.
+ cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8);
// Sigtrap is used in error reporting.
cf.handle_sigtrap = kHandleSignalExclusive;
diff --git a/lib/hwasan/hwasan_poisoning.cc b/lib/hwasan/hwasan_poisoning.cc
index 9c8e16b..6fb7d15 100644
--- a/lib/hwasan/hwasan_poisoning.cc
+++ b/lib/hwasan/hwasan_poisoning.cc
@@ -16,6 +16,7 @@
#include "hwasan_mapping.h"
#include "interception/interception.h"
#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_linux.h"
namespace __hwasan {
@@ -24,7 +25,22 @@
CHECK(IsAligned(size, kShadowAlignment));
uptr shadow_start = MemToShadow(p);
uptr shadow_size = MemToShadowSize(size);
- internal_memset((void *)shadow_start, tag, shadow_size);
+
+ uptr page_size = GetPageSizeCached();
+ uptr page_start = RoundUpTo(shadow_start, page_size);
+ uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size);
+ uptr threshold = common_flags()->clear_shadow_mmap_threshold;
+ if (SANITIZER_LINUX &&
+ UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
+ internal_memset((void *)shadow_start, tag, page_start - shadow_start);
+ internal_memset((void *)page_end, tag,
+ shadow_start + shadow_size - page_end);
+ // For an anonymous private mapping MADV_DONTNEED will return a zero page on
+ // Linux.
+ ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end);
+ } else {
+ internal_memset((void *)shadow_start, tag, shadow_size);
+ }
return AddTagToPointer(p, tag);
}
diff --git a/lib/sanitizer_common/sanitizer_linux.h b/lib/sanitizer_common/sanitizer_linux.h
index c309e33..522eb09 100644
--- a/lib/sanitizer_common/sanitizer_linux.h
+++ b/lib/sanitizer_common/sanitizer_linux.h
@@ -106,6 +106,17 @@
// Call cb for each region mapped by map.
void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr));
+// Releases memory pages entirely within the [beg, end] address range.
+// The pages no longer count toward RSS; reads are guaranteed to return 0.
+// Requires (but does not verify!) that pages are MAP_PRIVATE.
+INLINE void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {
+ // man madvise on Linux promises zero-fill for anonymous private pages.
+ // Testing shows the same behaviour for private (but not anonymous) mappings
+ // of shm_open() files, as long as the underlying file is untouched.
+ CHECK(SANITIZER_LINUX);
+ ReleaseMemoryPagesToOS(beg, end);
+}
+
#if SANITIZER_ANDROID
#if defined(__aarch64__)
diff --git a/test/hwasan/TestCases/Linux/release-shadow.c b/test/hwasan/TestCases/Linux/release-shadow.c
new file mode 100644
index 0000000..9aae350
--- /dev/null
+++ b/test/hwasan/TestCases/Linux/release-shadow.c
@@ -0,0 +1,70 @@
+// Test that tagging a large region to 0 reduces RSS.
+// RUN: %clang_hwasan -mllvm -hwasan-instrument-stack=0 %s -o %t && %run %t 2>&1
+
+#include <assert.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <sanitizer/hwasan_interface.h>
+
+const unsigned char kTag = 42;
+const size_t kNumShadowPages = 256;
+const size_t kNumPages = 16 * kNumShadowPages;
+const size_t kPageSize = 4096;
+const size_t kMapSize = kNumPages * kPageSize;
+
+void sync_rss() {
+ char *page = (char *)mmap(0, kPageSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
+ // Linux kernel updates RSS counters after a set number of page faults.
+ for (int i = 0; i < 1000; ++i) {
+ page[0] = 42;
+ madvise(page, kPageSize, MADV_DONTNEED);
+ }
+ munmap(page, kPageSize);
+}
+
+size_t current_rss() {
+ sync_rss();
+ int statm_fd = open("/proc/self/statm", O_RDONLY);
+ assert(statm_fd >= 0);
+
+ char buf[100];
+ assert(read(statm_fd, &buf, sizeof(buf)) > 0);
+ size_t size, rss;
+ assert(sscanf(buf, "%zu %zu", &size, &rss) == 2);
+
+ close(statm_fd);
+ return rss;
+}
+
+void test_rss_difference(void *p) {
+ __hwasan_tag_memory(p, kTag, kMapSize);
+ size_t rss_before = current_rss();
+ __hwasan_tag_memory(p, 0, kMapSize);
+ size_t rss_after = current_rss();
+ fprintf(stderr, "%zu -> %zu\n", rss_before, rss_after);
+ assert(rss_before > rss_after);
+ size_t diff = rss_before - rss_after;
+ fprintf(stderr, "diff %zu\n", diff);
+ // Check that the difference is at least close to kNumShadowPages.
+ assert(diff > kNumShadowPages / 4 * 3);
+}
+
+int main() {
+ fprintf(stderr, "starting rss %zu\n", current_rss());
+ fprintf(stderr, "shadow pages: %zu\n", kNumShadowPages);
+
+ void *p = mmap(0, kMapSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
+ fprintf(stderr, "p = %p\n", p);
+
+ test_rss_difference(p);
+ test_rss_difference(p);
+ test_rss_difference(p);
+
+ return 0;
+}