hwasan: Read shadow address from ifunc if we don't need a frame record.
This saves a cbz+cold call in the interceptor ABI, as well as a realign
in both ABIs, trading off a dcache entry against some branch predictor
entries and some code size.
Unfortunately the functionality is hidden behind a flag because ifunc is
known to be broken on static binaries on Android.
Differential Revision: https://reviews.llvm.org/D57084
Change-Id: I71c1d6a84294504a429fb9e725da471bd9bfc10b
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@351989 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index bf3f8f7..837220c 100644
--- a/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -156,6 +156,10 @@
cl::desc("inline all checks"),
cl::Hidden, cl::init(false));
+static cl::opt<bool> ClAllowIfunc("hwasan-allow-ifunc",
+ cl::desc("allow the use of ifunc"),
+ cl::Hidden, cl::init(false));
+
namespace {
/// An instrumentation pass implementing detection of addressability bugs
@@ -179,6 +183,7 @@
void initializeCallbacks(Module &M);
+ Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
Value *getDynamicShadowNonTls(IRBuilder<> &IRB);
void untagPointerOperand(Instruction *I, Value *Addr);
@@ -377,9 +382,18 @@
HwasanGenerateTagFunc = checkSanitizerInterfaceFunction(
M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty));
- if (Mapping.InGlobal)
- ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
- ArrayType::get(IRB.getInt8Ty(), 0));
+ ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
+ ArrayType::get(IRB.getInt8Ty(), 0));
+}
+
+Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
+ // An empty inline asm with input reg == output reg.
+ // An opaque no-op cast, basically.
+ InlineAsm *Asm = InlineAsm::get(
+ FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false),
+ StringRef(""), StringRef("=r,0"),
+ /*hasSideEffects=*/false);
+ return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
}
Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) {
@@ -388,13 +402,7 @@
return nullptr;
if (Mapping.InGlobal) {
- // An empty inline asm with input reg == output reg.
- // An opaque no-op cast, basically.
- InlineAsm *Asm = InlineAsm::get(
- FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false),
- StringRef(""), StringRef("=r,0"),
- /*hasSideEffects=*/false);
- return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
+ return getDynamicShadowIfunc(IRB);
} else {
Value *GlobalDynamicAddress =
IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
@@ -782,6 +790,9 @@
if (!Mapping.InTls)
return getDynamicShadowNonTls(IRB);
+ if (ClAllowIfunc && !WithFrameRecord && TargetTriple.isAndroid())
+ return getDynamicShadowIfunc(IRB);
+
Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
assert(SlotPtr);
diff --git a/test/Instrumentation/HWAddressSanitizer/prologue.ll b/test/Instrumentation/HWAddressSanitizer/prologue.ll
index 5cbfa67..1163dbe 100644
--- a/test/Instrumentation/HWAddressSanitizer/prologue.ll
+++ b/test/Instrumentation/HWAddressSanitizer/prologue.ll
@@ -1,15 +1,15 @@
; Test -hwasan-with-ifunc flag.
;
-; RUN: opt -hwasan -S < %s | \
-; RUN: FileCheck %s --check-prefixes=CHECK,CHECK-TLS,CHECK-HISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=1 < %s | \
-; RUN: FileCheck %s --check-prefixes=CHECK,CHECK-TLS,CHECK-HISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=0 < %s | \
-; RUN: FileCheck %s --check-prefixes=CHECK,CHECK-TLS,CHECK-NOHISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=0 < %s | \
+; RUN: opt -hwasan -hwasan-allow-ifunc -S < %s | \
+; RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-TLS,CHECK-HISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=1 < %s | \
+; RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-TLS,CHECK-HISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=0 < %s | \
+; RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-IFUNC,CHECK-NOHISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=0 -hwasan-with-tls=0 < %s | \
; RUN: FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL,CHECK-NOHISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=1 -hwasan-with-tls=0 < %s | \
-; RUN: FileCheck %s --check-prefixes=CHECK,CHECK-IFUNC,CHECK-NOHISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=1 -hwasan-with-tls=0 < %s | \
+; RUN: FileCheck %s --check-prefixes=CHECK,CHECk-NOGLOBAL,CHECK-IFUNC,CHECK-NOHISTORY
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-android22"
@@ -23,18 +23,11 @@
; CHECK-LABEL: @test_load
; CHECK: entry:
-; CHECK-IFUNC: %[[A:[^ ]*]] = call i8* asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
-; CHECK-IFUNC: @llvm.hwasan.check.memaccess(i8* %[[A]]
+; CHECK-NOGLOBAL: %[[A:[^ ]*]] = call i8* asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
+; CHECK-NOGLOBAL: @llvm.hwasan.check.memaccess(i8* %[[A]]
; CHECK-GLOBAL: load i8*, i8** @__hwasan_shadow_memory_dynamic_address
-; CHECK-TLS: %[[A:[^ ]*]] = call i8* @llvm.thread.pointer()
-; CHECK-TLS: %[[B:[^ ]*]] = getelementptr i8, i8* %[[A]], i32 48
-; CHECK-TLS: %[[C:[^ ]*]] = bitcast i8* %[[B]] to i64*
-; CHECK-TLS: %[[D:[^ ]*]] = load i64, i64* %[[C]]
-; CHECK-TLS: %[[E:[^ ]*]] = or i64 %[[D]], 4294967295
-; CHECK-TLS: = add i64 %[[E]], 1
-
; "store i64" is only used to update stack history (this input IR intentionally does not use any i64)
; W/o any allocas, the history is not updated, even if it is enabled explicitly with -hwasan-record-stack-history=1
; CHECK-NOT: store i64
