Implement arm64 TLSDESC
Each TLSDESC relocation relocates a 2-word descriptor in the GOT that
contains:
- the address of a TLS resolver function
- an argument to pass (indirectly) to the resolver function
(Specifically, the address of the 2-word descriptor is passed to the
resolver.)
The loader resolves R_GENERIC_TLSDESC relocations using one of three
resolver functions that it defines:
- tlsdesc_resolver_static
- tlsdesc_resolver_dynamic
- tlsdesc_resolver_unresolved_weak
The resolver functions are written in assembly because they have a
restrictive calling convention. They're only allowed to modify x0 and
(apparently) the condition codes.
For a relocation to memory in static TLS (i.e. the executable or an solib
loaded initially), the loader uses a simple resolver function,
tlsdesc_resolver_static, that returns the static offset it receives from
the loader.
For relocations to dynamic TLS memory (i.e. memory in a dlopen'ed solib),
the loader uses tlsdesc_resolver_dynamic, which allocates TLS memory on
demand. It inlines the fast path of __tls_get_addr, then falls back to
__tls_get_addr when it needs to allocate memory. The loader handles these
dynamic TLS relocations in two passes:
- In the first pass, it allocates a table of TlsDynamicResolverArg
objects, one per dynamic TLSDESC relocation.
- In the second pass, once the table is finalized, it writes the
addresses of the TlsDynamicResolverArg objects into the TLSDESC
relocations.
tlsdesc_resolver_unresolved_weak returns a negated thread pointer so that
taking the address of an unresolved weak TLS symbols produces NULL.
The loader handles R_GENERIC_TLSDESC in a target-independent way, but
only for arm64, because Bionic has only implemented the resolver functions
for arm64.
Bug: http://b/78026329
Test: bionic unit tests
Test: check that backtrace works inside a resolver function and inside
__tls_get_addr called from a resolver
(gdbclient.py, b __tls_get_addr, bt)
Change-Id: I752e59ff986292449892c449dad2546e6f0ff7b6
diff --git a/linker/Android.bp b/linker/Android.bp
index 5ae09ba..033860a 100644
--- a/linker/Android.bp
+++ b/linker/Android.bp
@@ -103,6 +103,7 @@
name: "linker_sources_arm64",
srcs: [
"arch/arm64/begin.S",
+ "arch/arm64/tlsdesc_resolver.S",
],
}
diff --git a/linker/arch/arm64/tlsdesc_resolver.S b/linker/arch/arm64/tlsdesc_resolver.S
new file mode 100644
index 0000000..ef46839
--- /dev/null
+++ b/linker/arch/arm64/tlsdesc_resolver.S
@@ -0,0 +1,203 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <private/bionic_asm.h>
+#include <private/bionic_asm_tls.h>
+
+.globl __tls_get_addr
+
+// These resolver functions must preserve every register except x0. They set x0
+// to the offset of the TLS symbol relative to the thread pointer.
+
+ENTRY_PRIVATE(tlsdesc_resolver_static)
+ ldr x0, [x0, #8]
+ ret
+END(tlsdesc_resolver_static)
+
+ENTRY_PRIVATE(tlsdesc_resolver_dynamic)
+ stp x19, x20, [sp, #-32]!
+ .cfi_def_cfa_offset 32
+ .cfi_rel_offset x19, 0
+ .cfi_rel_offset x20, 8
+ stp x21, x22, [sp, #16]
+ .cfi_rel_offset x21, 16
+ .cfi_rel_offset x22, 24
+
+ mrs x19, tpidr_el0 // __get_tls()
+ ldr x20, [x19, #(TLS_SLOT_DTV * 8)]
+ ldr x21, [x20] // TlsDtv::generation
+
+ ldr x0, [x0, #8] // TlsDynamicResolverArg*
+ ldr x22, [x0] // TlsDynamicResolverArg::generation
+
+ cmp x21, x22
+ b.lo .fallback
+
+ ldr x21, [x0, #8] // TlsIndex::module
+ ldr x22, [x0, #16] // TlsIndex::offset
+ ldr x21, [x20, x21, lsl #3] // TlsDtv::modules[module]
+ cbz x21, .fallback
+ add x0, x21, x22
+ sub x0, x0, x19
+
+ ldp x21, x22, [sp, #16]
+ .cfi_remember_state
+ .cfi_restore x21
+ .cfi_restore x22
+ ldp x19, x20, [sp], #32
+ .cfi_adjust_cfa_offset -32
+ .cfi_restore x19
+ .cfi_restore x20
+ ret
+
+.fallback:
+ .cfi_restore_state
+ ldp x21, x22, [sp, #16]
+ .cfi_restore x21
+ .cfi_restore x22
+ ldp x19, x20, [sp], #32
+ .cfi_adjust_cfa_offset -32
+ .cfi_restore x19
+ .cfi_restore x20
+ b tlsdesc_resolver_dynamic_slow_path
+END(tlsdesc_resolver_dynamic)
+
+#define SAVE_REG(x, slot) \
+ str x, [sp, #((slot) * 8)]; \
+ .cfi_rel_offset x, (slot) * 8; \
+
+#define SAVE_GPR_PAIR(x, y, slot) \
+ stp x, y, [sp, #((slot) * 8)]; \
+ .cfi_rel_offset x, (slot) * 8; \
+ .cfi_rel_offset y, ((slot) + 1) * 8; \
+
+#define SAVE_VEC_PAIR(x, y, slot) \
+ stp x, y, [sp, #((slot) * 8)]; \
+ .cfi_rel_offset x, (slot) * 8; \
+ .cfi_rel_offset y, ((slot) + 2) * 8; \
+
+#define RESTORE_REG(x, slot) \
+ ldr x, [sp, #((slot) * 8)]; \
+ .cfi_restore x; \
+
+#define RESTORE_REG_PAIR(x, y, slot) \
+ ldp x, y, [sp, #((slot) * 8)]; \
+ .cfi_restore x; \
+ .cfi_restore y; \
+
+// On entry, x0 is the address of a TlsDynamicResolverArg object rather than
+// the TlsDescriptor address passed to the original resolver function.
+ENTRY_PRIVATE(tlsdesc_resolver_dynamic_slow_path)
+ sub sp, sp, #(8 * 84)
+ .cfi_def_cfa_offset (8 * 84)
+ SAVE_GPR_PAIR(x29, x30, 0)
+ mov x29, sp
+
+ // Avoid leaking the contents of the shadow call stack register (x18) into
+ // memory. x19 through x29 are callee-save registers, so we do not need to
+ // save them.
+ SAVE_GPR_PAIR(x1, x2, 2)
+ SAVE_GPR_PAIR(x3, x4, 4)
+ SAVE_GPR_PAIR(x5, x6, 6)
+ SAVE_GPR_PAIR(x7, x8, 8)
+ SAVE_GPR_PAIR(x9, x10, 10)
+ SAVE_GPR_PAIR(x11, x12, 12)
+ SAVE_GPR_PAIR(x13, x14, 14)
+ SAVE_GPR_PAIR(x15, x16, 16)
+ SAVE_REG(x17, 18)
+
+ SAVE_VEC_PAIR(q0, q1, 20)
+ SAVE_VEC_PAIR(q2, q3, 24)
+ SAVE_VEC_PAIR(q4, q5, 28)
+ SAVE_VEC_PAIR(q6, q7, 32)
+ SAVE_VEC_PAIR(q8, q9, 36)
+ SAVE_VEC_PAIR(q10, q11, 40)
+ SAVE_VEC_PAIR(q12, q13, 44)
+ SAVE_VEC_PAIR(q14, q15, 48)
+ SAVE_VEC_PAIR(q16, q17, 52)
+ SAVE_VEC_PAIR(q18, q19, 56)
+ SAVE_VEC_PAIR(q20, q21, 60)
+ SAVE_VEC_PAIR(q22, q23, 64)
+ SAVE_VEC_PAIR(q24, q25, 68)
+ SAVE_VEC_PAIR(q26, q27, 72)
+ SAVE_VEC_PAIR(q28, q29, 76)
+ SAVE_VEC_PAIR(q30, q31, 80)
+
+ add x0, x0, #8
+ bl __tls_get_addr
+ mrs x1, tpidr_el0 // __get_tls()
+ sub x0, x0, x1
+
+ RESTORE_REG_PAIR(q30, q31, 80)
+ RESTORE_REG_PAIR(q28, q29, 76)
+ RESTORE_REG_PAIR(q26, q27, 72)
+ RESTORE_REG_PAIR(q24, q25, 68)
+ RESTORE_REG_PAIR(q22, q23, 64)
+ RESTORE_REG_PAIR(q20, q21, 60)
+ RESTORE_REG_PAIR(q18, q19, 56)
+ RESTORE_REG_PAIR(q16, q17, 52)
+ RESTORE_REG_PAIR(q14, q15, 48)
+ RESTORE_REG_PAIR(q12, q13, 44)
+ RESTORE_REG_PAIR(q10, q11, 40)
+ RESTORE_REG_PAIR(q8, q9, 36)
+ RESTORE_REG_PAIR(q6, q7, 32)
+ RESTORE_REG_PAIR(q4, q5, 28)
+ RESTORE_REG_PAIR(q2, q3, 24)
+ RESTORE_REG_PAIR(q0, q1, 20)
+
+ RESTORE_REG(x17, 18)
+ RESTORE_REG_PAIR(x15, x16, 16)
+ RESTORE_REG_PAIR(x13, x14, 14)
+ RESTORE_REG_PAIR(x11, x12, 12)
+ RESTORE_REG_PAIR(x9, x10, 10)
+ RESTORE_REG_PAIR(x7, x8, 8)
+ RESTORE_REG_PAIR(x5, x6, 6)
+ RESTORE_REG_PAIR(x3, x4, 4)
+ RESTORE_REG_PAIR(x1, x2, 2)
+
+ RESTORE_REG_PAIR(x29, x30, 0)
+ add sp, sp, #(8 * 84)
+ .cfi_def_cfa_offset 0
+ ret
+END(tlsdesc_resolver_dynamic_slow_path)
+
+// The address of an unresolved weak TLS symbol evaluates to NULL with TLSDESC.
+// The value returned by this function is added to the thread pointer, so return
+// a negated thread pointer to cancel it out.
+ENTRY_PRIVATE(tlsdesc_resolver_unresolved_weak)
+ str x19, [sp, #-16]!
+ .cfi_def_cfa_offset 16
+ .cfi_rel_offset x19, 0
+ ldr x19, [x0, #8]
+ mrs x0, tpidr_el0 // __get_tls()
+ sub x0, x19, x0
+ ldr x19, [sp], #16
+ .cfi_def_cfa_offset 0
+ .cfi_restore x19
+ ret
+END(tlsdesc_resolver_unresolved_weak)
diff --git a/linker/linker.cpp b/linker/linker.cpp
index 4dcdf7e..d0c740b 100644
--- a/linker/linker.cpp
+++ b/linker/linker.cpp
@@ -2698,6 +2698,7 @@
bool soinfo::relocate(const VersionTracker& version_tracker, ElfRelIteratorT&& rel_iterator,
const soinfo_list_t& global_group, const soinfo_list_t& local_group) {
const size_t tls_tp_base = __libc_shared_globals()->static_tls_layout.offset_thread_pointer();
+ std::vector<std::pair<TlsDescriptor*, size_t>> deferred_tlsdesc_relocs;
for (size_t idx = 0; rel_iterator.has_next(); ++idx) {
const auto rel = rel_iterator.next();
@@ -2977,6 +2978,49 @@
#endif // !defined(__aarch64__)
#if defined(__aarch64__)
+ // Bionic currently only implements TLSDESC for arm64. This implementation should work with
+ // other architectures, as long as the resolver functions are implemented.
+ case R_GENERIC_TLSDESC:
+ count_relocation(kRelocRelative);
+ MARK(rel->r_offset);
+ {
+ TlsDescriptor* desc = reinterpret_cast<TlsDescriptor*>(reloc);
+ if (lsi == nullptr) {
+ // Unresolved weak relocation.
+ desc->func = tlsdesc_resolver_unresolved_weak;
+ desc->arg = addend;
+ TRACE_TYPE(RELO, "RELO TLSDESC %16p <- unresolved weak 0x%zx %s\n",
+ reinterpret_cast<void*>(reloc), static_cast<size_t>(addend), sym_name);
+ } else {
+ CHECK(lsi->get_tls() != nullptr); // We rejected a missing TLS segment above.
+ size_t module_id = lsi->get_tls()->module_id;
+ const TlsModule& mod = get_tls_module(module_id);
+ if (mod.static_offset != SIZE_MAX) {
+ desc->func = tlsdesc_resolver_static;
+ desc->arg = mod.static_offset - tls_tp_base + sym_addr + addend;
+ TRACE_TYPE(RELO, "RELO TLSDESC %16p <- static (0x%zx - 0x%zx + 0x%zx + 0x%zx) %s\n",
+ reinterpret_cast<void*>(reloc), mod.static_offset, tls_tp_base,
+ static_cast<size_t>(sym_addr), static_cast<size_t>(addend), sym_name);
+ } else {
+ tlsdesc_args_.push_back({
+ .generation = mod.first_generation,
+ .index.module_id = module_id,
+ .index.offset = sym_addr + addend,
+ });
+ // Defer the TLSDESC relocation until the address of the TlsDynamicResolverArg object
+ // is finalized.
+ deferred_tlsdesc_relocs.push_back({ desc, tlsdesc_args_.size() - 1 });
+ const TlsDynamicResolverArg& desc_arg = tlsdesc_args_.back();
+ TRACE_TYPE(RELO, "RELO TLSDESC %16p <- dynamic (gen %zu, mod %zu, off %zu) %s",
+ reinterpret_cast<void*>(reloc), desc_arg.generation,
+ desc_arg.index.module_id, desc_arg.index.offset, sym_name);
+ }
+ }
+ }
+ break;
+#endif // defined(R_GENERIC_TLSDESC)
+
+#if defined(__aarch64__)
case R_AARCH64_ABS64:
count_relocation(kRelocAbsolute);
MARK(rel->r_offset);
@@ -3145,6 +3189,13 @@
return false;
}
}
+
+ for (const std::pair<TlsDescriptor*, size_t>& pair : deferred_tlsdesc_relocs) {
+ TlsDescriptor* desc = pair.first;
+ desc->func = tlsdesc_resolver_dynamic;
+ desc->arg = reinterpret_cast<size_t>(&tlsdesc_args_[pair.second]);
+ }
+
return true;
}
#endif // !defined(__mips__)
diff --git a/linker/linker_soinfo.h b/linker/linker_soinfo.h
index 3499cf7..dd9c6aa 100644
--- a/linker/linker_soinfo.h
+++ b/linker/linker_soinfo.h
@@ -32,6 +32,7 @@
#include <memory>
#include <string>
+#include <vector>
#include "private/bionic_elf_tls.h"
#include "linker_namespaces.h"
@@ -379,6 +380,7 @@
// version >= 5
std::unique_ptr<soinfo_tls> tls_;
+ std::vector<TlsDynamicResolverArg> tlsdesc_args_;
};
// This function is used by dlvsym() to calculate hash of sym_ver
diff --git a/linker/linker_tls.h b/linker/linker_tls.h
index fbb1dcf..87e1f0d 100644
--- a/linker/linker_tls.h
+++ b/linker/linker_tls.h
@@ -30,6 +30,8 @@
#include <stdlib.h>
+#include "private/bionic_elf_tls.h"
+
struct TlsModule;
struct soinfo;
@@ -40,3 +42,24 @@
void unregister_soinfo_tls(soinfo* si);
const TlsModule& get_tls_module(size_t module_id);
+
+typedef size_t TlsDescResolverFunc(size_t);
+
+struct TlsDescriptor {
+#if defined(__arm__)
+ size_t arg;
+ TlsDescResolverFunc* func;
+#else
+ TlsDescResolverFunc* func;
+ size_t arg;
+#endif
+};
+
+struct TlsDynamicResolverArg {
+ size_t generation;
+ TlsIndex index;
+};
+
+__LIBC_HIDDEN__ extern "C" size_t tlsdesc_resolver_static(size_t);
+__LIBC_HIDDEN__ extern "C" size_t tlsdesc_resolver_dynamic(size_t);
+__LIBC_HIDDEN__ extern "C" size_t tlsdesc_resolver_unresolved_weak(size_t);
