Move VDSO pointers to a shared globals struct.
Change-Id: I01cbc9cf0917dc1fac52d9205bda2c68529d12ef
diff --git a/libc/bionic/libc_init_common.cpp b/libc/bionic/libc_init_common.cpp
index 3bbb210..5b9edaf 100644
--- a/libc/bionic/libc_init_common.cpp
+++ b/libc/bionic/libc_init_common.cpp
@@ -42,10 +42,12 @@
#include <unistd.h>
#include "private/bionic_auxv.h"
+#include "private/bionic_globals.h"
#include "private/bionic_ssp.h"
#include "private/bionic_tls.h"
#include "private/KernelArgumentBlock.h"
#include "private/libc_logging.h"
+#include "private/WriteProtected.h"
#include "pthread_internal.h"
extern "C" abort_msg_t** __abort_message_ptr;
@@ -54,7 +56,7 @@
extern "C" int __set_tls(void* ptr);
extern "C" int __set_tid_address(int* tid_address);
-__LIBC_HIDDEN__ void __libc_init_vdso();
+__LIBC_HIDDEN__ WriteProtected<libc_globals> __libc_globals;
// Not public, but well-known in the BSDs.
const char* __progname;
@@ -105,6 +107,16 @@
__init_alternate_signal_stack(&main_thread);
}
+void __libc_init_globals(KernelArgumentBlock& args) {
+ // Initialize libc globals that are needed in both the linker and in libc.
+ // In dynamic binaries, this is run at least twice for different copies of the
+ // globals, once for the linker's copy and once for the one in libc.so.
+ __libc_globals.initialize();
+ __libc_globals.mutate([&args](libc_globals* globals) {
+ __libc_init_vdso(globals, args);
+ });
+}
+
void __libc_init_common(KernelArgumentBlock& args) {
// Initialize various globals.
environ = args.envp;
@@ -121,9 +133,7 @@
__pthread_internal_add(main_thread);
__system_properties_init(); // Requires 'environ'.
-
__bionic_setjmp_cookie_init();
- __libc_init_vdso();
}
__noreturn static void __early_abort(int line) {
diff --git a/libc/bionic/libc_init_common.h b/libc/bionic/libc_init_common.h
index 673ad5b..5066652 100644
--- a/libc/bionic/libc_init_common.h
+++ b/libc/bionic/libc_init_common.h
@@ -51,6 +51,9 @@
#if defined(__cplusplus)
class KernelArgumentBlock;
+
+__LIBC_HIDDEN__ void __libc_init_globals(KernelArgumentBlock& args);
+
__LIBC_HIDDEN__ void __libc_init_common(KernelArgumentBlock& args);
__LIBC_HIDDEN__ void __libc_init_AT_SECURE(KernelArgumentBlock& args);
diff --git a/libc/bionic/libc_init_dynamic.cpp b/libc/bionic/libc_init_dynamic.cpp
index 78125f9..3bb6e89 100644
--- a/libc/bionic/libc_init_dynamic.cpp
+++ b/libc/bionic/libc_init_dynamic.cpp
@@ -74,6 +74,7 @@
// __libc_init_common() will change the TLS area so the old one won't be accessible anyway.
*args_slot = NULL;
+ __libc_init_globals(*args);
__libc_init_common(*args);
// Hooks for various libraries to let them know that we're starting up.
diff --git a/libc/bionic/libc_init_static.cpp b/libc/bionic/libc_init_static.cpp
index 2b1a8cb..2fe86d0 100644
--- a/libc/bionic/libc_init_static.cpp
+++ b/libc/bionic/libc_init_static.cpp
@@ -84,7 +84,12 @@
int (*slingshot)(int, char**, char**),
structors_array_t const * const structors) {
KernelArgumentBlock args(raw_args);
+
__libc_init_main_thread(args);
+
+ // Initializing the globals requires TLS to be available for errno.
+ __libc_init_globals(args);
+
__libc_init_AT_SECURE(args);
__libc_init_common(args);
diff --git a/libc/bionic/vdso.cpp b/libc/bionic/vdso.cpp
index 9eae3d5..029c148 100644
--- a/libc/bionic/vdso.cpp
+++ b/libc/bionic/vdso.cpp
@@ -14,66 +14,47 @@
* limitations under the License.
*/
-#include <link.h>
-#include <string.h>
-#include <sys/auxv.h>
-#include <unistd.h>
+#include "private/bionic_globals.h"
+#include "private/bionic_vdso.h"
#if defined(__aarch64__) || defined(__x86_64__) || defined (__i386__)
-#if defined(__aarch64__)
-#define VDSO_CLOCK_GETTIME_SYMBOL "__kernel_clock_gettime"
-#define VDSO_GETTIMEOFDAY_SYMBOL "__kernel_gettimeofday"
-#elif defined(__x86_64__) || defined(__i386__)
-#define VDSO_CLOCK_GETTIME_SYMBOL "__vdso_clock_gettime"
-#define VDSO_GETTIMEOFDAY_SYMBOL "__vdso_gettimeofday"
-#endif
-
-#include <errno.h>
#include <limits.h>
-#include <sys/mman.h>
+#include <link.h>
+#include <string.h>
+#include <sys/cdefs.h>
+#include <sys/time.h>
#include <time.h>
-
-#include "private/bionic_prctl.h"
-#include "private/libc_logging.h"
-
-extern "C" int __clock_gettime(int, timespec*);
-extern "C" int __gettimeofday(timeval*, struct timezone*);
-
-struct vdso_entry {
- const char* name;
- void* fn;
-};
-
-enum {
- VDSO_CLOCK_GETTIME = 0,
- VDSO_GETTIMEOFDAY,
- VDSO_END
-};
-
-static union {
- vdso_entry entries[VDSO_END];
- char padding[PAGE_SIZE];
-} vdso __attribute__((aligned(PAGE_SIZE))) = {{
- [VDSO_CLOCK_GETTIME] = { VDSO_CLOCK_GETTIME_SYMBOL, reinterpret_cast<void*>(__clock_gettime) },
- [VDSO_GETTIMEOFDAY] = { VDSO_GETTIMEOFDAY_SYMBOL, reinterpret_cast<void*>(__gettimeofday) },
-}};
+#include <unistd.h>
+#include "private/KernelArgumentBlock.h"
int clock_gettime(int clock_id, timespec* tp) {
- int (*vdso_clock_gettime)(int, timespec*) =
- reinterpret_cast<int (*)(int, timespec*)>(vdso.entries[VDSO_CLOCK_GETTIME].fn);
- return vdso_clock_gettime(clock_id, tp);
+ auto vdso_clock_gettime = reinterpret_cast<decltype(&clock_gettime)>(
+ __libc_globals->vdso[VDSO_CLOCK_GETTIME].fn);
+ if (__predict_true(vdso_clock_gettime)) {
+ return vdso_clock_gettime(clock_id, tp);
+ }
+ return __clock_gettime(clock_id, tp);
}
int gettimeofday(timeval* tv, struct timezone* tz) {
- int (*vdso_gettimeofday)(timeval*, struct timezone*) =
- reinterpret_cast<int (*)(timeval*, struct timezone*)>(vdso.entries[VDSO_GETTIMEOFDAY].fn);
- return vdso_gettimeofday(tv, tz);
+ auto vdso_gettimeofday = reinterpret_cast<decltype(&gettimeofday)>(
+ __libc_globals->vdso[VDSO_GETTIMEOFDAY].fn);
+ if (__predict_true(vdso_gettimeofday)) {
+ return vdso_gettimeofday(tv, tz);
+ }
+ return __gettimeofday(tv, tz);
}
-static void __libc_init_vdso_entries() {
+void __libc_init_vdso(libc_globals* globals, KernelArgumentBlock& args) {
+ auto&& vdso = globals->vdso;
+ vdso[VDSO_CLOCK_GETTIME] = { VDSO_CLOCK_GETTIME_SYMBOL,
+ reinterpret_cast<void*>(__clock_gettime) };
+ vdso[VDSO_GETTIMEOFDAY] = { VDSO_GETTIMEOFDAY_SYMBOL,
+ reinterpret_cast<void*>(__gettimeofday) };
+
// Do we have a vdso?
- uintptr_t vdso_ehdr_addr = getauxval(AT_SYSINFO_EHDR);
+ uintptr_t vdso_ehdr_addr = args.getauxval(AT_SYSINFO_EHDR);
ElfW(Ehdr)* vdso_ehdr = reinterpret_cast<ElfW(Ehdr)*>(vdso_ehdr_addr);
if (vdso_ehdr == nullptr) {
return;
@@ -123,27 +104,16 @@
// Are there any symbols we want?
for (size_t i = 0; i < symbol_count; ++i) {
for (size_t j = 0; j < VDSO_END; ++j) {
- if (strcmp(vdso.entries[j].name, strtab + symtab[i].st_name) == 0) {
- vdso.entries[j].fn = reinterpret_cast<void*>(vdso_addr + symtab[i].st_value);
+ if (strcmp(vdso[j].name, strtab + symtab[i].st_name) == 0) {
+ vdso[j].fn = reinterpret_cast<void*>(vdso_addr + symtab[i].st_value);
}
}
}
}
-void __libc_init_vdso() {
- __libc_init_vdso_entries();
-
- // We can't use PR_SET_VMA because this isn't an anonymous region.
- // Long-term we should be able to replace all of this with ifuncs.
- static_assert(PAGE_SIZE == sizeof(vdso), "sizeof(vdso) too large");
- if (mprotect(vdso.entries, sizeof(vdso), PROT_READ) == -1) {
- __libc_fatal("failed to mprotect PROT_READ vdso function pointer table: %s", strerror(errno));
- }
-}
-
#else
-void __libc_init_vdso() {
+void __libc_init_vdso(libc_globals*, KernelArgumentBlock&) {
}
#endif
diff --git a/libc/private/WriteProtected.h b/libc/private/WriteProtected.h
new file mode 100644
index 0000000..1133e2a
--- /dev/null
+++ b/libc/private/WriteProtected.h
@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _PRIVATE_WRITEPROTECTED_H
+#define _PRIVATE_WRITEPROTECTED_H
+
+#include <errno.h>
+#include <string.h>
+#include <sys/cdefs.h>
+#include <sys/mman.h>
+#include <sys/user.h>
+
+#include "private/bionic_macros.h"
+#include "private/bionic_prctl.h"
+#include "private/libc_logging.h"
+
+template <typename T>
+union WriteProtectedContents {
+ T value;
+ char padding[PAGE_SIZE];
+
+ WriteProtectedContents() = default;
+ DISALLOW_COPY_AND_ASSIGN(WriteProtectedContents);
+} __attribute__((aligned(PAGE_SIZE)));
+
+// Write protected wrapper class that aligns its contents to a page boundary,
+// and sets the memory protection to be non-writable, except when being modified
+// explicitly.
+template <typename T>
+class WriteProtected {
+ static_assert(sizeof(T) < PAGE_SIZE,
+ "WriteProtected only supports contents up to PAGE_SIZE");
+ static_assert(__is_pod(T), "WriteProtected only supports POD contents");
+
+ WriteProtectedContents<T> contents;
+
+ public:
+ WriteProtected() = default;
+ DISALLOW_COPY_AND_ASSIGN(WriteProtected);
+
+ void initialize() {
+ // Not strictly necessary, but this will hopefully segfault if we initialize
+ // multiple times by accident.
+ memset(&contents, 0, sizeof(contents));
+
+ if (mprotect(&contents, PAGE_SIZE, PROT_READ)) {
+ __libc_fatal("failed to make WriteProtected nonwritable in initialize");
+ }
+ }
+
+ const T* operator->() {
+ return &contents.value;
+ }
+
+ const T& operator*() {
+ return contents.value;
+ }
+
+ template <typename Mutator>
+ void mutate(Mutator mutator) {
+ if (mprotect(&contents, PAGE_SIZE, PROT_READ | PROT_WRITE) != 0) {
+ __libc_fatal("failed to make WriteProtected writable in mutate: %s",
+ strerror(errno));
+ }
+ mutator(&contents.value);
+ if (mprotect(&contents, PAGE_SIZE, PROT_READ) != 0) {
+ __libc_fatal("failed to make WriteProtected nonwritable in mutate: %s",
+ strerror(errno));
+ }
+ }
+};
+
+#endif
diff --git a/libc/private/bionic_globals.h b/libc/private/bionic_globals.h
new file mode 100644
index 0000000..788a759
--- /dev/null
+++ b/libc/private/bionic_globals.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#ifndef _PRIVATE_BIONIC_GLOBALS_H
+#define _PRIVATE_BIONIC_GLOBALS_H
+
+#include <sys/cdefs.h>
+#include "private/bionic_vdso.h"
+#include "private/WriteProtected.h"
+
+struct libc_globals {
+ vdso_entry vdso[VDSO_END];
+};
+
+__LIBC_HIDDEN__ extern WriteProtected<libc_globals> __libc_globals;
+
+class KernelArgumentBlock;
+__LIBC_HIDDEN__ void __libc_init_vdso(libc_globals* globals,
+ KernelArgumentBlock& args);
+
+#endif
diff --git a/libc/private/bionic_vdso.h b/libc/private/bionic_vdso.h
new file mode 100644
index 0000000..5400de5
--- /dev/null
+++ b/libc/private/bionic_vdso.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#ifndef _PRIVATE_BIONIC_VDSO_H
+#define _PRIVATE_BIONIC_VDSO_H
+
+#include <time.h>
+
+#if defined(__aarch64__)
+#define VDSO_CLOCK_GETTIME_SYMBOL "__kernel_clock_gettime"
+#define VDSO_GETTIMEOFDAY_SYMBOL "__kernel_gettimeofday"
+#elif defined(__x86_64__) || defined(__i386__)
+#define VDSO_CLOCK_GETTIME_SYMBOL "__vdso_clock_gettime"
+#define VDSO_GETTIMEOFDAY_SYMBOL "__vdso_gettimeofday"
+#endif
+
+extern "C" int __clock_gettime(int, timespec*);
+extern "C" int __gettimeofday(timeval*, struct timezone*);
+
+struct vdso_entry {
+ const char* name;
+ void* fn;
+};
+
+enum {
+ VDSO_CLOCK_GETTIME = 0,
+ VDSO_GETTIMEOFDAY,
+ VDSO_END
+};
+
+#endif // _PRIVATE_BIONIC_VDSO_H
diff --git a/linker/linker.cpp b/linker/linker.cpp
index 5f4e35c..841b957 100644
--- a/linker/linker.cpp
+++ b/linker/linker.cpp
@@ -63,6 +63,7 @@
#include "base/strings.h"
#include "ziparchive/zip_archive.h"
+extern void __libc_init_globals(KernelArgumentBlock&);
extern void __libc_init_AT_SECURE(KernelArgumentBlock&);
// Override macros to use C++ style casts.
@@ -3579,6 +3580,9 @@
__libc_init_main_thread(args);
+ // Initialize the linker's static libc's globals
+ __libc_init_globals(args);
+
// Initialize the linker's own global variables
linker_so.call_constructors();
