libc/bionic/malloc_heapprofd.cpp - platform/bionic - Git at Google

 /*
  * Copyright (C) 2019 The Android Open Source Project
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *  * Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  *  * Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in
  *    the documentation and/or other materials provided with the
  *    distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
  * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */

 #if defined(LIBC_STATIC)
 #error This file should not be compiled for static targets.
 #endif

 #include <dlfcn.h>
 #include <fcntl.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>

 #include <platform/bionic/malloc.h>
 #include <private/bionic_config.h>
 #include <private/bionic_malloc_dispatch.h>
 #include <sys/system_properties.h>

 #include "gwp_asan_wrappers.h"
 #include "malloc_common.h"
 #include "malloc_common_dynamic.h"
 #include "malloc_heapprofd.h"
 #include "malloc_limit.h"

 // Installing heapprofd hooks is a multi step process, as outlined below.
 //
 // The incremental hooking and a dedicated task thread are used since we cannot
 // do heavy work within a signal handler, or when blocking a malloc invocation.
 //
 // +--->+-------------+------------------+
 // | +->+kInitialState+----------------+ |  malloc functions are not intercepted in any way.
 // | |  +-------+-----+                | |
 // | |          | HandleHeapprofd      | |
 // | |          v Signal()             | |
 // | |  +-------+----------------+     | |  currently installing the ephemeral hooks.
 // | |  |kInstallingEphemeralHook|<--+ | |
 // | |  +-------+----------------+   | | |
 // | |          |                    | | |
 // | |          v                    | | |
 // | |  +-------+---------------+    | | |  ephemeral hooks are installed. on the first call to
 // | |  |kEphemeralHookInstalled|    | | |  malloc these hooks spawn a thread that installs the
 // | |  +-------+---------------+    A B C  heapprofd hooks.
 // | |          | MallocInit         | | |
 // | |          v HeapprofdHook ()   | | |
 // | |  +-------+--------------+     | | |  first call to malloc happened. the hooks are reset to
 // | +--|kRemovingEphemeralHook|     | | |  kInitialState.
 // |    +----------------------+     | | |
 // |                                 | | |
 // |                                 | | |
 // |    +---------------+            | | |  currently installing the heapprofd hook
 // |    |kInstallingHook|<-----------|-+ |
 // |    +-------+-------+            |   |
 // |            |                    |   |
 // |            v                    |   |
 // |    +-------+------+             |   |  heapprofd hooks are installed. these forward calls to
 // |    |kHookInstalled|-------------+   |  malloc / free / etc. to heapprofd_client.so.
 // |    +-------+------+                 |
 // |            | DispatchReset()        |
 // |            v                        |
 // |    +-------+---------+              |  currently resetting the hooks to default.
 // |----+kUninstallingHook|              |
 //      +-----------------+              |
 //                                       |
 //                                       |
 //      +------------------+             |  malloc debug / malloc hooks are active. these take
 //      |kIncompatibleHooks+<------------+  precendence over heapprofd, so heapprofd will not get
 //      +------------------+                enabled. this is a terminal state.
 //
 //
 // A) HandleHeapprofdSignal()
 // B) HeapprofdInstallHooksAtInit() / InitHeapprofd()
 // C) HeapprofdRememberHookConflict()
 enum MallocHeapprofdState : uint8_t {
   kInitialState,
   kInstallingEphemeralHook,
   kEphemeralHookInstalled,
   kRemovingEphemeralHook,
   kInstallingHook,
   kHookInstalled,
   kUninstallingHook,
   kIncompatibleHooks
 };

 enum ModifyGlobalsMode {
   kWithLock,   // all calls to MaybeModifyGlobals with kWithLock will serialise. they can fail
                // due to a concurrent call with kWithoutLock.
   kWithoutLock // calls to MaybeModifyGlobals with kWithoutLock do not serialise. they can fail
                // due to concurrent calls with kWithoutLock or kWithLock.
 };

 // Provide mutual exclusion so no two threads try to modify the globals at the same time.
 template <typename Fn>
 bool MaybeModifyGlobals(ModifyGlobalsMode mode, Fn f) {
   bool success = false;
   if (mode == kWithLock) {
     pthread_mutex_lock(&gGlobalsMutateLock);
   }
   // As we have grabbed the mutex, the following condition should always hold, except
   // if we are currently running HandleHeapprofdSignal.
   if (!atomic_exchange(&gGlobalsMutating, true)) {
     f();
     success = true;
     atomic_store(&gGlobalsMutating, false);
   } else {
     error_log("%s: heapprofd client: concurrent modification.", getprogname());
   }
   if (mode == kWithLock) {
     pthread_mutex_unlock(&gGlobalsMutateLock);
   }
   return success;
 }

 extern "C" void* MallocInitHeapprofdHook(size_t);

 static constexpr char kHeapprofdSharedLib[] = "heapprofd_client.so";
 static constexpr char kHeapprofdPrefix[] = "heapprofd";
 static constexpr char kHeapprofdPropertyEnable[] = "heapprofd.enable";

 constexpr char kHeapprofdProgramPropertyPrefix[] = "heapprofd.enable.";
 constexpr size_t kHeapprofdProgramPropertyPrefixSize = sizeof(kHeapprofdProgramPropertyPrefix) - 1;
 constexpr size_t kMaxCmdlineSize = 512;

 // The handle returned by dlopen when previously loading the heapprofd
 // hooks. nullptr if shared library has not been already been loaded.
 static _Atomic (void*) gHeapprofdHandle = nullptr;
 static _Atomic MallocHeapprofdState gHeapprofdState = kInitialState;

 static bool GetHeapprofdProgramProperty(char* data, size_t size) {
   if (size < kHeapprofdProgramPropertyPrefixSize) {
     error_log("%s: Overflow constructing heapprofd property", getprogname());
     return false;
   }
   memcpy(data, kHeapprofdProgramPropertyPrefix, kHeapprofdProgramPropertyPrefixSize);

   int fd = open("/proc/self/cmdline", O_RDONLY | O_CLOEXEC);
   if (fd == -1) {
     error_log("%s: Failed to open /proc/self/cmdline", getprogname());
     return false;
   }
   char cmdline[kMaxCmdlineSize];
   ssize_t rd = read(fd, cmdline, sizeof(cmdline) - 1);
   close(fd);
   if (rd == -1) {
     error_log("%s: Failed to read /proc/self/cmdline", getprogname());
     return false;
   }
   cmdline[rd] = '\0';
   char* first_arg = static_cast<char*>(memchr(cmdline, '\0', rd));
   if (first_arg == nullptr) {
     error_log("%s: Overflow reading cmdline", getprogname());
     return false;
   }
   // For consistency with what we do with Java app cmdlines, trim everything
   // after the @ sign of the first arg.
   char* first_at = static_cast<char*>(memchr(cmdline, '@', rd));
   if (first_at != nullptr && first_at < first_arg) {
     *first_at = '\0';
     first_arg = first_at;
   }

   char* start = static_cast<char*>(memrchr(cmdline, '/', first_arg - cmdline));
   if (start == first_arg) {
     // The first argument ended in a slash.
     error_log("%s: cmdline ends in /", getprogname());
     return false;
   } else if (start == nullptr) {
     start = cmdline;
   } else {
     // Skip the /.
     start++;
   }

   size_t name_size = static_cast<size_t>(first_arg - start);
   if (name_size >= size - kHeapprofdProgramPropertyPrefixSize) {
     error_log("%s: overflow constructing heapprofd property.", getprogname());
     return false;
   }
   // + 1 to also copy the trailing null byte.
   memcpy(data + kHeapprofdProgramPropertyPrefixSize, start, name_size + 1);
   return true;
 }

 // Runtime triggering entry-point. Two possible call sites:
 // * when receiving a profiling signal with a si_value indicating heapprofd.
 // * when a Zygote child is marking itself as profileable, and there's a
 //   matching profiling request for this process (in which case heapprofd client
 //   is loaded synchronously).
 // In both cases, the caller is responsible for verifying that the process is
 // considered profileable.

 // Previously installed default dispatch table, if it exists. This is used to
 // load heapprofd properly when GWP-ASan was already installed. If GWP-ASan was
 // already installed, heapprofd will take over the dispatch table, but will use
 // GWP-ASan as the backing dispatch. Writes to this variable is atomically
 // protected by MaybeModifyGlobals.
 // Reads are not protected, so this is atomic. We cannot fail the call in
 // MallocInitHeapprofdHook.
 static _Atomic (const MallocDispatch*) gPreviousDefaultDispatchTable = nullptr;
 static MallocDispatch gEphemeralDispatch;

 void HandleHeapprofdSignal() {
   if (atomic_load(&gHeapprofdState) == kIncompatibleHooks) {
     error_log("%s: not enabling heapprofd, malloc_debug/malloc_hooks are enabled.", getprogname());
     return;
   }

   // We cannot grab the mutex here, as this is used in a signal handler.
   MaybeModifyGlobals(kWithoutLock, [] {
     MallocHeapprofdState expected = kInitialState;
     // If hooks are already installed, we still want to install ephemeral hooks to retrigger
     // heapprofd client initialization.
     MallocHeapprofdState expected2 = kHookInstalled;
     if (atomic_compare_exchange_strong(&gHeapprofdState, &expected,
           kInstallingEphemeralHook)) {
       const MallocDispatch* default_dispatch = GetDefaultDispatchTable();

       // Below, we initialize heapprofd lazily by redirecting libc's malloc() to
       // call MallocInitHeapprofdHook, which spawns off a thread and initializes
       // heapprofd. During the short period between now and when heapprofd is
       // initialized, allocations may need to be serviced. There are three
       // possible configurations:

       if (DispatchIsGwpAsan(default_dispatch)) {
         //  1. GWP-ASan was installed. We should use GWP-ASan for everything but
         //  malloc() in the interim period before heapprofd is properly
         //  installed. After heapprofd is finished installing, we will use
         //  GWP-ASan as heapprofd's backing allocator to allow heapprofd and
         //  GWP-ASan to coexist.
         atomic_store(&gPreviousDefaultDispatchTable, default_dispatch);
         gEphemeralDispatch = *default_dispatch;
       } else {
         // Either,
         // 2. No malloc hooking has been done (heapprofd, GWP-ASan, etc.). In
         // this case, everything but malloc() should come from the system
         // allocator.
         //
         // or,
         //
         // 3. It may be possible at this point in time that heapprofd is
         // *already* the default dispatch, and when it was initialized there
         // was no default dispatch installed. As such we don't want to use
         // heapprofd as the backing store for itself (otherwise infinite
         // recursion occurs). We will use the system allocator functions. Note:
         // We've checked that no other malloc interceptors are being used by
         // validating `gHeapprofdIncompatibleHooks` above, so we don't need to
         // worry about that case here.
         atomic_store(&gPreviousDefaultDispatchTable, nullptr);
         gEphemeralDispatch = *NativeAllocatorDispatch();
       }
     } else if (expected == kEphemeralHookInstalled) {
       // Nothing to do here. The ephemeral hook was installed, but
       // MallocInitHeapprofdHook() was never called. Since the ephemeral hook
       // is already there, no need to reinstall it.
       return;
     } else if (atomic_compare_exchange_strong(&gHeapprofdState, &expected2,
                                               kInstallingEphemeralHook)) {
       // if we still have hook installed, we can reuse the previous
       // decision. THIS IS REQUIRED FOR CORRECTNESS, because otherwise the
       // following can happen
       // 1. Assume DispatchIsGwpAsan(default_dispatch)
       // 2. This function is ran, sets gPreviousDefaultDispatchTable to
       //    GWP ASan.
       // 3. The sessions ends, DispatchReset FAILS due to a race. Now
       //    heapprofd hooks are default dispatch.
       // 4. We re-enter this function later. If we did NOT look at the
       //    previously recorded gPreviousDefaultDispatchTable, we would
       //    incorrectly reach case 3. below.
       // 5. The session ends, DispatchReset now resets the hooks to the
       //    system allocator. This is incorrect.
       const MallocDispatch* prev_dispatch =
         atomic_load(&gPreviousDefaultDispatchTable);
       gEphemeralDispatch = prev_dispatch ? *prev_dispatch : *NativeAllocatorDispatch();
     } else {
       error_log("%s: heapprofd: failed to transition kInitialState -> kInstallingEphemeralHook. "
           "current state (possible race): %d", getprogname(), expected2);
       return;
     }
     // Now, replace the malloc function so that the next call to malloc() will
     // initialize heapprofd.
     gEphemeralDispatch.malloc = MallocInitHeapprofdHook;

     // And finally, install these new malloc-family interceptors.
     __libc_globals.mutate([](libc_globals* globals) {
       atomic_store(&globals->default_dispatch_table, &gEphemeralDispatch);
       if (!MallocLimitInstalled()) {
         atomic_store(&globals->current_dispatch_table, &gEphemeralDispatch);
       }
     });
     atomic_store(&gHeapprofdState, kEphemeralHookInstalled);
   });
   // Otherwise, we're racing against malloc_limit's enable logic (at most once
   // per process, and a niche feature). This is highly unlikely, so simply give
   // up if it does happen.
 }

 bool HeapprofdShouldLoad() {
   // First check for heapprofd.enable. If it is set to "all", enable
   // heapprofd for all processes. Otherwise, check heapprofd.enable.${prog},
   // if it is set and not 0, enable heap profiling for this process.
   char property_value[PROP_VALUE_MAX];
   if (__system_property_get(kHeapprofdPropertyEnable, property_value) == 0) {
     return false;
   }
   if (strcmp(property_value, "all") == 0) {
     return true;
   }

   char program_property[kHeapprofdProgramPropertyPrefixSize + kMaxCmdlineSize];
   if (!GetHeapprofdProgramProperty(program_property,
                                    sizeof(program_property))) {
     return false;
   }
   if (__system_property_get(program_property, property_value) == 0) {
     return false;
   }
   return property_value[0] != '\0';
 }

 void HeapprofdRememberHookConflict() {
   atomic_store(&gHeapprofdState, kIncompatibleHooks);
 }

 static void CommonInstallHooks(libc_globals* globals) {
   void* impl_handle = atomic_load(&gHeapprofdHandle);
   if (impl_handle == nullptr) {
     impl_handle = LoadSharedLibrary(kHeapprofdSharedLib, kHeapprofdPrefix, &globals->malloc_dispatch_table);
     if (impl_handle == nullptr) {
       return;
     }
     atomic_store(&gHeapprofdHandle, impl_handle);
   } else if (!InitSharedLibrary(impl_handle, kHeapprofdSharedLib, kHeapprofdPrefix, &globals->malloc_dispatch_table)) {
     return;
   }

   FinishInstallHooks(globals, nullptr, kHeapprofdPrefix);
 }

 void HeapprofdInstallHooksAtInit(libc_globals *globals) {
   // Before we set the new default_dispatch_table in FinishInstallHooks, save
   // the previous dispatch table. If DispatchReset() gets called later, we want
   // to be able to restore the dispatch. We're still under
   // MaybeModifyGlobals locks at this point.
   atomic_store(&gPreviousDefaultDispatchTable, GetDefaultDispatchTable());
   MaybeModifyGlobals(kWithoutLock, [globals] {
     MallocHeapprofdState expected = kInitialState;
     if (atomic_compare_exchange_strong(&gHeapprofdState, &expected, kInstallingHook)) {
       CommonInstallHooks(globals);
       atomic_store(&gHeapprofdState, kHookInstalled);
     } else {
       error_log("%s: heapprofd: failed to transition kInitialState -> kInstallingHook. "
           "current state (possible race): %d", getprogname(), expected);
     }
   });
 }

 static void* InitHeapprofd(void*) {
   MaybeModifyGlobals(kWithLock, [] {
     MallocHeapprofdState expected = kInitialState;
     if (atomic_compare_exchange_strong(&gHeapprofdState, &expected, kInstallingHook)) {
       __libc_globals.mutate([](libc_globals* globals) {
         CommonInstallHooks(globals);
       });
       atomic_store(&gHeapprofdState, kHookInstalled);
     } else {
       error_log("%s: heapprofd: failed to transition kInitialState -> kInstallingHook. "
           "current state (possible race): %d", getprogname(), expected);
     }
   });
   return nullptr;
 }

 extern "C" void* MallocInitHeapprofdHook(size_t bytes) {
   MaybeModifyGlobals(kWithLock, [] {
     MallocHeapprofdState expected = kEphemeralHookInstalled;
     if (atomic_compare_exchange_strong(&gHeapprofdState, &expected, kRemovingEphemeralHook)) {
       __libc_globals.mutate([](libc_globals* globals) {
         const MallocDispatch* previous_dispatch = atomic_load(&gPreviousDefaultDispatchTable);
         atomic_store(&globals->default_dispatch_table, previous_dispatch);
         if (!MallocLimitInstalled()) {
           atomic_store(&globals->current_dispatch_table, previous_dispatch);
         }
       });
       atomic_store(&gHeapprofdState, kInitialState);

       pthread_t thread_id;
       if (pthread_create(&thread_id, nullptr, InitHeapprofd, nullptr) != 0) {
         error_log("%s: heapprofd: failed to pthread_create.", getprogname());
       } else if (pthread_setname_np(thread_id, "heapprofdinit") != 0) {
         error_log("%s: heapprod: failed to pthread_setname_np", getprogname());
       } else if (pthread_detach(thread_id) != 0) {
         error_log("%s: heapprofd: failed to pthread_detach", getprogname());
       }
     } else {
       warning_log("%s: heapprofd: could not transition kEphemeralHookInstalled -> "
           "kRemovingEphemeralHook. current state (possible race): %d. this can be benign "
           "if two threads try this transition at the same time", getprogname(),
           expected);
     }
   });
   // If we had a previous dispatch table, use that to service the allocation,
   // otherwise fall back to the native allocator.
   // This could be modified by a concurrent HandleHeapprofdSignal, but that is
   // benign as we will dispatch to the ephemeral handler, which will then dispatch
   // to the underlying one.
   const MallocDispatch* previous_dispatch = atomic_load(&gPreviousDefaultDispatchTable);
   if (previous_dispatch) {
     return previous_dispatch->malloc(bytes);
   }
   return NativeAllocatorDispatch()->malloc(bytes);
 }

 bool HeapprofdInitZygoteChildProfiling() {
   // Conditionally start "from startup" profiling.
   if (HeapprofdShouldLoad()) {
     // Directly call the signal handler codepath (properly protects against
     // concurrent invocations).
     HandleHeapprofdSignal();
   }
   return true;
 }

 static bool DispatchReset() {
   if (atomic_load(&gHeapprofdState) == kInitialState) {
     return true;
   }

   bool success = false;
   MaybeModifyGlobals(kWithLock, [&success] {
     MallocHeapprofdState expected = kHookInstalled;

     if(atomic_compare_exchange_strong(&gHeapprofdState, &expected, kUninstallingHook)){
       __libc_globals.mutate([](libc_globals* globals) {
         const MallocDispatch* previous_dispatch = atomic_load(&gPreviousDefaultDispatchTable);
         atomic_store(&globals->default_dispatch_table, previous_dispatch);
         if (!MallocLimitInstalled()) {
           atomic_store(&globals->current_dispatch_table, previous_dispatch);
         }
       });
       atomic_store(&gHeapprofdState, kInitialState);
       success = true;
     } else {
       error_log("%s: heapprofd: failed to transition kHookInstalled -> kUninstallingHook. "
           "current state (possible race): %d", getprogname(),
           expected);
     }
   });
   if (!success) {
     errno = EAGAIN;
   }
   return success;
 }

 bool HeapprofdMallopt(int opcode, void* arg, size_t arg_size) {
   if (opcode == M_RESET_HOOKS) {
     if (arg != nullptr || arg_size != 0) {
       errno = EINVAL;
       return false;
     }
     return DispatchReset();
   }
   errno = ENOTSUP;
   return false;
 }
	/*
	* Copyright (C) 2019 The Android Open Source Project
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* * Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* * Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
	* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
	* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
	* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#if defined(LIBC_STATIC)
	#error This file should not be compiled for static targets.
	#endif

	#include <dlfcn.h>
	#include <fcntl.h>
	#include <signal.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <unistd.h>

	#include <platform/bionic/malloc.h>
	#include <private/bionic_config.h>
	#include <private/bionic_malloc_dispatch.h>
	#include <sys/system_properties.h>

	#include "gwp_asan_wrappers.h"
	#include "malloc_common.h"
	#include "malloc_common_dynamic.h"
	#include "malloc_heapprofd.h"
	#include "malloc_limit.h"

	// Installing heapprofd hooks is a multi step process, as outlined below.
	//
	// The incremental hooking and a dedicated task thread are used since we cannot
	// do heavy work within a signal handler, or when blocking a malloc invocation.
	//
	// +--->+-------------+------------------+
	// \| +->+kInitialState+----------------+ \| malloc functions are not intercepted in any way.
	// \| \| +-------+-----+ \| \|
	// \| \| \| HandleHeapprofd \| \|
	// \| \| v Signal() \| \|
	// \| \| +-------+----------------+ \| \| currently installing the ephemeral hooks.
	// \| \| \|kInstallingEphemeralHook\|<--+ \| \|
	// \| \| +-------+----------------+ \| \| \|
	// \| \| \| \| \| \|
	// \| \| v \| \| \|
	// \| \| +-------+---------------+ \| \| \| ephemeral hooks are installed. on the first call to
	// \| \| \|kEphemeralHookInstalled\| \| \| \| malloc these hooks spawn a thread that installs the
	// \| \| +-------+---------------+ A B C heapprofd hooks.
	// \| \| \| MallocInit \| \| \|
	// \| \| v HeapprofdHook () \| \| \|
	// \| \| +-------+--------------+ \| \| \| first call to malloc happened. the hooks are reset to
	// \| +--\|kRemovingEphemeralHook\| \| \| \| kInitialState.
	// \| +----------------------+ \| \| \|
	// \| \| \| \|
	// \| \| \| \|
	// \| +---------------+ \| \| \| currently installing the heapprofd hook
	// \| \|kInstallingHook\|<-----------\|-+ \|
	// \| +-------+-------+ \| \|
	// \| \| \| \|
	// \| v \| \|
	// \| +-------+------+ \| \| heapprofd hooks are installed. these forward calls to
	// \| \|kHookInstalled\|-------------+ \| malloc / free / etc. to heapprofd_client.so.
	// \| +-------+------+ \|
	// \| \| DispatchReset() \|
	// \| v \|
	// \| +-------+---------+ \| currently resetting the hooks to default.
	// \|----+kUninstallingHook\| \|
	// +-----------------+ \|
	// \|
	// \|
	// +------------------+ \| malloc debug / malloc hooks are active. these take
	// \|kIncompatibleHooks+<------------+ precendence over heapprofd, so heapprofd will not get
	// +------------------+ enabled. this is a terminal state.
	//
	//
	// A) HandleHeapprofdSignal()
	// B) HeapprofdInstallHooksAtInit() / InitHeapprofd()
	// C) HeapprofdRememberHookConflict()
	enum MallocHeapprofdState : uint8_t {
	kInitialState,
	kInstallingEphemeralHook,
	kEphemeralHookInstalled,
	kRemovingEphemeralHook,
	kInstallingHook,
	kHookInstalled,
	kUninstallingHook,
	kIncompatibleHooks
	};

	enum ModifyGlobalsMode {
	kWithLock, // all calls to MaybeModifyGlobals with kWithLock will serialise. they can fail
	// due to a concurrent call with kWithoutLock.
	kWithoutLock // calls to MaybeModifyGlobals with kWithoutLock do not serialise. they can fail
	// due to concurrent calls with kWithoutLock or kWithLock.
	};

	// Provide mutual exclusion so no two threads try to modify the globals at the same time.
	template <typename Fn>
	bool MaybeModifyGlobals(ModifyGlobalsMode mode, Fn f) {
	bool success = false;
	if (mode == kWithLock) {
	pthread_mutex_lock(&gGlobalsMutateLock);
	}
	// As we have grabbed the mutex, the following condition should always hold, except
	// if we are currently running HandleHeapprofdSignal.
	if (!atomic_exchange(&gGlobalsMutating, true)) {
	f();
	success = true;
	atomic_store(&gGlobalsMutating, false);
	} else {
	error_log("%s: heapprofd client: concurrent modification.", getprogname());
	}
	if (mode == kWithLock) {
	pthread_mutex_unlock(&gGlobalsMutateLock);
	}
	return success;
	}

	extern "C" void* MallocInitHeapprofdHook(size_t);

	static constexpr char kHeapprofdSharedLib[] = "heapprofd_client.so";
	static constexpr char kHeapprofdPrefix[] = "heapprofd";
	static constexpr char kHeapprofdPropertyEnable[] = "heapprofd.enable";

	constexpr char kHeapprofdProgramPropertyPrefix[] = "heapprofd.enable.";
	constexpr size_t kHeapprofdProgramPropertyPrefixSize = sizeof(kHeapprofdProgramPropertyPrefix) - 1;
	constexpr size_t kMaxCmdlineSize = 512;

	// The handle returned by dlopen when previously loading the heapprofd
	// hooks. nullptr if shared library has not been already been loaded.
	static _Atomic (void*) gHeapprofdHandle = nullptr;
	static _Atomic MallocHeapprofdState gHeapprofdState = kInitialState;

	static bool GetHeapprofdProgramProperty(char* data, size_t size) {
	if (size < kHeapprofdProgramPropertyPrefixSize) {
	error_log("%s: Overflow constructing heapprofd property", getprogname());
	return false;
	}
	memcpy(data, kHeapprofdProgramPropertyPrefix, kHeapprofdProgramPropertyPrefixSize);

	int fd = open("/proc/self/cmdline", O_RDONLY \| O_CLOEXEC);
	if (fd == -1) {
	error_log("%s: Failed to open /proc/self/cmdline", getprogname());
	return false;
	}
	char cmdline[kMaxCmdlineSize];
	ssize_t rd = read(fd, cmdline, sizeof(cmdline) - 1);
	close(fd);
	if (rd == -1) {
	error_log("%s: Failed to read /proc/self/cmdline", getprogname());
	return false;
	}
	cmdline[rd] = '\0';
	char* first_arg = static_cast<char*>(memchr(cmdline, '\0', rd));
	if (first_arg == nullptr) {
	error_log("%s: Overflow reading cmdline", getprogname());
	return false;
	}
	// For consistency with what we do with Java app cmdlines, trim everything
	// after the @ sign of the first arg.
	char* first_at = static_cast<char*>(memchr(cmdline, '@', rd));
	if (first_at != nullptr && first_at < first_arg) {
	*first_at = '\0';
	first_arg = first_at;
	}

	char* start = static_cast<char*>(memrchr(cmdline, '/', first_arg - cmdline));
	if (start == first_arg) {
	// The first argument ended in a slash.
	error_log("%s: cmdline ends in /", getprogname());
	return false;
	} else if (start == nullptr) {
	start = cmdline;
	} else {
	// Skip the /.
	start++;
	}

	size_t name_size = static_cast<size_t>(first_arg - start);
	if (name_size >= size - kHeapprofdProgramPropertyPrefixSize) {
	error_log("%s: overflow constructing heapprofd property.", getprogname());
	return false;
	}
	// + 1 to also copy the trailing null byte.
	memcpy(data + kHeapprofdProgramPropertyPrefixSize, start, name_size + 1);
	return true;
	}

	// Runtime triggering entry-point. Two possible call sites:
	// * when receiving a profiling signal with a si_value indicating heapprofd.
	// * when a Zygote child is marking itself as profileable, and there's a
	// matching profiling request for this process (in which case heapprofd client
	// is loaded synchronously).
	// In both cases, the caller is responsible for verifying that the process is
	// considered profileable.

	// Previously installed default dispatch table, if it exists. This is used to
	// load heapprofd properly when GWP-ASan was already installed. If GWP-ASan was
	// already installed, heapprofd will take over the dispatch table, but will use
	// GWP-ASan as the backing dispatch. Writes to this variable is atomically
	// protected by MaybeModifyGlobals.
	// Reads are not protected, so this is atomic. We cannot fail the call in
	// MallocInitHeapprofdHook.
	static _Atomic (const MallocDispatch*) gPreviousDefaultDispatchTable = nullptr;
	static MallocDispatch gEphemeralDispatch;

	void HandleHeapprofdSignal() {
	if (atomic_load(&gHeapprofdState) == kIncompatibleHooks) {
	error_log("%s: not enabling heapprofd, malloc_debug/malloc_hooks are enabled.", getprogname());
	return;
	}

	// We cannot grab the mutex here, as this is used in a signal handler.
	MaybeModifyGlobals(kWithoutLock, [] {
	MallocHeapprofdState expected = kInitialState;
	// If hooks are already installed, we still want to install ephemeral hooks to retrigger
	// heapprofd client initialization.
	MallocHeapprofdState expected2 = kHookInstalled;
	if (atomic_compare_exchange_strong(&gHeapprofdState, &expected,
	kInstallingEphemeralHook)) {
	const MallocDispatch* default_dispatch = GetDefaultDispatchTable();

	// Below, we initialize heapprofd lazily by redirecting libc's malloc() to
	// call MallocInitHeapprofdHook, which spawns off a thread and initializes
	// heapprofd. During the short period between now and when heapprofd is
	// initialized, allocations may need to be serviced. There are three
	// possible configurations:

	if (DispatchIsGwpAsan(default_dispatch)) {
	// 1. GWP-ASan was installed. We should use GWP-ASan for everything but
	// malloc() in the interim period before heapprofd is properly
	// installed. After heapprofd is finished installing, we will use
	// GWP-ASan as heapprofd's backing allocator to allow heapprofd and
	// GWP-ASan to coexist.
	atomic_store(&gPreviousDefaultDispatchTable, default_dispatch);
	gEphemeralDispatch = *default_dispatch;
	} else {
	// Either,
	// 2. No malloc hooking has been done (heapprofd, GWP-ASan, etc.). In
	// this case, everything but malloc() should come from the system
	// allocator.
	//
	// or,
	//
	// 3. It may be possible at this point in time that heapprofd is
	// already the default dispatch, and when it was initialized there
	// was no default dispatch installed. As such we don't want to use
	// heapprofd as the backing store for itself (otherwise infinite
	// recursion occurs). We will use the system allocator functions. Note:
	// We've checked that no other malloc interceptors are being used by
	// validating `gHeapprofdIncompatibleHooks` above, so we don't need to
	// worry about that case here.
	atomic_store(&gPreviousDefaultDispatchTable, nullptr);
	gEphemeralDispatch = *NativeAllocatorDispatch();
	}
	} else if (expected == kEphemeralHookInstalled) {
	// Nothing to do here. The ephemeral hook was installed, but
	// MallocInitHeapprofdHook() was never called. Since the ephemeral hook
	// is already there, no need to reinstall it.
	return;
	} else if (atomic_compare_exchange_strong(&gHeapprofdState, &expected2,
	kInstallingEphemeralHook)) {
	// if we still have hook installed, we can reuse the previous
	// decision. THIS IS REQUIRED FOR CORRECTNESS, because otherwise the
	// following can happen
	// 1. Assume DispatchIsGwpAsan(default_dispatch)
	// 2. This function is ran, sets gPreviousDefaultDispatchTable to
	// GWP ASan.
	// 3. The sessions ends, DispatchReset FAILS due to a race. Now
	// heapprofd hooks are default dispatch.
	// 4. We re-enter this function later. If we did NOT look at the
	// previously recorded gPreviousDefaultDispatchTable, we would
	// incorrectly reach case 3. below.
	// 5. The session ends, DispatchReset now resets the hooks to the
	// system allocator. This is incorrect.
	const MallocDispatch* prev_dispatch =
	atomic_load(&gPreviousDefaultDispatchTable);
	gEphemeralDispatch = prev_dispatch ? prev_dispatch : NativeAllocatorDispatch();
	} else {
	error_log("%s: heapprofd: failed to transition kInitialState -> kInstallingEphemeralHook. "
	"current state (possible race): %d", getprogname(), expected2);
	return;
	}
	// Now, replace the malloc function so that the next call to malloc() will
	// initialize heapprofd.
	gEphemeralDispatch.malloc = MallocInitHeapprofdHook;

	// And finally, install these new malloc-family interceptors.
	__libc_globals.mutate([](libc_globals* globals) {
	atomic_store(&globals->default_dispatch_table, &gEphemeralDispatch);
	if (!MallocLimitInstalled()) {
	atomic_store(&globals->current_dispatch_table, &gEphemeralDispatch);
	}
	});
	atomic_store(&gHeapprofdState, kEphemeralHookInstalled);
	});
	// Otherwise, we're racing against malloc_limit's enable logic (at most once
	// per process, and a niche feature). This is highly unlikely, so simply give
	// up if it does happen.
	}

	bool HeapprofdShouldLoad() {
	// First check for heapprofd.enable. If it is set to "all", enable
	// heapprofd for all processes. Otherwise, check heapprofd.enable.${prog},
	// if it is set and not 0, enable heap profiling for this process.
	char property_value[PROP_VALUE_MAX];
	if (__system_property_get(kHeapprofdPropertyEnable, property_value) == 0) {
	return false;
	}
	if (strcmp(property_value, "all") == 0) {
	return true;
	}

	char program_property[kHeapprofdProgramPropertyPrefixSize + kMaxCmdlineSize];
	if (!GetHeapprofdProgramProperty(program_property,
	sizeof(program_property))) {
	return false;
	}
	if (__system_property_get(program_property, property_value) == 0) {
	return false;
	}
	return property_value[0] != '\0';
	}

	void HeapprofdRememberHookConflict() {
	atomic_store(&gHeapprofdState, kIncompatibleHooks);
	}

	static void CommonInstallHooks(libc_globals* globals) {
	void* impl_handle = atomic_load(&gHeapprofdHandle);
	if (impl_handle == nullptr) {
	impl_handle = LoadSharedLibrary(kHeapprofdSharedLib, kHeapprofdPrefix, &globals->malloc_dispatch_table);
	if (impl_handle == nullptr) {
	return;
	}
	atomic_store(&gHeapprofdHandle, impl_handle);
	} else if (!InitSharedLibrary(impl_handle, kHeapprofdSharedLib, kHeapprofdPrefix, &globals->malloc_dispatch_table)) {
	return;
	}

	FinishInstallHooks(globals, nullptr, kHeapprofdPrefix);
	}

	void HeapprofdInstallHooksAtInit(libc_globals *globals) {
	// Before we set the new default_dispatch_table in FinishInstallHooks, save
	// the previous dispatch table. If DispatchReset() gets called later, we want
	// to be able to restore the dispatch. We're still under
	// MaybeModifyGlobals locks at this point.
	atomic_store(&gPreviousDefaultDispatchTable, GetDefaultDispatchTable());
	MaybeModifyGlobals(kWithoutLock, [globals] {
	MallocHeapprofdState expected = kInitialState;
	if (atomic_compare_exchange_strong(&gHeapprofdState, &expected, kInstallingHook)) {
	CommonInstallHooks(globals);
	atomic_store(&gHeapprofdState, kHookInstalled);
	} else {
	error_log("%s: heapprofd: failed to transition kInitialState -> kInstallingHook. "
	"current state (possible race): %d", getprogname(), expected);
	}
	});
	}

	static void* InitHeapprofd(void*) {
	MaybeModifyGlobals(kWithLock, [] {
	MallocHeapprofdState expected = kInitialState;
	if (atomic_compare_exchange_strong(&gHeapprofdState, &expected, kInstallingHook)) {
	__libc_globals.mutate([](libc_globals* globals) {
	CommonInstallHooks(globals);
	});
	atomic_store(&gHeapprofdState, kHookInstalled);
	} else {
	error_log("%s: heapprofd: failed to transition kInitialState -> kInstallingHook. "
	"current state (possible race): %d", getprogname(), expected);
	}
	});
	return nullptr;
	}

	extern "C" void* MallocInitHeapprofdHook(size_t bytes) {
	MaybeModifyGlobals(kWithLock, [] {
	MallocHeapprofdState expected = kEphemeralHookInstalled;
	if (atomic_compare_exchange_strong(&gHeapprofdState, &expected, kRemovingEphemeralHook)) {
	__libc_globals.mutate([](libc_globals* globals) {
	const MallocDispatch* previous_dispatch = atomic_load(&gPreviousDefaultDispatchTable);
	atomic_store(&globals->default_dispatch_table, previous_dispatch);
	if (!MallocLimitInstalled()) {
	atomic_store(&globals->current_dispatch_table, previous_dispatch);
	}
	});
	atomic_store(&gHeapprofdState, kInitialState);

	pthread_t thread_id;
	if (pthread_create(&thread_id, nullptr, InitHeapprofd, nullptr) != 0) {
	error_log("%s: heapprofd: failed to pthread_create.", getprogname());
	} else if (pthread_setname_np(thread_id, "heapprofdinit") != 0) {
	error_log("%s: heapprod: failed to pthread_setname_np", getprogname());
	} else if (pthread_detach(thread_id) != 0) {
	error_log("%s: heapprofd: failed to pthread_detach", getprogname());
	}
	} else {
	warning_log("%s: heapprofd: could not transition kEphemeralHookInstalled -> "
	"kRemovingEphemeralHook. current state (possible race): %d. this can be benign "
	"if two threads try this transition at the same time", getprogname(),
	expected);
	}
	});
	// If we had a previous dispatch table, use that to service the allocation,
	// otherwise fall back to the native allocator.
	// This could be modified by a concurrent HandleHeapprofdSignal, but that is
	// benign as we will dispatch to the ephemeral handler, which will then dispatch
	// to the underlying one.
	const MallocDispatch* previous_dispatch = atomic_load(&gPreviousDefaultDispatchTable);
	if (previous_dispatch) {
	return previous_dispatch->malloc(bytes);
	}
	return NativeAllocatorDispatch()->malloc(bytes);
	}

	bool HeapprofdInitZygoteChildProfiling() {
	// Conditionally start "from startup" profiling.
	if (HeapprofdShouldLoad()) {
	// Directly call the signal handler codepath (properly protects against
	// concurrent invocations).
	HandleHeapprofdSignal();
	}
	return true;
	}

	static bool DispatchReset() {
	if (atomic_load(&gHeapprofdState) == kInitialState) {
	return true;
	}

	bool success = false;
	MaybeModifyGlobals(kWithLock, [&success] {
	MallocHeapprofdState expected = kHookInstalled;

	if(atomic_compare_exchange_strong(&gHeapprofdState, &expected, kUninstallingHook)){
	__libc_globals.mutate([](libc_globals* globals) {
	const MallocDispatch* previous_dispatch = atomic_load(&gPreviousDefaultDispatchTable);
	atomic_store(&globals->default_dispatch_table, previous_dispatch);
	if (!MallocLimitInstalled()) {
	atomic_store(&globals->current_dispatch_table, previous_dispatch);
	}
	});
	atomic_store(&gHeapprofdState, kInitialState);
	success = true;
	} else {
	error_log("%s: heapprofd: failed to transition kHookInstalled -> kUninstallingHook. "
	"current state (possible race): %d", getprogname(),
	expected);
	}
	});
	if (!success) {
	errno = EAGAIN;
	}
	return success;
	}

	bool HeapprofdMallopt(int opcode, void* arg, size_t arg_size) {
	if (opcode == M_RESET_HOOKS) {
	if (arg != nullptr \|\| arg_size != 0) {
	errno = EINVAL;
	return false;
	}
	return DispatchReset();
	}
	errno = ENOTSUP;
	return false;
	}