runtime/gc/space/bump_pointer_space.cc - platform/art - Git at Google

 /*
  * Copyright (C) 2013 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 #include "bump_pointer_space.h"
 #include "bump_pointer_space-inl.h"
 #include "mirror/object-inl.h"
 #include "mirror/class-inl.h"
 #include "thread_list.h"

 namespace art {
 namespace gc {
 namespace space {

 BumpPointerSpace* BumpPointerSpace::Create(const std::string& name, size_t capacity,
                                            byte* requested_begin) {
   capacity = RoundUp(capacity, kPageSize);
   std::string error_msg;
   std::unique_ptr<MemMap> mem_map(MemMap::MapAnonymous(name.c_str(), requested_begin, capacity,
                                                  PROT_READ | PROT_WRITE, true, &error_msg));
   if (mem_map.get() == nullptr) {
     LOG(ERROR) << "Failed to allocate pages for alloc space (" << name << ") of size "
         << PrettySize(capacity) << " with message " << error_msg;
     return nullptr;
   }
   return new BumpPointerSpace(name, mem_map.release());
 }

 BumpPointerSpace* BumpPointerSpace::CreateFromMemMap(const std::string& name, MemMap* mem_map) {
   return new BumpPointerSpace(name, mem_map);
 }

 BumpPointerSpace::BumpPointerSpace(const std::string& name, byte* begin, byte* limit)
     : ContinuousMemMapAllocSpace(name, nullptr, begin, begin, limit,
                                  kGcRetentionPolicyAlwaysCollect),
       growth_end_(limit),
       objects_allocated_(0), bytes_allocated_(0),
       block_lock_("Block lock"),
       main_block_size_(0),
       num_blocks_(0) {
 }

 BumpPointerSpace::BumpPointerSpace(const std::string& name, MemMap* mem_map)
     : ContinuousMemMapAllocSpace(name, mem_map, mem_map->Begin(), mem_map->Begin(), mem_map->End(),
                                  kGcRetentionPolicyAlwaysCollect),
       growth_end_(mem_map->End()),
       objects_allocated_(0), bytes_allocated_(0),
       block_lock_("Block lock"),
       main_block_size_(0),
       num_blocks_(0) {
 }

 void BumpPointerSpace::Clear() {
   // Release the pages back to the operating system.
   if (!kMadviseZeroes) {
     memset(Begin(), 0, Limit() - Begin());
   }
   CHECK_NE(madvise(Begin(), Limit() - Begin(), MADV_DONTNEED), -1) << "madvise failed";
   // Reset the end of the space back to the beginning, we move the end forward as we allocate
   // objects.
   SetEnd(Begin());
   objects_allocated_.StoreRelaxed(0);
   bytes_allocated_.StoreRelaxed(0);
   growth_end_ = Limit();
   {
     MutexLock mu(Thread::Current(), block_lock_);
     num_blocks_ = 0;
     main_block_size_ = 0;
   }
 }

 void BumpPointerSpace::Dump(std::ostream& os) const {
   os << GetName() << " "
       << reinterpret_cast<void*>(Begin()) << "-" << reinterpret_cast<void*>(End()) << " - "
       << reinterpret_cast<void*>(Limit());
 }

 mirror::Object* BumpPointerSpace::GetNextObject(mirror::Object* obj) {
   const uintptr_t position = reinterpret_cast<uintptr_t>(obj) + obj->SizeOf();
   return reinterpret_cast<mirror::Object*>(RoundUp(position, kAlignment));
 }

 void BumpPointerSpace::RevokeThreadLocalBuffers(Thread* thread) {
   MutexLock mu(Thread::Current(), block_lock_);
   RevokeThreadLocalBuffersLocked(thread);
 }

 void BumpPointerSpace::RevokeAllThreadLocalBuffers() {
   Thread* self = Thread::Current();
   MutexLock mu(self, *Locks::runtime_shutdown_lock_);
   MutexLock mu2(self, *Locks::thread_list_lock_);
   // TODO: Not do a copy of the thread list?
   std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList();
   for (Thread* thread : thread_list) {
     RevokeThreadLocalBuffers(thread);
   }
 }

 void BumpPointerSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) {
   if (kIsDebugBuild) {
     MutexLock mu(Thread::Current(), block_lock_);
     DCHECK(!thread->HasTlab());
   }
 }

 void BumpPointerSpace::AssertAllThreadLocalBuffersAreRevoked() {
   if (kIsDebugBuild) {
     Thread* self = Thread::Current();
     MutexLock mu(self, *Locks::runtime_shutdown_lock_);
     MutexLock mu2(self, *Locks::thread_list_lock_);
     // TODO: Not do a copy of the thread list?
     std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList();
     for (Thread* thread : thread_list) {
       AssertThreadLocalBuffersAreRevoked(thread);
     }
   }
 }

 void BumpPointerSpace::UpdateMainBlock() {
   DCHECK_EQ(num_blocks_, 0U);
   main_block_size_ = Size();
 }

 // Returns the start of the storage.
 byte* BumpPointerSpace::AllocBlock(size_t bytes) {
   bytes = RoundUp(bytes, kAlignment);
   if (!num_blocks_) {
     UpdateMainBlock();
   }
   byte* storage = reinterpret_cast<byte*>(
       AllocNonvirtualWithoutAccounting(bytes + sizeof(BlockHeader)));
   if (LIKELY(storage != nullptr)) {
     BlockHeader* header = reinterpret_cast<BlockHeader*>(storage);
     header->size_ = bytes;  // Write out the block header.
     storage += sizeof(BlockHeader);
     ++num_blocks_;
   }
   return storage;
 }

 void BumpPointerSpace::Walk(ObjectCallback* callback, void* arg) {
   byte* pos = Begin();
   byte* end = End();
   byte* main_end = pos;
   {
     MutexLock mu(Thread::Current(), block_lock_);
     // If we have 0 blocks then we need to update the main header since we have bump pointer style
     // allocation into an unbounded region (actually bounded by Capacity()).
     if (num_blocks_ == 0) {
       UpdateMainBlock();
     }
     main_end = Begin() + main_block_size_;
     if (num_blocks_ == 0) {
       // We don't have any other blocks, this means someone else may be allocating into the main
       // block. In this case, we don't want to try and visit the other blocks after the main block
       // since these could actually be part of the main block.
       end = main_end;
     }
   }
   // Walk all of the objects in the main block first.
   while (pos < main_end) {
     mirror::Object* obj = reinterpret_cast<mirror::Object*>(pos);
     if (obj->GetClass() == nullptr) {
       // There is a race condition where a thread has just allocated an object but not set the
       // class. We can't know the size of this object, so we don't visit it and exit the function
       // since there is guaranteed to be not other blocks.
       return;
     } else {
       callback(obj, arg);
       pos = reinterpret_cast<byte*>(GetNextObject(obj));
     }
   }
   // Walk the other blocks (currently only TLABs).
   while (pos < end) {
     BlockHeader* header = reinterpret_cast<BlockHeader*>(pos);
     size_t block_size = header->size_;
     pos += sizeof(BlockHeader);  // Skip the header so that we know where the objects
     mirror::Object* obj = reinterpret_cast<mirror::Object*>(pos);
     const mirror::Object* end = reinterpret_cast<const mirror::Object*>(pos + block_size);
     CHECK_LE(reinterpret_cast<const byte*>(end), End());
     // We don't know how many objects are allocated in the current block. When we hit a null class
     // assume its the end. TODO: Have a thread update the header when it flushes the block?
     while (obj < end && obj->GetClass() != nullptr) {
       callback(obj, arg);
       obj = GetNextObject(obj);
     }
     pos += block_size;
   }
 }

 accounting::ContinuousSpaceBitmap::SweepCallback* BumpPointerSpace::GetSweepCallback() {
   LOG(FATAL) << "Unimplemented";
   return nullptr;
 }

 uint64_t BumpPointerSpace::GetBytesAllocated() {
   // Start out pre-determined amount (blocks which are not being allocated into).
   uint64_t total = static_cast<uint64_t>(bytes_allocated_.LoadRelaxed());
   Thread* self = Thread::Current();
   MutexLock mu(self, *Locks::runtime_shutdown_lock_);
   MutexLock mu2(self, *Locks::thread_list_lock_);
   std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList();
   MutexLock mu3(Thread::Current(), block_lock_);
   // If we don't have any blocks, we don't have any thread local buffers. This check is required
   // since there can exist multiple bump pointer spaces which exist at the same time.
   if (num_blocks_ > 0) {
     for (Thread* thread : thread_list) {
       total += thread->GetThreadLocalBytesAllocated();
     }
   }
   return total;
 }

 uint64_t BumpPointerSpace::GetObjectsAllocated() {
   // Start out pre-determined amount (blocks which are not being allocated into).
   uint64_t total = static_cast<uint64_t>(objects_allocated_.LoadRelaxed());
   Thread* self = Thread::Current();
   MutexLock mu(self, *Locks::runtime_shutdown_lock_);
   MutexLock mu2(self, *Locks::thread_list_lock_);
   std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList();
   MutexLock mu3(Thread::Current(), block_lock_);
   // If we don't have any blocks, we don't have any thread local buffers. This check is required
   // since there can exist multiple bump pointer spaces which exist at the same time.
   if (num_blocks_ > 0) {
     for (Thread* thread : thread_list) {
       total += thread->GetThreadLocalObjectsAllocated();
     }
   }
   return total;
 }

 void BumpPointerSpace::RevokeThreadLocalBuffersLocked(Thread* thread) {
   objects_allocated_.FetchAndAddSequentiallyConsistent(thread->GetThreadLocalObjectsAllocated());
   bytes_allocated_.FetchAndAddSequentiallyConsistent(thread->GetThreadLocalBytesAllocated());
   thread->SetTlab(nullptr, nullptr);
 }

 bool BumpPointerSpace::AllocNewTlab(Thread* self, size_t bytes) {
   MutexLock mu(Thread::Current(), block_lock_);
   RevokeThreadLocalBuffersLocked(self);
   byte* start = AllocBlock(bytes);
   if (start == nullptr) {
     return false;
   }
   self->SetTlab(start, start + bytes);
   return true;
 }

 }  // namespace space
 }  // namespace gc
 }  // namespace art
	/*
	* Copyright (C) 2013 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#include "bump_pointer_space.h"
	#include "bump_pointer_space-inl.h"
	#include "mirror/object-inl.h"
	#include "mirror/class-inl.h"
	#include "thread_list.h"

	namespace art {
	namespace gc {
	namespace space {

	BumpPointerSpace* BumpPointerSpace::Create(const std::string& name, size_t capacity,
	byte* requested_begin) {
	capacity = RoundUp(capacity, kPageSize);
	std::string error_msg;
	std::unique_ptr<MemMap> mem_map(MemMap::MapAnonymous(name.c_str(), requested_begin, capacity,
	PROT_READ \| PROT_WRITE, true, &error_msg));
	if (mem_map.get() == nullptr) {
	LOG(ERROR) << "Failed to allocate pages for alloc space (" << name << ") of size "
	<< PrettySize(capacity) << " with message " << error_msg;
	return nullptr;
	}
	return new BumpPointerSpace(name, mem_map.release());
	}

	BumpPointerSpace* BumpPointerSpace::CreateFromMemMap(const std::string& name, MemMap* mem_map) {
	return new BumpPointerSpace(name, mem_map);
	}

	BumpPointerSpace::BumpPointerSpace(const std::string& name, byte* begin, byte* limit)
	: ContinuousMemMapAllocSpace(name, nullptr, begin, begin, limit,
	kGcRetentionPolicyAlwaysCollect),
	growth_end_(limit),
	objects_allocated_(0), bytes_allocated_(0),
	block_lock_("Block lock"),
	main_block_size_(0),
	num_blocks_(0) {
	}

	BumpPointerSpace::BumpPointerSpace(const std::string& name, MemMap* mem_map)
	: ContinuousMemMapAllocSpace(name, mem_map, mem_map->Begin(), mem_map->Begin(), mem_map->End(),
	kGcRetentionPolicyAlwaysCollect),
	growth_end_(mem_map->End()),
	objects_allocated_(0), bytes_allocated_(0),
	block_lock_("Block lock"),
	main_block_size_(0),
	num_blocks_(0) {
	}

	void BumpPointerSpace::Clear() {
	// Release the pages back to the operating system.
	if (!kMadviseZeroes) {
	memset(Begin(), 0, Limit() - Begin());
	}
	CHECK_NE(madvise(Begin(), Limit() - Begin(), MADV_DONTNEED), -1) << "madvise failed";
	// Reset the end of the space back to the beginning, we move the end forward as we allocate
	// objects.
	SetEnd(Begin());
	objects_allocated_.StoreRelaxed(0);
	bytes_allocated_.StoreRelaxed(0);
	growth_end_ = Limit();
	{
	MutexLock mu(Thread::Current(), block_lock_);
	num_blocks_ = 0;
	main_block_size_ = 0;
	}
	}

	void BumpPointerSpace::Dump(std::ostream& os) const {
	os << GetName() << " "
	<< reinterpret_cast<void>(Begin()) << "-" << reinterpret_cast<void>(End()) << " - "
	<< reinterpret_cast<void*>(Limit());
	}

	mirror::Object* BumpPointerSpace::GetNextObject(mirror::Object* obj) {
	const uintptr_t position = reinterpret_cast<uintptr_t>(obj) + obj->SizeOf();
	return reinterpret_cast<mirror::Object*>(RoundUp(position, kAlignment));
	}

	void BumpPointerSpace::RevokeThreadLocalBuffers(Thread* thread) {
	MutexLock mu(Thread::Current(), block_lock_);
	RevokeThreadLocalBuffersLocked(thread);
	}

	void BumpPointerSpace::RevokeAllThreadLocalBuffers() {
	Thread* self = Thread::Current();
	MutexLock mu(self, *Locks::runtime_shutdown_lock_);
	MutexLock mu2(self, *Locks::thread_list_lock_);
	// TODO: Not do a copy of the thread list?
	std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList();
	for (Thread* thread : thread_list) {
	RevokeThreadLocalBuffers(thread);
	}
	}

	void BumpPointerSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) {
	if (kIsDebugBuild) {
	MutexLock mu(Thread::Current(), block_lock_);
	DCHECK(!thread->HasTlab());
	}
	}

	void BumpPointerSpace::AssertAllThreadLocalBuffersAreRevoked() {
	if (kIsDebugBuild) {
	Thread* self = Thread::Current();
	MutexLock mu(self, *Locks::runtime_shutdown_lock_);
	MutexLock mu2(self, *Locks::thread_list_lock_);
	// TODO: Not do a copy of the thread list?
	std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList();
	for (Thread* thread : thread_list) {
	AssertThreadLocalBuffersAreRevoked(thread);
	}
	}
	}

	void BumpPointerSpace::UpdateMainBlock() {
	DCHECK_EQ(num_blocks_, 0U);
	main_block_size_ = Size();
	}

	// Returns the start of the storage.
	byte* BumpPointerSpace::AllocBlock(size_t bytes) {
	bytes = RoundUp(bytes, kAlignment);
	if (!num_blocks_) {
	UpdateMainBlock();
	}
	byte* storage = reinterpret_cast<byte*>(
	AllocNonvirtualWithoutAccounting(bytes + sizeof(BlockHeader)));
	if (LIKELY(storage != nullptr)) {
	BlockHeader* header = reinterpret_cast<BlockHeader*>(storage);
	header->size_ = bytes; // Write out the block header.
	storage += sizeof(BlockHeader);
	++num_blocks_;
	}
	return storage;
	}

	void BumpPointerSpace::Walk(ObjectCallback* callback, void* arg) {
	byte* pos = Begin();
	byte* end = End();
	byte* main_end = pos;
	{
	MutexLock mu(Thread::Current(), block_lock_);
	// If we have 0 blocks then we need to update the main header since we have bump pointer style
	// allocation into an unbounded region (actually bounded by Capacity()).
	if (num_blocks_ == 0) {
	UpdateMainBlock();
	}
	main_end = Begin() + main_block_size_;
	if (num_blocks_ == 0) {
	// We don't have any other blocks, this means someone else may be allocating into the main
	// block. In this case, we don't want to try and visit the other blocks after the main block
	// since these could actually be part of the main block.
	end = main_end;
	}
	}
	// Walk all of the objects in the main block first.
	while (pos < main_end) {
	mirror::Object* obj = reinterpret_cast<mirror::Object*>(pos);
	if (obj->GetClass() == nullptr) {
	// There is a race condition where a thread has just allocated an object but not set the
	// class. We can't know the size of this object, so we don't visit it and exit the function
	// since there is guaranteed to be not other blocks.
	return;
	} else {
	callback(obj, arg);
	pos = reinterpret_cast<byte*>(GetNextObject(obj));
	}
	}
	// Walk the other blocks (currently only TLABs).
	while (pos < end) {
	BlockHeader* header = reinterpret_cast<BlockHeader*>(pos);
	size_t block_size = header->size_;
	pos += sizeof(BlockHeader); // Skip the header so that we know where the objects
	mirror::Object* obj = reinterpret_cast<mirror::Object*>(pos);
	const mirror::Object* end = reinterpret_cast<const mirror::Object*>(pos + block_size);
	CHECK_LE(reinterpret_cast<const byte*>(end), End());
	// We don't know how many objects are allocated in the current block. When we hit a null class
	// assume its the end. TODO: Have a thread update the header when it flushes the block?
	while (obj < end && obj->GetClass() != nullptr) {
	callback(obj, arg);
	obj = GetNextObject(obj);
	}
	pos += block_size;
	}
	}

	accounting::ContinuousSpaceBitmap::SweepCallback* BumpPointerSpace::GetSweepCallback() {
	LOG(FATAL) << "Unimplemented";
	return nullptr;
	}

	uint64_t BumpPointerSpace::GetBytesAllocated() {
	// Start out pre-determined amount (blocks which are not being allocated into).
	uint64_t total = static_cast<uint64_t>(bytes_allocated_.LoadRelaxed());
	Thread* self = Thread::Current();
	MutexLock mu(self, *Locks::runtime_shutdown_lock_);
	MutexLock mu2(self, *Locks::thread_list_lock_);
	std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList();
	MutexLock mu3(Thread::Current(), block_lock_);
	// If we don't have any blocks, we don't have any thread local buffers. This check is required
	// since there can exist multiple bump pointer spaces which exist at the same time.
	if (num_blocks_ > 0) {
	for (Thread* thread : thread_list) {
	total += thread->GetThreadLocalBytesAllocated();
	}
	}
	return total;
	}

	uint64_t BumpPointerSpace::GetObjectsAllocated() {
	// Start out pre-determined amount (blocks which are not being allocated into).
	uint64_t total = static_cast<uint64_t>(objects_allocated_.LoadRelaxed());
	Thread* self = Thread::Current();
	MutexLock mu(self, *Locks::runtime_shutdown_lock_);
	MutexLock mu2(self, *Locks::thread_list_lock_);
	std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList();
	MutexLock mu3(Thread::Current(), block_lock_);
	// If we don't have any blocks, we don't have any thread local buffers. This check is required
	// since there can exist multiple bump pointer spaces which exist at the same time.
	if (num_blocks_ > 0) {
	for (Thread* thread : thread_list) {
	total += thread->GetThreadLocalObjectsAllocated();
	}
	}
	return total;
	}

	void BumpPointerSpace::RevokeThreadLocalBuffersLocked(Thread* thread) {
	objects_allocated_.FetchAndAddSequentiallyConsistent(thread->GetThreadLocalObjectsAllocated());
	bytes_allocated_.FetchAndAddSequentiallyConsistent(thread->GetThreadLocalBytesAllocated());
	thread->SetTlab(nullptr, nullptr);
	}

	bool BumpPointerSpace::AllocNewTlab(Thread* self, size_t bytes) {
	MutexLock mu(Thread::Current(), block_lock_);
	RevokeThreadLocalBuffersLocked(self);
	byte* start = AllocBlock(bytes);
	if (start == nullptr) {
	return false;
	}
	self->SetTlab(start, start + bytes);
	return true;
	}

	} // namespace space
	} // namespace gc
	} // namespace art