Write out image bitmap inside of image file.
We now create the image bitmap when we generate the image. The image
bitmap is written after the image inside of the image file. This
speeds up dex2oat by making walking the image during heap creation
unnecessary. This should also help memory pressure by enabling the
image bitmap to be swappable.
Bug: 10432288
Change-Id: Idebf459ed15edbb41a7d9b9b353934155bce2f19
diff --git a/compiler/image_test.cc b/compiler/image_test.cc
index dcafc19..6980091 100644
--- a/compiler/image_test.cc
+++ b/compiler/image_test.cc
@@ -85,6 +85,8 @@
ImageHeader image_header;
file->ReadFully(&image_header, sizeof(image_header));
ASSERT_TRUE(image_header.IsValid());
+ ASSERT_GE(image_header.GetImageBitmapOffset(), sizeof(image_header));
+ ASSERT_NE(0U, image_header.GetImageBitmapSize());
gc::Heap* heap = Runtime::Current()->GetHeap();
ASSERT_EQ(1U, heap->GetContinuousSpaces().size());
@@ -136,6 +138,7 @@
ASSERT_TRUE(heap->GetContinuousSpaces()[1]->IsDlMallocSpace());
gc::space::ImageSpace* image_space = heap->GetImageSpace();
+ image_space->VerifyImageAllocations();
byte* image_begin = image_space->Begin();
byte* image_end = image_space->End();
CHECK_EQ(requested_image_base, reinterpret_cast<uintptr_t>(image_begin));
diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc
index 548ea9e..d1859e6 100644
--- a/compiler/image_writer.cc
+++ b/compiler/image_writer.cc
@@ -29,6 +29,7 @@
#include "elf_writer.h"
#include "gc/accounting/card_table-inl.h"
#include "gc/accounting/heap_bitmap.h"
+#include "gc/accounting/space_bitmap-inl.h"
#include "gc/heap.h"
#include "gc/space/large_object_space.h"
#include "gc/space/space-inl.h"
@@ -136,9 +137,12 @@
CalculateNewObjectOffsets(oat_loaded_size, oat_data_offset);
CopyAndFixupObjects();
PatchOatCodeAndMethods();
+ // Record allocations into the image bitmap.
+ RecordImageAllocations();
Thread::Current()->TransitionFromRunnableToSuspended(kNative);
UniquePtr<File> image_file(OS::CreateEmptyFile(image_filename.c_str()));
+ ImageHeader* image_header = reinterpret_cast<ImageHeader*>(image_->Begin());
if (image_file.get() == NULL) {
LOG(ERROR) << "Failed to open image file " << image_filename;
return false;
@@ -147,14 +151,37 @@
PLOG(ERROR) << "Failed to make image file world readable: " << image_filename;
return EXIT_FAILURE;
}
- bool success = image_file->WriteFully(image_->Begin(), image_end_);
- if (!success) {
+
+ // Write out the image.
+ CHECK_EQ(image_end_, image_header->GetImageSize());
+ if (!image_file->WriteFully(image_->Begin(), image_end_)) {
PLOG(ERROR) << "Failed to write image file " << image_filename;
return false;
}
+
+ // Write out the image bitmap at the page aligned start of the image end.
+ CHECK_ALIGNED(image_header->GetImageBitmapOffset(), kPageSize);
+ if (!image_file->Write(reinterpret_cast<char*>(image_bitmap_->Begin()),
+ image_header->GetImageBitmapSize(),
+ image_header->GetImageBitmapOffset())) {
+ PLOG(ERROR) << "Failed to write image file " << image_filename;
+ return false;
+ }
+
return true;
}
+void ImageWriter::RecordImageAllocations() {
+ uint64_t start_time = NanoTime();
+ CHECK(image_bitmap_.get() != nullptr);
+ for (const auto& it : offsets_) {
+ mirror::Object* obj = reinterpret_cast<mirror::Object*>(image_->Begin() + it.second);
+ DCHECK_ALIGNED(obj, kObjectAlignment);
+ image_bitmap_->Set(obj);
+ }
+ LOG(INFO) << "RecordImageAllocations took " << PrettyDuration(NanoTime() - start_time);
+}
+
bool ImageWriter::AllocMemory() {
size_t size = 0;
for (const auto& space : Runtime::Current()->GetHeap()->GetContinuousSpaces()) {
@@ -388,7 +415,7 @@
DCHECK(!spaces.empty());
DCHECK_EQ(0U, image_end_);
- // leave space for the header, but do not write it yet, we need to
+ // Leave space for the header, but do not write it yet, we need to
// know where image_roots is going to end up
image_end_ += RoundUp(sizeof(ImageHeader), 8); // 64-bit-alignment
@@ -406,13 +433,20 @@
self->EndAssertNoThreadSuspension(old);
}
+ // Create the image bitmap.
+ image_bitmap_.reset(gc::accounting::SpaceBitmap::Create("image bitmap", image_->Begin(),
+ image_end_));
const byte* oat_file_begin = image_begin_ + RoundUp(image_end_, kPageSize);
const byte* oat_file_end = oat_file_begin + oat_loaded_size;
oat_data_begin_ = oat_file_begin + oat_data_offset;
const byte* oat_data_end = oat_data_begin_ + oat_file_->Size();
- // return to write header at start of image with future location of image_roots
+ // Return to write header at start of image with future location of image_roots. At this point,
+ // image_end_ is the size of the image (excluding bitmaps).
ImageHeader image_header(reinterpret_cast<uint32_t>(image_begin_),
+ static_cast<uint32_t>(image_end_),
+ RoundUp(image_end_, kPageSize),
+ image_bitmap_->Size(),
reinterpret_cast<uint32_t>(GetImageAddress(image_roots.get())),
oat_file_->GetOatHeader().GetChecksum(),
reinterpret_cast<uint32_t>(oat_file_begin),
diff --git a/compiler/image_writer.h b/compiler/image_writer.h
index 6a126b8..0d85f36 100644
--- a/compiler/image_writer.h
+++ b/compiler/image_writer.h
@@ -58,7 +58,10 @@
private:
bool AllocMemory();
- // we use the lock word to store the offset of the object in the image
+ // Mark the objects defined in this space in the given live bitmap.
+ void RecordImageAllocations() SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+
+ // We use the lock word to store the offset of the object in the image.
void AssignImageOffset(mirror::Object* object)
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) {
DCHECK(object != NULL);
@@ -194,6 +197,9 @@
// Beginning target oat address for the pointers from the output image to its oat file.
const byte* oat_data_begin_;
+ // Image bitmap which lets us know where the objects inside of the image reside.
+ UniquePtr<gc::accounting::SpaceBitmap> image_bitmap_;
+
// Offset from oat_data_begin_ to the stubs.
uint32_t interpreter_to_interpreter_bridge_offset_;
uint32_t interpreter_to_compiled_code_bridge_offset_;
diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc
index 8bc7877..50f3ede 100644
--- a/oatdump/oatdump.cc
+++ b/oatdump/oatdump.cc
@@ -675,6 +675,9 @@
os << "IMAGE BEGIN: " << reinterpret_cast<void*>(image_header_.GetImageBegin()) << "\n\n";
+ os << "IMAGE BITMAP OFFSET: " << reinterpret_cast<void*>(image_header_.GetImageBitmapOffset())
+ << " SIZE: " << reinterpret_cast<void*>(image_header_.GetImageBitmapSize()) << "\n\n";
+
os << "OAT CHECKSUM: " << StringPrintf("0x%08x\n\n", image_header_.GetOatChecksum());
os << "OAT FILE BEGIN:" << reinterpret_cast<void*>(image_header_.GetOatFileBegin()) << "\n\n";
diff --git a/runtime/gc/accounting/space_bitmap.cc b/runtime/gc/accounting/space_bitmap.cc
index 702e162..63b24ff 100644
--- a/runtime/gc/accounting/space_bitmap.cc
+++ b/runtime/gc/accounting/space_bitmap.cc
@@ -45,11 +45,19 @@
}
void SpaceSetMap::Walk(SpaceBitmap::Callback* callback, void* arg) {
- for (Objects::iterator it = contained_.begin(); it != contained_.end(); ++it) {
- callback(const_cast<mirror::Object*>(*it), arg);
+ for (const mirror::Object* obj : contained_) {
+ callback(const_cast<mirror::Object*>(obj), arg);
}
}
+SpaceBitmap* SpaceBitmap::CreateFromMemMap(const std::string& name, MemMap* mem_map,
+ byte* heap_begin, size_t heap_capacity) {
+ CHECK(mem_map != nullptr);
+ word* bitmap_begin = reinterpret_cast<word*>(mem_map->Begin());
+ size_t bitmap_size = OffsetToIndex(RoundUp(heap_capacity, kAlignment * kBitsPerWord)) * kWordSize;
+ return new SpaceBitmap(name, mem_map, bitmap_begin, bitmap_size, heap_begin);
+}
+
SpaceBitmap* SpaceBitmap::Create(const std::string& name, byte* heap_begin, size_t heap_capacity) {
CHECK(heap_begin != NULL);
// Round up since heap_capacity is not necessarily a multiple of kAlignment * kBitsPerWord.
@@ -59,8 +67,7 @@
LOG(ERROR) << "Failed to allocate bitmap " << name;
return NULL;
}
- word* bitmap_begin = reinterpret_cast<word*>(mem_map->Begin());
- return new SpaceBitmap(name, mem_map.release(), bitmap_begin, bitmap_size, heap_begin);
+ return CreateFromMemMap(name, mem_map.release(), heap_begin, heap_capacity);
}
// Clean up any resources associated with the bitmap.
@@ -74,14 +81,11 @@
}
// Not sure if doing this trim is necessary, since nothing past the end of the heap capacity
// should be marked.
- // TODO: Fix this code is, it broken and causes rare heap corruption!
- // mem_map_->Trim(reinterpret_cast<byte*>(heap_begin_ + bitmap_size_));
}
void SpaceBitmap::Clear() {
if (bitmap_begin_ != NULL) {
- // This returns the memory to the system. Successive page faults
- // will return zeroed memory.
+ // This returns the memory to the system. Successive page faults will return zeroed memory.
int result = madvise(bitmap_begin_, bitmap_size_, MADV_DONTNEED);
if (result == -1) {
PLOG(FATAL) << "madvise failed";
diff --git a/runtime/gc/accounting/space_bitmap.h b/runtime/gc/accounting/space_bitmap.h
index 26ab1de..f975692 100644
--- a/runtime/gc/accounting/space_bitmap.h
+++ b/runtime/gc/accounting/space_bitmap.h
@@ -48,10 +48,16 @@
typedef void SweepCallback(size_t ptr_count, mirror::Object** ptrs, void* arg);
- // Initialize a HeapBitmap so that it points to a bitmap large enough to cover a heap at
+ // Initialize a space bitmap so that it points to a bitmap large enough to cover a heap at
// heap_begin of heap_capacity bytes, where objects are guaranteed to be kAlignment-aligned.
static SpaceBitmap* Create(const std::string& name, byte* heap_begin, size_t heap_capacity);
+ // Initialize a space bitmap using the provided mem_map as the live bits. Takes ownership of the
+ // mem map. The address range covered starts at heap_begin and is of size equal to heap_capacity.
+ // Objects are kAlignement-aligned.
+ static SpaceBitmap* CreateFromMemMap(const std::string& name, MemMap* mem_map,
+ byte* heap_begin, size_t heap_capacity);
+
~SpaceBitmap();
// <offset> is the difference from .base to a pointer address.
diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc
index e20c2c5..e0048a0 100644
--- a/runtime/gc/heap.cc
+++ b/runtime/gc/heap.cc
@@ -187,14 +187,6 @@
heap_capacity += continuous_spaces_.back()->AsDlMallocSpace()->NonGrowthLimitCapacity();
}
- // Mark image objects in the live bitmap.
- for (const auto& space : continuous_spaces_) {
- if (space->IsImageSpace()) {
- space::ImageSpace* image_space = space->AsImageSpace();
- image_space->RecordImageAllocations(image_space->GetLiveBitmap());
- }
- }
-
// Allocate the card table.
card_table_.reset(accounting::CardTable::Create(heap_begin, heap_capacity));
CHECK(card_table_.get() != NULL) << "Failed to create card table";
diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc
index f959cff..e2e5bf7 100644
--- a/runtime/gc/space/image_space.cc
+++ b/runtime/gc/space/image_space.cc
@@ -35,15 +35,13 @@
namespace gc {
namespace space {
-size_t ImageSpace::bitmap_index_ = 0;
+AtomicInteger ImageSpace::bitmap_index_(0);
-ImageSpace::ImageSpace(const std::string& name, MemMap* mem_map)
-: MemMapSpace(name, mem_map, mem_map->Size(), kGcRetentionPolicyNeverCollect) {
- const size_t bitmap_index = bitmap_index_++;
- live_bitmap_.reset(accounting::SpaceBitmap::Create(
- StringPrintf("imagespace %s live-bitmap %d", name.c_str(), static_cast<int>(bitmap_index)),
- Begin(), Capacity()));
- DCHECK(live_bitmap_.get() != NULL) << "could not create imagespace live bitmap #" << bitmap_index;
+ImageSpace::ImageSpace(const std::string& name, MemMap* mem_map,
+ accounting::SpaceBitmap* live_bitmap)
+ : MemMapSpace(name, mem_map, mem_map->Size(), kGcRetentionPolicyNeverCollect) {
+ DCHECK(live_bitmap != NULL);
+ live_bitmap_.reset(live_bitmap);
}
static bool GenerateImage(const std::string& image_file_name) {
@@ -151,6 +149,17 @@
return space::ImageSpace::Init(image_file_name, true);
}
+void ImageSpace::VerifyImageAllocations() {
+ byte* current = Begin() + RoundUp(sizeof(ImageHeader), kObjectAlignment);
+ while (current < End()) {
+ DCHECK_ALIGNED(current, kObjectAlignment);
+ const mirror::Object* obj = reinterpret_cast<const mirror::Object*>(current);
+ CHECK(live_bitmap_->Test(obj));
+ CHECK(obj->GetClass() != nullptr) << "Image object at address " << obj << " has null class";
+ current += RoundUp(obj->SizeOf(), kObjectAlignment);
+ }
+}
+
ImageSpace* ImageSpace::Init(const std::string& image_file_name, bool validate_oat_file) {
CHECK(!image_file_name.empty());
@@ -171,8 +180,10 @@
LOG(ERROR) << "Invalid image header " << image_file_name;
return NULL;
}
+
+ // Note: The image header is part of the image due to mmap page alignment required of offset.
UniquePtr<MemMap> map(MemMap::MapFileAtAddress(image_header.GetImageBegin(),
- file->GetLength(),
+ image_header.GetImageSize(),
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_FIXED,
file->Fd(),
@@ -185,6 +196,20 @@
CHECK_EQ(image_header.GetImageBegin(), map->Begin());
DCHECK_EQ(0, memcmp(&image_header, map->Begin(), sizeof(ImageHeader)));
+ UniquePtr<MemMap> image_map(MemMap::MapFileAtAddress(nullptr, image_header.GetImageBitmapSize(),
+ PROT_READ, MAP_PRIVATE,
+ file->Fd(), image_header.GetBitmapOffset(),
+ false));
+ CHECK(image_map.get() != nullptr) << "failed to map image bitmap";
+ size_t bitmap_index = bitmap_index_.fetch_add(1);
+ std::string bitmap_name(StringPrintf("imagespace %s live-bitmap %u", image_file_name.c_str(),
+ bitmap_index));
+ UniquePtr<accounting::SpaceBitmap> bitmap(
+ accounting::SpaceBitmap::CreateFromMemMap(bitmap_name, image_map.release(),
+ reinterpret_cast<byte*>(map->Begin()),
+ map->Size()));
+ CHECK(bitmap.get() != nullptr) << "could not create " << bitmap_name;
+
Runtime* runtime = Runtime::Current();
mirror::Object* resolution_method = image_header.GetImageRoot(ImageHeader::kResolutionMethod);
runtime->SetResolutionMethod(down_cast<mirror::ArtMethod*>(resolution_method));
@@ -196,7 +221,10 @@
callee_save_method = image_header.GetImageRoot(ImageHeader::kRefsAndArgsSaveMethod);
runtime->SetCalleeSaveMethod(down_cast<mirror::ArtMethod*>(callee_save_method), Runtime::kRefsAndArgs);
- UniquePtr<ImageSpace> space(new ImageSpace(image_file_name, map.release()));
+ UniquePtr<ImageSpace> space(new ImageSpace(image_file_name, map.release(), bitmap.release()));
+ if (kIsDebugBuild) {
+ space->VerifyImageAllocations();
+ }
space->oat_file_.reset(space->OpenOatFile());
if (space->oat_file_.get() == NULL) {
@@ -245,9 +273,7 @@
bool ImageSpace::ValidateOatFile() const {
CHECK(oat_file_.get() != NULL);
- std::vector<const OatFile::OatDexFile*> oat_dex_files = oat_file_->GetOatDexFiles();
- for (size_t i = 0; i < oat_dex_files.size(); i++) {
- const OatFile::OatDexFile* oat_dex_file = oat_dex_files[i];
+ for (const OatFile::OatDexFile* oat_dex_file : oat_file_->GetOatDexFiles()) {
const std::string& dex_file_location = oat_dex_file->GetDexFileLocation();
uint32_t dex_file_location_checksum;
if (!DexFile::GetChecksum(dex_file_location.c_str(), dex_file_location_checksum)) {
@@ -270,28 +296,6 @@
return *oat_file_.release();
}
-void ImageSpace::RecordImageAllocations(accounting::SpaceBitmap* live_bitmap) const {
- uint64_t start_time = 0;
- if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
- LOG(INFO) << "ImageSpace::RecordImageAllocations entering";
- start_time = NanoTime();
- }
- DCHECK(!Runtime::Current()->IsStarted());
- CHECK(live_bitmap != NULL);
- byte* current = Begin() + RoundUp(sizeof(ImageHeader), kObjectAlignment);
- byte* end = End();
- while (current < end) {
- DCHECK_ALIGNED(current, kObjectAlignment);
- const mirror::Object* obj = reinterpret_cast<const mirror::Object*>(current);
- live_bitmap->Set(obj);
- current += RoundUp(obj->SizeOf(), kObjectAlignment);
- }
- if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
- LOG(INFO) << "ImageSpace::RecordImageAllocations exiting ("
- << PrettyDuration(NanoTime() - start_time) << ")";
- }
-}
-
void ImageSpace::Dump(std::ostream& os) const {
os << GetType()
<< "begin=" << reinterpret_cast<void*>(Begin())
diff --git a/runtime/gc/space/image_space.h b/runtime/gc/space/image_space.h
index bdda9fa..381a98e 100644
--- a/runtime/gc/space/image_space.h
+++ b/runtime/gc/space/image_space.h
@@ -53,6 +53,9 @@
OatFile& ReleaseOatFile()
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+ void VerifyImageAllocations()
+ SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+
const ImageHeader& GetImageHeader() const {
return *reinterpret_cast<ImageHeader*>(Begin());
}
@@ -61,10 +64,6 @@
return GetName();
}
- // Mark the objects defined in this space in the given live bitmap
- void RecordImageAllocations(accounting::SpaceBitmap* live_bitmap) const
- SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
-
accounting::SpaceBitmap* GetLiveBitmap() const {
return live_bitmap_.get();
}
@@ -96,11 +95,11 @@
friend class Space;
- static size_t bitmap_index_;
+ static AtomicInteger bitmap_index_;
UniquePtr<accounting::SpaceBitmap> live_bitmap_;
- ImageSpace(const std::string& name, MemMap* mem_map);
+ ImageSpace(const std::string& name, MemMap* mem_map, accounting::SpaceBitmap* live_bitmap);
// The OatFile associated with the image during early startup to
// reserve space contiguous to the image. It is later released to
diff --git a/runtime/image.cc b/runtime/image.cc
index 686a117..d11594c 100644
--- a/runtime/image.cc
+++ b/runtime/image.cc
@@ -24,9 +24,12 @@
namespace art {
const byte ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' };
-const byte ImageHeader::kImageVersion[] = { '0', '0', '4', '\0' };
+const byte ImageHeader::kImageVersion[] = { '0', '0', '5', '\0' };
ImageHeader::ImageHeader(uint32_t image_begin,
+ uint32_t image_size,
+ uint32_t image_bitmap_offset,
+ uint32_t image_bitmap_size,
uint32_t image_roots,
uint32_t oat_checksum,
uint32_t oat_file_begin,
@@ -34,6 +37,9 @@
uint32_t oat_data_end,
uint32_t oat_file_end)
: image_begin_(image_begin),
+ image_size_(image_size),
+ image_bitmap_offset_(image_bitmap_offset),
+ image_bitmap_size_(image_bitmap_size),
oat_checksum_(oat_checksum),
oat_file_begin_(oat_file_begin),
oat_data_begin_(oat_data_begin),
diff --git a/runtime/image.h b/runtime/image.h
index 35e4c5ca..5119e3a 100644
--- a/runtime/image.h
+++ b/runtime/image.h
@@ -21,6 +21,7 @@
#include "globals.h"
#include "mirror/object.h"
+#include "utils.h"
namespace art {
@@ -30,6 +31,9 @@
ImageHeader() {}
ImageHeader(uint32_t image_begin,
+ uint32_t image_size_,
+ uint32_t image_bitmap_offset,
+ uint32_t image_bitmap_size,
uint32_t image_roots,
uint32_t oat_checksum,
uint32_t oat_file_begin,
@@ -56,6 +60,18 @@
return reinterpret_cast<byte*>(image_begin_);
}
+ size_t GetImageSize() const {
+ return static_cast<uint32_t>(image_size_);
+ }
+
+ size_t GetImageBitmapOffset() const {
+ return image_bitmap_offset_;
+ }
+
+ size_t GetImageBitmapSize() const {
+ return image_bitmap_size_;
+ }
+
uint32_t GetOatChecksum() const {
return oat_checksum_;
}
@@ -80,6 +96,10 @@
return reinterpret_cast<byte*>(oat_file_end_);
}
+ size_t GetBitmapOffset() const {
+ return RoundUp(image_size_, kPageSize);
+ }
+
enum ImageRoot {
kResolutionMethod,
kCalleeSaveMethod,
@@ -106,6 +126,15 @@
// Required base address for mapping the image.
uint32_t image_begin_;
+ // Image size, not page aligned.
+ uint32_t image_size_;
+
+ // Image bitmap offset in the file.
+ uint32_t image_bitmap_offset_;
+
+ // Size of the image bitmap.
+ uint32_t image_bitmap_size_;
+
// Checksum of the oat file we link to for load time sanity check.
uint32_t oat_checksum_;
