Write dex files to oat file early.
Write dex files to oat file before we actually open and
verify them. Instead, open and verify the copies from the
oat file and use these. This way, in the most common case
of zipped dex files, we have mmapped dex files instead of
inflated dex files. That reduces the number of dirty pages
used by dex2oat.
Reading /proc/self/statm after we write the oat file for
a compilation of a certain large app on Nexus 5 AOSP build
with -j1, three attempts before and after this CL gave
before: 346061 189462 6269 26 0 140723 0
346189 189450 6269 26 0 140851 0
346061 189463 6269 26 0 140723 0
after: 346186 185808 23040 27 0 140468 0
346186 185819 23040 27 0 140468 0
346186 185822 23040 27 0 140468 0
These values are in pages (4KiB), so while the "size"
(=VmSize) is essentially unchanged, the "resident" (=VmRSS)
is over 14MiB less and the "shared" (i.e. backed by a file)
is 65.5MiB more. That is, the amount of dirty non-pageable
memory used is reduced by about 80MiB.
The oat file format has changed slightly, the class offset
table has been moved from the OatDexFile to its own section.
This actually fixes the alignment of these offsets as they
could have been unaligned previously, yet accessed as normal
with significant performance impact if the kernel has to
emulate the unaligned access (say, mips).
Change-Id: I0f4799bb1f1ca28e3533156a3494f55345c3e10a
diff --git a/compiler/elf_builder.h b/compiler/elf_builder.h
index a7461a5..46484b1 100644
--- a/compiler/elf_builder.h
+++ b/compiler/elf_builder.h
@@ -100,12 +100,6 @@
header_.sh_entsize = entsize;
}
- ~Section() OVERRIDE {
- if (started_) {
- CHECK(finished_);
- }
- }
-
// Start writing of this section.
void Start() {
CHECK(!started_);
diff --git a/compiler/image_test.cc b/compiler/image_test.cc
index 6859605..12132c0 100644
--- a/compiler/image_test.cc
+++ b/compiler/image_test.cc
@@ -95,25 +95,37 @@
t.NewTiming("WriteElf");
SafeMap<std::string, std::string> key_value_store;
- OatWriter oat_writer(class_linker->GetBootClassPath(),
- 0,
- 0,
- 0,
- compiler_driver_.get(),
- writer.get(),
- /*compiling_boot_image*/true,
- &timings,
- &key_value_store);
+ const std::vector<const DexFile*>& dex_files = class_linker->GetBootClassPath();
std::unique_ptr<ElfWriter> elf_writer = CreateElfWriterQuick(
compiler_driver_->GetInstructionSet(),
&compiler_driver_->GetCompilerOptions(),
oat_file.GetFile());
- bool success = writer->PrepareImageAddressSpace();
- ASSERT_TRUE(success);
-
elf_writer->Start();
-
+ OatWriter oat_writer(/*compiling_boot_image*/true, &timings);
OutputStream* rodata = elf_writer->StartRoData();
+ for (const DexFile* dex_file : dex_files) {
+ ArrayRef<const uint8_t> raw_dex_file(
+ reinterpret_cast<const uint8_t*>(&dex_file->GetHeader()),
+ dex_file->GetHeader().file_size_);
+ oat_writer.AddRawDexFileSource(raw_dex_file,
+ dex_file->GetLocation().c_str(),
+ dex_file->GetLocationChecksum());
+ }
+ std::unique_ptr<MemMap> opened_dex_files_map;
+ std::vector<std::unique_ptr<const DexFile>> opened_dex_files;
+ bool dex_files_ok = oat_writer.WriteAndOpenDexFiles(
+ rodata,
+ oat_file.GetFile(),
+ compiler_driver_->GetInstructionSet(),
+ compiler_driver_->GetInstructionSetFeatures(),
+ &key_value_store,
+ &opened_dex_files_map,
+ &opened_dex_files);
+ ASSERT_TRUE(dex_files_ok);
+ oat_writer.PrepareLayout(compiler_driver_.get(), writer.get(), dex_files);
+ bool image_space_ok = writer->PrepareImageAddressSpace();
+ ASSERT_TRUE(image_space_ok);
+
bool rodata_ok = oat_writer.WriteRodata(rodata);
ASSERT_TRUE(rodata_ok);
elf_writer->EndRoData(rodata);
@@ -123,12 +135,15 @@
ASSERT_TRUE(text_ok);
elf_writer->EndText(text);
+ bool header_ok = oat_writer.WriteHeader(elf_writer->GetStream(), 0u, 0u, 0u);
+ ASSERT_TRUE(header_ok);
+
elf_writer->SetBssSize(oat_writer.GetBssSize());
elf_writer->WriteDynamicSection();
elf_writer->WriteDebugInfo(oat_writer.GetMethodDebugInfo());
elf_writer->WritePatchLocations(oat_writer.GetAbsolutePatchLocations());
- success = elf_writer->End();
+ bool success = elf_writer->End();
ASSERT_TRUE(success);
}
diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc
index 9f7ffa5..1fd5bdf 100644
--- a/compiler/oat_test.cc
+++ b/compiler/oat_test.cc
@@ -38,6 +38,7 @@
#include "oat_file-inl.h"
#include "oat_writer.h"
#include "scoped_thread_state_change.h"
+#include "utils/test_dex_file_builder.h"
namespace art {
@@ -129,23 +130,74 @@
const std::vector<const DexFile*>& dex_files,
SafeMap<std::string, std::string>& key_value_store) {
TimingLogger timings("WriteElf", false, false);
- OatWriter oat_writer(dex_files,
- 42U,
- 4096U,
- 0,
- compiler_driver_.get(),
- nullptr,
- /*compiling_boot_image*/false,
- &timings,
- &key_value_store);
+ OatWriter oat_writer(/*compiling_boot_image*/false, &timings);
+ for (const DexFile* dex_file : dex_files) {
+ ArrayRef<const uint8_t> raw_dex_file(
+ reinterpret_cast<const uint8_t*>(&dex_file->GetHeader()),
+ dex_file->GetHeader().file_size_);
+ if (!oat_writer.AddRawDexFileSource(raw_dex_file,
+ dex_file->GetLocation().c_str(),
+ dex_file->GetLocationChecksum())) {
+ return false;
+ }
+ }
+ return DoWriteElf(file, oat_writer, key_value_store);
+ }
+
+ bool WriteElf(File* file,
+ const std::vector<const char*>& dex_filenames,
+ SafeMap<std::string, std::string>& key_value_store) {
+ TimingLogger timings("WriteElf", false, false);
+ OatWriter oat_writer(/*compiling_boot_image*/false, &timings);
+ for (const char* dex_filename : dex_filenames) {
+ if (!oat_writer.AddDexFileSource(dex_filename, dex_filename)) {
+ return false;
+ }
+ }
+ return DoWriteElf(file, oat_writer, key_value_store);
+ }
+
+ bool WriteElf(File* file,
+ ScopedFd&& zip_fd,
+ const char* location,
+ SafeMap<std::string, std::string>& key_value_store) {
+ TimingLogger timings("WriteElf", false, false);
+ OatWriter oat_writer(/*compiling_boot_image*/false, &timings);
+ if (!oat_writer.AddZippedDexFilesSource(std::move(zip_fd), location)) {
+ return false;
+ }
+ return DoWriteElf(file, oat_writer, key_value_store);
+ }
+
+ bool DoWriteElf(File* file,
+ OatWriter& oat_writer,
+ SafeMap<std::string, std::string>& key_value_store) {
std::unique_ptr<ElfWriter> elf_writer = CreateElfWriterQuick(
compiler_driver_->GetInstructionSet(),
&compiler_driver_->GetCompilerOptions(),
file);
-
elf_writer->Start();
-
OutputStream* rodata = elf_writer->StartRoData();
+ std::unique_ptr<MemMap> opened_dex_files_map;
+ std::vector<std::unique_ptr<const DexFile>> opened_dex_files;
+ if (!oat_writer.WriteAndOpenDexFiles(rodata,
+ file,
+ compiler_driver_->GetInstructionSet(),
+ compiler_driver_->GetInstructionSetFeatures(),
+ &key_value_store,
+ &opened_dex_files_map,
+ &opened_dex_files)) {
+ return false;
+ }
+ Runtime* runtime = Runtime::Current();
+ ClassLinker* const class_linker = runtime->GetClassLinker();
+ std::vector<const DexFile*> dex_files;
+ for (const std::unique_ptr<const DexFile>& dex_file : opened_dex_files) {
+ dex_files.push_back(dex_file.get());
+ ScopedObjectAccess soa(Thread::Current());
+ class_linker->RegisterDexFile(*dex_file, runtime->GetLinearAlloc());
+ }
+ oat_writer.PrepareLayout(compiler_driver_.get(), nullptr, dex_files);
if (!oat_writer.WriteRodata(rodata)) {
return false;
}
@@ -157,6 +209,10 @@
}
elf_writer->EndText(text);
+ if (!oat_writer.WriteHeader(elf_writer->GetStream(), 42U, 4096U, 0)) {
+ return false;
+ }
+
elf_writer->SetBssSize(oat_writer.GetBssSize());
elf_writer->WriteDynamicSection();
elf_writer->WriteDebugInfo(oat_writer.GetMethodDebugInfo());
@@ -169,6 +225,117 @@
std::unique_ptr<QuickCompilerCallbacks> callbacks_;
};
+class ZipBuilder {
+ public:
+ explicit ZipBuilder(File* zip_file) : zip_file_(zip_file) { }
+
+ bool AddFile(const char* location, const void* data, size_t size) {
+ off_t offset = lseek(zip_file_->Fd(), 0, SEEK_CUR);
+ if (offset == static_cast<off_t>(-1)) {
+ return false;
+ }
+
+ ZipFileHeader file_header;
+ file_header.crc32 = crc32(0u, reinterpret_cast<const Bytef*>(data), size);
+ file_header.compressed_size = size;
+ file_header.uncompressed_size = size;
+ file_header.filename_length = strlen(location);
+
+ if (!zip_file_->WriteFully(&file_header, sizeof(file_header)) ||
+ !zip_file_->WriteFully(location, file_header.filename_length) ||
+ !zip_file_->WriteFully(data, size)) {
+ return false;
+ }
+
+ CentralDirectoryFileHeader cdfh;
+ cdfh.crc32 = file_header.crc32;
+ cdfh.compressed_size = size;
+ cdfh.uncompressed_size = size;
+ cdfh.filename_length = file_header.filename_length;
+ cdfh.relative_offset_of_local_file_header = offset;
+ file_data_.push_back(FileData { cdfh, location });
+ return true;
+ }
+
+ bool Finish() {
+ off_t offset = lseek(zip_file_->Fd(), 0, SEEK_CUR);
+ if (offset == static_cast<off_t>(-1)) {
+ return false;
+ }
+
+ size_t central_directory_size = 0u;
+ for (const FileData& file_data : file_data_) {
+ if (!zip_file_->WriteFully(&file_data.cdfh, sizeof(file_data.cdfh)) ||
+ !zip_file_->WriteFully(file_data.location, file_data.cdfh.filename_length)) {
+ return false;
+ }
+ central_directory_size += sizeof(file_data.cdfh) + file_data.cdfh.filename_length;
+ }
+ EndOfCentralDirectoryRecord eocd_record;
+ eocd_record.number_of_central_directory_records_on_this_disk = file_data_.size();
+ eocd_record.total_number_of_central_directory_records = file_data_.size();
+ eocd_record.size_of_central_directory = central_directory_size;
+ eocd_record.offset_of_start_of_central_directory = offset;
+ return
+ zip_file_->WriteFully(&eocd_record, sizeof(eocd_record)) &&
+ zip_file_->Flush() == 0;
+ }
+
+ private:
+ struct PACKED(1) ZipFileHeader {
+ uint32_t signature = 0x04034b50;
+ uint16_t version_needed_to_extract = 10;
+ uint16_t general_purpose_bit_flag = 0;
+ uint16_t compression_method = 0; // 0 = store only.
+ uint16_t file_last_modification_time = 0u;
+ uint16_t file_last_modification_date = 0u;
+ uint32_t crc32;
+ uint32_t compressed_size;
+ uint32_t uncompressed_size;
+ uint16_t filename_length;
+ uint16_t extra_field_length = 0u; // No extra fields.
+ };
+
+ struct PACKED(1) CentralDirectoryFileHeader {
+ uint32_t signature = 0x02014b50;
+ uint16_t version_made_by = 10;
+ uint16_t version_needed_to_extract = 10;
+ uint16_t general_purpose_bit_flag = 0;
+ uint16_t compression_method = 0; // 0 = store only.
+ uint16_t file_last_modification_time = 0u;
+ uint16_t file_last_modification_date = 0u;
+ uint32_t crc32;
+ uint32_t compressed_size;
+ uint32_t uncompressed_size;
+ uint16_t filename_length;
+ uint16_t extra_field_length = 0u; // No extra fields.
+ uint16_t file_comment_length = 0u; // No file comment.
+ uint16_t disk_number_where_file_starts = 0u;
+ uint16_t internal_file_attributes = 0u;
+ uint32_t external_file_attributes = 0u;
+ uint32_t relative_offset_of_local_file_header;
+ };
+
+ struct PACKED(1) EndOfCentralDirectoryRecord {
+ uint32_t signature = 0x06054b50;
+ uint16_t number_of_this_disk = 0u;
+ uint16_t disk_where_central_directory_starts = 0u;
+ uint16_t number_of_central_directory_records_on_this_disk;
+ uint16_t total_number_of_central_directory_records;
+ uint32_t size_of_central_directory;
+ uint32_t offset_of_start_of_central_directory;
+ uint16_t comment_length = 0u; // No file comment.
+ };
+
+ struct FileData {
+ CentralDirectoryFileHeader cdfh;
+ const char* location;
+ };
+
+ File* zip_file_;
+ std::vector<FileData> file_data_;
+};
+
TEST_F(OatTest, WriteRead) {
TimingLogger timings("OatTest::WriteRead", false, false);
ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
@@ -329,4 +496,189 @@
EXPECT_LT(static_cast<size_t>(oat_file->Size()), static_cast<size_t>(tmp.GetFile()->GetLength()));
}
+TEST_F(OatTest, DexFileInput) {
+ TimingLogger timings("OatTest::DexFileInput", false, false);
+
+ std::vector<const char*> input_filenames;
+
+ ScratchFile dex_file1;
+ TestDexFileBuilder builder1;
+ builder1.AddField("Lsome.TestClass;", "int", "someField");
+ builder1.AddMethod("Lsome.TestClass;", "()I", "foo");
+ std::unique_ptr<const DexFile> dex_file1_data = builder1.Build(dex_file1.GetFilename());
+ bool success = dex_file1.GetFile()->WriteFully(&dex_file1_data->GetHeader(),
+ dex_file1_data->GetHeader().file_size_);
+ ASSERT_TRUE(success);
+ success = dex_file1.GetFile()->Flush() == 0;
+ ASSERT_TRUE(success);
+ input_filenames.push_back(dex_file1.GetFilename().c_str());
+
+ ScratchFile dex_file2;
+ TestDexFileBuilder builder2;
+ builder2.AddField("Land.AnotherTestClass;", "boolean", "someOtherField");
+ builder2.AddMethod("Land.AnotherTestClass;", "()J", "bar");
+ std::unique_ptr<const DexFile> dex_file2_data = builder2.Build(dex_file2.GetFilename());
+ success = dex_file2.GetFile()->WriteFully(&dex_file2_data->GetHeader(),
+ dex_file2_data->GetHeader().file_size_);
+ ASSERT_TRUE(success);
+ success = dex_file2.GetFile()->Flush() == 0;
+ ASSERT_TRUE(success);
+ input_filenames.push_back(dex_file2.GetFilename().c_str());
+
+ ScratchFile oat_file;
+ SafeMap<std::string, std::string> key_value_store;
+ key_value_store.Put(OatHeader::kImageLocationKey, "test.art");
+ success = WriteElf(oat_file.GetFile(), input_filenames, key_value_store);
+ ASSERT_TRUE(success);
+
+ std::string error_msg;
+ std::unique_ptr<OatFile> opened_oat_file(OatFile::Open(oat_file.GetFilename(),
+ oat_file.GetFilename(),
+ nullptr,
+ nullptr,
+ false,
+ nullptr,
+ &error_msg));
+ ASSERT_TRUE(opened_oat_file != nullptr);
+ ASSERT_EQ(2u, opened_oat_file->GetOatDexFiles().size());
+ std::unique_ptr<const DexFile> opened_dex_file1 =
+ opened_oat_file->GetOatDexFiles()[0]->OpenDexFile(&error_msg);
+ std::unique_ptr<const DexFile> opened_dex_file2 =
+ opened_oat_file->GetOatDexFiles()[1]->OpenDexFile(&error_msg);
+
+ ASSERT_EQ(dex_file1_data->GetHeader().file_size_, opened_dex_file1->GetHeader().file_size_);
+ ASSERT_EQ(0, memcmp(&dex_file1_data->GetHeader(),
+ &opened_dex_file1->GetHeader(),
+ dex_file1_data->GetHeader().file_size_));
+ ASSERT_EQ(dex_file1_data->GetLocation(), opened_dex_file1->GetLocation());
+
+ ASSERT_EQ(dex_file2_data->GetHeader().file_size_, opened_dex_file2->GetHeader().file_size_);
+ ASSERT_EQ(0, memcmp(&dex_file2_data->GetHeader(),
+ &opened_dex_file2->GetHeader(),
+ dex_file2_data->GetHeader().file_size_));
+ ASSERT_EQ(dex_file2_data->GetLocation(), opened_dex_file2->GetLocation());
+}
+
+TEST_F(OatTest, ZipFileInput) {
+ TimingLogger timings("OatTest::DexFileInput", false, false);
+
+ ScratchFile zip_file;
+ ZipBuilder zip_builder(zip_file.GetFile());
+
+ ScratchFile dex_file1;
+ TestDexFileBuilder builder1;
+ builder1.AddField("Lsome.TestClass;", "long", "someField");
+ builder1.AddMethod("Lsome.TestClass;", "()D", "foo");
+ std::unique_ptr<const DexFile> dex_file1_data = builder1.Build(dex_file1.GetFilename());
+ bool success = dex_file1.GetFile()->WriteFully(&dex_file1_data->GetHeader(),
+ dex_file1_data->GetHeader().file_size_);
+ ASSERT_TRUE(success);
+ success = dex_file1.GetFile()->Flush() == 0;
+ ASSERT_TRUE(success);
+ success = zip_builder.AddFile("classes.dex",
+ &dex_file1_data->GetHeader(),
+ dex_file1_data->GetHeader().file_size_);
+ ASSERT_TRUE(success);
+
+ ScratchFile dex_file2;
+ TestDexFileBuilder builder2;
+ builder2.AddField("Land.AnotherTestClass;", "boolean", "someOtherField");
+ builder2.AddMethod("Land.AnotherTestClass;", "()J", "bar");
+ std::unique_ptr<const DexFile> dex_file2_data = builder2.Build(dex_file2.GetFilename());
+ success = dex_file2.GetFile()->WriteFully(&dex_file2_data->GetHeader(),
+ dex_file2_data->GetHeader().file_size_);
+ ASSERT_TRUE(success);
+ success = dex_file2.GetFile()->Flush() == 0;
+ ASSERT_TRUE(success);
+ success = zip_builder.AddFile("classes2.dex",
+ &dex_file2_data->GetHeader(),
+ dex_file2_data->GetHeader().file_size_);
+ ASSERT_TRUE(success);
+
+ success = zip_builder.Finish();
+ ASSERT_TRUE(success) << strerror(errno);
+
+ SafeMap<std::string, std::string> key_value_store;
+ key_value_store.Put(OatHeader::kImageLocationKey, "test.art");
+ {
+ // Test using the AddDexFileSource() interface with the zip file.
+ std::vector<const char*> input_filenames { zip_file.GetFilename().c_str() }; // NOLINT [readability/braces] [4]
+
+ ScratchFile oat_file;
+ success = WriteElf(oat_file.GetFile(), input_filenames, key_value_store);
+ ASSERT_TRUE(success);
+
+ std::string error_msg;
+ std::unique_ptr<OatFile> opened_oat_file(OatFile::Open(oat_file.GetFilename(),
+ oat_file.GetFilename(),
+ nullptr,
+ nullptr,
+ false,
+ nullptr,
+ &error_msg));
+ ASSERT_TRUE(opened_oat_file != nullptr);
+ ASSERT_EQ(2u, opened_oat_file->GetOatDexFiles().size());
+ std::unique_ptr<const DexFile> opened_dex_file1 =
+ opened_oat_file->GetOatDexFiles()[0]->OpenDexFile(&error_msg);
+ std::unique_ptr<const DexFile> opened_dex_file2 =
+ opened_oat_file->GetOatDexFiles()[1]->OpenDexFile(&error_msg);
+
+ ASSERT_EQ(dex_file1_data->GetHeader().file_size_, opened_dex_file1->GetHeader().file_size_);
+ ASSERT_EQ(0, memcmp(&dex_file1_data->GetHeader(),
+ &opened_dex_file1->GetHeader(),
+ dex_file1_data->GetHeader().file_size_));
+ ASSERT_EQ(DexFile::GetMultiDexLocation(0, zip_file.GetFilename().c_str()),
+ opened_dex_file1->GetLocation());
+
+ ASSERT_EQ(dex_file2_data->GetHeader().file_size_, opened_dex_file2->GetHeader().file_size_);
+ ASSERT_EQ(0, memcmp(&dex_file2_data->GetHeader(),
+ &opened_dex_file2->GetHeader(),
+ dex_file2_data->GetHeader().file_size_));
+ ASSERT_EQ(DexFile::GetMultiDexLocation(1, zip_file.GetFilename().c_str()),
+ opened_dex_file2->GetLocation());
+ }
+
+ {
+ // Test using the AddZipDexFileSource() interface with the zip file handle.
+ ScopedFd zip_fd(dup(zip_file.GetFd()));
+ ASSERT_NE(-1, zip_fd.get());
+
+ ScratchFile oat_file;
+ success = WriteElf(oat_file.GetFile(),
+ std::move(zip_fd),
+ zip_file.GetFilename().c_str(),
+ key_value_store);
+ ASSERT_TRUE(success);
+
+ std::string error_msg;
+ std::unique_ptr<OatFile> opened_oat_file(OatFile::Open(oat_file.GetFilename(),
+ oat_file.GetFilename(),
+ nullptr,
+ nullptr,
+ false,
+ nullptr,
+ &error_msg));
+ ASSERT_TRUE(opened_oat_file != nullptr);
+ ASSERT_EQ(2u, opened_oat_file->GetOatDexFiles().size());
+ std::unique_ptr<const DexFile> opened_dex_file1 =
+ opened_oat_file->GetOatDexFiles()[0]->OpenDexFile(&error_msg);
+ std::unique_ptr<const DexFile> opened_dex_file2 =
+ opened_oat_file->GetOatDexFiles()[1]->OpenDexFile(&error_msg);
+
+ ASSERT_EQ(dex_file1_data->GetHeader().file_size_, opened_dex_file1->GetHeader().file_size_);
+ ASSERT_EQ(0, memcmp(&dex_file1_data->GetHeader(),
+ &opened_dex_file1->GetHeader(),
+ dex_file1_data->GetHeader().file_size_));
+ ASSERT_EQ(DexFile::GetMultiDexLocation(0, zip_file.GetFilename().c_str()),
+ opened_dex_file1->GetLocation());
+
+ ASSERT_EQ(dex_file2_data->GetHeader().file_size_, opened_dex_file2->GetHeader().file_size_);
+ ASSERT_EQ(0, memcmp(&dex_file2_data->GetHeader(),
+ &opened_dex_file2->GetHeader(),
+ dex_file2_data->GetHeader().file_size_));
+ ASSERT_EQ(DexFile::GetMultiDexLocation(1, zip_file.GetFilename().c_str()),
+ opened_dex_file2->GetLocation());
+ }
+}
+
} // namespace art
diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc
index 025e35e..c74c41f0 100644
--- a/compiler/oat_writer.cc
+++ b/compiler/oat_writer.cc
@@ -16,12 +16,14 @@
#include "oat_writer.h"
+#include <unistd.h>
#include <zlib.h>
#include "arch/arm64/instruction_set_features_arm64.h"
#include "art_method-inl.h"
#include "base/allocator.h"
#include "base/bit_vector.h"
+#include "base/file_magic.h"
#include "base/stl_util.h"
#include "base/unix_file/fd_file.h"
#include "class_linker.h"
@@ -49,9 +51,77 @@
#include "type_lookup_table.h"
#include "utils/dex_cache_arrays_layout-inl.h"
#include "verifier/method_verifier.h"
+#include "zip_archive.h"
namespace art {
+namespace { // anonymous namespace
+
+typedef DexFile::Header __attribute__((aligned(1))) UnalignedDexFileHeader;
+
+const UnalignedDexFileHeader* AsUnalignedDexFileHeader(const uint8_t* raw_data) {
+ return reinterpret_cast<const UnalignedDexFileHeader*>(raw_data);
+}
+
+} // anonymous namespace
+
+// Defines the location of the raw dex file to write.
+class OatWriter::DexFileSource {
+ public:
+ explicit DexFileSource(ZipEntry* zip_entry)
+ : type_(kZipEntry), source_(zip_entry) {
+ DCHECK(source_ != nullptr);
+ }
+
+ explicit DexFileSource(File* raw_file)
+ : type_(kRawFile), source_(raw_file) {
+ DCHECK(source_ != nullptr);
+ }
+
+ explicit DexFileSource(const uint8_t* dex_file)
+ : type_(kRawData), source_(dex_file) {
+ DCHECK(source_ != nullptr);
+ }
+
+ bool IsZipEntry() const { return type_ == kZipEntry; }
+ bool IsRawFile() const { return type_ == kRawFile; }
+ bool IsRawData() const { return type_ == kRawData; }
+
+ ZipEntry* GetZipEntry() const {
+ DCHECK(IsZipEntry());
+ DCHECK(source_ != nullptr);
+ return static_cast<ZipEntry*>(const_cast<void*>(source_));
+ }
+
+ File* GetRawFile() const {
+ DCHECK(IsRawFile());
+ DCHECK(source_ != nullptr);
+ return static_cast<File*>(const_cast<void*>(source_));
+ }
+
+ const uint8_t* GetRawData() const {
+ DCHECK(IsRawData());
+ DCHECK(source_ != nullptr);
+ return static_cast<const uint8_t*>(source_);
+ }
+
+ void Clear() {
+ type_ = kNone;
+ source_ = nullptr;
+ }
+
+ private:
+ enum Type {
+ kNone,
+ kZipEntry,
+ kRawFile,
+ kRawData,
+ };
+
+ Type type_;
+ const void* source_;
+};
+
class OatWriter::OatClass {
public:
OatClass(size_t offset,
@@ -116,11 +186,30 @@
class OatWriter::OatDexFile {
public:
- OatDexFile(size_t offset, const DexFile& dex_file);
+ OatDexFile(const char* dex_file_location,
+ DexFileSource source,
+ CreateTypeLookupTable create_type_lookup_table);
OatDexFile(OatDexFile&& src) = default;
+ const char* GetLocation() const {
+ return dex_file_location_data_;
+ }
+
+ void ReserveTypeLookupTable(OatWriter* oat_writer);
+ void ReserveClassOffsets(OatWriter* oat_writer);
+
size_t SizeOf() const;
- bool Write(OatWriter* oat_writer, OutputStream* out, const size_t file_offset) const;
+ bool Write(OatWriter* oat_writer, OutputStream* out) const;
+ bool WriteClassOffsets(OatWriter* oat_writer, OutputStream* out);
+
+ // The source of the dex file.
+ DexFileSource source_;
+
+ // Whether to create the type lookup table.
+ CreateTypeLookupTable create_type_lookup_table_;
+
+ // Dex file size. Initialized when writing the dex file.
+ size_t dex_file_size_;
// Offset of start of OatDexFile from beginning of OatHeader. It is
// used to validate file position when writing.
@@ -128,11 +217,13 @@
// Data to write.
uint32_t dex_file_location_size_;
- const uint8_t* dex_file_location_data_;
+ const char* dex_file_location_data_;
uint32_t dex_file_location_checksum_;
uint32_t dex_file_offset_;
+ uint32_t class_offsets_offset_;
uint32_t lookup_table_offset_;
- TypeLookupTable* lookup_table_; // Owned by the dex file.
+
+ // Data to write to a separate section.
dchecked_vector<uint32_t> class_offsets_;
private:
@@ -151,26 +242,20 @@
DCHECK_EQ(static_cast<off_t>(file_offset + offset_), out->Seek(0, kSeekCurrent)) \
<< "file_offset=" << file_offset << " offset_=" << offset_
-OatWriter::OatWriter(const std::vector<const DexFile*>& dex_files,
- uint32_t image_file_location_oat_checksum,
- uintptr_t image_file_location_oat_begin,
- int32_t image_patch_delta,
- const CompilerDriver* compiler,
- ImageWriter* image_writer,
- bool compiling_boot_image,
- TimingLogger* timings,
- SafeMap<std::string, std::string>* key_value_store)
- : compiler_driver_(compiler),
- image_writer_(image_writer),
+OatWriter::OatWriter(bool compiling_boot_image, TimingLogger* timings)
+ : write_state_(WriteState::kAddingDexFileSources),
+ timings_(timings),
+ raw_dex_files_(),
+ zip_archives_(),
+ zipped_dex_files_(),
+ zipped_dex_file_locations_(),
+ compiler_driver_(nullptr),
+ image_writer_(nullptr),
compiling_boot_image_(compiling_boot_image),
- dex_files_(&dex_files),
+ dex_files_(nullptr),
size_(0u),
bss_size_(0u),
oat_data_offset_(0u),
- image_file_location_oat_checksum_(image_file_location_oat_checksum),
- image_file_location_oat_begin_(image_file_location_oat_begin),
- image_patch_delta_(image_patch_delta),
- key_value_store_(key_value_store),
oat_header_(nullptr),
size_dex_file_alignment_(0),
size_executable_offset_alignment_(0),
@@ -197,55 +282,192 @@
size_oat_dex_file_location_data_(0),
size_oat_dex_file_location_checksum_(0),
size_oat_dex_file_offset_(0),
+ size_oat_dex_file_class_offsets_offset_(0),
size_oat_dex_file_lookup_table_offset_(0),
- size_oat_dex_file_class_offsets_(0),
size_oat_lookup_table_alignment_(0),
size_oat_lookup_table_(0),
+ size_oat_class_offsets_alignment_(0),
+ size_oat_class_offsets_(0),
size_oat_class_type_(0),
size_oat_class_status_(0),
size_oat_class_method_bitmaps_(0),
size_oat_class_method_offsets_(0),
method_offset_map_() {
- CHECK(key_value_store != nullptr);
- if (compiling_boot_image) {
- CHECK(image_writer != nullptr);
+}
+
+bool OatWriter::AddDexFileSource(const char* filename,
+ const char* location,
+ CreateTypeLookupTable create_type_lookup_table) {
+ DCHECK(write_state_ == WriteState::kAddingDexFileSources);
+ uint32_t magic;
+ std::string error_msg;
+ ScopedFd fd(OpenAndReadMagic(filename, &magic, &error_msg));
+ if (fd.get() == -1) {
+ PLOG(ERROR) << "Failed to read magic number from dex file: '" << filename << "'";
+ return false;
+ } else if (IsDexMagic(magic)) {
+ // The file is open for reading, not writing, so it's OK to let the File destructor
+ // close it without checking for explicit Close(), so pass checkUsage = false.
+ raw_dex_files_.emplace_back(new File(fd.release(), location, /* checkUsage */ false));
+ oat_dex_files_.emplace_back(location,
+ DexFileSource(raw_dex_files_.back().get()),
+ create_type_lookup_table);
+ } else if (IsZipMagic(magic)) {
+ if (!AddZippedDexFilesSource(std::move(fd), location, create_type_lookup_table)) {
+ return false;
+ }
+ } else {
+ LOG(ERROR) << "Expected valid zip or dex file: '" << filename << "'";
+ return false;
+ }
+ return true;
+}
+
+// Add dex file source(s) from a zip file specified by a file handle.
+bool OatWriter::AddZippedDexFilesSource(ScopedFd&& zip_fd,
+ const char* location,
+ CreateTypeLookupTable create_type_lookup_table) {
+ DCHECK(write_state_ == WriteState::kAddingDexFileSources);
+ std::string error_msg;
+ zip_archives_.emplace_back(ZipArchive::OpenFromFd(zip_fd.release(), location, &error_msg));
+ ZipArchive* zip_archive = zip_archives_.back().get();
+ if (zip_archive == nullptr) {
+ LOG(ERROR) << "Failed to open zip from file descriptor for '" << location << "': "
+ << error_msg;
+ return false;
+ }
+ for (size_t i = 0; ; ++i) {
+ std::string entry_name = DexFile::GetMultiDexClassesDexName(i);
+ std::unique_ptr<ZipEntry> entry(zip_archive->Find(entry_name.c_str(), &error_msg));
+ if (entry == nullptr) {
+ break;
+ }
+ zipped_dex_files_.push_back(std::move(entry));
+ zipped_dex_file_locations_.push_back(DexFile::GetMultiDexLocation(i, location));
+ const char* full_location = zipped_dex_file_locations_.back().c_str();
+ oat_dex_files_.emplace_back(full_location,
+ DexFileSource(zipped_dex_files_.back().get()),
+ create_type_lookup_table);
+ }
+ if (zipped_dex_file_locations_.empty()) {
+ LOG(ERROR) << "No dex files in zip file '" << location << "': " << error_msg;
+ return false;
+ }
+ return true;
+}
+
+// Add dex file source from raw memory.
+bool OatWriter::AddRawDexFileSource(const ArrayRef<const uint8_t>& data,
+ const char* location,
+ uint32_t location_checksum,
+ CreateTypeLookupTable create_type_lookup_table) {
+ DCHECK(write_state_ == WriteState::kAddingDexFileSources);
+ if (data.size() < sizeof(DexFile::Header)) {
+ LOG(ERROR) << "Provided data is shorter than dex file header. size: "
+ << data.size() << " File: " << location;
+ return false;
+ }
+ if (!ValidateDexFileHeader(data.data(), location)) {
+ return false;
+ }
+ const UnalignedDexFileHeader* header = AsUnalignedDexFileHeader(data.data());
+ if (data.size() < header->file_size_) {
+ LOG(ERROR) << "Truncated dex file data. Data size: " << data.size()
+ << " file size from header: " << header->file_size_ << " File: " << location;
+ return false;
+ }
+
+ oat_dex_files_.emplace_back(location, DexFileSource(data.data()), create_type_lookup_table);
+ oat_dex_files_.back().dex_file_location_checksum_ = location_checksum;
+ return true;
+}
+
+dchecked_vector<const char*> OatWriter::GetSourceLocations() const {
+ dchecked_vector<const char*> locations;
+ locations.reserve(oat_dex_files_.size());
+ for (const OatDexFile& oat_dex_file : oat_dex_files_) {
+ locations.push_back(oat_dex_file.GetLocation());
+ }
+ return locations;
+}
+
+bool OatWriter::WriteAndOpenDexFiles(
+ OutputStream* rodata,
+ File* file,
+ InstructionSet instruction_set,
+ const InstructionSetFeatures* instruction_set_features,
+ SafeMap<std::string, std::string>* key_value_store,
+ /*out*/ std::unique_ptr<MemMap>* opened_dex_files_map,
+ /*out*/ std::vector<std::unique_ptr<const DexFile>>* opened_dex_files) {
+ CHECK(write_state_ == WriteState::kAddingDexFileSources);
+
+ size_t offset = InitOatHeader(instruction_set,
+ instruction_set_features,
+ dchecked_integral_cast<uint32_t>(oat_dex_files_.size()),
+ key_value_store);
+ offset = InitOatDexFiles(offset);
+ size_ = offset;
+
+ std::unique_ptr<MemMap> dex_files_map;
+ std::vector<std::unique_ptr<const DexFile>> dex_files;
+ if (!WriteDexFiles(rodata, file)) {
+ return false;
+ }
+ // Reserve space for type lookup tables and update type_lookup_table_offset_.
+ for (OatDexFile& oat_dex_file : oat_dex_files_) {
+ oat_dex_file.ReserveTypeLookupTable(this);
+ }
+ size_t size_after_type_lookup_tables = size_;
+ // Reserve space for class offsets and update class_offsets_offset_.
+ for (OatDexFile& oat_dex_file : oat_dex_files_) {
+ oat_dex_file.ReserveClassOffsets(this);
+ }
+ if (!WriteOatDexFiles(rodata) ||
+ !ExtendForTypeLookupTables(rodata, file, size_after_type_lookup_tables) ||
+ !OpenDexFiles(file, &dex_files_map, &dex_files) ||
+ !WriteTypeLookupTables(dex_files_map.get(), dex_files)) {
+ return false;
+ }
+
+ *opened_dex_files_map = std::move(dex_files_map);
+ *opened_dex_files = std::move(dex_files);
+ write_state_ = WriteState::kPrepareLayout;
+ return true;
+}
+
+void OatWriter::PrepareLayout(const CompilerDriver* compiler,
+ ImageWriter* image_writer,
+ const std::vector<const DexFile*>& dex_files) {
+ CHECK(write_state_ == WriteState::kPrepareLayout);
+
+ dex_files_ = &dex_files;
+
+ compiler_driver_ = compiler;
+ image_writer_ = image_writer;
+ if (compiling_boot_image_) {
+ CHECK(image_writer_ != nullptr);
}
InstructionSet instruction_set = compiler_driver_->GetInstructionSet();
+ CHECK_EQ(instruction_set, oat_header_->GetInstructionSet());
const InstructionSetFeatures* features = compiler_driver_->GetInstructionSetFeatures();
relative_patcher_ = linker::RelativePatcher::Create(instruction_set, features,
&method_offset_map_);
- size_t offset;
+ uint32_t offset = size_;
{
- TimingLogger::ScopedTiming split("InitOatHeader", timings);
- offset = InitOatHeader();
- }
- {
- TimingLogger::ScopedTiming split("InitOatDexFiles", timings);
- offset = InitOatDexFiles(offset);
- }
- {
- TimingLogger::ScopedTiming split("InitDexFiles", timings);
- offset = InitDexFiles(offset);
- }
- {
- TimingLogger::ScopedTiming split("InitLookupTables", timings);
- offset = InitLookupTables(offset);
- }
- {
- TimingLogger::ScopedTiming split("InitOatClasses", timings);
+ TimingLogger::ScopedTiming split("InitOatClasses", timings_);
offset = InitOatClasses(offset);
}
{
- TimingLogger::ScopedTiming split("InitOatMaps", timings);
+ TimingLogger::ScopedTiming split("InitOatMaps", timings_);
offset = InitOatMaps(offset);
}
{
- TimingLogger::ScopedTiming split("InitOatCode", timings);
+ TimingLogger::ScopedTiming split("InitOatCode", timings_);
offset = InitOatCode(offset);
}
{
- TimingLogger::ScopedTiming split("InitOatCodeDexFiles", timings);
+ TimingLogger::ScopedTiming split("InitOatCodeDexFiles", timings_);
offset = InitOatCodeDexFiles(offset);
}
size_ = offset;
@@ -255,7 +477,7 @@
size_t bss_start = RoundUp(size_, kPageSize);
size_t pointer_size = GetInstructionSetPointerSize(instruction_set);
bss_size_ = 0u;
- for (const DexFile* dex_file : dex_files) {
+ for (const DexFile* dex_file : *dex_files_) {
dex_cache_arrays_offsets_.Put(dex_file, bss_start + bss_size_);
DexCacheArraysLayout layout(pointer_size, dex_file);
bss_size_ += layout.Size();
@@ -265,9 +487,10 @@
CHECK_EQ(dex_files_->size(), oat_dex_files_.size());
if (compiling_boot_image_) {
CHECK_EQ(image_writer_ != nullptr,
- key_value_store_->find(OatHeader::kImageLocationKey) == key_value_store_->end());
+ oat_header_->GetStoreValueByKey(OatHeader::kImageLocationKey) == nullptr);
}
- CHECK_ALIGNED(image_patch_delta_, kPageSize);
+
+ write_state_ = WriteState::kWriteRoData;
}
OatWriter::~OatWriter() {
@@ -1134,59 +1357,26 @@
return true;
}
-size_t OatWriter::InitOatHeader() {
- oat_header_.reset(OatHeader::Create(compiler_driver_->GetInstructionSet(),
- compiler_driver_->GetInstructionSetFeatures(),
- dchecked_integral_cast<uint32_t>(dex_files_->size()),
- key_value_store_));
- oat_header_->SetImageFileLocationOatChecksum(image_file_location_oat_checksum_);
- oat_header_->SetImageFileLocationOatDataBegin(image_file_location_oat_begin_);
-
+size_t OatWriter::InitOatHeader(InstructionSet instruction_set,
+ const InstructionSetFeatures* instruction_set_features,
+ uint32_t num_dex_files,
+ SafeMap<std::string, std::string>* key_value_store) {
+ TimingLogger::ScopedTiming split("InitOatHeader", timings_);
+ oat_header_.reset(OatHeader::Create(instruction_set,
+ instruction_set_features,
+ num_dex_files,
+ key_value_store));
+ size_oat_header_ += sizeof(OatHeader);
+ size_oat_header_key_value_store_ += oat_header_->GetHeaderSize() - sizeof(OatHeader);
return oat_header_->GetHeaderSize();
}
size_t OatWriter::InitOatDexFiles(size_t offset) {
- // create the OatDexFiles
- for (size_t i = 0; i != dex_files_->size(); ++i) {
- const DexFile* dex_file = (*dex_files_)[i];
- CHECK(dex_file != nullptr);
- oat_dex_files_.emplace_back(offset, *dex_file);
- offset += oat_dex_files_.back().SizeOf();
- }
- return offset;
-}
-
-size_t OatWriter::InitDexFiles(size_t offset) {
- // calculate the offsets within OatDexFiles to the DexFiles
- for (size_t i = 0; i != dex_files_->size(); ++i) {
- // dex files are required to be 4 byte aligned
- size_t original_offset = offset;
- offset = RoundUp(offset, 4);
- size_dex_file_alignment_ += offset - original_offset;
-
- // set offset in OatDexFile to DexFile
- oat_dex_files_[i].dex_file_offset_ = offset;
-
- const DexFile* dex_file = (*dex_files_)[i];
-
- // Initialize type lookup table
- oat_dex_files_[i].lookup_table_ = dex_file->GetTypeLookupTable();
-
- offset += dex_file->GetHeader().file_size_;
- }
- return offset;
-}
-
-size_t OatWriter::InitLookupTables(size_t offset) {
+ TimingLogger::ScopedTiming split("InitOatDexFiles", timings_);
+ // Initialize offsets of dex files.
for (OatDexFile& oat_dex_file : oat_dex_files_) {
- if (oat_dex_file.lookup_table_ != nullptr) {
- uint32_t aligned_offset = RoundUp(offset, 4);
- oat_dex_file.lookup_table_offset_ = aligned_offset;
- size_oat_lookup_table_alignment_ += aligned_offset - offset;
- offset = aligned_offset + oat_dex_file.lookup_table_->RawDataLength();
- } else {
- oat_dex_file.lookup_table_offset_ = 0;
- }
+ oat_dex_file.offset_ = offset;
+ offset += oat_dex_file.SizeOf();
}
return offset;
}
@@ -1239,7 +1429,6 @@
oat_header_->SetExecutableOffset(offset);
size_executable_offset_alignment_ = offset - old_offset;
if (compiler_driver_->IsBootImage()) {
- CHECK_EQ(image_patch_delta_, 0);
InstructionSet instruction_set = compiler_driver_->GetInstructionSet();
#define DO_TRAMPOLINE(field, fn_name) \
@@ -1264,7 +1453,6 @@
oat_header_->SetQuickImtConflictTrampolineOffset(0);
oat_header_->SetQuickResolutionTrampolineOffset(0);
oat_header_->SetQuickToInterpreterBridgeOffset(0);
- oat_header_->SetImagePatchDelta(image_patch_delta_);
}
return offset;
}
@@ -1289,22 +1477,15 @@
}
bool OatWriter::WriteRodata(OutputStream* out) {
- if (!GetOatDataOffset(out)) {
+ CHECK(write_state_ == WriteState::kWriteRoData);
+
+ if (!WriteClassOffsets(out)) {
+ LOG(ERROR) << "Failed to write class offsets to " << out->GetLocation();
return false;
}
- const size_t file_offset = oat_data_offset_;
- // Reserve space for header. It will be written last - after updating the checksum.
- size_t header_size = oat_header_->GetHeaderSize();
- if (out->Seek(header_size, kSeekCurrent) == static_cast<off_t>(-1)) {
- PLOG(ERROR) << "Failed to reserve space for oat header in " << out->GetLocation();
- return false;
- }
- size_oat_header_ += sizeof(OatHeader);
- size_oat_header_key_value_store_ += oat_header_->GetHeaderSize() - sizeof(OatHeader);
-
- if (!WriteTables(out, file_offset)) {
- LOG(ERROR) << "Failed to write oat tables to " << out->GetLocation();
+ if (!WriteClasses(out)) {
+ LOG(ERROR) << "Failed to write classes to " << out->GetLocation();
return false;
}
@@ -1313,6 +1494,7 @@
LOG(ERROR) << "Failed to seek to oat code position in " << out->GetLocation();
return false;
}
+ size_t file_offset = oat_data_offset_;
size_t relative_offset = static_cast<size_t>(tables_end_offset) - file_offset;
relative_offset = WriteMaps(out, file_offset, relative_offset);
if (relative_offset == 0) {
@@ -1332,11 +1514,13 @@
}
DCHECK_OFFSET();
+ write_state_ = WriteState::kWriteText;
return true;
}
bool OatWriter::WriteCode(OutputStream* out) {
- size_t header_size = oat_header_->GetHeaderSize();
+ CHECK(write_state_ == WriteState::kWriteText);
+
const size_t file_offset = oat_data_offset_;
size_t relative_offset = oat_header_->GetExecutableOffset();
DCHECK_OFFSET();
@@ -1390,10 +1574,12 @@
DO_STAT(size_oat_dex_file_location_data_);
DO_STAT(size_oat_dex_file_location_checksum_);
DO_STAT(size_oat_dex_file_offset_);
+ DO_STAT(size_oat_dex_file_class_offsets_offset_);
DO_STAT(size_oat_dex_file_lookup_table_offset_);
- DO_STAT(size_oat_dex_file_class_offsets_);
DO_STAT(size_oat_lookup_table_alignment_);
DO_STAT(size_oat_lookup_table_);
+ DO_STAT(size_oat_class_offsets_alignment_);
+ DO_STAT(size_oat_class_offsets_);
DO_STAT(size_oat_class_type_);
DO_STAT(size_oat_class_status_);
DO_STAT(size_oat_class_method_bitmaps_);
@@ -1408,88 +1594,90 @@
CHECK_EQ(file_offset + size_, static_cast<size_t>(oat_end_file_offset));
CHECK_EQ(size_, relative_offset);
- // Finalize the header checksum.
+ write_state_ = WriteState::kWriteHeader;
+ return true;
+}
+
+bool OatWriter::WriteHeader(OutputStream* out,
+ uint32_t image_file_location_oat_checksum,
+ uintptr_t image_file_location_oat_begin,
+ int32_t image_patch_delta) {
+ CHECK(write_state_ == WriteState::kWriteHeader);
+
+ oat_header_->SetImageFileLocationOatChecksum(image_file_location_oat_checksum);
+ oat_header_->SetImageFileLocationOatDataBegin(image_file_location_oat_begin);
+ if (compiler_driver_->IsBootImage()) {
+ CHECK_EQ(image_patch_delta, 0);
+ CHECK_EQ(oat_header_->GetImagePatchDelta(), 0);
+ } else {
+ CHECK_ALIGNED(image_patch_delta, kPageSize);
+ oat_header_->SetImagePatchDelta(image_patch_delta);
+ }
oat_header_->UpdateChecksumWithHeaderData();
- // Write the header now that the checksum is final.
+ const size_t file_offset = oat_data_offset_;
+
+ off_t current_offset = out->Seek(0, kSeekCurrent);
+ if (current_offset == static_cast<off_t>(-1)) {
+ PLOG(ERROR) << "Failed to get current offset from " << out->GetLocation();
+ return false;
+ }
if (out->Seek(file_offset, kSeekSet) == static_cast<off_t>(-1)) {
PLOG(ERROR) << "Failed to seek to oat header position in " << out->GetLocation();
return false;
}
DCHECK_EQ(file_offset, static_cast<size_t>(out->Seek(0, kSeekCurrent)));
+
+ // Flush all other data before writing the header.
+ if (!out->Flush()) {
+ PLOG(ERROR) << "Failed to flush before writing oat header to " << out->GetLocation();
+ return false;
+ }
+ // Write the header.
+ size_t header_size = oat_header_->GetHeaderSize();
if (!out->WriteFully(oat_header_.get(), header_size)) {
PLOG(ERROR) << "Failed to write oat header to " << out->GetLocation();
return false;
}
- if (out->Seek(oat_end_file_offset, kSeekSet) == static_cast<off_t>(-1)) {
- PLOG(ERROR) << "Failed to seek to end after writing oat header to " << out->GetLocation();
+ // Flush the header data.
+ if (!out->Flush()) {
+ PLOG(ERROR) << "Failed to flush after writing oat header to " << out->GetLocation();
return false;
}
- DCHECK_EQ(oat_end_file_offset, out->Seek(0, kSeekCurrent));
+ if (out->Seek(current_offset, kSeekSet) == static_cast<off_t>(-1)) {
+ PLOG(ERROR) << "Failed to seek back after writing oat header to " << out->GetLocation();
+ return false;
+ }
+ DCHECK_EQ(current_offset, out->Seek(0, kSeekCurrent));
+
+ write_state_ = WriteState::kDone;
return true;
}
-bool OatWriter::WriteTables(OutputStream* out, const size_t file_offset) {
- for (size_t i = 0; i != oat_dex_files_.size(); ++i) {
- if (!oat_dex_files_[i].Write(this, out, file_offset)) {
- PLOG(ERROR) << "Failed to write oat dex information to " << out->GetLocation();
- return false;
- }
- }
- for (size_t i = 0; i != oat_dex_files_.size(); ++i) {
- uint32_t expected_offset = file_offset + oat_dex_files_[i].dex_file_offset_;
- off_t actual_offset = out->Seek(expected_offset, kSeekSet);
- if (static_cast<uint32_t>(actual_offset) != expected_offset) {
- const DexFile* dex_file = (*dex_files_)[i];
- PLOG(ERROR) << "Failed to seek to dex file section. Actual: " << actual_offset
- << " Expected: " << expected_offset << " File: " << dex_file->GetLocation();
- return false;
- }
- const DexFile* dex_file = (*dex_files_)[i];
- if (!out->WriteFully(&dex_file->GetHeader(), dex_file->GetHeader().file_size_)) {
- PLOG(ERROR) << "Failed to write dex file " << dex_file->GetLocation()
- << " to " << out->GetLocation();
- return false;
- }
- size_dex_file_ += dex_file->GetHeader().file_size_;
- }
- if (!WriteLookupTables(out, file_offset)) {
- return false;
- }
- for (size_t i = 0; i != oat_classes_.size(); ++i) {
- if (!oat_classes_[i].Write(this, out, file_offset)) {
- PLOG(ERROR) << "Failed to write oat methods information to " << out->GetLocation();
- return false;
- }
- }
- return true;
-}
-
-bool OatWriter::WriteLookupTables(OutputStream* out, const size_t file_offset) {
- for (size_t i = 0; i < oat_dex_files_.size(); ++i) {
- const uint32_t lookup_table_offset = oat_dex_files_[i].lookup_table_offset_;
- const TypeLookupTable* table = oat_dex_files_[i].lookup_table_;
- DCHECK_EQ(lookup_table_offset == 0, table == nullptr);
- if (lookup_table_offset == 0) {
- continue;
- }
- const uint32_t expected_offset = file_offset + lookup_table_offset;
- off_t actual_offset = out->Seek(expected_offset, kSeekSet);
- if (static_cast<uint32_t>(actual_offset) != expected_offset) {
- const DexFile* dex_file = (*dex_files_)[i];
- PLOG(ERROR) << "Failed to seek to lookup table section. Actual: " << actual_offset
- << " Expected: " << expected_offset << " File: " << dex_file->GetLocation();
- return false;
- }
- if (table != nullptr) {
- if (!WriteData(out, table->RawData(), table->RawDataLength())) {
- const DexFile* dex_file = (*dex_files_)[i];
- PLOG(ERROR) << "Failed to write lookup table for " << dex_file->GetLocation()
- << " to " << out->GetLocation();
+bool OatWriter::WriteClassOffsets(OutputStream* out) {
+ for (OatDexFile& oat_dex_file : oat_dex_files_) {
+ if (oat_dex_file.class_offsets_offset_ != 0u) {
+ uint32_t expected_offset = oat_data_offset_ + oat_dex_file.class_offsets_offset_;
+ off_t actual_offset = out->Seek(expected_offset, kSeekSet);
+ if (static_cast<uint32_t>(actual_offset) != expected_offset) {
+ PLOG(ERROR) << "Failed to seek to oat class offsets section. Actual: " << actual_offset
+ << " Expected: " << expected_offset << " File: " << oat_dex_file.GetLocation();
return false;
}
- size_oat_lookup_table_ += table->RawDataLength();
+ if (!oat_dex_file.WriteClassOffsets(this, out)) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+bool OatWriter::WriteClasses(OutputStream* out) {
+ for (OatClass& oat_class : oat_classes_) {
+ if (!oat_class.Write(this, out, oat_data_offset_)) {
+ PLOG(ERROR) << "Failed to write oat methods information to " << out->GetLocation();
+ return false;
}
}
return true;
@@ -1585,6 +1773,455 @@
return true;
}
+bool OatWriter::ReadDexFileHeader(File* file, OatDexFile* oat_dex_file) {
+ // Read the dex file header and perform minimal verification.
+ uint8_t raw_header[sizeof(DexFile::Header)];
+ if (!file->ReadFully(&raw_header, sizeof(DexFile::Header))) {
+ PLOG(ERROR) << "Failed to read dex file header. Actual: "
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+ if (!ValidateDexFileHeader(raw_header, oat_dex_file->GetLocation())) {
+ return false;
+ }
+
+ const UnalignedDexFileHeader* header = AsUnalignedDexFileHeader(raw_header);
+ oat_dex_file->dex_file_size_ = header->file_size_;
+ oat_dex_file->dex_file_location_checksum_ = header->checksum_;
+ oat_dex_file->class_offsets_.resize(header->class_defs_size_);
+ return true;
+}
+
+bool OatWriter::ValidateDexFileHeader(const uint8_t* raw_header, const char* location) {
+ if (!DexFile::IsMagicValid(raw_header)) {
+ LOG(ERROR) << "Invalid magic number in dex file header. " << " File: " << location;
+ return false;
+ }
+ if (!DexFile::IsVersionValid(raw_header)) {
+ LOG(ERROR) << "Invalid version number in dex file header. " << " File: " << location;
+ return false;
+ }
+ const UnalignedDexFileHeader* header = AsUnalignedDexFileHeader(raw_header);
+ if (header->file_size_ < sizeof(DexFile::Header)) {
+ LOG(ERROR) << "Dex file header specifies file size insufficient to contain the header."
+ << " File: " << location;
+ return false;
+ }
+ return true;
+}
+
+bool OatWriter::WriteDexFiles(OutputStream* rodata, File* file) {
+ TimingLogger::ScopedTiming split("WriteDexFiles", timings_);
+
+ // Get the elf file offset of the oat file.
+ if (!GetOatDataOffset(rodata)) {
+ return false;
+ }
+
+ // Write dex files.
+ for (OatDexFile& oat_dex_file : oat_dex_files_) {
+ if (!WriteDexFile(rodata, file, &oat_dex_file)) {
+ return false;
+ }
+ }
+
+ // Close sources.
+ for (OatDexFile& oat_dex_file : oat_dex_files_) {
+ oat_dex_file.source_.Clear(); // Get rid of the reference, it's about to be invalidated.
+ }
+ zipped_dex_files_.clear();
+ zip_archives_.clear();
+ raw_dex_files_.clear();
+ return true;
+}
+
+bool OatWriter::WriteDexFile(OutputStream* rodata, File* file, OatDexFile* oat_dex_file) {
+ if (!SeekToDexFile(rodata, file, oat_dex_file)) {
+ return false;
+ }
+ if (oat_dex_file->source_.IsZipEntry()) {
+ if (!WriteDexFile(rodata, file, oat_dex_file, oat_dex_file->source_.GetZipEntry())) {
+ return false;
+ }
+ } else if (oat_dex_file->source_.IsRawFile()) {
+ if (!WriteDexFile(rodata, file, oat_dex_file, oat_dex_file->source_.GetRawFile())) {
+ return false;
+ }
+ } else {
+ DCHECK(oat_dex_file->source_.IsRawData());
+ if (!WriteDexFile(rodata, oat_dex_file, oat_dex_file->source_.GetRawData())) {
+ return false;
+ }
+ }
+
+ // Update current size and account for the written data.
+ DCHECK_EQ(size_, oat_dex_file->dex_file_offset_);
+ size_ += oat_dex_file->dex_file_size_;
+ size_dex_file_ += oat_dex_file->dex_file_size_;
+ return true;
+}
+
+bool OatWriter::SeekToDexFile(OutputStream* out, File* file, OatDexFile* oat_dex_file) {
+ // Dex files are required to be 4 byte aligned.
+ size_t original_offset = size_;
+ size_t offset = RoundUp(original_offset, 4);
+ size_dex_file_alignment_ += offset - original_offset;
+
+ // Seek to the start of the dex file and flush any pending operations in the stream.
+ // Verify that, after flushing the stream, the file is at the same offset as the stream.
+ uint32_t start_offset = oat_data_offset_ + offset;
+ off_t actual_offset = out->Seek(start_offset, kSeekSet);
+ if (actual_offset != static_cast<off_t>(start_offset)) {
+ PLOG(ERROR) << "Failed to seek to dex file section. Actual: " << actual_offset
+ << " Expected: " << start_offset
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+ if (!out->Flush()) {
+ PLOG(ERROR) << "Failed to flush before writing dex file."
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+ actual_offset = lseek(file->Fd(), 0, SEEK_CUR);
+ if (actual_offset != static_cast<off_t>(start_offset)) {
+ PLOG(ERROR) << "Stream/file position mismatch! Actual: " << actual_offset
+ << " Expected: " << start_offset
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+
+ size_ = offset;
+ oat_dex_file->dex_file_offset_ = offset;
+ return true;
+}
+
+bool OatWriter::WriteDexFile(OutputStream* rodata,
+ File* file,
+ OatDexFile* oat_dex_file,
+ ZipEntry* dex_file) {
+ size_t start_offset = oat_data_offset_ + size_;
+ DCHECK_EQ(static_cast<off_t>(start_offset), rodata->Seek(0, kSeekCurrent));
+
+ // Extract the dex file and get the extracted size.
+ std::string error_msg;
+ if (!dex_file->ExtractToFile(*file, &error_msg)) {
+ LOG(ERROR) << "Failed to extract dex file from ZIP entry: " << error_msg
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+ if (file->Flush() != 0) {
+ PLOG(ERROR) << "Failed to flush dex file from ZIP entry."
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+ off_t extracted_end = lseek(file->Fd(), 0, SEEK_CUR);
+ if (extracted_end == static_cast<off_t>(-1)) {
+ PLOG(ERROR) << "Failed get end offset after writing dex file from ZIP entry."
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+ if (extracted_end < static_cast<off_t>(start_offset)) {
+ LOG(ERROR) << "Dex file end position is before start position! End: " << extracted_end
+ << " Start: " << start_offset
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+ uint64_t extracted_size = static_cast<uint64_t>(extracted_end - start_offset);
+ if (extracted_size < sizeof(DexFile::Header)) {
+ LOG(ERROR) << "Extracted dex file is shorter than dex file header. size: "
+ << extracted_size << " File: " << oat_dex_file->GetLocation();
+ return false;
+ }
+
+ // Read the dex file header and extract required data to OatDexFile.
+ off_t actual_offset = lseek(file->Fd(), start_offset, SEEK_SET);
+ if (actual_offset != static_cast<off_t>(start_offset)) {
+ PLOG(ERROR) << "Failed to seek back to dex file header. Actual: " << actual_offset
+ << " Expected: " << start_offset
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+ if (!ReadDexFileHeader(file, oat_dex_file)) {
+ return false;
+ }
+ if (extracted_size < oat_dex_file->dex_file_size_) {
+ LOG(ERROR) << "Extracted truncated dex file. Extracted size: " << extracted_size
+ << " file size from header: " << oat_dex_file->dex_file_size_
+ << " File: " << oat_dex_file->GetLocation();
+ return false;
+ }
+
+ // Override the checksum from header with the CRC from ZIP entry.
+ oat_dex_file->dex_file_location_checksum_ = dex_file->GetCrc32();
+
+ // Seek both file and stream to the end offset.
+ size_t end_offset = start_offset + oat_dex_file->dex_file_size_;
+ actual_offset = lseek(file->Fd(), end_offset, SEEK_SET);
+ if (actual_offset != static_cast<off_t>(end_offset)) {
+ PLOG(ERROR) << "Failed to seek to end of dex file. Actual: " << actual_offset
+ << " Expected: " << end_offset
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+ actual_offset = rodata->Seek(end_offset, kSeekSet);
+ if (actual_offset != static_cast<off_t>(end_offset)) {
+ PLOG(ERROR) << "Failed to seek stream to end of dex file. Actual: " << actual_offset
+ << " Expected: " << end_offset << " File: " << oat_dex_file->GetLocation();
+ return false;
+ }
+ if (!rodata->Flush()) {
+ PLOG(ERROR) << "Failed to flush stream after seeking over dex file."
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+
+ // If we extracted more than the size specified in the header, truncate the file.
+ if (extracted_size > oat_dex_file->dex_file_size_) {
+ if (file->SetLength(end_offset) != 0) {
+ PLOG(ERROR) << "Failed to truncate excessive dex file length."
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool OatWriter::WriteDexFile(OutputStream* rodata,
+ File* file,
+ OatDexFile* oat_dex_file,
+ File* dex_file) {
+ size_t start_offset = oat_data_offset_ + size_;
+ DCHECK_EQ(static_cast<off_t>(start_offset), rodata->Seek(0, kSeekCurrent));
+
+ off_t input_offset = lseek(dex_file->Fd(), 0, SEEK_SET);
+ if (input_offset != static_cast<off_t>(0)) {
+ PLOG(ERROR) << "Failed to seek to dex file header. Actual: " << input_offset
+ << " Expected: 0"
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+ if (!ReadDexFileHeader(dex_file, oat_dex_file)) {
+ return false;
+ }
+
+ // Copy the input dex file using sendfile().
+ if (!file->Copy(dex_file, 0, oat_dex_file->dex_file_size_)) {
+ PLOG(ERROR) << "Failed to copy dex file to oat file."
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+ if (file->Flush() != 0) {
+ PLOG(ERROR) << "Failed to flush dex file."
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+
+ // Check file position and seek the stream to the end offset.
+ size_t end_offset = start_offset + oat_dex_file->dex_file_size_;
+ off_t actual_offset = lseek(file->Fd(), 0, SEEK_CUR);
+ if (actual_offset != static_cast<off_t>(end_offset)) {
+ PLOG(ERROR) << "Unexpected file position after copying dex file. Actual: " << actual_offset
+ << " Expected: " << end_offset
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+ actual_offset = rodata->Seek(end_offset, kSeekSet);
+ if (actual_offset != static_cast<off_t>(end_offset)) {
+ PLOG(ERROR) << "Failed to seek stream to end of dex file. Actual: " << actual_offset
+ << " Expected: " << end_offset << " File: " << oat_dex_file->GetLocation();
+ return false;
+ }
+ if (!rodata->Flush()) {
+ PLOG(ERROR) << "Failed to flush stream after seeking over dex file."
+ << " File: " << oat_dex_file->GetLocation() << " Output: " << file->GetPath();
+ return false;
+ }
+
+ return true;
+}
+
+bool OatWriter::WriteDexFile(OutputStream* rodata,
+ OatDexFile* oat_dex_file,
+ const uint8_t* dex_file) {
+ // Note: The raw data has already been checked to contain the header
+ // and all the data that the header specifies as the file size.
+ DCHECK(dex_file != nullptr);
+ DCHECK(ValidateDexFileHeader(dex_file, oat_dex_file->GetLocation()));
+ const UnalignedDexFileHeader* header = AsUnalignedDexFileHeader(dex_file);
+
+ if (!rodata->WriteFully(dex_file, header->file_size_)) {
+ PLOG(ERROR) << "Failed to write dex file " << oat_dex_file->GetLocation()
+ << " to " << rodata->GetLocation();
+ return false;
+ }
+ if (!rodata->Flush()) {
+ PLOG(ERROR) << "Failed to flush stream after writing dex file."
+ << " File: " << oat_dex_file->GetLocation();
+ return false;
+ }
+
+ // Update dex file size and resize class offsets in the OatDexFile.
+ // Note: For raw data, the checksum is passed directly to AddRawDexFileSource().
+ oat_dex_file->dex_file_size_ = header->file_size_;
+ oat_dex_file->class_offsets_.resize(header->class_defs_size_);
+ return true;
+}
+
+bool OatWriter::WriteOatDexFiles(OutputStream* rodata) {
+ TimingLogger::ScopedTiming split("WriteOatDexFiles", timings_);
+
+ // Seek to the start of OatDexFiles, i.e. to the end of the OatHeader. If there are
+ // no OatDexFiles, no data is actually written to .rodata before WriteHeader() and
+ // this Seek() ensures that we reserve the space for OatHeader in .rodata.
+ DCHECK(oat_dex_files_.empty() || oat_dex_files_[0u].offset_ == oat_header_->GetHeaderSize());
+ uint32_t expected_offset = oat_data_offset_ + oat_header_->GetHeaderSize();
+ off_t actual_offset = rodata->Seek(expected_offset, kSeekSet);
+ if (static_cast<uint32_t>(actual_offset) != expected_offset) {
+ PLOG(ERROR) << "Failed to seek to OatDexFile table section. Actual: " << actual_offset
+ << " Expected: " << expected_offset << " File: " << rodata->GetLocation();
+ return false;
+ }
+
+ for (size_t i = 0, size = oat_dex_files_.size(); i != size; ++i) {
+ OatDexFile* oat_dex_file = &oat_dex_files_[i];
+
+ DCHECK_EQ(oat_data_offset_ + oat_dex_file->offset_,
+ static_cast<size_t>(rodata->Seek(0, kSeekCurrent)));
+
+ // Write OatDexFile.
+ if (!oat_dex_file->Write(this, rodata)) {
+ PLOG(ERROR) << "Failed to write oat dex information to " << rodata->GetLocation();
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool OatWriter::ExtendForTypeLookupTables(OutputStream* rodata, File* file, size_t offset) {
+ TimingLogger::ScopedTiming split("ExtendForTypeLookupTables", timings_);
+
+ int64_t new_length = oat_data_offset_ + dchecked_integral_cast<int64_t>(offset);
+ if (file->SetLength(new_length) != 0) {
+ PLOG(ERROR) << "Failed to extend file for type lookup tables. new_length: " << new_length
+ << "File: " << file->GetPath();
+ return false;
+ }
+ off_t actual_offset = rodata->Seek(new_length, kSeekSet);
+ if (actual_offset != static_cast<off_t>(new_length)) {
+ PLOG(ERROR) << "Failed to seek stream after extending file for type lookup tables."
+ << " Actual: " << actual_offset << " Expected: " << new_length
+ << " File: " << rodata->GetLocation();
+ return false;
+ }
+ if (!rodata->Flush()) {
+ PLOG(ERROR) << "Failed to flush stream after extending for type lookup tables."
+ << " File: " << rodata->GetLocation();
+ return false;
+ }
+ return true;
+}
+
+bool OatWriter::OpenDexFiles(
+ File* file,
+ /*out*/ std::unique_ptr<MemMap>* opened_dex_files_map,
+ /*out*/ std::vector<std::unique_ptr<const DexFile>>* opened_dex_files) {
+ TimingLogger::ScopedTiming split("OpenDexFiles", timings_);
+
+ if (oat_dex_files_.empty()) {
+ // Nothing to do.
+ return true;
+ }
+
+ size_t map_offset = oat_dex_files_[0].dex_file_offset_;
+ size_t length = size_ - map_offset;
+ std::string error_msg;
+ std::unique_ptr<MemMap> dex_files_map(MemMap::MapFile(length,
+ PROT_READ | PROT_WRITE,
+ MAP_SHARED,
+ file->Fd(),
+ oat_data_offset_ + map_offset,
+ /* low_4gb */ false,
+ file->GetPath().c_str(),
+ &error_msg));
+ if (dex_files_map == nullptr) {
+ LOG(ERROR) << "Failed to mmap() dex files from oat file. File: " << file->GetPath()
+ << " error: " << error_msg;
+ return false;
+ }
+ std::vector<std::unique_ptr<const DexFile>> dex_files;
+ for (OatDexFile& oat_dex_file : oat_dex_files_) {
+ // Make sure no one messed with input files while we were copying data.
+ // At the very least we need consistent file size and number of class definitions.
+ const uint8_t* raw_dex_file =
+ dex_files_map->Begin() + oat_dex_file.dex_file_offset_ - map_offset;
+ if (!ValidateDexFileHeader(raw_dex_file, oat_dex_file.GetLocation())) {
+ // Note: ValidateDexFileHeader() already logged an error message.
+ LOG(ERROR) << "Failed to verify written dex file header!"
+ << " Output: " << file->GetPath() << " ~ " << std::hex << map_offset
+ << " ~ " << static_cast<const void*>(raw_dex_file);
+ return false;
+ }
+ const UnalignedDexFileHeader* header = AsUnalignedDexFileHeader(raw_dex_file);
+ if (header->file_size_ != oat_dex_file.dex_file_size_) {
+ LOG(ERROR) << "File size mismatch in written dex file header! Expected: "
+ << oat_dex_file.dex_file_size_ << " Actual: " << header->file_size_
+ << " Output: " << file->GetPath();
+ return false;
+ }
+ if (header->class_defs_size_ != oat_dex_file.class_offsets_.size()) {
+ LOG(ERROR) << "Class defs size mismatch in written dex file header! Expected: "
+ << oat_dex_file.class_offsets_.size() << " Actual: " << header->class_defs_size_
+ << " Output: " << file->GetPath();
+ return false;
+ }
+
+ // Now, open the dex file.
+ dex_files.emplace_back(DexFile::Open(raw_dex_file,
+ oat_dex_file.dex_file_size_,
+ oat_dex_file.GetLocation(),
+ oat_dex_file.dex_file_location_checksum_,
+ /* oat_dex_file */ nullptr,
+ &error_msg));
+ if (dex_files.back() == nullptr) {
+ LOG(ERROR) << "Failed to open dex file from oat file. File:" << oat_dex_file.GetLocation();
+ return false;
+ }
+ }
+
+ *opened_dex_files_map = std::move(dex_files_map);
+ *opened_dex_files = std::move(dex_files);
+ return true;
+}
+
+bool OatWriter::WriteTypeLookupTables(
+ MemMap* opened_dex_files_map,
+ const std::vector<std::unique_ptr<const DexFile>>& opened_dex_files) {
+ TimingLogger::ScopedTiming split("WriteTypeLookupTables", timings_);
+
+ DCHECK_EQ(opened_dex_files.size(), oat_dex_files_.size());
+ for (size_t i = 0, size = opened_dex_files.size(); i != size; ++i) {
+ OatDexFile* oat_dex_file = &oat_dex_files_[i];
+ if (oat_dex_file->lookup_table_offset_ != 0u) {
+ DCHECK(oat_dex_file->create_type_lookup_table_ == CreateTypeLookupTable::kCreate);
+ DCHECK_NE(oat_dex_file->class_offsets_.size(), 0u);
+ size_t map_offset = oat_dex_files_[0].dex_file_offset_;
+ size_t lookup_table_offset = oat_dex_file->lookup_table_offset_;
+ uint8_t* lookup_table = opened_dex_files_map->Begin() + (lookup_table_offset - map_offset);
+ opened_dex_files[i]->CreateTypeLookupTable(lookup_table);
+ }
+ }
+
+ DCHECK_EQ(opened_dex_files_map == nullptr, opened_dex_files.empty());
+ if (opened_dex_files_map != nullptr && !opened_dex_files_map->Sync()) {
+ PLOG(ERROR) << "Failed to Sync() type lookup tables. Map: " << opened_dex_files_map->GetName();
+ return false;
+ }
+
+ return true;
+}
+
bool OatWriter::WriteCodeAlignment(OutputStream* out, uint32_t aligned_code_delta) {
static const uint8_t kPadding[] = {
0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u
@@ -1611,15 +2248,20 @@
}
}
-OatWriter::OatDexFile::OatDexFile(size_t offset, const DexFile& dex_file) {
- offset_ = offset;
- const std::string& location(dex_file.GetLocation());
- dex_file_location_size_ = location.size();
- dex_file_location_data_ = reinterpret_cast<const uint8_t*>(location.data());
- dex_file_location_checksum_ = dex_file.GetLocationChecksum();
- dex_file_offset_ = 0;
- lookup_table_offset_ = 0;
- class_offsets_.resize(dex_file.NumClassDefs());
+OatWriter::OatDexFile::OatDexFile(const char* dex_file_location,
+ DexFileSource source,
+ CreateTypeLookupTable create_type_lookup_table)
+ : source_(source),
+ create_type_lookup_table_(create_type_lookup_table),
+ dex_file_size_(0),
+ offset_(0),
+ dex_file_location_size_(strlen(dex_file_location)),
+ dex_file_location_data_(dex_file_location),
+ dex_file_location_checksum_(0u),
+ dex_file_offset_(0u),
+ class_offsets_offset_(0u),
+ lookup_table_offset_(0u),
+ class_offsets_() {
}
size_t OatWriter::OatDexFile::SizeOf() const {
@@ -1627,24 +2269,54 @@
+ dex_file_location_size_
+ sizeof(dex_file_location_checksum_)
+ sizeof(dex_file_offset_)
- + sizeof(lookup_table_offset_)
- + (sizeof(class_offsets_[0]) * class_offsets_.size());
+ + sizeof(class_offsets_offset_)
+ + sizeof(lookup_table_offset_);
}
-bool OatWriter::OatDexFile::Write(OatWriter* oat_writer,
- OutputStream* out,
- const size_t file_offset) const {
+void OatWriter::OatDexFile::ReserveTypeLookupTable(OatWriter* oat_writer) {
+ DCHECK_EQ(lookup_table_offset_, 0u);
+ if (create_type_lookup_table_ == CreateTypeLookupTable::kCreate && !class_offsets_.empty()) {
+ size_t table_size = TypeLookupTable::RawDataLength(class_offsets_.size());
+ if (table_size != 0u) {
+ // Type tables are required to be 4 byte aligned.
+ size_t original_offset = oat_writer->size_;
+ size_t offset = RoundUp(original_offset, 4);
+ oat_writer->size_oat_lookup_table_alignment_ += offset - original_offset;
+ lookup_table_offset_ = offset;
+ oat_writer->size_ = offset + table_size;
+ oat_writer->size_oat_lookup_table_ += table_size;
+ }
+ }
+}
+
+void OatWriter::OatDexFile::ReserveClassOffsets(OatWriter* oat_writer) {
+ DCHECK_EQ(class_offsets_offset_, 0u);
+ if (!class_offsets_.empty()) {
+ // Class offsets are required to be 4 byte aligned.
+ size_t original_offset = oat_writer->size_;
+ size_t offset = RoundUp(original_offset, 4);
+ oat_writer->size_oat_class_offsets_alignment_ += offset - original_offset;
+ class_offsets_offset_ = offset;
+ oat_writer->size_ = offset + GetClassOffsetsRawSize();
+ }
+}
+
+bool OatWriter::OatDexFile::Write(OatWriter* oat_writer, OutputStream* out) const {
+ const size_t file_offset = oat_writer->oat_data_offset_;
DCHECK_OFFSET_();
+
if (!oat_writer->WriteData(out, &dex_file_location_size_, sizeof(dex_file_location_size_))) {
PLOG(ERROR) << "Failed to write dex file location length to " << out->GetLocation();
return false;
}
oat_writer->size_oat_dex_file_location_size_ += sizeof(dex_file_location_size_);
+
if (!oat_writer->WriteData(out, dex_file_location_data_, dex_file_location_size_)) {
PLOG(ERROR) << "Failed to write dex file location data to " << out->GetLocation();
return false;
}
oat_writer->size_oat_dex_file_location_data_ += dex_file_location_size_;
+
if (!oat_writer->WriteData(out,
&dex_file_location_checksum_,
sizeof(dex_file_location_checksum_))) {
@@ -1652,21 +2324,35 @@
return false;
}
oat_writer->size_oat_dex_file_location_checksum_ += sizeof(dex_file_location_checksum_);
+
if (!oat_writer->WriteData(out, &dex_file_offset_, sizeof(dex_file_offset_))) {
PLOG(ERROR) << "Failed to write dex file offset to " << out->GetLocation();
return false;
}
oat_writer->size_oat_dex_file_offset_ += sizeof(dex_file_offset_);
+
+ if (!oat_writer->WriteData(out, &class_offsets_offset_, sizeof(class_offsets_offset_))) {
+ PLOG(ERROR) << "Failed to write class offsets offset to " << out->GetLocation();
+ return false;
+ }
+ oat_writer->size_oat_dex_file_class_offsets_offset_ += sizeof(class_offsets_offset_);
+
if (!oat_writer->WriteData(out, &lookup_table_offset_, sizeof(lookup_table_offset_))) {
PLOG(ERROR) << "Failed to write lookup table offset to " << out->GetLocation();
return false;
}
oat_writer->size_oat_dex_file_lookup_table_offset_ += sizeof(lookup_table_offset_);
+
+ return true;
+}
+
+bool OatWriter::OatDexFile::WriteClassOffsets(OatWriter* oat_writer, OutputStream* out) {
if (!oat_writer->WriteData(out, class_offsets_.data(), GetClassOffsetsRawSize())) {
- PLOG(ERROR) << "Failed to write methods offsets to " << out->GetLocation();
+ PLOG(ERROR) << "Failed to write oat class offsets for " << GetLocation()
+ << " to " << out->GetLocation();
return false;
}
- oat_writer->size_oat_dex_file_class_offsets_ += GetClassOffsetsRawSize();
+ oat_writer->size_oat_class_offsets_ += GetClassOffsetsRawSize();
return true;
}
diff --git a/compiler/oat_writer.h b/compiler/oat_writer.h
index 5feb5fc..d681998 100644
--- a/compiler/oat_writer.h
+++ b/compiler/oat_writer.h
@@ -27,7 +27,9 @@
#include "method_reference.h"
#include "mirror/class.h"
#include "oat.h"
+#include "os.h"
#include "safe_map.h"
+#include "ScopedFd.h"
#include "utils/array_ref.h"
namespace art {
@@ -39,6 +41,7 @@
class OutputStream;
class TimingLogger;
class TypeLookupTable;
+class ZipEntry;
namespace dwarf {
struct MethodDebugInfo;
@@ -61,6 +64,11 @@
// ...
// TypeLookupTable[D]
//
+// ClassOffsets[0] one table of OatClass offsets for each class def for each OatDexFile.
+// ClassOffsets[1]
+// ...
+// ClassOffsets[D]
+//
// OatClass[0] one variable sized OatClass for each of C DexFile::ClassDefs
// OatClass[1] contains OatClass entries with class status, offsets to code, etc.
// ...
@@ -93,15 +101,65 @@
//
class OatWriter {
public:
- OatWriter(const std::vector<const DexFile*>& dex_files,
- uint32_t image_file_location_oat_checksum,
- uintptr_t image_file_location_oat_begin,
- int32_t image_patch_delta,
- const CompilerDriver* compiler,
- ImageWriter* image_writer,
- bool compiling_boot_image,
- TimingLogger* timings,
- SafeMap<std::string, std::string>* key_value_store);
+ enum class CreateTypeLookupTable {
+ kCreate,
+ kDontCreate,
+ kDefault = kCreate
+ };
+
+ OatWriter(bool compiling_boot_image, TimingLogger* timings);
+
+ // To produce a valid oat file, the user must first add sources with any combination of
+ // - AddDexFileSource(),
+ // - AddZippedDexFilesSource(),
+ // - AddRawDexFileSource().
+ // Then the user must call in order
+ // - WriteAndOpenDexFiles()
+ // - PrepareLayout(),
+ // - WriteRodata(),
+ // - WriteCode(),
+ // - WriteHeader().
+
+ // Add dex file source(s) from a file, either a plain dex file or
+ // a zip file with one or more dex files.
+ bool AddDexFileSource(
+ const char* filename,
+ const char* location,
+ CreateTypeLookupTable create_type_lookup_table = CreateTypeLookupTable::kDefault);
+ // Add dex file source(s) from a zip file specified by a file handle.
+ bool AddZippedDexFilesSource(
+ ScopedFd&& zip_fd,
+ const char* location,
+ CreateTypeLookupTable create_type_lookup_table = CreateTypeLookupTable::kDefault);
+ // Add dex file source from raw memory.
+ bool AddRawDexFileSource(
+ const ArrayRef<const uint8_t>& data,
+ const char* location,
+ uint32_t location_checksum,
+ CreateTypeLookupTable create_type_lookup_table = CreateTypeLookupTable::kDefault);
+ dchecked_vector<const char*> GetSourceLocations() const;
+
+ // Write raw dex files to the .rodata section and open them from the oat file.
+ bool WriteAndOpenDexFiles(OutputStream* rodata,
+ File* file,
+ InstructionSet instruction_set,
+ const InstructionSetFeatures* instruction_set_features,
+ SafeMap<std::string, std::string>* key_value_store,
+ /*out*/ std::unique_ptr<MemMap>* opened_dex_files_map,
+ /*out*/ std::vector<std::unique_ptr<const DexFile>>* opened_dex_files);
+ // Prepare layout of remaining data.
+ void PrepareLayout(const CompilerDriver* compiler,
+ ImageWriter* image_writer,
+ const std::vector<const DexFile*>& dex_files);
+ // Write the rest of .rodata section (ClassOffsets[], OatClass[], maps).
+ bool WriteRodata(OutputStream* out);
+ // Write the code to the .text section.
+ bool WriteCode(OutputStream* out);
+ // Write the oat header. This finalizes the oat file.
+ bool WriteHeader(OutputStream* out,
+ uint32_t image_file_location_oat_checksum,
+ uintptr_t image_file_location_oat_begin,
+ int32_t image_patch_delta);
// Returns whether the oat file has an associated image.
bool HasImage() const {
@@ -130,9 +188,6 @@
return ArrayRef<const uintptr_t>(absolute_patch_locations_);
}
- bool WriteRodata(OutputStream* out);
- bool WriteCode(OutputStream* out);
-
~OatWriter();
ArrayRef<const dwarf::MethodDebugInfo> GetMethodDebugInfo() const {
@@ -144,6 +199,7 @@
}
private:
+ class DexFileSource;
class OatClass;
class OatDexFile;
@@ -174,29 +230,65 @@
// with a given DexMethodVisitor.
bool VisitDexMethods(DexMethodVisitor* visitor);
- size_t InitOatHeader();
+ size_t InitOatHeader(InstructionSet instruction_set,
+ const InstructionSetFeatures* instruction_set_features,
+ uint32_t num_dex_files,
+ SafeMap<std::string, std::string>* key_value_store);
size_t InitOatDexFiles(size_t offset);
- size_t InitLookupTables(size_t offset);
- size_t InitDexFiles(size_t offset);
size_t InitOatClasses(size_t offset);
size_t InitOatMaps(size_t offset);
size_t InitOatCode(size_t offset);
size_t InitOatCodeDexFiles(size_t offset);
- bool WriteTables(OutputStream* out, const size_t file_offset);
- bool WriteLookupTables(OutputStream* out, const size_t file_offset);
+ bool WriteClassOffsets(OutputStream* out);
+ bool WriteClasses(OutputStream* out);
size_t WriteMaps(OutputStream* out, const size_t file_offset, size_t relative_offset);
size_t WriteCode(OutputStream* out, const size_t file_offset, size_t relative_offset);
size_t WriteCodeDexFiles(OutputStream* out, const size_t file_offset, size_t relative_offset);
bool GetOatDataOffset(OutputStream* out);
+ bool ReadDexFileHeader(File* file, OatDexFile* oat_dex_file);
+ bool ValidateDexFileHeader(const uint8_t* raw_header, const char* location);
+ bool WriteDexFiles(OutputStream* rodata, File* file);
+ bool WriteDexFile(OutputStream* rodata, File* file, OatDexFile* oat_dex_file);
+ bool SeekToDexFile(OutputStream* rodata, File* file, OatDexFile* oat_dex_file);
+ bool WriteDexFile(OutputStream* rodata, File* file, OatDexFile* oat_dex_file, ZipEntry* dex_file);
+ bool WriteDexFile(OutputStream* rodata, File* file, OatDexFile* oat_dex_file, File* dex_file);
+ bool WriteDexFile(OutputStream* rodata, OatDexFile* oat_dex_file, const uint8_t* dex_file);
+ bool WriteOatDexFiles(OutputStream* rodata);
+ bool ExtendForTypeLookupTables(OutputStream* rodata, File* file, size_t offset);
+ bool OpenDexFiles(File* file,
+ /*out*/ std::unique_ptr<MemMap>* opened_dex_files_map,
+ /*out*/ std::vector<std::unique_ptr<const DexFile>>* opened_dex_files);
+ bool WriteTypeLookupTables(MemMap* opened_dex_files_map,
+ const std::vector<std::unique_ptr<const DexFile>>& opened_dex_files);
bool WriteCodeAlignment(OutputStream* out, uint32_t aligned_code_delta);
bool WriteData(OutputStream* out, const void* data, size_t size);
+ enum class WriteState {
+ kAddingDexFileSources,
+ kPrepareLayout,
+ kWriteRoData,
+ kWriteText,
+ kWriteHeader,
+ kDone
+ };
+
+ WriteState write_state_;
+ TimingLogger* timings_;
+
+ std::vector<std::unique_ptr<File>> raw_dex_files_;
+ std::vector<std::unique_ptr<ZipArchive>> zip_archives_;
+ std::vector<std::unique_ptr<ZipEntry>> zipped_dex_files_;
+
+ // Using std::list<> which doesn't move elements around on push/emplace_back().
+ // We need this because we keep plain pointers to the strings' c_str().
+ std::list<std::string> zipped_dex_file_locations_;
+
dchecked_vector<dwarf::MethodDebugInfo> method_info_;
- const CompilerDriver* const compiler_driver_;
- ImageWriter* const image_writer_;
+ const CompilerDriver* compiler_driver_;
+ ImageWriter* image_writer_;
const bool compiling_boot_image_;
// note OatFile does not take ownership of the DexFiles
@@ -215,13 +307,7 @@
// Offset of the oat data from the start of the mmapped region of the elf file.
size_t oat_data_offset_;
- // dependencies on the image.
- uint32_t image_file_location_oat_checksum_;
- uintptr_t image_file_location_oat_begin_;
- int32_t image_patch_delta_;
-
// data to write
- SafeMap<std::string, std::string>* key_value_store_;
std::unique_ptr<OatHeader> oat_header_;
dchecked_vector<OatDexFile> oat_dex_files_;
dchecked_vector<OatClass> oat_classes_;
@@ -257,10 +343,12 @@
uint32_t size_oat_dex_file_location_data_;
uint32_t size_oat_dex_file_location_checksum_;
uint32_t size_oat_dex_file_offset_;
+ uint32_t size_oat_dex_file_class_offsets_offset_;
uint32_t size_oat_dex_file_lookup_table_offset_;
- uint32_t size_oat_dex_file_class_offsets_;
uint32_t size_oat_lookup_table_alignment_;
uint32_t size_oat_lookup_table_;
+ uint32_t size_oat_class_offsets_alignment_;
+ uint32_t size_oat_class_offsets_;
uint32_t size_oat_class_type_;
uint32_t size_oat_class_status_;
uint32_t size_oat_class_method_bitmaps_;
@@ -269,7 +357,7 @@
std::unique_ptr<linker::RelativePatcher> relative_patcher_;
// The locations of absolute patches relative to the start of the executable section.
- std::vector<uintptr_t> absolute_patch_locations_;
+ dchecked_vector<uintptr_t> absolute_patch_locations_;
// Map method reference to assigned offset.
// Wrap the map in a class implementing linker::RelativePatcherTargetProvider.
diff --git a/compiler/utils/test_dex_file_builder.h b/compiler/utils/test_dex_file_builder.h
index b6a228c..e57a540 100644
--- a/compiler/utils/test_dex_file_builder.h
+++ b/compiler/utils/test_dex_file_builder.h
@@ -21,6 +21,7 @@
#include <set>
#include <map>
#include <vector>
+#include <zlib.h>
#include "base/bit_utils.h"
#include "base/logging.h"
@@ -161,7 +162,6 @@
uint32_t total_size = data_section_offset + data_section_size;
dex_file_data_.resize(total_size);
- std::memcpy(&dex_file_data_[0], header_data.data, sizeof(DexFile::Header));
for (const auto& entry : strings_) {
CHECK_LT(entry.first.size(), 128u);
@@ -210,7 +210,12 @@
Write32(raw_offset + 4u, GetStringIdx(entry.first.name));
}
- // Leave checksum and signature as zeros.
+ // Leave signature as zeros.
+
+ header->file_size_ = dex_file_data_.size();
+ size_t skip = sizeof(header->magic_) + sizeof(header->checksum_);
+ header->checksum_ = adler32(0u, dex_file_data_.data() + skip, dex_file_data_.size() - skip);
+ std::memcpy(&dex_file_data_[0], header_data.data, sizeof(DexFile::Header));
std::string error_msg;
std::unique_ptr<const DexFile> dex_file(DexFile::Open(
diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc
index ea54239..b97e7ee 100644
--- a/dex2oat/dex2oat.cc
+++ b/dex2oat/dex2oat.cc
@@ -503,6 +503,10 @@
compiler_kind_(Compiler::kOptimizing),
instruction_set_(kRuntimeISA),
// Take the default set of instruction features from the build.
+ image_file_location_oat_checksum_(0),
+ image_file_location_oat_data_begin_(0),
+ image_patch_delta_(0),
+ key_value_store_(nullptr),
verification_results_(nullptr),
method_inliner_map_(),
runtime_(nullptr),
@@ -522,8 +526,14 @@
boot_image_(false),
multi_image_(false),
is_host_(false),
+ class_loader_(nullptr),
+ elf_writers_(),
+ oat_writers_(),
+ rodata_(),
image_writer_(nullptr),
driver_(nullptr),
+ opened_dex_files_maps_(),
+ opened_dex_files_(),
dump_stats_(false),
dump_passes_(false),
dump_timing_(false),
@@ -534,11 +544,6 @@
timings_(timings) {}
~Dex2Oat() {
- // Free opened dex files before deleting the runtime_, because ~DexFile
- // uses MemMap, which is shut down by ~Runtime.
- class_path_files_.clear();
- opened_dex_files_.clear();
-
// Log completion time before deleting the runtime_, because this accesses
// the runtime.
LogCompletionTime();
@@ -551,6 +556,9 @@
for (std::unique_ptr<const DexFile>& dex_file : opened_dex_files_) {
dex_file.release();
}
+ for (std::unique_ptr<MemMap>& map : opened_dex_files_maps_) {
+ map.release();
+ }
for (std::unique_ptr<File>& oat_file : oat_files_) {
oat_file.release();
}
@@ -1130,6 +1138,9 @@
// Check whether the oat output files are writable, and open them for later. Also open a swap
// file, if a name is given.
bool OpenFile() {
+ // Prune non-existent dex files now so that we don't create empty oat files for multi-image.
+ PruneNonExistentDexFiles();
+
// Expand oat and image filenames for multi image.
if (IsBootImage() && multi_image_) {
ExpandOatAndImageFilenames();
@@ -1201,9 +1212,6 @@
}
// Note that dex2oat won't close the swap_fd_. The compiler driver's swap space will do that.
- // Organize inputs, handling multi-dex and multiple oat file outputs.
- CreateDexOatMappings();
-
return true;
}
@@ -1246,89 +1254,135 @@
return false;
}
+ CreateOatWriters();
+ if (!AddDexFileSources()) {
+ return false;
+ }
+
+ if (IsBootImage() && image_filenames_.size() > 1) {
+ // If we're compiling the boot image, store the boot classpath into the Key-Value store.
+ // We need this for the multi-image case.
+ key_value_store_->Put(OatHeader::kBootClassPath, GetMultiImageBootClassPath());
+ }
+
+ if (!IsBootImage()) {
+ // When compiling an app, create the runtime early to retrieve
+ // the image location key needed for the oat header.
+ if (!CreateRuntime(std::move(runtime_options))) {
+ return false;
+ }
+
+ {
+ TimingLogger::ScopedTiming t3("Loading image checksum", timings_);
+ std::vector<gc::space::ImageSpace*> image_spaces =
+ Runtime::Current()->GetHeap()->GetBootImageSpaces();
+ image_file_location_oat_checksum_ = image_spaces[0]->GetImageHeader().GetOatChecksum();
+ image_file_location_oat_data_begin_ =
+ reinterpret_cast<uintptr_t>(image_spaces[0]->GetImageHeader().GetOatDataBegin());
+ image_patch_delta_ = image_spaces[0]->GetImageHeader().GetPatchDelta();
+ // Store the boot image filename(s).
+ std::vector<std::string> image_filenames;
+ for (const gc::space::ImageSpace* image_space : image_spaces) {
+ image_filenames.push_back(image_space->GetImageFilename());
+ }
+ std::string image_file_location = Join(image_filenames, ':');
+ if (!image_file_location.empty()) {
+ key_value_store_->Put(OatHeader::kImageLocationKey, image_file_location);
+ }
+ }
+
+ // Open dex files for class path.
+ const std::vector<std::string> class_path_locations =
+ GetClassPathLocations(runtime_->GetClassPathString());
+ OpenClassPathFiles(class_path_locations, &class_path_files_);
+
+ // Store the classpath we have right now.
+ std::vector<const DexFile*> class_path_files = MakeNonOwningPointerVector(class_path_files_);
+ key_value_store_->Put(OatHeader::kClassPathKey,
+ OatFile::EncodeDexFileDependencies(class_path_files));
+ }
+
+ // Now that we have finalized key_value_store_, start writing the oat file.
{
- TimingLogger::ScopedTiming t_runtime("Create runtime", timings_);
+ TimingLogger::ScopedTiming t_dex("Writing and opening dex files", timings_);
+ rodata_.reserve(oat_writers_.size());
+ for (size_t i = 0, size = oat_writers_.size(); i != size; ++i) {
+ rodata_.push_back(elf_writers_[i]->StartRoData());
+ // Unzip or copy dex files straight to the oat file.
+ std::unique_ptr<MemMap> opened_dex_files_map;
+ std::vector<std::unique_ptr<const DexFile>> opened_dex_files;
+ if (!oat_writers_[i]->WriteAndOpenDexFiles(rodata_.back(),
+ oat_files_[i].get(),
+ instruction_set_,
+ instruction_set_features_.get(),
+ key_value_store_.get(),
+ &opened_dex_files_map,
+ &opened_dex_files)) {
+ return false;
+ }
+ dex_files_per_oat_file_.push_back(MakeNonOwningPointerVector(opened_dex_files));
+ if (opened_dex_files_map != nullptr) {
+ opened_dex_files_maps_.push_back(std::move(opened_dex_files_map));
+ for (std::unique_ptr<const DexFile>& dex_file : opened_dex_files) {
+ dex_file_oat_filename_map_.emplace(dex_file.get(), oat_filenames_[i]);
+ opened_dex_files_.push_back(std::move(dex_file));
+ }
+ } else {
+ DCHECK(opened_dex_files.empty());
+ }
+ }
+ }
+
+ dex_files_ = MakeNonOwningPointerVector(opened_dex_files_);
+ if (IsBootImage()) {
+ // For boot image, pass opened dex files to the Runtime::Create().
+ // Note: Runtime acquires ownership of these dex files.
+ runtime_options.Set(RuntimeArgumentMap::BootClassPathDexList, &opened_dex_files_);
if (!CreateRuntime(std::move(runtime_options))) {
return false;
}
}
- // Runtime::Create acquired the mutator_lock_ that is normally given away when we
- // Runtime::Start, give it away now so that we don't starve GC.
- Thread* self = Thread::Current();
- self->TransitionFromRunnableToSuspended(kNative);
// If we're doing the image, override the compiler filter to force full compilation. Must be
// done ahead of WellKnownClasses::Init that causes verification. Note: doesn't force
// compilation of class initializers.
// Whilst we're in native take the opportunity to initialize well known classes.
+ Thread* self = Thread::Current();
WellKnownClasses::Init(self->GetJniEnv());
ClassLinker* const class_linker = Runtime::Current()->GetClassLinker();
- if (boot_image_filename_.empty()) {
- dex_files_ = class_linker->GetBootClassPath();
- // Prune invalid dex locations.
- for (size_t i = 0; i < dex_locations_.size(); i++) {
- const char* dex_location = dex_locations_[i];
- bool contains = false;
- for (const DexFile* dex_file : dex_files_) {
- if (strcmp(dex_location, dex_file->GetLocation().c_str()) == 0) {
- contains = true;
- break;
- }
- }
- if (!contains) {
- dex_locations_.erase(dex_locations_.begin() + i);
- i--;
- }
- }
- } else {
- TimingLogger::ScopedTiming t_dex("Opening dex files", timings_);
- if (dex_filenames_.empty()) {
- ATRACE_BEGIN("Opening zip archive from file descriptor");
- std::string error_msg;
- std::unique_ptr<ZipArchive> zip_archive(ZipArchive::OpenFromFd(zip_fd_,
- zip_location_.c_str(),
- &error_msg));
- if (zip_archive.get() == nullptr) {
- LOG(ERROR) << "Failed to open zip from file descriptor for '" << zip_location_ << "': "
- << error_msg;
- return false;
- }
- if (!DexFile::OpenFromZip(*zip_archive.get(), zip_location_, &error_msg, &opened_dex_files_)) {
- LOG(ERROR) << "Failed to open dex from file descriptor for zip file '" << zip_location_
- << "': " << error_msg;
- return false;
- }
- for (auto& dex_file : opened_dex_files_) {
- dex_files_.push_back(dex_file.get());
- }
- ATRACE_END();
- } else {
- size_t failure_count = OpenDexFiles(dex_filenames_, dex_locations_, &opened_dex_files_);
- if (failure_count > 0) {
- LOG(ERROR) << "Failed to open some dex files: " << failure_count;
- return false;
- }
- for (auto& dex_file : opened_dex_files_) {
- dex_files_.push_back(dex_file.get());
- }
- }
-
+ if (!IsBootImage()) {
constexpr bool kSaveDexInput = false;
if (kSaveDexInput) {
SaveDexInput();
}
+
+ // Handle and ClassLoader creation needs to come after Runtime::Create.
+ ScopedObjectAccess soa(self);
+
+ // Classpath: first the class-path given.
+ std::vector<const DexFile*> class_path_files = MakeNonOwningPointerVector(class_path_files_);
+
+ // Then the dex files we'll compile. Thus we'll resolve the class-path first.
+ class_path_files.insert(class_path_files.end(), dex_files_.begin(), dex_files_.end());
+
+ class_loader_ = class_linker->CreatePathClassLoader(self, class_path_files);
}
- // Ensure opened dex files are writable for dex-to-dex transformations. Also ensure that
- // the dex caches stay live since we don't want class unloading to occur during compilation.
- for (const auto& dex_file : dex_files_) {
- if (!dex_file->EnableWrite()) {
- PLOG(ERROR) << "Failed to make .dex file writeable '" << dex_file->GetLocation() << "'\n";
+
+ // Ensure opened dex files are writable for dex-to-dex transformations.
+ for (const std::unique_ptr<MemMap>& map : opened_dex_files_maps_) {
+ if (!map->Protect(PROT_READ | PROT_WRITE)) {
+ PLOG(ERROR) << "Failed to make .dex files writeable.";
+ return false;
}
+ }
+
+ // Ensure that the dex caches stay live since we don't want class unloading
+ // to occur during compilation.
+ for (const auto& dex_file : dex_files_) {
ScopedObjectAccess soa(self);
dex_caches_.push_back(soa.AddLocalReference<jobject>(
class_linker->RegisterDexFile(*dex_file, Runtime::Current()->GetLinearAlloc())));
- dex_file->CreateTypeLookupTable();
}
/*
@@ -1353,59 +1407,11 @@
return true;
}
- void CreateDexOatMappings() {
- if (oat_files_.size() > 1) {
- size_t index = 0;
- for (size_t i = 0; i < oat_files_.size(); ++i) {
- std::vector<const DexFile*> dex_files;
- if (index < dex_files_.size()) {
- dex_files.push_back(dex_files_[index]);
- dex_file_oat_filename_map_.emplace(dex_files_[index], oat_filenames_[i]);
- index++;
- while (index < dex_files_.size() &&
- (dex_files_[index]->GetBaseLocation() == dex_files_[index - 1]->GetBaseLocation())) {
- dex_file_oat_filename_map_.emplace(dex_files_[index], oat_filenames_[i]);
- dex_files.push_back(dex_files_[index]);
- index++;
- }
- }
- dex_files_per_oat_file_.push_back(std::move(dex_files));
- }
- } else {
- dex_files_per_oat_file_.push_back(dex_files_);
- for (const DexFile* dex_file : dex_files_) {
- dex_file_oat_filename_map_.emplace(dex_file, oat_filenames_[0]);
- }
- }
- }
-
// Create and invoke the compiler driver. This will compile all the dex files.
void Compile() {
TimingLogger::ScopedTiming t("dex2oat Compile", timings_);
compiler_phases_timings_.reset(new CumulativeLogger("compilation times"));
- // Handle and ClassLoader creation needs to come after Runtime::Create
- jobject class_loader = nullptr;
- Thread* self = Thread::Current();
-
- if (!boot_image_filename_.empty()) {
- ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
- OpenClassPathFiles(runtime_->GetClassPathString(), dex_files_, &class_path_files_);
- ScopedObjectAccess soa(self);
-
- // Classpath: first the class-path given.
- std::vector<const DexFile*> class_path_files = MakeNonOwningPointerVector(class_path_files_);
-
- // Store the classpath we have right now.
- key_value_store_->Put(OatHeader::kClassPathKey,
- OatFile::EncodeDexFileDependencies(class_path_files));
-
- // Then the dex files we'll compile. Thus we'll resolve the class-path first.
- class_path_files.insert(class_path_files.end(), dex_files_.begin(), dex_files_.end());
-
- class_loader = class_linker->CreatePathClassLoader(self, class_path_files);
- }
-
// Find the dex file we should not inline from.
// For now, on the host always have core-oj removed.
@@ -1453,49 +1459,6 @@
}
}
- if (IsBootImage() && image_filenames_.size() > 1) {
- // If we're compiling the boot image, store the boot classpath into the Key-Value store. If
- // the image filename was adapted (e.g., for our tests), we need to change this here, too, but
- // need to strip all path components (they will be re-established when loading).
- // We need this for the multi-image case.
- std::ostringstream bootcp_oss;
- bool first_bootcp = true;
- for (size_t i = 0; i < dex_locations_.size(); ++i) {
- if (!first_bootcp) {
- bootcp_oss << ":";
- }
-
- std::string dex_loc = dex_locations_[i];
- std::string image_filename = image_filenames_[i];
-
- // Use the dex_loc path, but the image_filename name (without path elements).
- size_t dex_last_slash = dex_loc.rfind('/');
-
- // npos is max(size_t). That makes this a bit ugly.
- size_t image_last_slash = image_filename.rfind('/');
- size_t image_last_at = image_filename.rfind('@');
- size_t image_last_sep = (image_last_slash == std::string::npos)
- ? image_last_at
- : (image_last_at == std::string::npos)
- ? std::string::npos
- : std::max(image_last_slash, image_last_at);
- // Note: whenever image_last_sep == npos, +1 overflow means using the full string.
-
- if (dex_last_slash == std::string::npos) {
- dex_loc = image_filename.substr(image_last_sep + 1);
- } else {
- dex_loc = dex_loc.substr(0, dex_last_slash + 1) +
- image_filename.substr(image_last_sep + 1);
- }
-
- // Image filenames already end with .art, no need to replace.
-
- bootcp_oss << dex_loc;
- first_bootcp = false;
- }
- key_value_store_->Put(OatHeader::kBootClassPath, bootcp_oss.str());
- }
-
driver_.reset(new CompilerDriver(compiler_options_.get(),
verification_results_.get(),
&method_inliner_map_,
@@ -1516,7 +1479,7 @@
&dex_file_oat_filename_map_,
profile_compilation_info_.get()));
driver_->SetDexFilesForOatFile(dex_files_);
- driver_->CompileAll(class_loader, dex_files_, timings_);
+ driver_->CompileAll(class_loader_, dex_files_, timings_);
}
// Notes on the interleaving of creating the images and oat files to
@@ -1584,19 +1547,18 @@
// ImageWriter, if necessary.
// Note: Flushing (and closing) the file is the caller's responsibility, except for the failure
// case (when the file will be explicitly erased).
- bool CreateOatFiles() {
- CHECK(key_value_store_.get() != nullptr);
-
+ bool WriteOatFiles() {
TimingLogger::ScopedTiming t("dex2oat Oat", timings_);
- std::vector<std::unique_ptr<OatWriter>> oat_writers;
- {
- TimingLogger::ScopedTiming t2("dex2oat OatWriter", timings_);
- std::string image_file_location;
- uint32_t image_file_location_oat_checksum = 0;
- uintptr_t image_file_location_oat_data_begin = 0;
- int32_t image_patch_delta = 0;
+ // Sync the data to the file, in case we did dex2dex transformations.
+ for (const std::unique_ptr<MemMap>& map : opened_dex_files_maps_) {
+ if (!map->Sync()) {
+ PLOG(ERROR) << "Failed to Sync() dex2dex output. Map: " << map->GetName();
+ return false;
+ }
+ }
+ if (IsImage()) {
if (app_image_ && image_base_ == 0) {
std::vector<gc::space::ImageSpace*> image_spaces =
Runtime::Current()->GetHeap()->GetBootImageSpaces();
@@ -1608,47 +1570,15 @@
VLOG(compiler) << "App image base=" << reinterpret_cast<void*>(image_base_);
}
- if (IsImage()) {
- PrepareImageWriter(image_base_);
- }
+ image_writer_.reset(new ImageWriter(*driver_,
+ image_base_,
+ compiler_options_->GetCompilePic(),
+ IsAppImage(),
+ image_storage_mode_,
+ oat_filenames_,
+ dex_file_oat_filename_map_));
- if (!IsBootImage()) {
- TimingLogger::ScopedTiming t3("Loading image checksum", timings_);
- std::vector<gc::space::ImageSpace*> image_spaces =
- Runtime::Current()->GetHeap()->GetBootImageSpaces();
- image_file_location_oat_checksum = image_spaces[0]->GetImageHeader().GetOatChecksum();
- image_file_location_oat_data_begin =
- reinterpret_cast<uintptr_t>(image_spaces[0]->GetImageHeader().GetOatDataBegin());
- image_patch_delta = image_spaces[0]->GetImageHeader().GetPatchDelta();
- std::vector<std::string> image_filenames;
- for (const gc::space::ImageSpace* image_space : image_spaces) {
- image_filenames.push_back(image_space->GetImageFilename());
- }
- image_file_location = Join(image_filenames, ':');
- }
-
- if (!image_file_location.empty()) {
- key_value_store_->Put(OatHeader::kImageLocationKey, image_file_location);
- }
-
- for (size_t i = 0; i < oat_files_.size(); ++i) {
- std::vector<const DexFile*>& dex_files = dex_files_per_oat_file_[i];
- std::unique_ptr<OatWriter> oat_writer(new OatWriter(dex_files,
- image_file_location_oat_checksum,
- image_file_location_oat_data_begin,
- image_patch_delta,
- driver_.get(),
- image_writer_.get(),
- IsBootImage(),
- timings_,
- key_value_store_.get()));
- oat_writers.push_back(std::move(oat_writer));
- }
- }
-
- if (IsImage()) {
- // The OatWriter constructor has already updated offsets in methods and we need to
- // prepare method offsets in the image address space for direct method patching.
+ // We need to prepare method offsets in the image address space for direct method patching.
TimingLogger::ScopedTiming t2("dex2oat Prepare image address space", timings_);
if (!image_writer_->PrepareImageAddressSpace()) {
LOG(ERROR) << "Failed to prepare image address space.";
@@ -1658,20 +1588,22 @@
{
TimingLogger::ScopedTiming t2("dex2oat Write ELF", timings_);
- for (size_t i = 0; i < oat_files_.size(); ++i) {
+ for (size_t i = 0, size = oat_files_.size(); i != size; ++i) {
std::unique_ptr<File>& oat_file = oat_files_[i];
- std::unique_ptr<OatWriter>& oat_writer = oat_writers[i];
- std::unique_ptr<ElfWriter> elf_writer =
- CreateElfWriterQuick(instruction_set_, compiler_options_.get(), oat_file.get());
+ std::unique_ptr<ElfWriter>& elf_writer = elf_writers_[i];
+ std::unique_ptr<OatWriter>& oat_writer = oat_writers_[i];
- elf_writer->Start();
+ std::vector<const DexFile*>& dex_files = dex_files_per_oat_file_[i];
+ oat_writer->PrepareLayout(driver_.get(), image_writer_.get(), dex_files);
- OutputStream* rodata = elf_writer->StartRoData();
+ OutputStream*& rodata = rodata_[i];
+ DCHECK(rodata != nullptr);
if (!oat_writer->WriteRodata(rodata)) {
LOG(ERROR) << "Failed to write .rodata section to the ELF file " << oat_file->GetPath();
return false;
}
elf_writer->EndRoData(rodata);
+ rodata = nullptr;
OutputStream* text = elf_writer->StartText();
if (!oat_writer->WriteCode(text)) {
@@ -1680,6 +1612,14 @@
}
elf_writer->EndText(text);
+ if (!oat_writer->WriteHeader(elf_writer->GetStream(),
+ image_file_location_oat_checksum_,
+ image_file_location_oat_data_begin_,
+ image_patch_delta_)) {
+ LOG(ERROR) << "Failed to write oat header to the ELF file " << oat_file->GetPath();
+ return false;
+ }
+
elf_writer->SetBssSize(oat_writer->GetBssSize());
elf_writer->WriteDynamicSection();
elf_writer->WriteDebugInfo(oat_writer->GetMethodDebugInfo());
@@ -1705,6 +1645,9 @@
}
VLOG(compiler) << "Oat file written successfully: " << oat_filenames_[i];
+
+ oat_writer.reset();
+ elf_writer.reset();
}
}
@@ -1852,65 +1795,78 @@
return result;
}
- static size_t OpenDexFiles(std::vector<const char*>& dex_filenames,
- std::vector<const char*>& dex_locations,
- std::vector<std::unique_ptr<const DexFile>>* dex_files) {
- DCHECK(dex_files != nullptr) << "OpenDexFiles out-param is nullptr";
- size_t failure_count = 0;
- for (size_t i = 0; i < dex_filenames.size(); i++) {
- const char* dex_filename = dex_filenames[i];
- const char* dex_location = dex_locations[i];
- ATRACE_BEGIN(StringPrintf("Opening dex file '%s'", dex_filenames[i]).c_str());
- std::string error_msg;
- if (!OS::FileExists(dex_filename)) {
- LOG(WARNING) << "Skipping non-existent dex file '" << dex_filename << "'";
- dex_filenames.erase(dex_filenames.begin() + i);
- dex_locations.erase(dex_locations.begin() + i);
- i--;
- continue;
+ std::string GetMultiImageBootClassPath() {
+ DCHECK(IsBootImage());
+ DCHECK_GT(oat_filenames_.size(), 1u);
+ // If the image filename was adapted (e.g., for our tests), we need to change this here,
+ // too, but need to strip all path components (they will be re-established when loading).
+ std::ostringstream bootcp_oss;
+ bool first_bootcp = true;
+ for (size_t i = 0; i < dex_locations_.size(); ++i) {
+ if (!first_bootcp) {
+ bootcp_oss << ":";
}
- if (!DexFile::Open(dex_filename, dex_location, &error_msg, dex_files)) {
- LOG(WARNING) << "Failed to open .dex from file '" << dex_filename << "': " << error_msg;
- ++failure_count;
+
+ std::string dex_loc = dex_locations_[i];
+ std::string image_filename = image_filenames_[i];
+
+ // Use the dex_loc path, but the image_filename name (without path elements).
+ size_t dex_last_slash = dex_loc.rfind('/');
+
+ // npos is max(size_t). That makes this a bit ugly.
+ size_t image_last_slash = image_filename.rfind('/');
+ size_t image_last_at = image_filename.rfind('@');
+ size_t image_last_sep = (image_last_slash == std::string::npos)
+ ? image_last_at
+ : (image_last_at == std::string::npos)
+ ? std::string::npos
+ : std::max(image_last_slash, image_last_at);
+ // Note: whenever image_last_sep == npos, +1 overflow means using the full string.
+
+ if (dex_last_slash == std::string::npos) {
+ dex_loc = image_filename.substr(image_last_sep + 1);
+ } else {
+ dex_loc = dex_loc.substr(0, dex_last_slash + 1) +
+ image_filename.substr(image_last_sep + 1);
}
- ATRACE_END();
+
+ // Image filenames already end with .art, no need to replace.
+
+ bootcp_oss << dex_loc;
+ first_bootcp = false;
}
- return failure_count;
+ return bootcp_oss.str();
}
- // Returns true if dex_files has a dex with the named location. We compare canonical locations,
- // so that relative and absolute paths will match. Not caching for the dex_files isn't very
- // efficient, but under normal circumstances the list is neither large nor is this part too
- // sensitive.
- static bool DexFilesContains(const std::vector<const DexFile*>& dex_files,
- const std::string& location) {
- std::string canonical_location(DexFile::GetDexCanonicalLocation(location.c_str()));
- for (size_t i = 0; i < dex_files.size(); ++i) {
- if (DexFile::GetDexCanonicalLocation(dex_files[i]->GetLocation().c_str()) ==
- canonical_location) {
- return true;
- }
+ std::vector<std::string> GetClassPathLocations(const std::string& class_path) {
+ // This function is used only for apps and for an app we have exactly one oat file.
+ DCHECK(!IsBootImage());
+ DCHECK_EQ(oat_writers_.size(), 1u);
+ std::vector<std::string> dex_files_canonical_locations;
+ for (const char* location : oat_writers_[0]->GetSourceLocations()) {
+ dex_files_canonical_locations.push_back(DexFile::GetDexCanonicalLocation(location));
}
- return false;
- }
- // Appends to opened_dex_files any elements of class_path that dex_files
- // doesn't already contain. This will open those dex files as necessary.
- static void OpenClassPathFiles(const std::string& class_path,
- std::vector<const DexFile*> dex_files,
- std::vector<std::unique_ptr<const DexFile>>* opened_dex_files) {
- DCHECK(opened_dex_files != nullptr) << "OpenClassPathFiles out-param is nullptr";
std::vector<std::string> parsed;
Split(class_path, ':', &parsed);
- // Take Locks::mutator_lock_ so that lock ordering on the ClassLinker::dex_lock_ is maintained.
- ScopedObjectAccess soa(Thread::Current());
- for (size_t i = 0; i < parsed.size(); ++i) {
- if (DexFilesContains(dex_files, parsed[i])) {
- continue;
- }
+ auto kept_it = std::remove_if(parsed.begin(),
+ parsed.end(),
+ [dex_files_canonical_locations](const std::string& location) {
+ return ContainsElement(dex_files_canonical_locations,
+ DexFile::GetDexCanonicalLocation(location.c_str()));
+ });
+ parsed.erase(kept_it, parsed.end());
+ return parsed;
+ }
+
+ // Opens requested class path files and appends them to opened_dex_files.
+ static void OpenClassPathFiles(const std::vector<std::string>& class_path_locations,
+ std::vector<std::unique_ptr<const DexFile>>* opened_dex_files) {
+ DCHECK(opened_dex_files != nullptr) << "OpenClassPathFiles out-param is nullptr";
+ for (const std::string& location : class_path_locations) {
std::string error_msg;
- if (!DexFile::Open(parsed[i].c_str(), parsed[i].c_str(), &error_msg, opened_dex_files)) {
- LOG(WARNING) << "Failed to open dex file '" << parsed[i] << "': " << error_msg;
+ if (!DexFile::Open(location.c_str(), location.c_str(), &error_msg, opened_dex_files)) {
+ LOG(WARNING) << "Failed to open dex file '" << location << "': " << error_msg;
}
}
}
@@ -1985,6 +1941,63 @@
return true;
}
+ void PruneNonExistentDexFiles() {
+ DCHECK_EQ(dex_filenames_.size(), dex_locations_.size());
+ size_t kept = 0u;
+ for (size_t i = 0, size = dex_filenames_.size(); i != size; ++i) {
+ if (!OS::FileExists(dex_filenames_[i])) {
+ LOG(WARNING) << "Skipping non-existent dex file '" << dex_filenames_[i] << "'";
+ } else {
+ dex_filenames_[kept] = dex_filenames_[i];
+ dex_locations_[kept] = dex_locations_[i];
+ ++kept;
+ }
+ }
+ dex_filenames_.resize(kept);
+ dex_locations_.resize(kept);
+ }
+
+ bool AddDexFileSources() {
+ TimingLogger::ScopedTiming t2("AddDexFileSources", timings_);
+ if (zip_fd_ != -1) {
+ DCHECK_EQ(oat_writers_.size(), 1u);
+ if (!oat_writers_[0]->AddZippedDexFilesSource(ScopedFd(zip_fd_), zip_location_.c_str())) {
+ return false;
+ }
+ } else if (oat_writers_.size() > 1u) {
+ // Multi-image.
+ DCHECK_EQ(oat_writers_.size(), dex_filenames_.size());
+ DCHECK_EQ(oat_writers_.size(), dex_locations_.size());
+ for (size_t i = 0, size = oat_writers_.size(); i != size; ++i) {
+ if (!oat_writers_[i]->AddDexFileSource(dex_filenames_[i], dex_locations_[i])) {
+ return false;
+ }
+ }
+ } else {
+ DCHECK_EQ(oat_writers_.size(), 1u);
+ DCHECK_EQ(dex_filenames_.size(), dex_locations_.size());
+ DCHECK_NE(dex_filenames_.size(), 0u);
+ for (size_t i = 0; i != dex_filenames_.size(); ++i) {
+ if (!oat_writers_[0]->AddDexFileSource(dex_filenames_[i], dex_locations_[i])) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ void CreateOatWriters() {
+ TimingLogger::ScopedTiming t2("CreateOatWriters", timings_);
+ elf_writers_.reserve(oat_files_.size());
+ oat_writers_.reserve(oat_files_.size());
+ for (const std::unique_ptr<File>& oat_file : oat_files_) {
+ elf_writers_.emplace_back(
+ CreateElfWriterQuick(instruction_set_, compiler_options_.get(), oat_file.get()));
+ elf_writers_.back()->Start();
+ oat_writers_.emplace_back(new OatWriter(IsBootImage(), timings_));
+ }
+ }
+
void SaveDexInput() {
for (size_t i = 0; i < dex_files_.size(); ++i) {
const DexFile* dex_file = dex_files_[i];
@@ -2044,8 +2057,8 @@
}
// Create a runtime necessary for compilation.
- bool CreateRuntime(RuntimeArgumentMap&& runtime_options)
- SHARED_TRYLOCK_FUNCTION(true, Locks::mutator_lock_) {
+ bool CreateRuntime(RuntimeArgumentMap&& runtime_options) {
+ TimingLogger::ScopedTiming t_runtime("Create runtime", timings_);
if (!Runtime::Create(std::move(runtime_options))) {
LOG(ERROR) << "Failed to create runtime";
return false;
@@ -2066,18 +2079,12 @@
runtime_->GetClassLinker()->RunRootClinits();
- return true;
- }
+ // Runtime::Create acquired the mutator_lock_ that is normally given away when we
+ // Runtime::Start, give it away now so that we don't starve GC.
+ Thread* self = Thread::Current();
+ self->TransitionFromRunnableToSuspended(kNative);
- void PrepareImageWriter(uintptr_t image_base) {
- DCHECK(IsImage());
- image_writer_.reset(new ImageWriter(*driver_,
- image_base,
- compiler_options_->GetCompilePic(),
- IsAppImage(),
- image_storage_mode_,
- oat_filenames_,
- dex_file_oat_filename_map_));
+ return true;
}
// Let the ImageWriter write the image files. If we do not compile PIC, also fix up the oat files.
@@ -2256,6 +2263,9 @@
InstructionSet instruction_set_;
std::unique_ptr<const InstructionSetFeatures> instruction_set_features_;
+ uint32_t image_file_location_oat_checksum_;
+ uintptr_t image_file_location_oat_data_begin_;
+ int32_t image_patch_delta_;
std::unique_ptr<SafeMap<std::string, std::string> > key_value_store_;
std::unique_ptr<VerificationResults> verification_results_;
@@ -2263,11 +2273,11 @@
DexFileToMethodInlinerMap method_inliner_map_;
std::unique_ptr<QuickCompilerCallbacks> callbacks_;
+ std::unique_ptr<Runtime> runtime_;
+
// Ownership for the class path files.
std::vector<std::unique_ptr<const DexFile>> class_path_files_;
- std::unique_ptr<Runtime> runtime_;
-
size_t thread_count_;
uint64_t start_ns_;
std::unique_ptr<WatchDog> watchdog_;
@@ -2302,11 +2312,17 @@
std::vector<const DexFile*> dex_files_;
std::string no_inline_from_string_;
std::vector<jobject> dex_caches_;
- std::vector<std::unique_ptr<const DexFile>> opened_dex_files_;
+ jobject class_loader_;
+ std::vector<std::unique_ptr<ElfWriter>> elf_writers_;
+ std::vector<std::unique_ptr<OatWriter>> oat_writers_;
+ std::vector<OutputStream*> rodata_;
std::unique_ptr<ImageWriter> image_writer_;
std::unique_ptr<CompilerDriver> driver_;
+ std::vector<std::unique_ptr<MemMap>> opened_dex_files_maps_;
+ std::vector<std::unique_ptr<const DexFile>> opened_dex_files_;
+
std::vector<std::string> verbose_methods_;
bool dump_stats_;
bool dump_passes_;
@@ -2355,7 +2371,7 @@
static int CompileImage(Dex2Oat& dex2oat) {
dex2oat.Compile();
- if (!dex2oat.CreateOatFiles()) {
+ if (!dex2oat.WriteOatFiles()) {
dex2oat.EraseOatFiles();
return EXIT_FAILURE;
}
@@ -2394,7 +2410,7 @@
static int CompileApp(Dex2Oat& dex2oat) {
dex2oat.Compile();
- if (!dex2oat.CreateOatFiles()) {
+ if (!dex2oat.WriteOatFiles()) {
dex2oat.EraseOatFiles();
return EXIT_FAILURE;
}
@@ -2451,6 +2467,11 @@
}
}
+ // Check early that the result of compilation can be written
+ if (!dex2oat.OpenFile()) {
+ return EXIT_FAILURE;
+ }
+
// Print the complete line when any of the following is true:
// 1) Debug build
// 2) Compiling an image
@@ -2464,11 +2485,6 @@
}
if (!dex2oat.Setup()) {
- return EXIT_FAILURE;
- }
-
- // Check early that the result of compilation can be written
- if (!dex2oat.OpenFile()) {
dex2oat.EraseOatFiles();
return EXIT_FAILURE;
}
diff --git a/runtime/dex_file.cc b/runtime/dex_file.cc
index bc8ba97..9b93c13 100644
--- a/runtime/dex_file.cc
+++ b/runtime/dex_file.cc
@@ -687,8 +687,8 @@
return nullptr;
}
-void DexFile::CreateTypeLookupTable() const {
- lookup_table_.reset(TypeLookupTable::Create(*this));
+void DexFile::CreateTypeLookupTable(uint8_t* storage) const {
+ lookup_table_.reset(TypeLookupTable::Create(*this, storage));
}
// Given a signature place the type ids into the given vector
diff --git a/runtime/dex_file.h b/runtime/dex_file.h
index 8a3db6c..facb6fd 100644
--- a/runtime/dex_file.h
+++ b/runtime/dex_file.h
@@ -1157,7 +1157,7 @@
return lookup_table_.get();
}
- void CreateTypeLookupTable() const;
+ void CreateTypeLookupTable(uint8_t* storage = nullptr) const;
private:
// Opens a .dex file
diff --git a/runtime/mem_map.cc b/runtime/mem_map.cc
index 3571edb..18c52e4 100644
--- a/runtime/mem_map.cc
+++ b/runtime/mem_map.cc
@@ -583,6 +583,10 @@
}
}
+bool MemMap::Sync() {
+ return msync(BaseBegin(), BaseSize(), MS_SYNC) == 0;
+}
+
bool MemMap::Protect(int prot) {
if (base_begin_ == nullptr && base_size_ == 0) {
prot_ = prot;
diff --git a/runtime/mem_map.h b/runtime/mem_map.h
index ed21365..ebd550a 100644
--- a/runtime/mem_map.h
+++ b/runtime/mem_map.h
@@ -126,6 +126,8 @@
return name_;
}
+ bool Sync();
+
bool Protect(int prot);
void MadviseDontNeedAndZero();
diff --git a/runtime/oat.h b/runtime/oat.h
index 13fd6a4..989e3f9 100644
--- a/runtime/oat.h
+++ b/runtime/oat.h
@@ -31,7 +31,7 @@
class PACKED(4) OatHeader {
public:
static constexpr uint8_t kOatMagic[] = { 'o', 'a', 't', '\n' };
- static constexpr uint8_t kOatVersion[] = { '0', '7', '4', '\0' };
+ static constexpr uint8_t kOatVersion[] = { '0', '7', '5', '\0' };
static constexpr const char* kImageLocationKey = "image-location";
static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline";
diff --git a/runtime/oat_file.cc b/runtime/oat_file.cc
index 83e594b..3babcf7 100644
--- a/runtime/oat_file.cc
+++ b/runtime/oat_file.cc
@@ -46,6 +46,7 @@
#include "oat_file_manager.h"
#include "os.h"
#include "runtime.h"
+#include "type_lookup_table.h"
#include "utils.h"
#include "utils/dex_cache_arrays_layout-inl.h"
#include "vmap_table.h"
@@ -266,16 +267,15 @@
i);
return false;
}
-
- const char* dex_file_location_data = reinterpret_cast<const char*>(oat);
- oat += dex_file_location_size;
- if (UNLIKELY(oat > End())) {
+ if (UNLIKELY(static_cast<size_t>(End() - oat) < dex_file_location_size)) {
*error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zu with truncated dex file "
"location",
GetLocation().c_str(),
i);
return false;
}
+ const char* dex_file_location_data = reinterpret_cast<const char*>(oat);
+ oat += dex_file_location_size;
std::string dex_file_location = ResolveRelativeEncodedDexLocation(
abs_dex_location,
@@ -318,6 +318,17 @@
Size());
return false;
}
+ if (UNLIKELY(Size() - dex_file_offset < sizeof(DexFile::Header))) {
+ *error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zu for '%s' with dex file "
+ "offset %u of %zu but the size of dex file header is %zu",
+ GetLocation().c_str(),
+ i,
+ dex_file_location.c_str(),
+ dex_file_offset,
+ Size(),
+ sizeof(DexFile::Header));
+ return false;
+ }
const uint8_t* dex_file_pointer = Begin() + dex_file_offset;
if (UNLIKELY(!DexFile::IsMagicValid(dex_file_pointer))) {
@@ -339,34 +350,75 @@
return false;
}
const DexFile::Header* header = reinterpret_cast<const DexFile::Header*>(dex_file_pointer);
+ if (Size() - dex_file_offset < header->file_size_) {
+ *error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zu for '%s' with dex file "
+ "offset %u and size %u truncated at %zu",
+ GetLocation().c_str(),
+ i,
+ dex_file_location.c_str(),
+ dex_file_offset,
+ header->file_size_,
+ Size());
+ return false;
+ }
- if (UNLIKELY(oat > End())) {
- *error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zd for '%s' with truncated "
- "lookup table offset", GetLocation().c_str(), i,
+ uint32_t class_offsets_offset;
+ if (UNLIKELY(!ReadOatDexFileData(*this, &oat, &class_offsets_offset))) {
+ *error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zu for '%s' truncated "
+ "after class offsets offset",
+ GetLocation().c_str(),
+ i,
dex_file_location.c_str());
return false;
}
- uint32_t lookup_table_offset = *reinterpret_cast<const uint32_t*>(oat);
- oat += sizeof(lookup_table_offset);
- if (Begin() + lookup_table_offset > End()) {
- *error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zd for '%s' with truncated "
- "lookup table", GetLocation().c_str(), i,
+ if (UNLIKELY(class_offsets_offset > Size()) ||
+ UNLIKELY((Size() - class_offsets_offset) / sizeof(uint32_t) < header->class_defs_size_)) {
+ *error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zu for '%s' with truncated "
+ "class offsets, offset %u of %zu, class defs %u",
+ GetLocation().c_str(),
+ i,
+ dex_file_location.c_str(),
+ class_offsets_offset,
+ Size(),
+ header->class_defs_size_);
+ return false;
+ }
+ if (UNLIKELY(!IsAligned<alignof(uint32_t)>(class_offsets_offset))) {
+ *error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zu for '%s' with unaligned "
+ "class offsets, offset %u",
+ GetLocation().c_str(),
+ i,
+ dex_file_location.c_str(),
+ class_offsets_offset);
+ return false;
+ }
+ const uint32_t* class_offsets_pointer =
+ reinterpret_cast<const uint32_t*>(Begin() + class_offsets_offset);
+
+ uint32_t lookup_table_offset;
+ if (UNLIKELY(!ReadOatDexFileData(*this, &oat, &lookup_table_offset))) {
+ *error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zd for '%s' truncated "
+ "after lookup table offset",
+ GetLocation().c_str(),
+ i,
dex_file_location.c_str());
return false;
}
const uint8_t* lookup_table_data = lookup_table_offset != 0u
? Begin() + lookup_table_offset
: nullptr;
-
- const uint32_t* methods_offsets_pointer = reinterpret_cast<const uint32_t*>(oat);
-
- oat += (sizeof(*methods_offsets_pointer) * header->class_defs_size_);
- if (UNLIKELY(oat > End())) {
+ if (lookup_table_offset != 0u &&
+ (UNLIKELY(lookup_table_offset > Size()) ||
+ UNLIKELY(Size() - lookup_table_offset <
+ TypeLookupTable::RawDataLength(header->class_defs_size_)))) {
*error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zu for '%s' with truncated "
- "method offsets",
+ "type lookup table, offset %u of %zu, class defs %u",
GetLocation().c_str(),
i,
- dex_file_location.c_str());
+ dex_file_location.c_str(),
+ lookup_table_offset,
+ Size(),
+ header->class_defs_size_);
return false;
}
@@ -398,7 +450,7 @@
dex_file_checksum,
dex_file_pointer,
lookup_table_data,
- methods_offsets_pointer,
+ class_offsets_pointer,
current_dex_cache_arrays);
oat_dex_files_storage_.push_back(oat_dex_file);
diff --git a/runtime/parsed_options.cc b/runtime/parsed_options.cc
index 2b92303..341be9a 100644
--- a/runtime/parsed_options.cc
+++ b/runtime/parsed_options.cc
@@ -309,8 +309,8 @@
const std::string option(options[i].first);
// TODO: support -Djava.class.path
if (option == "bootclasspath") {
- auto boot_class_path
- = reinterpret_cast<const std::vector<const DexFile*>*>(options[i].second);
+ auto boot_class_path = static_cast<std::vector<std::unique_ptr<const DexFile>>*>(
+ const_cast<void*>(options[i].second));
if (runtime_options != nullptr) {
runtime_options->Set(M::BootClassPathDexList, boot_class_path);
diff --git a/runtime/runtime.cc b/runtime/runtime.cc
index c4694ee..5b075b7 100644
--- a/runtime/runtime.cc
+++ b/runtime/runtime.cc
@@ -1131,10 +1131,14 @@
}
std::vector<std::unique_ptr<const DexFile>> boot_class_path;
- OpenDexFiles(dex_filenames,
- dex_locations,
- runtime_options.GetOrDefault(Opt::Image),
- &boot_class_path);
+ if (runtime_options.Exists(Opt::BootClassPathDexList)) {
+ boot_class_path.swap(*runtime_options.GetOrDefault(Opt::BootClassPathDexList));
+ } else {
+ OpenDexFiles(dex_filenames,
+ dex_locations,
+ runtime_options.GetOrDefault(Opt::Image),
+ &boot_class_path);
+ }
instruction_set_ = runtime_options.GetOrDefault(Opt::ImageInstructionSet);
std::string error_msg;
if (!class_linker_->InitWithoutImage(std::move(boot_class_path), &error_msg)) {
diff --git a/runtime/runtime_options.cc b/runtime/runtime_options.cc
index c54461e..e75481c 100644
--- a/runtime/runtime_options.cc
+++ b/runtime/runtime_options.cc
@@ -13,8 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
#include "runtime_options.h"
+#include <memory>
+
#include "gc/heap.h"
#include "monitor.h"
#include "runtime.h"
diff --git a/runtime/runtime_options.def b/runtime/runtime_options.def
index 5624285..c5b009d 100644
--- a/runtime/runtime_options.def
+++ b/runtime/runtime_options.def
@@ -117,8 +117,8 @@
// Not parse-able from command line, but can be provided explicitly.
// (Do not add anything here that is defined in ParsedOptions::MakeParser)
-RUNTIME_OPTIONS_KEY (const std::vector<const DexFile*>*, \
- BootClassPathDexList) // TODO: make unique_ptr
+RUNTIME_OPTIONS_KEY (std::vector<std::unique_ptr<const DexFile>>*, \
+ BootClassPathDexList)
RUNTIME_OPTIONS_KEY (InstructionSet, ImageInstructionSet, kRuntimeISA)
RUNTIME_OPTIONS_KEY (CompilerCallbacks*, CompilerCallbacksPtr) // TDOO: make unique_ptr
RUNTIME_OPTIONS_KEY (bool (*)(), HookIsSensitiveThread)
diff --git a/runtime/type_lookup_table.cc b/runtime/type_lookup_table.cc
index 0d40bb7..fc9faec 100644
--- a/runtime/type_lookup_table.cc
+++ b/runtime/type_lookup_table.cc
@@ -16,6 +16,7 @@
#include "type_lookup_table.h"
+#include "base/bit_utils.h"
#include "dex_file-inl.h"
#include "utf-inl.h"
#include "utils.h"
@@ -42,25 +43,39 @@
}
uint32_t TypeLookupTable::RawDataLength(const DexFile& dex_file) {
- return RoundUpToPowerOfTwo(dex_file.NumClassDefs()) * sizeof(Entry);
+ return RawDataLength(dex_file.NumClassDefs());
}
-TypeLookupTable* TypeLookupTable::Create(const DexFile& dex_file) {
+uint32_t TypeLookupTable::RawDataLength(uint32_t num_class_defs) {
+ return SupportedSize(num_class_defs) ? RoundUpToPowerOfTwo(num_class_defs) * sizeof(Entry) : 0u;
+}
+
+uint32_t TypeLookupTable::CalculateMask(uint32_t num_class_defs) {
+ return SupportedSize(num_class_defs) ? RoundUpToPowerOfTwo(num_class_defs) - 1u : 0u;
+}
+
+bool TypeLookupTable::SupportedSize(uint32_t num_class_defs) {
+ return num_class_defs != 0u && num_class_defs <= std::numeric_limits<uint16_t>::max();
+}
+
+TypeLookupTable* TypeLookupTable::Create(const DexFile& dex_file, uint8_t* storage) {
const uint32_t num_class_defs = dex_file.NumClassDefs();
- return (num_class_defs == 0 || num_class_defs > std::numeric_limits<uint16_t>::max())
- ? nullptr
- : new TypeLookupTable(dex_file);
+ return SupportedSize(num_class_defs)
+ ? new TypeLookupTable(dex_file, storage)
+ : nullptr;
}
TypeLookupTable* TypeLookupTable::Open(const uint8_t* raw_data, const DexFile& dex_file) {
return new TypeLookupTable(raw_data, dex_file);
}
-TypeLookupTable::TypeLookupTable(const DexFile& dex_file)
+TypeLookupTable::TypeLookupTable(const DexFile& dex_file, uint8_t* storage)
: dex_file_(dex_file),
- mask_(RoundUpToPowerOfTwo(dex_file.NumClassDefs()) - 1),
- entries_(new Entry[mask_ + 1]),
- owns_entries_(true) {
+ mask_(CalculateMask(dex_file.NumClassDefs())),
+ entries_(storage != nullptr ? reinterpret_cast<Entry*>(storage) : new Entry[mask_ + 1]),
+ owns_entries_(storage == nullptr) {
+ static_assert(alignof(Entry) == 4u, "Expecting Entry to be 4-byte aligned.");
+ DCHECK_ALIGNED(storage, alignof(Entry));
std::vector<uint16_t> conflict_class_defs;
// The first stage. Put elements on their initial positions. If an initial position is already
// occupied then delay the insertion of the element to the second stage to reduce probing
@@ -93,7 +108,7 @@
TypeLookupTable::TypeLookupTable(const uint8_t* raw_data, const DexFile& dex_file)
: dex_file_(dex_file),
- mask_(RoundUpToPowerOfTwo(dex_file.NumClassDefs()) - 1),
+ mask_(CalculateMask(dex_file.NumClassDefs())),
entries_(reinterpret_cast<Entry*>(const_cast<uint8_t*>(raw_data))),
owns_entries_(false) {}
diff --git a/runtime/type_lookup_table.h b/runtime/type_lookup_table.h
index 3c2295c..d74d01d 100644
--- a/runtime/type_lookup_table.h
+++ b/runtime/type_lookup_table.h
@@ -60,7 +60,7 @@
}
// Method creates lookup table for dex file
- static TypeLookupTable* Create(const DexFile& dex_file);
+ static TypeLookupTable* Create(const DexFile& dex_file, uint8_t* storage = nullptr);
// Method opens lookup table from binary data. Lookup table does not owns binary data.
static TypeLookupTable* Open(const uint8_t* raw_data, const DexFile& dex_file);
@@ -76,6 +76,9 @@
// Method returns length of binary data for the specified dex file.
static uint32_t RawDataLength(const DexFile& dex_file);
+ // Method returns length of binary data for the specified number of class definitions.
+ static uint32_t RawDataLength(uint32_t num_class_defs);
+
private:
/**
* To find element we need to compare strings.
@@ -109,8 +112,11 @@
}
};
+ static uint32_t CalculateMask(uint32_t num_class_defs);
+ static bool SupportedSize(uint32_t num_class_defs);
+
// Construct from a dex file.
- explicit TypeLookupTable(const DexFile& dex_file);
+ explicit TypeLookupTable(const DexFile& dex_file, uint8_t* storage);
// Construct from a dex file with existing data.
TypeLookupTable(const uint8_t* raw_data, const DexFile& dex_file);
