Merge "Fix null pointer in processing of enum annotations."
diff --git a/build/Android.common_test.mk b/build/Android.common_test.mk
index 420db43..edf107e 100644
--- a/build/Android.common_test.mk
+++ b/build/Android.common_test.mk
@@ -27,6 +27,10 @@
# rule name such as test-art-host-oat-optimizing-HelloWorld64.
ART_TEST_KNOWN_BROKEN :=
+# List of run-tests to skip running in any configuration. This needs to be the full name of the
+# run-test such as '457-regs'.
+ART_TEST_RUN_TEST_SKIP ?=
+
# Failing valgrind tests.
# Note: *all* 64b tests involving the runtime do not work currently. b/15170219.
diff --git a/build/Android.gtest.mk b/build/Android.gtest.mk
index 0afec2d..dcde5ab 100644
--- a/build/Android.gtest.mk
+++ b/build/Android.gtest.mk
@@ -188,6 +188,7 @@
runtime/gc/accounting/card_table_test.cc \
runtime/gc/accounting/mod_union_table_test.cc \
runtime/gc/accounting/space_bitmap_test.cc \
+ runtime/gc/collector/immune_spaces_test.cc \
runtime/gc/heap_test.cc \
runtime/gc/reference_queue_test.cc \
runtime/gc/space/dlmalloc_space_base_test.cc \
diff --git a/cmdline/cmdline_parser_test.cc b/cmdline/cmdline_parser_test.cc
index f34b5ed..529143d 100644
--- a/cmdline/cmdline_parser_test.cc
+++ b/cmdline/cmdline_parser_test.cc
@@ -457,8 +457,10 @@
EXPECT_SINGLE_PARSE_VALUE(false, "-Xusejit:false", M::UseJIT);
}
{
- EXPECT_SINGLE_PARSE_VALUE(MemoryKiB(16 * KB), "-Xjitcodecachesize:16K", M::JITCodeCacheCapacity);
- EXPECT_SINGLE_PARSE_VALUE(MemoryKiB(16 * MB), "-Xjitcodecachesize:16M", M::JITCodeCacheCapacity);
+ EXPECT_SINGLE_PARSE_VALUE(
+ MemoryKiB(16 * KB), "-Xjitinitialsize:16K", M::JITCodeCacheInitialCapacity);
+ EXPECT_SINGLE_PARSE_VALUE(
+ MemoryKiB(16 * MB), "-Xjitmaxsize:16M", M::JITCodeCacheMaxCapacity);
}
{
EXPECT_SINGLE_PARSE_VALUE(12345u, "-Xjitthreshold:12345", M::JITCompileThreshold);
diff --git a/compiler/common_compiler_test.cc b/compiler/common_compiler_test.cc
index c37ceca..e6cc50c 100644
--- a/compiler/common_compiler_test.cc
+++ b/compiler/common_compiler_test.cc
@@ -223,6 +223,11 @@
compiler_kind_ = compiler_kind;
}
+InstructionSet CommonCompilerTest::GetInstructionSet() const {
+ DCHECK(compiler_driver_.get() != nullptr);
+ return compiler_driver_->GetInstructionSet();
+}
+
void CommonCompilerTest::TearDown() {
timer_.reset();
compiler_driver_.reset();
diff --git a/compiler/common_compiler_test.h b/compiler/common_compiler_test.h
index 67b4428..7b0e5af 100644
--- a/compiler/common_compiler_test.h
+++ b/compiler/common_compiler_test.h
@@ -61,6 +61,8 @@
Compiler::Kind GetCompilerKind() const;
void SetCompilerKind(Compiler::Kind compiler_kind);
+ InstructionSet GetInstructionSet() const;
+
// Get the set of image classes given to the compiler-driver in SetUp. Note: the compiler
// driver assumes ownership of the set, so the test should properly release the set.
virtual std::unordered_set<std::string>* GetImageClasses();
@@ -115,6 +117,31 @@
return; \
}
+// TODO: When read barrier works with all compilers in use, get rid of this.
+#define TEST_DISABLED_FOR_READ_BARRIER_WITH_QUICK() \
+ if (kUseReadBarrier && GetCompilerKind() == Compiler::kQuick) { \
+ printf("WARNING: TEST DISABLED FOR READ BARRIER WITH QUICK\n"); \
+ return; \
+ }
+
+// TODO: When read barrier works with all Optimizing back ends, get rid of this.
+#define TEST_DISABLED_FOR_READ_BARRIER_WITH_OPTIMIZING_FOR_UNSUPPORTED_INSTRUCTION_SETS() \
+ if (kUseReadBarrier && GetCompilerKind() == Compiler::kOptimizing) { \
+ switch (GetInstructionSet()) { \
+ case kThumb2: \
+ case kX86: \
+ case kX86_64: \
+ /* Instruction set has read barrier support. */ \
+ break; \
+ \
+ default: \
+ /* Instruction set does not have barrier support. */ \
+ printf("WARNING: TEST DISABLED FOR READ BARRIER WITH OPTIMIZING " \
+ "FOR THIS INSTRUCTION SET\n"); \
+ return; \
+ } \
+ }
+
// TODO: When non-PIC works with all compilers in use, get rid of this.
#define TEST_DISABLED_FOR_NON_PIC_COMPILING_WITH_OPTIMIZING() \
if (GetCompilerKind() == Compiler::kOptimizing) { \
diff --git a/compiler/dex/quick/gen_common.cc b/compiler/dex/quick/gen_common.cc
index 2b60a51..5da7214 100644
--- a/compiler/dex/quick/gen_common.cc
+++ b/compiler/dex/quick/gen_common.cc
@@ -1104,7 +1104,11 @@
// access because the verifier was unable to?
const DexFile* dex_file = cu_->dex_file;
CompilerDriver* driver = cu_->compiler_driver;
- if (driver->CanAccessInstantiableTypeWithoutChecks(cu_->method_idx, *dex_file, type_idx)) {
+ bool finalizable;
+ if (driver->CanAccessInstantiableTypeWithoutChecks(cu_->method_idx,
+ *dex_file,
+ type_idx,
+ &finalizable)) {
bool is_type_initialized;
bool use_direct_type_ptr;
uintptr_t direct_type_ptr;
diff --git a/compiler/dex/quick/quick_compiler.cc b/compiler/dex/quick/quick_compiler.cc
index 6673ea8..05dde9f 100644
--- a/compiler/dex/quick/quick_compiler.cc
+++ b/compiler/dex/quick/quick_compiler.cc
@@ -673,6 +673,12 @@
return nullptr;
}
+ if (kEmitCompilerReadBarrier) {
+ VLOG(compiler) << "Skipping method : " << PrettyMethod(method_idx, dex_file)
+ << " Reason = Quick does not support read barrier.";
+ return nullptr;
+ }
+
// TODO: check method fingerprint here to determine appropriate backend type. Until then, use
// build default.
CompilerDriver* driver = GetCompilerDriver();
diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc
index bf3a865..e42a737 100644
--- a/compiler/driver/compiler_driver.cc
+++ b/compiler/driver/compiler_driver.cc
@@ -1208,7 +1208,8 @@
bool CompilerDriver::CanAccessInstantiableTypeWithoutChecks(uint32_t referrer_idx,
const DexFile& dex_file,
- uint32_t type_idx) {
+ uint32_t type_idx,
+ bool* finalizable) {
ScopedObjectAccess soa(Thread::Current());
mirror::DexCache* dex_cache = Runtime::Current()->GetClassLinker()->FindDexCache(
soa.Self(), dex_file, false);
@@ -1216,8 +1217,11 @@
mirror::Class* resolved_class = dex_cache->GetResolvedType(type_idx);
if (resolved_class == nullptr) {
stats_->TypeNeedsAccessCheck();
+ // Be conservative.
+ *finalizable = true;
return false; // Unknown class needs access checks.
}
+ *finalizable = resolved_class->IsFinalizable();
const DexFile::MethodId& method_id = dex_file.GetMethodId(referrer_idx);
mirror::Class* referrer_class = dex_cache->GetResolvedType(method_id.class_idx_);
if (referrer_class == nullptr) {
diff --git a/compiler/driver/compiler_driver.h b/compiler/driver/compiler_driver.h
index 5683b03..dae785b 100644
--- a/compiler/driver/compiler_driver.h
+++ b/compiler/driver/compiler_driver.h
@@ -211,8 +211,11 @@
REQUIRES(!Locks::mutator_lock_);
// Are runtime access and instantiable checks necessary in the code?
- bool CanAccessInstantiableTypeWithoutChecks(uint32_t referrer_idx, const DexFile& dex_file,
- uint32_t type_idx)
+ // out_is_finalizable is set to whether the type is finalizable.
+ bool CanAccessInstantiableTypeWithoutChecks(uint32_t referrer_idx,
+ const DexFile& dex_file,
+ uint32_t type_idx,
+ bool* out_is_finalizable)
REQUIRES(!Locks::mutator_lock_);
bool CanEmbedTypeInCode(const DexFile& dex_file, uint32_t type_idx,
diff --git a/compiler/driver/compiler_driver_test.cc b/compiler/driver/compiler_driver_test.cc
index 1107599..f8de9fa 100644
--- a/compiler/driver/compiler_driver_test.cc
+++ b/compiler/driver/compiler_driver_test.cc
@@ -147,6 +147,8 @@
TEST_F(CompilerDriverTest, AbstractMethodErrorStub) {
TEST_DISABLED_FOR_HEAP_REFERENCE_POISONING_WITH_QUICK();
+ TEST_DISABLED_FOR_READ_BARRIER_WITH_QUICK();
+ TEST_DISABLED_FOR_READ_BARRIER_WITH_OPTIMIZING_FOR_UNSUPPORTED_INSTRUCTION_SETS();
jobject class_loader;
{
ScopedObjectAccess soa(Thread::Current());
@@ -193,6 +195,8 @@
TEST_F(CompilerDriverMethodsTest, Selection) {
TEST_DISABLED_FOR_HEAP_REFERENCE_POISONING_WITH_QUICK();
+ TEST_DISABLED_FOR_READ_BARRIER_WITH_QUICK();
+ TEST_DISABLED_FOR_READ_BARRIER_WITH_OPTIMIZING_FOR_UNSUPPORTED_INSTRUCTION_SETS();
Thread* self = Thread::Current();
jobject class_loader;
{
diff --git a/compiler/dwarf/debug_info_entry_writer.h b/compiler/dwarf/debug_info_entry_writer.h
index d9b367b..aa31036 100644
--- a/compiler/dwarf/debug_info_entry_writer.h
+++ b/compiler/dwarf/debug_info_entry_writer.h
@@ -20,6 +20,7 @@
#include <cstdint>
#include <unordered_map>
+#include "base/casts.h"
#include "dwarf/dwarf_constants.h"
#include "dwarf/writer.h"
#include "leb128.h"
@@ -47,9 +48,9 @@
* It also handles generation of abbreviations.
*
* Usage:
- * StartTag(DW_TAG_compile_unit, DW_CHILDREN_yes);
+ * StartTag(DW_TAG_compile_unit);
* WriteStrp(DW_AT_producer, "Compiler name", debug_str);
- * StartTag(DW_TAG_subprogram, DW_CHILDREN_no);
+ * StartTag(DW_TAG_subprogram);
* WriteStrp(DW_AT_name, "Foo", debug_str);
* EndTag();
* EndTag();
@@ -59,36 +60,40 @@
static_assert(std::is_same<typename Vector::value_type, uint8_t>::value, "Invalid value type");
public:
+ static constexpr size_t kCompilationUnitHeaderSize = 11;
+
// Start debugging information entry.
- void StartTag(Tag tag, Children children) {
- DCHECK(has_children) << "This tag can not have nested tags";
+ // Returns offset of the entry in compilation unit.
+ size_t StartTag(Tag tag) {
if (inside_entry_) {
// Write abbrev code for the previous entry.
- this->UpdateUleb128(abbrev_code_offset_, EndAbbrev());
+ // Parent entry is finalized before any children are written.
+ this->UpdateUleb128(abbrev_code_offset_, EndAbbrev(DW_CHILDREN_yes));
inside_entry_ = false;
}
- StartAbbrev(tag, children);
+ StartAbbrev(tag);
// Abbrev code placeholder of sufficient size.
abbrev_code_offset_ = this->data()->size();
this->PushUleb128(NextAbbrevCode());
depth_++;
inside_entry_ = true;
- has_children = (children == DW_CHILDREN_yes);
+ return abbrev_code_offset_ + kCompilationUnitHeaderSize;
}
// End debugging information entry.
void EndTag() {
DCHECK_GT(depth_, 0);
if (inside_entry_) {
- // Write abbrev code for this tag.
- this->UpdateUleb128(abbrev_code_offset_, EndAbbrev());
+ // Write abbrev code for this entry.
+ this->UpdateUleb128(abbrev_code_offset_, EndAbbrev(DW_CHILDREN_no));
inside_entry_ = false;
- }
- if (has_children) {
- this->PushUint8(0); // End of children.
+ // This entry has no children and so there is no terminator.
+ } else {
+ // The entry has been already finalized so it must be parent entry
+ // and we need to write the terminator required by DW_CHILDREN_yes.
+ this->PushUint8(0);
}
depth_--;
- has_children = true; // Parent tag obviously has children.
}
void WriteAddr(Attribute attrib, uint64_t value) {
@@ -101,10 +106,10 @@
}
}
- void WriteBlock(Attribute attrib, const void* ptr, int size) {
+ void WriteBlock(Attribute attrib, const void* ptr, size_t num_bytes) {
AddAbbrevAttribute(attrib, DW_FORM_block);
- this->PushUleb128(size);
- this->PushData(ptr, size);
+ this->PushUleb128(num_bytes);
+ this->PushData(ptr, num_bytes);
}
void WriteData1(Attribute attrib, uint8_t value) {
@@ -147,12 +152,12 @@
this->PushUint8(value ? 1 : 0);
}
- void WriteRef4(Attribute attrib, int cu_offset) {
+ void WriteRef4(Attribute attrib, uint32_t cu_offset) {
AddAbbrevAttribute(attrib, DW_FORM_ref4);
this->PushUint32(cu_offset);
}
- void WriteRef(Attribute attrib, int cu_offset) {
+ void WriteRef(Attribute attrib, uint32_t cu_offset) {
AddAbbrevAttribute(attrib, DW_FORM_ref_udata);
this->PushUleb128(cu_offset);
}
@@ -162,16 +167,21 @@
this->PushString(value);
}
- void WriteStrp(Attribute attrib, int address) {
+ void WriteStrp(Attribute attrib, size_t debug_str_offset) {
AddAbbrevAttribute(attrib, DW_FORM_strp);
- this->PushUint32(address);
+ this->PushUint32(dchecked_integral_cast<uint32_t>(debug_str_offset));
}
- void WriteStrp(Attribute attrib, const char* value, std::vector<uint8_t>* debug_str) {
+ void WriteStrp(Attribute attrib, const char* str, size_t len,
+ std::vector<uint8_t>* debug_str) {
AddAbbrevAttribute(attrib, DW_FORM_strp);
- int address = debug_str->size();
- debug_str->insert(debug_str->end(), value, value + strlen(value) + 1);
- this->PushUint32(address);
+ this->PushUint32(debug_str->size());
+ debug_str->insert(debug_str->end(), str, str + len);
+ debug_str->push_back(0);
+ }
+
+ void WriteStrp(Attribute attrib, const char* str, std::vector<uint8_t>* debug_str) {
+ WriteStrp(attrib, str, strlen(str), debug_str);
}
bool Is64bit() const { return is64bit_; }
@@ -180,7 +190,11 @@
return patch_locations_;
}
+ int Depth() const { return depth_; }
+
using Writer<Vector>::data;
+ using Writer<Vector>::size;
+ using Writer<Vector>::UpdateUint32;
DebugInfoEntryWriter(bool is64bitArch,
Vector* debug_abbrev,
@@ -196,16 +210,17 @@
}
~DebugInfoEntryWriter() {
+ DCHECK(!inside_entry_);
DCHECK_EQ(depth_, 0);
}
private:
// Start abbreviation declaration.
- void StartAbbrev(Tag tag, Children children) {
- DCHECK(!inside_entry_);
+ void StartAbbrev(Tag tag) {
current_abbrev_.clear();
EncodeUnsignedLeb128(¤t_abbrev_, tag);
- current_abbrev_.push_back(children);
+ has_children_offset_ = current_abbrev_.size();
+ current_abbrev_.push_back(0); // Place-holder for DW_CHILDREN.
}
// Add attribute specification.
@@ -220,8 +235,9 @@
}
// End abbreviation declaration and return its code.
- int EndAbbrev() {
- DCHECK(inside_entry_);
+ int EndAbbrev(Children has_children) {
+ DCHECK(!current_abbrev_.empty());
+ current_abbrev_[has_children_offset_] = has_children;
auto it = abbrev_codes_.insert(std::make_pair(std::move(current_abbrev_),
NextAbbrevCode()));
int abbrev_code = it.first->second;
@@ -241,6 +257,7 @@
// Fields for writing and deduplication of abbrevs.
Writer<Vector> debug_abbrev_;
Vector current_abbrev_;
+ size_t has_children_offset_ = 0;
std::unordered_map<Vector, int,
FNVHash<Vector> > abbrev_codes_;
@@ -250,7 +267,6 @@
int depth_ = 0;
size_t abbrev_code_offset_ = 0; // Location to patch once we know the code.
bool inside_entry_ = false; // Entry ends at first child (if any).
- bool has_children = true;
std::vector<uintptr_t> patch_locations_;
};
diff --git a/compiler/dwarf/dedup_vector.h b/compiler/dwarf/dedup_vector.h
new file mode 100644
index 0000000..7fb21b7
--- /dev/null
+++ b/compiler/dwarf/dedup_vector.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_COMPILER_DWARF_DEDUP_VECTOR_H_
+#define ART_COMPILER_DWARF_DEDUP_VECTOR_H_
+
+#include <vector>
+#include <unordered_map>
+
+namespace art {
+namespace dwarf {
+ class DedupVector {
+ public:
+ // Returns an offset to previously inserted identical block of data,
+ // or appends the data at the end of the vector and returns offset to it.
+ size_t Insert(const uint8_t* ptr, size_t num_bytes) {
+ // See http://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
+ uint32_t hash = 2166136261u;
+ for (size_t i = 0; i < num_bytes; i++) {
+ hash = (hash ^ ptr[i]) * 16777619u;
+ }
+ // Try to find existing copy of the data.
+ const auto& range = hash_to_offset_.equal_range(hash);
+ for (auto it = range.first; it != range.second; ++it) {
+ const size_t offset = it->second;
+ if (offset + num_bytes <= vector_.size() &&
+ memcmp(vector_.data() + offset, ptr, num_bytes) == 0) {
+ return offset;
+ }
+ }
+ // Append the data at the end of the vector.
+ const size_t new_offset = vector_.size();
+ hash_to_offset_.emplace(hash, new_offset);
+ vector_.insert(vector_.end(), ptr, ptr + num_bytes);
+ return new_offset;
+ }
+
+ const std::vector<uint8_t>& Data() const { return vector_; }
+
+ private:
+ struct IdentityHash {
+ size_t operator()(uint32_t v) const { return v; }
+ };
+
+ // We store the full hash as the key to simplify growing of the table.
+ // It avoids storing or referencing the actual data in the hash-table.
+ std::unordered_multimap<uint32_t, size_t, IdentityHash> hash_to_offset_;
+
+ std::vector<uint8_t> vector_;
+ };
+} // namespace dwarf
+} // namespace art
+
+#endif // ART_COMPILER_DWARF_DEDUP_VECTOR_H_
diff --git a/compiler/dwarf/dwarf_test.cc b/compiler/dwarf/dwarf_test.cc
index 6bb22ed..e9cd421 100644
--- a/compiler/dwarf/dwarf_test.cc
+++ b/compiler/dwarf/dwarf_test.cc
@@ -285,7 +285,7 @@
constexpr bool is64bit = false;
DebugInfoEntryWriter<> info(is64bit, &debug_abbrev_data_);
DW_CHECK("Contents of the .debug_info section:");
- info.StartTag(dwarf::DW_TAG_compile_unit, dwarf::DW_CHILDREN_yes);
+ info.StartTag(dwarf::DW_TAG_compile_unit);
DW_CHECK("Abbrev Number: 1 (DW_TAG_compile_unit)");
info.WriteStrp(dwarf::DW_AT_producer, "Compiler name", &debug_str_data_);
DW_CHECK_NEXT("DW_AT_producer : (indirect string, offset: 0x0): Compiler name");
@@ -293,7 +293,7 @@
DW_CHECK_NEXT("DW_AT_low_pc : 0x1000000");
info.WriteAddr(dwarf::DW_AT_high_pc, 0x02000000);
DW_CHECK_NEXT("DW_AT_high_pc : 0x2000000");
- info.StartTag(dwarf::DW_TAG_subprogram, dwarf::DW_CHILDREN_no);
+ info.StartTag(dwarf::DW_TAG_subprogram);
DW_CHECK("Abbrev Number: 2 (DW_TAG_subprogram)");
info.WriteStrp(dwarf::DW_AT_name, "Foo", &debug_str_data_);
DW_CHECK_NEXT("DW_AT_name : (indirect string, offset: 0xe): Foo");
@@ -302,7 +302,7 @@
info.WriteAddr(dwarf::DW_AT_high_pc, 0x01020000);
DW_CHECK_NEXT("DW_AT_high_pc : 0x1020000");
info.EndTag(); // DW_TAG_subprogram
- info.StartTag(dwarf::DW_TAG_subprogram, dwarf::DW_CHILDREN_no);
+ info.StartTag(dwarf::DW_TAG_subprogram);
DW_CHECK("Abbrev Number: 2 (DW_TAG_subprogram)");
info.WriteStrp(dwarf::DW_AT_name, "Bar", &debug_str_data_);
DW_CHECK_NEXT("DW_AT_name : (indirect string, offset: 0x12): Bar");
@@ -313,7 +313,7 @@
info.EndTag(); // DW_TAG_subprogram
info.EndTag(); // DW_TAG_compile_unit
// Test that previous list was properly terminated and empty children.
- info.StartTag(dwarf::DW_TAG_compile_unit, dwarf::DW_CHILDREN_yes);
+ info.StartTag(dwarf::DW_TAG_compile_unit);
info.EndTag(); // DW_TAG_compile_unit
// The abbrev table is just side product, but check it as well.
@@ -327,7 +327,7 @@
DW_CHECK_NEXT("DW_AT_name DW_FORM_strp");
DW_CHECK_NEXT("DW_AT_low_pc DW_FORM_addr");
DW_CHECK_NEXT("DW_AT_high_pc DW_FORM_addr");
- DW_CHECK("3 DW_TAG_compile_unit [has children]");
+ DW_CHECK("3 DW_TAG_compile_unit [no children]");
std::vector<uintptr_t> debug_info_patches;
std::vector<uintptr_t> expected_patches { 16, 20, 29, 33, 42, 46 }; // NOLINT
diff --git a/compiler/dwarf/headers.h b/compiler/dwarf/headers.h
index 633e2f7..c75aeac 100644
--- a/compiler/dwarf/headers.h
+++ b/compiler/dwarf/headers.h
@@ -138,6 +138,7 @@
writer.PushUint32(debug_abbrev_offset);
writer.PushUint8(entries.Is64bit() ? 8 : 4);
size_t entries_offset = writer.data()->size();
+ DCHECK_EQ(entries_offset, DebugInfoEntryWriter<Vector>::kCompilationUnitHeaderSize);
writer.PushData(*entries.data());
writer.UpdateUint32(start, writer.data()->size() - start - 4);
// Copy patch locations and make them relative to .debug_info section.
diff --git a/compiler/dwarf/writer.h b/compiler/dwarf/writer.h
index 00b9dfa..d2add7f 100644
--- a/compiler/dwarf/writer.h
+++ b/compiler/dwarf/writer.h
@@ -114,9 +114,9 @@
data_->insert(data_->end(), value, value + strlen(value) + 1);
}
- void PushData(const void* ptr, size_t size) {
+ void PushData(const void* ptr, size_t num_bytes) {
const char* p = reinterpret_cast<const char*>(ptr);
- data_->insert(data_->end(), p, p + size);
+ data_->insert(data_->end(), p, p + num_bytes);
}
template<typename Vector2>
@@ -164,6 +164,10 @@
return data_;
}
+ size_t size() const {
+ return data_->size();
+ }
+
explicit Writer(Vector* buffer) : data_(buffer) { }
private:
diff --git a/compiler/elf_writer_debug.cc b/compiler/elf_writer_debug.cc
index e1ab340..5e2a8bf 100644
--- a/compiler/elf_writer_debug.cc
+++ b/compiler/elf_writer_debug.cc
@@ -19,9 +19,11 @@
#include <unordered_set>
#include "base/casts.h"
+#include "base/stl_util.h"
#include "compiled_method.h"
#include "driver/compiler_driver.h"
#include "dex_file-inl.h"
+#include "dwarf/dedup_vector.h"
#include "dwarf/headers.h"
#include "dwarf/register.h"
#include "elf_builder.h"
@@ -249,10 +251,217 @@
}
}
+struct CompilationUnit {
+ std::vector<const OatWriter::DebugInfo*> methods_;
+ size_t debug_line_offset_ = 0;
+ uint32_t low_pc_ = 0xFFFFFFFFU;
+ uint32_t high_pc_ = 0;
+};
+
+// Helper class to write .debug_info and its supporting sections.
template<typename ElfTypes>
class DebugInfoWriter {
typedef typename ElfTypes::Addr Elf_Addr;
+ // Helper class to write one compilation unit.
+ // It holds helper methods and temporary state.
+ class CompilationUnitWriter {
+ public:
+ explicit CompilationUnitWriter(DebugInfoWriter* owner)
+ : owner_(owner),
+ info_(Is64BitInstructionSet(owner_->builder_->GetIsa()), &debug_abbrev_) {
+ }
+
+ void Write(const CompilationUnit& compilation_unit) {
+ CHECK(!compilation_unit.methods_.empty());
+ const Elf_Addr text_address = owner_->builder_->GetText()->GetAddress();
+
+ info_.StartTag(DW_TAG_compile_unit);
+ info_.WriteStrp(DW_AT_producer, owner_->WriteString("Android dex2oat"));
+ info_.WriteData1(DW_AT_language, DW_LANG_Java);
+ info_.WriteAddr(DW_AT_low_pc, text_address + compilation_unit.low_pc_);
+ info_.WriteAddr(DW_AT_high_pc, text_address + compilation_unit.high_pc_);
+ info_.WriteData4(DW_AT_stmt_list, compilation_unit.debug_line_offset_);
+
+ const char* last_dex_class_desc = nullptr;
+ for (auto mi : compilation_unit.methods_) {
+ const DexFile* dex = mi->dex_file_;
+ const DexFile::MethodId& dex_method = dex->GetMethodId(mi->dex_method_index_);
+ const DexFile::ProtoId& dex_proto = dex->GetMethodPrototype(dex_method);
+ const DexFile::TypeList* dex_params = dex->GetProtoParameters(dex_proto);
+ const char* dex_class_desc = dex->GetMethodDeclaringClassDescriptor(dex_method);
+
+ // Enclose the method in correct class definition.
+ if (last_dex_class_desc != dex_class_desc) {
+ if (last_dex_class_desc != nullptr) {
+ EndClassTag(last_dex_class_desc);
+ }
+ size_t offset = StartClassTag(dex_class_desc);
+ type_cache_.emplace(dex_class_desc, offset);
+ // Check that each class is defined only once.
+ bool unique = owner_->defined_dex_classes_.insert(dex_class_desc).second;
+ CHECK(unique) << "Redefinition of " << dex_class_desc;
+ last_dex_class_desc = dex_class_desc;
+ }
+
+ std::vector<const char*> param_names;
+ if (mi->code_item_ != nullptr) {
+ const uint8_t* stream = dex->GetDebugInfoStream(mi->code_item_);
+ if (stream != nullptr) {
+ DecodeUnsignedLeb128(&stream); // line.
+ uint32_t parameters_size = DecodeUnsignedLeb128(&stream);
+ for (uint32_t i = 0; i < parameters_size; ++i) {
+ uint32_t id = DecodeUnsignedLeb128P1(&stream);
+ param_names.push_back(mi->dex_file_->StringDataByIdx(id));
+ }
+ }
+ }
+
+ int start_depth = info_.Depth();
+ info_.StartTag(DW_TAG_subprogram);
+ WriteName(dex->GetMethodName(dex_method));
+ info_.WriteAddr(DW_AT_low_pc, text_address + mi->low_pc_);
+ info_.WriteAddr(DW_AT_high_pc, text_address + mi->high_pc_);
+ WriteLazyType(dex->GetReturnTypeDescriptor(dex_proto));
+ if (dex_params != nullptr) {
+ for (uint32_t i = 0; i < dex_params->Size(); ++i) {
+ info_.StartTag(DW_TAG_formal_parameter);
+ // Parameter names may not be always available.
+ if (i < param_names.size() && param_names[i] != nullptr) {
+ WriteName(param_names[i]);
+ }
+ WriteLazyType(dex->StringByTypeIdx(dex_params->GetTypeItem(i).type_idx_));
+ info_.EndTag();
+ }
+ }
+ info_.EndTag();
+ CHECK_EQ(info_.Depth(), start_depth); // Balanced start/end.
+ }
+ if (last_dex_class_desc != nullptr) {
+ EndClassTag(last_dex_class_desc);
+ }
+ CHECK_EQ(info_.Depth(), 1);
+ FinishLazyTypes();
+ info_.EndTag(); // DW_TAG_compile_unit
+ std::vector<uint8_t> buffer;
+ buffer.reserve(info_.data()->size() + KB);
+ const size_t offset = owner_->builder_->GetDebugInfo()->GetSize();
+ const size_t debug_abbrev_offset =
+ owner_->debug_abbrev_.Insert(debug_abbrev_.data(), debug_abbrev_.size());
+ WriteDebugInfoCU(debug_abbrev_offset, info_, offset, &buffer, &owner_->debug_info_patches_);
+ owner_->builder_->GetDebugInfo()->WriteFully(buffer.data(), buffer.size());
+ }
+
+ // Some types are difficult to define as we go since they need
+ // to be enclosed in the right set of namespaces. Therefore we
+ // just define all types lazily at the end of compilation unit.
+ void WriteLazyType(const char* type_descriptor) {
+ DCHECK(type_descriptor != nullptr);
+ if (type_descriptor[0] != 'V') {
+ lazy_types_.emplace(type_descriptor, info_.size());
+ info_.WriteRef4(DW_AT_type, 0);
+ }
+ }
+
+ void FinishLazyTypes() {
+ for (const auto& lazy_type : lazy_types_) {
+ info_.UpdateUint32(lazy_type.second, WriteType(lazy_type.first));
+ }
+ lazy_types_.clear();
+ }
+
+ private:
+ void WriteName(const char* name) {
+ info_.WriteStrp(DW_AT_name, owner_->WriteString(name));
+ }
+
+ // Convert dex type descriptor to DWARF.
+ // Returns offset in the compilation unit.
+ size_t WriteType(const char* desc) {
+ const auto& it = type_cache_.find(desc);
+ if (it != type_cache_.end()) {
+ return it->second;
+ }
+
+ size_t offset;
+ if (*desc == 'L') {
+ // Class type. For example: Lpackage/name;
+ offset = StartClassTag(desc);
+ info_.WriteFlag(DW_AT_declaration, true);
+ EndClassTag(desc);
+ } else if (*desc == '[') {
+ // Array type.
+ size_t element_type = WriteType(desc + 1);
+ offset = info_.StartTag(DW_TAG_array_type);
+ info_.WriteRef(DW_AT_type, element_type);
+ info_.EndTag();
+ } else {
+ // Primitive types.
+ const char* name;
+ switch (*desc) {
+ case 'B': name = "byte"; break;
+ case 'C': name = "char"; break;
+ case 'D': name = "double"; break;
+ case 'F': name = "float"; break;
+ case 'I': name = "int"; break;
+ case 'J': name = "long"; break;
+ case 'S': name = "short"; break;
+ case 'Z': name = "boolean"; break;
+ case 'V': name = "void"; break;
+ default:
+ LOG(FATAL) << "Unknown dex type descriptor: " << desc;
+ UNREACHABLE();
+ }
+ offset = info_.StartTag(DW_TAG_base_type);
+ WriteName(name);
+ info_.EndTag();
+ }
+
+ type_cache_.emplace(desc, offset);
+ return offset;
+ }
+
+ // Start DW_TAG_class_type tag nested in DW_TAG_namespace tags.
+ // Returns offset of the class tag in the compilation unit.
+ size_t StartClassTag(const char* desc) {
+ DCHECK(desc != nullptr && desc[0] == 'L');
+ // Enclose the type in namespace tags.
+ const char* end;
+ for (desc = desc + 1; (end = strchr(desc, '/')) != nullptr; desc = end + 1) {
+ info_.StartTag(DW_TAG_namespace);
+ WriteName(std::string(desc, end - desc).c_str());
+ }
+ // Start the class tag.
+ size_t offset = info_.StartTag(DW_TAG_class_type);
+ end = strchr(desc, ';');
+ CHECK(end != nullptr);
+ WriteName(std::string(desc, end - desc).c_str());
+ return offset;
+ }
+
+ void EndClassTag(const char* desc) {
+ DCHECK(desc != nullptr && desc[0] == 'L');
+ // End the class tag.
+ info_.EndTag();
+ // Close namespace tags.
+ const char* end;
+ for (desc = desc + 1; (end = strchr(desc, '/')) != nullptr; desc = end + 1) {
+ info_.EndTag();
+ }
+ }
+
+ // For access to the ELF sections.
+ DebugInfoWriter<ElfTypes>* owner_;
+ // Debug abbrevs for this compilation unit only.
+ std::vector<uint8_t> debug_abbrev_;
+ // Temporary buffer to create and store the entries.
+ DebugInfoEntryWriter<> info_;
+ // Cache of already translated type descriptors.
+ std::map<const char*, size_t, CStringLess> type_cache_; // type_desc -> definition_offset.
+ // 32-bit references which need to be resolved to a type later.
+ std::multimap<const char*, size_t, CStringLess> lazy_types_; // type_desc -> patch_offset.
+ };
+
public:
explicit DebugInfoWriter(ElfBuilder<ElfTypes>* builder) : builder_(builder) {
}
@@ -261,54 +470,29 @@
builder_->GetDebugInfo()->Start();
}
- void Write(const std::vector<const OatWriter::DebugInfo*>& method_infos,
- size_t debug_line_offset) {
- const bool is64bit = Is64BitInstructionSet(builder_->GetIsa());
- const Elf_Addr text_address = builder_->GetText()->GetAddress();
- uint32_t cunit_low_pc = 0xFFFFFFFFU;
- uint32_t cunit_high_pc = 0;
- for (auto method_info : method_infos) {
- cunit_low_pc = std::min(cunit_low_pc, method_info->low_pc_);
- cunit_high_pc = std::max(cunit_high_pc, method_info->high_pc_);
- }
-
- size_t debug_abbrev_offset = debug_abbrev_.size();
- DebugInfoEntryWriter<> info(is64bit, &debug_abbrev_);
- info.StartTag(DW_TAG_compile_unit, DW_CHILDREN_yes);
- info.WriteStrp(DW_AT_producer, "Android dex2oat", &debug_str_);
- info.WriteData1(DW_AT_language, DW_LANG_Java);
- info.WriteAddr(DW_AT_low_pc, text_address + cunit_low_pc);
- info.WriteAddr(DW_AT_high_pc, text_address + cunit_high_pc);
- info.WriteData4(DW_AT_stmt_list, debug_line_offset);
- for (auto method_info : method_infos) {
- std::string method_name = PrettyMethod(method_info->dex_method_index_,
- *method_info->dex_file_, true);
- info.StartTag(DW_TAG_subprogram, DW_CHILDREN_no);
- info.WriteStrp(DW_AT_name, method_name.data(), &debug_str_);
- info.WriteAddr(DW_AT_low_pc, text_address + method_info->low_pc_);
- info.WriteAddr(DW_AT_high_pc, text_address + method_info->high_pc_);
- info.EndTag(); // DW_TAG_subprogram
- }
- info.EndTag(); // DW_TAG_compile_unit
- std::vector<uint8_t> buffer;
- buffer.reserve(info.data()->size() + KB);
- size_t offset = builder_->GetDebugInfo()->GetSize();
- WriteDebugInfoCU(debug_abbrev_offset, info, offset, &buffer, &debug_info_patches_);
- builder_->GetDebugInfo()->WriteFully(buffer.data(), buffer.size());
+ void WriteCompilationUnit(const CompilationUnit& compilation_unit) {
+ CompilationUnitWriter writer(this);
+ writer.Write(compilation_unit);
}
void End() {
builder_->GetDebugInfo()->End();
builder_->WritePatches(".debug_info.oat_patches", &debug_info_patches_);
- builder_->WriteSection(".debug_abbrev", &debug_abbrev_);
- builder_->WriteSection(".debug_str", &debug_str_);
+ builder_->WriteSection(".debug_abbrev", &debug_abbrev_.Data());
+ builder_->WriteSection(".debug_str", &debug_str_.Data());
}
private:
+ size_t WriteString(const char* str) {
+ return debug_str_.Insert(reinterpret_cast<const uint8_t*>(str), strlen(str) + 1);
+ }
+
ElfBuilder<ElfTypes>* builder_;
std::vector<uintptr_t> debug_info_patches_;
- std::vector<uint8_t> debug_abbrev_;
- std::vector<uint8_t> debug_str_;
+ DedupVector debug_abbrev_;
+ DedupVector debug_str_;
+
+ std::unordered_set<const char*> defined_dex_classes_; // For CHECKs only.
};
template<typename ElfTypes>
@@ -325,15 +509,11 @@
// Write line table for given set of methods.
// Returns the number of bytes written.
- size_t Write(const std::vector<const OatWriter::DebugInfo*>& method_infos) {
+ size_t WriteCompilationUnit(CompilationUnit& compilation_unit) {
const bool is64bit = Is64BitInstructionSet(builder_->GetIsa());
const Elf_Addr text_address = builder_->GetText()->GetAddress();
- uint32_t cunit_low_pc = 0xFFFFFFFFU;
- uint32_t cunit_high_pc = 0;
- for (auto method_info : method_infos) {
- cunit_low_pc = std::min(cunit_low_pc, method_info->low_pc_);
- cunit_high_pc = std::max(cunit_high_pc, method_info->high_pc_);
- }
+
+ compilation_unit.debug_line_offset_ = builder_->GetDebugLine()->GetSize();
std::vector<FileEntry> files;
std::unordered_map<std::string, size_t> files_map;
@@ -358,11 +538,17 @@
break;
}
DebugLineOpCodeWriter<> opcodes(is64bit, code_factor_bits_);
- opcodes.SetAddress(text_address + cunit_low_pc);
+ opcodes.SetAddress(text_address + compilation_unit.low_pc_);
if (dwarf_isa != -1) {
opcodes.SetISA(dwarf_isa);
}
- for (const OatWriter::DebugInfo* mi : method_infos) {
+ for (const OatWriter::DebugInfo* mi : compilation_unit.methods_) {
+ // Ignore function if we have already generated line table for the same address.
+ // It would confuse the debugger and the DWARF specification forbids it.
+ if (mi->deduped_) {
+ continue;
+ }
+
struct DebugInfoCallbacks {
static bool NewPosition(void* ctx, uint32_t address, uint32_t line) {
auto* context = reinterpret_cast<DebugInfoCallbacks*>(ctx);
@@ -461,7 +647,7 @@
opcodes.AddRow(method_address, 0);
}
}
- opcodes.AdvancePC(text_address + cunit_high_pc);
+ opcodes.AdvancePC(text_address + compilation_unit.high_pc_);
opcodes.EndSequence();
std::vector<uint8_t> buffer;
buffer.reserve(opcodes.data()->size() + KB);
@@ -484,36 +670,28 @@
template<typename ElfTypes>
void WriteDebugSections(ElfBuilder<ElfTypes>* builder,
const std::vector<OatWriter::DebugInfo>& method_infos) {
- struct CompilationUnit {
- std::vector<const OatWriter::DebugInfo*> methods_;
- size_t debug_line_offset_ = 0;
- };
-
// Group the methods into compilation units based on source file.
std::vector<CompilationUnit> compilation_units;
const char* last_source_file = nullptr;
for (const OatWriter::DebugInfo& mi : method_infos) {
- // Attribute given instruction range only to single method.
- // Otherwise the debugger might get really confused.
- if (!mi.deduped_) {
- auto& dex_class_def = mi.dex_file_->GetClassDef(mi.class_def_index_);
- const char* source_file = mi.dex_file_->GetSourceFile(dex_class_def);
- if (compilation_units.empty() || source_file != last_source_file) {
- compilation_units.push_back(CompilationUnit());
- }
- compilation_units.back().methods_.push_back(&mi);
- last_source_file = source_file;
+ auto& dex_class_def = mi.dex_file_->GetClassDef(mi.class_def_index_);
+ const char* source_file = mi.dex_file_->GetSourceFile(dex_class_def);
+ if (compilation_units.empty() || source_file != last_source_file) {
+ compilation_units.push_back(CompilationUnit());
}
+ CompilationUnit& cu = compilation_units.back();
+ cu.methods_.push_back(&mi);
+ cu.low_pc_ = std::min(cu.low_pc_, mi.low_pc_);
+ cu.high_pc_ = std::max(cu.high_pc_, mi.high_pc_);
+ last_source_file = source_file;
}
// Write .debug_line section.
{
DebugLineWriter<ElfTypes> line_writer(builder);
line_writer.Start();
- size_t offset = 0;
for (auto& compilation_unit : compilation_units) {
- compilation_unit.debug_line_offset_ = offset;
- offset += line_writer.Write(compilation_unit.methods_);
+ line_writer.WriteCompilationUnit(compilation_unit);
}
line_writer.End();
}
@@ -523,7 +701,7 @@
DebugInfoWriter<ElfTypes> info_writer(builder);
info_writer.Start();
for (const auto& compilation_unit : compilation_units) {
- info_writer.Write(compilation_unit.methods_, compilation_unit.debug_line_offset_);
+ info_writer.WriteCompilationUnit(compilation_unit);
}
info_writer.End();
}
diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc
index 3f18d9a..3d9e7e7 100644
--- a/compiler/image_writer.cc
+++ b/compiler/image_writer.cc
@@ -334,9 +334,9 @@
Thread* const self = Thread::Current();
ReaderMutexLock mu(self, *class_linker->DexLock());
uint32_t size = 0u;
- for (jobject weak_root : class_linker->GetDexCaches()) {
+ for (const ClassLinker::DexCacheData& data : class_linker->GetDexCachesData()) {
mirror::DexCache* dex_cache =
- down_cast<mirror::DexCache*>(self->DecodeJObject(weak_root));
+ down_cast<mirror::DexCache*>(self->DecodeJObject(data.weak_root));
if (dex_cache == nullptr || IsInBootImage(dex_cache)) {
continue;
}
@@ -683,8 +683,8 @@
ScopedAssertNoThreadSuspension sa(self, __FUNCTION__);
ReaderMutexLock mu(self, *Locks::classlinker_classes_lock_); // For ClassInClassTable
ReaderMutexLock mu2(self, *class_linker->DexLock());
- for (jobject weak_root : class_linker->GetDexCaches()) {
- mirror::DexCache* dex_cache = down_cast<mirror::DexCache*>(self->DecodeJObject(weak_root));
+ for (const ClassLinker::DexCacheData& data : class_linker->GetDexCachesData()) {
+ mirror::DexCache* dex_cache = down_cast<mirror::DexCache*>(self->DecodeJObject(data.weak_root));
if (dex_cache == nullptr) {
continue;
}
@@ -806,8 +806,9 @@
{
ReaderMutexLock mu(self, *class_linker->DexLock());
// Count number of dex caches not in the boot image.
- for (jobject weak_root : class_linker->GetDexCaches()) {
- mirror::DexCache* dex_cache = down_cast<mirror::DexCache*>(self->DecodeJObject(weak_root));
+ for (const ClassLinker::DexCacheData& data : class_linker->GetDexCachesData()) {
+ mirror::DexCache* dex_cache =
+ down_cast<mirror::DexCache*>(self->DecodeJObject(data.weak_root));
dex_cache_count += IsInBootImage(dex_cache) ? 0u : 1u;
}
}
@@ -818,15 +819,17 @@
ReaderMutexLock mu(self, *class_linker->DexLock());
size_t non_image_dex_caches = 0;
// Re-count number of non image dex caches.
- for (jobject weak_root : class_linker->GetDexCaches()) {
- mirror::DexCache* dex_cache = down_cast<mirror::DexCache*>(self->DecodeJObject(weak_root));
+ for (const ClassLinker::DexCacheData& data : class_linker->GetDexCachesData()) {
+ mirror::DexCache* dex_cache =
+ down_cast<mirror::DexCache*>(self->DecodeJObject(data.weak_root));
non_image_dex_caches += IsInBootImage(dex_cache) ? 0u : 1u;
}
CHECK_EQ(dex_cache_count, non_image_dex_caches)
<< "The number of non-image dex caches changed.";
size_t i = 0;
- for (jobject weak_root : class_linker->GetDexCaches()) {
- mirror::DexCache* dex_cache = down_cast<mirror::DexCache*>(self->DecodeJObject(weak_root));
+ for (const ClassLinker::DexCacheData& data : class_linker->GetDexCachesData()) {
+ mirror::DexCache* dex_cache =
+ down_cast<mirror::DexCache*>(self->DecodeJObject(data.weak_root));
if (!IsInBootImage(dex_cache)) {
dex_caches->Set<false>(i, dex_cache);
++i;
@@ -1438,34 +1441,28 @@
: reinterpret_cast<T*>(image_begin_ + NativeOffsetInImage(obj));
}
-void ImageWriter::FixupClass(mirror::Class* orig, mirror::Class* copy) {
- // Update the field arrays.
- copy->SetSFieldsPtrUnchecked(NativeLocationInImage(orig->GetSFieldsPtr()));
- copy->SetIFieldsPtrUnchecked(NativeLocationInImage(orig->GetIFieldsPtr()));
- // Update direct and virtual method arrays.
- copy->SetDirectMethodsPtrUnchecked(NativeLocationInImage(orig->GetDirectMethodsPtr()));
- copy->SetVirtualMethodsPtr(NativeLocationInImage(orig->GetVirtualMethodsPtr()));
- // Update dex cache strings.
- copy->SetDexCacheStrings(NativeLocationInImage(orig->GetDexCacheStrings()));
- // Fix up embedded tables.
- if (!orig->IsTemp()) {
- // TODO: Why do we have temp classes in some cases?
- if (orig->ShouldHaveEmbeddedImtAndVTable()) {
- for (int32_t i = 0; i < orig->GetEmbeddedVTableLength(); ++i) {
- ArtMethod* orig_method = orig->GetEmbeddedVTableEntry(i, target_ptr_size_);
- copy->SetEmbeddedVTableEntryUnchecked(
- i,
- NativeLocationInImage(orig_method),
- target_ptr_size_);
- }
- for (size_t i = 0; i < mirror::Class::kImtSize; ++i) {
- copy->SetEmbeddedImTableEntry(
- i,
- NativeLocationInImage(orig->GetEmbeddedImTableEntry(i, target_ptr_size_)),
- target_ptr_size_);
- }
- }
+template <typename T>
+T* ImageWriter::NativeCopyLocation(T* obj) {
+ return (obj == nullptr || IsInBootImage(obj))
+ ? obj
+ : reinterpret_cast<T*>(image_->Begin() + NativeOffsetInImage(obj));
+}
+
+class NativeLocationVisitor {
+ public:
+ explicit NativeLocationVisitor(ImageWriter* image_writer) : image_writer_(image_writer) {}
+
+ template <typename T>
+ T* operator()(T* ptr) const {
+ return image_writer_->NativeLocationInImage(ptr);
}
+
+ private:
+ ImageWriter* const image_writer_;
+};
+
+void ImageWriter::FixupClass(mirror::Class* orig, mirror::Class* copy) {
+ orig->FixupNativePointers(copy, target_ptr_size_, NativeLocationVisitor(this));
FixupClassVisitor visitor(this, copy);
static_cast<mirror::Object*>(orig)->VisitReferences(visitor, visitor);
}
@@ -1525,6 +1522,21 @@
}
}
+
+class ImageAddressVisitor {
+ public:
+ explicit ImageAddressVisitor(ImageWriter* image_writer) : image_writer_(image_writer) {}
+
+ template <typename T>
+ T* operator()(T* ptr) const SHARED_REQUIRES(Locks::mutator_lock_) {
+ return image_writer_->GetImageAddress(ptr);
+ }
+
+ private:
+ ImageWriter* const image_writer_;
+};
+
+
void ImageWriter::FixupDexCache(mirror::DexCache* orig_dex_cache,
mirror::DexCache* copy_dex_cache) {
// Though the DexCache array fields are usually treated as native pointers, we set the full
@@ -1533,52 +1545,39 @@
// static_cast<int64_t>(reinterpret_cast<uintptr_t>(image_begin_ + offset))).
GcRoot<mirror::String>* orig_strings = orig_dex_cache->GetStrings();
if (orig_strings != nullptr) {
- uintptr_t copy_strings_offset = NativeOffsetInImage(orig_strings);
- copy_dex_cache->SetField64<false>(
- mirror::DexCache::StringsOffset(),
- static_cast<int64_t>(reinterpret_cast<uintptr_t>(image_begin_ + copy_strings_offset)));
- GcRoot<mirror::String>* copy_strings =
- reinterpret_cast<GcRoot<mirror::String>*>(image_->Begin() + copy_strings_offset);
- for (size_t i = 0, num = orig_dex_cache->NumStrings(); i != num; ++i) {
- copy_strings[i] = GcRoot<mirror::String>(GetImageAddress(orig_strings[i].Read()));
- }
+ copy_dex_cache->SetFieldPtrWithSize<false>(mirror::DexCache::StringsOffset(),
+ NativeLocationInImage(orig_strings),
+ /*pointer size*/8u);
+ orig_dex_cache->FixupStrings(NativeCopyLocation(orig_strings), ImageAddressVisitor(this));
}
GcRoot<mirror::Class>* orig_types = orig_dex_cache->GetResolvedTypes();
if (orig_types != nullptr) {
- uintptr_t copy_types_offset = NativeOffsetInImage(orig_types);
- copy_dex_cache->SetField64<false>(
- mirror::DexCache::ResolvedTypesOffset(),
- static_cast<int64_t>(reinterpret_cast<uintptr_t>(image_begin_ + copy_types_offset)));
- GcRoot<mirror::Class>* copy_types =
- reinterpret_cast<GcRoot<mirror::Class>*>(image_->Begin() + copy_types_offset);
- for (size_t i = 0, num = orig_dex_cache->NumResolvedTypes(); i != num; ++i) {
- copy_types[i] = GcRoot<mirror::Class>(GetImageAddress(orig_types[i].Read()));
- }
+ copy_dex_cache->SetFieldPtrWithSize<false>(mirror::DexCache::ResolvedTypesOffset(),
+ NativeLocationInImage(orig_types),
+ /*pointer size*/8u);
+ orig_dex_cache->FixupResolvedTypes(NativeCopyLocation(orig_types), ImageAddressVisitor(this));
}
ArtMethod** orig_methods = orig_dex_cache->GetResolvedMethods();
if (orig_methods != nullptr) {
- uintptr_t copy_methods_offset = NativeOffsetInImage(orig_methods);
- copy_dex_cache->SetField64<false>(
- mirror::DexCache::ResolvedMethodsOffset(),
- static_cast<int64_t>(reinterpret_cast<uintptr_t>(image_begin_ + copy_methods_offset)));
- ArtMethod** copy_methods =
- reinterpret_cast<ArtMethod**>(image_->Begin() + copy_methods_offset);
+ copy_dex_cache->SetFieldPtrWithSize<false>(mirror::DexCache::ResolvedMethodsOffset(),
+ NativeLocationInImage(orig_methods),
+ /*pointer size*/8u);
+ ArtMethod** copy_methods = NativeCopyLocation(orig_methods);
for (size_t i = 0, num = orig_dex_cache->NumResolvedMethods(); i != num; ++i) {
ArtMethod* orig = mirror::DexCache::GetElementPtrSize(orig_methods, i, target_ptr_size_);
- ArtMethod* copy = IsInBootImage(orig) ? orig : NativeLocationInImage(orig);
+ ArtMethod* copy = NativeLocationInImage(orig);
mirror::DexCache::SetElementPtrSize(copy_methods, i, copy, target_ptr_size_);
}
}
ArtField** orig_fields = orig_dex_cache->GetResolvedFields();
if (orig_fields != nullptr) {
- uintptr_t copy_fields_offset = NativeOffsetInImage(orig_fields);
- copy_dex_cache->SetField64<false>(
- mirror::DexCache::ResolvedFieldsOffset(),
- static_cast<int64_t>(reinterpret_cast<uintptr_t>(image_begin_ + copy_fields_offset)));
- ArtField** copy_fields = reinterpret_cast<ArtField**>(image_->Begin() + copy_fields_offset);
+ copy_dex_cache->SetFieldPtrWithSize<false>(mirror::DexCache::ResolvedFieldsOffset(),
+ NativeLocationInImage(orig_fields),
+ /*pointer size*/8u);
+ ArtField** copy_fields = NativeCopyLocation(orig_fields);
for (size_t i = 0, num = orig_dex_cache->NumResolvedFields(); i != num; ++i) {
ArtField* orig = mirror::DexCache::GetElementPtrSize(orig_fields, i, target_ptr_size_);
- ArtField* copy = IsInBootImage(orig) ? orig : NativeLocationInImage(orig);
+ ArtField* copy = NativeLocationInImage(orig);
mirror::DexCache::SetElementPtrSize(copy_fields, i, copy, target_ptr_size_);
}
}
diff --git a/compiler/image_writer.h b/compiler/image_writer.h
index a0a785e..22cb91a 100644
--- a/compiler/image_writer.h
+++ b/compiler/image_writer.h
@@ -347,9 +347,14 @@
uintptr_t NativeOffsetInImage(void* obj);
+ // Location of where the object will be when the image is loaded at runtime.
template <typename T>
T* NativeLocationInImage(T* obj);
+ // Location of where the temporary copy of the object currently is.
+ template <typename T>
+ T* NativeCopyLocation(T* obj);
+
// Return true of obj is inside of the boot image space. This may only return true if we are
// compiling an app image.
bool IsInBootImage(const void* obj) const;
@@ -446,6 +451,7 @@
friend class FixupClassVisitor;
friend class FixupRootVisitor;
friend class FixupVisitor;
+ friend class NativeLocationVisitor;
friend class NonImageClassesVisitor;
DISALLOW_COPY_AND_ASSIGN(ImageWriter);
};
diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc
index 16f641a..030451c 100644
--- a/compiler/oat_test.cc
+++ b/compiler/oat_test.cc
@@ -232,7 +232,7 @@
EXPECT_EQ(72U, sizeof(OatHeader));
EXPECT_EQ(4U, sizeof(OatMethodOffsets));
EXPECT_EQ(28U, sizeof(OatQuickMethodHeader));
- EXPECT_EQ(113 * GetInstructionSetPointerSize(kRuntimeISA), sizeof(QuickEntryPoints));
+ EXPECT_EQ(114 * GetInstructionSetPointerSize(kRuntimeISA), sizeof(QuickEntryPoints));
}
TEST_F(OatTest, OatHeaderIsValid) {
diff --git a/compiler/optimizing/builder.cc b/compiler/optimizing/builder.cc
index 167c35d..3257de1 100644
--- a/compiler/optimizing/builder.cc
+++ b/compiler/optimizing/builder.cc
@@ -1449,7 +1449,8 @@
uint32_t* args,
uint32_t register_index) {
HInstruction* length = graph_->GetIntConstant(number_of_vreg_arguments, dex_pc);
- QuickEntrypointEnum entrypoint = NeedsAccessCheck(type_index)
+ bool finalizable;
+ QuickEntrypointEnum entrypoint = NeedsAccessCheck(type_index, &finalizable)
? kQuickAllocArrayWithAccessCheck
: kQuickAllocArray;
HInstruction* object = new (arena_) HNewArray(length,
@@ -1629,9 +1630,9 @@
}
}
-bool HGraphBuilder::NeedsAccessCheck(uint32_t type_index) const {
+bool HGraphBuilder::NeedsAccessCheck(uint32_t type_index, bool* finalizable) const {
return !compiler_driver_->CanAccessInstantiableTypeWithoutChecks(
- dex_compilation_unit_->GetDexMethodIndex(), *dex_file_, type_index);
+ dex_compilation_unit_->GetDexMethodIndex(), *dex_file_, type_index, finalizable);
}
void HGraphBuilder::BuildSwitchJumpTable(const SwitchTable& table,
@@ -2508,7 +2509,9 @@
current_block_->AddInstruction(fake_string);
UpdateLocal(register_index, fake_string, dex_pc);
} else {
- QuickEntrypointEnum entrypoint = NeedsAccessCheck(type_index)
+ bool finalizable;
+ bool can_throw = NeedsAccessCheck(type_index, &finalizable);
+ QuickEntrypointEnum entrypoint = can_throw
? kQuickAllocObjectWithAccessCheck
: kQuickAllocObject;
@@ -2517,6 +2520,8 @@
dex_pc,
type_index,
*dex_compilation_unit_->GetDexFile(),
+ can_throw,
+ finalizable,
entrypoint));
UpdateLocal(instruction.VRegA(), current_block_->GetLastInstruction(), dex_pc);
}
@@ -2526,7 +2531,8 @@
case Instruction::NEW_ARRAY: {
uint16_t type_index = instruction.VRegC_22c();
HInstruction* length = LoadLocal(instruction.VRegB_22c(), Primitive::kPrimInt, dex_pc);
- QuickEntrypointEnum entrypoint = NeedsAccessCheck(type_index)
+ bool finalizable;
+ QuickEntrypointEnum entrypoint = NeedsAccessCheck(type_index, &finalizable)
? kQuickAllocArrayWithAccessCheck
: kQuickAllocArray;
current_block_->AddInstruction(new (arena_) HNewArray(length,
diff --git a/compiler/optimizing/builder.h b/compiler/optimizing/builder.h
index 9eaa4b6..f857ef0 100644
--- a/compiler/optimizing/builder.h
+++ b/compiler/optimizing/builder.h
@@ -138,7 +138,10 @@
HInstruction* LoadLocal(uint32_t register_index, Primitive::Type type, uint32_t dex_pc) const;
void PotentiallyAddSuspendCheck(HBasicBlock* target, uint32_t dex_pc);
void InitializeParameters(uint16_t number_of_parameters);
- bool NeedsAccessCheck(uint32_t type_index) const;
+
+ // Returns whether the current method needs access check for the type.
+ // Output parameter finalizable is set to whether the type is finalizable.
+ bool NeedsAccessCheck(uint32_t type_index, /*out*/bool* finalizable) const;
template<typename T>
void Unop_12x(const Instruction& instruction, Primitive::Type type, uint32_t dex_pc);
diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc
index 5188e11..0baa0e3 100644
--- a/compiler/optimizing/code_generator.cc
+++ b/compiler/optimizing/code_generator.cc
@@ -310,7 +310,7 @@
void CodeGenerator::InitializeCodeGeneration(size_t number_of_spill_slots,
size_t maximum_number_of_live_core_registers,
- size_t maximum_number_of_live_fp_registers,
+ size_t maximum_number_of_live_fpu_registers,
size_t number_of_out_slots,
const ArenaVector<HBasicBlock*>& block_order) {
block_order_ = &block_order;
@@ -324,14 +324,14 @@
&& IsLeafMethod()
&& !RequiresCurrentMethod()) {
DCHECK_EQ(maximum_number_of_live_core_registers, 0u);
- DCHECK_EQ(maximum_number_of_live_fp_registers, 0u);
+ DCHECK_EQ(maximum_number_of_live_fpu_registers, 0u);
SetFrameSize(CallPushesPC() ? GetWordSize() : 0);
} else {
SetFrameSize(RoundUp(
number_of_spill_slots * kVRegSize
+ number_of_out_slots * kVRegSize
+ maximum_number_of_live_core_registers * GetWordSize()
- + maximum_number_of_live_fp_registers * GetFloatingPointSpillSlotSize()
+ + maximum_number_of_live_fpu_registers * GetFloatingPointSpillSlotSize()
+ FrameEntrySpillSize(),
kStackAlignment));
}
@@ -383,11 +383,11 @@
HInvokeStaticOrDirect* call = invoke->AsInvokeStaticOrDirect();
switch (call->GetMethodLoadKind()) {
case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
- locations->SetInAt(call->GetCurrentMethodInputIndex(), visitor->GetMethodLocation());
+ locations->SetInAt(call->GetSpecialInputIndex(), visitor->GetMethodLocation());
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod:
locations->AddTemp(visitor->GetMethodLocation());
- locations->SetInAt(call->GetCurrentMethodInputIndex(), Location::RequiresRegister());
+ locations->SetInAt(call->GetSpecialInputIndex(), Location::RequiresRegister());
break;
default:
locations->AddTemp(visitor->GetMethodLocation());
@@ -547,15 +547,19 @@
}
}
+// TODO: Remove argument `code_generator_supports_read_barrier` when
+// all code generators have read barrier support.
void CodeGenerator::CreateLoadClassLocationSummary(HLoadClass* cls,
Location runtime_type_index_location,
- Location runtime_return_location) {
+ Location runtime_return_location,
+ bool code_generator_supports_read_barrier) {
ArenaAllocator* allocator = cls->GetBlock()->GetGraph()->GetArena();
LocationSummary::CallKind call_kind = cls->NeedsAccessCheck()
? LocationSummary::kCall
- : (cls->CanCallRuntime()
- ? LocationSummary::kCallOnSlowPath
- : LocationSummary::kNoCall);
+ : (((code_generator_supports_read_barrier && kEmitCompilerReadBarrier) ||
+ cls->CanCallRuntime())
+ ? LocationSummary::kCallOnSlowPath
+ : LocationSummary::kNoCall);
LocationSummary* locations = new (allocator) LocationSummary(cls, call_kind);
if (cls->NeedsAccessCheck()) {
locations->SetInAt(0, Location::NoLocation());
@@ -1320,21 +1324,38 @@
// coherent with the runtime call generated, and that the GC side effect is
// set when required.
if (slow_path == nullptr) {
- DCHECK(instruction->GetLocations()->WillCall()) << instruction->DebugName();
+ DCHECK(instruction->GetLocations()->WillCall())
+ << "instruction->DebugName()=" << instruction->DebugName();
DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()))
- << instruction->DebugName() << instruction->GetSideEffects().ToString();
+ << "instruction->DebugName()=" << instruction->DebugName()
+ << " instruction->GetSideEffects().ToString()=" << instruction->GetSideEffects().ToString();
} else {
DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath() || slow_path->IsFatal())
- << instruction->DebugName() << slow_path->GetDescription();
+ << "instruction->DebugName()=" << instruction->DebugName()
+ << " slow_path->GetDescription()=" << slow_path->GetDescription();
DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()) ||
// Control flow would not come back into the code if a fatal slow
// path is taken, so we do not care if it triggers GC.
slow_path->IsFatal() ||
// HDeoptimize is a special case: we know we are not coming back from
// it into the code.
- instruction->IsDeoptimize())
- << instruction->DebugName() << instruction->GetSideEffects().ToString()
- << slow_path->GetDescription();
+ instruction->IsDeoptimize() ||
+ // When read barriers are enabled, some instructions use a
+ // slow path to emit a read barrier, which does not trigger
+ // GC, is not fatal, nor is emitted by HDeoptimize
+ // instructions.
+ (kEmitCompilerReadBarrier &&
+ (instruction->IsInstanceFieldGet() ||
+ instruction->IsStaticFieldGet() ||
+ instruction->IsArraySet() ||
+ instruction->IsArrayGet() ||
+ instruction->IsLoadClass() ||
+ instruction->IsLoadString() ||
+ instruction->IsInstanceOf() ||
+ instruction->IsCheckCast())))
+ << "instruction->DebugName()=" << instruction->DebugName()
+ << " instruction->GetSideEffects().ToString()=" << instruction->GetSideEffects().ToString()
+ << " slow_path->GetDescription()=" << slow_path->GetDescription();
}
// Check the coherency of leaf information.
@@ -1346,11 +1367,12 @@
}
void SlowPathCode::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
- RegisterSet* register_set = locations->GetLiveRegisters();
+ RegisterSet* live_registers = locations->GetLiveRegisters();
size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
+
for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
if (!codegen->IsCoreCalleeSaveRegister(i)) {
- if (register_set->ContainsCoreRegister(i)) {
+ if (live_registers->ContainsCoreRegister(i)) {
// If the register holds an object, update the stack mask.
if (locations->RegisterContainsObject(i)) {
locations->SetStackBit(stack_offset / kVRegSize);
@@ -1365,7 +1387,7 @@
for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
if (!codegen->IsFloatingPointCalleeSaveRegister(i)) {
- if (register_set->ContainsFloatingPointRegister(i)) {
+ if (live_registers->ContainsFloatingPointRegister(i)) {
DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
saved_fpu_stack_offsets_[i] = stack_offset;
@@ -1376,12 +1398,14 @@
}
void SlowPathCode::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
- RegisterSet* register_set = locations->GetLiveRegisters();
+ RegisterSet* live_registers = locations->GetLiveRegisters();
size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
+
for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
if (!codegen->IsCoreCalleeSaveRegister(i)) {
- if (register_set->ContainsCoreRegister(i)) {
+ if (live_registers->ContainsCoreRegister(i)) {
DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
+ DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
stack_offset += codegen->RestoreCoreRegister(stack_offset, i);
}
}
@@ -1389,8 +1413,9 @@
for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
if (!codegen->IsFloatingPointCalleeSaveRegister(i)) {
- if (register_set->ContainsFloatingPointRegister(i)) {
+ if (live_registers->ContainsFloatingPointRegister(i)) {
DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
+ DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, i);
}
}
diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h
index 2108abe..114d97b 100644
--- a/compiler/optimizing/code_generator.h
+++ b/compiler/optimizing/code_generator.h
@@ -201,7 +201,7 @@
virtual uintptr_t GetAddressOf(HBasicBlock* block) const = 0;
void InitializeCodeGeneration(size_t number_of_spill_slots,
size_t maximum_number_of_live_core_registers,
- size_t maximum_number_of_live_fp_registers,
+ size_t maximum_number_of_live_fpu_registers,
size_t number_of_out_slots,
const ArenaVector<HBasicBlock*>& block_order);
int32_t GetStackSlot(HLocal* local) const;
@@ -250,6 +250,15 @@
// Returns whether we should split long moves in parallel moves.
virtual bool ShouldSplitLongMoves() const { return false; }
+ size_t GetNumberOfCoreCalleeSaveRegisters() const {
+ return POPCOUNT(core_callee_save_mask_);
+ }
+
+ size_t GetNumberOfCoreCallerSaveRegisters() const {
+ DCHECK_GE(GetNumberOfCoreRegisters(), GetNumberOfCoreCalleeSaveRegisters());
+ return GetNumberOfCoreRegisters() - GetNumberOfCoreCalleeSaveRegisters();
+ }
+
bool IsCoreCalleeSaveRegister(int reg) const {
return (core_callee_save_mask_ & (1 << reg)) != 0;
}
@@ -416,7 +425,8 @@
// TODO: This overlaps a bit with MoveFromReturnRegister. Refactor for a better design.
static void CreateLoadClassLocationSummary(HLoadClass* cls,
Location runtime_type_index_location,
- Location runtime_return_location);
+ Location runtime_return_location,
+ bool code_generator_supports_read_barrier = false);
static void CreateSystemArrayCopyLocationSummary(HInvoke* invoke);
diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc
index 54c6cc8..cb6bed0 100644
--- a/compiler/optimizing/code_generator_arm.cc
+++ b/compiler/optimizing/code_generator_arm.cc
@@ -34,6 +34,9 @@
namespace art {
+template<class MirrorType>
+class GcRoot;
+
namespace arm {
static bool ExpectedPairLayout(Location location) {
@@ -286,15 +289,6 @@
CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
__ Bind(GetEntryLabel());
- if (instruction_->IsCheckCast()) {
- // The codegen for the instruction overwrites `temp`, so put it back in place.
- Register obj = locations->InAt(0).AsRegister<Register>();
- Register temp = locations->GetTemp(0).AsRegister<Register>();
- uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
- __ LoadFromOffset(kLoadWord, temp, obj, class_offset);
- __ MaybeUnpoisonHeapReference(temp);
- }
-
if (!is_fatal_) {
SaveLiveRegisters(codegen, locations);
}
@@ -315,6 +309,8 @@
instruction_,
instruction_->GetDexPc(),
this);
+ CheckEntrypointTypes<
+ kQuickInstanceofNonTrivial, uint32_t, const mirror::Class*, const mirror::Class*>();
arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0));
} else {
DCHECK(instruction_->IsCheckCast());
@@ -322,6 +318,7 @@
instruction_,
instruction_->GetDexPc(),
this);
+ CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
}
if (!is_fatal_) {
@@ -408,6 +405,221 @@
DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM);
};
+// Slow path generating a read barrier for a heap reference.
+class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCode {
+ public:
+ ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index)
+ : instruction_(instruction),
+ out_(out),
+ ref_(ref),
+ obj_(obj),
+ offset_(offset),
+ index_(index) {
+ DCHECK(kEmitCompilerReadBarrier);
+ // If `obj` is equal to `out` or `ref`, it means the initial object
+ // has been overwritten by (or after) the heap object reference load
+ // to be instrumented, e.g.:
+ //
+ // __ LoadFromOffset(kLoadWord, out, out, offset);
+ // codegen_->GenerateReadBarrier(instruction, out_loc, out_loc, out_loc, offset);
+ //
+ // In that case, we have lost the information about the original
+ // object, and the emitted read barrier cannot work properly.
+ DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out;
+ DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref;
+ }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
+ LocationSummary* locations = instruction_->GetLocations();
+ Register reg_out = out_.AsRegister<Register>();
+ DCHECK(locations->CanCall());
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
+ DCHECK(!instruction_->IsInvoke() ||
+ (instruction_->IsInvokeStaticOrDirect() &&
+ instruction_->GetLocations()->Intrinsified()));
+
+ __ Bind(GetEntryLabel());
+ SaveLiveRegisters(codegen, locations);
+
+ // We may have to change the index's value, but as `index_` is a
+ // constant member (like other "inputs" of this slow path),
+ // introduce a copy of it, `index`.
+ Location index = index_;
+ if (index_.IsValid()) {
+ // Handle `index_` for HArrayGet and intrinsic UnsafeGetObject.
+ if (instruction_->IsArrayGet()) {
+ // Compute the actual memory offset and store it in `index`.
+ Register index_reg = index_.AsRegister<Register>();
+ DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg));
+ if (codegen->IsCoreCalleeSaveRegister(index_reg)) {
+ // We are about to change the value of `index_reg` (see the
+ // calls to art::arm::Thumb2Assembler::Lsl and
+ // art::arm::Thumb2Assembler::AddConstant below), but it has
+ // not been saved by the previous call to
+ // art::SlowPathCode::SaveLiveRegisters, as it is a
+ // callee-save register --
+ // art::SlowPathCode::SaveLiveRegisters does not consider
+ // callee-save registers, as it has been designed with the
+ // assumption that callee-save registers are supposed to be
+ // handled by the called function. So, as a callee-save
+ // register, `index_reg` _would_ eventually be saved onto
+ // the stack, but it would be too late: we would have
+ // changed its value earlier. Therefore, we manually save
+ // it here into another freely available register,
+ // `free_reg`, chosen of course among the caller-save
+ // registers (as a callee-save `free_reg` register would
+ // exhibit the same problem).
+ //
+ // Note we could have requested a temporary register from
+ // the register allocator instead; but we prefer not to, as
+ // this is a slow path, and we know we can find a
+ // caller-save register that is available.
+ Register free_reg = FindAvailableCallerSaveRegister(codegen);
+ __ Mov(free_reg, index_reg);
+ index_reg = free_reg;
+ index = Location::RegisterLocation(index_reg);
+ } else {
+ // The initial register stored in `index_` has already been
+ // saved in the call to art::SlowPathCode::SaveLiveRegisters
+ // (as it is not a callee-save register), so we can freely
+ // use it.
+ }
+ // Shifting the index value contained in `index_reg` by the scale
+ // factor (2) cannot overflow in practice, as the runtime is
+ // unable to allocate object arrays with a size larger than
+ // 2^26 - 1 (that is, 2^28 - 4 bytes).
+ __ Lsl(index_reg, index_reg, TIMES_4);
+ static_assert(
+ sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
+ "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
+ __ AddConstant(index_reg, index_reg, offset_);
+ } else {
+ DCHECK(instruction_->IsInvoke());
+ DCHECK(instruction_->GetLocations()->Intrinsified());
+ DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) ||
+ (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile))
+ << instruction_->AsInvoke()->GetIntrinsic();
+ DCHECK_EQ(offset_, 0U);
+ DCHECK(index_.IsRegisterPair());
+ // UnsafeGet's offset location is a register pair, the low
+ // part contains the correct offset.
+ index = index_.ToLow();
+ }
+ }
+
+ // We're moving two or three locations to locations that could
+ // overlap, so we need a parallel move resolver.
+ InvokeRuntimeCallingConvention calling_convention;
+ HParallelMove parallel_move(codegen->GetGraph()->GetArena());
+ parallel_move.AddMove(ref_,
+ Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
+ Primitive::kPrimNot,
+ nullptr);
+ parallel_move.AddMove(obj_,
+ Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
+ Primitive::kPrimNot,
+ nullptr);
+ if (index.IsValid()) {
+ parallel_move.AddMove(index,
+ Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
+ Primitive::kPrimInt,
+ nullptr);
+ codegen->GetMoveResolver()->EmitNativeCode(¶llel_move);
+ } else {
+ codegen->GetMoveResolver()->EmitNativeCode(¶llel_move);
+ __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_);
+ }
+ arm_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierSlow),
+ instruction_,
+ instruction_->GetDexPc(),
+ this);
+ CheckEntrypointTypes<
+ kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>();
+ arm_codegen->Move32(out_, Location::RegisterLocation(R0));
+
+ RestoreLiveRegisters(codegen, locations);
+ __ b(GetExitLabel());
+ }
+
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; }
+
+ private:
+ Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) {
+ size_t ref = static_cast<int>(ref_.AsRegister<Register>());
+ size_t obj = static_cast<int>(obj_.AsRegister<Register>());
+ for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
+ if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) {
+ return static_cast<Register>(i);
+ }
+ }
+ // We shall never fail to find a free caller-save register, as
+ // there are more than two core caller-save registers on ARM
+ // (meaning it is possible to find one which is different from
+ // `ref` and `obj`).
+ DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u);
+ LOG(FATAL) << "Could not find a free caller-save register";
+ UNREACHABLE();
+ }
+
+ HInstruction* const instruction_;
+ const Location out_;
+ const Location ref_;
+ const Location obj_;
+ const uint32_t offset_;
+ // An additional location containing an index to an array.
+ // Only used for HArrayGet and the UnsafeGetObject &
+ // UnsafeGetObjectVolatile intrinsics.
+ const Location index_;
+
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM);
+};
+
+// Slow path generating a read barrier for a GC root.
+class ReadBarrierForRootSlowPathARM : public SlowPathCode {
+ public:
+ ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root)
+ : instruction_(instruction), out_(out), root_(root) {}
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ Register reg_out = out_.AsRegister<Register>();
+ DCHECK(locations->CanCall());
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
+ DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString());
+
+ __ Bind(GetEntryLabel());
+ SaveLiveRegisters(codegen, locations);
+
+ InvokeRuntimeCallingConvention calling_convention;
+ CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
+ arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_);
+ arm_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierForRootSlow),
+ instruction_,
+ instruction_->GetDexPc(),
+ this);
+ CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>();
+ arm_codegen->Move32(out_, Location::RegisterLocation(R0));
+
+ RestoreLiveRegisters(codegen, locations);
+ __ b(GetExitLabel());
+ }
+
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; }
+
+ private:
+ HInstruction* const instruction_;
+ const Location out_;
+ const Location root_;
+
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM);
+};
+
#undef __
#define __ down_cast<ArmAssembler*>(GetAssembler())->
@@ -581,7 +793,7 @@
LOG(FATAL) << "Unreachable type " << type;
}
- return Location();
+ return Location::NoLocation();
}
void CodeGeneratorARM::SetupBlockedRegisters(bool is_baseline) const {
@@ -820,7 +1032,7 @@
LOG(FATAL) << "Unexpected parameter type " << type;
break;
}
- return Location();
+ return Location::NoLocation();
}
Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const {
@@ -847,7 +1059,7 @@
}
case Primitive::kPrimVoid:
- return Location();
+ return Location::NoLocation();
}
UNREACHABLE();
@@ -1240,26 +1452,19 @@
__ b(true_label, final_condition);
}
-void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HIf* if_instr,
- HCondition* condition,
- Label* true_target,
- Label* false_target,
- Label* always_true_target) {
+void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition,
+ Label* true_target_in,
+ Label* false_target_in) {
+ // Generated branching requires both targets to be explicit. If either of the
+ // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead.
+ Label fallthrough_target;
+ Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in;
+ Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in;
+
LocationSummary* locations = condition->GetLocations();
Location left = locations->InAt(0);
Location right = locations->InAt(1);
- // We don't want true_target as a nullptr.
- if (true_target == nullptr) {
- true_target = always_true_target;
- }
- bool falls_through = (false_target == nullptr);
-
- // FP compares don't like null false_targets.
- if (false_target == nullptr) {
- false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
- }
-
Primitive::Type type = condition->InputAt(0)->GetType();
switch (type) {
case Primitive::kPrimLong:
@@ -1278,117 +1483,125 @@
LOG(FATAL) << "Unexpected compare type " << type;
}
- if (!falls_through) {
+ if (false_target != &fallthrough_target) {
__ b(false_target);
}
+
+ if (fallthrough_target.IsLinked()) {
+ __ Bind(&fallthrough_target);
+ }
}
void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction,
+ size_t condition_input_index,
Label* true_target,
- Label* false_target,
- Label* always_true_target) {
- HInstruction* cond = instruction->InputAt(0);
- if (cond->IsIntConstant()) {
+ Label* false_target) {
+ HInstruction* cond = instruction->InputAt(condition_input_index);
+
+ if (true_target == nullptr && false_target == nullptr) {
+ // Nothing to do. The code always falls through.
+ return;
+ } else if (cond->IsIntConstant()) {
// Constant condition, statically compared against 1.
- int32_t cond_value = cond->AsIntConstant()->GetValue();
- if (cond_value == 1) {
- if (always_true_target != nullptr) {
- __ b(always_true_target);
+ if (cond->AsIntConstant()->IsOne()) {
+ if (true_target != nullptr) {
+ __ b(true_target);
}
- return;
} else {
- DCHECK_EQ(cond_value, 0);
+ DCHECK(cond->AsIntConstant()->IsZero());
+ if (false_target != nullptr) {
+ __ b(false_target);
+ }
+ }
+ return;
+ }
+
+ // The following code generates these patterns:
+ // (1) true_target == nullptr && false_target != nullptr
+ // - opposite condition true => branch to false_target
+ // (2) true_target != nullptr && false_target == nullptr
+ // - condition true => branch to true_target
+ // (3) true_target != nullptr && false_target != nullptr
+ // - condition true => branch to true_target
+ // - branch to false_target
+ if (IsBooleanValueOrMaterializedCondition(cond)) {
+ // Condition has been materialized, compare the output to 0.
+ Location cond_val = instruction->GetLocations()->InAt(condition_input_index);
+ DCHECK(cond_val.IsRegister());
+ if (true_target == nullptr) {
+ __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target);
+ } else {
+ __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target);
}
} else {
- // Can we optimize the jump if we know that the next block is the true case?
+ // Condition has not been materialized. Use its inputs as the comparison and
+ // its condition as the branch condition.
HCondition* condition = cond->AsCondition();
- bool can_jump_to_false = CanReverseCondition(always_true_target, false_target, condition);
- if (condition == nullptr || condition->NeedsMaterialization()) {
- // Condition has been materialized, compare the output to 0.
- DCHECK(instruction->GetLocations()->InAt(0).IsRegister());
- if (can_jump_to_false) {
- __ CompareAndBranchIfZero(instruction->GetLocations()->InAt(0).AsRegister<Register>(),
- false_target);
- return;
- }
- __ CompareAndBranchIfNonZero(instruction->GetLocations()->InAt(0).AsRegister<Register>(),
- true_target);
+
+ // If this is a long or FP comparison that has been folded into
+ // the HCondition, generate the comparison directly.
+ Primitive::Type type = condition->InputAt(0)->GetType();
+ if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
+ GenerateCompareTestAndBranch(condition, true_target, false_target);
+ return;
+ }
+
+ LocationSummary* locations = cond->GetLocations();
+ DCHECK(locations->InAt(0).IsRegister());
+ Register left = locations->InAt(0).AsRegister<Register>();
+ Location right = locations->InAt(1);
+ if (right.IsRegister()) {
+ __ cmp(left, ShifterOperand(right.AsRegister<Register>()));
} else {
- // Condition has not been materialized, use its inputs as the
- // comparison and its condition as the branch condition.
- Primitive::Type type = (condition != nullptr)
- ? cond->InputAt(0)->GetType()
- : Primitive::kPrimInt;
- // Is this a long or FP comparison that has been folded into the HCondition?
- if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
- // Generate the comparison directly.
- GenerateCompareTestAndBranch(instruction->AsIf(), condition,
- true_target, false_target, always_true_target);
- return;
- }
-
- LocationSummary* locations = cond->GetLocations();
- DCHECK(locations->InAt(0).IsRegister()) << locations->InAt(0);
- Register left = locations->InAt(0).AsRegister<Register>();
- Location right = locations->InAt(1);
- if (right.IsRegister()) {
- __ cmp(left, ShifterOperand(right.AsRegister<Register>()));
- } else {
- DCHECK(right.IsConstant());
- GenerateCompareWithImmediate(left, CodeGenerator::GetInt32ValueOf(right.GetConstant()));
- }
- if (can_jump_to_false) {
- __ b(false_target, ARMCondition(condition->GetOppositeCondition()));
- return;
- }
-
+ DCHECK(right.IsConstant());
+ GenerateCompareWithImmediate(left, CodeGenerator::GetInt32ValueOf(right.GetConstant()));
+ }
+ if (true_target == nullptr) {
+ __ b(false_target, ARMCondition(condition->GetOppositeCondition()));
+ } else {
__ b(true_target, ARMCondition(condition->GetCondition()));
}
}
- if (false_target != nullptr) {
+
+ // If neither branch falls through (case 3), the conditional branch to `true_target`
+ // was already emitted (case 2) and we need to emit a jump to `false_target`.
+ if (true_target != nullptr && false_target != nullptr) {
__ b(false_target);
}
}
void LocationsBuilderARM::VisitIf(HIf* if_instr) {
- LocationSummary* locations =
- new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
- HInstruction* cond = if_instr->InputAt(0);
- if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
+ LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
+ if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
locations->SetInAt(0, Location::RequiresRegister());
}
}
void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) {
- Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
- Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
- Label* always_true_target = true_target;
- if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
- if_instr->IfTrueSuccessor())) {
- always_true_target = nullptr;
- }
- if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
- if_instr->IfFalseSuccessor())) {
- false_target = nullptr;
- }
- GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
+ HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
+ HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
+ Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
+ nullptr : codegen_->GetLabelOf(true_successor);
+ Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
+ nullptr : codegen_->GetLabelOf(false_successor);
+ GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
}
void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) {
LocationSummary* locations = new (GetGraph()->GetArena())
LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
- HInstruction* cond = deoptimize->InputAt(0);
- if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
+ if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
locations->SetInAt(0, Location::RequiresRegister());
}
}
void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) {
- SlowPathCode* slow_path = new (GetGraph()->GetArena())
- DeoptimizationSlowPathARM(deoptimize);
+ SlowPathCode* slow_path = new (GetGraph()->GetArena()) DeoptimizationSlowPathARM(deoptimize);
codegen_->AddSlowPath(slow_path);
- Label* slow_path_entry = slow_path->GetEntryLabel();
- GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
+ GenerateTestAndBranch(deoptimize,
+ /* condition_input_index */ 0,
+ slow_path->GetEntryLabel(),
+ /* false_target */ nullptr);
}
void LocationsBuilderARM::VisitCondition(HCondition* cond) {
@@ -1761,29 +1974,39 @@
void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) {
// TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
- Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
+ LocationSummary* locations = invoke->GetLocations();
+ Register temp = locations->GetTemp(0).AsRegister<Register>();
+ Register hidden_reg = locations->GetTemp(1).AsRegister<Register>();
uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
invoke->GetImtIndex() % mirror::Class::kImtSize, kArmPointerSize).Uint32Value();
- LocationSummary* locations = invoke->GetLocations();
Location receiver = locations->InAt(0);
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
- // Set the hidden argument.
- __ LoadImmediate(invoke->GetLocations()->GetTemp(1).AsRegister<Register>(),
- invoke->GetDexMethodIndex());
+ // Set the hidden argument. This is safe to do this here, as R12
+ // won't be modified thereafter, before the `blx` (call) instruction.
+ DCHECK_EQ(R12, hidden_reg);
+ __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex());
- // temp = object->GetClass();
if (receiver.IsStackSlot()) {
__ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex());
+ // /* HeapReference<Class> */ temp = temp->klass_
__ LoadFromOffset(kLoadWord, temp, temp, class_offset);
} else {
+ // /* HeapReference<Class> */ temp = receiver->klass_
__ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset);
}
codegen_->MaybeRecordImplicitNullCheck(invoke);
+ // Instead of simply (possibly) unpoisoning `temp` here, we should
+ // emit a read barrier for the previous class reference load.
+ // However this is not required in practice, as this is an
+ // intermediate/temporary reference and because the current
+ // concurrent copying collector keeps the from-space memory
+ // intact/accessible until the end of the marking phase (the
+ // concurrent copying collector may not in the future).
__ MaybeUnpoisonHeapReference(temp);
// temp = temp->GetImtEntryAt(method_offset);
- uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(
- kArmWordSize).Int32Value();
+ uint32_t entry_point =
+ ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmWordSize).Int32Value();
__ LoadFromOffset(kLoadWord, temp, temp, method_offset);
// LR = temp->GetEntryPoint();
__ LoadFromOffset(kLoadWord, LR, temp, entry_point);
@@ -3406,6 +3629,9 @@
Register out_lo,
Register out_hi) {
if (offset != 0) {
+ // Ensure `out_lo` is different from `addr`, so that loading
+ // `offset` into `out_lo` does not clutter `addr`.
+ DCHECK_NE(out_lo, addr);
__ LoadImmediate(out_lo, offset);
__ add(IP, addr, ShifterOperand(out_lo));
addr = IP;
@@ -3593,14 +3819,26 @@
void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
+
+ bool object_field_get_with_read_barrier =
+ kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot);
LocationSummary* locations =
- new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
+ new (GetGraph()->GetArena()) LocationSummary(instruction,
+ object_field_get_with_read_barrier ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall);
locations->SetInAt(0, Location::RequiresRegister());
bool volatile_for_double = field_info.IsVolatile()
&& (field_info.GetFieldType() == Primitive::kPrimDouble)
&& !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd();
- bool overlap = field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong);
+ // The output overlaps in case of volatile long: we don't want the
+ // code generated by GenerateWideAtomicLoad to overwrite the
+ // object's location. Likewise, in the case of an object field get
+ // with read barriers enabled, we do not want the load to overwrite
+ // the object's location, as we need it to emit the read barrier.
+ bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) ||
+ object_field_get_with_read_barrier;
if (Primitive::IsFloatingPointType(instruction->GetType())) {
locations->SetOut(Location::RequiresFpuRegister());
@@ -3666,7 +3904,8 @@
DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
LocationSummary* locations = instruction->GetLocations();
- Register base = locations->InAt(0).AsRegister<Register>();
+ Location base_loc = locations->InAt(0);
+ Register base = base_loc.AsRegister<Register>();
Location out = locations->Out();
bool is_volatile = field_info.IsVolatile();
bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd();
@@ -3746,7 +3985,7 @@
}
if (field_type == Primitive::kPrimNot) {
- __ MaybeUnpoisonHeapReference(out.AsRegister<Register>());
+ codegen_->MaybeGenerateReadBarrier(instruction, out, out, base_loc, offset);
}
}
@@ -3890,20 +4129,31 @@
}
void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) {
+ bool object_array_get_with_read_barrier =
+ kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
LocationSummary* locations =
- new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
+ new (GetGraph()->GetArena()) LocationSummary(instruction,
+ object_array_get_with_read_barrier ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
if (Primitive::IsFloatingPointType(instruction->GetType())) {
locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
} else {
- locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
+ // The output overlaps in the case of an object array get with
+ // read barriers enabled: we do not want the move to overwrite the
+ // array's location, as we need it to emit the read barrier.
+ locations->SetOut(
+ Location::RequiresRegister(),
+ object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap);
}
}
void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) {
LocationSummary* locations = instruction->GetLocations();
- Register obj = locations->InAt(0).AsRegister<Register>();
+ Location obj_loc = locations->InAt(0);
+ Register obj = obj_loc.AsRegister<Register>();
Location index = locations->InAt(1);
Primitive::Type type = instruction->GetType();
@@ -3966,8 +4216,9 @@
case Primitive::kPrimInt:
case Primitive::kPrimNot: {
- static_assert(sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
- "art::mirror::HeapReference<mirror::Object> and int32_t have different sizes.");
+ static_assert(
+ sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
+ "art::mirror::HeapReference<mirror::Object> and int32_t have different sizes.");
uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
Register out = locations->Out().AsRegister<Register>();
if (index.IsConstant()) {
@@ -4030,8 +4281,17 @@
codegen_->MaybeRecordImplicitNullCheck(instruction);
if (type == Primitive::kPrimNot) {
- Register out = locations->Out().AsRegister<Register>();
- __ MaybeUnpoisonHeapReference(out);
+ static_assert(
+ sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
+ "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
+ uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
+ Location out = locations->Out();
+ if (index.IsConstant()) {
+ uint32_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
+ codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, offset);
+ } else {
+ codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, data_offset, index);
+ }
}
}
@@ -4040,11 +4300,16 @@
bool needs_write_barrier =
CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
- bool may_need_runtime_call = instruction->NeedsTypeCheck();
+ bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
+ bool object_array_set_with_read_barrier =
+ kEmitCompilerReadBarrier && (value_type == Primitive::kPrimNot);
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
instruction,
- may_need_runtime_call ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall);
+ (may_need_runtime_call_for_type_check || object_array_set_with_read_barrier) ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall);
+
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
if (Primitive::IsFloatingPointType(value_type)) {
@@ -4052,20 +4317,20 @@
} else {
locations->SetInAt(2, Location::RequiresRegister());
}
-
if (needs_write_barrier) {
// Temporary registers for the write barrier.
locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too.
- locations->AddTemp(Location::RequiresRegister());
+ locations->AddTemp(Location::RequiresRegister()); // Possibly used for read barrier too.
}
}
void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) {
LocationSummary* locations = instruction->GetLocations();
- Register array = locations->InAt(0).AsRegister<Register>();
+ Location array_loc = locations->InAt(0);
+ Register array = array_loc.AsRegister<Register>();
Location index = locations->InAt(1);
Primitive::Type value_type = instruction->GetComponentType();
- bool may_need_runtime_call = locations->CanCall();
+ bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
bool needs_write_barrier =
CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
@@ -4102,7 +4367,8 @@
case Primitive::kPrimNot: {
uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
- Register value = locations->InAt(2).AsRegister<Register>();
+ Location value_loc = locations->InAt(2);
+ Register value = value_loc.AsRegister<Register>();
Register source = value;
if (instruction->InputAt(2)->IsNullConstant()) {
@@ -4116,6 +4382,8 @@
__ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4));
__ StoreToOffset(kStoreWord, source, IP, data_offset);
}
+ DCHECK(!needs_write_barrier);
+ DCHECK(!may_need_runtime_call_for_type_check);
break;
}
@@ -4128,7 +4396,7 @@
Label done;
SlowPathCode* slow_path = nullptr;
- if (may_need_runtime_call) {
+ if (may_need_runtime_call_for_type_check) {
slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction);
codegen_->AddSlowPath(slow_path);
if (instruction->GetValueCanBeNull()) {
@@ -4148,23 +4416,63 @@
__ Bind(&non_zero);
}
- __ LoadFromOffset(kLoadWord, temp1, array, class_offset);
- codegen_->MaybeRecordImplicitNullCheck(instruction);
- __ MaybeUnpoisonHeapReference(temp1);
- __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset);
- __ LoadFromOffset(kLoadWord, temp2, value, class_offset);
- // No need to poison/unpoison, we're comparing two poisoined references.
- __ cmp(temp1, ShifterOperand(temp2));
- if (instruction->StaticTypeOfArrayIsObjectArray()) {
- Label do_put;
- __ b(&do_put, EQ);
- __ MaybeUnpoisonHeapReference(temp1);
- __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset);
- // No need to poison/unpoison, we're comparing against null.
- __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel());
- __ Bind(&do_put);
+ if (kEmitCompilerReadBarrier) {
+ // When read barriers are enabled, the type checking
+ // instrumentation requires two read barriers:
+ //
+ // __ Mov(temp2, temp1);
+ // // /* HeapReference<Class> */ temp1 = temp1->component_type_
+ // __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset);
+ // codegen_->GenerateReadBarrier(
+ // instruction, temp1_loc, temp1_loc, temp2_loc, component_offset);
+ //
+ // // /* HeapReference<Class> */ temp2 = value->klass_
+ // __ LoadFromOffset(kLoadWord, temp2, value, class_offset);
+ // codegen_->GenerateReadBarrier(
+ // instruction, temp2_loc, temp2_loc, value_loc, class_offset, temp1_loc);
+ //
+ // __ cmp(temp1, ShifterOperand(temp2));
+ //
+ // However, the second read barrier may trash `temp`, as it
+ // is a temporary register, and as such would not be saved
+ // along with live registers before calling the runtime (nor
+ // restored afterwards). So in this case, we bail out and
+ // delegate the work to the array set slow path.
+ //
+ // TODO: Extend the register allocator to support a new
+ // "(locally) live temp" location so as to avoid always
+ // going into the slow path when read barriers are enabled.
+ __ b(slow_path->GetEntryLabel());
} else {
- __ b(slow_path->GetEntryLabel(), NE);
+ // /* HeapReference<Class> */ temp1 = array->klass_
+ __ LoadFromOffset(kLoadWord, temp1, array, class_offset);
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
+ __ MaybeUnpoisonHeapReference(temp1);
+
+ // /* HeapReference<Class> */ temp1 = temp1->component_type_
+ __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset);
+ // /* HeapReference<Class> */ temp2 = value->klass_
+ __ LoadFromOffset(kLoadWord, temp2, value, class_offset);
+ // If heap poisoning is enabled, no need to unpoison `temp1`
+ // nor `temp2`, as we are comparing two poisoned references.
+ __ cmp(temp1, ShifterOperand(temp2));
+
+ if (instruction->StaticTypeOfArrayIsObjectArray()) {
+ Label do_put;
+ __ b(&do_put, EQ);
+ // If heap poisoning is enabled, the `temp1` reference has
+ // not been unpoisoned yet; unpoison it now.
+ __ MaybeUnpoisonHeapReference(temp1);
+
+ // /* HeapReference<Class> */ temp1 = temp1->super_class_
+ __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset);
+ // If heap poisoning is enabled, no need to unpoison
+ // `temp1`, as we are comparing against null below.
+ __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel());
+ __ Bind(&do_put);
+ } else {
+ __ b(slow_path->GetEntryLabel(), NE);
+ }
}
}
@@ -4188,7 +4496,7 @@
__ StoreToOffset(kStoreWord, source, IP, data_offset);
}
- if (!may_need_runtime_call) {
+ if (!may_need_runtime_call_for_type_check) {
codegen_->MaybeRecordImplicitNullCheck(instruction);
}
@@ -4617,7 +4925,8 @@
CodeGenerator::CreateLoadClassLocationSummary(
cls,
Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
- Location::RegisterLocation(R0));
+ Location::RegisterLocation(R0),
+ /* code_generator_supports_read_barrier */ true);
}
void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) {
@@ -4631,21 +4940,42 @@
return;
}
- Register out = locations->Out().AsRegister<Register>();
+ Location out_loc = locations->Out();
+ Register out = out_loc.AsRegister<Register>();
Register current_method = locations->InAt(0).AsRegister<Register>();
+
if (cls->IsReferrersClass()) {
DCHECK(!cls->CanCallRuntime());
DCHECK(!cls->MustGenerateClinitCheck());
- __ LoadFromOffset(
- kLoadWord, out, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
+ uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value();
+ if (kEmitCompilerReadBarrier) {
+ // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_)
+ __ AddConstant(out, current_method, declaring_class_offset);
+ // /* mirror::Class* */ out = out->Read()
+ codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc);
+ } else {
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ __ LoadFromOffset(kLoadWord, out, current_method, declaring_class_offset);
+ }
} else {
DCHECK(cls->CanCallRuntime());
+ // /* GcRoot<mirror::Class>[] */ out =
+ // current_method.ptr_sized_fields_->dex_cache_resolved_types_
__ LoadFromOffset(kLoadWord,
out,
current_method,
ArtMethod::DexCacheResolvedTypesOffset(kArmPointerSize).Int32Value());
- __ LoadFromOffset(kLoadWord, out, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex()));
- // TODO: We will need a read barrier here.
+
+ size_t cache_offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex());
+ if (kEmitCompilerReadBarrier) {
+ // /* GcRoot<mirror::Class>* */ out = &out[type_index]
+ __ AddConstant(out, out, cache_offset);
+ // /* mirror::Class* */ out = out->Read()
+ codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc);
+ } else {
+ // /* GcRoot<mirror::Class> */ out = out[type_index]
+ __ LoadFromOffset(kLoadWord, out, out, cache_offset);
+ }
SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM(
cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
@@ -4700,13 +5030,35 @@
codegen_->AddSlowPath(slow_path);
LocationSummary* locations = load->GetLocations();
- Register out = locations->Out().AsRegister<Register>();
+ Location out_loc = locations->Out();
+ Register out = out_loc.AsRegister<Register>();
Register current_method = locations->InAt(0).AsRegister<Register>();
- __ LoadFromOffset(
- kLoadWord, out, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
+
+ uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value();
+ if (kEmitCompilerReadBarrier) {
+ // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_)
+ __ AddConstant(out, current_method, declaring_class_offset);
+ // /* mirror::Class* */ out = out->Read()
+ codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc);
+ } else {
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ __ LoadFromOffset(kLoadWord, out, current_method, declaring_class_offset);
+ }
+
+ // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
__ LoadFromOffset(kLoadWord, out, out, mirror::Class::DexCacheStringsOffset().Int32Value());
- __ LoadFromOffset(kLoadWord, out, out, CodeGenerator::GetCacheOffset(load->GetStringIndex()));
- // TODO: We will need a read barrier here.
+
+ size_t cache_offset = CodeGenerator::GetCacheOffset(load->GetStringIndex());
+ if (kEmitCompilerReadBarrier) {
+ // /* GcRoot<mirror::String>* */ out = &out[string_index]
+ __ AddConstant(out, out, cache_offset);
+ // /* mirror::String* */ out = out->Read()
+ codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc);
+ } else {
+ // /* GcRoot<mirror::String> */ out = out[string_index]
+ __ LoadFromOffset(kLoadWord, out, out, cache_offset);
+ }
+
__ CompareAndBranchIfZero(out, slow_path->GetEntryLabel());
__ Bind(slow_path->GetExitLabel());
}
@@ -4749,41 +5101,45 @@
void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) {
LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
- switch (instruction->GetTypeCheckKind()) {
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
+ switch (type_check_kind) {
case TypeCheckKind::kExactCheck:
case TypeCheckKind::kAbstractClassCheck:
case TypeCheckKind::kClassHierarchyCheck:
case TypeCheckKind::kArrayObjectCheck:
- call_kind = LocationSummary::kNoCall;
- break;
- case TypeCheckKind::kUnresolvedCheck:
- case TypeCheckKind::kInterfaceCheck:
- call_kind = LocationSummary::kCall;
+ call_kind =
+ kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall;
break;
case TypeCheckKind::kArrayCheck:
+ case TypeCheckKind::kUnresolvedCheck:
+ case TypeCheckKind::kInterfaceCheck:
call_kind = LocationSummary::kCallOnSlowPath;
break;
}
+
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
- if (call_kind != LocationSummary::kCall) {
- locations->SetInAt(0, Location::RequiresRegister());
- locations->SetInAt(1, Location::RequiresRegister());
- // The out register is used as a temporary, so it overlaps with the inputs.
- // Note that TypeCheckSlowPathARM uses this register too.
- locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
- } else {
- InvokeRuntimeCallingConvention calling_convention;
- locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
- locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
- locations->SetOut(Location::RegisterLocation(R0));
+ locations->SetInAt(0, Location::RequiresRegister());
+ locations->SetInAt(1, Location::RequiresRegister());
+ // The "out" register is used as a temporary, so it overlaps with the inputs.
+ // Note that TypeCheckSlowPathARM uses this register too.
+ locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
+ // When read barriers are enabled, we need a temporary register for
+ // some cases.
+ if (kEmitCompilerReadBarrier &&
+ (type_check_kind == TypeCheckKind::kAbstractClassCheck ||
+ type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
+ type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
+ locations->AddTemp(Location::RequiresRegister());
}
}
void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) {
LocationSummary* locations = instruction->GetLocations();
- Register obj = locations->InAt(0).AsRegister<Register>();
+ Location obj_loc = locations->InAt(0);
+ Register obj = obj_loc.AsRegister<Register>();
Register cls = locations->InAt(1).AsRegister<Register>();
- Register out = locations->Out().AsRegister<Register>();
+ Location out_loc = locations->Out();
+ Register out = out_loc.AsRegister<Register>();
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
@@ -4797,15 +5153,9 @@
__ CompareAndBranchIfZero(obj, &zero);
}
- // In case of an interface/unresolved check, we put the object class into the object register.
- // This is safe, as the register is caller-save, and the object must be in another
- // register if it survives the runtime call.
- Register target = (instruction->GetTypeCheckKind() == TypeCheckKind::kInterfaceCheck) ||
- (instruction->GetTypeCheckKind() == TypeCheckKind::kUnresolvedCheck)
- ? obj
- : out;
- __ LoadFromOffset(kLoadWord, target, obj, class_offset);
- __ MaybeUnpoisonHeapReference(target);
+ // /* HeapReference<Class> */ out = obj->klass_
+ __ LoadFromOffset(kLoadWord, out, obj, class_offset);
+ codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, obj_loc, class_offset);
switch (instruction->GetTypeCheckKind()) {
case TypeCheckKind::kExactCheck: {
@@ -4816,13 +5166,23 @@
__ b(&done);
break;
}
+
case TypeCheckKind::kAbstractClassCheck: {
// If the class is abstract, we eagerly fetch the super class of the
// object to avoid doing a comparison we know will fail.
Label loop;
__ Bind(&loop);
+ Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `out` into `temp` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ Register temp = temp_loc.AsRegister<Register>();
+ __ Mov(temp, out);
+ }
+ // /* HeapReference<Class> */ out = out->super_class_
__ LoadFromOffset(kLoadWord, out, out, super_offset);
- __ MaybeUnpoisonHeapReference(out);
+ codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset);
// If `out` is null, we use it for the result, and jump to `done`.
__ CompareAndBranchIfZero(out, &done);
__ cmp(out, ShifterOperand(cls));
@@ -4833,14 +5193,24 @@
}
break;
}
+
case TypeCheckKind::kClassHierarchyCheck: {
// Walk over the class hierarchy to find a match.
Label loop, success;
__ Bind(&loop);
__ cmp(out, ShifterOperand(cls));
__ b(&success, EQ);
+ Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `out` into `temp` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ Register temp = temp_loc.AsRegister<Register>();
+ __ Mov(temp, out);
+ }
+ // /* HeapReference<Class> */ out = out->super_class_
__ LoadFromOffset(kLoadWord, out, out, super_offset);
- __ MaybeUnpoisonHeapReference(out);
+ codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset);
__ CompareAndBranchIfNonZero(out, &loop);
// If `out` is null, we use it for the result, and jump to `done`.
__ b(&done);
@@ -4851,14 +5221,24 @@
}
break;
}
+
case TypeCheckKind::kArrayObjectCheck: {
// Do an exact check.
Label exact_check;
__ cmp(out, ShifterOperand(cls));
__ b(&exact_check, EQ);
- // Otherwise, we need to check that the object's class is a non primitive array.
+ // Otherwise, we need to check that the object's class is a non-primitive array.
+ Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `out` into `temp` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ Register temp = temp_loc.AsRegister<Register>();
+ __ Mov(temp, out);
+ }
+ // /* HeapReference<Class> */ out = out->component_type_
__ LoadFromOffset(kLoadWord, out, out, component_offset);
- __ MaybeUnpoisonHeapReference(out);
+ codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, component_offset);
// If `out` is null, we use it for the result, and jump to `done`.
__ CompareAndBranchIfZero(out, &done);
__ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset);
@@ -4869,11 +5249,12 @@
__ b(&done);
break;
}
+
case TypeCheckKind::kArrayCheck: {
__ cmp(out, ShifterOperand(cls));
DCHECK(locations->OnlyCallsOnSlowPath());
- slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(
- instruction, /* is_fatal */ false);
+ slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction,
+ /* is_fatal */ false);
codegen_->AddSlowPath(slow_path);
__ b(slow_path->GetEntryLabel(), NE);
__ LoadImmediate(out, 1);
@@ -4882,13 +5263,25 @@
}
break;
}
+
case TypeCheckKind::kUnresolvedCheck:
- case TypeCheckKind::kInterfaceCheck:
- default: {
- codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
- instruction,
- instruction->GetDexPc(),
- nullptr);
+ case TypeCheckKind::kInterfaceCheck: {
+ // Note that we indeed only call on slow path, but we always go
+ // into the slow path for the unresolved & interface check
+ // cases.
+ //
+ // We cannot directly call the InstanceofNonTrivial runtime
+ // entry point without resorting to a type checking slow path
+ // here (i.e. by calling InvokeRuntime directly), as it would
+ // require to assign fixed registers for the inputs of this
+ // HInstanceOf instruction (following the runtime calling
+ // convention), which might be cluttered by the potential first
+ // read barrier emission at the beginning of this method.
+ DCHECK(locations->OnlyCallsOnSlowPath());
+ slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction,
+ /* is_fatal */ false);
+ codegen_->AddSlowPath(slow_path);
+ __ b(slow_path->GetEntryLabel());
if (zero.IsLinked()) {
__ b(&done);
}
@@ -4914,57 +5307,61 @@
LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
- switch (instruction->GetTypeCheckKind()) {
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
+ switch (type_check_kind) {
case TypeCheckKind::kExactCheck:
case TypeCheckKind::kAbstractClassCheck:
case TypeCheckKind::kClassHierarchyCheck:
case TypeCheckKind::kArrayObjectCheck:
- call_kind = throws_into_catch
- ? LocationSummary::kCallOnSlowPath
- : LocationSummary::kNoCall;
- break;
- case TypeCheckKind::kUnresolvedCheck:
- case TypeCheckKind::kInterfaceCheck:
- call_kind = LocationSummary::kCall;
+ call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path.
break;
case TypeCheckKind::kArrayCheck:
+ case TypeCheckKind::kUnresolvedCheck:
+ case TypeCheckKind::kInterfaceCheck:
call_kind = LocationSummary::kCallOnSlowPath;
break;
}
- LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
- instruction, call_kind);
- if (call_kind != LocationSummary::kCall) {
- locations->SetInAt(0, Location::RequiresRegister());
- locations->SetInAt(1, Location::RequiresRegister());
- // Note that TypeCheckSlowPathARM uses this register too.
+ LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
+ locations->SetInAt(0, Location::RequiresRegister());
+ locations->SetInAt(1, Location::RequiresRegister());
+ // Note that TypeCheckSlowPathARM uses this "temp" register too.
+ locations->AddTemp(Location::RequiresRegister());
+ // When read barriers are enabled, we need an additional temporary
+ // register for some cases.
+ if (kEmitCompilerReadBarrier &&
+ (type_check_kind == TypeCheckKind::kAbstractClassCheck ||
+ type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
+ type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
locations->AddTemp(Location::RequiresRegister());
- } else {
- InvokeRuntimeCallingConvention calling_convention;
- locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
- locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
}
}
void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) {
LocationSummary* locations = instruction->GetLocations();
- Register obj = locations->InAt(0).AsRegister<Register>();
+ Location obj_loc = locations->InAt(0);
+ Register obj = obj_loc.AsRegister<Register>();
Register cls = locations->InAt(1).AsRegister<Register>();
- Register temp = locations->WillCall()
- ? Register(kNoRegister)
- : locations->GetTemp(0).AsRegister<Register>();
-
+ Location temp_loc = locations->GetTemp(0);
+ Register temp = temp_loc.AsRegister<Register>();
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
- SlowPathCode* slow_path = nullptr;
- if (!locations->WillCall()) {
- slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(
- instruction, !locations->CanCall());
- codegen_->AddSlowPath(slow_path);
- }
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
+ bool is_type_check_slow_path_fatal =
+ (type_check_kind == TypeCheckKind::kExactCheck ||
+ type_check_kind == TypeCheckKind::kAbstractClassCheck ||
+ type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
+ type_check_kind == TypeCheckKind::kArrayObjectCheck) &&
+ !instruction->CanThrowIntoCatchBlock();
+ SlowPathCode* type_check_slow_path =
+ new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction,
+ is_type_check_slow_path_fatal);
+ codegen_->AddSlowPath(type_check_slow_path);
Label done;
// Avoid null check if we know obj is not null.
@@ -4972,76 +5369,159 @@
__ CompareAndBranchIfZero(obj, &done);
}
- if (locations->WillCall()) {
- __ LoadFromOffset(kLoadWord, obj, obj, class_offset);
- __ MaybeUnpoisonHeapReference(obj);
- } else {
- __ LoadFromOffset(kLoadWord, temp, obj, class_offset);
- __ MaybeUnpoisonHeapReference(temp);
- }
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadWord, temp, obj, class_offset);
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
- switch (instruction->GetTypeCheckKind()) {
+ switch (type_check_kind) {
case TypeCheckKind::kExactCheck:
case TypeCheckKind::kArrayCheck: {
__ cmp(temp, ShifterOperand(cls));
// Jump to slow path for throwing the exception or doing a
// more involved array check.
- __ b(slow_path->GetEntryLabel(), NE);
+ __ b(type_check_slow_path->GetEntryLabel(), NE);
break;
}
+
case TypeCheckKind::kAbstractClassCheck: {
// If the class is abstract, we eagerly fetch the super class of the
// object to avoid doing a comparison we know will fail.
- Label loop;
+ Label loop, compare_classes;
__ Bind(&loop);
+ Location temp2_loc =
+ kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `temp` into `temp2` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ Register temp2 = temp2_loc.AsRegister<Register>();
+ __ Mov(temp2, temp);
+ }
+ // /* HeapReference<Class> */ temp = temp->super_class_
__ LoadFromOffset(kLoadWord, temp, temp, super_offset);
- __ MaybeUnpoisonHeapReference(temp);
- // Jump to the slow path to throw the exception.
- __ CompareAndBranchIfZero(temp, slow_path->GetEntryLabel());
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset);
+
+ // If the class reference currently in `temp` is not null, jump
+ // to the `compare_classes` label to compare it with the checked
+ // class.
+ __ CompareAndBranchIfNonZero(temp, &compare_classes);
+ // Otherwise, jump to the slow path to throw the exception.
+ //
+ // But before, move back the object's class into `temp` before
+ // going into the slow path, as it has been overwritten in the
+ // meantime.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadWord, temp, obj, class_offset);
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
+ __ b(type_check_slow_path->GetEntryLabel());
+
+ __ Bind(&compare_classes);
__ cmp(temp, ShifterOperand(cls));
__ b(&loop, NE);
break;
}
+
case TypeCheckKind::kClassHierarchyCheck: {
// Walk over the class hierarchy to find a match.
Label loop;
__ Bind(&loop);
__ cmp(temp, ShifterOperand(cls));
__ b(&done, EQ);
+
+ Location temp2_loc =
+ kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `temp` into `temp2` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ Register temp2 = temp2_loc.AsRegister<Register>();
+ __ Mov(temp2, temp);
+ }
+ // /* HeapReference<Class> */ temp = temp->super_class_
__ LoadFromOffset(kLoadWord, temp, temp, super_offset);
- __ MaybeUnpoisonHeapReference(temp);
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset);
+
+ // If the class reference currently in `temp` is not null, jump
+ // back at the beginning of the loop.
__ CompareAndBranchIfNonZero(temp, &loop);
- // Jump to the slow path to throw the exception.
- __ b(slow_path->GetEntryLabel());
+ // Otherwise, jump to the slow path to throw the exception.
+ //
+ // But before, move back the object's class into `temp` before
+ // going into the slow path, as it has been overwritten in the
+ // meantime.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadWord, temp, obj, class_offset);
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
+ __ b(type_check_slow_path->GetEntryLabel());
break;
}
+
case TypeCheckKind::kArrayObjectCheck: {
// Do an exact check.
+ Label check_non_primitive_component_type;
__ cmp(temp, ShifterOperand(cls));
__ b(&done, EQ);
- // Otherwise, we need to check that the object's class is a non primitive array.
+
+ // Otherwise, we need to check that the object's class is a non-primitive array.
+ Location temp2_loc =
+ kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `temp` into `temp2` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ Register temp2 = temp2_loc.AsRegister<Register>();
+ __ Mov(temp2, temp);
+ }
+ // /* HeapReference<Class> */ temp = temp->component_type_
__ LoadFromOffset(kLoadWord, temp, temp, component_offset);
- __ MaybeUnpoisonHeapReference(temp);
- __ CompareAndBranchIfZero(temp, slow_path->GetEntryLabel());
+ codegen_->MaybeGenerateReadBarrier(
+ instruction, temp_loc, temp_loc, temp2_loc, component_offset);
+
+ // If the component type is not null (i.e. the object is indeed
+ // an array), jump to label `check_non_primitive_component_type`
+ // to further check that this component type is not a primitive
+ // type.
+ __ CompareAndBranchIfNonZero(temp, &check_non_primitive_component_type);
+ // Otherwise, jump to the slow path to throw the exception.
+ //
+ // But before, move back the object's class into `temp` before
+ // going into the slow path, as it has been overwritten in the
+ // meantime.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadWord, temp, obj, class_offset);
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
+ __ b(type_check_slow_path->GetEntryLabel());
+
+ __ Bind(&check_non_primitive_component_type);
__ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset);
- static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
- __ CompareAndBranchIfNonZero(temp, slow_path->GetEntryLabel());
+ static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot");
+ __ CompareAndBranchIfZero(temp, &done);
+ // Same comment as above regarding `temp` and the slow path.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadWord, temp, obj, class_offset);
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
+ __ b(type_check_slow_path->GetEntryLabel());
break;
}
+
case TypeCheckKind::kUnresolvedCheck:
case TypeCheckKind::kInterfaceCheck:
- default:
- codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
- instruction,
- instruction->GetDexPc(),
- nullptr);
+ // We always go into the type check slow path for the unresolved &
+ // interface check cases.
+ //
+ // We cannot directly call the CheckCast runtime entry point
+ // without resorting to a type checking slow path here (i.e. by
+ // calling InvokeRuntime directly), as it would require to
+ // assign fixed registers for the inputs of this HInstanceOf
+ // instruction (following the runtime calling convention), which
+ // might be cluttered by the potential first read barrier
+ // emission at the beginning of this method.
+ __ b(type_check_slow_path->GetEntryLabel());
break;
}
__ Bind(&done);
- if (slow_path != nullptr) {
- __ Bind(slow_path->GetExitLabel());
- }
+ __ Bind(type_check_slow_path->GetExitLabel());
}
void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) {
@@ -5215,6 +5695,82 @@
}
}
+void CodeGeneratorARM::GenerateReadBarrier(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index) {
+ DCHECK(kEmitCompilerReadBarrier);
+
+ // If heap poisoning is enabled, the unpoisoning of the loaded
+ // reference will be carried out by the runtime within the slow
+ // path.
+ //
+ // Note that `ref` currently does not get unpoisoned (when heap
+ // poisoning is enabled), which is alright as the `ref` argument is
+ // not used by the artReadBarrierSlow entry point.
+ //
+ // TODO: Unpoison `ref` when it is used by artReadBarrierSlow.
+ SlowPathCode* slow_path = new (GetGraph()->GetArena())
+ ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index);
+ AddSlowPath(slow_path);
+
+ // TODO: When read barrier has a fast path, add it here.
+ /* Currently the read barrier call is inserted after the original load.
+ * However, if we have a fast path, we need to perform the load of obj.LockWord *before* the
+ * original load. This load-load ordering is required by the read barrier.
+ * The fast path/slow path (for Baker's algorithm) should look like:
+ *
+ * bool isGray = obj.LockWord & kReadBarrierMask;
+ * lfence; // load fence or artificial data dependence to prevent load-load reordering
+ * ref = obj.field; // this is the original load
+ * if (isGray) {
+ * ref = Mark(ref); // ideally the slow path just does Mark(ref)
+ * }
+ */
+
+ __ b(slow_path->GetEntryLabel());
+ __ Bind(slow_path->GetExitLabel());
+}
+
+void CodeGeneratorARM::MaybeGenerateReadBarrier(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index) {
+ if (kEmitCompilerReadBarrier) {
+ // If heap poisoning is enabled, unpoisoning will be taken care of
+ // by the runtime within the slow path.
+ GenerateReadBarrier(instruction, out, ref, obj, offset, index);
+ } else if (kPoisonHeapReferences) {
+ __ UnpoisonHeapReference(out.AsRegister<Register>());
+ }
+}
+
+void CodeGeneratorARM::GenerateReadBarrierForRoot(HInstruction* instruction,
+ Location out,
+ Location root) {
+ DCHECK(kEmitCompilerReadBarrier);
+
+ // Note that GC roots are not affected by heap poisoning, so we do
+ // not need to do anything special for this here.
+ SlowPathCode* slow_path =
+ new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root);
+ AddSlowPath(slow_path);
+
+ // TODO: Implement a fast path for ReadBarrierForRoot, performing
+ // the following operation (for Baker's algorithm):
+ //
+ // if (thread.tls32_.is_gc_marking) {
+ // root = Mark(root);
+ // }
+
+ __ b(slow_path->GetEntryLabel());
+ __ Bind(slow_path->GetExitLabel());
+}
+
HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch(
const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
MethodReference target_method) {
@@ -5272,7 +5828,7 @@
__ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, invoke->GetStringInitOffset());
break;
case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
- callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
+ callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
__ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress());
@@ -5287,7 +5843,7 @@
LOG(FATAL) << "Unsupported";
UNREACHABLE();
case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
- Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
+ Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
Register method_reg;
Register reg = temp.AsRegister<Register>();
if (current_method.IsRegister()) {
@@ -5298,10 +5854,11 @@
method_reg = reg;
__ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset);
}
- // temp = current_method->dex_cache_resolved_methods_;
- __ LoadFromOffset(
- kLoadWord, reg, method_reg, ArtMethod::DexCacheResolvedMethodsOffset(
- kArmPointerSize).Int32Value());
+ // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_;
+ __ LoadFromOffset(kLoadWord,
+ reg,
+ method_reg,
+ ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value());
// temp = temp[index_in_cache]
uint32_t index_in_cache = invoke->GetTargetMethod().dex_method_index;
__ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache));
@@ -5345,10 +5902,17 @@
LocationSummary* locations = invoke->GetLocations();
Location receiver = locations->InAt(0);
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
- // temp = object->GetClass();
DCHECK(receiver.IsRegister());
+ // /* HeapReference<Class> */ temp = receiver->klass_
__ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset);
MaybeRecordImplicitNullCheck(invoke);
+ // Instead of simply (possibly) unpoisoning `temp` here, we should
+ // emit a read barrier for the previous class reference load.
+ // However this is not required in practice, as this is an
+ // intermediate/temporary reference and because the current
+ // concurrent copying collector keeps the from-space memory
+ // intact/accessible until the end of the marking phase (the
+ // concurrent copying collector may not in the future).
__ MaybeUnpoisonHeapReference(temp);
// temp = temp->GetMethodAt(method_offset);
uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(
diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h
index cef1095..89de4f8 100644
--- a/compiler/optimizing/code_generator_arm.h
+++ b/compiler/optimizing/code_generator_arm.h
@@ -228,15 +228,13 @@
void GenerateImplicitNullCheck(HNullCheck* instruction);
void GenerateExplicitNullCheck(HNullCheck* instruction);
void GenerateTestAndBranch(HInstruction* instruction,
+ size_t condition_input_index,
Label* true_target,
- Label* false_target,
- Label* always_true_target);
+ Label* false_target);
void GenerateCompareWithImmediate(Register left, int32_t right);
- void GenerateCompareTestAndBranch(HIf* if_instr,
- HCondition* condition,
+ void GenerateCompareTestAndBranch(HCondition* condition,
Label* true_target,
- Label* false_target,
- Label* always_true_target);
+ Label* false_target);
void GenerateFPJumps(HCondition* cond, Label* true_label, Label* false_label);
void GenerateLongComparesAndJumps(HCondition* cond, Label* true_label, Label* false_label);
void DivRemOneOrMinusOne(HBinaryOperation* instruction);
@@ -375,6 +373,51 @@
void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE;
+ // Generate a read barrier for a heap reference within `instruction`.
+ //
+ // A read barrier for an object reference read from the heap is
+ // implemented as a call to the artReadBarrierSlow runtime entry
+ // point, which is passed the values in locations `ref`, `obj`, and
+ // `offset`:
+ //
+ // mirror::Object* artReadBarrierSlow(mirror::Object* ref,
+ // mirror::Object* obj,
+ // uint32_t offset);
+ //
+ // The `out` location contains the value returned by
+ // artReadBarrierSlow.
+ //
+ // When `index` is provided (i.e. for array accesses), the offset
+ // value passed to artReadBarrierSlow is adjusted to take `index`
+ // into account.
+ void GenerateReadBarrier(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index = Location::NoLocation());
+
+ // If read barriers are enabled, generate a read barrier for a heap reference.
+ // If heap poisoning is enabled, also unpoison the reference in `out`.
+ void MaybeGenerateReadBarrier(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index = Location::NoLocation());
+
+ // Generate a read barrier for a GC root within `instruction`.
+ //
+ // A read barrier for an object reference GC root is implemented as
+ // a call to the artReadBarrierForRootSlow runtime entry point,
+ // which is passed the value in location `root`:
+ //
+ // mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root);
+ //
+ // The `out` location contains the value returned by
+ // artReadBarrierForRootSlow.
+ void GenerateReadBarrierForRoot(HInstruction* instruction, Location out, Location root);
+
private:
using MethodToLiteralMap = ArenaSafeMap<MethodReference, Literal*, MethodReferenceComparator>;
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index 7e248b4..2776b7d 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -68,6 +68,10 @@
using helpers::ArtVixlRegCodeCoherentForRegSet;
static constexpr int kCurrentMethodStackOffset = 0;
+// The compare/jump sequence will generate about (2 * num_entries + 1) instructions. While jump
+// table version generates 7 instructions and num_entries literals. Compare/jump sequence will
+// generates less code/data with a small num_entries.
+static constexpr uint32_t kPackedSwitchJumpTableThreshold = 6;
inline Condition ARM64Condition(IfCondition cond) {
switch (cond) {
@@ -545,6 +549,28 @@
DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM64);
};
+void JumpTableARM64::EmitTable(CodeGeneratorARM64* codegen) {
+ uint32_t num_entries = switch_instr_->GetNumEntries();
+ DCHECK_GE(num_entries, kPackedSwitchJumpTableThreshold);
+
+ // We are about to use the assembler to place literals directly. Make sure we have enough
+ // underlying code buffer and we have generated the jump table with right size.
+ CodeBufferCheckScope scope(codegen->GetVIXLAssembler(), num_entries * sizeof(int32_t),
+ CodeBufferCheckScope::kCheck, CodeBufferCheckScope::kExactSize);
+
+ __ Bind(&table_start_);
+ const ArenaVector<HBasicBlock*>& successors = switch_instr_->GetBlock()->GetSuccessors();
+ for (uint32_t i = 0; i < num_entries; i++) {
+ vixl::Label* target_label = codegen->GetLabelOf(successors[i]);
+ DCHECK(target_label->IsBound());
+ ptrdiff_t jump_offset = target_label->location() - table_start_.location();
+ DCHECK_GT(jump_offset, std::numeric_limits<int32_t>::min());
+ DCHECK_LE(jump_offset, std::numeric_limits<int32_t>::max());
+ Literal<int32_t> literal(jump_offset);
+ __ place(&literal);
+ }
+}
+
#undef __
Location InvokeDexCallingConventionVisitorARM64::GetNextLocation(Primitive::Type type) {
@@ -587,6 +613,7 @@
compiler_options,
stats),
block_labels_(nullptr),
+ jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
location_builder_(graph, this),
instruction_visitor_(graph, this),
move_resolver_(graph->GetArena(), this),
@@ -603,10 +630,16 @@
AddAllocatedRegister(LocationFrom(lr));
}
-#undef __
#define __ GetVIXLAssembler()->
+void CodeGeneratorARM64::EmitJumpTables() {
+ for (auto jump_table : jump_tables_) {
+ jump_table->EmitTable(this);
+ }
+}
+
void CodeGeneratorARM64::Finalize(CodeAllocator* allocator) {
+ EmitJumpTables();
// Ensure we emit the literal pool.
__ FinalizeCode();
@@ -2283,38 +2316,56 @@
}
void InstructionCodeGeneratorARM64::GenerateTestAndBranch(HInstruction* instruction,
+ size_t condition_input_index,
vixl::Label* true_target,
- vixl::Label* false_target,
- vixl::Label* always_true_target) {
- HInstruction* cond = instruction->InputAt(0);
- HCondition* condition = cond->AsCondition();
+ vixl::Label* false_target) {
+ // FP branching requires both targets to be explicit. If either of the targets
+ // is nullptr (fallthrough) use and bind `fallthrough_target` instead.
+ vixl::Label fallthrough_target;
+ HInstruction* cond = instruction->InputAt(condition_input_index);
- if (cond->IsIntConstant()) {
- int32_t cond_value = cond->AsIntConstant()->GetValue();
- if (cond_value == 1) {
- if (always_true_target != nullptr) {
- __ B(always_true_target);
+ if (true_target == nullptr && false_target == nullptr) {
+ // Nothing to do. The code always falls through.
+ return;
+ } else if (cond->IsIntConstant()) {
+ // Constant condition, statically compared against 1.
+ if (cond->AsIntConstant()->IsOne()) {
+ if (true_target != nullptr) {
+ __ B(true_target);
}
- return;
} else {
- DCHECK_EQ(cond_value, 0);
+ DCHECK(cond->AsIntConstant()->IsZero());
+ if (false_target != nullptr) {
+ __ B(false_target);
+ }
}
- } else if (!cond->IsCondition() || condition->NeedsMaterialization()) {
+ return;
+ }
+
+ // The following code generates these patterns:
+ // (1) true_target == nullptr && false_target != nullptr
+ // - opposite condition true => branch to false_target
+ // (2) true_target != nullptr && false_target == nullptr
+ // - condition true => branch to true_target
+ // (3) true_target != nullptr && false_target != nullptr
+ // - condition true => branch to true_target
+ // - branch to false_target
+ if (IsBooleanValueOrMaterializedCondition(cond)) {
// The condition instruction has been materialized, compare the output to 0.
- Location cond_val = instruction->GetLocations()->InAt(0);
+ Location cond_val = instruction->GetLocations()->InAt(condition_input_index);
DCHECK(cond_val.IsRegister());
- __ Cbnz(InputRegisterAt(instruction, 0), true_target);
+ if (true_target == nullptr) {
+ __ Cbz(InputRegisterAt(instruction, condition_input_index), false_target);
+ } else {
+ __ Cbnz(InputRegisterAt(instruction, condition_input_index), true_target);
+ }
} else {
// The condition instruction has not been materialized, use its inputs as
// the comparison and its condition as the branch condition.
- Primitive::Type type =
- cond->IsCondition() ? cond->InputAt(0)->GetType() : Primitive::kPrimInt;
+ HCondition* condition = cond->AsCondition();
+ Primitive::Type type = condition->InputAt(0)->GetType();
if (Primitive::IsFloatingPointType(type)) {
- // FP compares don't like null false_targets.
- if (false_target == nullptr) {
- false_target = codegen_->GetLabelOf(instruction->AsIf()->IfFalseSuccessor());
- }
FPRegister lhs = InputFPRegisterAt(condition, 0);
if (condition->GetLocations()->InAt(1).IsConstant()) {
DCHECK(IsFloatingPointZeroConstant(condition->GetLocations()->InAt(1).GetConstant()));
@@ -2324,31 +2375,45 @@
__ Fcmp(lhs, InputFPRegisterAt(condition, 1));
}
if (condition->IsFPConditionTrueIfNaN()) {
- __ B(vs, true_target); // VS for unordered.
+ __ B(vs, true_target == nullptr ? &fallthrough_target : true_target);
} else if (condition->IsFPConditionFalseIfNaN()) {
- __ B(vs, false_target); // VS for unordered.
+ __ B(vs, false_target == nullptr ? &fallthrough_target : false_target);
}
- __ B(ARM64Condition(condition->GetCondition()), true_target);
+ if (true_target == nullptr) {
+ __ B(ARM64Condition(condition->GetOppositeCondition()), false_target);
+ } else {
+ __ B(ARM64Condition(condition->GetCondition()), true_target);
+ }
} else {
// Integer cases.
Register lhs = InputRegisterAt(condition, 0);
Operand rhs = InputOperandAt(condition, 1);
- Condition arm64_cond = ARM64Condition(condition->GetCondition());
+
+ Condition arm64_cond;
+ vixl::Label* non_fallthrough_target;
+ if (true_target == nullptr) {
+ arm64_cond = ARM64Condition(condition->GetOppositeCondition());
+ non_fallthrough_target = false_target;
+ } else {
+ arm64_cond = ARM64Condition(condition->GetCondition());
+ non_fallthrough_target = true_target;
+ }
+
if ((arm64_cond != gt && arm64_cond != le) && rhs.IsImmediate() && (rhs.immediate() == 0)) {
switch (arm64_cond) {
case eq:
- __ Cbz(lhs, true_target);
+ __ Cbz(lhs, non_fallthrough_target);
break;
case ne:
- __ Cbnz(lhs, true_target);
+ __ Cbnz(lhs, non_fallthrough_target);
break;
case lt:
// Test the sign bit and branch accordingly.
- __ Tbnz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, true_target);
+ __ Tbnz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, non_fallthrough_target);
break;
case ge:
// Test the sign bit and branch accordingly.
- __ Tbz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, true_target);
+ __ Tbz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, non_fallthrough_target);
break;
default:
// Without the `static_cast` the compiler throws an error for
@@ -2357,43 +2422,43 @@
}
} else {
__ Cmp(lhs, rhs);
- __ B(arm64_cond, true_target);
+ __ B(arm64_cond, non_fallthrough_target);
}
}
}
- if (false_target != nullptr) {
+
+ // If neither branch falls through (case 3), the conditional branch to `true_target`
+ // was already emitted (case 2) and we need to emit a jump to `false_target`.
+ if (true_target != nullptr && false_target != nullptr) {
__ B(false_target);
}
+
+ if (fallthrough_target.IsLinked()) {
+ __ Bind(&fallthrough_target);
+ }
}
void LocationsBuilderARM64::VisitIf(HIf* if_instr) {
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
- HInstruction* cond = if_instr->InputAt(0);
- if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
+ if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
locations->SetInAt(0, Location::RequiresRegister());
}
}
void InstructionCodeGeneratorARM64::VisitIf(HIf* if_instr) {
- vixl::Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
- vixl::Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
- vixl::Label* always_true_target = true_target;
- if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
- if_instr->IfTrueSuccessor())) {
- always_true_target = nullptr;
- }
- if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
- if_instr->IfFalseSuccessor())) {
- false_target = nullptr;
- }
- GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
+ HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
+ HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
+ vixl::Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
+ nullptr : codegen_->GetLabelOf(true_successor);
+ vixl::Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
+ nullptr : codegen_->GetLabelOf(false_successor);
+ GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
}
void LocationsBuilderARM64::VisitDeoptimize(HDeoptimize* deoptimize) {
LocationSummary* locations = new (GetGraph()->GetArena())
LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
- HInstruction* cond = deoptimize->InputAt(0);
- if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
+ if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
locations->SetInAt(0, Location::RequiresRegister());
}
}
@@ -2402,8 +2467,10 @@
SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena())
DeoptimizationSlowPathARM64(deoptimize);
codegen_->AddSlowPath(slow_path);
- vixl::Label* slow_path_entry = slow_path->GetEntryLabel();
- GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
+ GenerateTestAndBranch(deoptimize,
+ /* condition_input_index */ 0,
+ slow_path->GetEntryLabel(),
+ /* false_target */ nullptr);
}
void LocationsBuilderARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
@@ -2856,18 +2923,18 @@
switch (invoke->GetMethodLoadKind()) {
case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
// temp = thread->string_init_entrypoint
- __ Ldr(XRegisterFrom(temp).X(), MemOperand(tr, invoke->GetStringInitOffset()));
+ __ Ldr(XRegisterFrom(temp), MemOperand(tr, invoke->GetStringInitOffset()));
break;
case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
- callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
+ callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
// Load method address from literal pool.
- __ Ldr(XRegisterFrom(temp).X(), DeduplicateUint64Literal(invoke->GetMethodAddress()));
+ __ Ldr(XRegisterFrom(temp), DeduplicateUint64Literal(invoke->GetMethodAddress()));
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
// Load method address from literal pool with a link-time patch.
- __ Ldr(XRegisterFrom(temp).X(),
+ __ Ldr(XRegisterFrom(temp),
DeduplicateMethodAddressLiteral(invoke->GetTargetMethod()));
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
@@ -2877,20 +2944,23 @@
vixl::Label* pc_insn_label = &pc_relative_dex_cache_patches_.back().label;
{
vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
- __ adrp(XRegisterFrom(temp).X(), 0);
+ __ Bind(pc_insn_label);
+ __ adrp(XRegisterFrom(temp), 0);
}
- __ Bind(pc_insn_label); // Bind after ADRP.
pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
// Add LDR with its PC-relative DexCache access patch.
pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
invoke->GetDexCacheArrayOffset());
- __ Ldr(XRegisterFrom(temp).X(), MemOperand(XRegisterFrom(temp).X(), 0));
- __ Bind(&pc_relative_dex_cache_patches_.back().label); // Bind after LDR.
- pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
+ {
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(&pc_relative_dex_cache_patches_.back().label);
+ __ ldr(XRegisterFrom(temp), MemOperand(XRegisterFrom(temp), 0));
+ pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
+ }
break;
}
case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
- Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
+ Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
Register reg = XRegisterFrom(temp);
Register method_reg;
if (current_method.IsRegister()) {
@@ -2920,8 +2990,9 @@
case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
relative_call_patches_.emplace_back(invoke->GetTargetMethod());
vixl::Label* label = &relative_call_patches_.back().label;
- __ Bl(label); // Arbitrarily branch to the instruction after BL, override at link time.
- __ Bind(label); // Bind after BL.
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(label);
+ __ bl(0); // Branch and link to itself. This will be overriden at link time.
break;
}
case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
@@ -2934,7 +3005,7 @@
case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
// LR = callee_method->entry_point_from_quick_compiled_code_;
__ Ldr(lr, MemOperand(
- XRegisterFrom(callee_method).X(),
+ XRegisterFrom(callee_method),
ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize).Int32Value()));
// lr()
__ Blr(lr);
@@ -2990,14 +3061,14 @@
target_method.dex_method_index));
}
for (const MethodPatchInfo<vixl::Label>& info : relative_call_patches_) {
- linker_patches->push_back(LinkerPatch::RelativeCodePatch(info.label.location() - 4u,
+ linker_patches->push_back(LinkerPatch::RelativeCodePatch(info.label.location(),
info.target_method.dex_file,
info.target_method.dex_method_index));
}
for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) {
- linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.location() - 4u,
+ linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.location(),
&info.target_dex_file,
- info.pc_insn_label->location() - 4u,
+ info.pc_insn_label->location(),
info.element_offset));
}
}
@@ -3810,26 +3881,73 @@
void InstructionCodeGeneratorARM64::VisitPackedSwitch(HPackedSwitch* switch_instr) {
int32_t lower_bound = switch_instr->GetStartValue();
- int32_t num_entries = switch_instr->GetNumEntries();
+ uint32_t num_entries = switch_instr->GetNumEntries();
Register value_reg = InputRegisterAt(switch_instr, 0);
HBasicBlock* default_block = switch_instr->GetDefaultBlock();
- // Create a series of compare/jumps.
- const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors();
- for (int32_t i = 0; i < num_entries; i++) {
- int32_t case_value = lower_bound + i;
- vixl::Label* succ = codegen_->GetLabelOf(successors[i]);
- if (case_value == 0) {
- __ Cbz(value_reg, succ);
- } else {
- __ Cmp(value_reg, vixl::Operand(case_value));
- __ B(eq, succ);
- }
- }
+ // Roughly set 16 as max average assemblies generated per HIR in a graph.
+ static constexpr int32_t kMaxExpectedSizePerHInstruction = 16 * vixl::kInstructionSize;
+ // ADR has a limited range(+/-1MB), so we set a threshold for the number of HIRs in the graph to
+ // make sure we don't emit it if the target may run out of range.
+ // TODO: Instead of emitting all jump tables at the end of the code, we could keep track of ADR
+ // ranges and emit the tables only as required.
+ static constexpr int32_t kJumpTableInstructionThreshold = 1* MB / kMaxExpectedSizePerHInstruction;
- // And the default for any other value.
- if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) {
- __ B(codegen_->GetLabelOf(default_block));
+ if (num_entries < kPackedSwitchJumpTableThreshold ||
+ // Current instruction id is an upper bound of the number of HIRs in the graph.
+ GetGraph()->GetCurrentInstructionId() > kJumpTableInstructionThreshold) {
+ // Create a series of compare/jumps.
+ const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors();
+ for (uint32_t i = 0; i < num_entries; i++) {
+ int32_t case_value = lower_bound + i;
+ vixl::Label* succ = codegen_->GetLabelOf(successors[i]);
+ if (case_value == 0) {
+ __ Cbz(value_reg, succ);
+ } else {
+ __ Cmp(value_reg, Operand(case_value));
+ __ B(eq, succ);
+ }
+ }
+
+ // And the default for any other value.
+ if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) {
+ __ B(codegen_->GetLabelOf(default_block));
+ }
+ } else {
+ JumpTableARM64* jump_table = new (GetGraph()->GetArena()) JumpTableARM64(switch_instr);
+ codegen_->AddJumpTable(jump_table);
+
+ UseScratchRegisterScope temps(codegen_->GetVIXLAssembler());
+
+ // Below instructions should use at most one blocked register. Since there are two blocked
+ // registers, we are free to block one.
+ Register temp_w = temps.AcquireW();
+ Register index;
+ // Remove the bias.
+ if (lower_bound != 0) {
+ index = temp_w;
+ __ Sub(index, value_reg, Operand(lower_bound));
+ } else {
+ index = value_reg;
+ }
+
+ // Jump to default block if index is out of the range.
+ __ Cmp(index, Operand(num_entries));
+ __ B(hs, codegen_->GetLabelOf(default_block));
+
+ // In current VIXL implementation, it won't require any blocked registers to encode the
+ // immediate value for Adr. So we are free to use both VIXL blocked registers to reduce the
+ // register pressure.
+ Register table_base = temps.AcquireX();
+ // Load jump offset from the table.
+ __ Adr(table_base, jump_table->GetTableStartLabel());
+ Register jump_offset = temp_w;
+ __ Ldr(jump_offset, MemOperand(table_base, index, UXTW, 2));
+
+ // Jump to target block by branching to table_base(pc related) + offset.
+ Register target_address = table_base;
+ __ Add(target_address, table_base, Operand(jump_offset, SXTW));
+ __ Br(target_address);
}
}
diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h
index aa5ad38..881afcc 100644
--- a/compiler/optimizing/code_generator_arm64.h
+++ b/compiler/optimizing/code_generator_arm64.h
@@ -81,6 +81,22 @@
DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM64);
};
+class JumpTableARM64 : public ArenaObject<kArenaAllocSwitchTable> {
+ public:
+ explicit JumpTableARM64(HPackedSwitch* switch_instr)
+ : switch_instr_(switch_instr), table_start_() {}
+
+ vixl::Label* GetTableStartLabel() { return &table_start_; }
+
+ void EmitTable(CodeGeneratorARM64* codegen);
+
+ private:
+ HPackedSwitch* const switch_instr_;
+ vixl::Label table_start_;
+
+ DISALLOW_COPY_AND_ASSIGN(JumpTableARM64);
+};
+
static const vixl::Register kRuntimeParameterCoreRegisters[] =
{ vixl::x0, vixl::x1, vixl::x2, vixl::x3, vixl::x4, vixl::x5, vixl::x6, vixl::x7 };
static constexpr size_t kRuntimeParameterCoreRegistersLength =
@@ -203,9 +219,9 @@
void GenerateImplicitNullCheck(HNullCheck* instruction);
void GenerateExplicitNullCheck(HNullCheck* instruction);
void GenerateTestAndBranch(HInstruction* instruction,
+ size_t condition_input_index,
vixl::Label* true_target,
- vixl::Label* false_target,
- vixl::Label* always_true_target);
+ vixl::Label* false_target);
void DivRemOneOrMinusOne(HBinaryOperation* instruction);
void DivRemByPowerOfTwo(HBinaryOperation* instruction);
void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction);
@@ -358,6 +374,10 @@
block_labels_ = CommonInitializeLabels<vixl::Label>();
}
+ void AddJumpTable(JumpTableARM64* jump_table) {
+ jump_tables_.push_back(jump_table);
+ }
+
void Finalize(CodeAllocator* allocator) OVERRIDE;
// Code generation helpers.
@@ -422,15 +442,16 @@
const DexFile& target_dex_file;
uint32_t element_offset;
- // NOTE: Labels are bound to the end of the patched instruction because
- // we don't know if there will be a veneer or how big it will be.
vixl::Label label;
vixl::Label* pc_insn_label;
};
+ void EmitJumpTables();
+
// Labels for each block that will be compiled.
vixl::Label* block_labels_; // Indexed by block id.
vixl::Label frame_entry_label_;
+ ArenaVector<JumpTableARM64*> jump_tables_;
LocationsBuilderARM64 location_builder_;
InstructionCodeGeneratorARM64 instruction_visitor_;
diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc
index 959adb4..801e203 100644
--- a/compiler/optimizing/code_generator_mips.cc
+++ b/compiler/optimizing/code_generator_mips.cc
@@ -2420,30 +2420,51 @@
}
void InstructionCodeGeneratorMIPS::GenerateTestAndBranch(HInstruction* instruction,
+ size_t condition_input_index,
MipsLabel* true_target,
- MipsLabel* false_target,
- MipsLabel* always_true_target) {
- HInstruction* cond = instruction->InputAt(0);
- HCondition* condition = cond->AsCondition();
+ MipsLabel* false_target) {
+ HInstruction* cond = instruction->InputAt(condition_input_index);
- if (cond->IsIntConstant()) {
- int32_t cond_value = cond->AsIntConstant()->GetValue();
- if (cond_value == 1) {
- if (always_true_target != nullptr) {
- __ B(always_true_target);
+ if (true_target == nullptr && false_target == nullptr) {
+ // Nothing to do. The code always falls through.
+ return;
+ } else if (cond->IsIntConstant()) {
+ // Constant condition, statically compared against 1.
+ if (cond->AsIntConstant()->IsOne()) {
+ if (true_target != nullptr) {
+ __ B(true_target);
}
- return;
} else {
- DCHECK_EQ(cond_value, 0);
+ DCHECK(cond->AsIntConstant()->IsZero());
+ if (false_target != nullptr) {
+ __ B(false_target);
+ }
}
- } else if (!cond->IsCondition() || condition->NeedsMaterialization()) {
+ return;
+ }
+
+ // The following code generates these patterns:
+ // (1) true_target == nullptr && false_target != nullptr
+ // - opposite condition true => branch to false_target
+ // (2) true_target != nullptr && false_target == nullptr
+ // - condition true => branch to true_target
+ // (3) true_target != nullptr && false_target != nullptr
+ // - condition true => branch to true_target
+ // - branch to false_target
+ if (IsBooleanValueOrMaterializedCondition(cond)) {
// The condition instruction has been materialized, compare the output to 0.
- Location cond_val = instruction->GetLocations()->InAt(0);
+ Location cond_val = instruction->GetLocations()->InAt(condition_input_index);
DCHECK(cond_val.IsRegister());
- __ Bnez(cond_val.AsRegister<Register>(), true_target);
+ if (true_target == nullptr) {
+ __ Beqz(cond_val.AsRegister<Register>(), false_target);
+ } else {
+ __ Bnez(cond_val.AsRegister<Register>(), true_target);
+ }
} else {
// The condition instruction has not been materialized, use its inputs as
// the comparison and its condition as the branch condition.
+ HCondition* condition = cond->AsCondition();
+
Register lhs = condition->GetLocations()->InAt(0).AsRegister<Register>();
Location rhs_location = condition->GetLocations()->InAt(1);
Register rhs_reg = ZERO;
@@ -2455,37 +2476,46 @@
rhs_reg = rhs_location.AsRegister<Register>();
}
- IfCondition if_cond = condition->GetCondition();
+ IfCondition if_cond;
+ MipsLabel* non_fallthrough_target;
+ if (true_target == nullptr) {
+ if_cond = condition->GetOppositeCondition();
+ non_fallthrough_target = false_target;
+ } else {
+ if_cond = condition->GetCondition();
+ non_fallthrough_target = true_target;
+ }
+
if (use_imm && rhs_imm == 0) {
switch (if_cond) {
case kCondEQ:
- __ Beqz(lhs, true_target);
+ __ Beqz(lhs, non_fallthrough_target);
break;
case kCondNE:
- __ Bnez(lhs, true_target);
+ __ Bnez(lhs, non_fallthrough_target);
break;
case kCondLT:
- __ Bltz(lhs, true_target);
+ __ Bltz(lhs, non_fallthrough_target);
break;
case kCondGE:
- __ Bgez(lhs, true_target);
+ __ Bgez(lhs, non_fallthrough_target);
break;
case kCondLE:
- __ Blez(lhs, true_target);
+ __ Blez(lhs, non_fallthrough_target);
break;
case kCondGT:
- __ Bgtz(lhs, true_target);
+ __ Bgtz(lhs, non_fallthrough_target);
break;
case kCondB:
break; // always false
case kCondBE:
- __ Beqz(lhs, true_target); // <= 0 if zero
+ __ Beqz(lhs, non_fallthrough_target); // <= 0 if zero
break;
case kCondA:
- __ Bnez(lhs, true_target); // > 0 if non-zero
+ __ Bnez(lhs, non_fallthrough_target); // > 0 if non-zero
break;
case kCondAE:
- __ B(true_target); // always true
+ __ B(non_fallthrough_target); // always true
break;
}
} else {
@@ -2496,81 +2526,78 @@
}
switch (if_cond) {
case kCondEQ:
- __ Beq(lhs, rhs_reg, true_target);
+ __ Beq(lhs, rhs_reg, non_fallthrough_target);
break;
case kCondNE:
- __ Bne(lhs, rhs_reg, true_target);
+ __ Bne(lhs, rhs_reg, non_fallthrough_target);
break;
case kCondLT:
- __ Blt(lhs, rhs_reg, true_target);
+ __ Blt(lhs, rhs_reg, non_fallthrough_target);
break;
case kCondGE:
- __ Bge(lhs, rhs_reg, true_target);
+ __ Bge(lhs, rhs_reg, non_fallthrough_target);
break;
case kCondLE:
- __ Bge(rhs_reg, lhs, true_target);
+ __ Bge(rhs_reg, lhs, non_fallthrough_target);
break;
case kCondGT:
- __ Blt(rhs_reg, lhs, true_target);
+ __ Blt(rhs_reg, lhs, non_fallthrough_target);
break;
case kCondB:
- __ Bltu(lhs, rhs_reg, true_target);
+ __ Bltu(lhs, rhs_reg, non_fallthrough_target);
break;
case kCondAE:
- __ Bgeu(lhs, rhs_reg, true_target);
+ __ Bgeu(lhs, rhs_reg, non_fallthrough_target);
break;
case kCondBE:
- __ Bgeu(rhs_reg, lhs, true_target);
+ __ Bgeu(rhs_reg, lhs, non_fallthrough_target);
break;
case kCondA:
- __ Bltu(rhs_reg, lhs, true_target);
+ __ Bltu(rhs_reg, lhs, non_fallthrough_target);
break;
}
}
}
- if (false_target != nullptr) {
+
+ // If neither branch falls through (case 3), the conditional branch to `true_target`
+ // was already emitted (case 2) and we need to emit a jump to `false_target`.
+ if (true_target != nullptr && false_target != nullptr) {
__ B(false_target);
}
}
void LocationsBuilderMIPS::VisitIf(HIf* if_instr) {
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
- HInstruction* cond = if_instr->InputAt(0);
- if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
+ if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
locations->SetInAt(0, Location::RequiresRegister());
}
}
void InstructionCodeGeneratorMIPS::VisitIf(HIf* if_instr) {
- MipsLabel* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
- MipsLabel* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
- MipsLabel* always_true_target = true_target;
- if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
- if_instr->IfTrueSuccessor())) {
- always_true_target = nullptr;
- }
- if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
- if_instr->IfFalseSuccessor())) {
- false_target = nullptr;
- }
- GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
+ HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
+ HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
+ MipsLabel* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
+ nullptr : codegen_->GetLabelOf(true_successor);
+ MipsLabel* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
+ nullptr : codegen_->GetLabelOf(false_successor);
+ GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
}
void LocationsBuilderMIPS::VisitDeoptimize(HDeoptimize* deoptimize) {
LocationSummary* locations = new (GetGraph()->GetArena())
LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
- HInstruction* cond = deoptimize->InputAt(0);
- if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
+ if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
locations->SetInAt(0, Location::RequiresRegister());
}
}
void InstructionCodeGeneratorMIPS::VisitDeoptimize(HDeoptimize* deoptimize) {
- SlowPathCodeMIPS* slow_path = new (GetGraph()->GetArena())
- DeoptimizationSlowPathMIPS(deoptimize);
+ SlowPathCodeMIPS* slow_path = new (GetGraph()->GetArena()) DeoptimizationSlowPathMIPS(deoptimize);
codegen_->AddSlowPath(slow_path);
- MipsLabel* slow_path_entry = slow_path->GetEntryLabel();
- GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
+ GenerateTestAndBranch(deoptimize,
+ /* condition_input_index */ 0,
+ slow_path->GetEntryLabel(),
+ /* false_target */ nullptr);
}
void LocationsBuilderMIPS::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
@@ -3004,7 +3031,7 @@
invoke->GetStringInitOffset());
break;
case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
- callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
+ callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
__ LoadConst32(temp.AsRegister<Register>(), invoke->GetMethodAddress());
@@ -3016,7 +3043,7 @@
LOG(FATAL) << "Unsupported";
UNREACHABLE();
case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
- Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
+ Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
Register reg = temp.AsRegister<Register>();
Register method_reg;
if (current_method.IsRegister()) {
diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h
index 059131d..e3a2cb4 100644
--- a/compiler/optimizing/code_generator_mips.h
+++ b/compiler/optimizing/code_generator_mips.h
@@ -226,9 +226,9 @@
void GenerateImplicitNullCheck(HNullCheck* instruction);
void GenerateExplicitNullCheck(HNullCheck* instruction);
void GenerateTestAndBranch(HInstruction* instruction,
+ size_t condition_input_index,
MipsLabel* true_target,
- MipsLabel* false_target,
- MipsLabel* always_true_target);
+ MipsLabel* false_target);
void HandleGoto(HInstruction* got, HBasicBlock* successor);
MipsAssembler* const assembler_;
diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc
index 9b78dec..7b33075 100644
--- a/compiler/optimizing/code_generator_mips64.cc
+++ b/compiler/optimizing/code_generator_mips64.cc
@@ -420,7 +420,7 @@
: CodeGenerator(graph,
kNumberOfGpuRegisters,
kNumberOfFpuRegisters,
- 0, // kNumberOfRegisterPairs
+ /* number_of_register_pairs */ 0,
ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
arraysize(kCoreCalleeSaves)),
ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves),
@@ -2340,30 +2340,51 @@
}
void InstructionCodeGeneratorMIPS64::GenerateTestAndBranch(HInstruction* instruction,
+ size_t condition_input_index,
Label* true_target,
- Label* false_target,
- Label* always_true_target) {
- HInstruction* cond = instruction->InputAt(0);
- HCondition* condition = cond->AsCondition();
+ Label* false_target) {
+ HInstruction* cond = instruction->InputAt(condition_input_index);
- if (cond->IsIntConstant()) {
- int32_t cond_value = cond->AsIntConstant()->GetValue();
- if (cond_value == 1) {
- if (always_true_target != nullptr) {
- __ B(always_true_target);
+ if (true_target == nullptr && false_target == nullptr) {
+ // Nothing to do. The code always falls through.
+ return;
+ } else if (cond->IsIntConstant()) {
+ // Constant condition, statically compared against 1.
+ if (cond->AsIntConstant()->IsOne()) {
+ if (true_target != nullptr) {
+ __ B(true_target);
}
- return;
} else {
- DCHECK_EQ(cond_value, 0);
+ DCHECK(cond->AsIntConstant()->IsZero());
+ if (false_target != nullptr) {
+ __ B(false_target);
+ }
}
- } else if (!cond->IsCondition() || condition->NeedsMaterialization()) {
+ return;
+ }
+
+ // The following code generates these patterns:
+ // (1) true_target == nullptr && false_target != nullptr
+ // - opposite condition true => branch to false_target
+ // (2) true_target != nullptr && false_target == nullptr
+ // - condition true => branch to true_target
+ // (3) true_target != nullptr && false_target != nullptr
+ // - condition true => branch to true_target
+ // - branch to false_target
+ if (IsBooleanValueOrMaterializedCondition(cond)) {
// The condition instruction has been materialized, compare the output to 0.
- Location cond_val = instruction->GetLocations()->InAt(0);
+ Location cond_val = instruction->GetLocations()->InAt(condition_input_index);
DCHECK(cond_val.IsRegister());
- __ Bnezc(cond_val.AsRegister<GpuRegister>(), true_target);
+ if (true_target == nullptr) {
+ __ Beqzc(cond_val.AsRegister<GpuRegister>(), false_target);
+ } else {
+ __ Bnezc(cond_val.AsRegister<GpuRegister>(), true_target);
+ }
} else {
// The condition instruction has not been materialized, use its inputs as
// the comparison and its condition as the branch condition.
+ HCondition* condition = cond->AsCondition();
+
GpuRegister lhs = condition->GetLocations()->InAt(0).AsRegister<GpuRegister>();
Location rhs_location = condition->GetLocations()->InAt(1);
GpuRegister rhs_reg = ZERO;
@@ -2375,37 +2396,46 @@
rhs_reg = rhs_location.AsRegister<GpuRegister>();
}
- IfCondition if_cond = condition->GetCondition();
+ IfCondition if_cond;
+ Label* non_fallthrough_target;
+ if (true_target == nullptr) {
+ if_cond = condition->GetOppositeCondition();
+ non_fallthrough_target = false_target;
+ } else {
+ if_cond = condition->GetCondition();
+ non_fallthrough_target = true_target;
+ }
+
if (use_imm && rhs_imm == 0) {
switch (if_cond) {
case kCondEQ:
- __ Beqzc(lhs, true_target);
+ __ Beqzc(lhs, non_fallthrough_target);
break;
case kCondNE:
- __ Bnezc(lhs, true_target);
+ __ Bnezc(lhs, non_fallthrough_target);
break;
case kCondLT:
- __ Bltzc(lhs, true_target);
+ __ Bltzc(lhs, non_fallthrough_target);
break;
case kCondGE:
- __ Bgezc(lhs, true_target);
+ __ Bgezc(lhs, non_fallthrough_target);
break;
case kCondLE:
- __ Blezc(lhs, true_target);
+ __ Blezc(lhs, non_fallthrough_target);
break;
case kCondGT:
- __ Bgtzc(lhs, true_target);
+ __ Bgtzc(lhs, non_fallthrough_target);
break;
case kCondB:
break; // always false
case kCondBE:
- __ Beqzc(lhs, true_target); // <= 0 if zero
+ __ Beqzc(lhs, non_fallthrough_target); // <= 0 if zero
break;
case kCondA:
- __ Bnezc(lhs, true_target); // > 0 if non-zero
+ __ Bnezc(lhs, non_fallthrough_target); // > 0 if non-zero
break;
case kCondAE:
- __ B(true_target); // always true
+ __ B(non_fallthrough_target); // always true
break;
}
} else {
@@ -2424,7 +2454,7 @@
case kCondBE:
case kCondAE:
// if lhs == rhs for a positive condition, then it is a branch
- __ B(true_target);
+ __ B(non_fallthrough_target);
break;
case kCondNE:
case kCondLT:
@@ -2437,72 +2467,68 @@
} else {
switch (if_cond) {
case kCondEQ:
- __ Beqc(lhs, rhs_reg, true_target);
+ __ Beqc(lhs, rhs_reg, non_fallthrough_target);
break;
case kCondNE:
- __ Bnec(lhs, rhs_reg, true_target);
+ __ Bnec(lhs, rhs_reg, non_fallthrough_target);
break;
case kCondLT:
- __ Bltc(lhs, rhs_reg, true_target);
+ __ Bltc(lhs, rhs_reg, non_fallthrough_target);
break;
case kCondGE:
- __ Bgec(lhs, rhs_reg, true_target);
+ __ Bgec(lhs, rhs_reg, non_fallthrough_target);
break;
case kCondLE:
- __ Bgec(rhs_reg, lhs, true_target);
+ __ Bgec(rhs_reg, lhs, non_fallthrough_target);
break;
case kCondGT:
- __ Bltc(rhs_reg, lhs, true_target);
+ __ Bltc(rhs_reg, lhs, non_fallthrough_target);
break;
case kCondB:
- __ Bltuc(lhs, rhs_reg, true_target);
+ __ Bltuc(lhs, rhs_reg, non_fallthrough_target);
break;
case kCondAE:
- __ Bgeuc(lhs, rhs_reg, true_target);
+ __ Bgeuc(lhs, rhs_reg, non_fallthrough_target);
break;
case kCondBE:
- __ Bgeuc(rhs_reg, lhs, true_target);
+ __ Bgeuc(rhs_reg, lhs, non_fallthrough_target);
break;
case kCondA:
- __ Bltuc(rhs_reg, lhs, true_target);
+ __ Bltuc(rhs_reg, lhs, non_fallthrough_target);
break;
}
}
}
}
- if (false_target != nullptr) {
+
+ // If neither branch falls through (case 3), the conditional branch to `true_target`
+ // was already emitted (case 2) and we need to emit a jump to `false_target`.
+ if (true_target != nullptr && false_target != nullptr) {
__ B(false_target);
}
}
void LocationsBuilderMIPS64::VisitIf(HIf* if_instr) {
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
- HInstruction* cond = if_instr->InputAt(0);
- if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
+ if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
locations->SetInAt(0, Location::RequiresRegister());
}
}
void InstructionCodeGeneratorMIPS64::VisitIf(HIf* if_instr) {
- Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
- Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
- Label* always_true_target = true_target;
- if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
- if_instr->IfTrueSuccessor())) {
- always_true_target = nullptr;
- }
- if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
- if_instr->IfFalseSuccessor())) {
- false_target = nullptr;
- }
- GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
+ HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
+ HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
+ Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
+ nullptr : codegen_->GetLabelOf(true_successor);
+ Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
+ nullptr : codegen_->GetLabelOf(false_successor);
+ GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
}
void LocationsBuilderMIPS64::VisitDeoptimize(HDeoptimize* deoptimize) {
LocationSummary* locations = new (GetGraph()->GetArena())
LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
- HInstruction* cond = deoptimize->InputAt(0);
- if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
+ if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
locations->SetInAt(0, Location::RequiresRegister());
}
}
@@ -2511,8 +2537,10 @@
SlowPathCodeMIPS64* slow_path = new (GetGraph()->GetArena())
DeoptimizationSlowPathMIPS64(deoptimize);
codegen_->AddSlowPath(slow_path);
- Label* slow_path_entry = slow_path->GetEntryLabel();
- GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
+ GenerateTestAndBranch(deoptimize,
+ /* condition_input_index */ 0,
+ slow_path->GetEntryLabel(),
+ /* false_target */ nullptr);
}
void LocationsBuilderMIPS64::HandleFieldGet(HInstruction* instruction,
@@ -2794,9 +2822,9 @@
// sorted out.
if (invoke->HasCurrentMethodInput()) {
LocationSummary* locations = invoke->GetLocations();
- Location location = locations->InAt(invoke->GetCurrentMethodInputIndex());
+ Location location = locations->InAt(invoke->GetSpecialInputIndex());
if (location.IsUnallocated() && location.GetPolicy() == Location::kRequiresRegister) {
- locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::NoLocation());
+ locations->SetInAt(invoke->GetSpecialInputIndex(), Location::NoLocation());
}
}
}
@@ -2854,7 +2882,7 @@
invoke->GetStringInitOffset());
break;
case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
- callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
+ callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
__ LoadConst64(temp.AsRegister<GpuRegister>(), invoke->GetMethodAddress());
@@ -2866,7 +2894,7 @@
LOG(FATAL) << "Unsupported";
UNREACHABLE();
case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
- Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
+ Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
GpuRegister reg = temp.AsRegister<GpuRegister>();
GpuRegister method_reg;
if (current_method.IsRegister()) {
diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h
index ac3162f..a078dd1 100644
--- a/compiler/optimizing/code_generator_mips64.h
+++ b/compiler/optimizing/code_generator_mips64.h
@@ -230,9 +230,9 @@
void GenerateImplicitNullCheck(HNullCheck* instruction);
void GenerateExplicitNullCheck(HNullCheck* instruction);
void GenerateTestAndBranch(HInstruction* instruction,
+ size_t condition_input_index,
Label* true_target,
- Label* false_target,
- Label* always_true_target);
+ Label* false_target);
void DivRemOneOrMinusOne(HBinaryOperation* instruction);
void DivRemByPowerOfTwo(HBinaryOperation* instruction);
void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction);
diff --git a/compiler/optimizing/code_generator_utils.cc b/compiler/optimizing/code_generator_utils.cc
index bf354e7..644a3fb 100644
--- a/compiler/optimizing/code_generator_utils.cc
+++ b/compiler/optimizing/code_generator_utils.cc
@@ -95,19 +95,8 @@
*shift = is_long ? p - 64 : p - 32;
}
-// Is it valid to reverse the condition? Uses the values supplied to
-// GenerateTestAndBranch() in instruction generators.
-bool CanReverseCondition(Label* always_true_target,
- Label* false_target,
- HCondition* condition) {
- // 'always_true_target' is null when the 'true' path is to the next
- // block to be generated. Check the type of the condition to ensure that
- // FP conditions are not swapped. This is for future fusing of HCompare and
- // HCondition.
- // Note: If the condition is nullptr, then it is always okay to reverse.
- return always_true_target == nullptr && false_target != nullptr &&
- (condition == nullptr ||
- !Primitive::IsFloatingPointType(condition->InputAt(0)->GetType()));
+bool IsBooleanValueOrMaterializedCondition(HInstruction* cond_input) {
+ return !cond_input->IsCondition() || cond_input->AsCondition()->NeedsMaterialization();
}
} // namespace art
diff --git a/compiler/optimizing/code_generator_utils.h b/compiler/optimizing/code_generator_utils.h
index 628eee8..7efed8c 100644
--- a/compiler/optimizing/code_generator_utils.h
+++ b/compiler/optimizing/code_generator_utils.h
@@ -21,18 +21,16 @@
namespace art {
-class Label;
-class HCondition;
+class HInstruction;
// Computes the magic number and the shift needed in the div/rem by constant algorithm, as out
// arguments `magic` and `shift`
void CalculateMagicAndShiftForDivRem(int64_t divisor, bool is_long, int64_t* magic, int* shift);
-// Is it valid to reverse the condition? Uses the values supplied to
-// GenerateTestAndBranch() in instruction generators.
-bool CanReverseCondition(Label* always_true_target,
- Label* false_target,
- HCondition* condition);
+// Returns true if `cond_input` is expected to have a location. Assumes that
+// `cond_input` is a conditional input of the currently emitted instruction and
+// that it has been previously visited by the InstructionCodeGenerator.
+bool IsBooleanValueOrMaterializedCondition(HInstruction* cond_input);
} // namespace art
diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc
index 0147b01..a87e8ed 100644
--- a/compiler/optimizing/code_generator_x86.cc
+++ b/compiler/optimizing/code_generator_x86.cc
@@ -35,6 +35,9 @@
namespace art {
+template<class MirrorType>
+class GcRoot;
+
namespace x86 {
static constexpr int kCurrentMethodStackOffset = 0;
@@ -300,15 +303,6 @@
CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
__ Bind(GetEntryLabel());
- if (instruction_->IsCheckCast()) {
- // The codegen for the instruction overwrites `temp`, so put it back in place.
- Register obj = locations->InAt(0).AsRegister<Register>();
- Register temp = locations->GetTemp(0).AsRegister<Register>();
- uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
- __ movl(temp, Address(obj, class_offset));
- __ MaybeUnpoisonHeapReference(temp);
- }
-
if (!is_fatal_) {
SaveLiveRegisters(codegen, locations);
}
@@ -329,12 +323,15 @@
instruction_,
instruction_->GetDexPc(),
this);
+ CheckEntrypointTypes<
+ kQuickInstanceofNonTrivial, uint32_t, const mirror::Class*, const mirror::Class*>();
} else {
DCHECK(instruction_->IsCheckCast());
x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
instruction_,
instruction_->GetDexPc(),
this);
+ CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
}
if (!is_fatal_) {
@@ -425,6 +422,221 @@
DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathX86);
};
+// Slow path generating a read barrier for a heap reference.
+class ReadBarrierForHeapReferenceSlowPathX86 : public SlowPathCode {
+ public:
+ ReadBarrierForHeapReferenceSlowPathX86(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index)
+ : instruction_(instruction),
+ out_(out),
+ ref_(ref),
+ obj_(obj),
+ offset_(offset),
+ index_(index) {
+ DCHECK(kEmitCompilerReadBarrier);
+ // If `obj` is equal to `out` or `ref`, it means the initial object
+ // has been overwritten by (or after) the heap object reference load
+ // to be instrumented, e.g.:
+ //
+ // __ movl(out, Address(out, offset));
+ // codegen_->GenerateReadBarrier(instruction, out_loc, out_loc, out_loc, offset);
+ //
+ // In that case, we have lost the information about the original
+ // object, and the emitted read barrier cannot work properly.
+ DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out;
+ DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref;
+ }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
+ LocationSummary* locations = instruction_->GetLocations();
+ Register reg_out = out_.AsRegister<Register>();
+ DCHECK(locations->CanCall());
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
+ DCHECK(!instruction_->IsInvoke() ||
+ (instruction_->IsInvokeStaticOrDirect() &&
+ instruction_->GetLocations()->Intrinsified()));
+
+ __ Bind(GetEntryLabel());
+ SaveLiveRegisters(codegen, locations);
+
+ // We may have to change the index's value, but as `index_` is a
+ // constant member (like other "inputs" of this slow path),
+ // introduce a copy of it, `index`.
+ Location index = index_;
+ if (index_.IsValid()) {
+ // Handle `index_` for HArrayGet and intrinsic UnsafeGetObject.
+ if (instruction_->IsArrayGet()) {
+ // Compute the actual memory offset and store it in `index`.
+ Register index_reg = index_.AsRegister<Register>();
+ DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg));
+ if (codegen->IsCoreCalleeSaveRegister(index_reg)) {
+ // We are about to change the value of `index_reg` (see the
+ // calls to art::x86::X86Assembler::shll and
+ // art::x86::X86Assembler::AddImmediate below), but it has
+ // not been saved by the previous call to
+ // art::SlowPathCode::SaveLiveRegisters, as it is a
+ // callee-save register --
+ // art::SlowPathCode::SaveLiveRegisters does not consider
+ // callee-save registers, as it has been designed with the
+ // assumption that callee-save registers are supposed to be
+ // handled by the called function. So, as a callee-save
+ // register, `index_reg` _would_ eventually be saved onto
+ // the stack, but it would be too late: we would have
+ // changed its value earlier. Therefore, we manually save
+ // it here into another freely available register,
+ // `free_reg`, chosen of course among the caller-save
+ // registers (as a callee-save `free_reg` register would
+ // exhibit the same problem).
+ //
+ // Note we could have requested a temporary register from
+ // the register allocator instead; but we prefer not to, as
+ // this is a slow path, and we know we can find a
+ // caller-save register that is available.
+ Register free_reg = FindAvailableCallerSaveRegister(codegen);
+ __ movl(free_reg, index_reg);
+ index_reg = free_reg;
+ index = Location::RegisterLocation(index_reg);
+ } else {
+ // The initial register stored in `index_` has already been
+ // saved in the call to art::SlowPathCode::SaveLiveRegisters
+ // (as it is not a callee-save register), so we can freely
+ // use it.
+ }
+ // Shifting the index value contained in `index_reg` by the scale
+ // factor (2) cannot overflow in practice, as the runtime is
+ // unable to allocate object arrays with a size larger than
+ // 2^26 - 1 (that is, 2^28 - 4 bytes).
+ __ shll(index_reg, Immediate(TIMES_4));
+ static_assert(
+ sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
+ "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
+ __ AddImmediate(index_reg, Immediate(offset_));
+ } else {
+ DCHECK(instruction_->IsInvoke());
+ DCHECK(instruction_->GetLocations()->Intrinsified());
+ DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) ||
+ (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile))
+ << instruction_->AsInvoke()->GetIntrinsic();
+ DCHECK_EQ(offset_, 0U);
+ DCHECK(index_.IsRegisterPair());
+ // UnsafeGet's offset location is a register pair, the low
+ // part contains the correct offset.
+ index = index_.ToLow();
+ }
+ }
+
+ // We're moving two or three locations to locations that could
+ // overlap, so we need a parallel move resolver.
+ InvokeRuntimeCallingConvention calling_convention;
+ HParallelMove parallel_move(codegen->GetGraph()->GetArena());
+ parallel_move.AddMove(ref_,
+ Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
+ Primitive::kPrimNot,
+ nullptr);
+ parallel_move.AddMove(obj_,
+ Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
+ Primitive::kPrimNot,
+ nullptr);
+ if (index.IsValid()) {
+ parallel_move.AddMove(index,
+ Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
+ Primitive::kPrimInt,
+ nullptr);
+ codegen->GetMoveResolver()->EmitNativeCode(¶llel_move);
+ } else {
+ codegen->GetMoveResolver()->EmitNativeCode(¶llel_move);
+ __ movl(calling_convention.GetRegisterAt(2), Immediate(offset_));
+ }
+ x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierSlow),
+ instruction_,
+ instruction_->GetDexPc(),
+ this);
+ CheckEntrypointTypes<
+ kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>();
+ x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
+
+ RestoreLiveRegisters(codegen, locations);
+ __ jmp(GetExitLabel());
+ }
+
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathX86"; }
+
+ private:
+ Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) {
+ size_t ref = static_cast<int>(ref_.AsRegister<Register>());
+ size_t obj = static_cast<int>(obj_.AsRegister<Register>());
+ for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
+ if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) {
+ return static_cast<Register>(i);
+ }
+ }
+ // We shall never fail to find a free caller-save register, as
+ // there are more than two core caller-save registers on x86
+ // (meaning it is possible to find one which is different from
+ // `ref` and `obj`).
+ DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u);
+ LOG(FATAL) << "Could not find a free caller-save register";
+ UNREACHABLE();
+ }
+
+ HInstruction* const instruction_;
+ const Location out_;
+ const Location ref_;
+ const Location obj_;
+ const uint32_t offset_;
+ // An additional location containing an index to an array.
+ // Only used for HArrayGet and the UnsafeGetObject &
+ // UnsafeGetObjectVolatile intrinsics.
+ const Location index_;
+
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathX86);
+};
+
+// Slow path generating a read barrier for a GC root.
+class ReadBarrierForRootSlowPathX86 : public SlowPathCode {
+ public:
+ ReadBarrierForRootSlowPathX86(HInstruction* instruction, Location out, Location root)
+ : instruction_(instruction), out_(out), root_(root) {}
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ Register reg_out = out_.AsRegister<Register>();
+ DCHECK(locations->CanCall());
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
+ DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString());
+
+ __ Bind(GetEntryLabel());
+ SaveLiveRegisters(codegen, locations);
+
+ InvokeRuntimeCallingConvention calling_convention;
+ CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
+ x86_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_);
+ x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierForRootSlow),
+ instruction_,
+ instruction_->GetDexPc(),
+ this);
+ CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>();
+ x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
+
+ RestoreLiveRegisters(codegen, locations);
+ __ jmp(GetExitLabel());
+ }
+
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathX86"; }
+
+ private:
+ HInstruction* const instruction_;
+ const Location out_;
+ const Location root_;
+
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathX86);
+};
+
#undef __
#define __ down_cast<X86Assembler*>(GetAssembler())->
@@ -513,9 +725,9 @@
}
CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
- const X86InstructionSetFeatures& isa_features,
- const CompilerOptions& compiler_options,
- OptimizingCompilerStats* stats)
+ const X86InstructionSetFeatures& isa_features,
+ const CompilerOptions& compiler_options,
+ OptimizingCompilerStats* stats)
: CodeGenerator(graph,
kNumberOfCpuRegisters,
kNumberOfXmmRegisters,
@@ -582,7 +794,7 @@
LOG(FATAL) << "Unreachable type " << type;
}
- return Location();
+ return Location::NoLocation();
}
void CodeGeneratorX86::SetupBlockedRegisters(bool is_baseline) const {
@@ -783,7 +995,7 @@
LOG(FATAL) << "Unexpected parameter type " << type;
break;
}
- return Location();
+ return Location::NoLocation();
}
void CodeGeneratorX86::Move32(Location destination, Location source) {
@@ -1158,26 +1370,19 @@
__ j(final_condition, true_label);
}
-void InstructionCodeGeneratorX86::GenerateCompareTestAndBranch(HIf* if_instr,
- HCondition* condition,
- Label* true_target,
- Label* false_target,
- Label* always_true_target) {
+void InstructionCodeGeneratorX86::GenerateCompareTestAndBranch(HCondition* condition,
+ Label* true_target_in,
+ Label* false_target_in) {
+ // Generated branching requires both targets to be explicit. If either of the
+ // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead.
+ Label fallthrough_target;
+ Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in;
+ Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in;
+
LocationSummary* locations = condition->GetLocations();
Location left = locations->InAt(0);
Location right = locations->InAt(1);
- // We don't want true_target as a nullptr.
- if (true_target == nullptr) {
- true_target = always_true_target;
- }
- bool falls_through = (false_target == nullptr);
-
- // FP compares don't like null false_targets.
- if (false_target == nullptr) {
- false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
- }
-
Primitive::Type type = condition->InputAt(0)->GetType();
switch (type) {
case Primitive::kPrimLong:
@@ -1195,138 +1400,141 @@
LOG(FATAL) << "Unexpected compare type " << type;
}
- if (!falls_through) {
+ if (false_target != &fallthrough_target) {
__ jmp(false_target);
}
+
+ if (fallthrough_target.IsLinked()) {
+ __ Bind(&fallthrough_target);
+ }
}
+static bool AreEflagsSetFrom(HInstruction* cond, HInstruction* branch) {
+ // Moves may affect the eflags register (move zero uses xorl), so the EFLAGS
+ // are set only strictly before `branch`. We can't use the eflags on long/FP
+ // conditions if they are materialized due to the complex branching.
+ return cond->IsCondition() &&
+ cond->GetNext() == branch &&
+ cond->InputAt(0)->GetType() != Primitive::kPrimLong &&
+ !Primitive::IsFloatingPointType(cond->InputAt(0)->GetType());
+}
+
void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
+ size_t condition_input_index,
Label* true_target,
- Label* false_target,
- Label* always_true_target) {
- HInstruction* cond = instruction->InputAt(0);
- if (cond->IsIntConstant()) {
+ Label* false_target) {
+ HInstruction* cond = instruction->InputAt(condition_input_index);
+
+ if (true_target == nullptr && false_target == nullptr) {
+ // Nothing to do. The code always falls through.
+ return;
+ } else if (cond->IsIntConstant()) {
// Constant condition, statically compared against 1.
- int32_t cond_value = cond->AsIntConstant()->GetValue();
- if (cond_value == 1) {
- if (always_true_target != nullptr) {
- __ jmp(always_true_target);
+ if (cond->AsIntConstant()->IsOne()) {
+ if (true_target != nullptr) {
+ __ jmp(true_target);
}
- return;
} else {
- DCHECK_EQ(cond_value, 0);
+ DCHECK(cond->AsIntConstant()->IsZero());
+ if (false_target != nullptr) {
+ __ jmp(false_target);
+ }
+ }
+ return;
+ }
+
+ // The following code generates these patterns:
+ // (1) true_target == nullptr && false_target != nullptr
+ // - opposite condition true => branch to false_target
+ // (2) true_target != nullptr && false_target == nullptr
+ // - condition true => branch to true_target
+ // (3) true_target != nullptr && false_target != nullptr
+ // - condition true => branch to true_target
+ // - branch to false_target
+ if (IsBooleanValueOrMaterializedCondition(cond)) {
+ if (AreEflagsSetFrom(cond, instruction)) {
+ if (true_target == nullptr) {
+ __ j(X86Condition(cond->AsCondition()->GetOppositeCondition()), false_target);
+ } else {
+ __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
+ }
+ } else {
+ // Materialized condition, compare against 0.
+ Location lhs = instruction->GetLocations()->InAt(condition_input_index);
+ if (lhs.IsRegister()) {
+ __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
+ } else {
+ __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
+ }
+ if (true_target == nullptr) {
+ __ j(kEqual, false_target);
+ } else {
+ __ j(kNotEqual, true_target);
+ }
}
} else {
+ // Condition has not been materialized, use its inputs as the comparison and
+ // its condition as the branch condition.
HCondition* condition = cond->AsCondition();
- bool is_materialized =
- condition == nullptr || condition->NeedsMaterialization();
- // Moves do not affect the eflags register, so if the condition is
- // evaluated just before the if, we don't need to evaluate it
- // again. We can't use the eflags on long/FP conditions if they are
- // materialized due to the complex branching.
- Primitive::Type type = (condition != nullptr)
- ? cond->InputAt(0)->GetType()
- : Primitive::kPrimInt;
- bool eflags_set = condition != nullptr
- && condition->IsBeforeWhenDisregardMoves(instruction)
- && (type != Primitive::kPrimLong && !Primitive::IsFloatingPointType(type));
- // Can we optimize the jump if we know that the next block is the true case?
- bool can_jump_to_false = CanReverseCondition(always_true_target, false_target, condition);
- if (is_materialized) {
- if (!eflags_set) {
- // Materialized condition, compare against 0.
- Location lhs = instruction->GetLocations()->InAt(0);
- if (lhs.IsRegister()) {
- __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
- } else {
- __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
- }
- if (can_jump_to_false) {
- __ j(kEqual, false_target);
- return;
- }
- __ j(kNotEqual, true_target);
+
+ // If this is a long or FP comparison that has been folded into
+ // the HCondition, generate the comparison directly.
+ Primitive::Type type = condition->InputAt(0)->GetType();
+ if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
+ GenerateCompareTestAndBranch(condition, true_target, false_target);
+ return;
+ }
+
+ Location lhs = condition->GetLocations()->InAt(0);
+ Location rhs = condition->GetLocations()->InAt(1);
+ // LHS is guaranteed to be in a register (see LocationsBuilderX86::VisitCondition).
+ if (rhs.IsRegister()) {
+ __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
+ } else if (rhs.IsConstant()) {
+ int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
+ if (constant == 0) {
+ __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
} else {
- if (can_jump_to_false) {
- __ j(X86Condition(condition->GetOppositeCondition()), false_target);
- return;
- }
- __ j(X86Condition(condition->GetCondition()), true_target);
+ __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
}
} else {
- // Condition has not been materialized, use its inputs as the
- // comparison and its condition as the branch condition.
-
- // Is this a long or FP comparison that has been folded into the HCondition?
- if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
- // Generate the comparison directly.
- GenerateCompareTestAndBranch(instruction->AsIf(),
- condition,
- true_target,
- false_target,
- always_true_target);
- return;
- }
-
- Location lhs = cond->GetLocations()->InAt(0);
- Location rhs = cond->GetLocations()->InAt(1);
- // LHS is guaranteed to be in a register (see
- // LocationsBuilderX86::VisitCondition).
- if (rhs.IsRegister()) {
- __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
- } else if (rhs.IsConstant()) {
- int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
- if (constant == 0) {
- __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
- } else {
- __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
- }
- } else {
- __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
- }
-
- if (can_jump_to_false) {
- __ j(X86Condition(condition->GetOppositeCondition()), false_target);
- return;
- }
-
+ __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
+ }
+ if (true_target == nullptr) {
+ __ j(X86Condition(condition->GetOppositeCondition()), false_target);
+ } else {
__ j(X86Condition(condition->GetCondition()), true_target);
}
}
- if (false_target != nullptr) {
+
+ // If neither branch falls through (case 3), the conditional branch to `true_target`
+ // was already emitted (case 2) and we need to emit a jump to `false_target`.
+ if (true_target != nullptr && false_target != nullptr) {
__ jmp(false_target);
}
}
void LocationsBuilderX86::VisitIf(HIf* if_instr) {
- LocationSummary* locations =
- new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
- HInstruction* cond = if_instr->InputAt(0);
- if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
+ LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
+ if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
locations->SetInAt(0, Location::Any());
}
}
void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
- Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
- Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
- Label* always_true_target = true_target;
- if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
- if_instr->IfTrueSuccessor())) {
- always_true_target = nullptr;
- }
- if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
- if_instr->IfFalseSuccessor())) {
- false_target = nullptr;
- }
- GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
+ HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
+ HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
+ Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
+ nullptr : codegen_->GetLabelOf(true_successor);
+ Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
+ nullptr : codegen_->GetLabelOf(false_successor);
+ GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
}
void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
LocationSummary* locations = new (GetGraph()->GetArena())
LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
- HInstruction* cond = deoptimize->InputAt(0);
- if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
+ if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
locations->SetInAt(0, Location::Any());
}
}
@@ -1335,8 +1543,10 @@
SlowPathCode* slow_path = new (GetGraph()->GetArena())
DeoptimizationSlowPathX86(deoptimize);
codegen_->AddSlowPath(slow_path);
- Label* slow_path_entry = slow_path->GetEntryLabel();
- GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
+ GenerateTestAndBranch(deoptimize,
+ /* condition_input_index */ 0,
+ slow_path->GetEntryLabel(),
+ /* false_target */ nullptr);
}
void LocationsBuilderX86::VisitLocal(HLocal* local) {
@@ -1698,7 +1908,7 @@
IntrinsicLocationsBuilderX86 intrinsic(codegen_);
if (intrinsic.TryDispatch(invoke)) {
if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) {
- invoke->GetLocations()->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::Any());
+ invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any());
}
return;
}
@@ -1707,7 +1917,7 @@
// For PC-relative dex cache the invoke has an extra input, the PC-relative address base.
if (invoke->HasPcRelativeDexCache()) {
- invoke->GetLocations()->SetInAt(invoke->GetCurrentMethodInputIndex(),
+ invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(),
Location::RequiresRegister());
}
@@ -1716,9 +1926,9 @@
// needs a register. We therefore do not require a register for it, and let
// the code generation of the invoke handle it.
LocationSummary* locations = invoke->GetLocations();
- Location location = locations->InAt(invoke->GetCurrentMethodInputIndex());
+ Location location = locations->InAt(invoke->GetSpecialInputIndex());
if (location.IsUnallocated() && location.GetPolicy() == Location::kRequiresRegister) {
- locations->SetInAt(invoke->GetCurrentMethodInputIndex(), Location::NoLocation());
+ locations->SetInAt(invoke->GetSpecialInputIndex(), Location::NoLocation());
}
}
}
@@ -1767,6 +1977,9 @@
}
void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
+ // This call to HandleInvoke allocates a temporary (core) register
+ // which is also used to transfer the hidden argument from FP to
+ // core register.
HandleInvoke(invoke);
// Add the hidden argument.
invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
@@ -1774,31 +1987,42 @@
void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
// TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
- Register temp = invoke->GetLocations()->GetTemp(0).AsRegister<Register>();
+ LocationSummary* locations = invoke->GetLocations();
+ Register temp = locations->GetTemp(0).AsRegister<Register>();
+ XmmRegister hidden_reg = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
invoke->GetImtIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
- LocationSummary* locations = invoke->GetLocations();
Location receiver = locations->InAt(0);
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
- // Set the hidden argument.
+ // Set the hidden argument. This is safe to do this here, as XMM7
+ // won't be modified thereafter, before the `call` instruction.
+ DCHECK_EQ(XMM7, hidden_reg);
__ movl(temp, Immediate(invoke->GetDexMethodIndex()));
- __ movd(invoke->GetLocations()->GetTemp(1).AsFpuRegister<XmmRegister>(), temp);
+ __ movd(hidden_reg, temp);
- // temp = object->GetClass();
if (receiver.IsStackSlot()) {
__ movl(temp, Address(ESP, receiver.GetStackIndex()));
+ // /* HeapReference<Class> */ temp = temp->klass_
__ movl(temp, Address(temp, class_offset));
} else {
+ // /* HeapReference<Class> */ temp = receiver->klass_
__ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
}
codegen_->MaybeRecordImplicitNullCheck(invoke);
+ // Instead of simply (possibly) unpoisoning `temp` here, we should
+ // emit a read barrier for the previous class reference load.
+ // However this is not required in practice, as this is an
+ // intermediate/temporary reference and because the current
+ // concurrent copying collector keeps the from-space memory
+ // intact/accessible until the end of the marking phase (the
+ // concurrent copying collector may not in the future).
__ MaybeUnpoisonHeapReference(temp);
// temp = temp->GetImtEntryAt(method_offset);
__ movl(temp, Address(temp, method_offset));
// call temp->GetEntryPoint();
- __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
- kX86WordSize).Int32Value()));
+ __ call(Address(temp,
+ ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
DCHECK(!codegen_->IsLeafMethod());
codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
@@ -3808,7 +4032,7 @@
Register CodeGeneratorX86::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke,
Register temp) {
DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u);
- Location location = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
+ Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
if (!invoke->GetLocations()->Intrinsified()) {
return location.AsRegister<Register>();
}
@@ -3839,7 +4063,7 @@
__ fs()->movl(temp.AsRegister<Register>(), Address::Absolute(invoke->GetStringInitOffset()));
break;
case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
- callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
+ callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
__ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress()));
@@ -3860,7 +4084,7 @@
break;
}
case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
- Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
+ Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
Register method_reg;
Register reg = temp.AsRegister<Register>();
if (current_method.IsRegister()) {
@@ -3871,7 +4095,7 @@
method_reg = reg;
__ movl(reg, Address(ESP, kCurrentMethodStackOffset));
}
- // temp = temp->dex_cache_resolved_methods_;
+ // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_;
__ movl(reg, Address(method_reg,
ArtMethod::DexCacheResolvedMethodsOffset(kX86PointerSize).Int32Value()));
// temp = temp[index_in_cache]
@@ -3915,10 +4139,17 @@
LocationSummary* locations = invoke->GetLocations();
Location receiver = locations->InAt(0);
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
- // temp = object->GetClass();
DCHECK(receiver.IsRegister());
+ // /* HeapReference<Class> */ temp = receiver->klass_
__ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
MaybeRecordImplicitNullCheck(invoke);
+ // Instead of simply (possibly) unpoisoning `temp` here, we should
+ // emit a read barrier for the previous class reference load.
+ // However this is not required in practice, as this is an
+ // intermediate/temporary reference and because the current
+ // concurrent copying collector keeps the from-space memory
+ // intact/accessible until the end of the marking phase (the
+ // concurrent copying collector may not in the future).
__ MaybeUnpoisonHeapReference(temp);
// temp = temp->GetMethodAt(method_offset);
__ movl(temp, Address(temp, method_offset));
@@ -3980,18 +4211,29 @@
void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
+
+ bool object_field_get_with_read_barrier =
+ kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
LocationSummary* locations =
- new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
+ new (GetGraph()->GetArena()) LocationSummary(instruction,
+ kEmitCompilerReadBarrier ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall);
locations->SetInAt(0, Location::RequiresRegister());
if (Primitive::IsFloatingPointType(instruction->GetType())) {
locations->SetOut(Location::RequiresFpuRegister());
} else {
- // The output overlaps in case of long: we don't want the low move to overwrite
- // the object's location.
- locations->SetOut(Location::RequiresRegister(),
- (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
- : Location::kNoOutputOverlap);
+ // The output overlaps in case of long: we don't want the low move
+ // to overwrite the object's location. Likewise, in the case of
+ // an object field get with read barriers enabled, we do not want
+ // the move to overwrite the object's location, as we need it to emit
+ // the read barrier.
+ locations->SetOut(
+ Location::RequiresRegister(),
+ (object_field_get_with_read_barrier || instruction->GetType() == Primitive::kPrimLong) ?
+ Location::kOutputOverlap :
+ Location::kNoOutputOverlap);
}
if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
@@ -4007,7 +4249,8 @@
DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
LocationSummary* locations = instruction->GetLocations();
- Register base = locations->InAt(0).AsRegister<Register>();
+ Location base_loc = locations->InAt(0);
+ Register base = base_loc.AsRegister<Register>();
Location out = locations->Out();
bool is_volatile = field_info.IsVolatile();
Primitive::Type field_type = field_info.GetFieldType();
@@ -4082,7 +4325,7 @@
}
if (field_type == Primitive::kPrimNot) {
- __ MaybeUnpoisonHeapReference(out.AsRegister<Register>());
+ codegen_->MaybeGenerateReadBarrier(instruction, out, out, base_loc, offset);
}
}
@@ -4410,24 +4653,35 @@
}
void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
+ bool object_array_get_with_read_barrier =
+ kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
LocationSummary* locations =
- new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
+ new (GetGraph()->GetArena()) LocationSummary(instruction,
+ object_array_get_with_read_barrier ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
if (Primitive::IsFloatingPointType(instruction->GetType())) {
locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
} else {
- // The output overlaps in case of long: we don't want the low move to overwrite
- // the array's location.
- locations->SetOut(Location::RequiresRegister(),
- (instruction->GetType() == Primitive::kPrimLong) ? Location::kOutputOverlap
- : Location::kNoOutputOverlap);
+ // The output overlaps in case of long: we don't want the low move
+ // to overwrite the array's location. Likewise, in the case of an
+ // object array get with read barriers enabled, we do not want the
+ // move to overwrite the array's location, as we need it to emit
+ // the read barrier.
+ locations->SetOut(
+ Location::RequiresRegister(),
+ (instruction->GetType() == Primitive::kPrimLong || object_array_get_with_read_barrier) ?
+ Location::kOutputOverlap :
+ Location::kNoOutputOverlap);
}
}
void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
LocationSummary* locations = instruction->GetLocations();
- Register obj = locations->InAt(0).AsRegister<Register>();
+ Location obj_loc = locations->InAt(0);
+ Register obj = obj_loc.AsRegister<Register>();
Location index = locations->InAt(1);
Primitive::Type type = instruction->GetType();
@@ -4482,6 +4736,9 @@
case Primitive::kPrimInt:
case Primitive::kPrimNot: {
+ static_assert(
+ sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
+ "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
Register out = locations->Out().AsRegister<Register>();
if (index.IsConstant()) {
@@ -4546,8 +4803,17 @@
}
if (type == Primitive::kPrimNot) {
- Register out = locations->Out().AsRegister<Register>();
- __ MaybeUnpoisonHeapReference(out);
+ static_assert(
+ sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
+ "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
+ uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
+ Location out = locations->Out();
+ if (index.IsConstant()) {
+ uint32_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
+ codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, offset);
+ } else {
+ codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, data_offset, index);
+ }
}
}
@@ -4558,14 +4824,18 @@
// optimization.
Primitive::Type value_type = instruction->GetComponentType();
+
bool needs_write_barrier =
CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
-
- bool may_need_runtime_call = instruction->NeedsTypeCheck();
+ bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
+ bool object_array_set_with_read_barrier =
+ kEmitCompilerReadBarrier && (value_type == Primitive::kPrimNot);
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
instruction,
- may_need_runtime_call ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall);
+ (may_need_runtime_call_for_type_check || object_array_set_with_read_barrier) ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall);
bool is_byte_type = (value_type == Primitive::kPrimBoolean)
|| (value_type == Primitive::kPrimByte);
@@ -4586,20 +4856,21 @@
// Temporary registers for the write barrier.
locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too.
// Ensure the card is in a byte register.
- locations->AddTemp(Location::RegisterLocation(ECX));
+ locations->AddTemp(Location::RegisterLocation(ECX)); // Possibly used for read barrier too.
}
}
void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
LocationSummary* locations = instruction->GetLocations();
- Register array = locations->InAt(0).AsRegister<Register>();
+ Location array_loc = locations->InAt(0);
+ Register array = array_loc.AsRegister<Register>();
Location index = locations->InAt(1);
Location value = locations->InAt(2);
Primitive::Type value_type = instruction->GetComponentType();
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
- bool may_need_runtime_call = locations->CanCall();
+ bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
bool needs_write_barrier =
CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
@@ -4639,6 +4910,7 @@
Address address = index.IsConstant()
? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
: Address(array, index.AsRegister<Register>(), TIMES_4, offset);
+
if (!value.IsRegister()) {
// Just setting null.
DCHECK(instruction->InputAt(2)->IsNullConstant());
@@ -4646,7 +4918,7 @@
__ movl(address, Immediate(0));
codegen_->MaybeRecordImplicitNullCheck(instruction);
DCHECK(!needs_write_barrier);
- DCHECK(!may_need_runtime_call);
+ DCHECK(!may_need_runtime_call_for_type_check);
break;
}
@@ -4655,7 +4927,7 @@
NearLabel done, not_null, do_put;
SlowPathCode* slow_path = nullptr;
Register temp = locations->GetTemp(0).AsRegister<Register>();
- if (may_need_runtime_call) {
+ if (may_need_runtime_call_for_type_check) {
slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathX86(instruction);
codegen_->AddSlowPath(slow_path);
if (instruction->GetValueCanBeNull()) {
@@ -4667,22 +4939,62 @@
__ Bind(¬_null);
}
- __ movl(temp, Address(array, class_offset));
- codegen_->MaybeRecordImplicitNullCheck(instruction);
- __ MaybeUnpoisonHeapReference(temp);
- __ movl(temp, Address(temp, component_offset));
- // No need to poison/unpoison, we're comparing two poisoned references.
- __ cmpl(temp, Address(register_value, class_offset));
- if (instruction->StaticTypeOfArrayIsObjectArray()) {
- __ j(kEqual, &do_put);
- __ MaybeUnpoisonHeapReference(temp);
- __ movl(temp, Address(temp, super_offset));
- // No need to unpoison, we're comparing against null..
- __ testl(temp, temp);
- __ j(kNotEqual, slow_path->GetEntryLabel());
- __ Bind(&do_put);
+ if (kEmitCompilerReadBarrier) {
+ // When read barriers are enabled, the type checking
+ // instrumentation requires two read barriers:
+ //
+ // __ movl(temp2, temp);
+ // // /* HeapReference<Class> */ temp = temp->component_type_
+ // __ movl(temp, Address(temp, component_offset));
+ // codegen_->GenerateReadBarrier(
+ // instruction, temp_loc, temp_loc, temp2_loc, component_offset);
+ //
+ // // /* HeapReference<Class> */ temp2 = register_value->klass_
+ // __ movl(temp2, Address(register_value, class_offset));
+ // codegen_->GenerateReadBarrier(
+ // instruction, temp2_loc, temp2_loc, value, class_offset, temp_loc);
+ //
+ // __ cmpl(temp, temp2);
+ //
+ // However, the second read barrier may trash `temp`, as it
+ // is a temporary register, and as such would not be saved
+ // along with live registers before calling the runtime (nor
+ // restored afterwards). So in this case, we bail out and
+ // delegate the work to the array set slow path.
+ //
+ // TODO: Extend the register allocator to support a new
+ // "(locally) live temp" location so as to avoid always
+ // going into the slow path when read barriers are enabled.
+ __ jmp(slow_path->GetEntryLabel());
} else {
- __ j(kNotEqual, slow_path->GetEntryLabel());
+ // /* HeapReference<Class> */ temp = array->klass_
+ __ movl(temp, Address(array, class_offset));
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
+ __ MaybeUnpoisonHeapReference(temp);
+
+ // /* HeapReference<Class> */ temp = temp->component_type_
+ __ movl(temp, Address(temp, component_offset));
+ // If heap poisoning is enabled, no need to unpoison `temp`
+ // nor the object reference in `register_value->klass`, as
+ // we are comparing two poisoned references.
+ __ cmpl(temp, Address(register_value, class_offset));
+
+ if (instruction->StaticTypeOfArrayIsObjectArray()) {
+ __ j(kEqual, &do_put);
+ // If heap poisoning is enabled, the `temp` reference has
+ // not been unpoisoned yet; unpoison it now.
+ __ MaybeUnpoisonHeapReference(temp);
+
+ // /* HeapReference<Class> */ temp = temp->super_class_
+ __ movl(temp, Address(temp, super_offset));
+ // If heap poisoning is enabled, no need to unpoison
+ // `temp`, as we are comparing against null below.
+ __ testl(temp, temp);
+ __ j(kNotEqual, slow_path->GetEntryLabel());
+ __ Bind(&do_put);
+ } else {
+ __ j(kNotEqual, slow_path->GetEntryLabel());
+ }
}
}
@@ -4693,7 +5005,7 @@
} else {
__ movl(address, register_value);
}
- if (!may_need_runtime_call) {
+ if (!may_need_runtime_call_for_type_check) {
codegen_->MaybeRecordImplicitNullCheck(instruction);
}
@@ -4708,6 +5020,7 @@
break;
}
+
case Primitive::kPrimInt: {
uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
Address address = index.IsConstant()
@@ -5178,7 +5491,8 @@
CodeGenerator::CreateLoadClassLocationSummary(
cls,
Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
- Location::RegisterLocation(EAX));
+ Location::RegisterLocation(EAX),
+ /* code_generator_supports_read_barrier */ true);
}
void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
@@ -5192,18 +5506,40 @@
return;
}
- Register out = locations->Out().AsRegister<Register>();
+ Location out_loc = locations->Out();
+ Register out = out_loc.AsRegister<Register>();
Register current_method = locations->InAt(0).AsRegister<Register>();
+
if (cls->IsReferrersClass()) {
DCHECK(!cls->CanCallRuntime());
DCHECK(!cls->MustGenerateClinitCheck());
- __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
+ uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value();
+ if (kEmitCompilerReadBarrier) {
+ // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_)
+ __ leal(out, Address(current_method, declaring_class_offset));
+ // /* mirror::Class* */ out = out->Read()
+ codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc);
+ } else {
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ __ movl(out, Address(current_method, declaring_class_offset));
+ }
} else {
DCHECK(cls->CanCallRuntime());
- __ movl(out, Address(
- current_method, ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value()));
- __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
- // TODO: We will need a read barrier here.
+ // /* GcRoot<mirror::Class>[] */ out =
+ // current_method.ptr_sized_fields_->dex_cache_resolved_types_
+ __ movl(out, Address(current_method,
+ ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value()));
+
+ size_t cache_offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex());
+ if (kEmitCompilerReadBarrier) {
+ // /* GcRoot<mirror::Class>* */ out = &out[type_index]
+ __ leal(out, Address(out, cache_offset));
+ // /* mirror::Class* */ out = out->Read()
+ codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc);
+ } else {
+ // /* GcRoot<mirror::Class> */ out = out[type_index]
+ __ movl(out, Address(out, cache_offset));
+ }
SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
@@ -5257,12 +5593,35 @@
codegen_->AddSlowPath(slow_path);
LocationSummary* locations = load->GetLocations();
- Register out = locations->Out().AsRegister<Register>();
+ Location out_loc = locations->Out();
+ Register out = out_loc.AsRegister<Register>();
Register current_method = locations->InAt(0).AsRegister<Register>();
- __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
+
+ uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value();
+ if (kEmitCompilerReadBarrier) {
+ // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_)
+ __ leal(out, Address(current_method, declaring_class_offset));
+ // /* mirror::Class* */ out = out->Read()
+ codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc);
+ } else {
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ __ movl(out, Address(current_method, declaring_class_offset));
+ }
+
+ // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
__ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
- __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
- // TODO: We will need a read barrier here.
+
+ size_t cache_offset = CodeGenerator::GetCacheOffset(load->GetStringIndex());
+ if (kEmitCompilerReadBarrier) {
+ // /* GcRoot<mirror::String>* */ out = &out[string_index]
+ __ leal(out, Address(out, cache_offset));
+ // /* mirror::String* */ out = out->Read()
+ codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc);
+ } else {
+ // /* GcRoot<mirror::String> */ out = out[string_index]
+ __ movl(out, Address(out, cache_offset));
+ }
+
__ testl(out, out);
__ j(kEqual, slow_path->GetEntryLabel());
__ Bind(slow_path->GetExitLabel());
@@ -5306,40 +5665,44 @@
void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
- switch (instruction->GetTypeCheckKind()) {
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
+ switch (type_check_kind) {
case TypeCheckKind::kExactCheck:
case TypeCheckKind::kAbstractClassCheck:
case TypeCheckKind::kClassHierarchyCheck:
case TypeCheckKind::kArrayObjectCheck:
- call_kind = LocationSummary::kNoCall;
- break;
- case TypeCheckKind::kUnresolvedCheck:
- case TypeCheckKind::kInterfaceCheck:
- call_kind = LocationSummary::kCall;
+ call_kind =
+ kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall;
break;
case TypeCheckKind::kArrayCheck:
+ case TypeCheckKind::kUnresolvedCheck:
+ case TypeCheckKind::kInterfaceCheck:
call_kind = LocationSummary::kCallOnSlowPath;
break;
}
+
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
- if (call_kind != LocationSummary::kCall) {
- locations->SetInAt(0, Location::RequiresRegister());
- locations->SetInAt(1, Location::Any());
- // Note that TypeCheckSlowPathX86 uses this register too.
- locations->SetOut(Location::RequiresRegister());
- } else {
- InvokeRuntimeCallingConvention calling_convention;
- locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
- locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
- locations->SetOut(Location::RegisterLocation(EAX));
+ locations->SetInAt(0, Location::RequiresRegister());
+ locations->SetInAt(1, Location::Any());
+ // Note that TypeCheckSlowPathX86 uses this "out" register too.
+ locations->SetOut(Location::RequiresRegister());
+ // When read barriers are enabled, we need a temporary register for
+ // some cases.
+ if (kEmitCompilerReadBarrier &&
+ (type_check_kind == TypeCheckKind::kAbstractClassCheck ||
+ type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
+ type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
+ locations->AddTemp(Location::RequiresRegister());
}
}
void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
LocationSummary* locations = instruction->GetLocations();
- Register obj = locations->InAt(0).AsRegister<Register>();
+ Location obj_loc = locations->InAt(0);
+ Register obj = obj_loc.AsRegister<Register>();
Location cls = locations->InAt(1);
- Register out = locations->Out().AsRegister<Register>();
+ Location out_loc = locations->Out();
+ Register out = out_loc.AsRegister<Register>();
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
@@ -5354,15 +5717,9 @@
__ j(kEqual, &zero);
}
- // In case of an interface/unresolved check, we put the object class into the object register.
- // This is safe, as the register is caller-save, and the object must be in another
- // register if it survives the runtime call.
- Register target = (instruction->GetTypeCheckKind() == TypeCheckKind::kInterfaceCheck) ||
- (instruction->GetTypeCheckKind() == TypeCheckKind::kUnresolvedCheck)
- ? obj
- : out;
- __ movl(target, Address(obj, class_offset));
- __ MaybeUnpoisonHeapReference(target);
+ // /* HeapReference<Class> */ out = obj->klass_
+ __ movl(out, Address(obj, class_offset));
+ codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, obj_loc, class_offset);
switch (instruction->GetTypeCheckKind()) {
case TypeCheckKind::kExactCheck: {
@@ -5379,13 +5736,23 @@
__ jmp(&done);
break;
}
+
case TypeCheckKind::kAbstractClassCheck: {
// If the class is abstract, we eagerly fetch the super class of the
// object to avoid doing a comparison we know will fail.
NearLabel loop;
__ Bind(&loop);
+ Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `out` into `temp` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ Register temp = temp_loc.AsRegister<Register>();
+ __ movl(temp, out);
+ }
+ // /* HeapReference<Class> */ out = out->super_class_
__ movl(out, Address(out, super_offset));
- __ MaybeUnpoisonHeapReference(out);
+ codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset);
__ testl(out, out);
// If `out` is null, we use it for the result, and jump to `done`.
__ j(kEqual, &done);
@@ -5402,6 +5769,7 @@
}
break;
}
+
case TypeCheckKind::kClassHierarchyCheck: {
// Walk over the class hierarchy to find a match.
NearLabel loop, success;
@@ -5413,8 +5781,17 @@
__ cmpl(out, Address(ESP, cls.GetStackIndex()));
}
__ j(kEqual, &success);
+ Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `out` into `temp` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ Register temp = temp_loc.AsRegister<Register>();
+ __ movl(temp, out);
+ }
+ // /* HeapReference<Class> */ out = out->super_class_
__ movl(out, Address(out, super_offset));
- __ MaybeUnpoisonHeapReference(out);
+ codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset);
__ testl(out, out);
__ j(kNotEqual, &loop);
// If `out` is null, we use it for the result, and jump to `done`.
@@ -5426,6 +5803,7 @@
}
break;
}
+
case TypeCheckKind::kArrayObjectCheck: {
// Do an exact check.
NearLabel exact_check;
@@ -5436,9 +5814,18 @@
__ cmpl(out, Address(ESP, cls.GetStackIndex()));
}
__ j(kEqual, &exact_check);
- // Otherwise, we need to check that the object's class is a non primitive array.
+ // Otherwise, we need to check that the object's class is a non-primitive array.
+ Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `out` into `temp` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ Register temp = temp_loc.AsRegister<Register>();
+ __ movl(temp, out);
+ }
+ // /* HeapReference<Class> */ out = out->component_type_
__ movl(out, Address(out, component_offset));
- __ MaybeUnpoisonHeapReference(out);
+ codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, component_offset);
__ testl(out, out);
// If `out` is null, we use it for the result, and jump to `done`.
__ j(kEqual, &done);
@@ -5449,6 +5836,7 @@
__ jmp(&done);
break;
}
+
case TypeCheckKind::kArrayCheck: {
if (cls.IsRegister()) {
__ cmpl(out, cls.AsRegister<Register>());
@@ -5457,8 +5845,8 @@
__ cmpl(out, Address(ESP, cls.GetStackIndex()));
}
DCHECK(locations->OnlyCallsOnSlowPath());
- slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
- instruction, /* is_fatal */ false);
+ slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
+ /* is_fatal */ false);
codegen_->AddSlowPath(slow_path);
__ j(kNotEqual, slow_path->GetEntryLabel());
__ movl(out, Immediate(1));
@@ -5467,13 +5855,25 @@
}
break;
}
+
case TypeCheckKind::kUnresolvedCheck:
- case TypeCheckKind::kInterfaceCheck:
- default: {
- codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
- instruction,
- instruction->GetDexPc(),
- nullptr);
+ case TypeCheckKind::kInterfaceCheck: {
+ // Note that we indeed only call on slow path, but we always go
+ // into the slow path for the unresolved & interface check
+ // cases.
+ //
+ // We cannot directly call the InstanceofNonTrivial runtime
+ // entry point without resorting to a type checking slow path
+ // here (i.e. by calling InvokeRuntime directly), as it would
+ // require to assign fixed registers for the inputs of this
+ // HInstanceOf instruction (following the runtime calling
+ // convention), which might be cluttered by the potential first
+ // read barrier emission at the beginning of this method.
+ DCHECK(locations->OnlyCallsOnSlowPath());
+ slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
+ /* is_fatal */ false);
+ codegen_->AddSlowPath(slow_path);
+ __ jmp(slow_path->GetEntryLabel());
if (zero.IsLinked()) {
__ jmp(&done);
}
@@ -5498,75 +5898,73 @@
void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
-
- switch (instruction->GetTypeCheckKind()) {
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
+ switch (type_check_kind) {
case TypeCheckKind::kExactCheck:
case TypeCheckKind::kAbstractClassCheck:
case TypeCheckKind::kClassHierarchyCheck:
case TypeCheckKind::kArrayObjectCheck:
- call_kind = throws_into_catch
- ? LocationSummary::kCallOnSlowPath
- : LocationSummary::kNoCall;
- break;
- case TypeCheckKind::kInterfaceCheck:
- case TypeCheckKind::kUnresolvedCheck:
- call_kind = LocationSummary::kCall;
+ call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path.
break;
case TypeCheckKind::kArrayCheck:
+ case TypeCheckKind::kUnresolvedCheck:
+ case TypeCheckKind::kInterfaceCheck:
call_kind = LocationSummary::kCallOnSlowPath;
break;
}
-
- LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
- instruction, call_kind);
- if (call_kind != LocationSummary::kCall) {
- locations->SetInAt(0, Location::RequiresRegister());
- locations->SetInAt(1, Location::Any());
- // Note that TypeCheckSlowPathX86 uses this register too.
+ LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
+ locations->SetInAt(0, Location::RequiresRegister());
+ locations->SetInAt(1, Location::Any());
+ // Note that TypeCheckSlowPathX86 uses this "temp" register too.
+ locations->AddTemp(Location::RequiresRegister());
+ // When read barriers are enabled, we need an additional temporary
+ // register for some cases.
+ if (kEmitCompilerReadBarrier &&
+ (type_check_kind == TypeCheckKind::kAbstractClassCheck ||
+ type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
+ type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
locations->AddTemp(Location::RequiresRegister());
- } else {
- InvokeRuntimeCallingConvention calling_convention;
- locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
- locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
}
}
void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
LocationSummary* locations = instruction->GetLocations();
- Register obj = locations->InAt(0).AsRegister<Register>();
+ Location obj_loc = locations->InAt(0);
+ Register obj = obj_loc.AsRegister<Register>();
Location cls = locations->InAt(1);
- Register temp = locations->WillCall()
- ? kNoRegister
- : locations->GetTemp(0).AsRegister<Register>();
-
+ Location temp_loc = locations->GetTemp(0);
+ Register temp = temp_loc.AsRegister<Register>();
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
- SlowPathCode* slow_path = nullptr;
- if (!locations->WillCall()) {
- slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(
- instruction, !locations->CanCall());
- codegen_->AddSlowPath(slow_path);
- }
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
+ bool is_type_check_slow_path_fatal =
+ (type_check_kind == TypeCheckKind::kExactCheck ||
+ type_check_kind == TypeCheckKind::kAbstractClassCheck ||
+ type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
+ type_check_kind == TypeCheckKind::kArrayObjectCheck) &&
+ !instruction->CanThrowIntoCatchBlock();
+ SlowPathCode* type_check_slow_path =
+ new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
+ is_type_check_slow_path_fatal);
+ codegen_->AddSlowPath(type_check_slow_path);
- NearLabel done, abstract_entry;
+ NearLabel done;
// Avoid null check if we know obj is not null.
if (instruction->MustDoNullCheck()) {
__ testl(obj, obj);
__ j(kEqual, &done);
}
- if (locations->WillCall()) {
- __ movl(obj, Address(obj, class_offset));
- __ MaybeUnpoisonHeapReference(obj);
- } else {
- __ movl(temp, Address(obj, class_offset));
- __ MaybeUnpoisonHeapReference(temp);
- }
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ movl(temp, Address(obj, class_offset));
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
- switch (instruction->GetTypeCheckKind()) {
+ switch (type_check_kind) {
case TypeCheckKind::kExactCheck:
case TypeCheckKind::kArrayCheck: {
if (cls.IsRegister()) {
@@ -5577,19 +5975,44 @@
}
// Jump to slow path for throwing the exception or doing a
// more involved array check.
- __ j(kNotEqual, slow_path->GetEntryLabel());
+ __ j(kNotEqual, type_check_slow_path->GetEntryLabel());
break;
}
+
case TypeCheckKind::kAbstractClassCheck: {
// If the class is abstract, we eagerly fetch the super class of the
// object to avoid doing a comparison we know will fail.
- NearLabel loop, success;
+ NearLabel loop, compare_classes;
__ Bind(&loop);
+ Location temp2_loc =
+ kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `temp` into `temp2` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ Register temp2 = temp2_loc.AsRegister<Register>();
+ __ movl(temp2, temp);
+ }
+ // /* HeapReference<Class> */ temp = temp->super_class_
__ movl(temp, Address(temp, super_offset));
- __ MaybeUnpoisonHeapReference(temp);
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset);
+
+ // If the class reference currently in `temp` is not null, jump
+ // to the `compare_classes` label to compare it with the checked
+ // class.
__ testl(temp, temp);
- // Jump to the slow path to throw the exception.
- __ j(kEqual, slow_path->GetEntryLabel());
+ __ j(kNotEqual, &compare_classes);
+ // Otherwise, jump to the slow path to throw the exception.
+ //
+ // But before, move back the object's class into `temp` before
+ // going into the slow path, as it has been overwritten in the
+ // meantime.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ movl(temp, Address(obj, class_offset));
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
+ __ jmp(type_check_slow_path->GetEntryLabel());
+
+ __ Bind(&compare_classes);
if (cls.IsRegister()) {
__ cmpl(temp, cls.AsRegister<Register>());
} else {
@@ -5599,6 +6022,7 @@
__ j(kNotEqual, &loop);
break;
}
+
case TypeCheckKind::kClassHierarchyCheck: {
// Walk over the class hierarchy to find a match.
NearLabel loop;
@@ -5610,16 +6034,39 @@
__ cmpl(temp, Address(ESP, cls.GetStackIndex()));
}
__ j(kEqual, &done);
+
+ Location temp2_loc =
+ kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `temp` into `temp2` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ Register temp2 = temp2_loc.AsRegister<Register>();
+ __ movl(temp2, temp);
+ }
+ // /* HeapReference<Class> */ temp = temp->super_class_
__ movl(temp, Address(temp, super_offset));
- __ MaybeUnpoisonHeapReference(temp);
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset);
+
+ // If the class reference currently in `temp` is not null, jump
+ // back at the beginning of the loop.
__ testl(temp, temp);
__ j(kNotEqual, &loop);
- // Jump to the slow path to throw the exception.
- __ jmp(slow_path->GetEntryLabel());
+ // Otherwise, jump to the slow path to throw the exception.
+ //
+ // But before, move back the object's class into `temp` before
+ // going into the slow path, as it has been overwritten in the
+ // meantime.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ movl(temp, Address(obj, class_offset));
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
+ __ jmp(type_check_slow_path->GetEntryLabel());
break;
}
+
case TypeCheckKind::kArrayObjectCheck: {
// Do an exact check.
+ NearLabel check_non_primitive_component_type;
if (cls.IsRegister()) {
__ cmpl(temp, cls.AsRegister<Register>());
} else {
@@ -5627,29 +6074,67 @@
__ cmpl(temp, Address(ESP, cls.GetStackIndex()));
}
__ j(kEqual, &done);
- // Otherwise, we need to check that the object's class is a non primitive array.
+
+ // Otherwise, we need to check that the object's class is a non-primitive array.
+ Location temp2_loc =
+ kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `temp` into `temp2` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ Register temp2 = temp2_loc.AsRegister<Register>();
+ __ movl(temp2, temp);
+ }
+ // /* HeapReference<Class> */ temp = temp->component_type_
__ movl(temp, Address(temp, component_offset));
- __ MaybeUnpoisonHeapReference(temp);
+ codegen_->MaybeGenerateReadBarrier(
+ instruction, temp_loc, temp_loc, temp2_loc, component_offset);
+
+ // If the component type is not null (i.e. the object is indeed
+ // an array), jump to label `check_non_primitive_component_type`
+ // to further check that this component type is not a primitive
+ // type.
__ testl(temp, temp);
- __ j(kEqual, slow_path->GetEntryLabel());
+ __ j(kNotEqual, &check_non_primitive_component_type);
+ // Otherwise, jump to the slow path to throw the exception.
+ //
+ // But before, move back the object's class into `temp` before
+ // going into the slow path, as it has been overwritten in the
+ // meantime.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ movl(temp, Address(obj, class_offset));
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
+ __ jmp(type_check_slow_path->GetEntryLabel());
+
+ __ Bind(&check_non_primitive_component_type);
__ cmpw(Address(temp, primitive_offset), Immediate(Primitive::kPrimNot));
- __ j(kNotEqual, slow_path->GetEntryLabel());
+ __ j(kEqual, &done);
+ // Same comment as above regarding `temp` and the slow path.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ movl(temp, Address(obj, class_offset));
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
+ __ jmp(type_check_slow_path->GetEntryLabel());
break;
}
+
case TypeCheckKind::kUnresolvedCheck:
case TypeCheckKind::kInterfaceCheck:
- default:
- codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
- instruction,
- instruction->GetDexPc(),
- nullptr);
+ // We always go into the type check slow path for the unresolved &
+ // interface check cases.
+ //
+ // We cannot directly call the CheckCast runtime entry point
+ // without resorting to a type checking slow path here (i.e. by
+ // calling InvokeRuntime directly), as it would require to
+ // assign fixed registers for the inputs of this HInstanceOf
+ // instruction (following the runtime calling convention), which
+ // might be cluttered by the potential first read barrier
+ // emission at the beginning of this method.
+ __ jmp(type_check_slow_path->GetEntryLabel());
break;
}
__ Bind(&done);
- if (slow_path != nullptr) {
- __ Bind(slow_path->GetExitLabel());
- }
+ __ Bind(type_check_slow_path->GetExitLabel());
}
void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
@@ -5800,6 +6285,82 @@
}
}
+void CodeGeneratorX86::GenerateReadBarrier(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index) {
+ DCHECK(kEmitCompilerReadBarrier);
+
+ // If heap poisoning is enabled, the unpoisoning of the loaded
+ // reference will be carried out by the runtime within the slow
+ // path.
+ //
+ // Note that `ref` currently does not get unpoisoned (when heap
+ // poisoning is enabled), which is alright as the `ref` argument is
+ // not used by the artReadBarrierSlow entry point.
+ //
+ // TODO: Unpoison `ref` when it is used by artReadBarrierSlow.
+ SlowPathCode* slow_path = new (GetGraph()->GetArena())
+ ReadBarrierForHeapReferenceSlowPathX86(instruction, out, ref, obj, offset, index);
+ AddSlowPath(slow_path);
+
+ // TODO: When read barrier has a fast path, add it here.
+ /* Currently the read barrier call is inserted after the original load.
+ * However, if we have a fast path, we need to perform the load of obj.LockWord *before* the
+ * original load. This load-load ordering is required by the read barrier.
+ * The fast path/slow path (for Baker's algorithm) should look like:
+ *
+ * bool isGray = obj.LockWord & kReadBarrierMask;
+ * lfence; // load fence or artificial data dependence to prevent load-load reordering
+ * ref = obj.field; // this is the original load
+ * if (isGray) {
+ * ref = Mark(ref); // ideally the slow path just does Mark(ref)
+ * }
+ */
+
+ __ jmp(slow_path->GetEntryLabel());
+ __ Bind(slow_path->GetExitLabel());
+}
+
+void CodeGeneratorX86::MaybeGenerateReadBarrier(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index) {
+ if (kEmitCompilerReadBarrier) {
+ // If heap poisoning is enabled, unpoisoning will be taken care of
+ // by the runtime within the slow path.
+ GenerateReadBarrier(instruction, out, ref, obj, offset, index);
+ } else if (kPoisonHeapReferences) {
+ __ UnpoisonHeapReference(out.AsRegister<Register>());
+ }
+}
+
+void CodeGeneratorX86::GenerateReadBarrierForRoot(HInstruction* instruction,
+ Location out,
+ Location root) {
+ DCHECK(kEmitCompilerReadBarrier);
+
+ // Note that GC roots are not affected by heap poisoning, so we do
+ // not need to do anything special for this here.
+ SlowPathCode* slow_path =
+ new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathX86(instruction, out, root);
+ AddSlowPath(slow_path);
+
+ // TODO: Implement a fast path for ReadBarrierForRoot, performing
+ // the following operation (for Baker's algorithm):
+ //
+ // if (thread.tls32_.is_gc_marking) {
+ // root = Mark(root);
+ // }
+
+ __ jmp(slow_path->GetEntryLabel());
+ __ Bind(slow_path->GetExitLabel());
+}
+
void LocationsBuilderX86::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
// Nothing to do, this should be removed during prepare for register allocator.
LOG(FATAL) << "Unreachable";
diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h
index 177a059..064051c 100644
--- a/compiler/optimizing/code_generator_x86.h
+++ b/compiler/optimizing/code_generator_x86.h
@@ -227,14 +227,12 @@
void GenerateImplicitNullCheck(HNullCheck* instruction);
void GenerateExplicitNullCheck(HNullCheck* instruction);
void GenerateTestAndBranch(HInstruction* instruction,
+ size_t condition_input_index,
Label* true_target,
- Label* false_target,
- Label* always_true_target);
- void GenerateCompareTestAndBranch(HIf* if_inst,
- HCondition* condition,
+ Label* false_target);
+ void GenerateCompareTestAndBranch(HCondition* condition,
Label* true_target,
- Label* false_target,
- Label* always_true_target);
+ Label* false_target);
void GenerateFPJumps(HCondition* cond, Label* true_label, Label* false_label);
void GenerateLongComparesAndJumps(HCondition* cond, Label* true_label, Label* false_label);
void HandleGoto(HInstruction* got, HBasicBlock* successor);
@@ -397,6 +395,51 @@
void Finalize(CodeAllocator* allocator) OVERRIDE;
+ // Generate a read barrier for a heap reference within `instruction`.
+ //
+ // A read barrier for an object reference read from the heap is
+ // implemented as a call to the artReadBarrierSlow runtime entry
+ // point, which is passed the values in locations `ref`, `obj`, and
+ // `offset`:
+ //
+ // mirror::Object* artReadBarrierSlow(mirror::Object* ref,
+ // mirror::Object* obj,
+ // uint32_t offset);
+ //
+ // The `out` location contains the value returned by
+ // artReadBarrierSlow.
+ //
+ // When `index` is provided (i.e. for array accesses), the offset
+ // value passed to artReadBarrierSlow is adjusted to take `index`
+ // into account.
+ void GenerateReadBarrier(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index = Location::NoLocation());
+
+ // If read barriers are enabled, generate a read barrier for a heap reference.
+ // If heap poisoning is enabled, also unpoison the reference in `out`.
+ void MaybeGenerateReadBarrier(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index = Location::NoLocation());
+
+ // Generate a read barrier for a GC root within `instruction`.
+ //
+ // A read barrier for an object reference GC root is implemented as
+ // a call to the artReadBarrierForRootSlow runtime entry point,
+ // which is passed the value in location `root`:
+ //
+ // mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root);
+ //
+ // The `out` location contains the value returned by
+ // artReadBarrierForRootSlow.
+ void GenerateReadBarrierForRoot(HInstruction* instruction, Location out, Location root);
+
private:
Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, Register temp);
diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc
index e2ad667..dcc1808 100644
--- a/compiler/optimizing/code_generator_x86_64.cc
+++ b/compiler/optimizing/code_generator_x86_64.cc
@@ -34,6 +34,9 @@
namespace art {
+template<class MirrorType>
+class GcRoot;
+
namespace x86_64 {
static constexpr int kCurrentMethodStackOffset = 0;
@@ -52,16 +55,16 @@
explicit NullCheckSlowPathX86_64(HNullCheck* instruction) : instruction_(instruction) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
- CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
+ CodeGeneratorX86_64* x86_64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
__ Bind(GetEntryLabel());
if (instruction_->CanThrowIntoCatchBlock()) {
// Live registers will be restored in the catch block if caught.
SaveLiveRegisters(codegen, instruction_->GetLocations());
}
- x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowNullPointer),
- instruction_,
- instruction_->GetDexPc(),
- this);
+ x86_64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowNullPointer),
+ instruction_,
+ instruction_->GetDexPc(),
+ this);
}
bool IsFatal() const OVERRIDE { return true; }
@@ -78,16 +81,16 @@
explicit DivZeroCheckSlowPathX86_64(HDivZeroCheck* instruction) : instruction_(instruction) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
- CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
+ CodeGeneratorX86_64* x86_64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
__ Bind(GetEntryLabel());
if (instruction_->CanThrowIntoCatchBlock()) {
// Live registers will be restored in the catch block if caught.
SaveLiveRegisters(codegen, instruction_->GetLocations());
}
- x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowDivZero),
- instruction_,
- instruction_->GetDexPc(),
- this);
+ x86_64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowDivZero),
+ instruction_,
+ instruction_->GetDexPc(),
+ this);
}
bool IsFatal() const OVERRIDE { return true; }
@@ -139,18 +142,18 @@
: instruction_(instruction), successor_(successor) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
- CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
+ CodeGeneratorX86_64* x86_64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
__ Bind(GetEntryLabel());
SaveLiveRegisters(codegen, instruction_->GetLocations());
- x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pTestSuspend),
- instruction_,
- instruction_->GetDexPc(),
- this);
+ x86_64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pTestSuspend),
+ instruction_,
+ instruction_->GetDexPc(),
+ this);
RestoreLiveRegisters(codegen, instruction_->GetLocations());
if (successor_ == nullptr) {
__ jmp(GetReturnLabel());
} else {
- __ jmp(x64_codegen->GetLabelOf(successor_));
+ __ jmp(x86_64_codegen->GetLabelOf(successor_));
}
}
@@ -180,7 +183,7 @@
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
LocationSummary* locations = instruction_->GetLocations();
- CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
+ CodeGeneratorX86_64* x86_64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
__ Bind(GetEntryLabel());
if (instruction_->CanThrowIntoCatchBlock()) {
// Live registers will be restored in the catch block if caught.
@@ -196,8 +199,10 @@
locations->InAt(1),
Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
Primitive::kPrimInt);
- x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowArrayBounds),
- instruction_, instruction_->GetDexPc(), this);
+ x86_64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowArrayBounds),
+ instruction_,
+ instruction_->GetDexPc(),
+ this);
}
bool IsFatal() const OVERRIDE { return true; }
@@ -222,22 +227,25 @@
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
LocationSummary* locations = at_->GetLocations();
- CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
+ CodeGeneratorX86_64* x86_64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
__ Bind(GetEntryLabel());
SaveLiveRegisters(codegen, locations);
InvokeRuntimeCallingConvention calling_convention;
__ movl(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(cls_->GetTypeIndex()));
- x64_codegen->InvokeRuntime(do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
- : QUICK_ENTRY_POINT(pInitializeType),
- at_, dex_pc_, this);
+ x86_64_codegen->InvokeRuntime(do_clinit_ ?
+ QUICK_ENTRY_POINT(pInitializeStaticStorage) :
+ QUICK_ENTRY_POINT(pInitializeType),
+ at_,
+ dex_pc_,
+ this);
Location out = locations->Out();
// Move the class to the desired location.
if (out.IsValid()) {
DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
- x64_codegen->Move(out, Location::RegisterLocation(RAX));
+ x86_64_codegen->Move(out, Location::RegisterLocation(RAX));
}
RestoreLiveRegisters(codegen, locations);
@@ -271,18 +279,18 @@
LocationSummary* locations = instruction_->GetLocations();
DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
- CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
+ CodeGeneratorX86_64* x86_64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
__ Bind(GetEntryLabel());
SaveLiveRegisters(codegen, locations);
InvokeRuntimeCallingConvention calling_convention;
__ movl(CpuRegister(calling_convention.GetRegisterAt(0)),
Immediate(instruction_->GetStringIndex()));
- x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pResolveString),
- instruction_,
- instruction_->GetDexPc(),
- this);
- x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
+ x86_64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pResolveString),
+ instruction_,
+ instruction_->GetDexPc(),
+ this);
+ x86_64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
RestoreLiveRegisters(codegen, locations);
__ jmp(GetExitLabel());
}
@@ -308,18 +316,9 @@
DCHECK(instruction_->IsCheckCast()
|| !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
- CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
+ CodeGeneratorX86_64* x86_64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
__ Bind(GetEntryLabel());
- if (instruction_->IsCheckCast()) {
- // The codegen for the instruction overwrites `temp`, so put it back in place.
- CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
- CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
- uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
- __ movl(temp, Address(obj, class_offset));
- __ MaybeUnpoisonHeapReference(temp);
- }
-
if (!is_fatal_) {
SaveLiveRegisters(codegen, locations);
}
@@ -336,21 +335,24 @@
Primitive::kPrimNot);
if (instruction_->IsInstanceOf()) {
- x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
- instruction_,
- dex_pc,
- this);
+ x86_64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
+ instruction_,
+ dex_pc,
+ this);
+ CheckEntrypointTypes<
+ kQuickInstanceofNonTrivial, uint32_t, const mirror::Class*, const mirror::Class*>();
} else {
DCHECK(instruction_->IsCheckCast());
- x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
- instruction_,
- dex_pc,
- this);
+ x86_64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
+ instruction_,
+ dex_pc,
+ this);
+ CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
}
if (!is_fatal_) {
if (instruction_->IsInstanceOf()) {
- x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
+ x86_64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
}
RestoreLiveRegisters(codegen, locations);
@@ -375,15 +377,15 @@
: instruction_(instruction) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
- CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
+ CodeGeneratorX86_64* x86_64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
__ Bind(GetEntryLabel());
SaveLiveRegisters(codegen, instruction_->GetLocations());
DCHECK(instruction_->IsDeoptimize());
HDeoptimize* deoptimize = instruction_->AsDeoptimize();
- x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize),
- deoptimize,
- deoptimize->GetDexPc(),
- this);
+ x86_64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize),
+ deoptimize,
+ deoptimize->GetDexPc(),
+ this);
}
const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathX86_64"; }
@@ -421,11 +423,11 @@
nullptr);
codegen->GetMoveResolver()->EmitNativeCode(¶llel_move);
- CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
- x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
- instruction_,
- instruction_->GetDexPc(),
- this);
+ CodeGeneratorX86_64* x86_64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
+ x86_64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
+ instruction_,
+ instruction_->GetDexPc(),
+ this);
RestoreLiveRegisters(codegen, locations);
__ jmp(GetExitLabel());
}
@@ -438,6 +440,219 @@
DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathX86_64);
};
+// Slow path generating a read barrier for a heap reference.
+class ReadBarrierForHeapReferenceSlowPathX86_64 : public SlowPathCode {
+ public:
+ ReadBarrierForHeapReferenceSlowPathX86_64(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index)
+ : instruction_(instruction),
+ out_(out),
+ ref_(ref),
+ obj_(obj),
+ offset_(offset),
+ index_(index) {
+ DCHECK(kEmitCompilerReadBarrier);
+ // If `obj` is equal to `out` or `ref`, it means the initial
+ // object has been overwritten by (or after) the heap object
+ // reference load to be instrumented, e.g.:
+ //
+ // __ movl(out, Address(out, offset));
+ // codegen_->GenerateReadBarrier(instruction, out_loc, out_loc, out_loc, offset);
+ //
+ // In that case, we have lost the information about the original
+ // object, and the emitted read barrier cannot work properly.
+ DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out;
+ DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref;
+}
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ CodeGeneratorX86_64* x86_64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
+ LocationSummary* locations = instruction_->GetLocations();
+ CpuRegister reg_out = out_.AsRegister<CpuRegister>();
+ DCHECK(locations->CanCall());
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out.AsRegister())) << out_;
+ DCHECK(!instruction_->IsInvoke() ||
+ (instruction_->IsInvokeStaticOrDirect() &&
+ instruction_->GetLocations()->Intrinsified()));
+
+ __ Bind(GetEntryLabel());
+ SaveLiveRegisters(codegen, locations);
+
+ // We may have to change the index's value, but as `index_` is a
+ // constant member (like other "inputs" of this slow path),
+ // introduce a copy of it, `index`.
+ Location index = index_;
+ if (index_.IsValid()) {
+ // Handle `index_` for HArrayGet and intrinsic UnsafeGetObject.
+ if (instruction_->IsArrayGet()) {
+ // Compute real offset and store it in index_.
+ Register index_reg = index_.AsRegister<CpuRegister>().AsRegister();
+ DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg));
+ if (codegen->IsCoreCalleeSaveRegister(index_reg)) {
+ // We are about to change the value of `index_reg` (see the
+ // calls to art::x86_64::X86_64Assembler::shll and
+ // art::x86_64::X86_64Assembler::AddImmediate below), but it
+ // has not been saved by the previous call to
+ // art::SlowPathCode::SaveLiveRegisters, as it is a
+ // callee-save register --
+ // art::SlowPathCode::SaveLiveRegisters does not consider
+ // callee-save registers, as it has been designed with the
+ // assumption that callee-save registers are supposed to be
+ // handled by the called function. So, as a callee-save
+ // register, `index_reg` _would_ eventually be saved onto
+ // the stack, but it would be too late: we would have
+ // changed its value earlier. Therefore, we manually save
+ // it here into another freely available register,
+ // `free_reg`, chosen of course among the caller-save
+ // registers (as a callee-save `free_reg` register would
+ // exhibit the same problem).
+ //
+ // Note we could have requested a temporary register from
+ // the register allocator instead; but we prefer not to, as
+ // this is a slow path, and we know we can find a
+ // caller-save register that is available.
+ Register free_reg = FindAvailableCallerSaveRegister(codegen).AsRegister();
+ __ movl(CpuRegister(free_reg), CpuRegister(index_reg));
+ index_reg = free_reg;
+ index = Location::RegisterLocation(index_reg);
+ } else {
+ // The initial register stored in `index_` has already been
+ // saved in the call to art::SlowPathCode::SaveLiveRegisters
+ // (as it is not a callee-save register), so we can freely
+ // use it.
+ }
+ // Shifting the index value contained in `index_reg` by the
+ // scale factor (2) cannot overflow in practice, as the
+ // runtime is unable to allocate object arrays with a size
+ // larger than 2^26 - 1 (that is, 2^28 - 4 bytes).
+ __ shll(CpuRegister(index_reg), Immediate(TIMES_4));
+ static_assert(
+ sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
+ "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
+ __ AddImmediate(CpuRegister(index_reg), Immediate(offset_));
+ } else {
+ DCHECK(instruction_->IsInvoke());
+ DCHECK(instruction_->GetLocations()->Intrinsified());
+ DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) ||
+ (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile))
+ << instruction_->AsInvoke()->GetIntrinsic();
+ DCHECK_EQ(offset_, 0U);
+ DCHECK(index_.IsRegister());
+ }
+ }
+
+ // We're moving two or three locations to locations that could
+ // overlap, so we need a parallel move resolver.
+ InvokeRuntimeCallingConvention calling_convention;
+ HParallelMove parallel_move(codegen->GetGraph()->GetArena());
+ parallel_move.AddMove(ref_,
+ Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
+ Primitive::kPrimNot,
+ nullptr);
+ parallel_move.AddMove(obj_,
+ Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
+ Primitive::kPrimNot,
+ nullptr);
+ if (index.IsValid()) {
+ parallel_move.AddMove(index,
+ Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
+ Primitive::kPrimInt,
+ nullptr);
+ codegen->GetMoveResolver()->EmitNativeCode(¶llel_move);
+ } else {
+ codegen->GetMoveResolver()->EmitNativeCode(¶llel_move);
+ __ movl(CpuRegister(calling_convention.GetRegisterAt(2)), Immediate(offset_));
+ }
+ x86_64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierSlow),
+ instruction_,
+ instruction_->GetDexPc(),
+ this);
+ CheckEntrypointTypes<
+ kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>();
+ x86_64_codegen->Move(out_, Location::RegisterLocation(RAX));
+
+ RestoreLiveRegisters(codegen, locations);
+ __ jmp(GetExitLabel());
+ }
+
+ const char* GetDescription() const OVERRIDE {
+ return "ReadBarrierForHeapReferenceSlowPathX86_64";
+ }
+
+ private:
+ CpuRegister FindAvailableCallerSaveRegister(CodeGenerator* codegen) {
+ size_t ref = static_cast<int>(ref_.AsRegister<CpuRegister>().AsRegister());
+ size_t obj = static_cast<int>(obj_.AsRegister<CpuRegister>().AsRegister());
+ for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
+ if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) {
+ return static_cast<CpuRegister>(i);
+ }
+ }
+ // We shall never fail to find a free caller-save register, as
+ // there are more than two core caller-save registers on x86-64
+ // (meaning it is possible to find one which is different from
+ // `ref` and `obj`).
+ DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u);
+ LOG(FATAL) << "Could not find a free caller-save register";
+ UNREACHABLE();
+ }
+
+ HInstruction* const instruction_;
+ const Location out_;
+ const Location ref_;
+ const Location obj_;
+ const uint32_t offset_;
+ // An additional location containing an index to an array.
+ // Only used for HArrayGet and the UnsafeGetObject &
+ // UnsafeGetObjectVolatile intrinsics.
+ const Location index_;
+
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathX86_64);
+};
+
+// Slow path generating a read barrier for a GC root.
+class ReadBarrierForRootSlowPathX86_64 : public SlowPathCode {
+ public:
+ ReadBarrierForRootSlowPathX86_64(HInstruction* instruction, Location out, Location root)
+ : instruction_(instruction), out_(out), root_(root) {}
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ DCHECK(locations->CanCall());
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(out_.reg()));
+ DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString());
+
+ __ Bind(GetEntryLabel());
+ SaveLiveRegisters(codegen, locations);
+
+ InvokeRuntimeCallingConvention calling_convention;
+ CodeGeneratorX86_64* x86_64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
+ x86_64_codegen->Move(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_);
+ x86_64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierForRootSlow),
+ instruction_,
+ instruction_->GetDexPc(),
+ this);
+ CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>();
+ x86_64_codegen->Move(out_, Location::RegisterLocation(RAX));
+
+ RestoreLiveRegisters(codegen, locations);
+ __ jmp(GetExitLabel());
+ }
+
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathX86_64"; }
+
+ private:
+ HInstruction* const instruction_;
+ const Location out_;
+ const Location root_;
+
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathX86_64);
+};
+
#undef __
#define __ down_cast<X86_64Assembler*>(GetAssembler())->
@@ -503,7 +718,7 @@
Address::Absolute(invoke->GetStringInitOffset(), true));
break;
case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
- callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
+ callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
__ movq(temp.AsRegister<CpuRegister>(), Immediate(invoke->GetMethodAddress()));
@@ -522,7 +737,7 @@
__ Bind(&pc_relative_dex_cache_patches_.back().label);
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
- Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
+ Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
Register method_reg;
CpuRegister reg = temp.AsRegister<CpuRegister>();
if (current_method.IsRegister()) {
@@ -533,7 +748,7 @@
method_reg = reg.AsRegister();
__ movq(reg, Address(CpuRegister(RSP), kCurrentMethodStackOffset));
}
- // temp = temp->dex_cache_resolved_methods_;
+ // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_;
__ movq(reg,
Address(CpuRegister(method_reg),
ArtMethod::DexCacheResolvedMethodsOffset(kX86_64PointerSize).SizeValue()));
@@ -578,10 +793,17 @@
LocationSummary* locations = invoke->GetLocations();
Location receiver = locations->InAt(0);
size_t class_offset = mirror::Object::ClassOffset().SizeValue();
- // temp = object->GetClass();
DCHECK(receiver.IsRegister());
+ // /* HeapReference<Class> */ temp = receiver->klass_
__ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset));
MaybeRecordImplicitNullCheck(invoke);
+ // Instead of simply (possibly) unpoisoning `temp` here, we should
+ // emit a read barrier for the previous class reference load.
+ // However this is not required in practice, as this is an
+ // intermediate/temporary reference and because the current
+ // concurrent copying collector keeps the from-space memory
+ // intact/accessible until the end of the marking phase (the
+ // concurrent copying collector may not in the future).
__ MaybeUnpoisonHeapReference(temp);
// temp = temp->GetMethodAt(method_offset);
__ movq(temp, Address(temp, method_offset));
@@ -672,9 +894,9 @@
// Use a fake return address register to mimic Quick.
static constexpr Register kFakeReturnRegister = Register(kLastCpuRegister + 1);
CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph,
- const X86_64InstructionSetFeatures& isa_features,
- const CompilerOptions& compiler_options,
- OptimizingCompilerStats* stats)
+ const X86_64InstructionSetFeatures& isa_features,
+ const CompilerOptions& compiler_options,
+ OptimizingCompilerStats* stats)
: CodeGenerator(graph,
kNumberOfCpuRegisters,
kNumberOfFloatRegisters,
@@ -728,7 +950,7 @@
LOG(FATAL) << "Unreachable type " << type;
}
- return Location();
+ return Location::NoLocation();
}
void CodeGeneratorX86_64::SetupBlockedRegisters(bool is_baseline) const {
@@ -1082,26 +1304,19 @@
__ j(X86_64FPCondition(cond->GetCondition()), true_label);
}
-void InstructionCodeGeneratorX86_64::GenerateCompareTestAndBranch(HIf* if_instr,
- HCondition* condition,
- Label* true_target,
- Label* false_target,
- Label* always_true_target) {
+void InstructionCodeGeneratorX86_64::GenerateCompareTestAndBranch(HCondition* condition,
+ Label* true_target_in,
+ Label* false_target_in) {
+ // Generated branching requires both targets to be explicit. If either of the
+ // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead.
+ Label fallthrough_target;
+ Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in;
+ Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in;
+
LocationSummary* locations = condition->GetLocations();
Location left = locations->InAt(0);
Location right = locations->InAt(1);
- // We don't want true_target as a nullptr.
- if (true_target == nullptr) {
- true_target = always_true_target;
- }
- bool falls_through = (false_target == nullptr);
-
- // FP compares don't like null false_targets.
- if (false_target == nullptr) {
- false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
- }
-
Primitive::Type type = condition->InputAt(0)->GetType();
switch (type) {
case Primitive::kPrimLong: {
@@ -1160,135 +1375,140 @@
LOG(FATAL) << "Unexpected condition type " << type;
}
- if (!falls_through) {
+ if (false_target != &fallthrough_target) {
__ jmp(false_target);
}
+
+ if (fallthrough_target.IsLinked()) {
+ __ Bind(&fallthrough_target);
+ }
}
+static bool AreEflagsSetFrom(HInstruction* cond, HInstruction* branch) {
+ // Moves may affect the eflags register (move zero uses xorl), so the EFLAGS
+ // are set only strictly before `branch`. We can't use the eflags on long
+ // conditions if they are materialized due to the complex branching.
+ return cond->IsCondition() &&
+ cond->GetNext() == branch &&
+ !Primitive::IsFloatingPointType(cond->InputAt(0)->GetType());
+}
+
void InstructionCodeGeneratorX86_64::GenerateTestAndBranch(HInstruction* instruction,
+ size_t condition_input_index,
Label* true_target,
- Label* false_target,
- Label* always_true_target) {
- HInstruction* cond = instruction->InputAt(0);
- if (cond->IsIntConstant()) {
+ Label* false_target) {
+ HInstruction* cond = instruction->InputAt(condition_input_index);
+
+ if (true_target == nullptr && false_target == nullptr) {
+ // Nothing to do. The code always falls through.
+ return;
+ } else if (cond->IsIntConstant()) {
// Constant condition, statically compared against 1.
- int32_t cond_value = cond->AsIntConstant()->GetValue();
- if (cond_value == 1) {
- if (always_true_target != nullptr) {
- __ jmp(always_true_target);
+ if (cond->AsIntConstant()->IsOne()) {
+ if (true_target != nullptr) {
+ __ jmp(true_target);
}
- return;
} else {
- DCHECK_EQ(cond_value, 0);
+ DCHECK(cond->AsIntConstant()->IsZero());
+ if (false_target != nullptr) {
+ __ jmp(false_target);
+ }
+ }
+ return;
+ }
+
+ // The following code generates these patterns:
+ // (1) true_target == nullptr && false_target != nullptr
+ // - opposite condition true => branch to false_target
+ // (2) true_target != nullptr && false_target == nullptr
+ // - condition true => branch to true_target
+ // (3) true_target != nullptr && false_target != nullptr
+ // - condition true => branch to true_target
+ // - branch to false_target
+ if (IsBooleanValueOrMaterializedCondition(cond)) {
+ if (AreEflagsSetFrom(cond, instruction)) {
+ if (true_target == nullptr) {
+ __ j(X86_64IntegerCondition(cond->AsCondition()->GetOppositeCondition()), false_target);
+ } else {
+ __ j(X86_64IntegerCondition(cond->AsCondition()->GetCondition()), true_target);
+ }
+ } else {
+ // Materialized condition, compare against 0.
+ Location lhs = instruction->GetLocations()->InAt(condition_input_index);
+ if (lhs.IsRegister()) {
+ __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
+ } else {
+ __ cmpl(Address(CpuRegister(RSP), lhs.GetStackIndex()), Immediate(0));
+ }
+ if (true_target == nullptr) {
+ __ j(kEqual, false_target);
+ } else {
+ __ j(kNotEqual, true_target);
+ }
}
} else {
+ // Condition has not been materialized, use its inputs as the
+ // comparison and its condition as the branch condition.
HCondition* condition = cond->AsCondition();
- bool is_materialized = condition == nullptr || condition->NeedsMaterialization();
- // Moves do not affect the eflags register, so if the condition is
- // evaluated just before the if, we don't need to evaluate it
- // again. We can't use the eflags on FP conditions if they are
- // materialized due to the complex branching.
- Primitive::Type type = (condition != nullptr)
- ? cond->InputAt(0)->GetType()
- : Primitive::kPrimInt;
- bool eflags_set = condition != nullptr
- && condition->IsBeforeWhenDisregardMoves(instruction)
- && !Primitive::IsFloatingPointType(type);
- // Can we optimize the jump if we know that the next block is the true case?
- bool can_jump_to_false = CanReverseCondition(always_true_target, false_target, condition);
- if (is_materialized) {
- if (!eflags_set) {
- // Materialized condition, compare against 0.
- Location lhs = instruction->GetLocations()->InAt(0);
- if (lhs.IsRegister()) {
- __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
- } else {
- __ cmpl(Address(CpuRegister(RSP), lhs.GetStackIndex()),
- Immediate(0));
- }
- if (can_jump_to_false) {
- __ j(kEqual, false_target);
- return;
- }
- __ j(kNotEqual, true_target);
+ // If this is a long or FP comparison that has been folded into
+ // the HCondition, generate the comparison directly.
+ Primitive::Type type = condition->InputAt(0)->GetType();
+ if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
+ GenerateCompareTestAndBranch(condition, true_target, false_target);
+ return;
+ }
+
+ Location lhs = condition->GetLocations()->InAt(0);
+ Location rhs = condition->GetLocations()->InAt(1);
+ if (rhs.IsRegister()) {
+ __ cmpl(lhs.AsRegister<CpuRegister>(), rhs.AsRegister<CpuRegister>());
+ } else if (rhs.IsConstant()) {
+ int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
+ if (constant == 0) {
+ __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
} else {
- if (can_jump_to_false) {
- __ j(X86_64IntegerCondition(condition->GetOppositeCondition()), false_target);
- return;
- }
- __ j(X86_64IntegerCondition(condition->GetCondition()), true_target);
+ __ cmpl(lhs.AsRegister<CpuRegister>(), Immediate(constant));
}
} else {
- // Condition has not been materialized, use its inputs as the
- // comparison and its condition as the branch condition.
-
- // Is this a long or FP comparison that has been folded into the HCondition?
- if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
- // Generate the comparison directly.
- GenerateCompareTestAndBranch(instruction->AsIf(), condition,
- true_target, false_target, always_true_target);
- return;
- }
-
- Location lhs = cond->GetLocations()->InAt(0);
- Location rhs = cond->GetLocations()->InAt(1);
- if (rhs.IsRegister()) {
- __ cmpl(lhs.AsRegister<CpuRegister>(), rhs.AsRegister<CpuRegister>());
- } else if (rhs.IsConstant()) {
- int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
- if (constant == 0) {
- __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
- } else {
- __ cmpl(lhs.AsRegister<CpuRegister>(), Immediate(constant));
- }
- } else {
- __ cmpl(lhs.AsRegister<CpuRegister>(),
- Address(CpuRegister(RSP), rhs.GetStackIndex()));
- }
-
- if (can_jump_to_false) {
- __ j(X86_64IntegerCondition(condition->GetOppositeCondition()), false_target);
- return;
- }
-
+ __ cmpl(lhs.AsRegister<CpuRegister>(),
+ Address(CpuRegister(RSP), rhs.GetStackIndex()));
+ }
+ if (true_target == nullptr) {
+ __ j(X86_64IntegerCondition(condition->GetOppositeCondition()), false_target);
+ } else {
__ j(X86_64IntegerCondition(condition->GetCondition()), true_target);
}
}
- if (false_target != nullptr) {
+
+ // If neither branch falls through (case 3), the conditional branch to `true_target`
+ // was already emitted (case 2) and we need to emit a jump to `false_target`.
+ if (true_target != nullptr && false_target != nullptr) {
__ jmp(false_target);
}
}
void LocationsBuilderX86_64::VisitIf(HIf* if_instr) {
- LocationSummary* locations =
- new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
- HInstruction* cond = if_instr->InputAt(0);
- if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
+ LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
+ if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
locations->SetInAt(0, Location::Any());
}
}
void InstructionCodeGeneratorX86_64::VisitIf(HIf* if_instr) {
- Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
- Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
- Label* always_true_target = true_target;
- if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
- if_instr->IfTrueSuccessor())) {
- always_true_target = nullptr;
- }
- if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
- if_instr->IfFalseSuccessor())) {
- false_target = nullptr;
- }
- GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
+ HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
+ HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
+ Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
+ nullptr : codegen_->GetLabelOf(true_successor);
+ Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
+ nullptr : codegen_->GetLabelOf(false_successor);
+ GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
}
void LocationsBuilderX86_64::VisitDeoptimize(HDeoptimize* deoptimize) {
LocationSummary* locations = new (GetGraph()->GetArena())
LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
- HInstruction* cond = deoptimize->InputAt(0);
- if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
+ if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
locations->SetInAt(0, Location::Any());
}
}
@@ -1297,8 +1517,10 @@
SlowPathCode* slow_path = new (GetGraph()->GetArena())
DeoptimizationSlowPathX86_64(deoptimize);
codegen_->AddSlowPath(slow_path);
- Label* slow_path_entry = slow_path->GetEntryLabel();
- GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
+ GenerateTestAndBranch(deoptimize,
+ /* condition_input_index */ 0,
+ slow_path->GetEntryLabel(),
+ /* false_target */ nullptr);
}
void LocationsBuilderX86_64::VisitLocal(HLocal* local) {
@@ -1836,7 +2058,7 @@
LOG(FATAL) << "Unexpected parameter type " << type;
break;
}
- return Location();
+ return Location::NoLocation();
}
void LocationsBuilderX86_64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
@@ -1907,7 +2129,6 @@
}
codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
-
DCHECK(!codegen_->IsLeafMethod());
codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
}
@@ -1920,31 +2141,41 @@
void InstructionCodeGeneratorX86_64::VisitInvokeInterface(HInvokeInterface* invoke) {
// TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
- CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>();
+ LocationSummary* locations = invoke->GetLocations();
+ CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
+ CpuRegister hidden_reg = locations->GetTemp(1).AsRegister<CpuRegister>();
uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
invoke->GetImtIndex() % mirror::Class::kImtSize, kX86_64PointerSize).Uint32Value();
- LocationSummary* locations = invoke->GetLocations();
Location receiver = locations->InAt(0);
size_t class_offset = mirror::Object::ClassOffset().SizeValue();
- // Set the hidden argument.
- CpuRegister hidden_reg = invoke->GetLocations()->GetTemp(1).AsRegister<CpuRegister>();
+ // Set the hidden argument. This is safe to do this here, as RAX
+ // won't be modified thereafter, before the `call` instruction.
+ DCHECK_EQ(RAX, hidden_reg.AsRegister());
codegen_->Load64BitValue(hidden_reg, invoke->GetDexMethodIndex());
- // temp = object->GetClass();
if (receiver.IsStackSlot()) {
__ movl(temp, Address(CpuRegister(RSP), receiver.GetStackIndex()));
+ // /* HeapReference<Class> */ temp = temp->klass_
__ movl(temp, Address(temp, class_offset));
} else {
+ // /* HeapReference<Class> */ temp = receiver->klass_
__ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset));
}
codegen_->MaybeRecordImplicitNullCheck(invoke);
+ // Instead of simply (possibly) unpoisoning `temp` here, we should
+ // emit a read barrier for the previous class reference load.
+ // However this is not required in practice, as this is an
+ // intermediate/temporary reference and because the current
+ // concurrent copying collector keeps the from-space memory
+ // intact/accessible until the end of the marking phase (the
+ // concurrent copying collector may not in the future).
__ MaybeUnpoisonHeapReference(temp);
// temp = temp->GetImtEntryAt(method_offset);
__ movq(temp, Address(temp, method_offset));
// call temp->GetEntryPoint();
- __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
- kX86_64WordSize).SizeValue()));
+ __ call(Address(temp,
+ ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86_64WordSize).SizeValue()));
DCHECK(!codegen_->IsLeafMethod());
codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
@@ -3686,13 +3917,23 @@
void LocationsBuilderX86_64::HandleFieldGet(HInstruction* instruction) {
DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
+ bool object_field_get_with_read_barrier =
+ kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
LocationSummary* locations =
- new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
+ new (GetGraph()->GetArena()) LocationSummary(instruction,
+ object_field_get_with_read_barrier ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall);
locations->SetInAt(0, Location::RequiresRegister());
if (Primitive::IsFloatingPointType(instruction->GetType())) {
locations->SetOut(Location::RequiresFpuRegister());
} else {
- locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
+ // The output overlaps for an object field get when read barriers
+ // are enabled: we do not want the move to overwrite the object's
+ // location, as we need it to emit the read barrier.
+ locations->SetOut(
+ Location::RequiresRegister(),
+ object_field_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap);
}
}
@@ -3701,7 +3942,8 @@
DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
LocationSummary* locations = instruction->GetLocations();
- CpuRegister base = locations->InAt(0).AsRegister<CpuRegister>();
+ Location base_loc = locations->InAt(0);
+ CpuRegister base = base_loc.AsRegister<CpuRegister>();
Location out = locations->Out();
bool is_volatile = field_info.IsVolatile();
Primitive::Type field_type = field_info.GetFieldType();
@@ -3761,7 +4003,7 @@
}
if (field_type == Primitive::kPrimNot) {
- __ MaybeUnpoisonHeapReference(out.AsRegister<CpuRegister>());
+ codegen_->MaybeGenerateReadBarrier(instruction, out, out, base_loc, offset);
}
}
@@ -4079,20 +4321,31 @@
}
void LocationsBuilderX86_64::VisitArrayGet(HArrayGet* instruction) {
+ bool object_array_get_with_read_barrier =
+ kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
LocationSummary* locations =
- new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
+ new (GetGraph()->GetArena()) LocationSummary(instruction,
+ object_array_get_with_read_barrier ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
if (Primitive::IsFloatingPointType(instruction->GetType())) {
locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
} else {
- locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
+ // The output overlaps for an object array get when read barriers
+ // are enabled: we do not want the move to overwrite the array's
+ // location, as we need it to emit the read barrier.
+ locations->SetOut(
+ Location::RequiresRegister(),
+ object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap);
}
}
void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) {
LocationSummary* locations = instruction->GetLocations();
- CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
+ Location obj_loc = locations->InAt(0);
+ CpuRegister obj = obj_loc.AsRegister<CpuRegister>();
Location index = locations->InAt(1);
Primitive::Type type = instruction->GetType();
@@ -4147,8 +4400,9 @@
case Primitive::kPrimInt:
case Primitive::kPrimNot: {
- static_assert(sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
- "art::mirror::HeapReference<mirror::Object> and int32_t have different sizes.");
+ static_assert(
+ sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
+ "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
CpuRegister out = locations->Out().AsRegister<CpuRegister>();
if (index.IsConstant()) {
@@ -4203,8 +4457,17 @@
codegen_->MaybeRecordImplicitNullCheck(instruction);
if (type == Primitive::kPrimNot) {
- CpuRegister out = locations->Out().AsRegister<CpuRegister>();
- __ MaybeUnpoisonHeapReference(out);
+ static_assert(
+ sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
+ "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
+ uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
+ Location out = locations->Out();
+ if (index.IsConstant()) {
+ uint32_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
+ codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, offset);
+ } else {
+ codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, data_offset, index);
+ }
}
}
@@ -4214,10 +4477,14 @@
bool needs_write_barrier =
CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
bool may_need_runtime_call = instruction->NeedsTypeCheck();
+ bool object_array_set_with_read_barrier =
+ kEmitCompilerReadBarrier && (value_type == Primitive::kPrimNot);
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
instruction,
- may_need_runtime_call ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall);
+ (may_need_runtime_call || object_array_set_with_read_barrier) ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
@@ -4229,18 +4496,24 @@
if (needs_write_barrier) {
// Temporary registers for the write barrier.
- locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too.
+
+ // This first temporary register is possibly used for heap
+ // reference poisoning and/or read barrier emission too.
+ locations->AddTemp(Location::RequiresRegister());
+ // This second temporary register is possibly used for read
+ // barrier emission too.
locations->AddTemp(Location::RequiresRegister());
}
}
void InstructionCodeGeneratorX86_64::VisitArraySet(HArraySet* instruction) {
LocationSummary* locations = instruction->GetLocations();
- CpuRegister array = locations->InAt(0).AsRegister<CpuRegister>();
+ Location array_loc = locations->InAt(0);
+ CpuRegister array = array_loc.AsRegister<CpuRegister>();
Location index = locations->InAt(1);
Location value = locations->InAt(2);
Primitive::Type value_type = instruction->GetComponentType();
- bool may_need_runtime_call = locations->CanCall();
+ bool may_need_runtime_call = instruction->NeedsTypeCheck();
bool needs_write_barrier =
CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
@@ -4284,6 +4557,7 @@
Address address = index.IsConstant()
? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
: Address(array, index.AsRegister<CpuRegister>(), TIMES_4, offset);
+
if (!value.IsRegister()) {
// Just setting null.
DCHECK(instruction->InputAt(2)->IsNullConstant());
@@ -4312,22 +4586,62 @@
__ Bind(¬_null);
}
- __ movl(temp, Address(array, class_offset));
- codegen_->MaybeRecordImplicitNullCheck(instruction);
- __ MaybeUnpoisonHeapReference(temp);
- __ movl(temp, Address(temp, component_offset));
- // No need to poison/unpoison, we're comparing two poisoned references.
- __ cmpl(temp, Address(register_value, class_offset));
- if (instruction->StaticTypeOfArrayIsObjectArray()) {
- __ j(kEqual, &do_put);
- __ MaybeUnpoisonHeapReference(temp);
- __ movl(temp, Address(temp, super_offset));
- // No need to unpoison the result, we're comparing against null.
- __ testl(temp, temp);
- __ j(kNotEqual, slow_path->GetEntryLabel());
- __ Bind(&do_put);
+ if (kEmitCompilerReadBarrier) {
+ // When read barriers are enabled, the type checking
+ // instrumentation requires two read barriers:
+ //
+ // __ movl(temp2, temp);
+ // // /* HeapReference<Class> */ temp = temp->component_type_
+ // __ movl(temp, Address(temp, component_offset));
+ // codegen_->GenerateReadBarrier(
+ // instruction, temp_loc, temp_loc, temp2_loc, component_offset);
+ //
+ // // /* HeapReference<Class> */ temp2 = register_value->klass_
+ // __ movl(temp2, Address(register_value, class_offset));
+ // codegen_->GenerateReadBarrier(
+ // instruction, temp2_loc, temp2_loc, value, class_offset, temp_loc);
+ //
+ // __ cmpl(temp, temp2);
+ //
+ // However, the second read barrier may trash `temp`, as it
+ // is a temporary register, and as such would not be saved
+ // along with live registers before calling the runtime (nor
+ // restored afterwards). So in this case, we bail out and
+ // delegate the work to the array set slow path.
+ //
+ // TODO: Extend the register allocator to support a new
+ // "(locally) live temp" location so as to avoid always
+ // going into the slow path when read barriers are enabled.
+ __ jmp(slow_path->GetEntryLabel());
} else {
- __ j(kNotEqual, slow_path->GetEntryLabel());
+ // /* HeapReference<Class> */ temp = array->klass_
+ __ movl(temp, Address(array, class_offset));
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
+ __ MaybeUnpoisonHeapReference(temp);
+
+ // /* HeapReference<Class> */ temp = temp->component_type_
+ __ movl(temp, Address(temp, component_offset));
+ // If heap poisoning is enabled, no need to unpoison `temp`
+ // nor the object reference in `register_value->klass`, as
+ // we are comparing two poisoned references.
+ __ cmpl(temp, Address(register_value, class_offset));
+
+ if (instruction->StaticTypeOfArrayIsObjectArray()) {
+ __ j(kEqual, &do_put);
+ // If heap poisoning is enabled, the `temp` reference has
+ // not been unpoisoned yet; unpoison it now.
+ __ MaybeUnpoisonHeapReference(temp);
+
+ // /* HeapReference<Class> */ temp = temp->super_class_
+ __ movl(temp, Address(temp, super_offset));
+ // If heap poisoning is enabled, no need to unpoison
+ // `temp`, as we are comparing against null below.
+ __ testl(temp, temp);
+ __ j(kNotEqual, slow_path->GetEntryLabel());
+ __ Bind(&do_put);
+ } else {
+ __ j(kNotEqual, slow_path->GetEntryLabel());
+ }
}
}
@@ -4353,6 +4667,7 @@
break;
}
+
case Primitive::kPrimInt: {
uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
Address address = index.IsConstant()
@@ -4802,7 +5117,8 @@
CodeGenerator::CreateLoadClassLocationSummary(
cls,
Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
- Location::RegisterLocation(RAX));
+ Location::RegisterLocation(RAX),
+ /* code_generator_supports_read_barrier */ true);
}
void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) {
@@ -4816,18 +5132,40 @@
return;
}
- CpuRegister out = locations->Out().AsRegister<CpuRegister>();
+ Location out_loc = locations->Out();
+ CpuRegister out = out_loc.AsRegister<CpuRegister>();
CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>();
+
if (cls->IsReferrersClass()) {
DCHECK(!cls->CanCallRuntime());
DCHECK(!cls->MustGenerateClinitCheck());
- __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
+ uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value();
+ if (kEmitCompilerReadBarrier) {
+ // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_)
+ __ leaq(out, Address(current_method, declaring_class_offset));
+ // /* mirror::Class* */ out = out->Read()
+ codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc);
+ } else {
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ __ movl(out, Address(current_method, declaring_class_offset));
+ }
} else {
DCHECK(cls->CanCallRuntime());
- __ movq(out, Address(
- current_method, ArtMethod::DexCacheResolvedTypesOffset(kX86_64PointerSize).Int32Value()));
- __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
- // TODO: We will need a read barrier here.
+ // /* GcRoot<mirror::Class>[] */ out =
+ // current_method.ptr_sized_fields_->dex_cache_resolved_types_
+ __ movq(out, Address(current_method,
+ ArtMethod::DexCacheResolvedTypesOffset(kX86_64PointerSize).Int32Value()));
+
+ size_t cache_offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex());
+ if (kEmitCompilerReadBarrier) {
+ // /* GcRoot<mirror::Class>* */ out = &out[type_index]
+ __ leaq(out, Address(out, cache_offset));
+ // /* mirror::Class* */ out = out->Read()
+ codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc);
+ } else {
+ // /* GcRoot<mirror::Class> */ out = out[type_index]
+ __ movl(out, Address(out, cache_offset));
+ }
SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86_64(
cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
@@ -4872,12 +5210,35 @@
codegen_->AddSlowPath(slow_path);
LocationSummary* locations = load->GetLocations();
- CpuRegister out = locations->Out().AsRegister<CpuRegister>();
+ Location out_loc = locations->Out();
+ CpuRegister out = out_loc.AsRegister<CpuRegister>();
CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>();
- __ movl(out, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
- __ movq(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
- __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
- // TODO: We will need a read barrier here.
+
+ uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value();
+ if (kEmitCompilerReadBarrier) {
+ // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_)
+ __ leaq(out, Address(current_method, declaring_class_offset));
+ // /* mirror::Class* */ out = out->Read()
+ codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc);
+ } else {
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ __ movl(out, Address(current_method, declaring_class_offset));
+ }
+
+ // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
+ __ movq(out, Address(out, mirror::Class::DexCacheStringsOffset().Uint32Value()));
+
+ size_t cache_offset = CodeGenerator::GetCacheOffset(load->GetStringIndex());
+ if (kEmitCompilerReadBarrier) {
+ // /* GcRoot<mirror::String>* */ out = &out[string_index]
+ __ leaq(out, Address(out, cache_offset));
+ // /* mirror::String* */ out = out->Read()
+ codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc);
+ } else {
+ // /* GcRoot<mirror::String> */ out = out[string_index]
+ __ movl(out, Address(out, cache_offset));
+ }
+
__ testl(out, out);
__ j(kEqual, slow_path->GetEntryLabel());
__ Bind(slow_path->GetExitLabel());
@@ -4921,40 +5282,44 @@
void LocationsBuilderX86_64::VisitInstanceOf(HInstanceOf* instruction) {
LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
- switch (instruction->GetTypeCheckKind()) {
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
+ switch (type_check_kind) {
case TypeCheckKind::kExactCheck:
case TypeCheckKind::kAbstractClassCheck:
case TypeCheckKind::kClassHierarchyCheck:
case TypeCheckKind::kArrayObjectCheck:
- call_kind = LocationSummary::kNoCall;
- break;
- case TypeCheckKind::kUnresolvedCheck:
- case TypeCheckKind::kInterfaceCheck:
- call_kind = LocationSummary::kCall;
+ call_kind =
+ kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall;
break;
case TypeCheckKind::kArrayCheck:
+ case TypeCheckKind::kUnresolvedCheck:
+ case TypeCheckKind::kInterfaceCheck:
call_kind = LocationSummary::kCallOnSlowPath;
break;
}
+
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
- if (call_kind != LocationSummary::kCall) {
- locations->SetInAt(0, Location::RequiresRegister());
- locations->SetInAt(1, Location::Any());
- // Note that TypeCheckSlowPathX86_64 uses this register too.
- locations->SetOut(Location::RequiresRegister());
- } else {
- InvokeRuntimeCallingConvention calling_convention;
- locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
- locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
- locations->SetOut(Location::RegisterLocation(RAX));
+ locations->SetInAt(0, Location::RequiresRegister());
+ locations->SetInAt(1, Location::Any());
+ // Note that TypeCheckSlowPathX86_64 uses this "out" register too.
+ locations->SetOut(Location::RequiresRegister());
+ // When read barriers are enabled, we need a temporary register for
+ // some cases.
+ if (kEmitCompilerReadBarrier &&
+ (type_check_kind == TypeCheckKind::kAbstractClassCheck ||
+ type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
+ type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
+ locations->AddTemp(Location::RequiresRegister());
}
}
void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) {
LocationSummary* locations = instruction->GetLocations();
- CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
+ Location obj_loc = locations->InAt(0);
+ CpuRegister obj = obj_loc.AsRegister<CpuRegister>();
Location cls = locations->InAt(1);
- CpuRegister out = locations->Out().AsRegister<CpuRegister>();
+ Location out_loc = locations->Out();
+ CpuRegister out = out_loc.AsRegister<CpuRegister>();
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
@@ -4969,15 +5334,9 @@
__ j(kEqual, &zero);
}
- // In case of an interface/unresolved check, we put the object class into the object register.
- // This is safe, as the register is caller-save, and the object must be in another
- // register if it survives the runtime call.
- CpuRegister target = (instruction->GetTypeCheckKind() == TypeCheckKind::kInterfaceCheck) ||
- (instruction->GetTypeCheckKind() == TypeCheckKind::kUnresolvedCheck)
- ? obj
- : out;
- __ movl(target, Address(obj, class_offset));
- __ MaybeUnpoisonHeapReference(target);
+ // /* HeapReference<Class> */ out = obj->klass_
+ __ movl(out, Address(obj, class_offset));
+ codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, obj_loc, class_offset);
switch (instruction->GetTypeCheckKind()) {
case TypeCheckKind::kExactCheck: {
@@ -4999,13 +5358,23 @@
}
break;
}
+
case TypeCheckKind::kAbstractClassCheck: {
// If the class is abstract, we eagerly fetch the super class of the
// object to avoid doing a comparison we know will fail.
NearLabel loop, success;
__ Bind(&loop);
+ Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `out` into `temp` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ CpuRegister temp = temp_loc.AsRegister<CpuRegister>();
+ __ movl(temp, out);
+ }
+ // /* HeapReference<Class> */ out = out->super_class_
__ movl(out, Address(out, super_offset));
- __ MaybeUnpoisonHeapReference(out);
+ codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset);
__ testl(out, out);
// If `out` is null, we use it for the result, and jump to `done`.
__ j(kEqual, &done);
@@ -5022,6 +5391,7 @@
}
break;
}
+
case TypeCheckKind::kClassHierarchyCheck: {
// Walk over the class hierarchy to find a match.
NearLabel loop, success;
@@ -5033,8 +5403,17 @@
__ cmpl(out, Address(CpuRegister(RSP), cls.GetStackIndex()));
}
__ j(kEqual, &success);
+ Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `out` into `temp` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ CpuRegister temp = temp_loc.AsRegister<CpuRegister>();
+ __ movl(temp, out);
+ }
+ // /* HeapReference<Class> */ out = out->super_class_
__ movl(out, Address(out, super_offset));
- __ MaybeUnpoisonHeapReference(out);
+ codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset);
__ testl(out, out);
__ j(kNotEqual, &loop);
// If `out` is null, we use it for the result, and jump to `done`.
@@ -5046,6 +5425,7 @@
}
break;
}
+
case TypeCheckKind::kArrayObjectCheck: {
// Do an exact check.
NearLabel exact_check;
@@ -5056,9 +5436,18 @@
__ cmpl(out, Address(CpuRegister(RSP), cls.GetStackIndex()));
}
__ j(kEqual, &exact_check);
- // Otherwise, we need to check that the object's class is a non primitive array.
+ // Otherwise, we need to check that the object's class is a non-primitive array.
+ Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `out` into `temp` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ CpuRegister temp = temp_loc.AsRegister<CpuRegister>();
+ __ movl(temp, out);
+ }
+ // /* HeapReference<Class> */ out = out->component_type_
__ movl(out, Address(out, component_offset));
- __ MaybeUnpoisonHeapReference(out);
+ codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, component_offset);
__ testl(out, out);
// If `out` is null, we use it for the result, and jump to `done`.
__ j(kEqual, &done);
@@ -5069,6 +5458,7 @@
__ jmp(&done);
break;
}
+
case TypeCheckKind::kArrayCheck: {
if (cls.IsRegister()) {
__ cmpl(out, cls.AsRegister<CpuRegister>());
@@ -5077,8 +5467,8 @@
__ cmpl(out, Address(CpuRegister(RSP), cls.GetStackIndex()));
}
DCHECK(locations->OnlyCallsOnSlowPath());
- slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(
- instruction, /* is_fatal */ false);
+ slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(instruction,
+ /* is_fatal */ false);
codegen_->AddSlowPath(slow_path);
__ j(kNotEqual, slow_path->GetEntryLabel());
__ movl(out, Immediate(1));
@@ -5087,13 +5477,25 @@
}
break;
}
+
case TypeCheckKind::kUnresolvedCheck:
- case TypeCheckKind::kInterfaceCheck:
- default: {
- codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
- instruction,
- instruction->GetDexPc(),
- nullptr);
+ case TypeCheckKind::kInterfaceCheck: {
+ // Note that we indeed only call on slow path, but we always go
+ // into the slow path for the unresolved & interface check
+ // cases.
+ //
+ // We cannot directly call the InstanceofNonTrivial runtime
+ // entry point without resorting to a type checking slow path
+ // here (i.e. by calling InvokeRuntime directly), as it would
+ // require to assign fixed registers for the inputs of this
+ // HInstanceOf instruction (following the runtime calling
+ // convention), which might be cluttered by the potential first
+ // read barrier emission at the beginning of this method.
+ DCHECK(locations->OnlyCallsOnSlowPath());
+ slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(instruction,
+ /* is_fatal */ false);
+ codegen_->AddSlowPath(slow_path);
+ __ jmp(slow_path->GetEntryLabel());
if (zero.IsLinked()) {
__ jmp(&done);
}
@@ -5118,58 +5520,60 @@
void LocationsBuilderX86_64::VisitCheckCast(HCheckCast* instruction) {
LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
-
- switch (instruction->GetTypeCheckKind()) {
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
+ switch (type_check_kind) {
case TypeCheckKind::kExactCheck:
case TypeCheckKind::kAbstractClassCheck:
case TypeCheckKind::kClassHierarchyCheck:
case TypeCheckKind::kArrayObjectCheck:
- call_kind = throws_into_catch
- ? LocationSummary::kCallOnSlowPath
- : LocationSummary::kNoCall;
- break;
- case TypeCheckKind::kUnresolvedCheck:
- case TypeCheckKind::kInterfaceCheck:
- call_kind = LocationSummary::kCall;
+ call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path.
break;
case TypeCheckKind::kArrayCheck:
+ case TypeCheckKind::kUnresolvedCheck:
+ case TypeCheckKind::kInterfaceCheck:
call_kind = LocationSummary::kCallOnSlowPath;
break;
}
-
- LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
- instruction, call_kind);
- if (call_kind != LocationSummary::kCall) {
- locations->SetInAt(0, Location::RequiresRegister());
- locations->SetInAt(1, Location::Any());
- // Note that TypeCheckSlowPathX86_64 uses this register too.
+ LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
+ locations->SetInAt(0, Location::RequiresRegister());
+ locations->SetInAt(1, Location::Any());
+ // Note that TypeCheckSlowPathX86_64 uses this "temp" register too.
+ locations->AddTemp(Location::RequiresRegister());
+ // When read barriers are enabled, we need an additional temporary
+ // register for some cases.
+ if (kEmitCompilerReadBarrier &&
+ (type_check_kind == TypeCheckKind::kAbstractClassCheck ||
+ type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
+ type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
locations->AddTemp(Location::RequiresRegister());
- } else {
- InvokeRuntimeCallingConvention calling_convention;
- locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
- locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
}
}
void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) {
LocationSummary* locations = instruction->GetLocations();
- CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
+ Location obj_loc = locations->InAt(0);
+ CpuRegister obj = obj_loc.AsRegister<CpuRegister>();
Location cls = locations->InAt(1);
- CpuRegister temp = locations->WillCall()
- ? CpuRegister(kNoRegister)
- : locations->GetTemp(0).AsRegister<CpuRegister>();
-
+ Location temp_loc = locations->GetTemp(0);
+ CpuRegister temp = temp_loc.AsRegister<CpuRegister>();
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
- SlowPathCode* slow_path = nullptr;
- if (!locations->WillCall()) {
- slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(
- instruction, !locations->CanCall());
- codegen_->AddSlowPath(slow_path);
- }
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
+ bool is_type_check_slow_path_fatal =
+ (type_check_kind == TypeCheckKind::kExactCheck ||
+ type_check_kind == TypeCheckKind::kAbstractClassCheck ||
+ type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
+ type_check_kind == TypeCheckKind::kArrayObjectCheck) &&
+ !instruction->CanThrowIntoCatchBlock();
+ SlowPathCode* type_check_slow_path =
+ new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(instruction,
+ is_type_check_slow_path_fatal);
+ codegen_->AddSlowPath(type_check_slow_path);
NearLabel done;
// Avoid null check if we know obj is not null.
@@ -5178,15 +5582,11 @@
__ j(kEqual, &done);
}
- if (locations->WillCall()) {
- __ movl(obj, Address(obj, class_offset));
- __ MaybeUnpoisonHeapReference(obj);
- } else {
- __ movl(temp, Address(obj, class_offset));
- __ MaybeUnpoisonHeapReference(temp);
- }
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ movl(temp, Address(obj, class_offset));
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
- switch (instruction->GetTypeCheckKind()) {
+ switch (type_check_kind) {
case TypeCheckKind::kExactCheck:
case TypeCheckKind::kArrayCheck: {
if (cls.IsRegister()) {
@@ -5197,19 +5597,44 @@
}
// Jump to slow path for throwing the exception or doing a
// more involved array check.
- __ j(kNotEqual, slow_path->GetEntryLabel());
+ __ j(kNotEqual, type_check_slow_path->GetEntryLabel());
break;
}
+
case TypeCheckKind::kAbstractClassCheck: {
// If the class is abstract, we eagerly fetch the super class of the
// object to avoid doing a comparison we know will fail.
- NearLabel loop;
+ NearLabel loop, compare_classes;
__ Bind(&loop);
+ Location temp2_loc =
+ kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `temp` into `temp2` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ CpuRegister temp2 = temp2_loc.AsRegister<CpuRegister>();
+ __ movl(temp2, temp);
+ }
+ // /* HeapReference<Class> */ temp = temp->super_class_
__ movl(temp, Address(temp, super_offset));
- __ MaybeUnpoisonHeapReference(temp);
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset);
+
+ // If the class reference currently in `temp` is not null, jump
+ // to the `compare_classes` label to compare it with the checked
+ // class.
__ testl(temp, temp);
- // Jump to the slow path to throw the exception.
- __ j(kEqual, slow_path->GetEntryLabel());
+ __ j(kNotEqual, &compare_classes);
+ // Otherwise, jump to the slow path to throw the exception.
+ //
+ // But before, move back the object's class into `temp` before
+ // going into the slow path, as it has been overwritten in the
+ // meantime.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ movl(temp, Address(obj, class_offset));
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
+ __ jmp(type_check_slow_path->GetEntryLabel());
+
+ __ Bind(&compare_classes);
if (cls.IsRegister()) {
__ cmpl(temp, cls.AsRegister<CpuRegister>());
} else {
@@ -5219,6 +5644,7 @@
__ j(kNotEqual, &loop);
break;
}
+
case TypeCheckKind::kClassHierarchyCheck: {
// Walk over the class hierarchy to find a match.
NearLabel loop;
@@ -5230,16 +5656,39 @@
__ cmpl(temp, Address(CpuRegister(RSP), cls.GetStackIndex()));
}
__ j(kEqual, &done);
+
+ Location temp2_loc =
+ kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `temp` into `temp2` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ CpuRegister temp2 = temp2_loc.AsRegister<CpuRegister>();
+ __ movl(temp2, temp);
+ }
+ // /* HeapReference<Class> */ temp = temp->super_class_
__ movl(temp, Address(temp, super_offset));
- __ MaybeUnpoisonHeapReference(temp);
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset);
+
+ // If the class reference currently in `temp` is not null, jump
+ // back at the beginning of the loop.
__ testl(temp, temp);
__ j(kNotEqual, &loop);
- // Jump to the slow path to throw the exception.
- __ jmp(slow_path->GetEntryLabel());
+ // Otherwise, jump to the slow path to throw the exception.
+ //
+ // But before, move back the object's class into `temp` before
+ // going into the slow path, as it has been overwritten in the
+ // meantime.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ movl(temp, Address(obj, class_offset));
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
+ __ jmp(type_check_slow_path->GetEntryLabel());
break;
}
+
case TypeCheckKind::kArrayObjectCheck: {
// Do an exact check.
+ NearLabel check_non_primitive_component_type;
if (cls.IsRegister()) {
__ cmpl(temp, cls.AsRegister<CpuRegister>());
} else {
@@ -5247,29 +5696,67 @@
__ cmpl(temp, Address(CpuRegister(RSP), cls.GetStackIndex()));
}
__ j(kEqual, &done);
- // Otherwise, we need to check that the object's class is a non primitive array.
+
+ // Otherwise, we need to check that the object's class is a non-primitive array.
+ Location temp2_loc =
+ kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
+ if (kEmitCompilerReadBarrier) {
+ // Save the value of `temp` into `temp2` before overwriting it
+ // in the following move operation, as we will need it for the
+ // read barrier below.
+ CpuRegister temp2 = temp2_loc.AsRegister<CpuRegister>();
+ __ movl(temp2, temp);
+ }
+ // /* HeapReference<Class> */ temp = temp->component_type_
__ movl(temp, Address(temp, component_offset));
- __ MaybeUnpoisonHeapReference(temp);
+ codegen_->MaybeGenerateReadBarrier(
+ instruction, temp_loc, temp_loc, temp2_loc, component_offset);
+
+ // If the component type is not null (i.e. the object is indeed
+ // an array), jump to label `check_non_primitive_component_type`
+ // to further check that this component type is not a primitive
+ // type.
__ testl(temp, temp);
- __ j(kEqual, slow_path->GetEntryLabel());
+ __ j(kNotEqual, &check_non_primitive_component_type);
+ // Otherwise, jump to the slow path to throw the exception.
+ //
+ // But before, move back the object's class into `temp` before
+ // going into the slow path, as it has been overwritten in the
+ // meantime.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ movl(temp, Address(obj, class_offset));
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
+ __ jmp(type_check_slow_path->GetEntryLabel());
+
+ __ Bind(&check_non_primitive_component_type);
__ cmpw(Address(temp, primitive_offset), Immediate(Primitive::kPrimNot));
- __ j(kNotEqual, slow_path->GetEntryLabel());
+ __ j(kEqual, &done);
+ // Same comment as above regarding `temp` and the slow path.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ movl(temp, Address(obj, class_offset));
+ codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
+ __ jmp(type_check_slow_path->GetEntryLabel());
break;
}
+
case TypeCheckKind::kUnresolvedCheck:
case TypeCheckKind::kInterfaceCheck:
- default:
- codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
- instruction,
- instruction->GetDexPc(),
- nullptr);
+ // We always go into the type check slow path for the unresolved &
+ // interface check cases.
+ //
+ // We cannot directly call the CheckCast runtime entry point
+ // without resorting to a type checking slow path here (i.e. by
+ // calling InvokeRuntime directly), as it would require to
+ // assign fixed registers for the inputs of this HInstanceOf
+ // instruction (following the runtime calling convention), which
+ // might be cluttered by the potential first read barrier
+ // emission at the beginning of this method.
+ __ jmp(type_check_slow_path->GetEntryLabel());
break;
}
__ Bind(&done);
- if (slow_path != nullptr) {
- __ Bind(slow_path->GetExitLabel());
- }
+ __ Bind(type_check_slow_path->GetExitLabel());
}
void LocationsBuilderX86_64::VisitMonitorOperation(HMonitorOperation* instruction) {
@@ -5402,6 +5889,82 @@
}
}
+void CodeGeneratorX86_64::GenerateReadBarrier(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index) {
+ DCHECK(kEmitCompilerReadBarrier);
+
+ // If heap poisoning is enabled, the unpoisoning of the loaded
+ // reference will be carried out by the runtime within the slow
+ // path.
+ //
+ // Note that `ref` currently does not get unpoisoned (when heap
+ // poisoning is enabled), which is alright as the `ref` argument is
+ // not used by the artReadBarrierSlow entry point.
+ //
+ // TODO: Unpoison `ref` when it is used by artReadBarrierSlow.
+ SlowPathCode* slow_path = new (GetGraph()->GetArena())
+ ReadBarrierForHeapReferenceSlowPathX86_64(instruction, out, ref, obj, offset, index);
+ AddSlowPath(slow_path);
+
+ // TODO: When read barrier has a fast path, add it here.
+ /* Currently the read barrier call is inserted after the original load.
+ * However, if we have a fast path, we need to perform the load of obj.LockWord *before* the
+ * original load. This load-load ordering is required by the read barrier.
+ * The fast path/slow path (for Baker's algorithm) should look like:
+ *
+ * bool isGray = obj.LockWord & kReadBarrierMask;
+ * lfence; // load fence or artificial data dependence to prevent load-load reordering
+ * ref = obj.field; // this is the original load
+ * if (isGray) {
+ * ref = Mark(ref); // ideally the slow path just does Mark(ref)
+ * }
+ */
+
+ __ jmp(slow_path->GetEntryLabel());
+ __ Bind(slow_path->GetExitLabel());
+}
+
+void CodeGeneratorX86_64::MaybeGenerateReadBarrier(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index) {
+ if (kEmitCompilerReadBarrier) {
+ // If heap poisoning is enabled, unpoisoning will be taken care of
+ // by the runtime within the slow path.
+ GenerateReadBarrier(instruction, out, ref, obj, offset, index);
+ } else if (kPoisonHeapReferences) {
+ __ UnpoisonHeapReference(out.AsRegister<CpuRegister>());
+ }
+}
+
+void CodeGeneratorX86_64::GenerateReadBarrierForRoot(HInstruction* instruction,
+ Location out,
+ Location root) {
+ DCHECK(kEmitCompilerReadBarrier);
+
+ // Note that GC roots are not affected by heap poisoning, so we do
+ // not need to do anything special for this here.
+ SlowPathCode* slow_path =
+ new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathX86_64(instruction, out, root);
+ AddSlowPath(slow_path);
+
+ // TODO: Implement a fast path for ReadBarrierForRoot, performing
+ // the following operation (for Baker's algorithm):
+ //
+ // if (thread.tls32_.is_gc_marking) {
+ // root = Mark(root);
+ // }
+
+ __ jmp(slow_path->GetEntryLabel());
+ __ Bind(slow_path->GetExitLabel());
+}
+
void LocationsBuilderX86_64::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
// Nothing to do, this should be removed during prepare for register allocator.
LOG(FATAL) << "Unreachable";
diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h
index 7084508..145b1f3 100644
--- a/compiler/optimizing/code_generator_x86_64.h
+++ b/compiler/optimizing/code_generator_x86_64.h
@@ -217,14 +217,12 @@
void PushOntoFPStack(Location source, uint32_t temp_offset,
uint32_t stack_adjustment, bool is_float);
void GenerateTestAndBranch(HInstruction* instruction,
+ size_t condition_input_index,
Label* true_target,
- Label* false_target,
- Label* always_true_target);
- void GenerateCompareTestAndBranch(HIf* if_inst,
- HCondition* condition,
+ Label* false_target);
+ void GenerateCompareTestAndBranch(HCondition* condition,
Label* true_target,
- Label* false_target,
- Label* always_true_target);
+ Label* false_target);
void GenerateFPJumps(HCondition* cond, Label* true_label, Label* false_label);
void HandleGoto(HInstruction* got, HBasicBlock* successor);
@@ -352,6 +350,51 @@
return isa_features_;
}
+ // Generate a read barrier for a heap reference within `instruction`.
+ //
+ // A read barrier for an object reference read from the heap is
+ // implemented as a call to the artReadBarrierSlow runtime entry
+ // point, which is passed the values in locations `ref`, `obj`, and
+ // `offset`:
+ //
+ // mirror::Object* artReadBarrierSlow(mirror::Object* ref,
+ // mirror::Object* obj,
+ // uint32_t offset);
+ //
+ // The `out` location contains the value returned by
+ // artReadBarrierSlow.
+ //
+ // When `index` provided (i.e., when it is different from
+ // Location::NoLocation()), the offset value passed to
+ // artReadBarrierSlow is adjusted to take `index` into account.
+ void GenerateReadBarrier(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index = Location::NoLocation());
+
+ // If read barriers are enabled, generate a read barrier for a heap reference.
+ // If heap poisoning is enabled, also unpoison the reference in `out`.
+ void MaybeGenerateReadBarrier(HInstruction* instruction,
+ Location out,
+ Location ref,
+ Location obj,
+ uint32_t offset,
+ Location index = Location::NoLocation());
+
+ // Generate a read barrier for a GC root within `instruction`.
+ //
+ // A read barrier for an object reference GC root is implemented as
+ // a call to the artReadBarrierForRootSlow runtime entry point,
+ // which is passed the value in location `root`:
+ //
+ // mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root);
+ //
+ // The `out` location contains the value returned by
+ // artReadBarrierForRootSlow.
+ void GenerateReadBarrierForRoot(HInstruction* instruction, Location out, Location root);
+
int ConstantAreaStart() const {
return constant_area_start_;
}
diff --git a/compiler/optimizing/graph_visualizer.cc b/compiler/optimizing/graph_visualizer.cc
index 2b77901..d166d00 100644
--- a/compiler/optimizing/graph_visualizer.cc
+++ b/compiler/optimizing/graph_visualizer.cc
@@ -397,6 +397,9 @@
<< invoke->IsRecursive()
<< std::noboolalpha;
StartAttributeStream("intrinsic") << invoke->GetIntrinsic();
+ if (invoke->IsStatic()) {
+ StartAttributeStream("clinit_check") << invoke->GetClinitCheckRequirement();
+ }
}
void VisitUnresolvedInstanceFieldGet(HUnresolvedInstanceFieldGet* field_access) OVERRIDE {
@@ -500,6 +503,18 @@
StartAttributeStream("exact") << std::boolalpha << info.IsExact() << std::noboolalpha;
} else if (instruction->IsLoadClass()) {
StartAttributeStream("klass") << "unresolved";
+ } else if (instruction->IsNullConstant()) {
+ // The NullConstant may be added to the graph during other passes that happen between
+ // ReferenceTypePropagation and Inliner (e.g. InstructionSimplifier). If the inliner
+ // doesn't run or doesn't inline anything, the NullConstant remains untyped.
+ // So we should check NullConstants for validity only after reference type propagation.
+ //
+ // Note: The infrastructure to properly type NullConstants everywhere is to complex to add
+ // for the benefits.
+ StartAttributeStream("klass") << "not_set";
+ DCHECK(!is_after_pass_
+ || !IsPass(ReferenceTypePropagation::kReferenceTypePropagationPassName))
+ << " Expected a valid rti after reference type propagation";
} else {
DCHECK(!is_after_pass_)
<< "Expected a valid rti after reference type propagation";
diff --git a/compiler/optimizing/instruction_simplifier.cc b/compiler/optimizing/instruction_simplifier.cc
index b97dc1a..9ad2dd1 100644
--- a/compiler/optimizing/instruction_simplifier.cc
+++ b/compiler/optimizing/instruction_simplifier.cc
@@ -796,6 +796,34 @@
HShl* shl = new(allocator) HShl(type, input_other, shift);
block->ReplaceAndRemoveInstructionWith(instruction, shl);
RecordSimplification();
+ } else if (IsPowerOfTwo(factor - 1)) {
+ // Transform code looking like
+ // MUL dst, src, (2^n + 1)
+ // into
+ // SHL tmp, src, n
+ // ADD dst, src, tmp
+ HShl* shl = new (allocator) HShl(type,
+ input_other,
+ GetGraph()->GetIntConstant(WhichPowerOf2(factor - 1)));
+ HAdd* add = new (allocator) HAdd(type, input_other, shl);
+
+ block->InsertInstructionBefore(shl, instruction);
+ block->ReplaceAndRemoveInstructionWith(instruction, add);
+ RecordSimplification();
+ } else if (IsPowerOfTwo(factor + 1)) {
+ // Transform code looking like
+ // MUL dst, src, (2^n - 1)
+ // into
+ // SHL tmp, src, n
+ // SUB dst, tmp, src
+ HShl* shl = new (allocator) HShl(type,
+ input_other,
+ GetGraph()->GetIntConstant(WhichPowerOf2(factor + 1)));
+ HSub* sub = new (allocator) HSub(type, shl, input_other);
+
+ block->InsertInstructionBefore(shl, instruction);
+ block->ReplaceAndRemoveInstructionWith(instruction, sub);
+ RecordSimplification();
}
}
}
diff --git a/compiler/optimizing/intrinsics_arm.cc b/compiler/optimizing/intrinsics_arm.cc
index 0a5acc3..d2017da 100644
--- a/compiler/optimizing/intrinsics_arm.cc
+++ b/compiler/optimizing/intrinsics_arm.cc
@@ -44,7 +44,23 @@
bool IntrinsicLocationsBuilderARM::TryDispatch(HInvoke* invoke) {
Dispatch(invoke);
LocationSummary* res = invoke->GetLocations();
- return res != nullptr && res->Intrinsified();
+ if (res == nullptr) {
+ return false;
+ }
+ if (kEmitCompilerReadBarrier && res->CanCall()) {
+ // Generating an intrinsic for this HInvoke may produce an
+ // IntrinsicSlowPathARM slow path. Currently this approach
+ // does not work when using read barriers, as the emitted
+ // calling sequence will make use of another slow path
+ // (ReadBarrierForRootSlowPathARM for HInvokeStaticOrDirect,
+ // ReadBarrierSlowPathARM for HInvokeVirtual). So we bail
+ // out in this case.
+ //
+ // TODO: Find a way to have intrinsics work with read barriers.
+ invoke->SetLocations(nullptr);
+ return false;
+ }
+ return res->Intrinsified();
}
#define __ assembler->
@@ -662,20 +678,23 @@
(type == Primitive::kPrimLong) ||
(type == Primitive::kPrimNot));
ArmAssembler* assembler = codegen->GetAssembler();
- Register base = locations->InAt(1).AsRegister<Register>(); // Object pointer.
- Register offset = locations->InAt(2).AsRegisterPairLow<Register>(); // Long offset, lo part only.
+ Location base_loc = locations->InAt(1);
+ Register base = base_loc.AsRegister<Register>(); // Object pointer.
+ Location offset_loc = locations->InAt(2);
+ Register offset = offset_loc.AsRegisterPairLow<Register>(); // Long offset, lo part only.
+ Location trg_loc = locations->Out();
if (type == Primitive::kPrimLong) {
- Register trg_lo = locations->Out().AsRegisterPairLow<Register>();
+ Register trg_lo = trg_loc.AsRegisterPairLow<Register>();
__ add(IP, base, ShifterOperand(offset));
if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) {
- Register trg_hi = locations->Out().AsRegisterPairHigh<Register>();
+ Register trg_hi = trg_loc.AsRegisterPairHigh<Register>();
__ ldrexd(trg_lo, trg_hi, IP);
} else {
__ ldrd(trg_lo, Address(IP));
}
} else {
- Register trg = locations->Out().AsRegister<Register>();
+ Register trg = trg_loc.AsRegister<Register>();
__ ldr(trg, Address(base, offset));
}
@@ -684,14 +703,18 @@
}
if (type == Primitive::kPrimNot) {
- Register trg = locations->Out().AsRegister<Register>();
- __ MaybeUnpoisonHeapReference(trg);
+ codegen->MaybeGenerateReadBarrier(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc);
}
}
static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
+ bool can_call = kEmitCompilerReadBarrier &&
+ (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject ||
+ invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile);
LocationSummary* locations = new (arena) LocationSummary(invoke,
- LocationSummary::kNoCall,
+ can_call ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall,
kIntrinsified);
locations->SetInAt(0, Location::NoLocation()); // Unused receiver.
locations->SetInAt(1, Location::RequiresRegister());
@@ -936,6 +959,7 @@
__ Bind(&loop_head);
__ ldrex(tmp_lo, tmp_ptr);
+ // TODO: Do we need a read barrier here when `type == Primitive::kPrimNot`?
__ subs(tmp_lo, tmp_lo, ShifterOperand(expected_lo));
@@ -964,7 +988,11 @@
// The UnsafeCASObject intrinsic does not always work when heap
// poisoning is enabled (it breaks run-test 004-UnsafeTest); turn it
// off temporarily as a quick fix.
+ //
// TODO(rpl): Fix it and turn it back on.
+ //
+ // TODO(rpl): Also, we should investigate whether we need a read
+ // barrier in the generated code.
if (kPoisonHeapReferences) {
return;
}
@@ -1400,6 +1428,10 @@
}
}
+// TODO: Implement read barriers in the SystemArrayCopy intrinsic.
+// Note that this code path is not used (yet) because we do not
+// intrinsify methods that can go into the IntrinsicSlowPathARM
+// slow path.
void IntrinsicCodeGeneratorARM::VisitSystemArrayCopy(HInvoke* invoke) {
ArmAssembler* assembler = GetAssembler();
LocationSummary* locations = invoke->GetLocations();
diff --git a/compiler/optimizing/intrinsics_mips.cc b/compiler/optimizing/intrinsics_mips.cc
index a94e3a8..3268445 100644
--- a/compiler/optimizing/intrinsics_mips.cc
+++ b/compiler/optimizing/intrinsics_mips.cc
@@ -138,6 +138,221 @@
#define __ assembler->
+static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
+ LocationSummary* locations = new (arena) LocationSummary(invoke,
+ LocationSummary::kNoCall,
+ kIntrinsified);
+ locations->SetInAt(0, Location::RequiresFpuRegister());
+ locations->SetOut(Location::RequiresRegister());
+}
+
+static void MoveFPToInt(LocationSummary* locations, bool is64bit, MipsAssembler* assembler) {
+ FRegister in = locations->InAt(0).AsFpuRegister<FRegister>();
+
+ if (is64bit) {
+ Register out_lo = locations->Out().AsRegisterPairLow<Register>();
+ Register out_hi = locations->Out().AsRegisterPairHigh<Register>();
+
+ __ Mfc1(out_lo, in);
+ __ Mfhc1(out_hi, in);
+ } else {
+ Register out = locations->Out().AsRegister<Register>();
+
+ __ Mfc1(out, in);
+ }
+}
+
+// long java.lang.Double.doubleToRawLongBits(double)
+void IntrinsicLocationsBuilderMIPS::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
+ CreateFPToIntLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorMIPS::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
+ MoveFPToInt(invoke->GetLocations(), true, GetAssembler());
+}
+
+// int java.lang.Float.floatToRawIntBits(float)
+void IntrinsicLocationsBuilderMIPS::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
+ CreateFPToIntLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorMIPS::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
+ MoveFPToInt(invoke->GetLocations(), false, GetAssembler());
+}
+
+static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
+ LocationSummary* locations = new (arena) LocationSummary(invoke,
+ LocationSummary::kNoCall,
+ kIntrinsified);
+ locations->SetInAt(0, Location::RequiresRegister());
+ locations->SetOut(Location::RequiresFpuRegister());
+}
+
+static void MoveIntToFP(LocationSummary* locations, bool is64bit, MipsAssembler* assembler) {
+ FRegister out = locations->Out().AsFpuRegister<FRegister>();
+
+ if (is64bit) {
+ Register in_lo = locations->InAt(0).AsRegisterPairLow<Register>();
+ Register in_hi = locations->InAt(0).AsRegisterPairHigh<Register>();
+
+ __ Mtc1(in_lo, out);
+ __ Mthc1(in_hi, out);
+ } else {
+ Register in = locations->InAt(0).AsRegister<Register>();
+
+ __ Mtc1(in, out);
+ }
+}
+
+// double java.lang.Double.longBitsToDouble(long)
+void IntrinsicLocationsBuilderMIPS::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
+ CreateIntToFPLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorMIPS::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
+ MoveIntToFP(invoke->GetLocations(), true, GetAssembler());
+}
+
+// float java.lang.Float.intBitsToFloat(int)
+void IntrinsicLocationsBuilderMIPS::VisitFloatIntBitsToFloat(HInvoke* invoke) {
+ CreateIntToFPLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorMIPS::VisitFloatIntBitsToFloat(HInvoke* invoke) {
+ MoveIntToFP(invoke->GetLocations(), false, GetAssembler());
+}
+
+static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
+ LocationSummary* locations = new (arena) LocationSummary(invoke,
+ LocationSummary::kNoCall,
+ kIntrinsified);
+ locations->SetInAt(0, Location::RequiresRegister());
+ locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
+}
+
+static void GenReverseBytes(LocationSummary* locations,
+ Primitive::Type type,
+ MipsAssembler* assembler,
+ bool isR2OrNewer) {
+ DCHECK(type == Primitive::kPrimShort ||
+ type == Primitive::kPrimInt ||
+ type == Primitive::kPrimLong);
+
+ if (type == Primitive::kPrimShort) {
+ Register in = locations->InAt(0).AsRegister<Register>();
+ Register out = locations->Out().AsRegister<Register>();
+
+ if (isR2OrNewer) {
+ __ Wsbh(out, in);
+ __ Seh(out, out);
+ } else {
+ __ Sll(TMP, in, 24);
+ __ Sra(TMP, TMP, 16);
+ __ Sll(out, in, 16);
+ __ Srl(out, out, 24);
+ __ Or(out, out, TMP);
+ }
+ } else if (type == Primitive::kPrimInt) {
+ Register in = locations->InAt(0).AsRegister<Register>();
+ Register out = locations->Out().AsRegister<Register>();
+
+ if (isR2OrNewer) {
+ __ Rotr(out, in, 16);
+ __ Wsbh(out, out);
+ } else {
+ // MIPS32r1
+ // __ Rotr(out, in, 16);
+ __ Sll(TMP, in, 16);
+ __ Srl(out, in, 16);
+ __ Or(out, out, TMP);
+ // __ Wsbh(out, out);
+ __ LoadConst32(AT, 0x00FF00FF);
+ __ And(TMP, out, AT);
+ __ Sll(TMP, TMP, 8);
+ __ Srl(out, out, 8);
+ __ And(out, out, AT);
+ __ Or(out, out, TMP);
+ }
+ } else if (type == Primitive::kPrimLong) {
+ Register in_lo = locations->InAt(0).AsRegisterPairLow<Register>();
+ Register in_hi = locations->InAt(0).AsRegisterPairHigh<Register>();
+ Register out_lo = locations->Out().AsRegisterPairLow<Register>();
+ Register out_hi = locations->Out().AsRegisterPairHigh<Register>();
+
+ if (isR2OrNewer) {
+ __ Rotr(AT, in_hi, 16);
+ __ Rotr(TMP, in_lo, 16);
+ __ Wsbh(out_lo, AT);
+ __ Wsbh(out_hi, TMP);
+ } else {
+ // When calling CreateIntToIntLocations() we promised that the
+ // use of the out_lo/out_hi wouldn't overlap with the use of
+ // in_lo/in_hi. Be very careful not to write to out_lo/out_hi
+ // until we're completely done reading from in_lo/in_hi.
+ // __ Rotr(TMP, in_lo, 16);
+ __ Sll(TMP, in_lo, 16);
+ __ Srl(AT, in_lo, 16);
+ __ Or(TMP, TMP, AT); // Hold in TMP until it's safe
+ // to write to out_hi.
+ // __ Rotr(out_lo, in_hi, 16);
+ __ Sll(AT, in_hi, 16);
+ __ Srl(out_lo, in_hi, 16); // Here we are finally done reading
+ // from in_lo/in_hi so it's okay to
+ // write to out_lo/out_hi.
+ __ Or(out_lo, out_lo, AT);
+ // __ Wsbh(out_hi, out_hi);
+ __ LoadConst32(AT, 0x00FF00FF);
+ __ And(out_hi, TMP, AT);
+ __ Sll(out_hi, out_hi, 8);
+ __ Srl(TMP, TMP, 8);
+ __ And(TMP, TMP, AT);
+ __ Or(out_hi, out_hi, TMP);
+ // __ Wsbh(out_lo, out_lo);
+ __ And(TMP, out_lo, AT); // AT already holds the correct mask value
+ __ Sll(TMP, TMP, 8);
+ __ Srl(out_lo, out_lo, 8);
+ __ And(out_lo, out_lo, AT);
+ __ Or(out_lo, out_lo, TMP);
+ }
+ }
+}
+
+// int java.lang.Integer.reverseBytes(int)
+void IntrinsicLocationsBuilderMIPS::VisitIntegerReverseBytes(HInvoke* invoke) {
+ CreateIntToIntLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorMIPS::VisitIntegerReverseBytes(HInvoke* invoke) {
+ GenReverseBytes(invoke->GetLocations(),
+ Primitive::kPrimInt,
+ GetAssembler(),
+ codegen_->GetInstructionSetFeatures().IsMipsIsaRevGreaterThanEqual2());
+}
+
+// long java.lang.Long.reverseBytes(long)
+void IntrinsicLocationsBuilderMIPS::VisitLongReverseBytes(HInvoke* invoke) {
+ CreateIntToIntLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorMIPS::VisitLongReverseBytes(HInvoke* invoke) {
+ GenReverseBytes(invoke->GetLocations(),
+ Primitive::kPrimLong,
+ GetAssembler(),
+ codegen_->GetInstructionSetFeatures().IsMipsIsaRevGreaterThanEqual2());
+}
+
+// short java.lang.Short.reverseBytes(short)
+void IntrinsicLocationsBuilderMIPS::VisitShortReverseBytes(HInvoke* invoke) {
+ CreateIntToIntLocations(arena_, invoke);
+}
+
+void IntrinsicCodeGeneratorMIPS::VisitShortReverseBytes(HInvoke* invoke) {
+ GenReverseBytes(invoke->GetLocations(),
+ Primitive::kPrimShort,
+ GetAssembler(),
+ codegen_->GetInstructionSetFeatures().IsMipsIsaRevGreaterThanEqual2());
+}
+
// boolean java.lang.String.equals(Object anObject)
void IntrinsicLocationsBuilderMIPS::VisitStringEquals(HInvoke* invoke) {
LocationSummary* locations = new (arena_) LocationSummary(invoke,
@@ -250,15 +465,8 @@
UNIMPLEMENTED_INTRINSIC(IntegerReverse)
UNIMPLEMENTED_INTRINSIC(LongReverse)
-UNIMPLEMENTED_INTRINSIC(ShortReverseBytes)
-UNIMPLEMENTED_INTRINSIC(IntegerReverseBytes)
-UNIMPLEMENTED_INTRINSIC(LongReverseBytes)
UNIMPLEMENTED_INTRINSIC(LongNumberOfLeadingZeros)
UNIMPLEMENTED_INTRINSIC(IntegerNumberOfLeadingZeros)
-UNIMPLEMENTED_INTRINSIC(FloatIntBitsToFloat)
-UNIMPLEMENTED_INTRINSIC(DoubleLongBitsToDouble)
-UNIMPLEMENTED_INTRINSIC(FloatFloatToRawIntBits)
-UNIMPLEMENTED_INTRINSIC(DoubleDoubleToRawLongBits)
UNIMPLEMENTED_INTRINSIC(MathAbsDouble)
UNIMPLEMENTED_INTRINSIC(MathAbsFloat)
UNIMPLEMENTED_INTRINSIC(MathAbsInt)
diff --git a/compiler/optimizing/intrinsics_mips64.cc b/compiler/optimizing/intrinsics_mips64.cc
index ff843eb..3654159 100644
--- a/compiler/optimizing/intrinsics_mips64.cc
+++ b/compiler/optimizing/intrinsics_mips64.cc
@@ -1391,6 +1391,108 @@
__ Bind(slow_path->GetExitLabel());
}
+// boolean java.lang.String.equals(Object anObject)
+void IntrinsicLocationsBuilderMIPS64::VisitStringEquals(HInvoke* invoke) {
+ LocationSummary* locations = new (arena_) LocationSummary(invoke,
+ LocationSummary::kNoCall,
+ kIntrinsified);
+ locations->SetInAt(0, Location::RequiresRegister());
+ locations->SetInAt(1, Location::RequiresRegister());
+ locations->SetOut(Location::RequiresRegister());
+
+ // Temporary registers to store lengths of strings and for calculations.
+ locations->AddTemp(Location::RequiresRegister());
+ locations->AddTemp(Location::RequiresRegister());
+ locations->AddTemp(Location::RequiresRegister());
+}
+
+void IntrinsicCodeGeneratorMIPS64::VisitStringEquals(HInvoke* invoke) {
+ Mips64Assembler* assembler = GetAssembler();
+ LocationSummary* locations = invoke->GetLocations();
+
+ GpuRegister str = locations->InAt(0).AsRegister<GpuRegister>();
+ GpuRegister arg = locations->InAt(1).AsRegister<GpuRegister>();
+ GpuRegister out = locations->Out().AsRegister<GpuRegister>();
+
+ GpuRegister temp1 = locations->GetTemp(0).AsRegister<GpuRegister>();
+ GpuRegister temp2 = locations->GetTemp(1).AsRegister<GpuRegister>();
+ GpuRegister temp3 = locations->GetTemp(2).AsRegister<GpuRegister>();
+
+ Label loop;
+ Label end;
+ Label return_true;
+ Label return_false;
+
+ // Get offsets of count, value, and class fields within a string object.
+ const int32_t count_offset = mirror::String::CountOffset().Int32Value();
+ const int32_t value_offset = mirror::String::ValueOffset().Int32Value();
+ const int32_t class_offset = mirror::Object::ClassOffset().Int32Value();
+
+ // Note that the null check must have been done earlier.
+ DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
+
+ // If the register containing the pointer to "this", and the register
+ // containing the pointer to "anObject" are the same register then
+ // "this", and "anObject" are the same object and we can
+ // short-circuit the logic to a true result.
+ if (str == arg) {
+ __ LoadConst64(out, 1);
+ return;
+ }
+
+ // Check if input is null, return false if it is.
+ __ Beqzc(arg, &return_false);
+
+ // Reference equality check, return true if same reference.
+ __ Beqc(str, arg, &return_true);
+
+ // Instanceof check for the argument by comparing class fields.
+ // All string objects must have the same type since String cannot be subclassed.
+ // Receiver must be a string object, so its class field is equal to all strings' class fields.
+ // If the argument is a string object, its class field must be equal to receiver's class field.
+ __ Lw(temp1, str, class_offset);
+ __ Lw(temp2, arg, class_offset);
+ __ Bnec(temp1, temp2, &return_false);
+
+ // Load lengths of this and argument strings.
+ __ Lw(temp1, str, count_offset);
+ __ Lw(temp2, arg, count_offset);
+ // Check if lengths are equal, return false if they're not.
+ __ Bnec(temp1, temp2, &return_false);
+ // Return true if both strings are empty.
+ __ Beqzc(temp1, &return_true);
+
+ // Don't overwrite input registers
+ __ Move(TMP, str);
+ __ Move(temp3, arg);
+
+ // Assertions that must hold in order to compare strings 4 characters at a time.
+ DCHECK_ALIGNED(value_offset, 8);
+ static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded");
+
+ // Loop to compare strings 4 characters at a time starting at the beginning of the string.
+ // Ok to do this because strings are zero-padded to be 8-byte aligned.
+ __ Bind(&loop);
+ __ Ld(out, TMP, value_offset);
+ __ Ld(temp2, temp3, value_offset);
+ __ Bnec(out, temp2, &return_false);
+ __ Daddiu(TMP, TMP, 8);
+ __ Daddiu(temp3, temp3, 8);
+ __ Addiu(temp1, temp1, -4);
+ __ Bgtzc(temp1, &loop);
+
+ // Return true and exit the function.
+ // If loop does not result in returning false, we return true.
+ __ Bind(&return_true);
+ __ LoadConst64(out, 1);
+ __ B(&end);
+
+ // Return false and exit the function.
+ __ Bind(&return_false);
+ __ LoadConst64(out, 0);
+ __ Bind(&end);
+}
+
static void GenerateStringIndexOf(HInvoke* invoke,
Mips64Assembler* assembler,
CodeGeneratorMIPS64* codegen,
@@ -1586,8 +1688,6 @@
UNIMPLEMENTED_INTRINSIC(MathRoundDouble)
UNIMPLEMENTED_INTRINSIC(MathRoundFloat)
-UNIMPLEMENTED_INTRINSIC(StringEquals)
-
UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent)
UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck)
UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar)
diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc
index 040bf6a..371588f 100644
--- a/compiler/optimizing/intrinsics_x86.cc
+++ b/compiler/optimizing/intrinsics_x86.cc
@@ -55,7 +55,23 @@
bool IntrinsicLocationsBuilderX86::TryDispatch(HInvoke* invoke) {
Dispatch(invoke);
LocationSummary* res = invoke->GetLocations();
- return res != nullptr && res->Intrinsified();
+ if (res == nullptr) {
+ return false;
+ }
+ if (kEmitCompilerReadBarrier && res->CanCall()) {
+ // Generating an intrinsic for this HInvoke may produce an
+ // IntrinsicSlowPathX86 slow path. Currently this approach
+ // does not work when using read barriers, as the emitted
+ // calling sequence will make use of another slow path
+ // (ReadBarrierForRootSlowPathX86 for HInvokeStaticOrDirect,
+ // ReadBarrierSlowPathX86 for HInvokeVirtual). So we bail
+ // out in this case.
+ //
+ // TODO: Find a way to have intrinsics work with read barriers.
+ invoke->SetLocations(nullptr);
+ return false;
+ }
+ return res->Intrinsified();
}
static void MoveArguments(HInvoke* invoke, CodeGeneratorX86* codegen) {
@@ -1571,26 +1587,32 @@
GetAssembler()->fs()->movl(out, Address::Absolute(Thread::PeerOffset<kX86WordSize>()));
}
-static void GenUnsafeGet(LocationSummary* locations, Primitive::Type type,
- bool is_volatile, X86Assembler* assembler) {
- Register base = locations->InAt(1).AsRegister<Register>();
- Register offset = locations->InAt(2).AsRegisterPairLow<Register>();
- Location output = locations->Out();
+static void GenUnsafeGet(HInvoke* invoke,
+ Primitive::Type type,
+ bool is_volatile,
+ CodeGeneratorX86* codegen) {
+ X86Assembler* assembler = down_cast<X86Assembler*>(codegen->GetAssembler());
+ LocationSummary* locations = invoke->GetLocations();
+ Location base_loc = locations->InAt(1);
+ Register base = base_loc.AsRegister<Register>();
+ Location offset_loc = locations->InAt(2);
+ Register offset = offset_loc.AsRegisterPairLow<Register>();
+ Location output_loc = locations->Out();
switch (type) {
case Primitive::kPrimInt:
case Primitive::kPrimNot: {
- Register output_reg = output.AsRegister<Register>();
- __ movl(output_reg, Address(base, offset, ScaleFactor::TIMES_1, 0));
+ Register output = output_loc.AsRegister<Register>();
+ __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0));
if (type == Primitive::kPrimNot) {
- __ MaybeUnpoisonHeapReference(output_reg);
+ codegen->MaybeGenerateReadBarrier(invoke, output_loc, output_loc, base_loc, 0U, offset_loc);
}
break;
}
case Primitive::kPrimLong: {
- Register output_lo = output.AsRegisterPairLow<Register>();
- Register output_hi = output.AsRegisterPairHigh<Register>();
+ Register output_lo = output_loc.AsRegisterPairLow<Register>();
+ Register output_hi = output_loc.AsRegisterPairHigh<Register>();
if (is_volatile) {
// Need to use a XMM to read atomically.
XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
@@ -1613,8 +1635,13 @@
static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke,
bool is_long, bool is_volatile) {
+ bool can_call = kEmitCompilerReadBarrier &&
+ (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject ||
+ invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile);
LocationSummary* locations = new (arena) LocationSummary(invoke,
- LocationSummary::kNoCall,
+ can_call ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall,
kIntrinsified);
locations->SetInAt(0, Location::NoLocation()); // Unused receiver.
locations->SetInAt(1, Location::RequiresRegister());
@@ -1653,22 +1680,22 @@
void IntrinsicCodeGeneratorX86::VisitUnsafeGet(HInvoke* invoke) {
- GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimInt, false, GetAssembler());
+ GenUnsafeGet(invoke, Primitive::kPrimInt, false, codegen_);
}
void IntrinsicCodeGeneratorX86::VisitUnsafeGetVolatile(HInvoke* invoke) {
- GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimInt, true, GetAssembler());
+ GenUnsafeGet(invoke, Primitive::kPrimInt, true, codegen_);
}
void IntrinsicCodeGeneratorX86::VisitUnsafeGetLong(HInvoke* invoke) {
- GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimLong, false, GetAssembler());
+ GenUnsafeGet(invoke, Primitive::kPrimLong, false, codegen_);
}
void IntrinsicCodeGeneratorX86::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
- GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimLong, true, GetAssembler());
+ GenUnsafeGet(invoke, Primitive::kPrimLong, true, codegen_);
}
void IntrinsicCodeGeneratorX86::VisitUnsafeGetObject(HInvoke* invoke) {
- GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimNot, false, GetAssembler());
+ GenUnsafeGet(invoke, Primitive::kPrimNot, false, codegen_);
}
void IntrinsicCodeGeneratorX86::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
- GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimNot, true, GetAssembler());
+ GenUnsafeGet(invoke, Primitive::kPrimNot, true, codegen_);
}
@@ -1890,13 +1917,18 @@
__ LockCmpxchgl(Address(base, offset, TIMES_1, 0), value);
- // locked cmpxchg has full barrier semantics, and we don't need
+ // LOCK CMPXCHG has full barrier semantics, and we don't need
// scheduling barriers at this time.
// Convert ZF into the boolean result.
__ setb(kZero, out.AsRegister<Register>());
__ movzxb(out.AsRegister<Register>(), out.AsRegister<ByteRegister>());
+ // In the case of the `UnsafeCASObject` intrinsic, accessing an
+ // object in the heap with LOCK CMPXCHG does not require a read
+ // barrier, as we do not keep a reference to this heap location.
+ // However, if heap poisoning is enabled, we need to unpoison the
+ // values that were poisoned earlier.
if (kPoisonHeapReferences) {
if (base_equals_value) {
// `value` has been moved to a temporary register, no need to
@@ -1929,8 +1961,8 @@
LOG(FATAL) << "Unexpected CAS type " << type;
}
- // locked cmpxchg has full barrier semantics, and we don't need
- // scheduling barriers at this time.
+ // LOCK CMPXCHG/LOCK CMPXCHG8B have full barrier semantics, and we
+ // don't need scheduling barriers at this time.
// Convert ZF into the boolean result.
__ setb(kZero, out.AsRegister<Register>());
diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc
index a29f3ef..2d9f01b 100644
--- a/compiler/optimizing/intrinsics_x86_64.cc
+++ b/compiler/optimizing/intrinsics_x86_64.cc
@@ -50,8 +50,24 @@
bool IntrinsicLocationsBuilderX86_64::TryDispatch(HInvoke* invoke) {
Dispatch(invoke);
- const LocationSummary* res = invoke->GetLocations();
- return res != nullptr && res->Intrinsified();
+ LocationSummary* res = invoke->GetLocations();
+ if (res == nullptr) {
+ return false;
+ }
+ if (kEmitCompilerReadBarrier && res->CanCall()) {
+ // Generating an intrinsic for this HInvoke may produce an
+ // IntrinsicSlowPathX86_64 slow path. Currently this approach
+ // does not work when using read barriers, as the emitted
+ // calling sequence will make use of another slow path
+ // (ReadBarrierForRootSlowPathX86_64 for HInvokeStaticOrDirect,
+ // ReadBarrierSlowPathX86_64 for HInvokeVirtual). So we bail
+ // out in this case.
+ //
+ // TODO: Find a way to have intrinsics work with read barriers.
+ invoke->SetLocations(nullptr);
+ return false;
+ }
+ return res->Intrinsified();
}
static void MoveArguments(HInvoke* invoke, CodeGeneratorX86_64* codegen) {
@@ -917,6 +933,10 @@
CodeGenerator::CreateSystemArrayCopyLocationSummary(invoke);
}
+// TODO: Implement read barriers in the SystemArrayCopy intrinsic.
+// Note that this code path is not used (yet) because we do not
+// intrinsify methods that can go into the IntrinsicSlowPathX86_64
+// slow path.
void IntrinsicCodeGeneratorX86_64::VisitSystemArrayCopy(HInvoke* invoke) {
X86_64Assembler* assembler = GetAssembler();
LocationSummary* locations = invoke->GetLocations();
@@ -1698,23 +1718,30 @@
GetAssembler()->gs()->movl(out, Address::Absolute(Thread::PeerOffset<kX86_64WordSize>(), true));
}
-static void GenUnsafeGet(LocationSummary* locations, Primitive::Type type,
- bool is_volatile ATTRIBUTE_UNUSED, X86_64Assembler* assembler) {
- CpuRegister base = locations->InAt(1).AsRegister<CpuRegister>();
- CpuRegister offset = locations->InAt(2).AsRegister<CpuRegister>();
- CpuRegister trg = locations->Out().AsRegister<CpuRegister>();
+static void GenUnsafeGet(HInvoke* invoke,
+ Primitive::Type type,
+ bool is_volatile ATTRIBUTE_UNUSED,
+ CodeGeneratorX86_64* codegen) {
+ X86_64Assembler* assembler = down_cast<X86_64Assembler*>(codegen->GetAssembler());
+ LocationSummary* locations = invoke->GetLocations();
+ Location base_loc = locations->InAt(1);
+ CpuRegister base = base_loc.AsRegister<CpuRegister>();
+ Location offset_loc = locations->InAt(2);
+ CpuRegister offset = offset_loc.AsRegister<CpuRegister>();
+ Location output_loc = locations->Out();
+ CpuRegister output = locations->Out().AsRegister<CpuRegister>();
switch (type) {
case Primitive::kPrimInt:
case Primitive::kPrimNot:
- __ movl(trg, Address(base, offset, ScaleFactor::TIMES_1, 0));
+ __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0));
if (type == Primitive::kPrimNot) {
- __ MaybeUnpoisonHeapReference(trg);
+ codegen->MaybeGenerateReadBarrier(invoke, output_loc, output_loc, base_loc, 0U, offset_loc);
}
break;
case Primitive::kPrimLong:
- __ movq(trg, Address(base, offset, ScaleFactor::TIMES_1, 0));
+ __ movq(output, Address(base, offset, ScaleFactor::TIMES_1, 0));
break;
default:
@@ -1724,8 +1751,13 @@
}
static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
+ bool can_call = kEmitCompilerReadBarrier &&
+ (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject ||
+ invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile);
LocationSummary* locations = new (arena) LocationSummary(invoke,
- LocationSummary::kNoCall,
+ can_call ?
+ LocationSummary::kCallOnSlowPath :
+ LocationSummary::kNoCall,
kIntrinsified);
locations->SetInAt(0, Location::NoLocation()); // Unused receiver.
locations->SetInAt(1, Location::RequiresRegister());
@@ -1754,22 +1786,22 @@
void IntrinsicCodeGeneratorX86_64::VisitUnsafeGet(HInvoke* invoke) {
- GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimInt, false, GetAssembler());
+ GenUnsafeGet(invoke, Primitive::kPrimInt, false, codegen_);
}
void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetVolatile(HInvoke* invoke) {
- GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimInt, true, GetAssembler());
+ GenUnsafeGet(invoke, Primitive::kPrimInt, true, codegen_);
}
void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetLong(HInvoke* invoke) {
- GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimLong, false, GetAssembler());
+ GenUnsafeGet(invoke, Primitive::kPrimLong, false, codegen_);
}
void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
- GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimLong, true, GetAssembler());
+ GenUnsafeGet(invoke, Primitive::kPrimLong, true, codegen_);
}
void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetObject(HInvoke* invoke) {
- GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimNot, false, GetAssembler());
+ GenUnsafeGet(invoke, Primitive::kPrimNot, false, codegen_);
}
void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
- GenUnsafeGet(invoke->GetLocations(), Primitive::kPrimNot, true, GetAssembler());
+ GenUnsafeGet(invoke, Primitive::kPrimNot, true, codegen_);
}
@@ -1961,13 +1993,18 @@
__ LockCmpxchgl(Address(base, offset, TIMES_1, 0), CpuRegister(value_reg));
- // locked cmpxchg has full barrier semantics, and we don't need
+ // LOCK CMPXCHG has full barrier semantics, and we don't need
// scheduling barriers at this time.
// Convert ZF into the boolean result.
__ setcc(kZero, out);
__ movzxb(out, out);
+ // In the case of the `UnsafeCASObject` intrinsic, accessing an
+ // object in the heap with LOCK CMPXCHG does not require a read
+ // barrier, as we do not keep a reference to this heap location.
+ // However, if heap poisoning is enabled, we need to unpoison the
+ // values that were poisoned earlier.
if (kPoisonHeapReferences) {
if (base_equals_value) {
// `value_reg` has been moved to a temporary register, no need
@@ -1992,7 +2029,7 @@
LOG(FATAL) << "Unexpected CAS type " << type;
}
- // locked cmpxchg has full barrier semantics, and we don't need
+ // LOCK CMPXCHG has full barrier semantics, and we don't need
// scheduling barriers at this time.
// Convert ZF into the boolean result.
diff --git a/compiler/optimizing/load_store_elimination.cc b/compiler/optimizing/load_store_elimination.cc
index 6fbb682..5b89cfe 100644
--- a/compiler/optimizing/load_store_elimination.cc
+++ b/compiler/optimizing/load_store_elimination.cc
@@ -119,19 +119,10 @@
: ref_info_(ref_info),
offset_(offset),
index_(index),
- declaring_class_def_index_(declaring_class_def_index),
- may_become_unknown_(true) {
+ declaring_class_def_index_(declaring_class_def_index) {
DCHECK(ref_info != nullptr);
DCHECK((offset == kInvalidFieldOffset && index != nullptr) ||
(offset != kInvalidFieldOffset && index == nullptr));
-
- if (ref_info->IsSingletonAndNotReturned()) {
- // We try to track stores to singletons that aren't returned to eliminate the stores
- // since values in singleton's fields cannot be killed due to aliasing. Those values
- // can still be killed due to merging values since we don't build phi for merging heap
- // values. SetMayBecomeUnknown(true) may be called later once such merge becomes possible.
- may_become_unknown_ = false;
- }
}
ReferenceInfo* GetReferenceInfo() const { return ref_info_; }
@@ -148,21 +139,11 @@
return index_ != nullptr;
}
- // Returns true if this heap location's value may become unknown after it's
- // set to a value, due to merge of values, or killed due to aliasing.
- bool MayBecomeUnknown() const {
- return may_become_unknown_;
- }
- void SetMayBecomeUnknown(bool val) {
- may_become_unknown_ = val;
- }
-
private:
ReferenceInfo* const ref_info_; // reference for instance/static field or array access.
const size_t offset_; // offset of static/instance field.
HInstruction* const index_; // index of an array element.
const int16_t declaring_class_def_index_; // declaring class's def's dex index.
- bool may_become_unknown_; // value may become kUnknownHeapValue.
DISALLOW_COPY_AND_ASSIGN(HeapLocation);
};
@@ -381,26 +362,13 @@
return heap_locations_[heap_location_idx];
}
- void VisitFieldAccess(HInstruction* field_access,
- HInstruction* ref,
- const FieldInfo& field_info,
- bool is_store) {
+ void VisitFieldAccess(HInstruction* ref, const FieldInfo& field_info) {
if (field_info.IsVolatile()) {
has_volatile_ = true;
}
const uint16_t declaring_class_def_index = field_info.GetDeclaringClassDefIndex();
const size_t offset = field_info.GetFieldOffset().SizeValue();
- HeapLocation* location = GetOrCreateHeapLocation(ref, offset, nullptr, declaring_class_def_index);
- // A store of a value may be eliminated if all future loads for that value can be eliminated.
- // For a value that's stored into a singleton field, the value will not be killed due
- // to aliasing. However if the value is set in a block that doesn't post dominate the definition,
- // the value may be killed due to merging later. Before we have post dominating info, we check
- // if the store is in the same block as the definition just to be conservative.
- if (is_store &&
- location->GetReferenceInfo()->IsSingletonAndNotReturned() &&
- field_access->GetBlock() != ref->GetBlock()) {
- location->SetMayBecomeUnknown(true);
- }
+ GetOrCreateHeapLocation(ref, offset, nullptr, declaring_class_def_index);
}
void VisitArrayAccess(HInstruction* array, HInstruction* index) {
@@ -409,20 +377,20 @@
}
void VisitInstanceFieldGet(HInstanceFieldGet* instruction) OVERRIDE {
- VisitFieldAccess(instruction, instruction->InputAt(0), instruction->GetFieldInfo(), false);
+ VisitFieldAccess(instruction->InputAt(0), instruction->GetFieldInfo());
}
void VisitInstanceFieldSet(HInstanceFieldSet* instruction) OVERRIDE {
- VisitFieldAccess(instruction, instruction->InputAt(0), instruction->GetFieldInfo(), true);
+ VisitFieldAccess(instruction->InputAt(0), instruction->GetFieldInfo());
has_heap_stores_ = true;
}
void VisitStaticFieldGet(HStaticFieldGet* instruction) OVERRIDE {
- VisitFieldAccess(instruction, instruction->InputAt(0), instruction->GetFieldInfo(), false);
+ VisitFieldAccess(instruction->InputAt(0), instruction->GetFieldInfo());
}
void VisitStaticFieldSet(HStaticFieldSet* instruction) OVERRIDE {
- VisitFieldAccess(instruction, instruction->InputAt(0), instruction->GetFieldInfo(), true);
+ VisitFieldAccess(instruction->InputAt(0), instruction->GetFieldInfo());
has_heap_stores_ = true;
}
@@ -464,9 +432,14 @@
};
// An unknown heap value. Loads with such a value in the heap location cannot be eliminated.
+// A heap location can be set to kUnknownHeapValue when:
+// - initially set a value.
+// - killed due to aliasing, merging, invocation, or loop side effects.
static HInstruction* const kUnknownHeapValue =
reinterpret_cast<HInstruction*>(static_cast<uintptr_t>(-1));
+
// Default heap value after an allocation.
+// A heap location can be set to that value right after an allocation.
static HInstruction* const kDefaultHeapValue =
reinterpret_cast<HInstruction*>(static_cast<uintptr_t>(-2));
@@ -484,29 +457,17 @@
kUnknownHeapValue,
graph->GetArena()->Adapter(kArenaAllocLSE)),
graph->GetArena()->Adapter(kArenaAllocLSE)),
- removed_instructions_(graph->GetArena()->Adapter(kArenaAllocLSE)),
- substitute_instructions_(graph->GetArena()->Adapter(kArenaAllocLSE)),
+ removed_loads_(graph->GetArena()->Adapter(kArenaAllocLSE)),
+ substitute_instructions_for_loads_(graph->GetArena()->Adapter(kArenaAllocLSE)),
+ possibly_removed_stores_(graph->GetArena()->Adapter(kArenaAllocLSE)),
singleton_new_instances_(graph->GetArena()->Adapter(kArenaAllocLSE)) {
}
void VisitBasicBlock(HBasicBlock* block) OVERRIDE {
- int block_id = block->GetBlockId();
- ArenaVector<HInstruction*>& heap_values = heap_values_for_[block_id];
+ // Populate the heap_values array for this block.
// TODO: try to reuse the heap_values array from one predecessor if possible.
if (block->IsLoopHeader()) {
- // We do a single pass in reverse post order. For loops, use the side effects as a hint
- // to see if the heap values should be killed.
- if (side_effects_.GetLoopEffects(block).DoesAnyWrite()) {
- // Leave all values as kUnknownHeapValue.
- } else {
- // Inherit the values from pre-header.
- HBasicBlock* pre_header = block->GetLoopInformation()->GetPreHeader();
- ArenaVector<HInstruction*>& pre_header_heap_values =
- heap_values_for_[pre_header->GetBlockId()];
- for (size_t i = 0; i < heap_values.size(); i++) {
- heap_values[i] = pre_header_heap_values[i];
- }
- }
+ HandleLoopSideEffects(block);
} else {
MergePredecessorValues(block);
}
@@ -515,23 +476,34 @@
// Remove recorded instructions that should be eliminated.
void RemoveInstructions() {
- size_t size = removed_instructions_.size();
- DCHECK_EQ(size, substitute_instructions_.size());
+ size_t size = removed_loads_.size();
+ DCHECK_EQ(size, substitute_instructions_for_loads_.size());
for (size_t i = 0; i < size; i++) {
- HInstruction* instruction = removed_instructions_[i];
- DCHECK(instruction != nullptr);
- HInstruction* substitute = substitute_instructions_[i];
- if (substitute != nullptr) {
- // Keep tracing substitute till one that's not removed.
- HInstruction* sub_sub = FindSubstitute(substitute);
- while (sub_sub != substitute) {
- substitute = sub_sub;
- sub_sub = FindSubstitute(substitute);
- }
- instruction->ReplaceWith(substitute);
+ HInstruction* load = removed_loads_[i];
+ DCHECK(load != nullptr);
+ DCHECK(load->IsInstanceFieldGet() ||
+ load->IsStaticFieldGet() ||
+ load->IsArrayGet());
+ HInstruction* substitute = substitute_instructions_for_loads_[i];
+ DCHECK(substitute != nullptr);
+ // Keep tracing substitute till one that's not removed.
+ HInstruction* sub_sub = FindSubstitute(substitute);
+ while (sub_sub != substitute) {
+ substitute = sub_sub;
+ sub_sub = FindSubstitute(substitute);
}
- instruction->GetBlock()->RemoveInstruction(instruction);
+ load->ReplaceWith(substitute);
+ load->GetBlock()->RemoveInstruction(load);
}
+
+ // At this point, stores in possibly_removed_stores_ can be safely removed.
+ size = possibly_removed_stores_.size();
+ for (size_t i = 0; i < size; i++) {
+ HInstruction* store = possibly_removed_stores_[i];
+ DCHECK(store->IsInstanceFieldSet() || store->IsStaticFieldSet() || store->IsArraySet());
+ store->GetBlock()->RemoveInstruction(store);
+ }
+
// TODO: remove unnecessary allocations.
// Eliminate instructions in singleton_new_instances_ that:
// - don't have uses,
@@ -541,6 +513,52 @@
}
private:
+ // If heap_values[index] is an instance field store, need to keep the store.
+ // This is necessary if a heap value is killed due to merging, or loop side
+ // effects (which is essentially merging also), since a load later from the
+ // location won't be eliminated.
+ void KeepIfIsStore(HInstruction* heap_value) {
+ if (heap_value == kDefaultHeapValue ||
+ heap_value == kUnknownHeapValue ||
+ !heap_value->IsInstanceFieldSet()) {
+ return;
+ }
+ auto idx = std::find(possibly_removed_stores_.begin(),
+ possibly_removed_stores_.end(), heap_value);
+ if (idx != possibly_removed_stores_.end()) {
+ // Make sure the store is kept.
+ possibly_removed_stores_.erase(idx);
+ }
+ }
+
+ void HandleLoopSideEffects(HBasicBlock* block) {
+ DCHECK(block->IsLoopHeader());
+ int block_id = block->GetBlockId();
+ ArenaVector<HInstruction*>& heap_values = heap_values_for_[block_id];
+ HBasicBlock* pre_header = block->GetLoopInformation()->GetPreHeader();
+ ArenaVector<HInstruction*>& pre_header_heap_values =
+ heap_values_for_[pre_header->GetBlockId()];
+ // We do a single pass in reverse post order. For loops, use the side effects as a hint
+ // to see if the heap values should be killed.
+ if (side_effects_.GetLoopEffects(block).DoesAnyWrite()) {
+ for (size_t i = 0; i < pre_header_heap_values.size(); i++) {
+ // heap value is killed by loop side effects, need to keep the last store.
+ KeepIfIsStore(pre_header_heap_values[i]);
+ }
+ if (kIsDebugBuild) {
+ // heap_values should all be kUnknownHeapValue that it is inited with.
+ for (size_t i = 0; i < heap_values.size(); i++) {
+ DCHECK_EQ(heap_values[i], kUnknownHeapValue);
+ }
+ }
+ } else {
+ // Inherit the values from pre-header.
+ for (size_t i = 0; i < heap_values.size(); i++) {
+ heap_values[i] = pre_header_heap_values[i];
+ }
+ }
+ }
+
void MergePredecessorValues(HBasicBlock* block) {
const ArenaVector<HBasicBlock*>& predecessors = block->GetPredecessors();
if (predecessors.size() == 0) {
@@ -548,16 +566,25 @@
}
ArenaVector<HInstruction*>& heap_values = heap_values_for_[block->GetBlockId()];
for (size_t i = 0; i < heap_values.size(); i++) {
- HInstruction* value = heap_values_for_[predecessors[0]->GetBlockId()][i];
- if (value != kUnknownHeapValue) {
+ HInstruction* pred0_value = heap_values_for_[predecessors[0]->GetBlockId()][i];
+ heap_values[i] = pred0_value;
+ if (pred0_value != kUnknownHeapValue) {
for (size_t j = 1; j < predecessors.size(); j++) {
- if (heap_values_for_[predecessors[j]->GetBlockId()][i] != value) {
- value = kUnknownHeapValue;
+ HInstruction* pred_value = heap_values_for_[predecessors[j]->GetBlockId()][i];
+ if (pred_value != pred0_value) {
+ heap_values[i] = kUnknownHeapValue;
break;
}
}
}
- heap_values[i] = value;
+
+ if (heap_values[i] == kUnknownHeapValue) {
+ // Keep the last store in each predecessor since future loads cannot be eliminated.
+ for (size_t j = 0; j < predecessors.size(); j++) {
+ ArenaVector<HInstruction*>& pred_values = heap_values_for_[predecessors[j]->GetBlockId()];
+ KeepIfIsStore(pred_values[i]);
+ }
+ }
}
}
@@ -616,21 +643,30 @@
HInstruction* heap_value = heap_values[idx];
if (heap_value == kDefaultHeapValue) {
HInstruction* constant = GetDefaultValue(instruction->GetType());
- removed_instructions_.push_back(instruction);
- substitute_instructions_.push_back(constant);
+ removed_loads_.push_back(instruction);
+ substitute_instructions_for_loads_.push_back(constant);
heap_values[idx] = constant;
return;
}
+ if (heap_value != kUnknownHeapValue && heap_value->IsInstanceFieldSet()) {
+ HInstruction* store = heap_value;
+ // This load must be from a singleton since it's from the same field
+ // that a "removed" store puts the value. That store must be to a singleton's field.
+ DCHECK(ref_info->IsSingleton());
+ // Get the real heap value of the store.
+ heap_value = store->InputAt(1);
+ }
if ((heap_value != kUnknownHeapValue) &&
// Keep the load due to possible I/F, J/D array aliasing.
// See b/22538329 for details.
(heap_value->GetType() == instruction->GetType())) {
- removed_instructions_.push_back(instruction);
- substitute_instructions_.push_back(heap_value);
+ removed_loads_.push_back(instruction);
+ substitute_instructions_for_loads_.push_back(heap_value);
TryRemovingNullCheck(instruction);
return;
}
+ // Load isn't eliminated.
if (heap_value == kUnknownHeapValue) {
// Put the load as the value into the HeapLocation.
// This acts like GVN but with better aliasing analysis.
@@ -662,51 +698,63 @@
ArenaVector<HInstruction*>& heap_values =
heap_values_for_[instruction->GetBlock()->GetBlockId()];
HInstruction* heap_value = heap_values[idx];
- bool redundant_store = false;
+ bool same_value = false;
+ bool possibly_redundant = false;
if (Equal(heap_value, value)) {
// Store into the heap location with the same value.
- redundant_store = true;
+ same_value = true;
} else if (index != nullptr) {
// For array element, don't eliminate stores since it can be easily aliased
// with non-constant index.
} else if (!heap_location_collector_.MayDeoptimize() &&
- ref_info->IsSingletonAndNotReturned() &&
- !heap_location_collector_.GetHeapLocation(idx)->MayBecomeUnknown()) {
- // Store into a field of a singleton that's not returned. And that value cannot be
- // killed due to merge. It's redundant since future loads will get the value
- // set by this instruction.
- Primitive::Type type = Primitive::kPrimVoid;
- if (instruction->IsInstanceFieldSet()) {
- type = instruction->AsInstanceFieldSet()->GetFieldInfo().GetFieldType();
- } else if (instruction->IsStaticFieldSet()) {
- type = instruction->AsStaticFieldSet()->GetFieldInfo().GetFieldType();
+ ref_info->IsSingletonAndNotReturned()) {
+ // Store into a field of a singleton that's not returned. The value cannot be
+ // killed due to aliasing/invocation. It can be redundant since future loads can
+ // directly get the value set by this instruction. The value can still be killed due to
+ // merging or loop side effects. Stores whose values are killed due to merging/loop side
+ // effects later will be removed from possibly_removed_stores_ when that is detected.
+ possibly_redundant = true;
+ HNewInstance* new_instance = ref_info->GetReference()->AsNewInstance();
+ DCHECK(new_instance != nullptr);
+ if (new_instance->IsFinalizable()) {
+ // Finalizable objects escape globally. Need to keep the store.
+ possibly_redundant = false;
} else {
- DCHECK(false) << "Must be an instance/static field set instruction.";
+ HLoopInformation* loop_info = instruction->GetBlock()->GetLoopInformation();
+ if (loop_info != nullptr) {
+ // instruction is a store in the loop so the loop must does write.
+ DCHECK(side_effects_.GetLoopEffects(loop_info->GetHeader()).DoesAnyWrite());
+
+ if (loop_info->IsLoopInvariant(original_ref, false)) {
+ DCHECK(original_ref->GetBlock()->Dominates(loop_info->GetPreHeader()));
+ // Keep the store since its value may be needed at the loop header.
+ possibly_redundant = false;
+ } else {
+ // The singleton is created inside the loop. Value stored to it isn't needed at
+ // the loop header. This is true for outer loops also.
+ }
+ }
}
- if (value->GetType() != type) {
- // I/F, J/D aliasing should not happen for fields.
- DCHECK(Primitive::IsIntegralType(value->GetType()));
- DCHECK(!Primitive::Is64BitType(value->GetType()));
- DCHECK(Primitive::IsIntegralType(type));
- DCHECK(!Primitive::Is64BitType(type));
- // Keep the store since the corresponding load isn't eliminated due to different types.
- // TODO: handle the different int types so that we can eliminate this store.
- redundant_store = false;
- } else {
- redundant_store = true;
- }
- // TODO: eliminate the store if the singleton object is not finalizable.
- redundant_store = false;
}
- if (redundant_store) {
- removed_instructions_.push_back(instruction);
- substitute_instructions_.push_back(nullptr);
- TryRemovingNullCheck(instruction);
+ if (same_value || possibly_redundant) {
+ possibly_removed_stores_.push_back(instruction);
}
- heap_values[idx] = value;
+ if (!same_value) {
+ if (possibly_redundant) {
+ DCHECK(instruction->IsInstanceFieldSet());
+ // Put the store as the heap value. If the value is loaded from heap
+ // by a load later, this store isn't really redundant.
+ heap_values[idx] = instruction;
+ } else {
+ heap_values[idx] = value;
+ }
+ }
// This store may kill values in other heap locations due to aliasing.
for (size_t i = 0; i < heap_values.size(); i++) {
+ if (i == idx) {
+ continue;
+ }
if (heap_values[i] == value) {
// Same value should be kept even if aliasing happens.
continue;
@@ -834,9 +882,10 @@
return;
}
if (!heap_location_collector_.MayDeoptimize() &&
- ref_info->IsSingletonAndNotReturned()) {
- // The allocation might be eliminated.
- singleton_new_instances_.push_back(new_instance);
+ ref_info->IsSingletonAndNotReturned() &&
+ !new_instance->IsFinalizable() &&
+ !new_instance->CanThrow()) {
+ // TODO: add new_instance to singleton_new_instances_ and enable allocation elimination.
}
ArenaVector<HInstruction*>& heap_values =
heap_values_for_[new_instance->GetBlock()->GetBlockId()];
@@ -854,10 +903,10 @@
// Find an instruction's substitute if it should be removed.
// Return the same instruction if it should not be removed.
HInstruction* FindSubstitute(HInstruction* instruction) {
- size_t size = removed_instructions_.size();
+ size_t size = removed_loads_.size();
for (size_t i = 0; i < size; i++) {
- if (removed_instructions_[i] == instruction) {
- return substitute_instructions_[i];
+ if (removed_loads_[i] == instruction) {
+ return substitute_instructions_for_loads_[i];
}
}
return instruction;
@@ -871,8 +920,13 @@
// We record the instructions that should be eliminated but may be
// used by heap locations. They'll be removed in the end.
- ArenaVector<HInstruction*> removed_instructions_;
- ArenaVector<HInstruction*> substitute_instructions_;
+ ArenaVector<HInstruction*> removed_loads_;
+ ArenaVector<HInstruction*> substitute_instructions_for_loads_;
+
+ // Stores in this list may be removed from the list later when it's
+ // found that the store cannot be eliminated.
+ ArenaVector<HInstruction*> possibly_removed_stores_;
+
ArenaVector<HInstruction*> singleton_new_instances_;
DISALLOW_COPY_AND_ASSIGN(LSEVisitor);
diff --git a/compiler/optimizing/locations.h b/compiler/optimizing/locations.h
index 1181007..63bbc2c 100644
--- a/compiler/optimizing/locations.h
+++ b/compiler/optimizing/locations.h
@@ -594,6 +594,10 @@
return intrinsified_;
}
+ void SetIntrinsified(bool intrinsified) {
+ intrinsified_ = intrinsified;
+ }
+
private:
ArenaVector<Location> inputs_;
ArenaVector<Location> temps_;
@@ -613,7 +617,7 @@
RegisterSet live_registers_;
// Whether these are locations for an intrinsified call.
- const bool intrinsified_;
+ bool intrinsified_;
ART_FRIEND_TEST(RegisterAllocatorTest, ExpectedInRegisterHint);
ART_FRIEND_TEST(RegisterAllocatorTest, SameAsFirstInputHint);
diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc
index 73a44ee..0a39ff3 100644
--- a/compiler/optimizing/nodes.cc
+++ b/compiler/optimizing/nodes.cc
@@ -2068,6 +2068,19 @@
}
}
+std::ostream& operator<<(std::ostream& os, HInvokeStaticOrDirect::ClinitCheckRequirement rhs) {
+ switch (rhs) {
+ case HInvokeStaticOrDirect::ClinitCheckRequirement::kExplicit:
+ return os << "explicit";
+ case HInvokeStaticOrDirect::ClinitCheckRequirement::kImplicit:
+ return os << "implicit";
+ case HInvokeStaticOrDirect::ClinitCheckRequirement::kNone:
+ return os << "none";
+ default:
+ return os << "unknown:" << static_cast<int>(rhs);
+ }
+}
+
void HInstruction::RemoveEnvironmentUsers() {
for (HUseIterator<HEnvironment*> use_it(GetEnvUses()); !use_it.Done(); use_it.Advance()) {
HUseListNode<HEnvironment*>* user_node = use_it.Current();
diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h
index 2878ac9..4f894b0 100644
--- a/compiler/optimizing/nodes.h
+++ b/compiler/optimizing/nodes.h
@@ -1436,7 +1436,7 @@
return flags_ == (kAllChangeBits | kAllDependOnBits);
}
- // Returns true if this may read something written by other.
+ // Returns true if `this` may read something written by `other`.
bool MayDependOn(SideEffects other) const {
const uint64_t depends_on_flags = (flags_ & kAllDependOnBits) >> kChangeBits;
return (other.flags_ & depends_on_flags);
@@ -3434,14 +3434,19 @@
DCHECK(had_current_method_input || !needs_current_method_input);
if (had_current_method_input && !needs_current_method_input) {
- DCHECK_EQ(InputAt(GetCurrentMethodInputIndex()), GetBlock()->GetGraph()->GetCurrentMethod());
- RemoveInputAt(GetCurrentMethodInputIndex());
+ DCHECK_EQ(InputAt(GetSpecialInputIndex()), GetBlock()->GetGraph()->GetCurrentMethod());
+ RemoveInputAt(GetSpecialInputIndex());
}
dispatch_info_ = dispatch_info;
}
- void InsertInputAt(size_t index, HInstruction* input);
- void RemoveInputAt(size_t index);
+ void AddSpecialInput(HInstruction* input) {
+ // We allow only one special input.
+ DCHECK(!IsStringInit() && !HasCurrentMethodInput());
+ DCHECK(InputCount() == GetSpecialInputIndex() ||
+ (InputCount() == GetSpecialInputIndex() + 1 && IsStaticWithExplicitClinitCheck()));
+ InsertInputAt(GetSpecialInputIndex(), input);
+ }
bool CanDoImplicitNullCheckOn(HInstruction* obj ATTRIBUTE_UNUSED) const OVERRIDE {
// We access the method via the dex cache so we can't do an implicit null check.
@@ -3453,13 +3458,20 @@
return return_type_ == Primitive::kPrimNot && !IsStringInit();
}
+ // Get the index of the special input, if any.
+ //
+ // If the invoke IsStringInit(), it initially has a HFakeString special argument
+ // which is removed by the instruction simplifier; if the invoke HasCurrentMethodInput(),
+ // the "special input" is the current method pointer; otherwise there may be one
+ // platform-specific special input, such as PC-relative addressing base.
+ uint32_t GetSpecialInputIndex() const { return GetNumberOfArguments(); }
+
InvokeType GetInvokeType() const { return invoke_type_; }
MethodLoadKind GetMethodLoadKind() const { return dispatch_info_.method_load_kind; }
CodePtrLocation GetCodePtrLocation() const { return dispatch_info_.code_ptr_location; }
bool IsRecursive() const { return GetMethodLoadKind() == MethodLoadKind::kRecursive; }
bool NeedsDexCacheOfDeclaringClass() const OVERRIDE;
bool IsStringInit() const { return GetMethodLoadKind() == MethodLoadKind::kStringInit; }
- uint32_t GetCurrentMethodInputIndex() const { return GetNumberOfArguments(); }
bool HasMethodAddress() const { return GetMethodLoadKind() == MethodLoadKind::kDirectAddress; }
bool HasPcRelativeDexCache() const {
return GetMethodLoadKind() == MethodLoadKind::kDexCachePcRelative;
@@ -3467,11 +3479,11 @@
bool HasCurrentMethodInput() const {
// This function can be called only after the invoke has been fully initialized by the builder.
if (NeedsCurrentMethodInput(GetMethodLoadKind())) {
- DCHECK(InputAt(GetCurrentMethodInputIndex())->IsCurrentMethod());
+ DCHECK(InputAt(GetSpecialInputIndex())->IsCurrentMethod());
return true;
} else {
- DCHECK(InputCount() == GetCurrentMethodInputIndex() ||
- !InputAt(GetCurrentMethodInputIndex())->IsCurrentMethod());
+ DCHECK(InputCount() == GetSpecialInputIndex() ||
+ !InputAt(GetSpecialInputIndex())->IsCurrentMethod());
return false;
}
}
@@ -3505,20 +3517,19 @@
return GetInvokeType() == kStatic;
}
- // Remove the art::HLoadClass instruction set as last input by
- // art::PrepareForRegisterAllocation::VisitClinitCheck in lieu of
- // the initial art::HClinitCheck instruction (only relevant for
- // static calls with explicit clinit check).
- void RemoveLoadClassAsLastInput() {
+ // Remove the HClinitCheck or the replacement HLoadClass (set as last input by
+ // PrepareForRegisterAllocation::VisitClinitCheck() in lieu of the initial HClinitCheck)
+ // instruction; only relevant for static calls with explicit clinit check.
+ void RemoveExplicitClinitCheck(ClinitCheckRequirement new_requirement) {
DCHECK(IsStaticWithExplicitClinitCheck());
size_t last_input_index = InputCount() - 1;
HInstruction* last_input = InputAt(last_input_index);
DCHECK(last_input != nullptr);
- DCHECK(last_input->IsLoadClass()) << last_input->DebugName();
+ DCHECK(last_input->IsLoadClass() || last_input->IsClinitCheck()) << last_input->DebugName();
RemoveAsUserOfInput(last_input_index);
inputs_.pop_back();
- clinit_check_requirement_ = ClinitCheckRequirement::kImplicit;
- DCHECK(IsStaticWithImplicitClinitCheck());
+ clinit_check_requirement_ = new_requirement;
+ DCHECK(!IsStaticWithExplicitClinitCheck());
}
bool IsStringFactoryFor(HFakeString* str) const {
@@ -3539,7 +3550,7 @@
}
// Is this a call to a static method whose declaring class has an
- // explicit intialization check in the graph?
+ // explicit initialization check in the graph?
bool IsStaticWithExplicitClinitCheck() const {
return IsStatic() && (clinit_check_requirement_ == ClinitCheckRequirement::kExplicit);
}
@@ -3572,6 +3583,9 @@
return input_record;
}
+ void InsertInputAt(size_t index, HInstruction* input);
+ void RemoveInputAt(size_t index);
+
private:
const InvokeType invoke_type_;
ClinitCheckRequirement clinit_check_requirement_;
@@ -3583,6 +3597,7 @@
DISALLOW_COPY_AND_ASSIGN(HInvokeStaticOrDirect);
};
+std::ostream& operator<<(std::ostream& os, HInvokeStaticOrDirect::ClinitCheckRequirement rhs);
class HInvokeVirtual : public HInvoke {
public:
@@ -3643,10 +3658,14 @@
uint32_t dex_pc,
uint16_t type_index,
const DexFile& dex_file,
+ bool can_throw,
+ bool finalizable,
QuickEntrypointEnum entrypoint)
: HExpression(Primitive::kPrimNot, SideEffects::CanTriggerGC(), dex_pc),
type_index_(type_index),
dex_file_(dex_file),
+ can_throw_(can_throw),
+ finalizable_(finalizable),
entrypoint_(entrypoint) {
SetRawInputAt(0, current_method);
}
@@ -3656,11 +3675,13 @@
// Calls runtime so needs an environment.
bool NeedsEnvironment() const OVERRIDE { return true; }
- // It may throw when called on:
- // - interfaces
- // - abstract/innaccessible/unknown classes
- // TODO: optimize when possible.
- bool CanThrow() const OVERRIDE { return true; }
+
+ // It may throw when called on type that's not instantiable/accessible.
+ // It can throw OOME.
+ // TODO: distinguish between the two cases so we can for example allow allocation elimination.
+ bool CanThrow() const OVERRIDE { return can_throw_ || true; }
+
+ bool IsFinalizable() const { return finalizable_; }
bool CanBeNull() const OVERRIDE { return false; }
@@ -3671,6 +3692,8 @@
private:
const uint16_t type_index_;
const DexFile& dex_file_;
+ const bool can_throw_;
+ const bool finalizable_;
const QuickEntrypointEnum entrypoint_;
DISALLOW_COPY_AND_ASSIGN(HNewInstance);
diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc
index 2be0680..2204921 100644
--- a/compiler/optimizing/optimizing_compiler.cc
+++ b/compiler/optimizing/optimizing_compiler.cc
@@ -383,6 +383,14 @@
|| instruction_set == kX86_64;
}
+// Read barrier are supported only on ARM, x86 and x86-64 at the moment.
+// TODO: Add support for other architectures and remove this function
+static bool InstructionSetSupportsReadBarrier(InstructionSet instruction_set) {
+ return instruction_set == kThumb2
+ || instruction_set == kX86
+ || instruction_set == kX86_64;
+}
+
static void RunOptimizations(HOptimization* optimizations[],
size_t length,
PassObserver* pass_observer) {
@@ -661,8 +669,8 @@
CompilerDriver* compiler_driver = GetCompilerDriver();
InstructionSet instruction_set = compiler_driver->GetInstructionSet();
- // Always use the thumb2 assembler: some runtime functionality (like implicit stack
- // overflow checks) assume thumb2.
+ // Always use the Thumb-2 assembler: some runtime functionality
+ // (like implicit stack overflow checks) assume Thumb-2.
if (instruction_set == kArm) {
instruction_set = kThumb2;
}
@@ -673,6 +681,12 @@
return nullptr;
}
+ // When read barriers are enabled, do not attempt to compile for
+ // instruction sets that have no read barrier support.
+ if (kEmitCompilerReadBarrier && !InstructionSetSupportsReadBarrier(instruction_set)) {
+ return nullptr;
+ }
+
if (Compiler::IsPathologicalCase(*code_item, method_idx, dex_file)) {
MaybeRecordStat(MethodCompilationStat::kNotCompiledPathological);
return nullptr;
@@ -841,9 +855,14 @@
if (kIsDebugBuild &&
IsCompilingWithCoreImage() &&
- IsInstructionSetSupported(compiler_driver->GetInstructionSet())) {
- // For testing purposes, we put a special marker on method names that should be compiled
- // with this compiler. This makes sure we're not regressing.
+ IsInstructionSetSupported(compiler_driver->GetInstructionSet()) &&
+ (!kEmitCompilerReadBarrier ||
+ InstructionSetSupportsReadBarrier(compiler_driver->GetInstructionSet()))) {
+ // For testing purposes, we put a special marker on method names
+ // that should be compiled with this compiler (when the the
+ // instruction set is supported -- and has support for read
+ // barriers, if they are enabled). This makes sure we're not
+ // regressing.
std::string method_name = PrettyMethod(method_idx, dex_file);
bool shouldCompile = method_name.find("$opt$") != std::string::npos;
DCHECK((method != nullptr) || !shouldCompile) << "Didn't compile " << method_name;
diff --git a/compiler/optimizing/pc_relative_fixups_x86.cc b/compiler/optimizing/pc_relative_fixups_x86.cc
index c2894c7..808a1dc 100644
--- a/compiler/optimizing/pc_relative_fixups_x86.cc
+++ b/compiler/optimizing/pc_relative_fixups_x86.cc
@@ -113,9 +113,8 @@
if (invoke_static_or_direct != nullptr && invoke_static_or_direct->HasPcRelativeDexCache()) {
InitializePCRelativeBasePointer(invoke);
// Add the extra parameter base_.
- uint32_t index = invoke_static_or_direct->GetCurrentMethodInputIndex();
DCHECK(!invoke_static_or_direct->HasCurrentMethodInput());
- invoke_static_or_direct->InsertInputAt(index, base_);
+ invoke_static_or_direct->AddSpecialInput(base_);
}
// Ensure that we can load FP arguments from the constant area.
for (size_t i = 0, e = invoke->InputCount(); i < e; i++) {
diff --git a/compiler/optimizing/prepare_for_register_allocation.cc b/compiler/optimizing/prepare_for_register_allocation.cc
index ca928ae..f3d075c 100644
--- a/compiler/optimizing/prepare_for_register_allocation.cc
+++ b/compiler/optimizing/prepare_for_register_allocation.cc
@@ -48,12 +48,46 @@
}
void PrepareForRegisterAllocation::VisitClinitCheck(HClinitCheck* check) {
- HLoadClass* cls = check->GetLoadClass();
- check->ReplaceWith(cls);
- if (check->GetPrevious() == cls) {
+ // Try to find a static invoke from which this check originated.
+ HInvokeStaticOrDirect* invoke = nullptr;
+ for (HUseIterator<HInstruction*> it(check->GetUses()); !it.Done(); it.Advance()) {
+ HInstruction* user = it.Current()->GetUser();
+ if (user->IsInvokeStaticOrDirect() && CanMoveClinitCheck(check, user)) {
+ invoke = user->AsInvokeStaticOrDirect();
+ DCHECK(invoke->IsStaticWithExplicitClinitCheck());
+ invoke->RemoveExplicitClinitCheck(HInvokeStaticOrDirect::ClinitCheckRequirement::kImplicit);
+ break;
+ }
+ }
+ // If we found a static invoke for merging, remove the check from all other static invokes.
+ if (invoke != nullptr) {
+ for (HUseIterator<HInstruction*> it(check->GetUses()); !it.Done(); ) {
+ HInstruction* user = it.Current()->GetUser();
+ DCHECK(invoke->StrictlyDominates(user)); // All other uses must be dominated.
+ it.Advance(); // Advance before we remove the node, reference to the next node is preserved.
+ if (user->IsInvokeStaticOrDirect()) {
+ user->AsInvokeStaticOrDirect()->RemoveExplicitClinitCheck(
+ HInvokeStaticOrDirect::ClinitCheckRequirement::kNone);
+ }
+ }
+ }
+
+ HLoadClass* load_class = check->GetLoadClass();
+ bool can_merge_with_load_class = CanMoveClinitCheck(load_class, check);
+
+ check->ReplaceWith(load_class);
+
+ if (invoke != nullptr) {
+ // Remove the check from the graph. It has been merged into the invoke.
+ check->GetBlock()->RemoveInstruction(check);
+ // Check if we can merge the load class as well.
+ if (can_merge_with_load_class && !load_class->HasUses()) {
+ load_class->GetBlock()->RemoveInstruction(load_class);
+ }
+ } else if (can_merge_with_load_class) {
// Pass the initialization duty to the `HLoadClass` instruction,
// and remove the instruction from the graph.
- cls->SetMustGenerateClinitCheck(true);
+ load_class->SetMustGenerateClinitCheck(true);
check->GetBlock()->RemoveInstruction(check);
}
}
@@ -86,30 +120,60 @@
DCHECK(last_input != nullptr)
<< "Last input is not HLoadClass. It is " << last_input->DebugName();
- // Remove a load class instruction as last input of a static
- // invoke, which has been added (along with a clinit check,
- // removed by PrepareForRegisterAllocation::VisitClinitCheck
- // previously) by the graph builder during the creation of the
- // static invoke instruction, but is no longer required at this
- // stage (i.e., after inlining has been performed).
- invoke->RemoveLoadClassAsLastInput();
+ // Detach the explicit class initialization check from the invoke.
+ // Keeping track of the initializing instruction is no longer required
+ // at this stage (i.e., after inlining has been performed).
+ invoke->RemoveExplicitClinitCheck(HInvokeStaticOrDirect::ClinitCheckRequirement::kNone);
- // The static call will initialize the class so there's no need for a clinit check if
- // it's the first user.
- // There is one special case where we still need the clinit check, when inlining. Because
- // currently the callee is responsible for reporting parameters to the GC, the code
- // that walks the stack during `artQuickResolutionTrampoline` cannot be interrupted for GC.
- // Therefore we cannot allocate any object in that code, including loading a new class.
- if (last_input == invoke->GetPrevious() && !invoke->IsFromInlinedInvoke()) {
- last_input->SetMustGenerateClinitCheck(false);
+ // Merging with load class should have happened in VisitClinitCheck().
+ DCHECK(!CanMoveClinitCheck(last_input, invoke));
+ }
+}
- // If the load class instruction is no longer used, remove it from
- // the graph.
- if (!last_input->HasUses()) {
- last_input->GetBlock()->RemoveInstruction(last_input);
- }
+bool PrepareForRegisterAllocation::CanMoveClinitCheck(HInstruction* input, HInstruction* user) {
+ // Determine if input and user come from the same dex instruction, so that we can move
+ // the clinit check responsibility from one to the other, i.e. from HClinitCheck (user)
+ // to HLoadClass (input), or from HClinitCheck (input) to HInvokeStaticOrDirect (user).
+
+ // Start with a quick dex pc check.
+ if (user->GetDexPc() != input->GetDexPc()) {
+ return false;
+ }
+
+ // Now do a thorough environment check that this is really coming from the same instruction in
+ // the same inlined graph. Unfortunately, we have to go through the whole environment chain.
+ HEnvironment* user_environment = user->GetEnvironment();
+ HEnvironment* input_environment = input->GetEnvironment();
+ while (user_environment != nullptr || input_environment != nullptr) {
+ if (user_environment == nullptr || input_environment == nullptr) {
+ // Different environment chain length. This happens when a method is called
+ // once directly and once indirectly through another inlined method.
+ return false;
+ }
+ if (user_environment->GetDexPc() != input_environment->GetDexPc() ||
+ user_environment->GetMethodIdx() != input_environment->GetMethodIdx() ||
+ !IsSameDexFile(user_environment->GetDexFile(), input_environment->GetDexFile())) {
+ return false;
+ }
+ user_environment = user_environment->GetParent();
+ input_environment = input_environment->GetParent();
+ }
+
+ // Check for code motion taking the input to a different block.
+ if (user->GetBlock() != input->GetBlock()) {
+ return false;
+ }
+
+ // In debug mode, check that we have not inserted a throwing instruction
+ // or an instruction with side effects between input and user.
+ if (kIsDebugBuild) {
+ for (HInstruction* between = input->GetNext(); between != user; between = between->GetNext()) {
+ CHECK(between != nullptr); // User must be after input in the same block.
+ CHECK(!between->CanThrow());
+ CHECK(!between->HasSideEffects());
}
}
+ return true;
}
} // namespace art
diff --git a/compiler/optimizing/prepare_for_register_allocation.h b/compiler/optimizing/prepare_for_register_allocation.h
index d7f277f..a70fb30 100644
--- a/compiler/optimizing/prepare_for_register_allocation.h
+++ b/compiler/optimizing/prepare_for_register_allocation.h
@@ -41,6 +41,8 @@
void VisitCondition(HCondition* condition) OVERRIDE;
void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE;
+ bool CanMoveClinitCheck(HInstruction* input, HInstruction* user);
+
DISALLOW_COPY_AND_ASSIGN(PrepareForRegisterAllocation);
};
diff --git a/compiler/optimizing/reference_type_propagation.cc b/compiler/optimizing/reference_type_propagation.cc
index ecc085b..0d05c49 100644
--- a/compiler/optimizing/reference_type_propagation.cc
+++ b/compiler/optimizing/reference_type_propagation.cc
@@ -614,23 +614,36 @@
}
bool is_exact = a.IsExact() && b.IsExact();
- Handle<mirror::Class> type_handle;
+ ReferenceTypeInfo::TypeHandle result_type_handle;
+ ReferenceTypeInfo::TypeHandle a_type_handle = a.GetTypeHandle();
+ ReferenceTypeInfo::TypeHandle b_type_handle = b.GetTypeHandle();
+ bool a_is_interface = a_type_handle->IsInterface();
+ bool b_is_interface = b_type_handle->IsInterface();
if (a.GetTypeHandle().Get() == b.GetTypeHandle().Get()) {
- type_handle = a.GetTypeHandle();
+ result_type_handle = a_type_handle;
} else if (a.IsSupertypeOf(b)) {
- type_handle = a.GetTypeHandle();
+ result_type_handle = a_type_handle;
is_exact = false;
} else if (b.IsSupertypeOf(a)) {
- type_handle = b.GetTypeHandle();
+ result_type_handle = b_type_handle;
+ is_exact = false;
+ } else if (!a_is_interface && !b_is_interface) {
+ result_type_handle = handles_->NewHandle(a_type_handle->GetCommonSuperClass(b_type_handle));
is_exact = false;
} else {
- // TODO: Find the first common super class.
- type_handle = object_class_handle_;
+ // This can happen if:
+ // - both types are interfaces. TODO(calin): implement
+ // - one is an interface, the other a class, and the type does not implement the interface
+ // e.g:
+ // void foo(Interface i, boolean cond) {
+ // Object o = cond ? i : new Object();
+ // }
+ result_type_handle = object_class_handle_;
is_exact = false;
}
- return ReferenceTypeInfo::Create(type_handle, is_exact);
+ return ReferenceTypeInfo::Create(result_type_handle, is_exact);
}
static void UpdateArrayGet(HArrayGet* instr,
@@ -719,14 +732,35 @@
instr->SetReferenceTypeInfo(new_rti);
}
+// NullConstant inputs are ignored during merging as they do not provide any useful information.
+// If all the inputs are NullConstants then the type of the phi will be set to Object.
void ReferenceTypePropagation::UpdatePhi(HPhi* instr) {
- ReferenceTypeInfo new_rti = instr->InputAt(0)->GetReferenceTypeInfo();
+ size_t input_count = instr->InputCount();
+ size_t first_input_index_not_null = 0;
+ while (first_input_index_not_null < input_count &&
+ instr->InputAt(first_input_index_not_null)->IsNullConstant()) {
+ first_input_index_not_null++;
+ }
+ if (first_input_index_not_null == input_count) {
+ // All inputs are NullConstants, set the type to object.
+ // This may happen in the presence of inlining.
+ instr->SetReferenceTypeInfo(
+ ReferenceTypeInfo::Create(object_class_handle_, /* is_exact */ false));
+ return;
+ }
+
+ ReferenceTypeInfo new_rti = instr->InputAt(first_input_index_not_null)->GetReferenceTypeInfo();
+
if (new_rti.IsValid() && new_rti.IsObjectClass() && !new_rti.IsExact()) {
// Early return if we are Object and inexact.
instr->SetReferenceTypeInfo(new_rti);
return;
}
- for (size_t i = 1; i < instr->InputCount(); i++) {
+
+ for (size_t i = first_input_index_not_null + 1; i < input_count; i++) {
+ if (instr->InputAt(i)->IsNullConstant()) {
+ continue;
+ }
new_rti = MergeTypes(new_rti, instr->InputAt(i)->GetReferenceTypeInfo());
if (new_rti.IsValid() && new_rti.IsObjectClass()) {
if (!new_rti.IsExact()) {
diff --git a/compiler/optimizing/side_effects_test.cc b/compiler/optimizing/side_effects_test.cc
index ec45d6b..9bbc354 100644
--- a/compiler/optimizing/side_effects_test.cc
+++ b/compiler/optimizing/side_effects_test.cc
@@ -129,13 +129,13 @@
TEST(SideEffectsTest, VolatileDependences) {
SideEffects volatile_write =
- SideEffects::FieldWriteOfType(Primitive::kPrimInt, true);
+ SideEffects::FieldWriteOfType(Primitive::kPrimInt, /* is_volatile */ true);
SideEffects any_write =
- SideEffects::FieldWriteOfType(Primitive::kPrimInt, false);
+ SideEffects::FieldWriteOfType(Primitive::kPrimInt, /* is_volatile */ false);
SideEffects volatile_read =
- SideEffects::FieldReadOfType(Primitive::kPrimByte, true);
+ SideEffects::FieldReadOfType(Primitive::kPrimByte, /* is_volatile */ true);
SideEffects any_read =
- SideEffects::FieldReadOfType(Primitive::kPrimByte, false);
+ SideEffects::FieldReadOfType(Primitive::kPrimByte, /* is_volatile */ false);
EXPECT_FALSE(volatile_write.MayDependOn(any_read));
EXPECT_TRUE(any_read.MayDependOn(volatile_write));
@@ -151,15 +151,15 @@
TEST(SideEffectsTest, SameWidthTypes) {
// Type I/F.
testWriteAndReadDependence(
- SideEffects::FieldWriteOfType(Primitive::kPrimInt, false),
- SideEffects::FieldReadOfType(Primitive::kPrimFloat, false));
+ SideEffects::FieldWriteOfType(Primitive::kPrimInt, /* is_volatile */ false),
+ SideEffects::FieldReadOfType(Primitive::kPrimFloat, /* is_volatile */ false));
testWriteAndReadDependence(
SideEffects::ArrayWriteOfType(Primitive::kPrimInt),
SideEffects::ArrayReadOfType(Primitive::kPrimFloat));
// Type L/D.
testWriteAndReadDependence(
- SideEffects::FieldWriteOfType(Primitive::kPrimLong, false),
- SideEffects::FieldReadOfType(Primitive::kPrimDouble, false));
+ SideEffects::FieldWriteOfType(Primitive::kPrimLong, /* is_volatile */ false),
+ SideEffects::FieldReadOfType(Primitive::kPrimDouble, /* is_volatile */ false));
testWriteAndReadDependence(
SideEffects::ArrayWriteOfType(Primitive::kPrimLong),
SideEffects::ArrayReadOfType(Primitive::kPrimDouble));
@@ -171,9 +171,9 @@
for (Primitive::Type type = Primitive::kPrimNot;
type < Primitive::kPrimVoid;
type = Primitive::Type(type + 1)) {
- s = s.Union(SideEffects::FieldWriteOfType(type, false));
+ s = s.Union(SideEffects::FieldWriteOfType(type, /* is_volatile */ false));
s = s.Union(SideEffects::ArrayWriteOfType(type));
- s = s.Union(SideEffects::FieldReadOfType(type, false));
+ s = s.Union(SideEffects::FieldReadOfType(type, /* is_volatile */ false));
s = s.Union(SideEffects::ArrayReadOfType(type));
}
EXPECT_TRUE(s.DoesAllReadWrite());
@@ -225,10 +225,10 @@
"||DJ|||||", // note: DJ alias
SideEffects::ArrayReadOfType(Primitive::kPrimDouble).ToString().c_str());
SideEffects s = SideEffects::None();
- s = s.Union(SideEffects::FieldWriteOfType(Primitive::kPrimChar, false));
- s = s.Union(SideEffects::FieldWriteOfType(Primitive::kPrimLong, false));
+ s = s.Union(SideEffects::FieldWriteOfType(Primitive::kPrimChar, /* is_volatile */ false));
+ s = s.Union(SideEffects::FieldWriteOfType(Primitive::kPrimLong, /* is_volatile */ false));
s = s.Union(SideEffects::ArrayWriteOfType(Primitive::kPrimShort));
- s = s.Union(SideEffects::FieldReadOfType(Primitive::kPrimInt, false));
+ s = s.Union(SideEffects::FieldReadOfType(Primitive::kPrimInt, /* is_volatile */ false));
s = s.Union(SideEffects::ArrayReadOfType(Primitive::kPrimFloat));
s = s.Union(SideEffects::ArrayReadOfType(Primitive::kPrimDouble));
EXPECT_STREQ(
diff --git a/compiler/utils/mips/assembler_mips.cc b/compiler/utils/mips/assembler_mips.cc
index aee6412..fc7ac70 100644
--- a/compiler/utils/mips/assembler_mips.cc
+++ b/compiler/utils/mips/assembler_mips.cc
@@ -310,15 +310,27 @@
EmitR(0x1f, static_cast<Register>(0), rt, rd, 0x18, 0x20);
}
+void MipsAssembler::Wsbh(Register rd, Register rt) {
+ EmitR(0x1f, static_cast<Register>(0), rt, rd, 2, 0x20);
+}
+
void MipsAssembler::Sll(Register rd, Register rt, int shamt) {
+ CHECK(IsUint<5>(shamt)) << shamt;
EmitR(0, static_cast<Register>(0), rt, rd, shamt, 0x00);
}
void MipsAssembler::Srl(Register rd, Register rt, int shamt) {
+ CHECK(IsUint<5>(shamt)) << shamt;
EmitR(0, static_cast<Register>(0), rt, rd, shamt, 0x02);
}
+void MipsAssembler::Rotr(Register rd, Register rt, int shamt) {
+ CHECK(IsUint<5>(shamt)) << shamt;
+ EmitR(0, static_cast<Register>(1), rt, rd, shamt, 0x02);
+}
+
void MipsAssembler::Sra(Register rd, Register rt, int shamt) {
+ CHECK(IsUint<5>(shamt)) << shamt;
EmitR(0, static_cast<Register>(0), rt, rd, shamt, 0x03);
}
diff --git a/compiler/utils/mips/assembler_mips.h b/compiler/utils/mips/assembler_mips.h
index 4038c1f..1ef0992 100644
--- a/compiler/utils/mips/assembler_mips.h
+++ b/compiler/utils/mips/assembler_mips.h
@@ -135,9 +135,11 @@
void Seb(Register rd, Register rt); // R2+
void Seh(Register rd, Register rt); // R2+
+ void Wsbh(Register rd, Register rt); // R2+
void Sll(Register rd, Register rt, int shamt);
void Srl(Register rd, Register rt, int shamt);
+ void Rotr(Register rd, Register rt, int shamt); // R2+
void Sra(Register rd, Register rt, int shamt);
void Sllv(Register rd, Register rt, Register rs);
void Srlv(Register rd, Register rt, Register rs);
diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc
index 68cf6d9..89c2a7c 100644
--- a/dex2oat/dex2oat.cc
+++ b/dex2oat/dex2oat.cc
@@ -1231,6 +1231,7 @@
// Handle and ClassLoader creation needs to come after Runtime::Create
jobject class_loader = nullptr;
+ jobject class_path_class_loader = nullptr;
Thread* self = Thread::Current();
if (!boot_image_option_.empty()) {
@@ -1248,10 +1249,12 @@
key_value_store_->Put(OatHeader::kClassPathKey,
OatFile::EncodeDexFileDependencies(class_path_files));
- // Then the dex files we'll compile. Thus we'll resolve the class-path first.
- class_path_files.insert(class_path_files.end(), dex_files_.begin(), dex_files_.end());
+ class_path_class_loader = class_linker->CreatePathClassLoader(self,
+ class_path_files,
+ nullptr);
- class_loader = class_linker->CreatePathClassLoader(self, class_path_files);
+ // Class path loader as parent so that we'll resolve there first.
+ class_loader = class_linker->CreatePathClassLoader(self, dex_files_, class_path_class_loader);
}
driver_.reset(new CompilerDriver(compiler_options_.get(),
diff --git a/dexdump/dexdump_main.cc b/dexdump/dexdump_main.cc
index 2466f33..dd1002c 100644
--- a/dexdump/dexdump_main.cc
+++ b/dexdump/dexdump_main.cc
@@ -40,11 +40,12 @@
*/
static void usage(void) {
fprintf(stderr, "Copyright (C) 2007 The Android Open Source Project\n\n");
- fprintf(stderr, "%s: [-c] [-d] [-f] [-h] [-i] [-l layout] [-o outfile]"
+ fprintf(stderr, "%s: [-c] [-d] [-e] [-f] [-h] [-i] [-l layout] [-o outfile]"
" [-t tempfile] dexfile...\n", gProgName);
fprintf(stderr, "\n");
fprintf(stderr, " -c : verify checksum and exit\n");
fprintf(stderr, " -d : disassemble code sections\n");
+ fprintf(stderr, " -e : display exported items only\n");
fprintf(stderr, " -f : display summary information from file header\n");
fprintf(stderr, " -g : dump CFG for dex\n");
fprintf(stderr, " -h : display file header details\n");
@@ -69,7 +70,7 @@
// Parse all arguments.
while (1) {
- const int ic = getopt(argc, argv, "cdfghil:t:o:");
+ const int ic = getopt(argc, argv, "cdefghil:t:o:");
if (ic < 0) {
break; // done
}
@@ -80,6 +81,9 @@
case 'd': // disassemble Dalvik instructions
gOptions.disassemble = true;
break;
+ case 'e': // exported items only
+ gOptions.exportsOnly = true;
+ break;
case 'f': // dump outer file header
gOptions.showFileHeaders = true;
break;
@@ -98,7 +102,6 @@
} else if (strcmp(optarg, "xml") == 0) {
gOptions.outputFormat = OUTPUT_XML;
gOptions.verbose = false;
- gOptions.exportsOnly = true;
} else {
wantUsage = true;
}
diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc
index 5a060af..94eb82b 100644
--- a/oatdump/oatdump.cc
+++ b/oatdump/oatdump.cc
@@ -1618,9 +1618,9 @@
dex_caches_.clear();
{
ReaderMutexLock mu(self, *class_linker->DexLock());
- for (jobject weak_root : class_linker->GetDexCaches()) {
+ for (const ClassLinker::DexCacheData& data : class_linker->GetDexCachesData()) {
mirror::DexCache* dex_cache =
- down_cast<mirror::DexCache*>(self->DecodeJObject(weak_root));
+ down_cast<mirror::DexCache*>(self->DecodeJObject(data.weak_root));
if (dex_cache != nullptr) {
dex_caches_.insert(dex_cache);
}
@@ -2412,7 +2412,7 @@
// Need a class loader.
// Fake that we're a compiler.
- jobject class_loader = class_linker->CreatePathClassLoader(self, class_path);
+ jobject class_loader = class_linker->CreatePathClassLoader(self, class_path, /*parent*/nullptr);
// Use the class loader while dumping.
StackHandleScope<1> scope(self);
diff --git a/patchoat/patchoat.cc b/patchoat/patchoat.cc
index c587f68..3d9f7dc 100644
--- a/patchoat/patchoat.cc
+++ b/patchoat/patchoat.cc
@@ -526,13 +526,26 @@
temp_table.VisitRoots(&visitor, kVisitRootFlagAllRoots);
}
+class RelocatedPointerVisitor {
+ public:
+ explicit RelocatedPointerVisitor(PatchOat* patch_oat) : patch_oat_(patch_oat) {}
+
+ template <typename T>
+ T* operator()(T* ptr) const {
+ return patch_oat_->RelocatedAddressOfPointer(ptr);
+ }
+
+ private:
+ PatchOat* const patch_oat_;
+};
+
void PatchOat::PatchDexFileArrays(mirror::ObjectArray<mirror::Object>* img_roots) {
auto* dex_caches = down_cast<mirror::ObjectArray<mirror::DexCache>*>(
img_roots->Get(ImageHeader::kDexCaches));
+ const size_t pointer_size = InstructionSetPointerSize(isa_);
for (size_t i = 0, count = dex_caches->GetLength(); i < count; ++i) {
auto* orig_dex_cache = dex_caches->GetWithoutChecks(i);
auto* copy_dex_cache = RelocatedCopyOf(orig_dex_cache);
- const size_t pointer_size = InstructionSetPointerSize(isa_);
// Though the DexCache array fields are usually treated as native pointers, we set the full
// 64-bit values here, clearing the top 32 bits for 32-bit targets. The zero-extension is
// done by casting to the unsigned type uintptr_t before casting to int64_t, i.e.
@@ -543,10 +556,7 @@
mirror::DexCache::StringsOffset(),
static_cast<int64_t>(reinterpret_cast<uintptr_t>(relocated_strings)));
if (orig_strings != nullptr) {
- GcRoot<mirror::String>* copy_strings = RelocatedCopyOf(orig_strings);
- for (size_t j = 0, num = orig_dex_cache->NumStrings(); j != num; ++j) {
- copy_strings[j] = GcRoot<mirror::String>(RelocatedAddressOfPointer(orig_strings[j].Read()));
- }
+ orig_dex_cache->FixupStrings(RelocatedCopyOf(orig_strings), RelocatedPointerVisitor(this));
}
GcRoot<mirror::Class>* orig_types = orig_dex_cache->GetResolvedTypes();
GcRoot<mirror::Class>* relocated_types = RelocatedAddressOfPointer(orig_types);
@@ -554,10 +564,8 @@
mirror::DexCache::ResolvedTypesOffset(),
static_cast<int64_t>(reinterpret_cast<uintptr_t>(relocated_types)));
if (orig_types != nullptr) {
- GcRoot<mirror::Class>* copy_types = RelocatedCopyOf(orig_types);
- for (size_t j = 0, num = orig_dex_cache->NumResolvedTypes(); j != num; ++j) {
- copy_types[j] = GcRoot<mirror::Class>(RelocatedAddressOfPointer(orig_types[j].Read()));
- }
+ orig_dex_cache->FixupResolvedTypes(RelocatedCopyOf(orig_types),
+ RelocatedPointerVisitor(this));
}
ArtMethod** orig_methods = orig_dex_cache->GetResolvedMethods();
ArtMethod** relocated_methods = RelocatedAddressOfPointer(orig_methods);
@@ -588,25 +596,6 @@
}
}
-void PatchOat::FixupNativePointerArray(mirror::PointerArray* object) {
- if (object->IsIntArray()) {
- mirror::IntArray* arr = object->AsIntArray();
- mirror::IntArray* copy_arr = down_cast<mirror::IntArray*>(RelocatedCopyOf(arr));
- for (size_t j = 0, count2 = arr->GetLength(); j < count2; ++j) {
- copy_arr->SetWithoutChecks<false>(
- j, RelocatedAddressOfIntPointer(arr->GetWithoutChecks(j)));
- }
- } else {
- CHECK(object->IsLongArray());
- mirror::LongArray* arr = object->AsLongArray();
- mirror::LongArray* copy_arr = down_cast<mirror::LongArray*>(RelocatedCopyOf(arr));
- for (size_t j = 0, count2 = arr->GetLength(); j < count2; ++j) {
- copy_arr->SetWithoutChecks<false>(
- j, RelocatedAddressOfIntPointer(arr->GetWithoutChecks(j)));
- }
- }
-}
-
bool PatchOat::PatchImage() {
ImageHeader* image_header = reinterpret_cast<ImageHeader*>(image_->Begin());
CHECK_GT(image_->Size(), sizeof(ImageHeader));
@@ -674,17 +663,14 @@
PatchOat::PatchVisitor visitor(this, copy);
object->VisitReferences<kVerifyNone>(visitor, visitor);
if (object->IsClass<kVerifyNone>()) {
- auto* klass = object->AsClass();
- auto* copy_klass = down_cast<mirror::Class*>(copy);
- copy_klass->SetDexCacheStrings(RelocatedAddressOfPointer(klass->GetDexCacheStrings()));
- copy_klass->SetSFieldsPtrUnchecked(RelocatedAddressOfPointer(klass->GetSFieldsPtr()));
- copy_klass->SetIFieldsPtrUnchecked(RelocatedAddressOfPointer(klass->GetIFieldsPtr()));
- copy_klass->SetDirectMethodsPtrUnchecked(
- RelocatedAddressOfPointer(klass->GetDirectMethodsPtr()));
- copy_klass->SetVirtualMethodsPtr(RelocatedAddressOfPointer(klass->GetVirtualMethodsPtr()));
+ const size_t pointer_size = InstructionSetPointerSize(isa_);
+ mirror::Class* klass = object->AsClass();
+ mirror::Class* copy_klass = down_cast<mirror::Class*>(copy);
+ RelocatedPointerVisitor native_visitor(this);
+ klass->FixupNativePointers(copy_klass, pointer_size, native_visitor);
auto* vtable = klass->GetVTable();
if (vtable != nullptr) {
- FixupNativePointerArray(vtable);
+ vtable->Fixup(RelocatedCopyOf(vtable), pointer_size, native_visitor);
}
auto* iftable = klass->GetIfTable();
if (iftable != nullptr) {
@@ -692,24 +678,12 @@
if (iftable->GetMethodArrayCount(i) > 0) {
auto* method_array = iftable->GetMethodArray(i);
CHECK(method_array != nullptr);
- FixupNativePointerArray(method_array);
+ method_array->Fixup(RelocatedCopyOf(method_array), pointer_size, native_visitor);
}
}
}
- if (klass->ShouldHaveEmbeddedImtAndVTable()) {
- const size_t pointer_size = InstructionSetPointerSize(isa_);
- for (int32_t i = 0; i < klass->GetEmbeddedVTableLength(); ++i) {
- copy_klass->SetEmbeddedVTableEntryUnchecked(i, RelocatedAddressOfPointer(
- klass->GetEmbeddedVTableEntry(i, pointer_size)), pointer_size);
- }
- for (size_t i = 0; i < mirror::Class::kImtSize; ++i) {
- copy_klass->SetEmbeddedImTableEntry(i, RelocatedAddressOfPointer(
- klass->GetEmbeddedImTableEntry(i, pointer_size)), pointer_size);
- }
- }
- }
- if (object->GetClass() == mirror::Method::StaticClass() ||
- object->GetClass() == mirror::Constructor::StaticClass()) {
+ } else if (object->GetClass() == mirror::Method::StaticClass() ||
+ object->GetClass() == mirror::Constructor::StaticClass()) {
// Need to go update the ArtMethod.
auto* dest = down_cast<mirror::AbstractMethod*>(copy);
auto* src = down_cast<mirror::AbstractMethod*>(object);
diff --git a/patchoat/patchoat.h b/patchoat/patchoat.h
index 87ecc61..0915014 100644
--- a/patchoat/patchoat.h
+++ b/patchoat/patchoat.h
@@ -102,8 +102,6 @@
SHARED_REQUIRES(Locks::mutator_lock_);
void FixupMethod(ArtMethod* object, ArtMethod* copy)
SHARED_REQUIRES(Locks::mutator_lock_);
- void FixupNativePointerArray(mirror::PointerArray* object)
- SHARED_REQUIRES(Locks::mutator_lock_);
bool InHeap(mirror::Object*);
// Patches oat in place, modifying the oat_file given to the constructor.
@@ -200,6 +198,7 @@
TimingLogger* timings_;
friend class FixupRootVisitor;
+ friend class RelocatedPointerVisitor;
friend class PatchOatArtFieldVisitor;
friend class PatchOatArtMethodVisitor;
DISALLOW_IMPLICIT_CONSTRUCTORS(PatchOat);
diff --git a/runtime/Android.mk b/runtime/Android.mk
index 1fdffe3..0b0f094 100644
--- a/runtime/Android.mk
+++ b/runtime/Android.mk
@@ -60,6 +60,7 @@
gc/collector/concurrent_copying.cc \
gc/collector/garbage_collector.cc \
gc/collector/immune_region.cc \
+ gc/collector/immune_spaces.cc \
gc/collector/mark_compact.cc \
gc/collector/mark_sweep.cc \
gc/collector/partial_mark_sweep.cc \
diff --git a/runtime/arch/arm/entrypoints_init_arm.cc b/runtime/arch/arm/entrypoints_init_arm.cc
index 76c7c4f..be33b0e 100644
--- a/runtime/arch/arm/entrypoints_init_arm.cc
+++ b/runtime/arch/arm/entrypoints_init_arm.cc
@@ -164,9 +164,10 @@
// Deoptimization from compiled code.
qpoints->pDeoptimize = art_quick_deoptimize_from_compiled_code;
- // Read barrier
+ // Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
qpoints->pReadBarrierSlow = artReadBarrierSlow;
+ qpoints->pReadBarrierForRootSlow = artReadBarrierForRootSlow;
}
} // namespace art
diff --git a/runtime/arch/arm64/entrypoints_init_arm64.cc b/runtime/arch/arm64/entrypoints_init_arm64.cc
index 371cbb2..63285a4 100644
--- a/runtime/arch/arm64/entrypoints_init_arm64.cc
+++ b/runtime/arch/arm64/entrypoints_init_arm64.cc
@@ -147,9 +147,10 @@
// Deoptimization from compiled code.
qpoints->pDeoptimize = art_quick_deoptimize_from_compiled_code;
- // Read barrier
+ // Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
qpoints->pReadBarrierSlow = artReadBarrierSlow;
+ qpoints->pReadBarrierForRootSlow = artReadBarrierForRootSlow;
};
} // namespace art
diff --git a/runtime/arch/arm64/instruction_set_features_arm64.cc b/runtime/arch/arm64/instruction_set_features_arm64.cc
index 395cee8..613bb5c 100644
--- a/runtime/arch/arm64/instruction_set_features_arm64.cc
+++ b/runtime/arch/arm64/instruction_set_features_arm64.cc
@@ -39,7 +39,7 @@
if (!needs_a53_835769_fix) {
// Check to see if this is an expected variant.
static const char* arm64_known_variants[] = {
- "denver64"
+ "denver64", "kryo"
};
if (!FindVariantInArray(arm64_known_variants, arraysize(arm64_known_variants), variant)) {
std::ostringstream os;
diff --git a/runtime/arch/mips/entrypoints_direct_mips.h b/runtime/arch/mips/entrypoints_direct_mips.h
index f9c5315..74e7638 100644
--- a/runtime/arch/mips/entrypoints_direct_mips.h
+++ b/runtime/arch/mips/entrypoints_direct_mips.h
@@ -45,7 +45,8 @@
entrypoint == kQuickCmpgFloat ||
entrypoint == kQuickCmplDouble ||
entrypoint == kQuickCmplFloat ||
- entrypoint == kQuickReadBarrierSlow;
+ entrypoint == kQuickReadBarrierSlow ||
+ entrypoint == kQuickReadBarrierForRootSlow;
}
} // namespace art
diff --git a/runtime/arch/mips/entrypoints_init_mips.cc b/runtime/arch/mips/entrypoints_init_mips.cc
index 59421dd..cba427d 100644
--- a/runtime/arch/mips/entrypoints_init_mips.cc
+++ b/runtime/arch/mips/entrypoints_init_mips.cc
@@ -271,10 +271,14 @@
qpoints->pA64Store = QuasiAtomic::Write64;
static_assert(IsDirectEntrypoint(kQuickA64Store), "Non-direct C stub marked direct.");
+ // Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
static_assert(!IsDirectEntrypoint(kQuickReadBarrierJni), "Non-direct C stub marked direct.");
qpoints->pReadBarrierSlow = artReadBarrierSlow;
static_assert(IsDirectEntrypoint(kQuickReadBarrierSlow), "Direct C stub not marked direct.");
+ qpoints->pReadBarrierForRootSlow = artReadBarrierForRootSlow;
+ static_assert(IsDirectEntrypoint(kQuickReadBarrierForRootSlow),
+ "Direct C stub not marked direct.");
};
} // namespace art
diff --git a/runtime/arch/mips64/entrypoints_init_mips64.cc b/runtime/arch/mips64/entrypoints_init_mips64.cc
index 417d5fc..89f54dd 100644
--- a/runtime/arch/mips64/entrypoints_init_mips64.cc
+++ b/runtime/arch/mips64/entrypoints_init_mips64.cc
@@ -178,9 +178,10 @@
qpoints->pA64Load = QuasiAtomic::Read64;
qpoints->pA64Store = QuasiAtomic::Write64;
- // Read barrier
+ // Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
qpoints->pReadBarrierSlow = artReadBarrierSlow;
+ qpoints->pReadBarrierForRootSlow = artReadBarrierForRootSlow;
};
} // namespace art
diff --git a/runtime/arch/stub_test.cc b/runtime/arch/stub_test.cc
index 1d10e5d..afa48cd 100644
--- a/runtime/arch/stub_test.cc
+++ b/runtime/arch/stub_test.cc
@@ -2151,4 +2151,39 @@
#endif
}
+TEST_F(StubTest, ReadBarrierForRoot) {
+#if defined(ART_USE_READ_BARRIER) && (defined(__i386__) || defined(__arm__) || \
+ defined(__aarch64__) || defined(__mips__) || (defined(__x86_64__) && !defined(__APPLE__)))
+ Thread* self = Thread::Current();
+
+ const uintptr_t readBarrierForRootSlow =
+ StubTest::GetEntrypoint(self, kQuickReadBarrierForRootSlow);
+
+ // Create an object
+ ScopedObjectAccess soa(self);
+ // garbage is created during ClassLinker::Init
+
+ StackHandleScope<1> hs(soa.Self());
+
+ Handle<mirror::String> obj(
+ hs.NewHandle(mirror::String::AllocFromModifiedUtf8(soa.Self(), "hello, world!")));
+
+ EXPECT_FALSE(self->IsExceptionPending());
+
+ GcRoot<mirror::Class>& root = mirror::String::java_lang_String_;
+ size_t result = Invoke3(reinterpret_cast<size_t>(&root), 0U, 0U, readBarrierForRootSlow, self);
+
+ EXPECT_FALSE(self->IsExceptionPending());
+ EXPECT_NE(reinterpret_cast<size_t>(nullptr), result);
+ mirror::Class* klass = reinterpret_cast<mirror::Class*>(result);
+ EXPECT_EQ(klass, obj->GetClass());
+
+ // Tests done.
+#else
+ LOG(INFO) << "Skipping read_barrier_for_root_slow";
+ // Force-print to std::cout so it's also outside the logcat.
+ std::cout << "Skipping read_barrier_for_root_slow" << std::endl;
+#endif
+}
+
} // namespace art
diff --git a/runtime/arch/x86/entrypoints_init_x86.cc b/runtime/arch/x86/entrypoints_init_x86.cc
index 019546f..e200018 100644
--- a/runtime/arch/x86/entrypoints_init_x86.cc
+++ b/runtime/arch/x86/entrypoints_init_x86.cc
@@ -29,6 +29,7 @@
// Read barrier entrypoints.
extern "C" mirror::Object* art_quick_read_barrier_slow(mirror::Object*, mirror::Object*, uint32_t);
+extern "C" mirror::Object* art_quick_read_barrier_for_root_slow(GcRoot<mirror::Object>*);
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
// JNI
@@ -136,9 +137,10 @@
// Deoptimize
qpoints->pDeoptimize = art_quick_deoptimize_from_compiled_code;
- // Read barrier
+ // Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
qpoints->pReadBarrierSlow = art_quick_read_barrier_slow;
+ qpoints->pReadBarrierForRootSlow = art_quick_read_barrier_for_root_slow;
};
} // namespace art
diff --git a/runtime/arch/x86/quick_entrypoints_x86.S b/runtime/arch/x86/quick_entrypoints_x86.S
index 2f485ae..463c9cf 100644
--- a/runtime/arch/x86/quick_entrypoints_x86.S
+++ b/runtime/arch/x86/quick_entrypoints_x86.S
@@ -788,7 +788,106 @@
// Generate the allocation entrypoints for each allocator.
GENERATE_ALLOC_ENTRYPOINTS_FOR_EACH_ALLOCATOR
-GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT(_rosalloc, RosAlloc)
+
+// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT(_rosalloc, RosAlloc).
+DEFINE_FUNCTION art_quick_alloc_object_rosalloc
+ // Fast path rosalloc allocation.
+ // eax: uint32_t type_idx/return value, ecx: ArtMethod*
+ // ebx, edx: free
+ PUSH edi
+ movl ART_METHOD_DEX_CACHE_TYPES_OFFSET_32(%ecx), %edx // Load dex cache resolved types array
+ // Load the class (edx)
+ movl 0(%edx, %eax, COMPRESSED_REFERENCE_SIZE), %edx
+ testl %edx, %edx // Check null class
+ jz .Lart_quick_alloc_object_rosalloc_slow_path
+ // Check class status
+ cmpl LITERAL(MIRROR_CLASS_STATUS_INITIALIZED), MIRROR_CLASS_STATUS_OFFSET(%edx)
+ jne .Lart_quick_alloc_object_rosalloc_slow_path
+ // No fake dependence needed on x86
+ // between status and flags load,
+ // since each load is a load-acquire,
+ // no loads reordering.
+ // Check access flags has
+ // kAccClassIsFinalizable
+ testl LITERAL(ACCESS_FLAGS_CLASS_IS_FINALIZABLE), MIRROR_CLASS_ACCESS_FLAGS_OFFSET(%edx)
+ jnz .Lart_quick_alloc_object_rosalloc_slow_path
+
+ movl %fs:THREAD_SELF_OFFSET, %ebx // ebx = thread
+ // Check if the thread local allocation
+ // stack has room
+ movl THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET(%ebx), %edi
+ cmpl THREAD_LOCAL_ALLOC_STACK_END_OFFSET(%ebx), %edi
+ jae .Lart_quick_alloc_object_rosalloc_slow_path
+
+ movl MIRROR_CLASS_OBJECT_SIZE_OFFSET(%edx), %edi // Load the object size (edi)
+ // Check if the size is for a thread
+ // local allocation
+ cmpl LITERAL(ROSALLOC_MAX_THREAD_LOCAL_BRACKET_SIZE), %edi
+ ja .Lart_quick_alloc_object_rosalloc_slow_path
+ decl %edi
+ shrl LITERAL(ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT), %edi // Calculate the rosalloc bracket index
+ // from object size.
+ // Align up the size by the rosalloc
+ // bracket quantum size and divide
+ // by the quantum size and subtract
+ // by 1. This code is a shorter but
+ // equivalent version.
+ // Load thread local rosalloc run (ebx)
+ movl THREAD_ROSALLOC_RUNS_OFFSET(%ebx, %edi, __SIZEOF_POINTER__), %ebx
+ // Load free_list head (edi),
+ // this will be the return value.
+ movl (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)(%ebx), %edi
+ test %edi, %edi
+ jz .Lart_quick_alloc_object_rosalloc_slow_path
+ // Point of no slow path. Won't go to
+ // the slow path from here on. Ok to
+ // clobber eax and ecx.
+ movl %edi, %eax
+ // Load the next pointer of the head
+ // and update head of free list with
+ // next pointer
+ movl ROSALLOC_SLOT_NEXT_OFFSET(%eax), %edi
+ movl %edi, (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)(%ebx)
+ // Decrement size of free list by 1
+ decl (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)(%ebx)
+ // Store the class pointer in the
+ // header. This also overwrites the
+ // next pointer. The offsets are
+ // asserted to match.
+#if ROSALLOC_SLOT_NEXT_OFFSET != MIRROR_OBJECT_CLASS_OFFSET
+#error "Class pointer needs to overwrite next pointer."
+#endif
+ POISON_HEAP_REF edx
+ movl %edx, MIRROR_OBJECT_CLASS_OFFSET(%eax)
+ movl %fs:THREAD_SELF_OFFSET, %ebx // ebx = thread
+ // Push the new object onto the thread
+ // local allocation stack and
+ // increment the thread local
+ // allocation stack top.
+ movl THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET(%ebx), %edi
+ movl %eax, (%edi)
+ addl LITERAL(COMPRESSED_REFERENCE_SIZE), %edi
+ movl %edi, THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET(%ebx)
+ // No fence needed for x86.
+ POP edi
+ ret
+.Lart_quick_alloc_object_rosalloc_slow_path:
+ POP edi
+ SETUP_REFS_ONLY_CALLEE_SAVE_FRAME ebx, ebx // save ref containing registers for GC
+ // Outgoing argument set up
+ PUSH eax // alignment padding
+ pushl %fs:THREAD_SELF_OFFSET // pass Thread::Current()
+ CFI_ADJUST_CFA_OFFSET(4)
+ PUSH ecx
+ PUSH eax
+ call SYMBOL(artAllocObjectFromCodeRosAlloc) // cxx_name(arg0, arg1, Thread*)
+ addl LITERAL(16), %esp // pop arguments
+ CFI_ADJUST_CFA_OFFSET(-16)
+ RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME // resotre frame up to return address
+ RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER // return or deliver exception
+END_FUNCTION art_quick_alloc_object_rosalloc
+
+
GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT(_tlab, TLAB)
ONE_ARG_DOWNCALL art_quick_resolve_string, artResolveStringFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER
@@ -1588,14 +1687,22 @@
END_FUNCTION art_nested_signal_return
DEFINE_FUNCTION art_quick_read_barrier_slow
- PUSH edx // pass arg3 - offset
- PUSH ecx // pass arg2 - obj
- PUSH eax // pass arg1 - ref
- call SYMBOL(artReadBarrierSlow) // artReadBarrierSlow(ref, obj, offset)
- addl LITERAL(12), %esp // pop arguments
+ PUSH edx // pass arg3 - offset
+ PUSH ecx // pass arg2 - obj
+ PUSH eax // pass arg1 - ref
+ call SYMBOL(artReadBarrierSlow) // artReadBarrierSlow(ref, obj, offset)
+ addl LITERAL(12), %esp // pop arguments
CFI_ADJUST_CFA_OFFSET(-12)
ret
END_FUNCTION art_quick_read_barrier_slow
+DEFINE_FUNCTION art_quick_read_barrier_for_root_slow
+ PUSH eax // pass arg1 - root
+ call SYMBOL(artReadBarrierForRootSlow) // artReadBarrierForRootSlow(root)
+ addl LITERAL(4), %esp // pop argument
+ CFI_ADJUST_CFA_OFFSET(-4)
+ ret
+END_FUNCTION art_quick_read_barrier_for_root_slow
+
// TODO: implement these!
UNIMPLEMENTED art_quick_memcmp16
diff --git a/runtime/arch/x86_64/entrypoints_init_x86_64.cc b/runtime/arch/x86_64/entrypoints_init_x86_64.cc
index eae09ee..2b38c9d 100644
--- a/runtime/arch/x86_64/entrypoints_init_x86_64.cc
+++ b/runtime/arch/x86_64/entrypoints_init_x86_64.cc
@@ -30,6 +30,7 @@
// Read barrier entrypoints.
extern "C" mirror::Object* art_quick_read_barrier_slow(mirror::Object*, mirror::Object*, uint32_t);
+extern "C" mirror::Object* art_quick_read_barrier_for_root_slow(GcRoot<mirror::Object>*);
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
#if defined(__APPLE__)
@@ -140,9 +141,10 @@
// Deoptimize
qpoints->pDeoptimize = art_quick_deoptimize_from_compiled_code;
- // Read barrier
+ // Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
qpoints->pReadBarrierSlow = art_quick_read_barrier_slow;
+ qpoints->pReadBarrierForRootSlow = art_quick_read_barrier_for_root_slow;
#endif // __APPLE__
};
diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
index 5fd8969..17d277e 100644
--- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S
+++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
@@ -1722,3 +1722,14 @@
RESTORE_FP_CALLEE_SAVE_FRAME
ret
END_FUNCTION art_quick_read_barrier_slow
+
+DEFINE_FUNCTION art_quick_read_barrier_for_root_slow
+ SETUP_FP_CALLEE_SAVE_FRAME
+ subq LITERAL(8), %rsp // Alignment padding.
+ CFI_ADJUST_CFA_OFFSET(8)
+ call SYMBOL(artReadBarrierForRootSlow) // artReadBarrierForRootSlow(root)
+ addq LITERAL(8), %rsp
+ CFI_ADJUST_CFA_OFFSET(-8)
+ RESTORE_FP_CALLEE_SAVE_FRAME
+ ret
+END_FUNCTION art_quick_read_barrier_for_root_slow
diff --git a/runtime/asm_support.h b/runtime/asm_support.h
index 69f6fe9..b548dfb 100644
--- a/runtime/asm_support.h
+++ b/runtime/asm_support.h
@@ -122,7 +122,7 @@
art::Thread::SelfOffset<__SIZEOF_POINTER__>().Int32Value())
// Offset of field Thread::tlsPtr_.thread_local_pos.
-#define THREAD_LOCAL_POS_OFFSET (THREAD_CARD_TABLE_OFFSET + 150 * __SIZEOF_POINTER__)
+#define THREAD_LOCAL_POS_OFFSET (THREAD_CARD_TABLE_OFFSET + 151 * __SIZEOF_POINTER__)
ADD_TEST_EQ(THREAD_LOCAL_POS_OFFSET,
art::Thread::ThreadLocalPosOffset<__SIZEOF_POINTER__>().Int32Value())
// Offset of field Thread::tlsPtr_.thread_local_end.
diff --git a/runtime/base/bit_utils.h b/runtime/base/bit_utils.h
index 9c78ee5..d6a44f7 100644
--- a/runtime/base/bit_utils.h
+++ b/runtime/base/bit_utils.h
@@ -53,6 +53,7 @@
: __builtin_ctzll(x);
}
+// Return the number of 1-bits in `x`.
template<typename T>
static constexpr int POPCOUNT(T x) {
return (sizeof(T) == sizeof(uint32_t))
diff --git a/runtime/base/stl_util.h b/runtime/base/stl_util.h
index 0949619..324ab21 100644
--- a/runtime/base/stl_util.h
+++ b/runtime/base/stl_util.h
@@ -149,6 +149,13 @@
return it != container.end();
}
+// const char* compare function suitable for std::map or std::set.
+struct CStringLess {
+ bool operator()(const char* lhs, const char* rhs) const {
+ return strcmp(lhs, rhs) < 0;
+ }
+};
+
} // namespace art
#endif // ART_RUNTIME_BASE_STL_UTIL_H_
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index f649972..2dd2a83 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -322,7 +322,8 @@
std::fill_n(find_array_class_cache_, kFindArrayCacheSize, GcRoot<mirror::Class>(nullptr));
}
-void ClassLinker::InitWithoutImage(std::vector<std::unique_ptr<const DexFile>> boot_class_path) {
+bool ClassLinker::InitWithoutImage(std::vector<std::unique_ptr<const DexFile>> boot_class_path,
+ std::string* error_msg) {
VLOG(startup) << "ClassLinker::Init";
Thread* const self = Thread::Current();
@@ -368,10 +369,12 @@
mirror::Class::SetStatus(java_lang_Object, mirror::Class::kStatusLoaded, self);
java_lang_Object->SetObjectSize(sizeof(mirror::Object));
- runtime->SetSentinel(heap->AllocObject<true>(self,
- java_lang_Object.Get(),
- java_lang_Object->GetObjectSize(),
- VoidFunctor()));
+ // Allocate in non-movable so that it's possible to check if a JNI weak global ref has been
+ // cleared without triggering the read barrier and unintentionally mark the sentinel alive.
+ runtime->SetSentinel(heap->AllocNonMovableObject<true>(self,
+ java_lang_Object.Get(),
+ java_lang_Object->GetObjectSize(),
+ VoidFunctor()));
// Object[] next to hold class roots.
Handle<mirror::Class> object_array_class(hs.NewHandle(
@@ -475,9 +478,15 @@
// Setup boot_class_path_ and register class_path now that we can use AllocObjectArray to create
// DexCache instances. Needs to be after String, Field, Method arrays since AllocDexCache uses
// these roots.
- CHECK_NE(0U, boot_class_path.size());
+ if (boot_class_path.empty()) {
+ *error_msg = "Boot classpath is empty.";
+ return false;
+ }
for (auto& dex_file : boot_class_path) {
- CHECK(dex_file.get() != nullptr);
+ if (dex_file.get() == nullptr) {
+ *error_msg = "Null dex file.";
+ return false;
+ }
AppendToBootClassPath(self, *dex_file);
opened_dex_files_.push_back(std::move(dex_file));
}
@@ -658,6 +667,8 @@
FinishInit(self);
VLOG(startup) << "ClassLinker::InitFromCompiler exiting";
+
+ return true;
}
void ClassLinker::FinishInit(Thread* self) {
@@ -848,7 +859,7 @@
DISALLOW_COPY_AND_ASSIGN(SetInterpreterEntrypointArtMethodVisitor);
};
-void ClassLinker::InitFromImage() {
+bool ClassLinker::InitFromImage(std::string* error_msg) {
VLOG(startup) << "ClassLinker::InitFromImage entering";
CHECK(!init_done_);
@@ -886,27 +897,39 @@
mirror::Class* java_lang_Object = GetClassRoot(kJavaLangObject);
java_lang_Object->SetObjectSize(sizeof(mirror::Object));
- Runtime::Current()->SetSentinel(heap->AllocObject<true>(self,
- java_lang_Object,
- java_lang_Object->GetObjectSize(),
- VoidFunctor()));
+ // Allocate in non-movable so that it's possible to check if a JNI weak global ref has been
+ // cleared without triggering the read barrier and unintentionally mark the sentinel alive.
+ runtime->SetSentinel(heap->AllocNonMovableObject<true>(self,
+ java_lang_Object,
+ java_lang_Object->GetObjectSize(),
+ VoidFunctor()));
- CHECK_EQ(oat_file->GetOatHeader().GetDexFileCount(),
- static_cast<uint32_t>(dex_caches->GetLength()));
+ if (oat_file->GetOatHeader().GetDexFileCount() !=
+ static_cast<uint32_t>(dex_caches->GetLength())) {
+ *error_msg = "Dex cache count and dex file count mismatch while trying to initialize from "
+ "image";
+ return false;
+ }
for (int32_t i = 0; i < dex_caches->GetLength(); i++) {
StackHandleScope<1> hs2(self);
Handle<mirror::DexCache> dex_cache(hs2.NewHandle(dex_caches->Get(i)));
const std::string& dex_file_location(dex_cache->GetLocation()->ToModifiedUtf8());
const OatFile::OatDexFile* oat_dex_file = oat_file->GetOatDexFile(dex_file_location.c_str(),
nullptr);
- CHECK(oat_dex_file != nullptr) << oat_file->GetLocation() << " " << dex_file_location;
- std::string error_msg;
- std::unique_ptr<const DexFile> dex_file = oat_dex_file->OpenDexFile(&error_msg);
+ if (oat_dex_file == nullptr) {
+ *error_msg = StringPrintf("Failed finding oat dex file for %s %s",
+ oat_file->GetLocation().c_str(),
+ dex_file_location.c_str());
+ return false;
+ }
+ std::string inner_error_msg;
+ std::unique_ptr<const DexFile> dex_file = oat_dex_file->OpenDexFile(&inner_error_msg);
if (dex_file == nullptr) {
- LOG(FATAL) << "Failed to open dex file " << dex_file_location
- << " from within oat file " << oat_file->GetLocation()
- << " error '" << error_msg << "'";
- UNREACHABLE();
+ *error_msg = StringPrintf("Failed to open dex file %s from within oat file %s error '%s'",
+ dex_file_location.c_str(),
+ oat_file->GetLocation().c_str(),
+ inner_error_msg.c_str());
+ return false;
}
if (kSanityCheckObjects) {
@@ -916,13 +939,22 @@
space);
}
- CHECK_EQ(dex_file->GetLocationChecksum(), oat_dex_file->GetDexFileLocationChecksum());
+ if (dex_file->GetLocationChecksum() != oat_dex_file->GetDexFileLocationChecksum()) {
+ *error_msg = StringPrintf("Checksums do not match for %s: %x vs %x",
+ dex_file_location.c_str(),
+ dex_file->GetLocationChecksum(),
+ oat_dex_file->GetDexFileLocationChecksum());
+ return false;
+ }
AppendToBootClassPath(*dex_file.get(), dex_cache);
opened_dex_files_.push_back(std::move(dex_file));
}
- CHECK(ValidPointerSize(image_pointer_size_)) << image_pointer_size_;
+ if (!ValidPointerSize(image_pointer_size_)) {
+ *error_msg = StringPrintf("Invalid image pointer size: %zu", image_pointer_size_);
+ return false;
+ }
// Set classes on AbstractMethod early so that IsMethod tests can be performed during the live
// bitmap walk.
@@ -930,7 +962,12 @@
// Only the Aot compiler supports having an image with a different pointer size than the
// runtime. This happens on the host for compile 32 bit tests since we use a 64 bit libart
// compiler. We may also use 32 bit dex2oat on a system with 64 bit apps.
- CHECK_EQ(image_pointer_size_, sizeof(void*));
+ if (image_pointer_size_ != sizeof(void*)) {
+ *error_msg = StringPrintf("Runtime must use current image pointer size: %zu vs %zu",
+ image_pointer_size_ ,
+ sizeof(void*));
+ return false;
+ }
}
if (kSanityCheckObjects) {
@@ -983,6 +1020,8 @@
FinishInit(self);
VLOG(startup) << "ClassLinker::InitFromImage exiting";
+
+ return true;
}
bool ClassLinker::ClassInClassTable(mirror::Class* klass) {
@@ -2324,17 +2363,22 @@
// Clean up pass to remove null dex caches.
// Null dex caches can occur due to class unloading and we are lazily removing null entries.
JavaVMExt* const vm = self->GetJniEnv()->vm;
- for (auto it = dex_caches_.begin(); it != dex_caches_.end();) {
- mirror::Object* dex_cache_root = self->DecodeJObject(*it);
- if (dex_cache_root == nullptr) {
- vm->DeleteWeakGlobalRef(self, *it);
+ for (auto it = dex_caches_.begin(); it != dex_caches_.end(); ) {
+ DexCacheData data = *it;
+ if (self->IsJWeakCleared(data.weak_root)) {
+ vm->DeleteWeakGlobalRef(self, data.weak_root);
it = dex_caches_.erase(it);
} else {
++it;
}
}
- dex_caches_.push_back(vm->AddWeakGlobalRef(self, dex_cache.Get()));
+ jweak dex_cache_jweak = vm->AddWeakGlobalRef(self, dex_cache.Get());
dex_cache->SetDexFile(&dex_file);
+ DexCacheData data;
+ data.weak_root = dex_cache_jweak;
+ data.dex_file = dex_cache->GetDexFile();
+ data.resolved_types = dex_cache->GetResolvedTypes();
+ dex_caches_.push_back(data);
}
mirror::DexCache* ClassLinker::RegisterDexFile(const DexFile& dex_file, LinearAlloc* linear_alloc) {
@@ -2381,10 +2425,16 @@
const DexFile& dex_file,
bool allow_failure) {
// Search assuming unique-ness of dex file.
- for (jweak weak_root : dex_caches_) {
- mirror::DexCache* dex_cache = down_cast<mirror::DexCache*>(self->DecodeJObject(weak_root));
- if (dex_cache != nullptr && dex_cache->GetDexFile() == &dex_file) {
- return dex_cache;
+ for (const DexCacheData& data : dex_caches_) {
+ // Avoid decoding (and read barriers) other unrelated dex caches.
+ if (data.dex_file == &dex_file) {
+ mirror::DexCache* dex_cache =
+ down_cast<mirror::DexCache*>(self->DecodeJObject(data.weak_root));
+ if (dex_cache != nullptr) {
+ return dex_cache;
+ } else {
+ break;
+ }
}
}
if (allow_failure) {
@@ -2392,8 +2442,8 @@
}
std::string location(dex_file.GetLocation());
// Failure, dump diagnostic and abort.
- for (jobject weak_root : dex_caches_) {
- mirror::DexCache* dex_cache = down_cast<mirror::DexCache*>(self->DecodeJObject(weak_root));
+ for (const DexCacheData& data : dex_caches_) {
+ mirror::DexCache* dex_cache = down_cast<mirror::DexCache*>(self->DecodeJObject(data.weak_root));
if (dex_cache != nullptr) {
LOG(ERROR) << "Registered dex file " << dex_cache->GetDexFile()->GetLocation();
}
@@ -2405,10 +2455,13 @@
void ClassLinker::FixupDexCaches(ArtMethod* resolution_method) {
Thread* const self = Thread::Current();
ReaderMutexLock mu(self, dex_lock_);
- for (jobject weak_root : dex_caches_) {
- mirror::DexCache* dex_cache = down_cast<mirror::DexCache*>(self->DecodeJObject(weak_root));
- if (dex_cache != nullptr) {
- dex_cache->Fixup(resolution_method, image_pointer_size_);
+ for (const DexCacheData& data : dex_caches_) {
+ if (!self->IsJWeakCleared(data.weak_root)) {
+ mirror::DexCache* dex_cache = down_cast<mirror::DexCache*>(
+ self->DecodeJObject(data.weak_root));
+ if (dex_cache != nullptr) {
+ dex_cache->Fixup(resolution_method, image_pointer_size_);
+ }
}
}
}
@@ -3346,15 +3399,18 @@
Thread* const self = Thread::Current();
ReaderMutexLock mu(self, dex_lock_);
// Locate the dex cache of the original interface/Object
- for (jobject weak_root : dex_caches_) {
- mirror::DexCache* dex_cache = down_cast<mirror::DexCache*>(self->DecodeJObject(weak_root));
- if (dex_cache != nullptr &&
- proxy_method->HasSameDexCacheResolvedTypes(dex_cache->GetResolvedTypes(),
+ for (const DexCacheData& data : dex_caches_) {
+ if (!self->IsJWeakCleared(data.weak_root) &&
+ proxy_method->HasSameDexCacheResolvedTypes(data.resolved_types,
image_pointer_size_)) {
- ArtMethod* resolved_method = dex_cache->GetResolvedMethod(
- proxy_method->GetDexMethodIndex(), image_pointer_size_);
- CHECK(resolved_method != nullptr);
- return resolved_method;
+ mirror::DexCache* dex_cache = down_cast<mirror::DexCache*>(
+ self->DecodeJObject(data.weak_root));
+ if (dex_cache != nullptr) {
+ ArtMethod* resolved_method = dex_cache->GetResolvedMethod(
+ proxy_method->GetDexMethodIndex(), image_pointer_size_);
+ CHECK(resolved_method != nullptr);
+ return resolved_method;
+ }
}
}
}
@@ -6573,7 +6629,9 @@
}
}
-jobject ClassLinker::CreatePathClassLoader(Thread* self, std::vector<const DexFile*>& dex_files) {
+jobject ClassLinker::CreatePathClassLoader(Thread* self,
+ std::vector<const DexFile*>& dex_files,
+ jobject parent_loader) {
// SOAAlreadyRunnable is protected, and we need something to add a global reference.
// We could move the jobject to the callers, but all call-sites do this...
ScopedObjectAccessUnchecked soa(self);
@@ -6604,8 +6662,8 @@
for (const DexFile* dex_file : dex_files) {
StackHandleScope<3> hs2(self);
- // CreatePathClassLoader is only used by gtests. Index 0 of h_long_array is supposed to be the
- // oat file but we can leave it null.
+ // CreatePathClassLoader is only used by gtests and dex2oat. Index 0 of h_long_array is
+ // supposed to be the oat file but we can leave it null.
Handle<mirror::LongArray> h_long_array = hs2.NewHandle(mirror::LongArray::Alloc(
self,
kDexFileIndexStart + 1));
@@ -6651,9 +6709,10 @@
mirror::Class::FindField(self, hs.NewHandle(h_path_class_loader->GetClass()), "parent",
"Ljava/lang/ClassLoader;");
DCHECK(parent_field != nullptr);
- mirror::Object* boot_cl =
- soa.Decode<mirror::Class*>(WellKnownClasses::java_lang_BootClassLoader)->AllocObject(self);
- parent_field->SetObject<false>(h_path_class_loader.Get(), boot_cl);
+ mirror::Object* parent = (parent_loader != nullptr)
+ ? soa.Decode<mirror::ClassLoader*>(parent_loader)
+ : soa.Decode<mirror::Class*>(WellKnownClasses::java_lang_BootClassLoader)->AllocObject(self);
+ parent_field->SetObject<false>(h_path_class_loader.Get(), parent);
// Make it a global ref and return.
ScopedLocalRef<jobject> local_ref(
diff --git a/runtime/class_linker.h b/runtime/class_linker.h
index 21f9e7b..29aac31 100644
--- a/runtime/class_linker.h
+++ b/runtime/class_linker.h
@@ -115,12 +115,15 @@
~ClassLinker();
// Initialize class linker by bootstraping from dex files.
- void InitWithoutImage(std::vector<std::unique_ptr<const DexFile>> boot_class_path)
+ bool InitWithoutImage(std::vector<std::unique_ptr<const DexFile>> boot_class_path,
+ std::string* error_msg)
SHARED_REQUIRES(Locks::mutator_lock_)
REQUIRES(!dex_lock_);
// Initialize class linker from one or more images.
- void InitFromImage() SHARED_REQUIRES(Locks::mutator_lock_) REQUIRES(!dex_lock_);
+ bool InitFromImage(std::string* error_msg)
+ SHARED_REQUIRES(Locks::mutator_lock_)
+ REQUIRES(!dex_lock_);
// Finds a class by its descriptor, loading it if necessary.
// If class_loader is null, searches boot_class_path_.
@@ -511,7 +514,10 @@
// Creates a GlobalRef PathClassLoader that can be used to load classes from the given dex files.
// Note: the objects are not completely set up. Do not use this outside of tests and the compiler.
- jobject CreatePathClassLoader(Thread* self, std::vector<const DexFile*>& dex_files)
+ // If parent_loader is null then we use the boot class loader.
+ jobject CreatePathClassLoader(Thread* self,
+ std::vector<const DexFile*>& dex_files,
+ jobject parent_loader)
SHARED_REQUIRES(Locks::mutator_lock_)
REQUIRES(!dex_lock_);
@@ -551,6 +557,17 @@
REQUIRES(!Locks::classlinker_classes_lock_)
SHARED_REQUIRES(Locks::mutator_lock_);
+ struct DexCacheData {
+ // Weak root to the DexCache. Note: Do not decode this unnecessarily or else class unloading may
+ // not work properly.
+ jweak weak_root;
+ // The following two fields are caches to the DexCache's fields and here to avoid unnecessary
+ // jweak decode that triggers read barriers (and mark them alive unnecessarily and mess with
+ // class unloading.)
+ const DexFile* dex_file;
+ GcRoot<mirror::Class>* resolved_types;
+ };
+
private:
struct ClassLoaderData {
jweak weak_root; // Weak root to enable class unloading.
@@ -902,7 +919,8 @@
size_t GetDexCacheCount() SHARED_REQUIRES(Locks::mutator_lock_, dex_lock_) {
return dex_caches_.size();
}
- const std::list<jweak>& GetDexCaches() SHARED_REQUIRES(Locks::mutator_lock_, dex_lock_) {
+ const std::list<DexCacheData>& GetDexCachesData()
+ SHARED_REQUIRES(Locks::mutator_lock_, dex_lock_) {
return dex_caches_;
}
@@ -965,9 +983,9 @@
std::vector<std::unique_ptr<const DexFile>> opened_dex_files_;
mutable ReaderWriterMutex dex_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER;
- // JNI weak globals to allow dex caches to get unloaded. We lazily delete weak globals when we
- // register new dex files.
- std::list<jweak> dex_caches_ GUARDED_BY(dex_lock_);
+ // JNI weak globals and side data to allow dex caches to get unloaded. We lazily delete weak
+ // globals when we register new dex files.
+ std::list<DexCacheData> dex_caches_ GUARDED_BY(dex_lock_);
// This contains the class loaders which have class tables. It is populated by
// InsertClassTableForClassLoader.
diff --git a/runtime/common_runtime_test.cc b/runtime/common_runtime_test.cc
index b6b5141..f705a50 100644
--- a/runtime/common_runtime_test.cc
+++ b/runtime/common_runtime_test.cc
@@ -553,7 +553,8 @@
Thread* self = Thread::Current();
jobject class_loader = Runtime::Current()->GetClassLinker()->CreatePathClassLoader(self,
- class_path);
+ class_path,
+ nullptr);
self->SetClassLoaderOverride(class_loader);
return class_loader;
}
diff --git a/runtime/common_runtime_test.h b/runtime/common_runtime_test.h
index a474ae6..6da2bef 100644
--- a/runtime/common_runtime_test.h
+++ b/runtime/common_runtime_test.h
@@ -174,13 +174,6 @@
DISALLOW_COPY_AND_ASSIGN(CheckJniAbortCatcher);
};
-// TODO: When read barrier works with the compiler, get rid of this.
-#define TEST_DISABLED_FOR_READ_BARRIER() \
- if (kUseReadBarrier) { \
- printf("WARNING: TEST DISABLED FOR READ BARRIER\n"); \
- return; \
- }
-
#define TEST_DISABLED_FOR_MIPS() \
if (kRuntimeISA == kMips) { \
printf("WARNING: TEST DISABLED FOR MIPS\n"); \
diff --git a/runtime/debugger.cc b/runtime/debugger.cc
index 13d0b84..32e77b7 100644
--- a/runtime/debugger.cc
+++ b/runtime/debugger.cc
@@ -2684,26 +2684,26 @@
case JDWP::JT_BOOLEAN:
case JDWP::JT_BYTE:
CHECK_EQ(width, 1U);
- if (!visitor.SetVRegFromDebugger(m, vreg, static_cast<uint32_t>(value), kIntVReg)) {
+ if (!visitor.SetVReg(m, vreg, static_cast<uint32_t>(value), kIntVReg)) {
return FailSetLocalValue(visitor, vreg, tag, static_cast<uint32_t>(value));
}
break;
case JDWP::JT_SHORT:
case JDWP::JT_CHAR:
CHECK_EQ(width, 2U);
- if (!visitor.SetVRegFromDebugger(m, vreg, static_cast<uint32_t>(value), kIntVReg)) {
+ if (!visitor.SetVReg(m, vreg, static_cast<uint32_t>(value), kIntVReg)) {
return FailSetLocalValue(visitor, vreg, tag, static_cast<uint32_t>(value));
}
break;
case JDWP::JT_INT:
CHECK_EQ(width, 4U);
- if (!visitor.SetVRegFromDebugger(m, vreg, static_cast<uint32_t>(value), kIntVReg)) {
+ if (!visitor.SetVReg(m, vreg, static_cast<uint32_t>(value), kIntVReg)) {
return FailSetLocalValue(visitor, vreg, tag, static_cast<uint32_t>(value));
}
break;
case JDWP::JT_FLOAT:
CHECK_EQ(width, 4U);
- if (!visitor.SetVRegFromDebugger(m, vreg, static_cast<uint32_t>(value), kFloatVReg)) {
+ if (!visitor.SetVReg(m, vreg, static_cast<uint32_t>(value), kFloatVReg)) {
return FailSetLocalValue(visitor, vreg, tag, static_cast<uint32_t>(value));
}
break;
@@ -2721,7 +2721,7 @@
VLOG(jdwp) << tag << " object " << o << " is an invalid object";
return JDWP::ERR_INVALID_OBJECT;
}
- if (!visitor.SetVRegFromDebugger(m, vreg, static_cast<uint32_t>(reinterpret_cast<uintptr_t>(o)),
+ if (!visitor.SetVReg(m, vreg, static_cast<uint32_t>(reinterpret_cast<uintptr_t>(o)),
kReferenceVReg)) {
return FailSetLocalValue(visitor, vreg, tag, reinterpret_cast<uintptr_t>(o));
}
@@ -2729,14 +2729,14 @@
}
case JDWP::JT_DOUBLE: {
CHECK_EQ(width, 8U);
- if (!visitor.SetVRegPairFromDebugger(m, vreg, value, kDoubleLoVReg, kDoubleHiVReg)) {
+ if (!visitor.SetVRegPair(m, vreg, value, kDoubleLoVReg, kDoubleHiVReg)) {
return FailSetLocalValue(visitor, vreg, tag, value);
}
break;
}
case JDWP::JT_LONG: {
CHECK_EQ(width, 8U);
- if (!visitor.SetVRegPairFromDebugger(m, vreg, value, kLongLoVReg, kLongHiVReg)) {
+ if (!visitor.SetVRegPair(m, vreg, value, kLongLoVReg, kLongHiVReg)) {
return FailSetLocalValue(visitor, vreg, tag, value);
}
break;
diff --git a/runtime/entrypoints/quick/quick_entrypoints.h b/runtime/entrypoints/quick/quick_entrypoints.h
index 3d3f7a1..27865e3 100644
--- a/runtime/entrypoints/quick/quick_entrypoints.h
+++ b/runtime/entrypoints/quick/quick_entrypoints.h
@@ -31,12 +31,12 @@
namespace mirror {
class Array;
class Class;
+template<class MirrorType> class CompressedReference;
class Object;
-template<class MirrorType>
-class CompressedReference;
} // namespace mirror
class ArtMethod;
+template<class MirrorType> class GcRoot;
class Thread;
// Pointers to functions that are called by quick compiler generated code via thread-local storage.
@@ -72,9 +72,14 @@
Thread* self)
NO_THREAD_SAFETY_ANALYSIS HOT_ATTR;
+
// Read barrier entrypoints.
-// Compilers for ARM, ARM64, MIPS, MIPS64 can insert a call to this function directly.
-// For x86 and x86_64, compilers need a wrapper assembly function, to handle mismatch in ABI.
+//
+// Compilers for ARM, ARM64, MIPS, MIPS64 can insert a call to these
+// functions directly. For x86 and x86-64, compilers need a wrapper
+// assembly function, to handle mismatch in ABI.
+
+// Read barrier entrypoint for heap references.
// This is the read barrier slow path for instance and static fields and reference-type arrays.
// TODO: Currently the read barrier does not have a fast path for compilers to directly generate.
// Ideally the slow path should only take one parameter "ref".
@@ -82,6 +87,10 @@
uint32_t offset)
SHARED_REQUIRES(Locks::mutator_lock_) HOT_ATTR;
+// Read barrier entrypoint for GC roots.
+extern "C" mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root)
+ SHARED_REQUIRES(Locks::mutator_lock_) HOT_ATTR;
+
} // namespace art
#endif // ART_RUNTIME_ENTRYPOINTS_QUICK_QUICK_ENTRYPOINTS_H_
diff --git a/runtime/entrypoints/quick/quick_entrypoints_list.h b/runtime/entrypoints/quick/quick_entrypoints_list.h
index 73d8ae7..ee7b986 100644
--- a/runtime/entrypoints/quick/quick_entrypoints_list.h
+++ b/runtime/entrypoints/quick/quick_entrypoints_list.h
@@ -146,7 +146,8 @@
V(NewStringFromStringBuilder, void) \
\
V(ReadBarrierJni, void, mirror::CompressedReference<mirror::Object>*, Thread*) \
- V(ReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t)
+ V(ReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t) \
+ V(ReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*)
#endif // ART_RUNTIME_ENTRYPOINTS_QUICK_QUICK_ENTRYPOINTS_LIST_H_
#undef ART_RUNTIME_ENTRYPOINTS_QUICK_QUICK_ENTRYPOINTS_LIST_H_ // #define is only for lint.
diff --git a/runtime/entrypoints/quick/quick_field_entrypoints.cc b/runtime/entrypoints/quick/quick_field_entrypoints.cc
index 7361d34..7ec5fc5 100644
--- a/runtime/entrypoints/quick/quick_field_entrypoints.cc
+++ b/runtime/entrypoints/quick/quick_field_entrypoints.cc
@@ -14,14 +14,16 @@
* limitations under the License.
*/
+#include <stdint.h>
+
#include "art_field-inl.h"
#include "art_method-inl.h"
#include "callee_save_frame.h"
#include "dex_file-inl.h"
#include "entrypoints/entrypoint_utils-inl.h"
+#include "gc_root-inl.h"
#include "mirror/class-inl.h"
-
-#include <stdint.h>
+#include "mirror/object_reference.h"
namespace art {
@@ -560,13 +562,25 @@
// TODO: Currently the read barrier does not have a fast path. Ideally the slow path should only
// take one parameter "ref", which is given by the fast path.
extern "C" mirror::Object* artReadBarrierSlow(mirror::Object* ref ATTRIBUTE_UNUSED,
- mirror::Object* obj, uint32_t offset) {
- DCHECK(kUseReadBarrier);
+ mirror::Object* obj,
+ uint32_t offset) {
+ DCHECK(kEmitCompilerReadBarrier);
uint8_t* raw_addr = reinterpret_cast<uint8_t*>(obj) + offset;
mirror::HeapReference<mirror::Object>* ref_addr =
- reinterpret_cast<mirror::HeapReference<mirror::Object>*>(raw_addr);
- return ReadBarrier::Barrier<mirror::Object, kWithReadBarrier, true>(obj, MemberOffset(offset),
- ref_addr);
+ reinterpret_cast<mirror::HeapReference<mirror::Object>*>(raw_addr);
+ constexpr ReadBarrierOption kReadBarrierOption =
+ kUseReadBarrier ? kWithReadBarrier : kWithoutReadBarrier;
+ mirror::Object* result =
+ ReadBarrier::Barrier<mirror::Object, kReadBarrierOption, true>(obj,
+ MemberOffset(offset),
+ ref_addr);
+ return result;
+}
+
+extern "C" mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root) {
+ DCHECK(kEmitCompilerReadBarrier);
+ // TODO: Pass a GcRootSource object as second argument to GcRoot::Read?
+ return root->Read();
}
} // namespace art
diff --git a/runtime/entrypoints_order_test.cc b/runtime/entrypoints_order_test.cc
index 78f56ee..8587ede 100644
--- a/runtime/entrypoints_order_test.cc
+++ b/runtime/entrypoints_order_test.cc
@@ -302,8 +302,10 @@
EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pNewStringFromStringBuilder, pReadBarrierJni,
sizeof(void*));
EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierJni, pReadBarrierSlow, sizeof(void*));
+ EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierSlow, pReadBarrierForRootSlow,
+ sizeof(void*));
- CHECKED(OFFSETOF_MEMBER(QuickEntryPoints, pReadBarrierSlow)
+ CHECKED(OFFSETOF_MEMBER(QuickEntryPoints, pReadBarrierForRootSlow)
+ sizeof(void*) == sizeof(QuickEntryPoints), QuickEntryPoints_all);
}
};
diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc
index f4cf3ae..bcfcb89 100644
--- a/runtime/gc/collector/concurrent_copying.cc
+++ b/runtime/gc/collector/concurrent_copying.cc
@@ -134,10 +134,10 @@
WriterMutexLock mu(self, *Locks::heap_bitmap_lock_);
// Mark all of the spaces we never collect as immune.
for (const auto& space : heap_->GetContinuousSpaces()) {
- if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect
- || space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect) {
+ if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect ||
+ space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect) {
CHECK(space->IsZygoteSpace() || space->IsImageSpace());
- CHECK(immune_region_.AddContinuousSpace(space)) << "Failed to add space " << *space;
+ immune_spaces_.AddSpace(space);
const char* bitmap_name = space->IsImageSpace() ? "cc image space bitmap" :
"cc zygote space bitmap";
// TODO: try avoiding using bitmaps for image/zygote to save space.
@@ -164,7 +164,7 @@
<< reinterpret_cast<void*>(region_space_->Limit());
}
CheckEmptyMarkStack();
- immune_region_.Reset();
+ immune_spaces_.Reset();
bytes_moved_.StoreRelaxed(0);
objects_moved_.StoreRelaxed(0);
if (GetCurrentIteration()->GetGcCause() == kGcCauseExplicit ||
@@ -177,7 +177,11 @@
BindBitmaps();
if (kVerboseMode) {
LOG(INFO) << "force_evacuate_all=" << force_evacuate_all_;
- LOG(INFO) << "Immune region: " << immune_region_.Begin() << "-" << immune_region_.End();
+ LOG(INFO) << "Largest immune region: " << immune_spaces_.GetLargestImmuneRegion().Begin()
+ << "-" << immune_spaces_.GetLargestImmuneRegion().End();
+ for (space::ContinuousSpace* space : immune_spaces_.GetSpaces()) {
+ LOG(INFO) << "Immune space: " << *space;
+ }
LOG(INFO) << "GC end of InitializePhase";
}
}
@@ -300,7 +304,7 @@
void operator()(mirror::Object* obj) const SHARED_REQUIRES(Locks::mutator_lock_)
SHARED_REQUIRES(Locks::heap_bitmap_lock_) {
DCHECK(obj != nullptr);
- DCHECK(collector_->immune_region_.ContainsObject(obj));
+ DCHECK(collector_->immune_spaces_.ContainsObject(obj));
accounting::ContinuousSpaceBitmap* cc_bitmap =
collector_->cc_heap_bitmap_->GetContinuousSpaceBitmap(obj);
DCHECK(cc_bitmap != nullptr)
@@ -383,15 +387,13 @@
}
// Immune spaces.
- for (auto& space : heap_->GetContinuousSpaces()) {
- if (immune_region_.ContainsSpace(space)) {
- DCHECK(space->IsImageSpace() || space->IsZygoteSpace());
- accounting::ContinuousSpaceBitmap* live_bitmap = space->GetLiveBitmap();
- ConcurrentCopyingImmuneSpaceObjVisitor visitor(this);
- live_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()),
- reinterpret_cast<uintptr_t>(space->Limit()),
- visitor);
- }
+ for (auto& space : immune_spaces_.GetSpaces()) {
+ DCHECK(space->IsImageSpace() || space->IsZygoteSpace());
+ accounting::ContinuousSpaceBitmap* live_bitmap = space->GetLiveBitmap();
+ ConcurrentCopyingImmuneSpaceObjVisitor visitor(this);
+ live_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()),
+ reinterpret_cast<uintptr_t>(space->Limit()),
+ visitor);
}
Thread* self = Thread::Current();
@@ -1078,7 +1080,7 @@
!IsInToSpace(to_ref->AsReference()->GetReferent<kWithoutReadBarrier>())))) {
// Leave this Reference gray in the queue so that GetReferent() will trigger a read barrier. We
// will change it to black or white later in ReferenceQueue::DequeuePendingReference().
- CHECK(to_ref->AsReference()->IsEnqueued()) << "Left unenqueued ref gray " << to_ref;
+ DCHECK(to_ref->AsReference()->IsEnqueued()) << "Left unenqueued ref gray " << to_ref;
} else {
// We may occasionally leave a Reference black or white in the queue if its referent happens to
// be concurrently marked after the Scan() call above has enqueued the Reference, in which case
@@ -1087,9 +1089,10 @@
if (kUseBakerReadBarrier) {
if (region_space_->IsInToSpace(to_ref)) {
// If to-space, change from gray to white.
- bool success = to_ref->AtomicSetReadBarrierPointer(ReadBarrier::GrayPtr(),
- ReadBarrier::WhitePtr());
- CHECK(success) << "Must succeed as we won the race.";
+ bool success = to_ref->AtomicSetReadBarrierPointer</*kCasRelease*/true>(
+ ReadBarrier::GrayPtr(),
+ ReadBarrier::WhitePtr());
+ DCHECK(success) << "Must succeed as we won the race.";
DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::WhitePtr());
} else {
// If non-moving space/unevac from space, change from gray
@@ -1099,9 +1102,10 @@
// indicate non-moving objects that have been marked
// through. Note we'd need to change from black to white
// later (concurrently).
- bool success = to_ref->AtomicSetReadBarrierPointer(ReadBarrier::GrayPtr(),
- ReadBarrier::BlackPtr());
- CHECK(success) << "Must succeed as we won the race.";
+ bool success = to_ref->AtomicSetReadBarrierPointer</*kCasRelease*/true>(
+ ReadBarrier::GrayPtr(),
+ ReadBarrier::BlackPtr());
+ DCHECK(success) << "Must succeed as we won the race.";
DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::BlackPtr());
}
}
@@ -1205,7 +1209,7 @@
for (const auto& space : GetHeap()->GetContinuousSpaces()) {
if (space->IsContinuousMemMapAllocSpace()) {
space::ContinuousMemMapAllocSpace* alloc_space = space->AsContinuousMemMapAllocSpace();
- if (space == region_space_ || immune_region_.ContainsSpace(space)) {
+ if (space == region_space_ || immune_spaces_.ContainsSpace(space)) {
continue;
}
TimingLogger::ScopedTiming split2(
@@ -1225,9 +1229,6 @@
public:
explicit ConcurrentCopyingClearBlackPtrsVisitor(ConcurrentCopying* cc)
: collector_(cc) {}
-#ifndef USE_BAKER_OR_BROOKS_READ_BARRIER
- NO_RETURN
-#endif
void operator()(mirror::Object* obj) const SHARED_REQUIRES(Locks::mutator_lock_)
SHARED_REQUIRES(Locks::heap_bitmap_lock_) {
DCHECK(obj != nullptr);
@@ -1507,8 +1508,8 @@
}
} else {
// In a non-moving space.
- if (immune_region_.ContainsObject(obj)) {
- LOG(INFO) << "holder is in the image or the zygote space.";
+ if (immune_spaces_.ContainsObject(obj)) {
+ LOG(INFO) << "holder is in an immune image or the zygote space.";
accounting::ContinuousSpaceBitmap* cc_bitmap =
cc_heap_bitmap_->GetContinuousSpaceBitmap(obj);
CHECK(cc_bitmap != nullptr)
@@ -1519,7 +1520,7 @@
LOG(INFO) << "holder is NOT marked in the bit map.";
}
} else {
- LOG(INFO) << "holder is in a non-moving (or main) space.";
+ LOG(INFO) << "holder is in a non-immune, non-moving (or main) space.";
accounting::ContinuousSpaceBitmap* mark_bitmap =
heap_mark_bitmap_->GetContinuousSpaceBitmap(obj);
accounting::LargeObjectBitmap* los_bitmap =
@@ -1547,7 +1548,7 @@
void ConcurrentCopying::AssertToSpaceInvariantInNonMovingSpace(mirror::Object* obj,
mirror::Object* ref) {
// In a non-moving spaces. Check that the ref is marked.
- if (immune_region_.ContainsObject(ref)) {
+ if (immune_spaces_.ContainsObject(ref)) {
accounting::ContinuousSpaceBitmap* cc_bitmap =
cc_heap_bitmap_->GetContinuousSpaceBitmap(ref);
CHECK(cc_bitmap != nullptr)
@@ -1932,7 +1933,7 @@
}
} else {
// from_ref is in a non-moving space.
- if (immune_region_.ContainsObject(from_ref)) {
+ if (immune_spaces_.ContainsObject(from_ref)) {
accounting::ContinuousSpaceBitmap* cc_bitmap =
cc_heap_bitmap_->GetContinuousSpaceBitmap(from_ref);
DCHECK(cc_bitmap != nullptr)
@@ -1986,7 +1987,7 @@
mirror::Object* ConcurrentCopying::MarkNonMoving(mirror::Object* ref) {
// ref is in a non-moving space (from_ref == to_ref).
DCHECK(!region_space_->HasAddress(ref)) << ref;
- if (immune_region_.ContainsObject(ref)) {
+ if (immune_spaces_.ContainsObject(ref)) {
accounting::ContinuousSpaceBitmap* cc_bitmap =
cc_heap_bitmap_->GetContinuousSpaceBitmap(ref);
DCHECK(cc_bitmap != nullptr)
diff --git a/runtime/gc/collector/concurrent_copying.h b/runtime/gc/collector/concurrent_copying.h
index 27726e2..5d21c59 100644
--- a/runtime/gc/collector/concurrent_copying.h
+++ b/runtime/gc/collector/concurrent_copying.h
@@ -19,7 +19,7 @@
#include "barrier.h"
#include "garbage_collector.h"
-#include "immune_region.h"
+#include "immune_spaces.h"
#include "jni.h"
#include "object_callbacks.h"
#include "offsets.h"
@@ -200,7 +200,7 @@
bool is_marking_; // True while marking is ongoing.
bool is_active_; // True while the collection is ongoing.
bool is_asserting_to_space_invariant_; // True while asserting the to-space invariant.
- ImmuneRegion immune_region_;
+ ImmuneSpaces immune_spaces_;
std::unique_ptr<accounting::HeapBitmap> cc_heap_bitmap_;
std::vector<accounting::SpaceBitmap<kObjectAlignment>*> cc_bitmaps_;
accounting::SpaceBitmap<kObjectAlignment>* region_space_bitmap_;
diff --git a/runtime/gc/collector/immune_region.cc b/runtime/gc/collector/immune_region.cc
index 3e1c944..8a04c17 100644
--- a/runtime/gc/collector/immune_region.cc
+++ b/runtime/gc/collector/immune_region.cc
@@ -32,39 +32,6 @@
SetEnd(nullptr);
}
-bool ImmuneRegion::AddContinuousSpace(space::ContinuousSpace* space) {
- // Bind live to mark bitmap if necessary.
- if (space->GetLiveBitmap() != space->GetMarkBitmap()) {
- CHECK(space->IsContinuousMemMapAllocSpace());
- space->AsContinuousMemMapAllocSpace()->BindLiveToMarkBitmap();
- }
- mirror::Object* space_begin = reinterpret_cast<mirror::Object*>(space->Begin());
- mirror::Object* space_limit = reinterpret_cast<mirror::Object*>(space->Limit());
- if (IsEmpty()) {
- SetBegin(space_begin);
- SetEnd(space_limit);
- } else {
- if (space_limit <= begin_) { // Space is before the immune region.
- SetBegin(space_begin);
- } else if (space_begin >= end_) { // Space is after the immune region.
- SetEnd(space_limit);
- } else {
- return false;
- }
- }
- return true;
-}
-
-bool ImmuneRegion::ContainsSpace(const space::ContinuousSpace* space) const {
- bool contains =
- begin_ <= reinterpret_cast<mirror::Object*>(space->Begin()) &&
- end_ >= reinterpret_cast<mirror::Object*>(space->Limit());
- if (kIsDebugBuild && contains) {
- // A bump pointer space shoult not be in the immune region.
- DCHECK(space->GetType() != space::kSpaceTypeBumpPointerSpace);
- }
- return contains;
-}
} // namespace collector
} // namespace gc
diff --git a/runtime/gc/collector/immune_region.h b/runtime/gc/collector/immune_region.h
index 3ead501..b60426d 100644
--- a/runtime/gc/collector/immune_region.h
+++ b/runtime/gc/collector/immune_region.h
@@ -39,35 +39,34 @@
class ImmuneRegion {
public:
ImmuneRegion();
+
void Reset();
- bool AddContinuousSpace(space::ContinuousSpace* space)
- REQUIRES(Locks::heap_bitmap_lock_);
- bool ContainsSpace(const space::ContinuousSpace* space) const;
+
// Returns true if an object is inside of the immune region (assumed to be marked).
- bool ContainsObject(const mirror::Object* obj) const ALWAYS_INLINE {
+ ALWAYS_INLINE bool ContainsObject(const mirror::Object* obj) const {
// Note: Relies on integer underflow behavior.
return reinterpret_cast<uintptr_t>(obj) - reinterpret_cast<uintptr_t>(begin_) < size_;
}
+
void SetBegin(mirror::Object* begin) {
begin_ = begin;
UpdateSize();
}
+
void SetEnd(mirror::Object* end) {
end_ = end;
UpdateSize();
}
- mirror::Object* Begin() {
+ mirror::Object* Begin() const {
return begin_;
}
- mirror::Object* End() {
+
+ mirror::Object* End() const {
return end_;
}
private:
- bool IsEmpty() const {
- return size_ == 0;
- }
void UpdateSize() {
size_ = reinterpret_cast<uintptr_t>(end_) - reinterpret_cast<uintptr_t>(begin_);
}
diff --git a/runtime/gc/collector/immune_spaces.cc b/runtime/gc/collector/immune_spaces.cc
new file mode 100644
index 0000000..8f9a9e2
--- /dev/null
+++ b/runtime/gc/collector/immune_spaces.cc
@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "immune_spaces.h"
+
+#include "gc/space/space-inl.h"
+#include "mirror/object.h"
+
+namespace art {
+namespace gc {
+namespace collector {
+
+void ImmuneSpaces::Reset() {
+ spaces_.clear();
+ largest_immune_region_.Reset();
+}
+
+void ImmuneSpaces::CreateLargestImmuneRegion() {
+ uintptr_t best_begin = 0u;
+ uintptr_t best_end = 0u;
+ uintptr_t cur_begin = 0u;
+ uintptr_t cur_end = 0u;
+ // TODO: If the last space is an image space, we may include its oat file in the immune region.
+ // This could potentially hide heap corruption bugs if there is invalid pointers that point into
+ // the boot oat code
+ for (space::ContinuousSpace* space : GetSpaces()) {
+ uintptr_t space_begin = reinterpret_cast<uintptr_t>(space->Begin());
+ uintptr_t space_end = reinterpret_cast<uintptr_t>(space->Limit());
+ if (space->IsImageSpace()) {
+ // For the boot image, the boot oat file is always directly after. For app images it may not
+ // be if the app image was mapped at a random address.
+ space::ImageSpace* image_space = space->AsImageSpace();
+ // Update the end to include the other non-heap sections.
+ space_end = RoundUp(reinterpret_cast<uintptr_t>(image_space->GetImageEnd()), kPageSize);
+ uintptr_t oat_begin = reinterpret_cast<uintptr_t>(image_space->GetOatFileBegin());
+ uintptr_t oat_end = reinterpret_cast<uintptr_t>(image_space->GetOatFileEnd());
+ if (space_end == oat_begin) {
+ DCHECK_GE(oat_end, oat_begin);
+ space_end = oat_end;
+ }
+ }
+ if (cur_begin == 0u) {
+ cur_begin = space_begin;
+ cur_end = space_end;
+ } else if (cur_end == space_begin) {
+ // Extend current region.
+ cur_end = space_end;
+ } else {
+ // Reset.
+ cur_begin = 0;
+ cur_end = 0;
+ }
+ if (cur_end - cur_begin > best_end - best_begin) {
+ // Improvement, update the best range.
+ best_begin = cur_begin;
+ best_end = cur_end;
+ }
+ }
+ largest_immune_region_.SetBegin(reinterpret_cast<mirror::Object*>(best_begin));
+ largest_immune_region_.SetEnd(reinterpret_cast<mirror::Object*>(best_end));
+}
+
+void ImmuneSpaces::AddSpace(space::ContinuousSpace* space) {
+ DCHECK(spaces_.find(space) == spaces_.end()) << *space;
+ // Bind live to mark bitmap if necessary.
+ if (space->GetLiveBitmap() != space->GetMarkBitmap()) {
+ CHECK(space->IsContinuousMemMapAllocSpace());
+ space->AsContinuousMemMapAllocSpace()->BindLiveToMarkBitmap();
+ }
+ spaces_.insert(space);
+ CreateLargestImmuneRegion();
+}
+
+bool ImmuneSpaces::CompareByBegin::operator()(space::ContinuousSpace* a, space::ContinuousSpace* b)
+ const {
+ return a->Begin() < b->Begin();
+}
+
+bool ImmuneSpaces::ContainsSpace(space::ContinuousSpace* space) const {
+ return spaces_.find(space) != spaces_.end();
+}
+
+} // namespace collector
+} // namespace gc
+} // namespace art
diff --git a/runtime/gc/collector/immune_spaces.h b/runtime/gc/collector/immune_spaces.h
new file mode 100644
index 0000000..72cb60d
--- /dev/null
+++ b/runtime/gc/collector/immune_spaces.h
@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_RUNTIME_GC_COLLECTOR_IMMUNE_SPACES_H_
+#define ART_RUNTIME_GC_COLLECTOR_IMMUNE_SPACES_H_
+
+#include "base/macros.h"
+#include "base/mutex.h"
+#include "gc/space/space.h"
+#include "immune_region.h"
+
+#include <set>
+
+namespace art {
+namespace gc {
+namespace space {
+class ContinuousSpace;
+} // namespace space
+
+namespace collector {
+
+// ImmuneSpaces is a set of spaces which are not going to have any objects become marked during the
+// GC.
+class ImmuneSpaces {
+ class CompareByBegin {
+ public:
+ bool operator()(space::ContinuousSpace* a, space::ContinuousSpace* b) const;
+ };
+
+ public:
+ ImmuneSpaces() {}
+ void Reset();
+
+ // Add a continuous space to the immune spaces set.
+ void AddSpace(space::ContinuousSpace* space) REQUIRES(Locks::heap_bitmap_lock_);
+
+ // Returns true if an object is inside of the immune region (assumed to be marked). Only returns
+ // true for the largest immune region. The object can still be inside of an immune space.
+ ALWAYS_INLINE bool IsInImmuneRegion(const mirror::Object* obj) const {
+ return largest_immune_region_.ContainsObject(obj);
+ }
+
+ // Return true if the spaces is contained.
+ bool ContainsSpace(space::ContinuousSpace* space) const;
+
+ // Return the set of spaces in the immune region.
+ const std::set<space::ContinuousSpace*, CompareByBegin>& GetSpaces() {
+ return spaces_;
+ }
+
+ // Return the associated largest immune region.
+ const ImmuneRegion& GetLargestImmuneRegion() const {
+ return largest_immune_region_;
+ }
+
+ // Return true if the object is contained by any of the immune space.s
+ ALWAYS_INLINE bool ContainsObject(const mirror::Object* obj) const {
+ if (largest_immune_region_.ContainsObject(obj)) {
+ return true;
+ }
+ for (space::ContinuousSpace* space : spaces_) {
+ if (space->HasAddress(obj)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private:
+ // Setup the immune region to the largest continuous set of immune spaces. The immune region is
+ // just the for the fast path lookup.
+ void CreateLargestImmuneRegion();
+
+ std::set<space::ContinuousSpace*, CompareByBegin> spaces_;
+ ImmuneRegion largest_immune_region_;
+};
+
+} // namespace collector
+} // namespace gc
+} // namespace art
+
+#endif // ART_RUNTIME_GC_COLLECTOR_IMMUNE_SPACES_H_
diff --git a/runtime/gc/collector/immune_spaces_test.cc b/runtime/gc/collector/immune_spaces_test.cc
new file mode 100644
index 0000000..f741117
--- /dev/null
+++ b/runtime/gc/collector/immune_spaces_test.cc
@@ -0,0 +1,150 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common_runtime_test.h"
+#include "gc/collector/immune_spaces.h"
+#include "gc/space/image_space.h"
+#include "gc/space/space-inl.h"
+#include "oat_file.h"
+#include "thread-inl.h"
+
+namespace art {
+namespace mirror {
+class Object;
+} // namespace mirror
+namespace gc {
+namespace collector {
+
+class ImmuneSpacesTest : public CommonRuntimeTest {};
+
+class DummySpace : public space::ContinuousSpace {
+ public:
+ DummySpace(uint8_t* begin, uint8_t* end)
+ : ContinuousSpace("DummySpace",
+ space::kGcRetentionPolicyNeverCollect,
+ begin,
+ end,
+ /*limit*/end) {}
+
+ space::SpaceType GetType() const OVERRIDE {
+ return space::kSpaceTypeMallocSpace;
+ }
+
+ bool CanMoveObjects() const OVERRIDE {
+ return false;
+ }
+
+ accounting::ContinuousSpaceBitmap* GetLiveBitmap() const OVERRIDE {
+ return nullptr;
+ }
+
+ accounting::ContinuousSpaceBitmap* GetMarkBitmap() const OVERRIDE {
+ return nullptr;
+ }
+};
+
+TEST_F(ImmuneSpacesTest, AppendBasic) {
+ ImmuneSpaces spaces;
+ uint8_t* const base = reinterpret_cast<uint8_t*>(0x1000);
+ DummySpace a(base, base + 45 * KB);
+ DummySpace b(a.Limit(), a.Limit() + 813 * KB);
+ {
+ WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
+ spaces.AddSpace(&a);
+ spaces.AddSpace(&b);
+ }
+ EXPECT_TRUE(spaces.ContainsSpace(&a));
+ EXPECT_TRUE(spaces.ContainsSpace(&b));
+ EXPECT_EQ(reinterpret_cast<uint8_t*>(spaces.GetLargestImmuneRegion().Begin()), a.Begin());
+ EXPECT_EQ(reinterpret_cast<uint8_t*>(spaces.GetLargestImmuneRegion().End()), b.Limit());
+}
+
+class DummyImageSpace : public space::ImageSpace {
+ public:
+ DummyImageSpace(MemMap* map, accounting::ContinuousSpaceBitmap* live_bitmap)
+ : ImageSpace("DummyImageSpace",
+ /*image_location*/"",
+ map,
+ live_bitmap,
+ map->End()) {}
+
+ // OatSize is how large the oat file is after the image.
+ static DummyImageSpace* Create(size_t size, size_t oat_size) {
+ std::string error_str;
+ std::unique_ptr<MemMap> map(MemMap::MapAnonymous("DummyImageSpace",
+ nullptr,
+ size,
+ PROT_READ | PROT_WRITE,
+ /*low_4gb*/true,
+ /*reuse*/false,
+ &error_str));
+ if (map == nullptr) {
+ LOG(ERROR) << error_str;
+ return nullptr;
+ }
+ std::unique_ptr<accounting::ContinuousSpaceBitmap> live_bitmap(
+ accounting::ContinuousSpaceBitmap::Create("bitmap", map->Begin(), map->Size()));
+ if (live_bitmap == nullptr) {
+ return nullptr;
+ }
+ // Create image header.
+ ImageSection sections[ImageHeader::kSectionCount];
+ new (map->Begin()) ImageHeader(
+ /*image_begin*/PointerToLowMemUInt32(map->Begin()),
+ /*image_size*/map->Size(),
+ sections,
+ /*image_roots*/PointerToLowMemUInt32(map->Begin()) + 1,
+ /*oat_checksum*/0u,
+ /*oat_file_begin*/PointerToLowMemUInt32(map->End()),
+ /*oat_data_begin*/PointerToLowMemUInt32(map->End()),
+ /*oat_data_end*/PointerToLowMemUInt32(map->End() + oat_size),
+ /*oat_file_end*/PointerToLowMemUInt32(map->End() + oat_size),
+ /*pointer_size*/sizeof(void*),
+ /*compile_pic*/false);
+ return new DummyImageSpace(map.release(), live_bitmap.release());
+ }
+};
+
+TEST_F(ImmuneSpacesTest, AppendAfterImage) {
+ ImmuneSpaces spaces;
+ constexpr size_t image_size = 123 * kPageSize;
+ constexpr size_t image_oat_size = 321 * kPageSize;
+ std::unique_ptr<DummyImageSpace> image_space(DummyImageSpace::Create(image_size, image_oat_size));
+ ASSERT_TRUE(image_space != nullptr);
+ const ImageHeader& image_header = image_space->GetImageHeader();
+ EXPECT_EQ(image_header.GetImageSize(), image_size);
+ EXPECT_EQ(static_cast<size_t>(image_header.GetOatFileEnd() - image_header.GetOatFileBegin()),
+ image_oat_size);
+ DummySpace space(image_header.GetOatFileEnd(), image_header.GetOatFileEnd() + 813 * kPageSize);
+ EXPECT_NE(image_space->Limit(), space.Begin());
+ {
+ WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
+ spaces.AddSpace(image_space.get());
+ spaces.AddSpace(&space);
+ }
+ EXPECT_TRUE(spaces.ContainsSpace(image_space.get()));
+ EXPECT_TRUE(spaces.ContainsSpace(&space));
+ // CreateLargestImmuneRegion should have coalesced the two spaces since the oat code after the
+ // image prevents gaps.
+ // Check that we have a continuous region.
+ EXPECT_EQ(reinterpret_cast<uint8_t*>(spaces.GetLargestImmuneRegion().Begin()),
+ image_space->Begin());
+ EXPECT_EQ(reinterpret_cast<uint8_t*>(spaces.GetLargestImmuneRegion().End()), space.Limit());
+}
+
+} // namespace collector
+} // namespace gc
+} // namespace art
diff --git a/runtime/gc/collector/mark_compact.cc b/runtime/gc/collector/mark_compact.cc
index f561764..ce6467a 100644
--- a/runtime/gc/collector/mark_compact.cc
+++ b/runtime/gc/collector/mark_compact.cc
@@ -45,7 +45,7 @@
for (const auto& space : GetHeap()->GetContinuousSpaces()) {
if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect ||
space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect) {
- CHECK(immune_region_.AddContinuousSpace(space)) << "Failed to add space " << *space;
+ immune_spaces_.AddSpace(space);
}
}
}
@@ -115,7 +115,7 @@
TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
mark_stack_ = heap_->GetMarkStack();
DCHECK(mark_stack_ != nullptr);
- immune_region_.Reset();
+ immune_spaces_.Reset();
CHECK(space_->CanMoveObjects()) << "Attempting compact non-movable space from " << *space_;
// TODO: I don't think we should need heap bitmap lock to Get the mark bitmap.
ReaderMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
@@ -148,7 +148,7 @@
// Verify all the objects have the correct forward pointer installed.
obj->AssertReadBarrierPointer();
}
- if (!immune_region_.ContainsObject(obj)) {
+ if (!immune_spaces_.IsInImmuneRegion(obj)) {
if (objects_before_forwarding_->HasAddress(obj)) {
if (!objects_before_forwarding_->Set(obj)) {
MarkStackPush(obj); // This object was not previously marked.
@@ -218,7 +218,7 @@
TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
for (auto& space : heap_->GetContinuousSpaces()) {
// If the space is immune then we need to mark the references to other spaces.
- if (immune_region_.ContainsSpace(space)) {
+ if (immune_spaces_.ContainsSpace(space)) {
accounting::ModUnionTable* table = heap_->FindModUnionTableFromSpace(space);
if (table != nullptr) {
// TODO: Improve naming.
@@ -475,7 +475,7 @@
}
mirror::Object* MarkCompact::IsMarked(mirror::Object* object) {
- if (immune_region_.ContainsObject(object)) {
+ if (immune_spaces_.IsInImmuneRegion(object)) {
return object;
}
if (updating_references_) {
@@ -498,7 +498,7 @@
}
bool MarkCompact::ShouldSweepSpace(space::ContinuousSpace* space) const {
- return space != space_ && !immune_region_.ContainsSpace(space);
+ return space != space_ && !immune_spaces_.ContainsSpace(space);
}
class MoveObjectVisitor {
diff --git a/runtime/gc/collector/mark_compact.h b/runtime/gc/collector/mark_compact.h
index 8d91939..8a12094 100644
--- a/runtime/gc/collector/mark_compact.h
+++ b/runtime/gc/collector/mark_compact.h
@@ -26,7 +26,7 @@
#include "garbage_collector.h"
#include "gc_root.h"
#include "gc/accounting/heap_bitmap.h"
-#include "immune_region.h"
+#include "immune_spaces.h"
#include "lock_word.h"
#include "object_callbacks.h"
#include "offsets.h"
@@ -194,8 +194,8 @@
accounting::ObjectStack* mark_stack_;
- // Immune region, every object inside the immune region is assumed to be marked.
- ImmuneRegion immune_region_;
+ // Every object inside the immune spaces is assumed to be marked.
+ ImmuneSpaces immune_spaces_;
// Bump pointer space which we are collecting.
space::BumpPointerSpace* space_;
diff --git a/runtime/gc/collector/mark_sweep.cc b/runtime/gc/collector/mark_sweep.cc
index db516a0..5427f88 100644
--- a/runtime/gc/collector/mark_sweep.cc
+++ b/runtime/gc/collector/mark_sweep.cc
@@ -86,7 +86,7 @@
// Mark all of the spaces we never collect as immune.
for (const auto& space : GetHeap()->GetContinuousSpaces()) {
if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect) {
- CHECK(immune_region_.AddContinuousSpace(space)) << "Failed to add space " << *space;
+ immune_spaces_.AddSpace(space);
}
}
}
@@ -115,7 +115,7 @@
TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
mark_stack_ = heap_->GetMarkStack();
DCHECK(mark_stack_ != nullptr);
- immune_region_.Reset();
+ immune_spaces_.Reset();
no_reference_class_count_.StoreRelaxed(0);
normal_count_.StoreRelaxed(0);
class_count_.StoreRelaxed(0);
@@ -268,16 +268,41 @@
PreCleanCards();
}
+class ScanObjectVisitor {
+ public:
+ explicit ScanObjectVisitor(MarkSweep* const mark_sweep) ALWAYS_INLINE
+ : mark_sweep_(mark_sweep) {}
+
+ void operator()(mirror::Object* obj) const
+ ALWAYS_INLINE
+ REQUIRES(Locks::heap_bitmap_lock_)
+ SHARED_REQUIRES(Locks::mutator_lock_) {
+ if (kCheckLocks) {
+ Locks::mutator_lock_->AssertSharedHeld(Thread::Current());
+ Locks::heap_bitmap_lock_->AssertExclusiveHeld(Thread::Current());
+ }
+ mark_sweep_->ScanObject(obj);
+ }
+
+ private:
+ MarkSweep* const mark_sweep_;
+};
+
void MarkSweep::UpdateAndMarkModUnion() {
- for (const auto& space : heap_->GetContinuousSpaces()) {
- if (immune_region_.ContainsSpace(space)) {
- const char* name = space->IsZygoteSpace()
- ? "UpdateAndMarkZygoteModUnionTable"
- : "UpdateAndMarkImageModUnionTable";
- TimingLogger::ScopedTiming t(name, GetTimings());
- accounting::ModUnionTable* mod_union_table = heap_->FindModUnionTableFromSpace(space);
- CHECK(mod_union_table != nullptr);
+ for (const auto& space : immune_spaces_.GetSpaces()) {
+ const char* name = space->IsZygoteSpace()
+ ? "UpdateAndMarkZygoteModUnionTable"
+ : "UpdateAndMarkImageModUnionTable";
+ DCHECK(space->IsZygoteSpace() || space->IsImageSpace()) << *space;
+ TimingLogger::ScopedTiming t(name, GetTimings());
+ accounting::ModUnionTable* mod_union_table = heap_->FindModUnionTableFromSpace(space);
+ if (mod_union_table != nullptr) {
mod_union_table->UpdateAndMarkReferences(this);
+ } else {
+ // No mod-union table, scan all the live bits. This can only occur for app images.
+ space->GetLiveBitmap()->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()),
+ reinterpret_cast<uintptr_t>(space->End()),
+ ScanObjectVisitor(this));
}
}
}
@@ -460,7 +485,7 @@
// Verify all the objects have the correct pointer installed.
obj->AssertReadBarrierPointer();
}
- if (immune_region_.ContainsObject(obj)) {
+ if (immune_spaces_.IsInImmuneRegion(obj)) {
if (kCountMarkedObjects) {
++mark_immune_count_;
}
@@ -501,7 +526,7 @@
// Verify all the objects have the correct pointer installed.
obj->AssertReadBarrierPointer();
}
- if (immune_region_.ContainsObject(obj)) {
+ if (immune_spaces_.IsInImmuneRegion(obj)) {
DCHECK(IsMarked(obj) != nullptr);
return false;
}
@@ -606,26 +631,6 @@
this, static_cast<VisitRootFlags>(flags | kVisitRootFlagNonMoving));
}
-class ScanObjectVisitor {
- public:
- explicit ScanObjectVisitor(MarkSweep* const mark_sweep) ALWAYS_INLINE
- : mark_sweep_(mark_sweep) {}
-
- void operator()(mirror::Object* obj) const
- ALWAYS_INLINE
- REQUIRES(Locks::heap_bitmap_lock_)
- SHARED_REQUIRES(Locks::mutator_lock_) {
- if (kCheckLocks) {
- Locks::mutator_lock_->AssertSharedHeld(Thread::Current());
- Locks::heap_bitmap_lock_->AssertExclusiveHeld(Thread::Current());
- }
- mark_sweep_->ScanObject(obj);
- }
-
- private:
- MarkSweep* const mark_sweep_;
-};
-
class DelayReferenceReferentVisitor {
public:
explicit DelayReferenceReferentVisitor(MarkSweep* collector) : collector_(collector) {}
@@ -1193,7 +1198,8 @@
std::vector<space::ContinuousSpace*> sweep_spaces;
space::ContinuousSpace* non_moving_space = nullptr;
for (space::ContinuousSpace* space : heap_->GetContinuousSpaces()) {
- if (space->IsAllocSpace() && !immune_region_.ContainsSpace(space) &&
+ if (space->IsAllocSpace() &&
+ !immune_spaces_.ContainsSpace(space) &&
space->GetLiveBitmap() != nullptr) {
if (space == heap_->GetNonMovingSpace()) {
non_moving_space = space;
@@ -1422,7 +1428,7 @@
}
inline mirror::Object* MarkSweep::IsMarked(mirror::Object* object) {
- if (immune_region_.ContainsObject(object)) {
+ if (immune_spaces_.IsInImmuneRegion(object)) {
return object;
}
if (current_space_bitmap_->HasAddress(object)) {
diff --git a/runtime/gc/collector/mark_sweep.h b/runtime/gc/collector/mark_sweep.h
index 8f7df78..245f96b 100644
--- a/runtime/gc/collector/mark_sweep.h
+++ b/runtime/gc/collector/mark_sweep.h
@@ -26,7 +26,7 @@
#include "garbage_collector.h"
#include "gc_root.h"
#include "gc/accounting/heap_bitmap.h"
-#include "immune_region.h"
+#include "immune_spaces.h"
#include "object_callbacks.h"
#include "offsets.h"
@@ -314,8 +314,9 @@
accounting::ObjectStack* mark_stack_;
- // Immune region, every object inside the immune range is assumed to be marked.
- ImmuneRegion immune_region_;
+ // Every object inside the immune spaces is assumed to be marked. Immune spaces that aren't in the
+ // immune region are handled by the normal marking logic.
+ ImmuneSpaces immune_spaces_;
// Parallel finger.
AtomicInteger atomic_finger_;
diff --git a/runtime/gc/collector/partial_mark_sweep.cc b/runtime/gc/collector/partial_mark_sweep.cc
index 15f782a..9847794 100644
--- a/runtime/gc/collector/partial_mark_sweep.cc
+++ b/runtime/gc/collector/partial_mark_sweep.cc
@@ -39,7 +39,7 @@
for (const auto& space : GetHeap()->GetContinuousSpaces()) {
if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect) {
CHECK(space->IsZygoteSpace());
- CHECK(immune_region_.AddContinuousSpace(space)) << "Failed to add space " << *space;
+ immune_spaces_.AddSpace(space);
}
}
}
diff --git a/runtime/gc/collector/semi_space-inl.h b/runtime/gc/collector/semi_space-inl.h
index 06d20f5..12cf3db 100644
--- a/runtime/gc/collector/semi_space-inl.h
+++ b/runtime/gc/collector/semi_space-inl.h
@@ -74,7 +74,7 @@
MarkStackPush(forward_address);
}
obj_ptr->Assign(forward_address);
- } else if (!collect_from_space_only_ && !immune_region_.ContainsObject(obj)) {
+ } else if (!collect_from_space_only_ && !immune_spaces_.IsInImmuneRegion(obj)) {
BitmapSetSlowPathVisitor visitor(this);
if (!mark_bitmap_->Set(obj, visitor)) {
// This object was not previously marked.
diff --git a/runtime/gc/collector/semi_space.cc b/runtime/gc/collector/semi_space.cc
index 7f57f30..e9497a2 100644
--- a/runtime/gc/collector/semi_space.cc
+++ b/runtime/gc/collector/semi_space.cc
@@ -66,8 +66,9 @@
for (const auto& space : GetHeap()->GetContinuousSpaces()) {
if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect ||
space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect) {
- CHECK(immune_region_.AddContinuousSpace(space)) << "Failed to add space " << *space;
+ immune_spaces_.AddSpace(space);
} else if (space->GetLiveBitmap() != nullptr) {
+ // TODO: We can probably also add this space to the immune region.
if (space == to_space_ || collect_from_space_only_) {
if (collect_from_space_only_) {
// Bind the bitmaps of the main free list space and the non-moving space we are doing a
@@ -144,7 +145,7 @@
TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
mark_stack_ = heap_->GetMarkStack();
DCHECK(mark_stack_ != nullptr);
- immune_region_.Reset();
+ immune_spaces_.Reset();
is_large_object_space_immune_ = false;
saved_bytes_ = 0;
bytes_moved_ = 0;
@@ -376,7 +377,13 @@
<< "generational_=" << generational_ << " "
<< "collect_from_space_only_=" << collect_from_space_only_;
accounting::RememberedSet* rem_set = GetHeap()->FindRememberedSetFromSpace(space);
- CHECK_EQ(rem_set != nullptr, kUseRememberedSet);
+ if (kUseRememberedSet) {
+ // App images currently do not have remembered sets.
+ DCHECK((space->IsImageSpace() && space != heap_->GetBootImageSpace()) ||
+ rem_set != nullptr);
+ } else {
+ DCHECK(rem_set == nullptr);
+ }
if (rem_set != nullptr) {
TimingLogger::ScopedTiming t2("UpdateAndMarkRememberedSet", GetTimings());
rem_set->UpdateAndMarkReferences(from_space_, this);
@@ -767,7 +774,8 @@
if (from_space_->HasAddress(obj)) {
// Returns either the forwarding address or null.
return GetForwardingAddressInFromSpace(obj);
- } else if (collect_from_space_only_ || immune_region_.ContainsObject(obj) ||
+ } else if (collect_from_space_only_ ||
+ immune_spaces_.IsInImmuneRegion(obj) ||
to_space_->HasAddress(obj)) {
return obj; // Already forwarded, must be marked.
}
diff --git a/runtime/gc/collector/semi_space.h b/runtime/gc/collector/semi_space.h
index b9246ca..a905904 100644
--- a/runtime/gc/collector/semi_space.h
+++ b/runtime/gc/collector/semi_space.h
@@ -25,7 +25,7 @@
#include "garbage_collector.h"
#include "gc_root.h"
#include "gc/accounting/heap_bitmap.h"
-#include "immune_region.h"
+#include "immune_spaces.h"
#include "mirror/object_reference.h"
#include "object_callbacks.h"
#include "offsets.h"
@@ -201,8 +201,8 @@
// object.
accounting::ObjectStack* mark_stack_;
- // Immune region, every object inside the immune region is assumed to be marked.
- ImmuneRegion immune_region_;
+ // Every object inside the immune spaces is assumed to be marked.
+ ImmuneSpaces immune_spaces_;
// If true, the large object space is immune.
bool is_large_object_space_immune_;
diff --git a/runtime/gc/space/dlmalloc_space.cc b/runtime/gc/space/dlmalloc_space.cc
index 77f606d..e754a52 100644
--- a/runtime/gc/space/dlmalloc_space.cc
+++ b/runtime/gc/space/dlmalloc_space.cc
@@ -20,6 +20,8 @@
#include "gc/accounting/card_table.h"
#include "gc/accounting/space_bitmap-inl.h"
#include "gc/heap.h"
+#include "jit/jit.h"
+#include "jit/jit_code_cache.h"
#include "memory_tool_malloc_space-inl.h"
#include "mirror/class-inl.h"
#include "mirror/object-inl.h"
@@ -318,10 +320,17 @@
// Implement the dlmalloc morecore callback.
void* ArtDlMallocMoreCore(void* mspace, intptr_t increment) {
- Heap* heap = Runtime::Current()->GetHeap();
+ Runtime* runtime = Runtime::Current();
+ Heap* heap = runtime->GetHeap();
::art::gc::space::DlMallocSpace* dlmalloc_space = heap->GetDlMallocSpace();
// Support for multiple DlMalloc provided by a slow path.
if (UNLIKELY(dlmalloc_space == nullptr || dlmalloc_space->GetMspace() != mspace)) {
+ if (LIKELY(runtime->GetJit() != nullptr)) {
+ jit::JitCodeCache* code_cache = runtime->GetJit()->GetCodeCache();
+ if (code_cache->OwnsSpace(mspace)) {
+ return code_cache->MoreCore(mspace, increment);
+ }
+ }
dlmalloc_space = nullptr;
for (space::ContinuousSpace* space : heap->GetContinuousSpaces()) {
if (space->IsDlMallocSpace()) {
diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc
index 1fe9a03..e2b2431 100644
--- a/runtime/gc/space/image_space.cc
+++ b/runtime/gc/space/image_space.cc
@@ -58,10 +58,7 @@
CHECK_ALIGNED(max_delta, kPageSize);
CHECK_LT(min_delta, max_delta);
- std::default_random_engine generator;
- generator.seed(NanoTime() * getpid());
- std::uniform_int_distribution<int32_t> distribution(min_delta, max_delta);
- int32_t r = distribution(generator);
+ int32_t r = GetRandomNumber<int32_t>(min_delta, max_delta);
if (r % 2 == 0) {
r = RoundUp(r, kPageSize);
} else {
diff --git a/runtime/gc/space/image_space.h b/runtime/gc/space/image_space.h
index 99207426..babd672 100644
--- a/runtime/gc/space/image_space.h
+++ b/runtime/gc/space/image_space.h
@@ -119,7 +119,22 @@
bool* has_data,
bool *is_global_cache);
- private:
+ // Return the end of the image which includes non-heap objects such as ArtMethods and ArtFields.
+ uint8_t* GetImageEnd() const {
+ return Begin() + GetImageHeader().GetImageSize();
+ }
+
+ // Return the start of the associated oat file.
+ uint8_t* GetOatFileBegin() const {
+ return GetImageHeader().GetOatFileBegin();
+ }
+
+ // Return the end of the associated oat file.
+ uint8_t* GetOatFileEnd() const {
+ return GetImageHeader().GetOatFileEnd();
+ }
+
+ protected:
// Tries to initialize an ImageSpace from the given image path,
// returning null on error.
//
@@ -157,6 +172,7 @@
const std::string image_location_;
+ private:
DISALLOW_COPY_AND_ASSIGN(ImageSpace);
};
diff --git a/runtime/gc_root.h b/runtime/gc_root.h
index 477e67b..3734bcc 100644
--- a/runtime/gc_root.h
+++ b/runtime/gc_root.h
@@ -198,7 +198,7 @@
ALWAYS_INLINE GcRoot(MirrorType* ref = nullptr) SHARED_REQUIRES(Locks::mutator_lock_);
private:
- // Root visitors take pointers to root_ and place the min CompressedReference** arrays. We use a
+ // Root visitors take pointers to root_ and place them in CompressedReference** arrays. We use a
// CompressedReference<mirror::Object> here since it violates strict aliasing requirements to
// cast CompressedReference<MirrorType>* to CompressedReference<mirror::Object>*.
mutable mirror::CompressedReference<mirror::Object> root_;
diff --git a/runtime/globals.h b/runtime/globals.h
index 987a94e..e7ea6f3 100644
--- a/runtime/globals.h
+++ b/runtime/globals.h
@@ -87,8 +87,18 @@
#endif
static constexpr bool kUseBakerOrBrooksReadBarrier = kUseBakerReadBarrier || kUseBrooksReadBarrier;
-static constexpr bool kUseReadBarrier = kUseBakerReadBarrier || kUseBrooksReadBarrier ||
- kUseTableLookupReadBarrier;
+static constexpr bool kUseReadBarrier =
+ kUseBakerReadBarrier || kUseBrooksReadBarrier || kUseTableLookupReadBarrier;
+
+// Debugging flag that forces the generation of read barriers, but
+// does not trigger the use of the concurrent copying GC.
+//
+// TODO: Remove this flag when the read barriers compiler
+// instrumentation is completed.
+static constexpr bool kForceReadBarrier = false;
+// TODO: Likewise, remove this flag when kForceReadBarrier is removed
+// and replace it with kUseReadBarrier.
+static constexpr bool kEmitCompilerReadBarrier = kForceReadBarrier || kUseReadBarrier;
// If true, references within the heap are poisoned (negated).
#ifdef USE_HEAP_POISONING
diff --git a/runtime/image.h b/runtime/image.h
index 20e4159..555cf5d 100644
--- a/runtime/image.h
+++ b/runtime/image.h
@@ -84,7 +84,7 @@
image_roots_(0U), pointer_size_(0U), compile_pic_(0) {}
ImageHeader(uint32_t image_begin,
- uint32_t image_size_,
+ uint32_t image_size,
ImageSection* sections,
uint32_t image_roots,
uint32_t oat_checksum,
@@ -93,7 +93,7 @@
uint32_t oat_data_end,
uint32_t oat_file_end,
uint32_t pointer_size,
- bool compile_pic_);
+ bool compile_pic);
bool IsValid() const;
const char* GetMagic() const;
diff --git a/runtime/java_vm_ext.cc b/runtime/java_vm_ext.cc
index b5e28e9..7cc05f7 100644
--- a/runtime/java_vm_ext.cc
+++ b/runtime/java_vm_ext.cc
@@ -56,15 +56,17 @@
class SharedLibrary {
public:
SharedLibrary(JNIEnv* env, Thread* self, const std::string& path, void* handle,
- jobject class_loader)
+ jobject class_loader, void* class_loader_allocator)
: path_(path),
handle_(handle),
needs_native_bridge_(false),
class_loader_(env->NewWeakGlobalRef(class_loader)),
+ class_loader_allocator_(class_loader_allocator),
jni_on_load_lock_("JNI_OnLoad lock"),
jni_on_load_cond_("JNI_OnLoad condition variable", jni_on_load_lock_),
jni_on_load_thread_id_(self->GetThreadId()),
jni_on_load_result_(kPending) {
+ CHECK(class_loader_allocator_ != nullptr);
}
~SharedLibrary() {
@@ -78,6 +80,10 @@
return class_loader_;
}
+ const void* GetClassLoaderAllocator() const {
+ return class_loader_allocator_;
+ }
+
const std::string& GetPath() const {
return path_;
}
@@ -169,6 +175,9 @@
// The ClassLoader this library is associated with, a weak global JNI reference that is
// created/deleted with the scope of the library.
const jweak class_loader_;
+ // Used to do equality check on class loaders so we can avoid decoding the weak root and read
+ // barriers that mess with class unloading.
+ const void* class_loader_allocator_;
// Guards remaining items.
Mutex jni_on_load_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER;
@@ -224,11 +233,15 @@
SHARED_REQUIRES(Locks::mutator_lock_) {
std::string jni_short_name(JniShortName(m));
std::string jni_long_name(JniLongName(m));
- const mirror::ClassLoader* declaring_class_loader = m->GetDeclaringClass()->GetClassLoader();
+ mirror::ClassLoader* const declaring_class_loader = m->GetDeclaringClass()->GetClassLoader();
ScopedObjectAccessUnchecked soa(Thread::Current());
+ void* const declaring_class_loader_allocator =
+ Runtime::Current()->GetClassLinker()->GetAllocatorForClassLoader(declaring_class_loader);
+ CHECK(declaring_class_loader_allocator != nullptr);
for (const auto& lib : libraries_) {
SharedLibrary* const library = lib.second;
- if (soa.Decode<mirror::ClassLoader*>(library->GetClassLoader()) != declaring_class_loader) {
+ // Use the allocator address for class loader equality to avoid unnecessary weak root decode.
+ if (library->GetClassLoaderAllocator() != declaring_class_loader_allocator) {
// We only search libraries loaded by the appropriate ClassLoader.
continue;
}
@@ -269,7 +282,7 @@
// If class_loader is a null jobject then it is the boot class loader. We should not unload
// the native libraries of the boot class loader.
if (class_loader != nullptr &&
- soa.Decode<mirror::ClassLoader*>(class_loader) == nullptr) {
+ soa.Self()->IsJWeakCleared(class_loader)) {
void* const sym = library->FindSymbol("JNI_OnUnload", nullptr);
if (sym == nullptr) {
VLOG(jni) << "[No JNI_OnUnload found in \"" << library->GetPath() << "\"]";
@@ -667,6 +680,19 @@
return weak_globals_.SynchronizedGet(ref);
}
+bool JavaVMExt::IsWeakGlobalCleared(Thread* self, IndirectRef ref) {
+ DCHECK_EQ(GetIndirectRefKind(ref), kWeakGlobal);
+ MutexLock mu(self, weak_globals_lock_);
+ while (UNLIKELY(!MayAccessWeakGlobals(self))) {
+ weak_globals_add_condition_.WaitHoldingLocks(self);
+ }
+ // When just checking a weak ref has been cleared, avoid triggering the read barrier in decode
+ // (DecodeWeakGlobal) so that we won't accidentally mark the object alive. Since the cleared
+ // sentinel is a non-moving object, we can compare the ref to it without the read barrier and
+ // decide if it's cleared.
+ return Runtime::Current()->IsClearedJniWeakGlobal(weak_globals_.Get<kWithoutReadBarrier>(ref));
+}
+
void JavaVMExt::UpdateWeakGlobal(Thread* self, IndirectRef ref, mirror::Object* result) {
MutexLock mu(self, weak_globals_lock_);
weak_globals_.Update(ref, result);
@@ -703,8 +729,19 @@
MutexLock mu(self, *Locks::jni_libraries_lock_);
library = libraries_->Get(path);
}
+ void* class_loader_allocator = nullptr;
+ {
+ ScopedObjectAccess soa(env);
+ // As the incoming class loader is reachable/alive during the call of this function,
+ // it's okay to decode it without worrying about unexpectedly marking it alive.
+ mirror::ClassLoader* loader = soa.Decode<mirror::ClassLoader*>(class_loader);
+ class_loader_allocator =
+ Runtime::Current()->GetClassLinker()->GetAllocatorForClassLoader(loader);
+ CHECK(class_loader_allocator != nullptr);
+ }
if (library != nullptr) {
- if (env->IsSameObject(library->GetClassLoader(), class_loader) == JNI_FALSE) {
+ // Use the allocator pointers for class loader equality to avoid unnecessary weak root decode.
+ if (library->GetClassLoaderAllocator() != class_loader_allocator) {
// The library will be associated with class_loader. The JNI
// spec says we can't load the same library into more than one
// class loader.
@@ -765,7 +802,7 @@
{
// Create SharedLibrary ahead of taking the libraries lock to maintain lock ordering.
std::unique_ptr<SharedLibrary> new_library(
- new SharedLibrary(env, self, path, handle, class_loader));
+ new SharedLibrary(env, self, path, handle, class_loader, class_loader_allocator));
MutexLock mu(self, *Locks::jni_libraries_lock_);
library = libraries_->Get(path);
if (library == nullptr) { // We won race to get libraries_lock.
diff --git a/runtime/java_vm_ext.h b/runtime/java_vm_ext.h
index c1fbdc0..618f6fa 100644
--- a/runtime/java_vm_ext.h
+++ b/runtime/java_vm_ext.h
@@ -149,6 +149,11 @@
SHARED_REQUIRES(Locks::mutator_lock_)
REQUIRES(!weak_globals_lock_);
+ // Checks if the weak global ref has been cleared by the GC without decode (read barrier.)
+ bool IsWeakGlobalCleared(Thread* self, IndirectRef ref)
+ SHARED_REQUIRES(Locks::mutator_lock_)
+ REQUIRES(!weak_globals_lock_);
+
Mutex& WeakGlobalsLock() RETURN_CAPABILITY(weak_globals_lock_) {
return weak_globals_lock_;
}
diff --git a/runtime/jit/jit.cc b/runtime/jit/jit.cc
index f691151..ecbf13c 100644
--- a/runtime/jit/jit.cc
+++ b/runtime/jit/jit.cc
@@ -34,8 +34,10 @@
JitOptions* JitOptions::CreateFromRuntimeArguments(const RuntimeArgumentMap& options) {
auto* jit_options = new JitOptions;
jit_options->use_jit_ = options.GetOrDefault(RuntimeArgumentMap::UseJIT);
- jit_options->code_cache_capacity_ =
- options.GetOrDefault(RuntimeArgumentMap::JITCodeCacheCapacity);
+ jit_options->code_cache_initial_capacity_ =
+ options.GetOrDefault(RuntimeArgumentMap::JITCodeCacheInitialCapacity);
+ jit_options->code_cache_max_capacity_ =
+ options.GetOrDefault(RuntimeArgumentMap::JITCodeCacheMaxCapacity);
jit_options->compile_threshold_ =
options.GetOrDefault(RuntimeArgumentMap::JITCompileThreshold);
jit_options->warmup_threshold_ =
@@ -69,13 +71,15 @@
if (!jit->LoadCompiler(error_msg)) {
return nullptr;
}
- jit->code_cache_.reset(JitCodeCache::Create(options->GetCodeCacheCapacity(), error_msg));
+ jit->code_cache_.reset(JitCodeCache::Create(
+ options->GetCodeCacheInitialCapacity(), options->GetCodeCacheMaxCapacity(), error_msg));
if (jit->GetCodeCache() == nullptr) {
return nullptr;
}
- LOG(INFO) << "JIT created with code_cache_capacity="
- << PrettySize(options->GetCodeCacheCapacity())
- << " compile_threshold=" << options->GetCompileThreshold();
+ LOG(INFO) << "JIT created with initial_capacity="
+ << PrettySize(options->GetCodeCacheInitialCapacity())
+ << ", max_capacity=" << PrettySize(options->GetCodeCacheMaxCapacity())
+ << ", compile_threshold=" << options->GetCompileThreshold();
return jit.release();
}
diff --git a/runtime/jit/jit.h b/runtime/jit/jit.h
index 1f89f9b..fc76549 100644
--- a/runtime/jit/jit.h
+++ b/runtime/jit/jit.h
@@ -102,8 +102,11 @@
size_t GetWarmupThreshold() const {
return warmup_threshold_;
}
- size_t GetCodeCacheCapacity() const {
- return code_cache_capacity_;
+ size_t GetCodeCacheInitialCapacity() const {
+ return code_cache_initial_capacity_;
+ }
+ size_t GetCodeCacheMaxCapacity() const {
+ return code_cache_max_capacity_;
}
bool DumpJitInfoOnShutdown() const {
return dump_info_on_shutdown_;
@@ -117,13 +120,18 @@
private:
bool use_jit_;
- size_t code_cache_capacity_;
+ size_t code_cache_initial_capacity_;
+ size_t code_cache_max_capacity_;
size_t compile_threshold_;
size_t warmup_threshold_;
bool dump_info_on_shutdown_;
- JitOptions() : use_jit_(false), code_cache_capacity_(0), compile_threshold_(0),
- dump_info_on_shutdown_(false) { }
+ JitOptions()
+ : use_jit_(false),
+ code_cache_initial_capacity_(0),
+ code_cache_max_capacity_(0),
+ compile_threshold_(0),
+ dump_info_on_shutdown_(false) { }
DISALLOW_COPY_AND_ASSIGN(JitOptions);
};
diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc
index a291a09..da79109 100644
--- a/runtime/jit/jit_code_cache.cc
+++ b/runtime/jit/jit_code_cache.cc
@@ -44,73 +44,89 @@
} \
} while (false) \
-JitCodeCache* JitCodeCache::Create(size_t capacity, std::string* error_msg) {
- CHECK_GT(capacity, 0U);
- CHECK_LT(capacity, kMaxCapacity);
+JitCodeCache* JitCodeCache::Create(size_t initial_capacity,
+ size_t max_capacity,
+ std::string* error_msg) {
+ CHECK_GE(max_capacity, initial_capacity);
+ // We need to have 32 bit offsets from method headers in code cache which point to things
+ // in the data cache. If the maps are more than 4G apart, having multiple maps wouldn't work.
+ // Ensure we're below 1 GB to be safe.
+ if (max_capacity > 1 * GB) {
+ std::ostringstream oss;
+ oss << "Maxium code cache capacity is limited to 1 GB, "
+ << PrettySize(max_capacity) << " is too big";
+ *error_msg = oss.str();
+ return nullptr;
+ }
+
std::string error_str;
// Map name specific for android_os_Debug.cpp accounting.
MemMap* data_map = MemMap::MapAnonymous(
- "data-code-cache", nullptr, capacity, kProtAll, false, false, &error_str);
+ "data-code-cache", nullptr, max_capacity, kProtAll, false, false, &error_str);
if (data_map == nullptr) {
std::ostringstream oss;
- oss << "Failed to create read write execute cache: " << error_str << " size=" << capacity;
+ oss << "Failed to create read write execute cache: " << error_str << " size=" << max_capacity;
*error_msg = oss.str();
return nullptr;
}
+ // Align both capacities to page size, as that's the unit mspaces use.
+ initial_capacity = RoundDown(initial_capacity, 2 * kPageSize);
+ max_capacity = RoundDown(max_capacity, 2 * kPageSize);
+
// Data cache is 1 / 2 of the map.
// TODO: Make this variable?
- size_t data_size = RoundUp(data_map->Size() / 2, kPageSize);
- size_t code_size = data_map->Size() - data_size;
+ size_t data_size = max_capacity / 2;
+ size_t code_size = max_capacity - data_size;
+ DCHECK_EQ(code_size + data_size, max_capacity);
uint8_t* divider = data_map->Begin() + data_size;
- // We need to have 32 bit offsets from method headers in code cache which point to things
- // in the data cache. If the maps are more than 4G apart, having multiple maps wouldn't work.
MemMap* code_map = data_map->RemapAtEnd(divider, "jit-code-cache", kProtAll, &error_str);
if (code_map == nullptr) {
std::ostringstream oss;
- oss << "Failed to create read write execute cache: " << error_str << " size=" << capacity;
+ oss << "Failed to create read write execute cache: " << error_str << " size=" << max_capacity;
*error_msg = oss.str();
return nullptr;
}
- DCHECK_EQ(code_map->Size(), code_size);
DCHECK_EQ(code_map->Begin(), divider);
- return new JitCodeCache(code_map, data_map);
+ data_size = initial_capacity / 2;
+ code_size = initial_capacity - data_size;
+ DCHECK_EQ(code_size + data_size, initial_capacity);
+ return new JitCodeCache(code_map, data_map, code_size, data_size, max_capacity);
}
-JitCodeCache::JitCodeCache(MemMap* code_map, MemMap* data_map)
+JitCodeCache::JitCodeCache(MemMap* code_map,
+ MemMap* data_map,
+ size_t initial_code_capacity,
+ size_t initial_data_capacity,
+ size_t max_capacity)
: lock_("Jit code cache", kJitCodeCacheLock),
lock_cond_("Jit code cache variable", lock_),
collection_in_progress_(false),
code_map_(code_map),
- data_map_(data_map) {
+ data_map_(data_map),
+ max_capacity_(max_capacity),
+ current_capacity_(initial_code_capacity + initial_data_capacity),
+ code_end_(initial_code_capacity),
+ data_end_(initial_data_capacity),
+ has_done_one_collection_(false) {
- code_mspace_ = create_mspace_with_base(code_map_->Begin(), code_map_->Size(), false /*locked*/);
- data_mspace_ = create_mspace_with_base(data_map_->Begin(), data_map_->Size(), false /*locked*/);
+ code_mspace_ = create_mspace_with_base(code_map_->Begin(), code_end_, false /*locked*/);
+ data_mspace_ = create_mspace_with_base(data_map_->Begin(), data_end_, false /*locked*/);
if (code_mspace_ == nullptr || data_mspace_ == nullptr) {
PLOG(FATAL) << "create_mspace_with_base failed";
}
- // Prevent morecore requests from the mspace.
- mspace_set_footprint_limit(code_mspace_, code_map_->Size());
- mspace_set_footprint_limit(data_mspace_, data_map_->Size());
+ SetFootprintLimit(current_capacity_);
CHECKED_MPROTECT(code_map_->Begin(), code_map_->Size(), kProtCode);
CHECKED_MPROTECT(data_map_->Begin(), data_map_->Size(), kProtData);
- live_bitmap_.reset(CodeCacheBitmap::Create("code-cache-bitmap",
- reinterpret_cast<uintptr_t>(code_map_->Begin()),
- reinterpret_cast<uintptr_t>(code_map_->End())));
-
- if (live_bitmap_.get() == nullptr) {
- PLOG(FATAL) << "creating bitmaps for the JIT code cache failed";
- }
-
- VLOG(jit) << "Created jit code cache: data size="
- << PrettySize(data_map_->Size())
- << ", code size="
- << PrettySize(code_map_->Size());
+ VLOG(jit) << "Created jit code cache: initial data size="
+ << PrettySize(initial_data_capacity)
+ << ", initial code size="
+ << PrettySize(initial_code_capacity);
}
bool JitCodeCache::ContainsPc(const void* ptr) const {
@@ -433,13 +449,48 @@
Barrier* const barrier_;
};
-void JitCodeCache::GarbageCollectCache(Thread* self) {
- if (!kIsDebugBuild || VLOG_IS_ON(jit)) {
- LOG(INFO) << "Clearing code cache, code="
- << PrettySize(CodeCacheSize())
- << ", data=" << PrettySize(DataCacheSize());
+void JitCodeCache::NotifyCollectionDone(Thread* self) {
+ collection_in_progress_ = false;
+ lock_cond_.Broadcast(self);
+}
+
+void JitCodeCache::SetFootprintLimit(size_t new_footprint) {
+ size_t per_space_footprint = new_footprint / 2;
+ DCHECK(IsAlignedParam(per_space_footprint, kPageSize));
+ DCHECK_EQ(per_space_footprint * 2, new_footprint);
+ mspace_set_footprint_limit(data_mspace_, per_space_footprint);
+ {
+ ScopedCodeCacheWrite scc(code_map_.get());
+ mspace_set_footprint_limit(code_mspace_, per_space_footprint);
+ }
+}
+
+bool JitCodeCache::IncreaseCodeCacheCapacity() {
+ if (current_capacity_ == max_capacity_) {
+ return false;
}
+ // Double the capacity if we're below 1MB, or increase it by 1MB if
+ // we're above.
+ if (current_capacity_ < 1 * MB) {
+ current_capacity_ *= 2;
+ } else {
+ current_capacity_ += 1 * MB;
+ }
+ if (current_capacity_ > max_capacity_) {
+ current_capacity_ = max_capacity_;
+ }
+
+ if (!kIsDebugBuild || VLOG_IS_ON(jit)) {
+ LOG(INFO) << "Increasing code cache capacity to " << PrettySize(current_capacity_);
+ }
+
+ SetFootprintLimit(current_capacity_);
+
+ return true;
+}
+
+void JitCodeCache::GarbageCollectCache(Thread* self) {
instrumentation::Instrumentation* instrumentation = Runtime::Current()->GetInstrumentation();
// Wait for an existing collection, or let everyone know we are starting one.
@@ -452,6 +503,28 @@
collection_in_progress_ = true;
}
}
+
+ // Check if we just need to grow the capacity. If we don't, allocate the bitmap while
+ // we hold the lock.
+ {
+ MutexLock mu(self, lock_);
+ if (has_done_one_collection_ && IncreaseCodeCacheCapacity()) {
+ has_done_one_collection_ = false;
+ NotifyCollectionDone(self);
+ return;
+ } else {
+ live_bitmap_.reset(CodeCacheBitmap::Create(
+ "code-cache-bitmap",
+ reinterpret_cast<uintptr_t>(code_map_->Begin()),
+ reinterpret_cast<uintptr_t>(code_map_->Begin() + current_capacity_ / 2)));
+ }
+ }
+
+ if (!kIsDebugBuild || VLOG_IS_ON(jit)) {
+ LOG(INFO) << "Clearing code cache, code="
+ << PrettySize(CodeCacheSize())
+ << ", data=" << PrettySize(DataCacheSize());
+ }
// Walk over all compiled methods and set the entry points of these
// methods to interpreter.
{
@@ -500,7 +573,6 @@
}
}
}
- GetLiveBitmap()->Bitmap::Clear();
// Free all profiling info.
for (ProfilingInfo* info : profiling_infos_) {
@@ -509,8 +581,9 @@
}
profiling_infos_.clear();
- collection_in_progress_ = false;
- lock_cond_.Broadcast(self);
+ live_bitmap_.reset(nullptr);
+ has_done_one_collection_ = true;
+ NotifyCollectionDone(self);
}
if (!kIsDebugBuild || VLOG_IS_ON(jit)) {
@@ -589,5 +662,20 @@
return info;
}
+// NO_THREAD_SAFETY_ANALYSIS as this is called from mspace code, at which point the lock
+// is already held.
+void* JitCodeCache::MoreCore(const void* mspace, intptr_t increment) NO_THREAD_SAFETY_ANALYSIS {
+ if (code_mspace_ == mspace) {
+ size_t result = code_end_;
+ code_end_ += increment;
+ return reinterpret_cast<void*>(result + code_map_->Begin());
+ } else {
+ DCHECK_EQ(data_mspace_, mspace);
+ size_t result = data_end_;
+ data_end_ += increment;
+ return reinterpret_cast<void*>(result + data_map_->Begin());
+ }
+}
+
} // namespace jit
} // namespace art
diff --git a/runtime/jit/jit_code_cache.h b/runtime/jit/jit_code_cache.h
index 131446c..13481e0 100644
--- a/runtime/jit/jit_code_cache.h
+++ b/runtime/jit/jit_code_cache.h
@@ -41,20 +41,20 @@
class JitInstrumentationCache;
-// Alignment that will suit all architectures.
+// Alignment in bits that will suit all architectures.
static constexpr int kJitCodeAlignment = 16;
using CodeCacheBitmap = gc::accounting::MemoryRangeBitmap<kJitCodeAlignment>;
class JitCodeCache {
public:
- static constexpr size_t kMaxCapacity = 1 * GB;
+ static constexpr size_t kMaxCapacity = 64 * MB;
// Put the default to a very low amount for debug builds to stress the code cache
// collection.
- static constexpr size_t kDefaultCapacity = kIsDebugBuild ? 20 * KB : 2 * MB;
+ static constexpr size_t kInitialCapacity = kIsDebugBuild ? 16 * KB : 64 * KB;
// Create the code cache with a code + data capacity equal to "capacity", error message is passed
// in the out arg error_msg.
- static JitCodeCache* Create(size_t capacity, std::string* error_msg);
+ static JitCodeCache* Create(size_t initial_capacity, size_t max_capacity, std::string* error_msg);
// Number of bytes allocated in the code cache.
size_t CodeCacheSize() REQUIRES(!lock_);
@@ -133,9 +133,19 @@
REQUIRES(!lock_)
SHARED_REQUIRES(Locks::mutator_lock_);
+ bool OwnsSpace(const void* mspace) const NO_THREAD_SAFETY_ANALYSIS {
+ return mspace == code_mspace_ || mspace == data_mspace_;
+ }
+
+ void* MoreCore(const void* mspace, intptr_t increment);
+
private:
- // Take ownership of code_mem_map.
- JitCodeCache(MemMap* code_map, MemMap* data_map);
+ // Take ownership of maps.
+ JitCodeCache(MemMap* code_map,
+ MemMap* data_map,
+ size_t initial_code_capacity,
+ size_t initial_data_capacity,
+ size_t max_capacity);
// Internal version of 'CommitCode' that will not retry if the
// allocation fails. Return null if the allocation fails.
@@ -172,6 +182,16 @@
// Number of bytes allocated in the data cache.
size_t DataCacheSizeLocked() REQUIRES(lock_);
+ // Notify all waiting threads that a collection is done.
+ void NotifyCollectionDone(Thread* self) REQUIRES(lock_);
+
+ // Try to increase the current capacity of the code cache. Return whether we
+ // succeeded at doing so.
+ bool IncreaseCodeCacheCapacity() REQUIRES(lock_);
+
+ // Set the footprint limit of the code cache.
+ void SetFootprintLimit(size_t new_footprint) REQUIRES(lock_);
+
// Lock for guarding allocations, collections, and the method_code_map_.
Mutex lock_;
// Condition to wait on during collection.
@@ -193,6 +213,21 @@
// ProfilingInfo objects we have allocated.
std::vector<ProfilingInfo*> profiling_infos_ GUARDED_BY(lock_);
+ // The maximum capacity in bytes this code cache can go to.
+ size_t max_capacity_ GUARDED_BY(lock_);
+
+ // The current capacity in bytes of the code cache.
+ size_t current_capacity_ GUARDED_BY(lock_);
+
+ // The current footprint in bytes of the code portion of the code cache.
+ size_t code_end_ GUARDED_BY(lock_);
+
+ // The current footprint in bytes of the data portion of the code cache.
+ size_t data_end_ GUARDED_BY(lock_);
+
+ // Whether a collection has already been done on the current capacity.
+ bool has_done_one_collection_ GUARDED_BY(lock_);
+
DISALLOW_IMPLICIT_CONSTRUCTORS(JitCodeCache);
};
diff --git a/runtime/lambda/box_table.cc b/runtime/lambda/box_table.cc
index 8eef10b..9918bb7 100644
--- a/runtime/lambda/box_table.cc
+++ b/runtime/lambda/box_table.cc
@@ -62,7 +62,7 @@
BoxTable::~BoxTable() {
// Free all the copies of our closures.
- for (auto map_iterator = map_.begin(); map_iterator != map_.end(); ++map_iterator) {
+ for (auto map_iterator = map_.begin(); map_iterator != map_.end(); ) {
std::pair<UnorderedMapKeyType, ValueType>& key_value_pair = *map_iterator;
Closure* closure = key_value_pair.first;
diff --git a/runtime/mirror/array-inl.h b/runtime/mirror/array-inl.h
index ec7d758..b6f424b 100644
--- a/runtime/mirror/array-inl.h
+++ b/runtime/mirror/array-inl.h
@@ -394,6 +394,19 @@
}
}
+template <typename Visitor>
+inline void PointerArray::Fixup(mirror::PointerArray* dest,
+ size_t pointer_size,
+ const Visitor& visitor) {
+ for (size_t i = 0, count = GetLength(); i < count; ++i) {
+ void* ptr = GetElementPtrSize<void*>(i, pointer_size);
+ void* new_ptr = visitor(ptr);
+ if (ptr != new_ptr) {
+ dest->SetElementPtrSize<false, true>(i, new_ptr, pointer_size);
+ }
+ }
+}
+
} // namespace mirror
} // namespace art
diff --git a/runtime/mirror/array.h b/runtime/mirror/array.h
index b27a884..50d77eb 100644
--- a/runtime/mirror/array.h
+++ b/runtime/mirror/array.h
@@ -122,7 +122,7 @@
T Get(int32_t i) ALWAYS_INLINE SHARED_REQUIRES(Locks::mutator_lock_);
T GetWithoutChecks(int32_t i) ALWAYS_INLINE SHARED_REQUIRES(Locks::mutator_lock_) {
- DCHECK(CheckIsValidIndex(i));
+ DCHECK(CheckIsValidIndex(i)) << "i=" << i << " length=" << GetLength();
return GetData()[i];
}
@@ -190,6 +190,12 @@
template<bool kTransactionActive = false, bool kUnchecked = false, typename T>
void SetElementPtrSize(uint32_t idx, T element, size_t ptr_size)
SHARED_REQUIRES(Locks::mutator_lock_);
+
+ // Fixup the pointers in the dest arrays by passing our pointers through the visitor. Only copies
+ // to dest if visitor(source_ptr) != source_ptr.
+ template <typename Visitor>
+ void Fixup(mirror::PointerArray* dest, size_t pointer_size, const Visitor& visitor)
+ SHARED_REQUIRES(Locks::mutator_lock_);
};
} // namespace mirror
diff --git a/runtime/mirror/class-inl.h b/runtime/mirror/class-inl.h
index 174de0e..9e416dc 100644
--- a/runtime/mirror/class-inl.h
+++ b/runtime/mirror/class-inl.h
@@ -937,6 +937,57 @@
return arr != nullptr ? arr->size() : 0u;
}
+template <typename Visitor>
+inline void Class::FixupNativePointers(mirror::Class* dest,
+ size_t pointer_size,
+ const Visitor& visitor) {
+ // Update the field arrays.
+ LengthPrefixedArray<ArtField>* const sfields = GetSFieldsPtr();
+ LengthPrefixedArray<ArtField>* const new_sfields = visitor(sfields);
+ if (sfields != new_sfields) {
+ dest->SetSFieldsPtrUnchecked(new_sfields);
+ }
+ LengthPrefixedArray<ArtField>* const ifields = GetIFieldsPtr();
+ LengthPrefixedArray<ArtField>* const new_ifields = visitor(ifields);
+ if (ifields != new_ifields) {
+ dest->SetIFieldsPtrUnchecked(new_ifields);
+ }
+ // Update direct and virtual method arrays.
+ LengthPrefixedArray<ArtMethod>* direct_methods = GetDirectMethodsPtr();
+ LengthPrefixedArray<ArtMethod>* new_direct_methods = visitor(direct_methods);
+ if (direct_methods != new_direct_methods) {
+ dest->SetDirectMethodsPtrUnchecked(new_direct_methods);
+ }
+ LengthPrefixedArray<ArtMethod>* virtual_methods = GetVirtualMethodsPtr();
+ LengthPrefixedArray<ArtMethod>* new_virtual_methods = visitor(virtual_methods);
+ if (virtual_methods != new_virtual_methods) {
+ dest->SetVirtualMethodsPtr(new_virtual_methods);
+ }
+ // Update dex cache strings.
+ GcRoot<mirror::String>* strings = GetDexCacheStrings();
+ GcRoot<mirror::String>* new_strings = visitor(strings);
+ if (strings != new_strings) {
+ dest->SetDexCacheStrings(new_strings);
+ }
+ // Fix up embedded tables.
+ if (!IsTemp() && ShouldHaveEmbeddedImtAndVTable()) {
+ for (int32_t i = 0, count = GetEmbeddedVTableLength(); i < count; ++i) {
+ ArtMethod* method = GetEmbeddedVTableEntry(i, pointer_size);
+ ArtMethod* new_method = visitor(method);
+ if (method != new_method) {
+ dest->SetEmbeddedVTableEntryUnchecked(i, new_method, pointer_size);
+ }
+ }
+ for (size_t i = 0; i < mirror::Class::kImtSize; ++i) {
+ ArtMethod* method = GetEmbeddedImTableEntry(i, pointer_size);
+ ArtMethod* new_method = visitor(method);
+ if (method != new_method) {
+ dest->SetEmbeddedImTableEntry(i, new_method, pointer_size);
+ }
+ }
+ }
+}
+
} // namespace mirror
} // namespace art
diff --git a/runtime/mirror/class.cc b/runtime/mirror/class.cc
index 3590586..05a9039 100644
--- a/runtime/mirror/class.cc
+++ b/runtime/mirror/class.cc
@@ -802,6 +802,18 @@
}
}
+mirror::Class* Class::GetCommonSuperClass(Handle<Class> klass) {
+ DCHECK(klass.Get() != nullptr);
+ DCHECK(!klass->IsInterface());
+ DCHECK(!IsInterface());
+ mirror::Class* common_super_class = this;
+ while (!common_super_class->IsAssignableFrom(klass.Get())) {
+ common_super_class = common_super_class->GetSuperClass();
+ }
+ DCHECK(common_super_class != nullptr);
+ return common_super_class;
+}
+
const char* Class::GetSourceFile() {
const DexFile& dex_file = GetDexFile();
const DexFile::ClassDef* dex_class_def = GetClassDef();
diff --git a/runtime/mirror/class.h b/runtime/mirror/class.h
index c4339b9..0ab5b97 100644
--- a/runtime/mirror/class.h
+++ b/runtime/mirror/class.h
@@ -657,6 +657,10 @@
ALWAYS_INLINE Class* GetSuperClass() SHARED_REQUIRES(Locks::mutator_lock_);
+ // Get first common super class. It will never return null.
+ // `This` and `klass` must be classes.
+ Class* GetCommonSuperClass(Handle<Class> klass) SHARED_REQUIRES(Locks::mutator_lock_);
+
void SetSuperClass(Class *new_super_class) SHARED_REQUIRES(Locks::mutator_lock_) {
// Super class is assigned once, except during class linker initialization.
Class* old_super_class = GetFieldObject<Class>(OFFSET_OF_OBJECT_MEMBER(Class, super_class_));
@@ -1157,6 +1161,13 @@
ALWAYS_INLINE LengthPrefixedArray<ArtMethod>* GetVirtualMethodsPtrUnchecked()
SHARED_REQUIRES(Locks::mutator_lock_);
+ // Fix up all of the native pointers in the class by running them through the visitor. Only sets
+ // the corresponding entry in dest if visitor(obj) != obj to prevent dirty memory. Dest should be
+ // initialized to a copy of *this to prevent issues.
+ template <typename Visitor>
+ void FixupNativePointers(mirror::Class* dest, size_t pointer_size, const Visitor& visitor)
+ SHARED_REQUIRES(Locks::mutator_lock_);
+
private:
void SetVerifyError(Object* klass) SHARED_REQUIRES(Locks::mutator_lock_);
@@ -1185,7 +1196,7 @@
bool ProxyDescriptorEquals(const char* match) SHARED_REQUIRES(Locks::mutator_lock_);
- // Check that the pointer size mathces the one in the class linker.
+ // Check that the pointer size matches the one in the class linker.
ALWAYS_INLINE static void CheckPointerSize(size_t pointer_size);
static MemberOffset EmbeddedImTableOffset(size_t pointer_size);
diff --git a/runtime/mirror/dex_cache-inl.h b/runtime/mirror/dex_cache-inl.h
index f8ccfb1..975af61 100644
--- a/runtime/mirror/dex_cache-inl.h
+++ b/runtime/mirror/dex_cache-inl.h
@@ -137,6 +137,32 @@
}
}
+template <typename Visitor>
+inline void DexCache::FixupStrings(GcRoot<mirror::String>* dest, const Visitor& visitor) {
+ GcRoot<mirror::String>* src = GetStrings();
+ for (size_t i = 0, count = NumStrings(); i < count; ++i) {
+ // TODO: Probably don't need read barrier for most callers.
+ mirror::String* source = src[i].Read();
+ mirror::String* new_source = visitor(source);
+ if (source != new_source) {
+ dest[i] = GcRoot<mirror::String>(new_source);
+ }
+ }
+}
+
+template <typename Visitor>
+inline void DexCache::FixupResolvedTypes(GcRoot<mirror::Class>* dest, const Visitor& visitor) {
+ GcRoot<mirror::Class>* src = GetResolvedTypes();
+ for (size_t i = 0, count = NumResolvedTypes(); i < count; ++i) {
+ // TODO: Probably don't need read barrier for most callers.
+ mirror::Class* source = src[i].Read();
+ mirror::Class* new_source = visitor(source);
+ if (source != new_source) {
+ dest[i] = GcRoot<mirror::Class>(new_source);
+ }
+ }
+}
+
} // namespace mirror
} // namespace art
diff --git a/runtime/mirror/dex_cache.h b/runtime/mirror/dex_cache.h
index 3144553..32eb595 100644
--- a/runtime/mirror/dex_cache.h
+++ b/runtime/mirror/dex_cache.h
@@ -61,6 +61,14 @@
void Fixup(ArtMethod* trampoline, size_t pointer_size)
SHARED_REQUIRES(Locks::mutator_lock_);
+ template <typename Visitor>
+ void FixupStrings(GcRoot<mirror::String>* dest, const Visitor& visitor)
+ SHARED_REQUIRES(Locks::mutator_lock_);
+
+ template <typename Visitor>
+ void FixupResolvedTypes(GcRoot<mirror::Class>* dest, const Visitor& visitor)
+ SHARED_REQUIRES(Locks::mutator_lock_);
+
String* GetLocation() SHARED_REQUIRES(Locks::mutator_lock_) {
return GetFieldObject<String>(OFFSET_OF_OBJECT_MEMBER(DexCache, location_));
}
diff --git a/runtime/mirror/object-inl.h b/runtime/mirror/object-inl.h
index 5c12091..4603428 100644
--- a/runtime/mirror/object-inl.h
+++ b/runtime/mirror/object-inl.h
@@ -163,6 +163,7 @@
#endif
}
+template<bool kCasRelease>
inline bool Object::AtomicSetReadBarrierPointer(Object* expected_rb_ptr, Object* rb_ptr) {
#ifdef USE_BAKER_READ_BARRIER
DCHECK(kUseBakerReadBarrier);
@@ -181,10 +182,13 @@
static_cast<uint32_t>(reinterpret_cast<uintptr_t>(expected_rb_ptr)));
new_lw = lw;
new_lw.SetReadBarrierState(static_cast<uint32_t>(reinterpret_cast<uintptr_t>(rb_ptr)));
- // This CAS is a CAS release so that when GC updates all the fields of an object and then
- // changes the object from gray to black, the field updates (stores) will be visible (won't be
- // reordered after this CAS.)
- } while (!CasLockWordWeakRelease(expected_lw, new_lw));
+ // ConcurrentCopying::ProcessMarkStackRef uses this with kCasRelease == true.
+ // If kCasRelease == true, use a CAS release so that when GC updates all the fields of
+ // an object and then changes the object from gray to black, the field updates (stores) will be
+ // visible (won't be reordered after this CAS.)
+ } while (!(kCasRelease ?
+ CasLockWordWeakRelease(expected_lw, new_lw) :
+ CasLockWordWeakRelaxed(expected_lw, new_lw)));
return true;
#elif USE_BROOKS_READ_BARRIER
DCHECK(kUseBrooksReadBarrier);
diff --git a/runtime/mirror/object.h b/runtime/mirror/object.h
index 5c6520f..71e704e 100644
--- a/runtime/mirror/object.h
+++ b/runtime/mirror/object.h
@@ -92,13 +92,13 @@
void SetClass(Class* new_klass) SHARED_REQUIRES(Locks::mutator_lock_);
Object* GetReadBarrierPointer() SHARED_REQUIRES(Locks::mutator_lock_);
+
#ifndef USE_BAKER_OR_BROOKS_READ_BARRIER
NO_RETURN
#endif
void SetReadBarrierPointer(Object* rb_ptr) SHARED_REQUIRES(Locks::mutator_lock_);
-#ifndef USE_BAKER_OR_BROOKS_READ_BARRIER
- NO_RETURN
-#endif
+
+ template<bool kCasRelease = false>
ALWAYS_INLINE bool AtomicSetReadBarrierPointer(Object* expected_rb_ptr, Object* rb_ptr)
SHARED_REQUIRES(Locks::mutator_lock_);
void AssertReadBarrierPointer() const SHARED_REQUIRES(Locks::mutator_lock_);
diff --git a/runtime/mirror/string.cc b/runtime/mirror/string.cc
index 45610dc..be869d4 100644
--- a/runtime/mirror/string.cc
+++ b/runtime/mirror/string.cc
@@ -254,7 +254,11 @@
StackHandleScope<1> hs(self);
Handle<String> string(hs.NewHandle(this));
CharArray* result = CharArray::Alloc(self, GetLength());
- memcpy(result->GetData(), string->GetValue(), string->GetLength() * sizeof(uint16_t));
+ if (result != nullptr) {
+ memcpy(result->GetData(), string->GetValue(), string->GetLength() * sizeof(uint16_t));
+ } else {
+ self->AssertPendingOOMException();
+ }
return result;
}
diff --git a/runtime/mirror/string.h b/runtime/mirror/string.h
index fbee2d7..80ebd2c 100644
--- a/runtime/mirror/string.h
+++ b/runtime/mirror/string.h
@@ -27,6 +27,7 @@
template<class T> class Handle;
struct StringOffsets;
class StringPiece;
+class StubTest_ReadBarrierForRoot_Test;
namespace mirror {
@@ -179,7 +180,7 @@
static GcRoot<Class> java_lang_String_;
friend struct art::StringOffsets; // for verifying offset information
- ART_FRIEND_TEST(ObjectTest, StringLength); // for SetOffset and SetCount
+ ART_FRIEND_TEST(art::StubTest, ReadBarrierForRoot); // For java_lang_String_.
DISALLOW_IMPLICIT_CONSTRUCTORS(String);
};
diff --git a/runtime/native/dalvik_system_DexFile.cc b/runtime/native/dalvik_system_DexFile.cc
index e85434ee..da6cf1f 100644
--- a/runtime/native/dalvik_system_DexFile.cc
+++ b/runtime/native/dalvik_system_DexFile.cc
@@ -155,7 +155,9 @@
jstring javaOutputName,
jint flags ATTRIBUTE_UNUSED,
// class_loader will be used for app images.
- jobject class_loader ATTRIBUTE_UNUSED) {
+ jobject class_loader ATTRIBUTE_UNUSED,
+ // dex_elements will be used for app images.
+ jobject dex_elements ATTRIBUTE_UNUSED) {
ScopedUtfChars sourceName(env, javaSourceName);
if (sourceName.c_str() == nullptr) {
return 0;
@@ -164,7 +166,6 @@
if (env->ExceptionCheck()) {
return 0;
}
-
Runtime* const runtime = Runtime::Current();
ClassLinker* linker = runtime->GetClassLinker();
std::vector<std::unique_ptr<const DexFile>> dex_files;
@@ -446,7 +447,12 @@
NATIVE_METHOD(DexFile, getDexOptNeeded,
"(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Z)I"),
NATIVE_METHOD(DexFile, openDexFileNative,
- "(Ljava/lang/String;Ljava/lang/String;ILjava/lang/ClassLoader;)Ljava/lang/Object;"),
+ "(Ljava/lang/String;"
+ "Ljava/lang/String;"
+ "I"
+ "Ljava/lang/ClassLoader;"
+ "[Ldalvik/system/DexPathList$Element;"
+ ")Ljava/lang/Object;"),
};
void register_dalvik_system_DexFile(JNIEnv* env) {
diff --git a/runtime/native/dalvik_system_ZygoteHooks.cc b/runtime/native/dalvik_system_ZygoteHooks.cc
index 1d06706..ae1a4d7 100644
--- a/runtime/native/dalvik_system_ZygoteHooks.cc
+++ b/runtime/native/dalvik_system_ZygoteHooks.cc
@@ -64,8 +64,7 @@
DEBUG_ENABLE_ASSERT = 1 << 2,
DEBUG_ENABLE_SAFEMODE = 1 << 3,
DEBUG_ENABLE_JNI_LOGGING = 1 << 4,
- DEBUG_ENABLE_JIT = 1 << 5,
- DEBUG_GENERATE_DEBUG_INFO = 1 << 6,
+ DEBUG_GENERATE_DEBUG_INFO = 1 << 5,
};
Runtime* const runtime = Runtime::Current();
@@ -97,21 +96,10 @@
if (safe_mode) {
// Ensure that any (secondary) oat files will be interpreted.
runtime->AddCompilerOption("--compiler-filter=interpret-only");
+ runtime->SetSafeMode(true);
debug_flags &= ~DEBUG_ENABLE_SAFEMODE;
}
- bool use_jit = false;
- if ((debug_flags & DEBUG_ENABLE_JIT) != 0) {
- if (safe_mode) {
- LOG(INFO) << "Not enabling JIT due to safe mode";
- } else {
- use_jit = true;
- LOG(INFO) << "Late-enabling JIT";
- }
- debug_flags &= ~DEBUG_ENABLE_JIT;
- }
- runtime->GetJITOptions()->SetUseJIT(use_jit);
-
const bool generate_debug_info = (debug_flags & DEBUG_GENERATE_DEBUG_INFO) != 0;
if (generate_debug_info) {
runtime->AddCompilerOption("--generate-debug-info");
@@ -171,23 +159,17 @@
proc_name = StringPrintf("%u", static_cast<uint32_t>(pid));
}
- std::string profiles_dir(GetDalvikCache("profiles", false /* create_if_absent */));
- if (!profiles_dir.empty()) {
- std::string trace_file = StringPrintf("%s/%s.trace.bin", profiles_dir.c_str(),
- proc_name.c_str());
- Trace::Start(trace_file.c_str(),
- -1,
- buffer_size,
- 0, // TODO: Expose flags.
- output_mode,
- trace_mode,
- 0); // TODO: Expose interval.
- if (thread->IsExceptionPending()) {
- ScopedObjectAccess soa(env);
- thread->ClearException();
- }
- } else {
- LOG(ERROR) << "Profiles dir is empty?!?!";
+ std::string trace_file = StringPrintf("/data/misc/trace/%s.trace.bin", proc_name.c_str());
+ Trace::Start(trace_file.c_str(),
+ -1,
+ buffer_size,
+ 0, // TODO: Expose flags.
+ output_mode,
+ trace_mode,
+ 0); // TODO: Expose interval.
+ if (thread->IsExceptionPending()) {
+ ScopedObjectAccess soa(env);
+ thread->ClearException();
}
}
}
@@ -199,9 +181,9 @@
if (isa != kNone && isa != kRuntimeISA) {
action = Runtime::NativeBridgeAction::kInitialize;
}
- Runtime::Current()->DidForkFromZygote(env, action, isa_string.c_str());
+ Runtime::Current()->InitNonZygoteOrPostFork(env, action, isa_string.c_str());
} else {
- Runtime::Current()->DidForkFromZygote(env, Runtime::NativeBridgeAction::kUnload, nullptr);
+ Runtime::Current()->InitNonZygoteOrPostFork(env, Runtime::NativeBridgeAction::kUnload, nullptr);
}
}
diff --git a/runtime/os.h b/runtime/os.h
index befe2e8..46d89fb 100644
--- a/runtime/os.h
+++ b/runtime/os.h
@@ -39,6 +39,10 @@
// already exists, it is *not* overwritten, but unlinked, and a new inode will be used.
static File* CreateEmptyFile(const char* name);
+ // Create an empty file with write access. This is a *new* file, that is, if the file
+ // already exists, it is *not* overwritten, but unlinked, and a new inode will be used.
+ static File* CreateEmptyFileWriteOnly(const char* name);
+
// Open a file with the specified open(2) flags.
static File* OpenFileWithFlags(const char* name, int flags);
diff --git a/runtime/os_linux.cc b/runtime/os_linux.cc
index 675699d..f45e9f6 100644
--- a/runtime/os_linux.cc
+++ b/runtime/os_linux.cc
@@ -35,12 +35,20 @@
return OpenFileWithFlags(name, O_RDWR);
}
-File* OS::CreateEmptyFile(const char* name) {
+static File* CreateEmptyFile(const char* name, int extra_flags) {
// In case the file exists, unlink it so we get a new file. This is necessary as the previous
// file may be in use and must not be changed.
unlink(name);
- return OpenFileWithFlags(name, O_RDWR | O_CREAT | O_TRUNC);
+ return OS::OpenFileWithFlags(name, O_CREAT | extra_flags);
+}
+
+File* OS::CreateEmptyFile(const char* name) {
+ return art::CreateEmptyFile(name, O_RDWR | O_TRUNC);
+}
+
+File* OS::CreateEmptyFileWriteOnly(const char* name) {
+ return art::CreateEmptyFile(name, O_WRONLY | O_TRUNC | O_NOFOLLOW | O_CLOEXEC);
}
File* OS::OpenFileWithFlags(const char* name, int flags) {
diff --git a/runtime/parsed_options.cc b/runtime/parsed_options.cc
index ae16c7f..dfd783b 100644
--- a/runtime/parsed_options.cc
+++ b/runtime/parsed_options.cc
@@ -152,9 +152,12 @@
.WithType<bool>()
.WithValueMap({{"false", false}, {"true", true}})
.IntoKey(M::UseJIT)
- .Define("-Xjitcodecachesize:_")
+ .Define("-Xjitinitialsize:_")
.WithType<MemoryKiB>()
- .IntoKey(M::JITCodeCacheCapacity)
+ .IntoKey(M::JITCodeCacheInitialCapacity)
+ .Define("-Xjitmaxsize:_")
+ .WithType<MemoryKiB>()
+ .IntoKey(M::JITCodeCacheMaxCapacity)
.Define("-Xjitthreshold:_")
.WithType<unsigned int>()
.IntoKey(M::JITCompileThreshold)
@@ -640,7 +643,6 @@
UsageMessage(stream, " -XX:ForegroundHeapGrowthMultiplier=doublevalue\n");
UsageMessage(stream, " -XX:LowMemoryMode\n");
UsageMessage(stream, " -Xprofile:{threadcpuclock,wallclock,dualclock}\n");
- UsageMessage(stream, " -Xjitcodecachesize:N\n");
UsageMessage(stream, " -Xjitthreshold:integervalue\n");
UsageMessage(stream, "\n");
@@ -684,6 +686,8 @@
UsageMessage(stream, " -Ximage-compiler-option dex2oat-option\n");
UsageMessage(stream, " -Xpatchoat:filename\n");
UsageMessage(stream, " -Xusejit:booleanvalue\n");
+ UsageMessage(stream, " -Xjitinitialsize:N\n");
+ UsageMessage(stream, " -Xjitmaxsize:N\n");
UsageMessage(stream, " -X[no]relocate\n");
UsageMessage(stream, " -X[no]dex2oat (Whether to invoke dex2oat on the application)\n");
UsageMessage(stream, " -X[no]image-dex2oat (Whether to create and use a boot image)\n");
@@ -718,6 +722,7 @@
UsageMessage(stream, " -Xjitblocking\n");
UsageMessage(stream, " -Xjitmethod:signature[,signature]* (eg Ljava/lang/String\\;replace)\n");
UsageMessage(stream, " -Xjitclass:classname[,classname]*\n");
+ UsageMessage(stream, " -Xjitcodecachesize:N\n");
UsageMessage(stream, " -Xjitoffset:offset[,offset]\n");
UsageMessage(stream, " -Xjitconfig:filename\n");
UsageMessage(stream, " -Xjitcheckcg\n");
diff --git a/runtime/reflection_test.cc b/runtime/reflection_test.cc
index c7c2709..6234720 100644
--- a/runtime/reflection_test.cc
+++ b/runtime/reflection_test.cc
@@ -507,6 +507,8 @@
TEST_F(ReflectionTest, StaticMainMethod) {
TEST_DISABLED_FOR_HEAP_REFERENCE_POISONING_WITH_QUICK();
+ TEST_DISABLED_FOR_READ_BARRIER_WITH_QUICK();
+ TEST_DISABLED_FOR_READ_BARRIER_WITH_OPTIMIZING_FOR_UNSUPPORTED_INSTRUCTION_SETS();
ScopedObjectAccess soa(Thread::Current());
jobject jclass_loader = LoadDex("Main");
StackHandleScope<1> hs(soa.Self());
diff --git a/runtime/runtime.cc b/runtime/runtime.cc
index 17f34c0..a210aa8 100644
--- a/runtime/runtime.cc
+++ b/runtime/runtime.cc
@@ -208,7 +208,8 @@
zygote_max_failed_boots_(0),
experimental_flags_(ExperimentalFlags::kNone),
oat_file_manager_(nullptr),
- is_low_memory_mode_(false) {
+ is_low_memory_mode_(false),
+ safe_mode_(false) {
CheckAsmSupportOffsetsAndSizes();
std::fill(callee_save_methods_, callee_save_methods_ + arraysize(callee_save_methods_), 0u);
}
@@ -573,8 +574,9 @@
if (is_native_bridge_loaded_) {
PreInitializeNativeBridge(".");
}
- DidForkFromZygote(self->GetJniEnv(), NativeBridgeAction::kInitialize,
- GetInstructionSetString(kRuntimeISA));
+ InitNonZygoteOrPostFork(self->GetJniEnv(),
+ NativeBridgeAction::kInitialize,
+ GetInstructionSetString(kRuntimeISA));
}
ATRACE_BEGIN("StartDaemonThreads");
@@ -663,7 +665,7 @@
#endif
}
-void Runtime::DidForkFromZygote(JNIEnv* env, NativeBridgeAction action, const char* isa) {
+void Runtime::InitNonZygoteOrPostFork(JNIEnv* env, NativeBridgeAction action, const char* isa) {
is_zygote_ = false;
if (is_native_bridge_loaded_) {
@@ -685,8 +687,9 @@
// before fork aren't attributed to an app.
heap_->ResetGcPerformanceInfo();
- if (jit_.get() == nullptr && jit_options_->UseJIT()) {
- // Create the JIT if the flag is set and we haven't already create it (happens for run-tests).
+ if (!safe_mode_ && jit_options_->UseJIT() && jit_.get() == nullptr) {
+ // Note that when running ART standalone (not zygote, nor zygote fork),
+ // the jit may have already been created.
CreateJit();
}
@@ -820,6 +823,7 @@
void Runtime::SetSentinel(mirror::Object* sentinel) {
CHECK(sentinel_.Read() == nullptr);
CHECK(sentinel != nullptr);
+ CHECK(!heap_->IsMovableObject(sentinel));
sentinel_ = GcRoot<mirror::Object>(sentinel);
}
@@ -1040,8 +1044,13 @@
class_linker_ = new ClassLinker(intern_table_);
if (GetHeap()->HasImageSpace()) {
ATRACE_BEGIN("InitFromImage");
- class_linker_->InitFromImage();
+ std::string error_msg;
+ bool result = class_linker_->InitFromImage(&error_msg);
ATRACE_END();
+ if (!result) {
+ LOG(ERROR) << "Could not initialize from image: " << error_msg;
+ return false;
+ }
if (kIsDebugBuild) {
GetHeap()->GetBootImageSpace()->VerifyImageAllocations();
}
@@ -1073,7 +1082,11 @@
runtime_options.GetOrDefault(Opt::Image),
&boot_class_path);
instruction_set_ = runtime_options.GetOrDefault(Opt::ImageInstructionSet);
- class_linker_->InitWithoutImage(std::move(boot_class_path));
+ std::string error_msg;
+ if (!class_linker_->InitWithoutImage(std::move(boot_class_path), &error_msg)) {
+ LOG(ERROR) << "Could not initialize without image: " << error_msg;
+ return false;
+ }
// TODO: Should we move the following to InitWithoutImage?
SetInstructionSet(instruction_set_);
diff --git a/runtime/runtime.h b/runtime/runtime.h
index a8ba19b..d61663c 100644
--- a/runtime/runtime.h
+++ b/runtime/runtime.h
@@ -447,7 +447,7 @@
void PreZygoteFork();
bool InitZygote();
- void DidForkFromZygote(JNIEnv* env, NativeBridgeAction action, const char* isa);
+ void InitNonZygoteOrPostFork(JNIEnv* env, NativeBridgeAction action, const char* isa);
const instrumentation::Instrumentation* GetInstrumentation() const {
return &instrumentation_;
@@ -588,6 +588,10 @@
double GetHashTableMinLoadFactor() const;
double GetHashTableMaxLoadFactor() const;
+ void SetSafeMode(bool mode) {
+ safe_mode_ = mode;
+ }
+
private:
static void InitPlatformSignalHandlers();
@@ -791,6 +795,9 @@
// Whether or not we are on a low RAM device.
bool is_low_memory_mode_;
+ // Whether the application should run in safe mode, that is, interpreter only.
+ bool safe_mode_;
+
DISALLOW_COPY_AND_ASSIGN(Runtime);
};
std::ostream& operator<<(std::ostream& os, const Runtime::CalleeSaveType& rhs);
diff --git a/runtime/runtime_options.def b/runtime/runtime_options.def
index 7b5bc1a..9051eda 100644
--- a/runtime/runtime_options.def
+++ b/runtime/runtime_options.def
@@ -69,7 +69,8 @@
RUNTIME_OPTIONS_KEY (bool, UseJIT, false)
RUNTIME_OPTIONS_KEY (unsigned int, JITCompileThreshold, jit::Jit::kDefaultCompileThreshold)
RUNTIME_OPTIONS_KEY (unsigned int, JITWarmupThreshold, jit::Jit::kDefaultWarmupThreshold)
-RUNTIME_OPTIONS_KEY (MemoryKiB, JITCodeCacheCapacity, jit::JitCodeCache::kDefaultCapacity)
+RUNTIME_OPTIONS_KEY (MemoryKiB, JITCodeCacheInitialCapacity, jit::JitCodeCache::kInitialCapacity)
+RUNTIME_OPTIONS_KEY (MemoryKiB, JITCodeCacheMaxCapacity, jit::JitCodeCache::kMaxCapacity)
RUNTIME_OPTIONS_KEY (MillisecondsToNanoseconds, \
HSpaceCompactForOOMMinIntervalsMs,\
MsToNs(100 * 1000)) // 100s
@@ -80,10 +81,7 @@
RUNTIME_OPTIONS_KEY (bool, Relocate, kDefaultMustRelocate)
RUNTIME_OPTIONS_KEY (bool, Dex2Oat, true)
RUNTIME_OPTIONS_KEY (bool, ImageDex2Oat, true)
- // kUseReadBarrier currently works with
- // the interpreter only.
- // TODO: make it work with the compiler.
-RUNTIME_OPTIONS_KEY (bool, Interpret, kUseReadBarrier) // -Xint
+RUNTIME_OPTIONS_KEY (bool, Interpret, false) // -Xint
// Disable the compiler for CC (for now).
RUNTIME_OPTIONS_KEY (XGcOption, GcOption) // -Xgc:
RUNTIME_OPTIONS_KEY (gc::space::LargeObjectSpaceType, \
@@ -97,7 +95,7 @@
RUNTIME_OPTIONS_KEY (unsigned int, LockProfThreshold)
RUNTIME_OPTIONS_KEY (std::string, StackTraceFile)
RUNTIME_OPTIONS_KEY (Unit, MethodTrace)
-RUNTIME_OPTIONS_KEY (std::string, MethodTraceFile, "/data/method-trace-file.bin")
+RUNTIME_OPTIONS_KEY (std::string, MethodTraceFile, "/data/misc/trace/method-trace-file.bin")
RUNTIME_OPTIONS_KEY (unsigned int, MethodTraceFileSize, 10 * MB)
RUNTIME_OPTIONS_KEY (Unit, MethodTraceStreaming)
RUNTIME_OPTIONS_KEY (TraceClockSource, ProfileClock, kDefaultTraceClockSource) // -Xprofile:
diff --git a/runtime/stack.cc b/runtime/stack.cc
index d7edfad..9098d38 100644
--- a/runtime/stack.cc
+++ b/runtime/stack.cc
@@ -265,7 +265,12 @@
}
} else {
DCHECK(cur_shadow_frame_ != nullptr);
- *val = cur_shadow_frame_->GetVReg(vreg);
+ if (kind == kReferenceVReg) {
+ *val = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(
+ cur_shadow_frame_->GetVRegReference(vreg)));
+ } else {
+ *val = cur_shadow_frame_->GetVReg(vreg);
+ }
return true;
}
}
@@ -481,52 +486,10 @@
return true;
}
-bool StackVisitor::SetVReg(ArtMethod* m, uint16_t vreg, uint32_t new_value,
+bool StackVisitor::SetVReg(ArtMethod* m,
+ uint16_t vreg,
+ uint32_t new_value,
VRegKind kind) {
- if (cur_quick_frame_ != nullptr) {
- DCHECK(context_ != nullptr); // You can't reliably write registers without a context.
- DCHECK(m == GetMethod());
- if (cur_oat_quick_method_header_->IsOptimized()) {
- return false;
- } else {
- return SetVRegFromQuickCode(m, vreg, new_value, kind);
- }
- } else {
- cur_shadow_frame_->SetVReg(vreg, new_value);
- return true;
- }
-}
-
-bool StackVisitor::SetVRegFromQuickCode(ArtMethod* m, uint16_t vreg, uint32_t new_value,
- VRegKind kind) {
- DCHECK(context_ != nullptr); // You can't reliably write registers without a context.
- DCHECK(m == GetMethod());
- const OatQuickMethodHeader* method_header = GetCurrentOatQuickMethodHeader();
- QuickMethodFrameInfo frame_info = method_header->GetFrameInfo();
- const VmapTable vmap_table(method_header->GetVmapTable());
- uint32_t vmap_offset;
- // TODO: IsInContext stops before spotting floating point registers.
- if (vmap_table.IsInContext(vreg, kind, &vmap_offset)) {
- bool is_float = (kind == kFloatVReg) || (kind == kDoubleLoVReg) || (kind == kDoubleHiVReg);
- uint32_t spill_mask = is_float ? frame_info.FpSpillMask() : frame_info.CoreSpillMask();
- uint32_t reg = vmap_table.ComputeRegister(spill_mask, vmap_offset, kind);
- return SetRegisterIfAccessible(reg, new_value, kind);
- } else {
- const DexFile::CodeItem* code_item = m->GetCodeItem();
- DCHECK(code_item != nullptr) << PrettyMethod(m); // Can't be null or how would we compile
- // its instructions?
- uint32_t* addr = GetVRegAddrFromQuickCode(
- cur_quick_frame_, code_item, frame_info.CoreSpillMask(),
- frame_info.FpSpillMask(), frame_info.FrameSizeInBytes(), vreg);
- *addr = new_value;
- return true;
- }
-}
-
-bool StackVisitor::SetVRegFromDebugger(ArtMethod* m,
- uint16_t vreg,
- uint32_t new_value,
- VRegKind kind) {
const DexFile::CodeItem* code_item = m->GetCodeItem();
if (code_item == nullptr) {
return false;
@@ -551,93 +514,11 @@
return true;
}
-bool StackVisitor::SetRegisterIfAccessible(uint32_t reg, uint32_t new_value, VRegKind kind) {
- const bool is_float = (kind == kFloatVReg) || (kind == kDoubleLoVReg) || (kind == kDoubleHiVReg);
- if (!IsAccessibleRegister(reg, is_float)) {
- return false;
- }
- const bool target64 = Is64BitInstructionSet(kRuntimeISA);
-
- // Create a new value that can hold both low 32 and high 32 bits, in
- // case we are running 64 bits.
- uintptr_t full_new_value = new_value;
- // Deal with 32 or 64-bit wide registers in a way that builds on all targets.
- if (target64) {
- bool wide_lo = (kind == kLongLoVReg) || (kind == kDoubleLoVReg);
- bool wide_hi = (kind == kLongHiVReg) || (kind == kDoubleHiVReg);
- if (wide_lo || wide_hi) {
- uintptr_t old_reg_val = GetRegister(reg, is_float);
- uint64_t new_vreg_portion = static_cast<uint64_t>(new_value);
- uint64_t old_reg_val_as_wide = static_cast<uint64_t>(old_reg_val);
- uint64_t mask = 0xffffffff;
- if (wide_lo) {
- mask = mask << 32;
- } else {
- new_vreg_portion = new_vreg_portion << 32;
- }
- full_new_value = static_cast<uintptr_t>((old_reg_val_as_wide & mask) | new_vreg_portion);
- }
- }
- SetRegister(reg, full_new_value, is_float);
- return true;
-}
-
-bool StackVisitor::SetVRegPair(ArtMethod* m, uint16_t vreg, uint64_t new_value,
- VRegKind kind_lo, VRegKind kind_hi) {
- if (kind_lo == kLongLoVReg) {
- DCHECK_EQ(kind_hi, kLongHiVReg);
- } else if (kind_lo == kDoubleLoVReg) {
- DCHECK_EQ(kind_hi, kDoubleHiVReg);
- } else {
- LOG(FATAL) << "Expected long or double: kind_lo=" << kind_lo << ", kind_hi=" << kind_hi;
- }
- if (cur_quick_frame_ != nullptr) {
- DCHECK(context_ != nullptr); // You can't reliably write registers without a context.
- DCHECK(m == GetMethod());
- if (cur_oat_quick_method_header_->IsOptimized()) {
- return false;
- } else {
- return SetVRegPairFromQuickCode(m, vreg, new_value, kind_lo, kind_hi);
- }
- } else {
- DCHECK(cur_shadow_frame_ != nullptr);
- cur_shadow_frame_->SetVRegLong(vreg, new_value);
- return true;
- }
-}
-
-bool StackVisitor::SetVRegPairFromQuickCode(
- ArtMethod* m, uint16_t vreg, uint64_t new_value, VRegKind kind_lo, VRegKind kind_hi) {
- DCHECK_EQ(m, GetMethod());
- const OatQuickMethodHeader* method_header = GetCurrentOatQuickMethodHeader();
- QuickMethodFrameInfo frame_info = method_header->GetFrameInfo();
- const VmapTable vmap_table(method_header->GetVmapTable());
- uint32_t vmap_offset_lo, vmap_offset_hi;
- // TODO: IsInContext stops before spotting floating point registers.
- if (vmap_table.IsInContext(vreg, kind_lo, &vmap_offset_lo) &&
- vmap_table.IsInContext(vreg + 1, kind_hi, &vmap_offset_hi)) {
- bool is_float = (kind_lo == kDoubleLoVReg);
- uint32_t spill_mask = is_float ? frame_info.FpSpillMask() : frame_info.CoreSpillMask();
- uint32_t reg_lo = vmap_table.ComputeRegister(spill_mask, vmap_offset_lo, kind_lo);
- uint32_t reg_hi = vmap_table.ComputeRegister(spill_mask, vmap_offset_hi, kind_hi);
- return SetRegisterPairIfAccessible(reg_lo, reg_hi, new_value, is_float);
- } else {
- const DexFile::CodeItem* code_item = m->GetCodeItem();
- DCHECK(code_item != nullptr) << PrettyMethod(m); // Can't be null or how would we compile
- // its instructions?
- uint32_t* addr = GetVRegAddrFromQuickCode(
- cur_quick_frame_, code_item, frame_info.CoreSpillMask(),
- frame_info.FpSpillMask(), frame_info.FrameSizeInBytes(), vreg);
- *reinterpret_cast<uint64_t*>(addr) = new_value;
- return true;
- }
-}
-
-bool StackVisitor::SetVRegPairFromDebugger(ArtMethod* m,
- uint16_t vreg,
- uint64_t new_value,
- VRegKind kind_lo,
- VRegKind kind_hi) {
+bool StackVisitor::SetVRegPair(ArtMethod* m,
+ uint16_t vreg,
+ uint64_t new_value,
+ VRegKind kind_lo,
+ VRegKind kind_hi) {
if (kind_lo == kLongLoVReg) {
DCHECK_EQ(kind_hi, kLongHiVReg);
} else if (kind_lo == kDoubleLoVReg) {
@@ -666,25 +547,6 @@
return true;
}
-bool StackVisitor::SetRegisterPairIfAccessible(uint32_t reg_lo, uint32_t reg_hi,
- uint64_t new_value, bool is_float) {
- if (!IsAccessibleRegister(reg_lo, is_float) || !IsAccessibleRegister(reg_hi, is_float)) {
- return false;
- }
- uintptr_t new_value_lo = static_cast<uintptr_t>(new_value & 0xFFFFFFFF);
- uintptr_t new_value_hi = static_cast<uintptr_t>(new_value >> 32);
- bool target64 = Is64BitInstructionSet(kRuntimeISA);
- // Deal with 32 or 64-bit wide registers in a way that builds on all targets.
- if (target64) {
- DCHECK_EQ(reg_lo, reg_hi);
- SetRegister(reg_lo, new_value, is_float);
- } else {
- SetRegister(reg_lo, new_value_lo, is_float);
- SetRegister(reg_hi, new_value_hi, is_float);
- }
- return true;
-}
-
bool StackVisitor::IsAccessibleGPR(uint32_t reg) const {
DCHECK(context_ != nullptr);
return context_->IsAccessibleGPR(reg);
@@ -702,12 +564,6 @@
return context_->GetGPR(reg);
}
-void StackVisitor::SetGPR(uint32_t reg, uintptr_t value) {
- DCHECK(cur_quick_frame_ != nullptr) << "This is a quick frame routine";
- DCHECK(context_ != nullptr);
- context_->SetGPR(reg, value);
-}
-
bool StackVisitor::IsAccessibleFPR(uint32_t reg) const {
DCHECK(context_ != nullptr);
return context_->IsAccessibleFPR(reg);
@@ -719,12 +575,6 @@
return context_->GetFPR(reg);
}
-void StackVisitor::SetFPR(uint32_t reg, uintptr_t value) {
- DCHECK(cur_quick_frame_ != nullptr) << "This is a quick frame routine";
- DCHECK(context_ != nullptr);
- context_->SetFPR(reg, value);
-}
-
uintptr_t StackVisitor::GetReturnPc() const {
uint8_t* sp = reinterpret_cast<uint8_t*>(GetCurrentQuickFrame());
DCHECK(sp != nullptr);
@@ -865,8 +715,8 @@
CHECK(code_start <= pc && pc <= (code_start + code_size))
<< PrettyMethod(method)
<< " pc=" << std::hex << pc
- << " code=" << code
- << " size=" << code_size;
+ << " code_start=" << code_start
+ << " code_size=" << code_size;
}
void StackVisitor::SanityCheckFrame() const {
diff --git a/runtime/stack.h b/runtime/stack.h
index aa7b616..a0c44cb 100644
--- a/runtime/stack.h
+++ b/runtime/stack.h
@@ -600,22 +600,18 @@
uint64_t* val) const
SHARED_REQUIRES(Locks::mutator_lock_);
+ // Values will be set in debugger shadow frames. Debugger will make sure deoptimization
+ // is triggered to make the values effective.
bool SetVReg(ArtMethod* m, uint16_t vreg, uint32_t new_value, VRegKind kind)
SHARED_REQUIRES(Locks::mutator_lock_);
// Values will be set in debugger shadow frames. Debugger will make sure deoptimization
// is triggered to make the values effective.
- bool SetVRegFromDebugger(ArtMethod* m, uint16_t vreg, uint32_t new_value, VRegKind kind)
- SHARED_REQUIRES(Locks::mutator_lock_);
-
- bool SetVRegPair(ArtMethod* m, uint16_t vreg, uint64_t new_value,
- VRegKind kind_lo, VRegKind kind_hi)
- SHARED_REQUIRES(Locks::mutator_lock_);
-
- // Values will be set in debugger shadow frames. Debugger will make sure deoptimization
- // is triggered to make the values effective.
- bool SetVRegPairFromDebugger(ArtMethod* m, uint16_t vreg, uint64_t new_value,
- VRegKind kind_lo, VRegKind kind_hi)
+ bool SetVRegPair(ArtMethod* m,
+ uint16_t vreg,
+ uint64_t new_value,
+ VRegKind kind_lo,
+ VRegKind kind_hi)
SHARED_REQUIRES(Locks::mutator_lock_);
uintptr_t* GetGPRAddress(uint32_t reg) const;
@@ -749,22 +745,12 @@
DCHECK(IsAccessibleRegister(reg, is_float));
return is_float ? GetFPR(reg) : GetGPR(reg);
}
- void SetRegister(uint32_t reg, uintptr_t value, bool is_float) {
- DCHECK(IsAccessibleRegister(reg, is_float));
- if (is_float) {
- SetFPR(reg, value);
- } else {
- SetGPR(reg, value);
- }
- }
bool IsAccessibleGPR(uint32_t reg) const;
uintptr_t GetGPR(uint32_t reg) const;
- void SetGPR(uint32_t reg, uintptr_t value);
bool IsAccessibleFPR(uint32_t reg) const;
uintptr_t GetFPR(uint32_t reg) const;
- void SetFPR(uint32_t reg, uintptr_t value);
bool GetVRegFromDebuggerShadowFrame(uint16_t vreg, VRegKind kind, uint32_t* val) const
SHARED_REQUIRES(Locks::mutator_lock_);
@@ -789,19 +775,6 @@
uint64_t* val) const
SHARED_REQUIRES(Locks::mutator_lock_);
- bool SetVRegFromQuickCode(ArtMethod* m, uint16_t vreg, uint32_t new_value,
- VRegKind kind)
- SHARED_REQUIRES(Locks::mutator_lock_);
- bool SetRegisterIfAccessible(uint32_t reg, uint32_t new_value, VRegKind kind)
- SHARED_REQUIRES(Locks::mutator_lock_);
-
- bool SetVRegPairFromQuickCode(ArtMethod* m, uint16_t vreg, uint64_t new_value,
- VRegKind kind_lo, VRegKind kind_hi)
- SHARED_REQUIRES(Locks::mutator_lock_);
- bool SetRegisterPairIfAccessible(uint32_t reg_lo, uint32_t reg_hi, uint64_t new_value,
- bool is_float)
- SHARED_REQUIRES(Locks::mutator_lock_);
-
void SanityCheckFrame() const SHARED_REQUIRES(Locks::mutator_lock_);
InlineInfo GetCurrentInlineInfo() const SHARED_REQUIRES(Locks::mutator_lock_);
diff --git a/runtime/thread.cc b/runtime/thread.cc
index b0cf418..63e6326 100644
--- a/runtime/thread.cc
+++ b/runtime/thread.cc
@@ -1886,6 +1886,14 @@
return result;
}
+bool Thread::IsJWeakCleared(jweak obj) const {
+ CHECK(obj != nullptr);
+ IndirectRef ref = reinterpret_cast<IndirectRef>(obj);
+ IndirectRefKind kind = GetIndirectRefKind(ref);
+ CHECK_EQ(kind, kWeakGlobal);
+ return tlsPtr_.jni_env->vm->IsWeakGlobalCleared(const_cast<Thread*>(this), ref);
+}
+
// Implements java.lang.Thread.interrupted.
bool Thread::Interrupted() {
MutexLock mu(Thread::Current(), *wait_mutex_);
@@ -2509,6 +2517,7 @@
QUICK_ENTRY_POINT_INFO(pNewStringFromStringBuilder)
QUICK_ENTRY_POINT_INFO(pReadBarrierJni)
QUICK_ENTRY_POINT_INFO(pReadBarrierSlow)
+ QUICK_ENTRY_POINT_INFO(pReadBarrierForRootSlow)
#undef QUICK_ENTRY_POINT_INFO
os << offset;
diff --git a/runtime/thread.h b/runtime/thread.h
index 138c143..4624f27 100644
--- a/runtime/thread.h
+++ b/runtime/thread.h
@@ -445,6 +445,8 @@
// Convert a jobject into a Object*
mirror::Object* DecodeJObject(jobject obj) const SHARED_REQUIRES(Locks::mutator_lock_);
+ // Checks if the weak global ref has been cleared by the GC without decoding it.
+ bool IsJWeakCleared(jweak obj) const SHARED_REQUIRES(Locks::mutator_lock_);
mirror::Object* GetMonitorEnterObject() const SHARED_REQUIRES(Locks::mutator_lock_) {
return tlsPtr_.monitor_enter_object;
diff --git a/runtime/thread_list.cc b/runtime/thread_list.cc
index b09b87f..a390908 100644
--- a/runtime/thread_list.cc
+++ b/runtime/thread_list.cc
@@ -948,7 +948,12 @@
Locks::mutator_lock_->ExclusiveLock(self);
Locks::mutator_lock_->ExclusiveUnlock(self);
#endif
- AssertThreadsAreSuspended(self, self, debug_thread);
+ // Disabled for the following race condition:
+ // Thread 1 calls SuspendAllForDebugger, gets preempted after pulsing the mutator lock.
+ // Thread 2 calls SuspendAll and SetStateUnsafe (perhaps from Dbg::Disconnected).
+ // Thread 1 fails assertion that all threads are suspended due to thread 2 being in a runnable
+ // state (from SetStateUnsafe).
+ // AssertThreadsAreSuspended(self, self, debug_thread);
VLOG(threads) << *self << " SuspendAllForDebugger complete";
}
diff --git a/runtime/trace.cc b/runtime/trace.cc
index ab342aa..5815f7a 100644
--- a/runtime/trace.cc
+++ b/runtime/trace.cc
@@ -331,7 +331,7 @@
std::unique_ptr<File> trace_file;
if (output_mode != TraceOutputMode::kDDMS) {
if (trace_fd < 0) {
- trace_file.reset(OS::CreateEmptyFile(trace_filename));
+ trace_file.reset(OS::CreateEmptyFileWriteOnly(trace_filename));
} else {
trace_file.reset(new File(trace_fd, "tracefile"));
trace_file->DisableAutoClose();
diff --git a/runtime/utils.h b/runtime/utils.h
index 3690f86..8b7941a 100644
--- a/runtime/utils.h
+++ b/runtime/utils.h
@@ -18,9 +18,11 @@
#define ART_RUNTIME_UTILS_H_
#include <pthread.h>
+#include <stdlib.h>
#include <limits>
#include <memory>
+#include <random>
#include <string>
#include <type_traits>
#include <vector>
@@ -350,6 +352,26 @@
double* parsed_value,
UsageFn Usage);
+#if defined(__BIONIC__)
+struct Arc4RandomGenerator {
+ typedef uint32_t result_type;
+ static constexpr uint32_t min() { return std::numeric_limits<uint32_t>::min(); }
+ static constexpr uint32_t max() { return std::numeric_limits<uint32_t>::max(); }
+ uint32_t operator() () { return arc4random(); }
+};
+using RNG = Arc4RandomGenerator;
+#else
+using RNG = std::random_device;
+#endif
+
+template <typename T>
+T GetRandomNumber(T min, T max) {
+ CHECK_LT(min, max);
+ std::uniform_int_distribution<T> dist(min, max);
+ RNG rng;
+ return dist(rng);
+}
+
} // namespace art
#endif // ART_RUNTIME_UTILS_H_
diff --git a/test/061-out-of-memory/expected.txt b/test/061-out-of-memory/expected.txt
index ca87629..c31980c 100644
--- a/test/061-out-of-memory/expected.txt
+++ b/test/061-out-of-memory/expected.txt
@@ -4,4 +4,5 @@
testOomeLarge succeeded
testOomeSmall beginning
testOomeSmall succeeded
+Got expected toCharArray OOM
tests succeeded
diff --git a/test/061-out-of-memory/src/Main.java b/test/061-out-of-memory/src/Main.java
index c812c81..bda978e 100644
--- a/test/061-out-of-memory/src/Main.java
+++ b/test/061-out-of-memory/src/Main.java
@@ -26,6 +26,7 @@
testHugeArray();
testOomeLarge();
testOomeSmall();
+ testOomeToCharArray();
System.out.println("tests succeeded");
}
@@ -106,4 +107,21 @@
}
System.out.println("testOomeSmall succeeded");
}
+
+ private static void testOomeToCharArray() {
+ Object[] o = new Object[2000000];
+ String test = "test";
+ int i = 0;
+ try {
+ for (; i < o.length; ++i) o[i] = new char[1000000];
+ } catch (OutOfMemoryError oom) {}
+ try {
+ for (; i < o.length; ++i) {
+ o[i] = test.toCharArray();
+ }
+ } catch (OutOfMemoryError oom) {
+ o = null;
+ System.out.println("Got expected toCharArray OOM");
+ }
+ }
}
diff --git a/test/450-checker-types/src/Main.java b/test/450-checker-types/src/Main.java
index accf70b..ec63057 100644
--- a/test/450-checker-types/src/Main.java
+++ b/test/450-checker-types/src/Main.java
@@ -14,11 +14,19 @@
* limitations under the License.
*/
-interface Interface {
+interface SuperInterface {
+ void superInterfaceMethod();
+}
+
+interface OtherInterface extends SuperInterface {
+}
+
+interface Interface extends SuperInterface {
void $noinline$f();
}
class Super implements Interface {
+ public void superInterfaceMethod() {}
public void $noinline$f() {
throw new RuntimeException();
}
@@ -548,33 +556,30 @@
private void argumentCheck(Super s, double d, SubclassA a, Final f) {
}
- /// CHECK-START: Main Main.getMain(boolean) reference_type_propagation (after)
- /// CHECK: <<Phi:l\d+>> Phi klass:java.lang.Object
- /// CHECK: Return [<<Phi>>]
- private Main getMain(boolean cond) {
- return cond ? null : new Main();
- }
-
private Main getNull() {
return null;
}
private int mainField = 0;
- /// CHECK-START: void Main.testInlinerWidensReturnType(boolean) inliner (before)
- /// CHECK: <<Int:i\d+>> IntConstant 0
- /// CHECK: <<Invoke:l\d+>> InvokeStaticOrDirect klass:Main
- /// CHECK: <<NullCheck:l\d+>> NullCheck [<<Invoke>>] klass:Main exact:false
- /// CHECK: InstanceFieldSet [<<NullCheck>>,<<Int>>]
-
- /// CHECK-START: void Main.testInlinerWidensReturnType(boolean) inliner (after)
- /// CHECK: <<Int:i\d+>> IntConstant 0
+ /// CHECK-START: SuperInterface Main.getWiderType(boolean, Interface, OtherInterface) reference_type_propagation (after)
/// CHECK: <<Phi:l\d+>> Phi klass:java.lang.Object
- /// CHECK: <<NullCheck:l\d+>> NullCheck [<<Phi>>] klass:Main exact:false
- /// CHECK: InstanceFieldSet [<<NullCheck>>,<<Int>>]
- private void testInlinerWidensReturnType(boolean cond) {
- Main o = getMain(cond);
- o.mainField = 0;
+ /// CHECK: Return [<<Phi>>]
+ private SuperInterface getWiderType(boolean cond, Interface a, OtherInterface b) {
+ return cond ? a : b;
+ }
+
+ /// CHECK-START: void Main.testInlinerWidensReturnType(boolean, Interface, OtherInterface) inliner (before)
+ /// CHECK: <<Invoke:l\d+>> InvokeStaticOrDirect klass:SuperInterface
+ /// CHECK: <<NullCheck:l\d+>> NullCheck [<<Invoke>>] klass:SuperInterface exact:false
+ /// CHECK: InvokeInterface [<<NullCheck>>]
+
+ /// CHECK-START: void Main.testInlinerWidensReturnType(boolean, Interface, OtherInterface) inliner (after)
+ /// CHECK: <<Phi:l\d+>> Phi klass:java.lang.Object
+ /// CHECK: <<NullCheck:l\d+>> NullCheck [<<Phi>>] klass:SuperInterface exact:false
+ /// CHECK: InvokeInterface [<<NullCheck>>]
+ private void testInlinerWidensReturnType(boolean cond, Interface a, OtherInterface b) {
+ getWiderType(cond, a, b).superInterfaceMethod();
}
/// CHECK-START: void Main.testInlinerReturnsNull() inliner (before)
@@ -593,6 +598,26 @@
o.mainField = 0;
}
+ /// CHECK-START: void Main.testPhiHasOnlyNullInputs(boolean) inliner (before)
+ /// CHECK: <<Int:i\d+>> IntConstant 0
+ /// CHECK: <<Phi:l\d+>> Phi klass:Main exact:false
+ /// CHECK: <<NullCheck:l\d+>> NullCheck [<<Phi>>] klass:Main exact:false
+ /// CHECK: InstanceFieldSet [<<NullCheck>>,<<Int>>]
+
+ /// CHECK-START: void Main.testPhiHasOnlyNullInputs(boolean) inliner (after)
+ /// CHECK: <<Int:i\d+>> IntConstant 0
+ /// CHECK: <<Null:l\d+>> NullConstant klass:java.lang.Object
+ /// CHECK: <<Phi:l\d+>> Phi [<<Null>>,<<Null>>] klass:java.lang.Object exact:false
+ /// CHECK: <<NullCheck:l\d+>> NullCheck [<<Phi>>] klass:java.lang.Object exact:false
+ /// CHECK: InstanceFieldSet [<<NullCheck>>,<<Int>>]
+ private void testPhiHasOnlyNullInputs(boolean cond) {
+ Main o = cond ? null : getNull();
+ o.mainField = 0;
+ // getSuper() will force a type propagation after inlining
+ // because returns a more precise type.
+ getSuper();
+ }
+
public static void main(String[] args) {
}
}
diff --git a/test/455-set-vreg/expected.txt b/test/455-set-vreg/expected.txt
deleted file mode 100644
index 6a5618e..0000000
--- a/test/455-set-vreg/expected.txt
+++ /dev/null
@@ -1 +0,0 @@
-JNI_OnLoad called
diff --git a/test/455-set-vreg/info.txt b/test/455-set-vreg/info.txt
deleted file mode 100644
index e8c57b5..0000000
--- a/test/455-set-vreg/info.txt
+++ /dev/null
@@ -1 +0,0 @@
-Tests for setting DEX registers in a Java method.
diff --git a/test/455-set-vreg/set_vreg_jni.cc b/test/455-set-vreg/set_vreg_jni.cc
deleted file mode 100644
index 21149f6..0000000
--- a/test/455-set-vreg/set_vreg_jni.cc
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright (C) 2015 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "arch/context.h"
-#include "art_method-inl.h"
-#include "jni.h"
-#include "scoped_thread_state_change.h"
-#include "stack.h"
-#include "thread.h"
-
-namespace art {
-
-namespace {
-
-class TestVisitor : public StackVisitor {
- public:
- TestVisitor(Thread* thread, Context* context, mirror::Object* this_value)
- SHARED_REQUIRES(Locks::mutator_lock_)
- : StackVisitor(thread, context, StackVisitor::StackWalkKind::kIncludeInlinedFrames),
- this_value_(this_value) {}
-
- bool VisitFrame() SHARED_REQUIRES(Locks::mutator_lock_) {
- ArtMethod* m = GetMethod();
- std::string m_name(m->GetName());
-
- if (m_name.compare("testIntVReg") == 0) {
- uint32_t value = 0;
- CHECK(GetVReg(m, 1, kReferenceVReg, &value));
- CHECK_EQ(reinterpret_cast<mirror::Object*>(value), this_value_);
-
- CHECK(SetVReg(m, 2, 5, kIntVReg));
- CHECK(SetVReg(m, 3, 4, kIntVReg));
- CHECK(SetVReg(m, 4, 3, kIntVReg));
- CHECK(SetVReg(m, 5, 2, kIntVReg));
- CHECK(SetVReg(m, 6, 1, kIntVReg));
- } else if (m_name.compare("testLongVReg") == 0) {
- uint32_t value = 0;
- CHECK(GetVReg(m, 3, kReferenceVReg, &value));
- CHECK_EQ(reinterpret_cast<mirror::Object*>(value), this_value_);
-
- CHECK(SetVRegPair(m, 4, std::numeric_limits<int64_t>::max(), kLongLoVReg, kLongHiVReg));
- CHECK(SetVRegPair(m, 6, 4, kLongLoVReg, kLongHiVReg));
- CHECK(SetVRegPair(m, 8, 3, kLongLoVReg, kLongHiVReg));
- CHECK(SetVRegPair(m, 10, 2, kLongLoVReg, kLongHiVReg));
- CHECK(SetVRegPair(m, 12, 1, kLongLoVReg, kLongHiVReg));
- } else if (m_name.compare("testFloatVReg") == 0) {
- uint32_t value = 0;
- CHECK(GetVReg(m, 1, kReferenceVReg, &value));
- CHECK_EQ(reinterpret_cast<mirror::Object*>(value), this_value_);
-
- CHECK(SetVReg(m, 2, bit_cast<uint32_t, float>(5.0f), kFloatVReg));
- CHECK(SetVReg(m, 3, bit_cast<uint32_t, float>(4.0f), kFloatVReg));
- CHECK(SetVReg(m, 4, bit_cast<uint32_t, float>(3.0f), kFloatVReg));
- CHECK(SetVReg(m, 5, bit_cast<uint32_t, float>(2.0f), kFloatVReg));
- CHECK(SetVReg(m, 6, bit_cast<uint32_t, float>(1.0f), kFloatVReg));
- } else if (m_name.compare("testDoubleVReg") == 0) {
- uint32_t value = 0;
- CHECK(GetVReg(m, 3, kReferenceVReg, &value));
- CHECK_EQ(reinterpret_cast<mirror::Object*>(value), this_value_);
-
- CHECK(SetVRegPair(m, 4, bit_cast<uint64_t, double>(5.0), kDoubleLoVReg, kDoubleHiVReg));
- CHECK(SetVRegPair(m, 6, bit_cast<uint64_t, double>(4.0), kDoubleLoVReg, kDoubleHiVReg));
- CHECK(SetVRegPair(m, 8, bit_cast<uint64_t, double>(3.0), kDoubleLoVReg, kDoubleHiVReg));
- CHECK(SetVRegPair(m, 10, bit_cast<uint64_t, double>(2.0), kDoubleLoVReg, kDoubleHiVReg));
- CHECK(SetVRegPair(m, 12, bit_cast<uint64_t, double>(1.0), kDoubleLoVReg, kDoubleHiVReg));
- }
-
- return true;
- }
-
- mirror::Object* this_value_;
-};
-
-extern "C" JNIEXPORT void JNICALL Java_Main_doNativeCallSetVReg(JNIEnv*, jobject value) {
- ScopedObjectAccess soa(Thread::Current());
- std::unique_ptr<Context> context(Context::Create());
- TestVisitor visitor(soa.Self(), context.get(), soa.Decode<mirror::Object*>(value));
- visitor.WalkStack();
-}
-
-} // namespace
-
-} // namespace art
diff --git a/test/455-set-vreg/src/Main.java b/test/455-set-vreg/src/Main.java
deleted file mode 100644
index 4db9d66..0000000
--- a/test/455-set-vreg/src/Main.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright (C) 2015 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-public class Main {
- public Main() {
- }
-
- int testIntVReg(int a, int b, int c, int d, int e) {
- doNativeCallSetVReg();
- return a - b - c - d - e;
- }
-
- long testLongVReg(long a, long b, long c, long d, long e) {
- doNativeCallSetVReg();
- return a - b - c - d - e;
- }
-
- float testFloatVReg(float a, float b, float c, float d, float e) {
- doNativeCallSetVReg();
- return a - b - c - d - e;
- }
-
- double testDoubleVReg(double a, double b, double c, double d, double e) {
- doNativeCallSetVReg();
- return a - b - c - d - e;
- }
-
- native void doNativeCallSetVReg();
-
- public static void main(String[] args) {
- System.loadLibrary(args[0]);
- Main rm = new Main();
- int intExpected = 5 - 4 - 3 - 2 - 1;
- int intResult = rm.testIntVReg(0, 0, 0, 0, 0);
- if (intResult != intExpected) {
- throw new Error("Expected " + intExpected + ", got " + intResult);
- }
-
- long longExpected = Long.MAX_VALUE - 4 - 3 - 2 - 1;
- long longResult = rm.testLongVReg(0, 0, 0, 0, 0);
- if (longResult != longExpected) {
- throw new Error("Expected " + longExpected + ", got " + longResult);
- }
-
- float floatExpected = 5.0f - 4.0f - 3.0f - 2.0f - 1.0f;
- float floatResult = rm.testFloatVReg(0.0f, 0.0f, 0.0f, 0.0f, 0.0f);
- if (floatResult != floatExpected) {
- throw new Error("Expected " + floatExpected + ", got " + floatResult);
- }
-
- double doubleExpected = 5.0 - 4.0 - 3.0 - 2.0 - 1.0;
- double doubleResult = rm.testDoubleVReg(0.0, 0.0, 0.0, 0.0, 0.0);
- if (doubleResult != doubleExpected) {
- throw new Error("Expected " + doubleExpected + ", got " + doubleResult);
- }
- }
-}
diff --git a/test/457-regs/regs_jni.cc b/test/457-regs/regs_jni.cc
index 64b2336..79fa8b0 100644
--- a/test/457-regs/regs_jni.cc
+++ b/test/457-regs/regs_jni.cc
@@ -68,7 +68,7 @@
CHECK(!success);
}
- CHECK(GetVReg(m, 3, kReferenceVReg, &value));
+ CHECK(GetVReg(m, 3, kIntVReg, &value));
CHECK_EQ(value, 1u);
CHECK(GetVReg(m, 4, kFloatVReg, &value));
diff --git a/test/458-checker-instruction-simplification/src/Main.java b/test/458-checker-instruction-simplification/src/Main.java
index c32d34a..d5fed2a 100644
--- a/test/458-checker-instruction-simplification/src/Main.java
+++ b/test/458-checker-instruction-simplification/src/Main.java
@@ -1226,6 +1226,46 @@
return arg / -0.25f;
}
+ /**
+ * Test strength reduction of factors of the form (2^n + 1).
+ */
+
+ /// CHECK-START: int Main.mulPow2Plus1(int) instruction_simplifier (before)
+ /// CHECK-DAG: <<Arg:i\d+>> ParameterValue
+ /// CHECK-DAG: <<Const9:i\d+>> IntConstant 9
+ /// CHECK: Mul [<<Arg>>,<<Const9>>]
+
+ /// CHECK-START: int Main.mulPow2Plus1(int) instruction_simplifier (after)
+ /// CHECK-DAG: <<Arg:i\d+>> ParameterValue
+ /// CHECK-DAG: <<Const3:i\d+>> IntConstant 3
+ /// CHECK: <<Shift:i\d+>> Shl [<<Arg>>,<<Const3>>]
+ /// CHECK-NEXT: Add [<<Arg>>,<<Shift>>]
+
+ public static int mulPow2Plus1(int arg) {
+ return arg * 9;
+ }
+
+
+ /**
+ * Test strength reduction of factors of the form (2^n - 1).
+ */
+
+ /// CHECK-START: long Main.mulPow2Minus1(long) instruction_simplifier (before)
+ /// CHECK-DAG: <<Arg:j\d+>> ParameterValue
+ /// CHECK-DAG: <<Const31:j\d+>> LongConstant 31
+ /// CHECK: Mul [<<Arg>>,<<Const31>>]
+
+ /// CHECK-START: long Main.mulPow2Minus1(long) instruction_simplifier (after)
+ /// CHECK-DAG: <<Arg:j\d+>> ParameterValue
+ /// CHECK-DAG: <<Const5:i\d+>> IntConstant 5
+ /// CHECK: <<Shift:j\d+>> Shl [<<Arg>>,<<Const5>>]
+ /// CHECK-NEXT: Sub [<<Shift>>,<<Arg>>]
+
+ public static long mulPow2Minus1(long arg) {
+ return arg * 31;
+ }
+
+
public static void main(String[] args) {
int arg = 123456;
@@ -1283,5 +1323,15 @@
assertLongEquals(Shr56And255(0xc123456787654321L), 0xc1L);
assertIntEquals(Shr24And127(0xc1234567), 0x41);
assertLongEquals(Shr56And127(0xc123456787654321L), 0x41L);
+ assertIntEquals(0, mulPow2Plus1(0));
+ assertIntEquals(9, mulPow2Plus1(1));
+ assertIntEquals(18, mulPow2Plus1(2));
+ assertIntEquals(900, mulPow2Plus1(100));
+ assertIntEquals(111105, mulPow2Plus1(12345));
+ assertLongEquals(0, mulPow2Minus1(0));
+ assertLongEquals(31, mulPow2Minus1(1));
+ assertLongEquals(62, mulPow2Minus1(2));
+ assertLongEquals(3100, mulPow2Minus1(100));
+ assertLongEquals(382695, mulPow2Minus1(12345));
}
}
diff --git a/test/478-checker-clinit-check-pruning/expected.txt b/test/478-checker-clinit-check-pruning/expected.txt
index 387e1a7..7de097f 100644
--- a/test/478-checker-clinit-check-pruning/expected.txt
+++ b/test/478-checker-clinit-check-pruning/expected.txt
@@ -4,3 +4,9 @@
Main$ClassWithClinit4's static initializer
Main$ClassWithClinit5's static initializer
Main$ClassWithClinit6's static initializer
+Main$ClassWithClinit7's static initializer
+Main$ClassWithClinit8's static initializer
+Main$ClassWithClinit9's static initializer
+Main$ClassWithClinit10's static initializer
+Main$ClassWithClinit11's static initializer
+Main$ClassWithClinit12's static initializer
diff --git a/test/478-checker-clinit-check-pruning/src/Main.java b/test/478-checker-clinit-check-pruning/src/Main.java
index cff6273..7993513 100644
--- a/test/478-checker-clinit-check-pruning/src/Main.java
+++ b/test/478-checker-clinit-check-pruning/src/Main.java
@@ -83,7 +83,7 @@
// before the next pass (liveness analysis) instead.
/// CHECK-START: void Main.invokeStaticNotInlined() liveness (before)
- /// CHECK: InvokeStaticOrDirect
+ /// CHECK: InvokeStaticOrDirect clinit_check:implicit
/// CHECK-START: void Main.invokeStaticNotInlined() liveness (before)
/// CHECK-NOT: LoadClass
@@ -269,7 +269,7 @@
/// CHECK-START: void Main.noClinitBecauseOfInvokeStatic() liveness (before)
/// CHECK-DAG: <<IntConstant:i\d+>> IntConstant 0
/// CHECK-DAG: <<LoadClass:l\d+>> LoadClass gen_clinit_check:false
- /// CHECK-DAG: InvokeStaticOrDirect
+ /// CHECK-DAG: InvokeStaticOrDirect clinit_check:implicit
/// CHECK-DAG: StaticFieldSet [<<LoadClass>>,<<IntConstant>>]
/// CHECK-START: void Main.noClinitBecauseOfInvokeStatic() liveness (before)
@@ -289,7 +289,7 @@
/// CHECK-DAG: <<IntConstant:i\d+>> IntConstant 0
/// CHECK-DAG: <<LoadClass:l\d+>> LoadClass gen_clinit_check:true
/// CHECK-DAG: StaticFieldSet [<<LoadClass>>,<<IntConstant>>]
- /// CHECK-DAG: InvokeStaticOrDirect
+ /// CHECK-DAG: InvokeStaticOrDirect clinit_check:none
/// CHECK-START: void Main.clinitBecauseOfFieldAccess() liveness (before)
/// CHECK-NOT: ClinitCheck
@@ -298,6 +298,206 @@
ClassWithClinit2.$noinline$staticMethod();
}
+ /*
+ * Verify that LoadClass from const-class is not merged with
+ * later invoke-static (or it's ClinitCheck).
+ */
+
+ /// CHECK-START: void Main.constClassAndInvokeStatic(java.lang.Iterable) liveness (before)
+ /// CHECK: LoadClass gen_clinit_check:false
+ /// CHECK: InvokeStaticOrDirect clinit_check:implicit
+
+ /// CHECK-START: void Main.constClassAndInvokeStatic(java.lang.Iterable) liveness (before)
+ /// CHECK-NOT: ClinitCheck
+
+ static void constClassAndInvokeStatic(Iterable it) {
+ $opt$inline$ignoreClass(ClassWithClinit7.class);
+ ClassWithClinit7.someStaticMethod(it);
+ }
+
+ static void $opt$inline$ignoreClass(Class c) {
+ }
+
+ static class ClassWithClinit7 {
+ static {
+ System.out.println("Main$ClassWithClinit7's static initializer");
+ }
+
+ // Note: not inlined from constClassAndInvokeStatic() but fully inlined from main().
+ static void someStaticMethod(Iterable it) {
+ // We're not inlining invoke-interface at the moment.
+ it.iterator();
+ }
+ }
+
+ /*
+ * Verify that LoadClass from sget is not merged with later invoke-static.
+ */
+
+ /// CHECK-START: void Main.sgetAndInvokeStatic(java.lang.Iterable) liveness (before)
+ /// CHECK: LoadClass gen_clinit_check:true
+ /// CHECK: InvokeStaticOrDirect clinit_check:none
+
+ /// CHECK-START: void Main.sgetAndInvokeStatic(java.lang.Iterable) liveness (before)
+ /// CHECK-NOT: ClinitCheck
+
+ static void sgetAndInvokeStatic(Iterable it) {
+ $opt$inline$ignoreInt(ClassWithClinit8.value);
+ ClassWithClinit8.someStaticMethod(it);
+ }
+
+ static void $opt$inline$ignoreInt(int i) {
+ }
+
+ static class ClassWithClinit8 {
+ public static int value = 0;
+ static {
+ System.out.println("Main$ClassWithClinit8's static initializer");
+ }
+
+ // Note: not inlined from sgetAndInvokeStatic() but fully inlined from main().
+ static void someStaticMethod(Iterable it) {
+ // We're not inlining invoke-interface at the moment.
+ it.iterator();
+ }
+ }
+
+ /*
+ * Verify that LoadClass from const-class, ClinitCheck from sget and
+ * InvokeStaticOrDirect from invoke-static are not merged.
+ */
+
+ /// CHECK-START: void Main.constClassSgetAndInvokeStatic(java.lang.Iterable) liveness (before)
+ /// CHECK: LoadClass gen_clinit_check:false
+ /// CHECK: ClinitCheck
+ /// CHECK: InvokeStaticOrDirect clinit_check:none
+
+ static void constClassSgetAndInvokeStatic(Iterable it) {
+ $opt$inline$ignoreClass(ClassWithClinit9.class);
+ $opt$inline$ignoreInt(ClassWithClinit9.value);
+ ClassWithClinit9.someStaticMethod(it);
+ }
+
+ static class ClassWithClinit9 {
+ public static int value = 0;
+ static {
+ System.out.println("Main$ClassWithClinit9's static initializer");
+ }
+
+ // Note: not inlined from constClassSgetAndInvokeStatic() but fully inlined from main().
+ static void someStaticMethod(Iterable it) {
+ // We're not inlining invoke-interface at the moment.
+ it.iterator();
+ }
+ }
+
+ /*
+ * Verify that LoadClass from a fully-inlined invoke-static is not merged
+ * with InvokeStaticOrDirect from a later invoke-static to the same method.
+ */
+
+ /// CHECK-START: void Main.inlinedInvokeStaticViaNonStatic(java.lang.Iterable) liveness (before)
+ /// CHECK: LoadClass gen_clinit_check:true
+ /// CHECK: InvokeStaticOrDirect clinit_check:none
+
+ /// CHECK-START: void Main.inlinedInvokeStaticViaNonStatic(java.lang.Iterable) liveness (before)
+ /// CHECK-NOT: ClinitCheck
+
+ static void inlinedInvokeStaticViaNonStatic(Iterable it) {
+ inlinedInvokeStaticViaNonStaticHelper(null);
+ inlinedInvokeStaticViaNonStaticHelper(it);
+ }
+
+ static void inlinedInvokeStaticViaNonStaticHelper(Iterable it) {
+ ClassWithClinit10.inlinedForNull(it);
+ }
+
+ static class ClassWithClinit10 {
+ public static int value = 0;
+ static {
+ System.out.println("Main$ClassWithClinit10's static initializer");
+ }
+
+ static void inlinedForNull(Iterable it) {
+ if (it != null) {
+ // We're not inlining invoke-interface at the moment.
+ it.iterator();
+ }
+ }
+ }
+
+ /*
+ * Check that the LoadClass from an invoke-static C.foo() doesn't get merged with
+ * an invoke-static inside C.foo(). This would mess up the stack walk in the
+ * resolution trampoline where we would have to load C (if C isn't loaded yet)
+ * which is not permitted there.
+ *
+ * Note: In case of failure, we would get an failed assertion during compilation,
+ * so we wouldn't really get to the checker tests below.
+ */
+
+ /// CHECK-START: void Main.inlinedInvokeStaticViaStatic(java.lang.Iterable) liveness (before)
+ /// CHECK: LoadClass gen_clinit_check:true
+ /// CHECK: InvokeStaticOrDirect clinit_check:none
+
+ /// CHECK-START: void Main.inlinedInvokeStaticViaStatic(java.lang.Iterable) liveness (before)
+ /// CHECK-NOT: ClinitCheck
+
+ static void inlinedInvokeStaticViaStatic(Iterable it) {
+ ClassWithClinit11.callInlinedForNull(it);
+ }
+
+ static class ClassWithClinit11 {
+ public static int value = 0;
+ static {
+ System.out.println("Main$ClassWithClinit11's static initializer");
+ }
+
+ static void callInlinedForNull(Iterable it) {
+ inlinedForNull(it);
+ }
+
+ static void inlinedForNull(Iterable it) {
+ // We're not inlining invoke-interface at the moment.
+ it.iterator();
+ }
+ }
+
+ /*
+ * A test similar to inlinedInvokeStaticViaStatic() but doing the indirect invoke
+ * twice with the first one to be fully inlined.
+ */
+
+ /// CHECK-START: void Main.inlinedInvokeStaticViaStaticTwice(java.lang.Iterable) liveness (before)
+ /// CHECK: LoadClass gen_clinit_check:true
+ /// CHECK: InvokeStaticOrDirect clinit_check:none
+
+ /// CHECK-START: void Main.inlinedInvokeStaticViaStaticTwice(java.lang.Iterable) liveness (before)
+ /// CHECK-NOT: ClinitCheck
+
+ static void inlinedInvokeStaticViaStaticTwice(Iterable it) {
+ ClassWithClinit12.callInlinedForNull(null);
+ ClassWithClinit12.callInlinedForNull(it);
+ }
+
+ static class ClassWithClinit12 {
+ public static int value = 0;
+ static {
+ System.out.println("Main$ClassWithClinit12's static initializer");
+ }
+
+ static void callInlinedForNull(Iterable it) {
+ inlinedForNull(it);
+ }
+
+ static void inlinedForNull(Iterable it) {
+ if (it != null) {
+ // We're not inlining invoke-interface at the moment.
+ it.iterator();
+ }
+ }
+ }
+
// TODO: Add a test for the case of a static method whose declaring
// class type index is not available (i.e. when `storage_index`
// equals `DexFile::kDexNoIndex` in
@@ -310,5 +510,12 @@
ClassWithClinit4.invokeStaticNotInlined();
SubClassOfClassWithClinit5.invokeStaticInlined();
SubClassOfClassWithClinit6.invokeStaticNotInlined();
+ Iterable it = new Iterable() { public java.util.Iterator iterator() { return null; } };
+ constClassAndInvokeStatic(it);
+ sgetAndInvokeStatic(it);
+ constClassSgetAndInvokeStatic(it);
+ inlinedInvokeStaticViaNonStatic(it);
+ inlinedInvokeStaticViaStatic(it);
+ inlinedInvokeStaticViaStaticTwice(it);
}
}
diff --git a/test/485-checker-dce-loop-update/smali/TestCase.smali b/test/485-checker-dce-loop-update/smali/TestCase.smali
index ab4afdb..1de0bae 100644
--- a/test/485-checker-dce-loop-update/smali/TestCase.smali
+++ b/test/485-checker-dce-loop-update/smali/TestCase.smali
@@ -136,11 +136,11 @@
## CHECK-DAG: <<Cst1:i\d+>> IntConstant 1
## CHECK-DAG: <<Cst5:i\d+>> IntConstant 5
## CHECK-DAG: <<Cst7:i\d+>> IntConstant 7
-## CHECK-DAG: <<Cst9:i\d+>> IntConstant 9
+## CHECK-DAG: <<Cst11:i\d+>> IntConstant 11
## CHECK-DAG: <<PhiX1:i\d+>> Phi [<<ArgX>>,<<Add5:i\d+>>,<<Add7:i\d+>>] loop:<<HeaderY:B\d+>>
## CHECK-DAG: If [<<ArgY>>] loop:<<HeaderY>>
## CHECK-DAG: If [<<ArgZ>>] loop:<<HeaderY>>
-## CHECK-DAG: <<Mul9:i\d+>> Mul [<<PhiX1>>,<<Cst9>>] loop:<<HeaderY>>
+## CHECK-DAG: <<Mul9:i\d+>> Mul [<<PhiX1>>,<<Cst11>>] loop:<<HeaderY>>
## CHECK-DAG: <<PhiX2:i\d+>> Phi [<<PhiX1>>,<<Mul9>>] loop:<<HeaderY>>
## CHECK-DAG: If [<<Cst1>>] loop:<<HeaderY>>
## CHECK-DAG: <<Add5>> Add [<<PhiX2>>,<<Cst5>>] loop:<<HeaderY>>
@@ -152,12 +152,12 @@
## CHECK-DAG: <<ArgY:z\d+>> ParameterValue
## CHECK-DAG: <<ArgZ:z\d+>> ParameterValue
## CHECK-DAG: <<Cst7:i\d+>> IntConstant 7
-## CHECK-DAG: <<Cst9:i\d+>> IntConstant 9
+## CHECK-DAG: <<Cst11:i\d+>> IntConstant 11
## CHECK-DAG: <<PhiX1:i\d+>> Phi [<<ArgX>>,<<Add7:i\d+>>] loop:<<HeaderY:B\d+>>
## CHECK-DAG: If [<<ArgY>>] loop:<<HeaderY>>
## CHECK-DAG: <<Add7>> Add [<<PhiX1>>,<<Cst7>>] loop:<<HeaderY>>
## CHECK-DAG: If [<<ArgZ>>] loop:none
-## CHECK-DAG: <<Mul9:i\d+>> Mul [<<PhiX1>>,<<Cst9>>] loop:none
+## CHECK-DAG: <<Mul9:i\d+>> Mul [<<PhiX1>>,<<Cst11>>] loop:none
## CHECK-DAG: <<PhiX2:i\d+>> Phi [<<PhiX1>>,<<Mul9>>] loop:none
## CHECK-DAG: Return [<<PhiX2>>] loop:none
@@ -177,7 +177,7 @@
# Additional logic which will end up outside the loop
if-eqz p2, :skip_if
- mul-int/lit8 p0, p0, 9
+ mul-int/lit8 p0, p0, 11
:skip_if
if-nez v0, :loop_end # will always take the branch
diff --git a/test/530-checker-lse/src/Main.java b/test/530-checker-lse/src/Main.java
index c766aaa..13c4722 100644
--- a/test/530-checker-lse/src/Main.java
+++ b/test/530-checker-lse/src/Main.java
@@ -22,7 +22,7 @@
return radius * radius * Math.PI;
}
private double radius;
-};
+}
class TestClass {
TestClass() {
@@ -35,17 +35,31 @@
int j;
volatile int k;
TestClass next;
+ String str;
static int si;
-};
+}
class SubTestClass extends TestClass {
int k;
-};
+}
class TestClass2 {
int i;
int j;
-};
+}
+
+class Finalizable {
+ static boolean sVisited = false;
+ static final int VALUE = 0xbeef;
+ int i;
+
+ protected void finalize() {
+ if (i != VALUE) {
+ System.out.println("Where is the beef?");
+ }
+ sVisited = true;
+ }
+}
public class Main {
@@ -56,7 +70,7 @@
/// CHECK-START: double Main.calcCircleArea(double) load_store_elimination (after)
/// CHECK: NewInstance
- /// CHECK: InstanceFieldSet
+ /// CHECK-NOT: InstanceFieldSet
/// CHECK-NOT: InstanceFieldGet
static double calcCircleArea(double radius) {
@@ -117,7 +131,7 @@
/// CHECK: InstanceFieldGet
/// CHECK: InstanceFieldSet
/// CHECK: NewInstance
- /// CHECK: InstanceFieldSet
+ /// CHECK-NOT: InstanceFieldSet
/// CHECK-NOT: InstanceFieldGet
// A new allocation shouldn't alias with pre-existing values.
@@ -223,7 +237,7 @@
/// CHECK-START: int Main.test8() load_store_elimination (after)
/// CHECK: NewInstance
- /// CHECK: InstanceFieldSet
+ /// CHECK-NOT: InstanceFieldSet
/// CHECK: InvokeVirtual
/// CHECK-NOT: NullCheck
/// CHECK-NOT: InstanceFieldGet
@@ -381,8 +395,8 @@
/// CHECK-START: int Main.test16() load_store_elimination (after)
/// CHECK: NewInstance
- /// CHECK-NOT: StaticFieldSet
- /// CHECK-NOT: StaticFieldGet
+ /// CHECK-NOT: InstanceFieldSet
+ /// CHECK-NOT: InstanceFieldGet
// Test inlined constructor.
static int test16() {
@@ -398,8 +412,8 @@
/// CHECK-START: int Main.test17() load_store_elimination (after)
/// CHECK: <<Const0:i\d+>> IntConstant 0
/// CHECK: NewInstance
- /// CHECK-NOT: StaticFieldSet
- /// CHECK-NOT: StaticFieldGet
+ /// CHECK-NOT: InstanceFieldSet
+ /// CHECK-NOT: InstanceFieldGet
/// CHECK: Return [<<Const0>>]
// Test getting default value.
@@ -455,6 +469,148 @@
return obj;
}
+ /// CHECK-START: void Main.test21() load_store_elimination (before)
+ /// CHECK: NewInstance
+ /// CHECK: InstanceFieldSet
+ /// CHECK: StaticFieldSet
+ /// CHECK: StaticFieldGet
+
+ /// CHECK-START: void Main.test21() load_store_elimination (after)
+ /// CHECK: NewInstance
+ /// CHECK: InstanceFieldSet
+ /// CHECK: StaticFieldSet
+ /// CHECK: InstanceFieldGet
+
+ // Loop side effects can kill heap values, stores need to be kept in that case.
+ static void test21() {
+ TestClass obj = new TestClass();
+ obj.str = "abc";
+ for (int i = 0; i < 2; i++) {
+ // Generate some loop side effect that does write.
+ obj.si = 1;
+ }
+ System.out.print(obj.str.substring(0, 0));
+ }
+
+ /// CHECK-START: int Main.test22() load_store_elimination (before)
+ /// CHECK: NewInstance
+ /// CHECK: InstanceFieldSet
+ /// CHECK: NewInstance
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldGet
+ /// CHECK: NewInstance
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldGet
+ /// CHECK: InstanceFieldGet
+
+ /// CHECK-START: int Main.test22() load_store_elimination (after)
+ /// CHECK: NewInstance
+ /// CHECK: InstanceFieldSet
+ /// CHECK: NewInstance
+ /// CHECK-NOT: InstanceFieldSet
+ /// CHECK-NOT: InstanceFieldGet
+ /// CHECK: NewInstance
+ /// CHECK-NOT: InstanceFieldSet
+ /// CHECK: InstanceFieldGet
+ /// CHECK-NOT: InstanceFieldGet
+
+ // Loop side effects only affects stores into singletons that dominiates the loop header.
+ static int test22() {
+ int sum = 0;
+ TestClass obj1 = new TestClass();
+ obj1.i = 2; // This store can't be eliminated since it can be killed by loop side effects.
+ for (int i = 0; i < 2; i++) {
+ TestClass obj2 = new TestClass();
+ obj2.i = 3; // This store can be eliminated since the singleton is inside the loop.
+ sum += obj2.i;
+ }
+ TestClass obj3 = new TestClass();
+ obj3.i = 5; // This store can be eliminated since the singleton is created after the loop.
+ sum += obj1.i + obj3.i;
+ return sum;
+ }
+
+ /// CHECK-START: int Main.test23(boolean) load_store_elimination (before)
+ /// CHECK: NewInstance
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldGet
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldGet
+ /// CHECK: Return
+ /// CHECK: InstanceFieldGet
+ /// CHECK: InstanceFieldSet
+
+ /// CHECK-START: int Main.test23(boolean) load_store_elimination (after)
+ /// CHECK: NewInstance
+ /// CHECK-NOT: InstanceFieldSet
+ /// CHECK-NOT: InstanceFieldGet
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldGet
+ /// CHECK: Return
+ /// CHECK-NOT: InstanceFieldGet
+ /// CHECK: InstanceFieldSet
+
+ // Test store elimination on merging.
+ static int test23(boolean b) {
+ TestClass obj = new TestClass();
+ obj.i = 3; // This store can be eliminated since the value flows into each branch.
+ if (b) {
+ obj.i += 1; // This store cannot be eliminated due to the merge later.
+ } else {
+ obj.i += 2; // This store cannot be eliminated due to the merge later.
+ }
+ return obj.i;
+ }
+
+ /// CHECK-START: void Main.testFinalizable() load_store_elimination (before)
+ /// CHECK: NewInstance
+ /// CHECK: InstanceFieldSet
+
+ /// CHECK-START: void Main.testFinalizable() load_store_elimination (after)
+ /// CHECK: NewInstance
+ /// CHECK: InstanceFieldSet
+
+ // Allocations and stores into finalizable objects cannot be eliminated.
+ static void testFinalizable() {
+ Finalizable finalizable = new Finalizable();
+ finalizable.i = Finalizable.VALUE;
+ }
+
+ static java.lang.ref.WeakReference<Object> getWeakReference() {
+ return new java.lang.ref.WeakReference<>(new Object());
+ }
+
+ static void testFinalizableByForcingGc() {
+ testFinalizable();
+ java.lang.ref.WeakReference<Object> reference = getWeakReference();
+
+ Runtime runtime = Runtime.getRuntime();
+ for (int i = 0; i < 20; ++i) {
+ runtime.gc();
+ System.runFinalization();
+ try {
+ Thread.sleep(1);
+ } catch (InterruptedException e) {
+ throw new AssertionError(e);
+ }
+
+ // Check to see if the weak reference has been garbage collected.
+ if (reference.get() == null) {
+ // A little bit more sleep time to make sure.
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ throw new AssertionError(e);
+ }
+ if (!Finalizable.sVisited) {
+ System.out.println("finalize() not called.");
+ }
+ return;
+ }
+ }
+ System.out.println("testFinalizableByForcingGc() failed to force gc.");
+ }
+
public static void assertIntEquals(int expected, int result) {
if (expected != result) {
throw new Error("Expected: " + expected + ", found: " + result);
@@ -508,5 +664,10 @@
float[] fa2 = { 1.8f };
assertFloatEquals(test19(fa1, fa2), 1.8f);
assertFloatEquals(test20().i, 0);
+ test21();
+ assertIntEquals(test22(), 13);
+ assertIntEquals(test23(true), 4);
+ assertIntEquals(test23(false), 5);
+ testFinalizableByForcingGc();
}
}
diff --git a/test/537-checker-jump-over-jump/src/Main.java b/test/537-checker-jump-over-jump/src/Main.java
index fb666ea..cf9a69d 100644
--- a/test/537-checker-jump-over-jump/src/Main.java
+++ b/test/537-checker-jump-over-jump/src/Main.java
@@ -20,13 +20,25 @@
public static int[] fibs;
/// CHECK-START-X86_64: int Main.test() disassembly (after)
- /// CHECK: If
- /// CHECK-NEXT: cmp
- /// CHECK-NEXT: jnl/ge
- /// CHECK-NOT: jmp
- /// CHECK: ArrayGet
- // Checks that there is no conditional jump over a jmp. The ArrayGet is in
- // the next block.
+ /// CHECK-DAG: <<Zero:i\d+>> IntConstant 0
+ //
+ /// CHECK: If
+ /// CHECK-NEXT: cmp
+ /// CHECK-NEXT: jnl/ge
+ //
+ /// CHECK-DAG: <<Fibs:l\d+>> StaticFieldGet
+ /// CHECK-DAG: NullCheck [<<Fibs>>]
+ /// CHECK-NOT: jmp
+ /// CHECK-DAG: <<FibsAtZero:i\d+>> ArrayGet [<<Fibs>>,<<Zero>>]
+ /// CHECK-DAG: Return [<<FibsAtZero>>]
+ //
+ // Checks that there is no conditional jump over a `jmp`
+ // instruction. The `ArrayGet` instruction is in the next block.
+ //
+ // Note that the `StaticFieldGet` HIR instruction above (captured as
+ // `Fibs`) can produce a `jmp` x86-64 instruction when read barriers
+ // are enabled (to jump into the read barrier slow path), which is
+ // different from the `jmp` in the `CHECK-NOT` assertion.
public static int test() {
for (int i = 1; ; i++) {
if (i >= FIBCOUNT) {
diff --git a/test/549-checker-types-merge/expected.txt b/test/549-checker-types-merge/expected.txt
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/549-checker-types-merge/expected.txt
diff --git a/test/549-checker-types-merge/info.txt b/test/549-checker-types-merge/info.txt
new file mode 100644
index 0000000..f174e20
--- /dev/null
+++ b/test/549-checker-types-merge/info.txt
@@ -0,0 +1 @@
+Checker test for testing the type merge during reference type propagation.
diff --git a/test/549-checker-types-merge/src/Main.java b/test/549-checker-types-merge/src/Main.java
new file mode 100644
index 0000000..dc27f10
--- /dev/null
+++ b/test/549-checker-types-merge/src/Main.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// To make it easier to follow the tests:
+// - all interfaces defined in this file extend InterfaceSuper (except InterfaceOtherSuper)
+// - all classes defined in this file extend ClassSuper (except ClassOtherSuper)
+
+interface InterfaceSuper {}
+interface InterfaceOtherSuper {}
+
+interface InterfaceA extends InterfaceSuper {}
+interface InterfaceB extends InterfaceSuper {}
+interface InterfaceExtendsA extends InterfaceA {}
+interface InterfaceExtendsB extends InterfaceB {}
+
+class ClassSuper {}
+class ClassOtherSuper {}
+
+class ClassA extends ClassSuper {}
+class ClassB extends ClassSuper {}
+class ClassExtendsA extends ClassA {}
+class ClassExtendsB extends ClassB {}
+
+class ClassImplementsInterfaceA extends ClassSuper implements InterfaceA {}
+
+public class Main {
+
+ /// CHECK-START: java.lang.Object Main.testMergeNullContant(boolean) reference_type_propagation (after)
+ /// CHECK: <<Phi:l\d+>> Phi klass:Main
+ /// CHECK: Return [<<Phi>>]
+ private Object testMergeNullContant(boolean cond) {
+ return cond ? null : new Main();
+ }
+
+ /// CHECK-START: java.lang.Object Main.testMergeClasses(boolean, ClassExtendsA, ClassExtendsB) reference_type_propagation (after)
+ /// CHECK: <<Phi:l\d+>> Phi klass:ClassSuper
+ /// CHECK: Return [<<Phi>>]
+ private Object testMergeClasses(boolean cond, ClassExtendsA a, ClassExtendsB b) {
+ // Different classes, have a common super type.
+ return cond ? a : b;
+ }
+
+ /// CHECK-START: java.lang.Object Main.testMergeClasses(boolean, ClassExtendsA, ClassSuper) reference_type_propagation (after)
+ /// CHECK: <<Phi:l\d+>> Phi klass:ClassSuper
+ /// CHECK: Return [<<Phi>>]
+ private Object testMergeClasses(boolean cond, ClassExtendsA a, ClassSuper b) {
+ // Different classes, one is the super type of the other.
+ return cond ? a : b;
+ }
+
+ /// CHECK-START: java.lang.Object Main.testMergeClasses(boolean, ClassSuper, ClassSuper) reference_type_propagation (after)
+ /// CHECK: <<Phi:l\d+>> Phi klass:ClassSuper
+ /// CHECK: Return [<<Phi>>]
+ private Object testMergeClasses(boolean cond, ClassSuper a, ClassSuper b) {
+ // Same classes.
+ return cond ? a : b;
+ }
+
+ /// CHECK-START: java.lang.Object Main.testMergeClasses(boolean, ClassOtherSuper, ClassSuper) reference_type_propagation (after)
+ /// CHECK: <<Phi:l\d+>> Phi klass:java.lang.Object
+ /// CHECK: Return [<<Phi>>]
+ private Object testMergeClasses(boolean cond, ClassOtherSuper a, ClassSuper b) {
+ // Different classes, have Object as the common super type.
+ return cond ? a : b;
+ }
+
+ /// CHECK-START: java.lang.Object Main.testMergeClassWithInterface(boolean, ClassImplementsInterfaceA, InterfaceSuper) reference_type_propagation (after)
+ /// CHECK: <<Phi:l\d+>> Phi klass:InterfaceSuper
+ /// CHECK: Return [<<Phi>>]
+ private Object testMergeClassWithInterface(boolean cond, ClassImplementsInterfaceA a, InterfaceSuper b) {
+ // Class implements interface.
+ return cond ? a : b;
+ }
+
+ /// CHECK-START: java.lang.Object Main.testMergeClassWithInterface(boolean, ClassSuper, InterfaceSuper) reference_type_propagation (after)
+ /// CHECK: <<Phi:l\d+>> Phi klass:java.lang.Object
+ /// CHECK: Return [<<Phi>>]
+ private Object testMergeClassWithInterface(boolean cond, ClassSuper a, InterfaceSuper b) {
+ // Class doesn't implement interface.
+ return cond ? a : b;
+ }
+
+ /// CHECK-START: java.lang.Object Main.testMergeInterfaces(boolean, InterfaceExtendsA, InterfaceSuper) reference_type_propagation (after)
+ /// CHECK: <<Phi:l\d+>> Phi klass:InterfaceSuper
+ /// CHECK: Return [<<Phi>>]
+ private Object testMergeInterfaces(boolean cond, InterfaceExtendsA a, InterfaceSuper b) {
+ // Different Interfaces, one implements the other.
+ return cond ? a : b;
+ }
+
+ /// CHECK-START: java.lang.Object Main.testMergeInterfaces(boolean, InterfaceSuper, InterfaceSuper) reference_type_propagation (after)
+ /// CHECK: <<Phi:l\d+>> Phi klass:InterfaceSuper
+ /// CHECK: Return [<<Phi>>]
+ private Object testMergeInterfaces(boolean cond, InterfaceSuper a, InterfaceSuper b) {
+ // Same interfaces.
+ return cond ? a : b;
+ }
+
+ /// CHECK-START: java.lang.Object Main.testMergeInterfaces(boolean, InterfaceExtendsA, InterfaceExtendsB) reference_type_propagation (after)
+ /// CHECK: <<Phi:l\d+>> Phi klass:java.lang.Object
+ /// CHECK: Return [<<Phi>>]
+ private Object testMergeInterfaces(boolean cond, InterfaceExtendsA a, InterfaceExtendsB b) {
+ // Different Interfaces, have a common super type.
+ return cond ? a : b;
+ }
+
+ /// CHECK-START: java.lang.Object Main.testMergeInterfaces(boolean, InterfaceSuper, InterfaceOtherSuper) reference_type_propagation (after)
+ /// CHECK: <<Phi:l\d+>> Phi klass:java.lang.Object
+ /// CHECK: Return [<<Phi>>]
+ private Object testMergeInterfaces(boolean cond, InterfaceSuper a, InterfaceOtherSuper b) {
+ // Different interfaces.
+ return cond ? a : b;
+ }
+
+ public static void main(String[] args) {
+ }
+}
diff --git a/test/960-default-smali/build b/test/960-default-smali/build
index 4dc848c..b72afcd 100755
--- a/test/960-default-smali/build
+++ b/test/960-default-smali/build
@@ -22,7 +22,7 @@
# Should we compile with Java source code. By default we will use Smali.
USES_JAVA_SOURCE="false"
-if [[ $ARGS == *"--jvm"* ]]; then
+if [[ $@ == *"--jvm"* ]]; then
USES_JAVA_SOURCE="true"
elif [[ "$USE_JACK" == "true" ]]; then
if $JACK -D jack.java.source.version=1.8 >& /dev/null; then
diff --git a/test/961-default-iface-resolution-generated/build b/test/961-default-iface-resolution-generated/build
index b4ced3e..005f76c 100755
--- a/test/961-default-iface-resolution-generated/build
+++ b/test/961-default-iface-resolution-generated/build
@@ -33,7 +33,7 @@
# Should we compile with Java source code. By default we will use Smali.
USES_JAVA_SOURCE="false"
-if [[ $ARGS == *"--jvm"* ]]; then
+if [[ $@ == *"--jvm"* ]]; then
USES_JAVA_SOURCE="true"
elif [[ $USE_JACK == "true" ]]; then
if "$JACK" -D jack.java.source.version=1.8 >& /dev/null; then
diff --git a/test/Android.libarttest.mk b/test/Android.libarttest.mk
index bffd0e0..7a22e1b 100644
--- a/test/Android.libarttest.mk
+++ b/test/Android.libarttest.mk
@@ -34,7 +34,6 @@
139-register-natives/regnative.cc \
141-class-unload/jni_unload.cc \
454-get-vreg/get_vreg_jni.cc \
- 455-set-vreg/set_vreg_jni.cc \
457-regs/regs_jni.cc \
461-get-reference-vreg/get_reference_vreg_jni.cc \
466-get-live-vreg/get_live_vreg_jni.cc \
diff --git a/test/Android.run-test.mk b/test/Android.run-test.mk
index b528721..c830ad4 100644
--- a/test/Android.run-test.mk
+++ b/test/Android.run-test.mk
@@ -208,6 +208,10 @@
$(shell echo $(1) | tr '[:lower:]' '[:upper:]' | tr '-' '_')
endef # name-to-var
+ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
+ $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
+ $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(ART_TEST_RUN_TEST_SKIP), $(ALL_ADDRESS_SIZES))
+
# Tests that are timing sensitive and flaky on heavily loaded systems.
TEST_ART_TIMING_SENSITIVE_RUN_TESTS := \
053-wait-some \
@@ -305,12 +309,13 @@
# Tests that are broken with GC stress.
# * 137-cfi needs to unwind a second forked process. We're using a primitive sleep to wait till we
# hope the second process got into the expected state. The slowness of gcstress makes this bad.
-# * 961-default-iface-resolution-generated is a very long test that often will take more than the
-# timeout to run when gcstress is enabled. This is because gcstress slows down allocations
-# significantly which this test does a lot.
+# * 961-default-iface-resolution-generated and 964-default-iface-init-generated are very long tests
+# that often will take more than the timeout to run when gcstress is enabled. This is because
+# gcstress slows down allocations significantly which these tests do a lot.
TEST_ART_BROKEN_GCSTRESS_RUN_TESTS := \
137-cfi \
- 961-default-iface-resolution-generated
+ 961-default-iface-resolution-generated \
+ 964-default-iface-init-generated
ifneq (,$(filter gcstress,$(GC_TYPES)))
ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
@@ -459,18 +464,6 @@
TEST_ART_BROKEN_OPTIMIZING_MIPS_RUN_TESTS :=
-# Known broken tests for the optimizing compiler.
-TEST_ART_BROKEN_OPTIMIZING_RUN_TESTS := \
- 455-set-vreg \
-
-ifneq (,$(filter optimizing,$(COMPILER_TYPES)))
- ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
- optimizing,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
- $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES),$(TEST_ART_BROKEN_OPTIMIZING_RUN_TESTS),$(ALL_ADDRESS_SIZES))
-endif
-
-TEST_ART_BROKEN_OPTIMIZING_RUN_TESTS :=
-
# Tests that should fail when the optimizing compiler compiles them non-debuggable.
TEST_ART_BROKEN_OPTIMIZING_NONDEBUGGABLE_RUN_TESTS := \
454-get-vreg \
@@ -496,16 +489,20 @@
TEST_ART_BROKEN_OPTIMIZING_DEBUGGABLE_RUN_TESTS :=
# Tests that should fail in the read barrier configuration.
+# 055: Exceeds run time limits due to read barrier instrumentation.
# 137: Read barrier forces interpreter. Cannot run this with the interpreter.
-# 141: Class unloading test is flaky with CC since CC seems to occasionally keep class loaders live.
+# 537: Expects an array copy to be intrinsified, but calling-on-slowpath intrinsics are not yet
+# handled in the read barrier configuration.
TEST_ART_BROKEN_READ_BARRIER_RUN_TESTS := \
- 137-cfi \
- 141-class-unload
+ 055-enum-performance \
+ 137-cfi \
+ 537-checker-arraycopy
ifeq ($(ART_USE_READ_BARRIER),true)
- ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
- $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
- $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES),$(TEST_ART_BROKEN_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES))
+ ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \
+ $(PREBUILD_TYPES),$(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES), \
+ $(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \
+ $(TEST_ART_BROKEN_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES))
endif
TEST_ART_BROKEN_READ_BARRIER_RUN_TESTS :=
diff --git a/test/dexdump/run-all-tests b/test/dexdump/run-all-tests
index 9cf7ab6..11ab55a 100755
--- a/test/dexdump/run-all-tests
+++ b/test/dexdump/run-all-tests
@@ -40,7 +40,7 @@
# Set up dexdump binary and flags to test.
DEXD="${ANDROID_HOST_OUT}/bin/dexdump2"
DEXDFLAGS1="-dfh"
-DEXDFLAGS2="-l xml"
+DEXDFLAGS2="-e -l xml"
# Set up dexlist binary and flags to test.
DEXL="${ANDROID_HOST_OUT}/bin/dexlist"
diff --git a/test/run-test b/test/run-test
index b2f6546..d0da34e 100755
--- a/test/run-test
+++ b/test/run-test
@@ -665,6 +665,15 @@
export TEST_NAME=`basename ${test_dir}`
+# arch_supports_read_barrier ARCH
+# -------------------------------
+# Return whether the Optimizing compiler has read barrier support for ARCH.
+function arch_supports_read_barrier() {
+ # Optimizing has read barrier support for ARM, x86 and x86-64 at the
+ # moment.
+ [ "x$1" = xarm ] || [ "x$1" = xx86 ] || [ "x$1" = xx86_64 ]
+}
+
# Tests named '<number>-checker-*' will also have their CFGs verified with
# Checker when compiled with Optimizing on host.
if [[ "$TEST_NAME" =~ ^[0-9]+-checker- ]]; then
@@ -678,9 +687,24 @@
USE_JACK="false"
if [ "$runtime" = "art" -a "$image_suffix" = "-optimizing" ]; then
+ # Optimizing has read barrier support for certain architectures
+ # only. On other architectures, compiling is disabled when read
+ # barriers are enabled, meaning that we do not produce a CFG file
+ # as a side-effect of compilation, thus the Checker assertions
+ # cannot be checked. Disable Checker for those cases.
+ #
+ # TODO: Enable Checker when read barrier support is added to more
+ # architectures (b/12687968).
+ if [ "x$ART_USE_READ_BARRIER" = xtrue ] \
+ && (([ "x$host_mode" = "xyes" ] \
+ && ! arch_supports_read_barrier "$host_arch_name") \
+ || ([ "x$target_mode" = "xyes" ] \
+ && ! arch_supports_read_barrier "$target_arch_name")); then
+ run_checker="no"
# In no-prebuild mode, the compiler is only invoked if both dex2oat and
# patchoat are available. Disable Checker otherwise (b/22552692).
- if [ "$prebuild_mode" = "yes" ] || [ "$have_patchoat" = "yes" -a "$have_dex2oat" = "yes" ]; then
+ elif [ "$prebuild_mode" = "yes" ] \
+ || [ "$have_patchoat" = "yes" -a "$have_dex2oat" = "yes" ]; then
run_checker="yes"
if [ "$target_mode" = "no" ]; then
diff --git a/tools/ahat/README.txt b/tools/ahat/README.txt
index aa548cc..362ae25 100644
--- a/tools/ahat/README.txt
+++ b/tools/ahat/README.txt
@@ -14,7 +14,6 @@
- Say how to enable allocation sites.
- Where to submit feedback, questions, and bug reports.
* Dim 'image' and 'zygote' heap sizes slightly? Why do we even show these?
- * Filter out RootObjs in mSnapshot.getGCRoots, not RootsHandler.
* Let user re-sort sites objects info by clicking column headers.
* Let user re-sort "Objects" list.
* Show site context and heap and class filter in "Objects" view?
diff --git a/tools/ahat/src/AhatHttpHandler.java b/tools/ahat/src/AhatHttpHandler.java
index 0553713..178747c 100644
--- a/tools/ahat/src/AhatHttpHandler.java
+++ b/tools/ahat/src/AhatHttpHandler.java
@@ -44,7 +44,7 @@
DocString menu = new DocString();
menu.appendLink(DocString.uri("/"), DocString.text("overview"));
menu.append(" - ");
- menu.appendLink(DocString.uri("roots"), DocString.text("roots"));
+ menu.appendLink(DocString.uri("rooted"), DocString.text("rooted"));
menu.append(" - ");
menu.appendLink(DocString.uri("sites"), DocString.text("allocations"));
menu.append(" - ");
diff --git a/tools/ahat/src/AhatSnapshot.java b/tools/ahat/src/AhatSnapshot.java
index 43658f3..0bf064e 100644
--- a/tools/ahat/src/AhatSnapshot.java
+++ b/tools/ahat/src/AhatSnapshot.java
@@ -19,7 +19,6 @@
import com.android.tools.perflib.heap.ClassObj;
import com.android.tools.perflib.heap.Heap;
import com.android.tools.perflib.heap.Instance;
-import com.android.tools.perflib.heap.RootObj;
import com.android.tools.perflib.heap.Snapshot;
import com.android.tools.perflib.heap.StackFrame;
import com.android.tools.perflib.heap.StackTrace;
@@ -30,7 +29,6 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -47,6 +45,9 @@
// Map from Instance to the list of Instances it immediately dominates.
private Map<Instance, List<Instance>> mDominated;
+ // Collection of objects whose immediate dominator is the SENTINEL_ROOT.
+ private List<Instance> mRooted;
+
private Site mRootSite;
private Map<Heap, Long> mHeapSizes;
@@ -70,6 +71,7 @@
mDominated = new HashMap<Instance, List<Instance>>();
mRootSite = new Site("ROOT");
mHeapSizes = new HashMap<Heap, Long>();
+ mRooted = new ArrayList<Instance>();
ClassObj javaLangClass = mSnapshot.findClass("java.lang.Class");
for (Heap heap : mHeaps) {
@@ -79,6 +81,10 @@
if (dominator != null) {
total += inst.getSize();
+ if (dominator == Snapshot.SENTINEL_ROOT) {
+ mRooted.add(inst);
+ }
+
// Properly label the class of a class object.
if (inst instanceof ClassObj && javaLangClass != null && inst.getClassObj() == null) {
inst.setClassId(javaLangClass.getId());
@@ -126,8 +132,12 @@
return mSnapshot.getHeap(name);
}
- public Collection<RootObj> getGCRoots() {
- return mSnapshot.getGCRoots();
+ /**
+ * Returns a collection of instances whose immediate dominator is the
+ * SENTINEL_ROOT.
+ */
+ public List<Instance> getRooted() {
+ return mRooted;
}
public List<Heap> getHeaps() {
diff --git a/tools/ahat/src/Main.java b/tools/ahat/src/Main.java
index 96fc53b..ebd49d7 100644
--- a/tools/ahat/src/Main.java
+++ b/tools/ahat/src/Main.java
@@ -74,7 +74,7 @@
InetSocketAddress addr = new InetSocketAddress(loopback, port);
HttpServer server = HttpServer.create(addr, 0);
server.createContext("/", new AhatHttpHandler(new OverviewHandler(ahat, hprof)));
- server.createContext("/roots", new AhatHttpHandler(new RootsHandler(ahat)));
+ server.createContext("/rooted", new AhatHttpHandler(new RootedHandler(ahat)));
server.createContext("/object", new AhatHttpHandler(new ObjectHandler(ahat)));
server.createContext("/objects", new AhatHttpHandler(new ObjectsHandler(ahat)));
server.createContext("/site", new AhatHttpHandler(new SiteHandler(ahat)));
diff --git a/tools/ahat/src/ObjectHandler.java b/tools/ahat/src/ObjectHandler.java
index 9e4ce56..1305070 100644
--- a/tools/ahat/src/ObjectHandler.java
+++ b/tools/ahat/src/ObjectHandler.java
@@ -215,7 +215,7 @@
public DocString render(Instance element) {
if (element == null) {
- return DocString.link(DocString.uri("roots"), DocString.text("ROOT"));
+ return DocString.link(DocString.uri("rooted"), DocString.text("ROOT"));
} else {
return DocString.text("→ ").append(Value.render(element));
}
diff --git a/tools/ahat/src/OverviewHandler.java b/tools/ahat/src/OverviewHandler.java
index e86679f..0fe4fba 100644
--- a/tools/ahat/src/OverviewHandler.java
+++ b/tools/ahat/src/OverviewHandler.java
@@ -50,7 +50,7 @@
printHeapSizes(doc, query);
DocString menu = new DocString();
- menu.appendLink(DocString.uri("roots"), DocString.text("Roots"));
+ menu.appendLink(DocString.uri("rooted"), DocString.text("Rooted"));
menu.append(" - ");
menu.appendLink(DocString.uri("site"), DocString.text("Allocations"));
menu.append(" - ");
diff --git a/tools/ahat/src/RootedHandler.java b/tools/ahat/src/RootedHandler.java
new file mode 100644
index 0000000..ec3272f
--- /dev/null
+++ b/tools/ahat/src/RootedHandler.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.ahat;
+
+import java.io.IOException;
+
+class RootedHandler implements AhatHandler {
+
+ private static final String ROOTED_ID = "rooted";
+
+ private AhatSnapshot mSnapshot;
+
+ public RootedHandler(AhatSnapshot snapshot) {
+ mSnapshot = snapshot;
+ }
+
+ @Override
+ public void handle(Doc doc, Query query) throws IOException {
+ doc.title("Rooted");
+ DominatedList.render(mSnapshot, doc, query, ROOTED_ID, mSnapshot.getRooted());
+ }
+}
diff --git a/tools/ahat/src/RootsHandler.java b/tools/ahat/src/RootsHandler.java
deleted file mode 100644
index 2a92c90..0000000
--- a/tools/ahat/src/RootsHandler.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (C) 2015 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.ahat;
-
-import com.android.tools.perflib.heap.Instance;
-import com.android.tools.perflib.heap.RootObj;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-class RootsHandler implements AhatHandler {
-
- private static final String ROOTS_ID = "roots";
-
- private AhatSnapshot mSnapshot;
-
- public RootsHandler(AhatSnapshot snapshot) {
- mSnapshot = snapshot;
- }
-
- @Override
- public void handle(Doc doc, Query query) throws IOException {
- doc.title("Roots");
-
- Set<Instance> rootset = new HashSet<Instance>();
- for (RootObj root : mSnapshot.getGCRoots()) {
- Instance inst = root.getReferredInstance();
- if (inst != null) {
- rootset.add(inst);
- }
- }
-
- List<Instance> roots = new ArrayList<Instance>();
- for (Instance inst : rootset) {
- roots.add(inst);
- }
- DominatedList.render(mSnapshot, doc, query, ROOTS_ID, roots);
- }
-}
-
diff --git a/tools/ahat/src/help.html b/tools/ahat/src/help.html
index b7ae2ce..92ec37d 100644
--- a/tools/ahat/src/help.html
+++ b/tools/ahat/src/help.html
@@ -20,7 +20,7 @@
<div class="menu">
<a href="/">overview</a> -
- <a href="roots">roots</a> -
+ <a href="rooted">rooted</a> -
<a href="sites">allocations</a> -
<a href="help">help</a>
</div>
@@ -29,7 +29,7 @@
<h2>Information shown by ahat:</h2>
<ul>
<li><a href="/">The total bytes retained by heap.</a></li>
- <li><a href="/roots">A list of root objects and their retained sizes for each heap.</a></li>
+ <li><a href="/rooted">A list of rooted objects and their retained sizes for each heap.</a></li>
<li>Information about each allocated object:
<ul>
<li>The allocation site (stack trace) of the object (if available).</li>
diff --git a/tools/buildbot-build.sh b/tools/buildbot-build.sh
index 047c24f..02787fb 100755
--- a/tools/buildbot-build.sh
+++ b/tools/buildbot-build.sh
@@ -21,7 +21,7 @@
out_dir=${OUT_DIR-out}
java_libraries_dir=${out_dir}/target/common/obj/JAVA_LIBRARIES
-common_targets="vogar ${java_libraries_dir}/core-tests_intermediates/javalib.jar apache-harmony-jdwp-tests-hostdex ${java_libraries_dir}/jsr166-tests_intermediates/javalib.jar"
+common_targets="vogar ${java_libraries_dir}/core-tests_intermediates/javalib.jar apache-harmony-jdwp-tests-hostdex ${java_libraries_dir}/jsr166-tests_intermediates/javalib.jar ${out_dir}/host/linux-x86/bin/jack"
mode="target"
j_arg="-j$(nproc)"
showcommands=
diff --git a/tools/run-jdwp-tests.sh b/tools/run-jdwp-tests.sh
index de27a6f..0747712 100755
--- a/tools/run-jdwp-tests.sh
+++ b/tools/run-jdwp-tests.sh
@@ -43,9 +43,11 @@
vm_args=""
# By default, we run the whole JDWP test suite.
test="org.apache.harmony.jpda.tests.share.AllTests"
+host="no"
while true; do
if [[ "$1" == "--mode=host" ]]; then
+ host="yes"
# Specify bash explicitly since the art script cannot, since it has to run on the device
# with mksh.
art="bash ${OUT_DIR-out}/host/linux-x86/bin/art"
@@ -118,3 +120,15 @@
--classpath $test_jar \
--vm-arg -Xcompiler-option --vm-arg --debuggable \
$test
+
+vogar_exit_status=$?
+
+echo "Killing stalled dalvikvm processes..."
+if [[ $host == "yes" ]]; then
+ pkill -9 -f /bin/dalvikvm
+else
+ adb shell pkill -9 -f /bin/dalvikvm
+fi
+echo "Done."
+
+exit $vogar_exit_status
