merge in oc-release history after reset to master
diff --git a/compiler/Android.bp b/compiler/Android.bp
index 1ee2a21..c59e36b 100644
--- a/compiler/Android.bp
+++ b/compiler/Android.bp
@@ -417,6 +417,7 @@
shared_libs: [
"libartd-compiler",
+ "libartd-simulator",
"libvixld-arm",
"libvixld-arm64",
diff --git a/compiler/compiled_method.h b/compiler/compiled_method.h
index e2a0942..00e2d62 100644
--- a/compiler/compiled_method.h
+++ b/compiler/compiled_method.h
@@ -170,7 +170,6 @@
// choose to squeeze the Type into fewer than 8 bits, we'll have to declare
// patch_type_ as an uintN_t and do explicit static_cast<>s.
enum class Type : uint8_t {
- kRecordPosition, // Just record patch position for patchoat.
kMethod,
kCall,
kCallRelative, // NOTE: Actual patching is instruction_set-dependent.
@@ -183,10 +182,6 @@
kDexCacheArray, // NOTE: Actual patching is instruction_set-dependent.
};
- static LinkerPatch RecordPosition(size_t literal_offset) {
- return LinkerPatch(literal_offset, Type::kRecordPosition, /* target_dex_file */ nullptr);
- }
-
static LinkerPatch MethodPatch(size_t literal_offset,
const DexFile* target_dex_file,
uint32_t target_method_idx) {
diff --git a/compiler/driver/compiler_driver.h b/compiler/driver/compiler_driver.h
index 1e5c43d..cbde587 100644
--- a/compiler/driver/compiler_driver.h
+++ b/compiler/driver/compiler_driver.h
@@ -355,6 +355,10 @@
return current_dex_to_dex_methods_;
}
+ const ProfileCompilationInfo* GetProfileCompilationInfo() const {
+ return profile_compilation_info_;
+ }
+
private:
// Can `referrer_class` access the resolved `member`?
// Dispatch call to mirror::Class::CanAccessResolvedField or
diff --git a/compiler/driver/compiler_driver_test.cc b/compiler/driver/compiler_driver_test.cc
index 562f97b..35aa1ee 100644
--- a/compiler/driver/compiler_driver_test.cc
+++ b/compiler/driver/compiler_driver_test.cc
@@ -133,9 +133,10 @@
<< " " << dex.GetMethodDeclaringClassDescriptor(dex.GetMethodId(i)) << " "
<< dex.GetMethodName(dex.GetMethodId(i));
}
- EXPECT_EQ(dex.NumFieldIds(), dex_cache->NumResolvedFields());
+ EXPECT_TRUE(dex_cache->StaticArtFieldSize() == dex_cache->NumResolvedFields()
+ || dex.NumFieldIds() == dex_cache->NumResolvedFields());
for (size_t i = 0; i < dex_cache->NumResolvedFields(); i++) {
- ArtField* field = cl->GetResolvedField(i, dex_cache);
+ ArtField* field = dex_cache->GetResolvedField(i, cl->GetImagePointerSize());
EXPECT_TRUE(field != nullptr) << "field_idx=" << i
<< " " << dex.GetFieldDeclaringClassDescriptor(dex.GetFieldId(i))
<< " " << dex.GetFieldName(dex.GetFieldId(i));
diff --git a/compiler/driver/compiler_options.cc b/compiler/driver/compiler_options.cc
index c222f90..34ad1c5 100644
--- a/compiler/driver/compiler_options.cc
+++ b/compiler/driver/compiler_options.cc
@@ -32,7 +32,6 @@
no_inline_from_(nullptr),
boot_image_(false),
app_image_(false),
- include_patch_information_(kDefaultIncludePatchInformation),
top_k_profile_threshold_(kDefaultTopKProfileThreshold),
debuggable_(false),
generate_debug_info_(kDefaultGenerateDebugInfo),
@@ -66,7 +65,6 @@
size_t inline_depth_limit,
size_t inline_max_code_units,
const std::vector<const DexFile*>* no_inline_from,
- bool include_patch_information,
double top_k_profile_threshold,
bool debuggable,
bool generate_debug_info,
@@ -93,7 +91,6 @@
no_inline_from_(no_inline_from),
boot_image_(false),
app_image_(false),
- include_patch_information_(include_patch_information),
top_k_profile_threshold_(top_k_profile_threshold),
debuggable_(debuggable),
generate_debug_info_(generate_debug_info),
@@ -206,10 +203,6 @@
debuggable_ = true;
} else if (option.starts_with("--top-k-profile-threshold=")) {
ParseDouble(option.data(), '=', 0.0, 100.0, &top_k_profile_threshold_, Usage);
- } else if (option == "--include-patch-information") {
- include_patch_information_ = true;
- } else if (option == "--no-include-patch-information") {
- include_patch_information_ = false;
} else if (option == "--abort-on-hard-verifier-error") {
abort_on_hard_verifier_failure_ = true;
} else if (option.starts_with("--dump-init-failures=")) {
diff --git a/compiler/driver/compiler_options.h b/compiler/driver/compiler_options.h
index 6894cd5..2e3e55f 100644
--- a/compiler/driver/compiler_options.h
+++ b/compiler/driver/compiler_options.h
@@ -46,7 +46,6 @@
static constexpr double kDefaultTopKProfileThreshold = 90.0;
static const bool kDefaultGenerateDebugInfo = false;
static const bool kDefaultGenerateMiniDebugInfo = false;
- static const bool kDefaultIncludePatchInformation = false;
static const size_t kDefaultInlineDepthLimit = 3;
static const size_t kDefaultInlineMaxCodeUnits = 32;
static constexpr size_t kUnsetInlineDepthLimit = -1;
@@ -68,7 +67,6 @@
size_t inline_depth_limit,
size_t inline_max_code_units,
const std::vector<const DexFile*>* no_inline_from,
- bool include_patch_information,
double top_k_profile_threshold,
bool debuggable,
bool generate_debug_info,
@@ -213,10 +211,6 @@
return implicit_suspend_checks_;
}
- bool GetIncludePatchInformation() const {
- return include_patch_information_;
- }
-
bool IsBootImage() const {
return boot_image_;
}
@@ -305,7 +299,6 @@
bool boot_image_;
bool app_image_;
- bool include_patch_information_;
// When using a profile file only the top K% of the profiled samples will be compiled.
double top_k_profile_threshold_;
bool debuggable_;
diff --git a/compiler/elf_writer.h b/compiler/elf_writer.h
index d55f745..7baae52 100644
--- a/compiler/elf_writer.h
+++ b/compiler/elf_writer.h
@@ -63,7 +63,6 @@
virtual void EndText(OutputStream* text) = 0;
virtual void WriteDynamicSection() = 0;
virtual void WriteDebugInfo(const ArrayRef<const debug::MethodDebugInfo>& method_infos) = 0;
- virtual void WritePatchLocations(const ArrayRef<const uintptr_t>& patch_locations) = 0;
virtual bool End() = 0;
// Get the ELF writer's stream. This stream can be used for writing data directly
diff --git a/compiler/elf_writer_quick.cc b/compiler/elf_writer_quick.cc
index 0d6575c..28c35e9 100644
--- a/compiler/elf_writer_quick.cc
+++ b/compiler/elf_writer_quick.cc
@@ -105,7 +105,6 @@
void EndText(OutputStream* text) OVERRIDE;
void WriteDynamicSection() OVERRIDE;
void WriteDebugInfo(const ArrayRef<const debug::MethodDebugInfo>& method_infos) OVERRIDE;
- void WritePatchLocations(const ArrayRef<const uintptr_t>& patch_locations) OVERRIDE;
bool End() OVERRIDE;
virtual OutputStream* GetStream() OVERRIDE;
@@ -268,17 +267,6 @@
}
template <typename ElfTypes>
-void ElfWriterQuick<ElfTypes>::WritePatchLocations(
- const ArrayRef<const uintptr_t>& patch_locations) {
- // Add relocation section for .text.
- if (compiler_options_->GetIncludePatchInformation()) {
- // Note that ElfWriter::Fixup will be called regardless and therefore
- // we need to include oat_patches for debug sections unconditionally.
- builder_->WritePatches(".text.oat_patches", patch_locations);
- }
-}
-
-template <typename ElfTypes>
bool ElfWriterQuick<ElfTypes>::End() {
builder_->End();
if (compiler_options_->GetGenerateBuildId()) {
diff --git a/compiler/image_test.cc b/compiler/image_test.cc
index b0225a3..89e8a67 100644
--- a/compiler/image_test.cc
+++ b/compiler/image_test.cc
@@ -318,7 +318,6 @@
elf_writer->WriteDynamicSection();
elf_writer->WriteDebugInfo(oat_writer->GetMethodDebugInfo());
- elf_writer->WritePatchLocations(oat_writer->GetAbsolutePatchLocations());
bool success = elf_writer->End();
ASSERT_TRUE(success);
diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc
index 65d82ed..a4a1fd3 100644
--- a/compiler/image_writer.cc
+++ b/compiler/image_writer.cc
@@ -968,11 +968,12 @@
<< Class::PrettyClass(declaring_class) << " not in class linker table";
}
}
- ArtField** resolved_fields = dex_cache->GetResolvedFields();
+ mirror::FieldDexCacheType* resolved_fields = dex_cache->GetResolvedFields();
for (size_t i = 0; i < dex_cache->NumResolvedFields(); i++) {
- ArtField* field = mirror::DexCache::GetElementPtrSize(resolved_fields, i, target_ptr_size_);
+ auto pair = mirror::DexCache::GetNativePairPtrSize(resolved_fields, i, target_ptr_size_);
+ ArtField* field = pair.object;
if (field != nullptr && !KeepClass(field->GetDeclaringClass().Ptr())) {
- dex_cache->SetResolvedField(i, nullptr, target_ptr_size_);
+ dex_cache->ClearResolvedField(pair.index, target_ptr_size_);
}
}
// Clean the dex field. It might have been populated during the initialization phase, but
@@ -1596,7 +1597,7 @@
break;
}
case kBinDexCacheArray:
- bin_offset = RoundUp(bin_offset, DexCacheArraysLayout::Alignment());
+ bin_offset = RoundUp(bin_offset, DexCacheArraysLayout::Alignment(target_ptr_size_));
break;
case kBinImTable:
case kBinIMTConflictTable: {
@@ -2236,16 +2237,17 @@
mirror::DexCache::SetElementPtrSize(copy_methods, i, copy, target_ptr_size_);
}
}
- ArtField** orig_fields = orig_dex_cache->GetResolvedFields();
+ mirror::FieldDexCacheType* orig_fields = orig_dex_cache->GetResolvedFields();
if (orig_fields != nullptr) {
copy_dex_cache->SetFieldPtrWithSize<false>(mirror::DexCache::ResolvedFieldsOffset(),
NativeLocationInImage(orig_fields),
PointerSize::k64);
- ArtField** copy_fields = NativeCopyLocation(orig_fields, orig_dex_cache);
+ mirror::FieldDexCacheType* copy_fields = NativeCopyLocation(orig_fields, orig_dex_cache);
for (size_t i = 0, num = orig_dex_cache->NumResolvedFields(); i != num; ++i) {
- ArtField* orig = mirror::DexCache::GetElementPtrSize(orig_fields, i, target_ptr_size_);
- ArtField* copy = NativeLocationInImage(orig);
- mirror::DexCache::SetElementPtrSize(copy_fields, i, copy, target_ptr_size_);
+ mirror::FieldDexCachePair orig =
+ mirror::DexCache::GetNativePairPtrSize(orig_fields, i, target_ptr_size_);
+ mirror::FieldDexCachePair copy(NativeLocationInImage(orig.object), orig.index);
+ mirror::DexCache::SetNativePairPtrSize(copy_fields, i, copy, target_ptr_size_);
}
}
mirror::MethodTypeDexCacheType* orig_method_types = orig_dex_cache->GetResolvedMethodTypes();
diff --git a/compiler/jit/jit_compiler.cc b/compiler/jit/jit_compiler.cc
index cbd831a..3ae7974 100644
--- a/compiler/jit/jit_compiler.cc
+++ b/compiler/jit/jit_compiler.cc
@@ -100,7 +100,6 @@
CompilerOptions::kDefaultInlineDepthLimit,
CompilerOptions::kDefaultInlineMaxCodeUnits,
/* no_inline_from */ nullptr,
- /* include_patch_information */ false,
CompilerOptions::kDefaultTopKProfileThreshold,
Runtime::Current()->IsJavaDebuggable(),
CompilerOptions::kDefaultGenerateDebugInfo,
diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc
index e2233e4..97b1374 100644
--- a/compiler/oat_test.cc
+++ b/compiler/oat_test.cc
@@ -250,7 +250,6 @@
elf_writer->WriteDynamicSection();
elf_writer->WriteDebugInfo(oat_writer.GetMethodDebugInfo());
- elf_writer->WritePatchLocations(oat_writer.GetAbsolutePatchLocations());
if (!elf_writer->End()) {
return false;
diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc
index e8de99a..afcdf5e 100644
--- a/compiler/oat_writer.cc
+++ b/compiler/oat_writer.cc
@@ -1225,7 +1225,7 @@
break;
}
default: {
- DCHECK_EQ(patch.GetType(), LinkerPatch::Type::kRecordPosition);
+ DCHECK(false) << "Unexpected linker patch type: " << patch.GetType();
break;
}
}
diff --git a/compiler/oat_writer.h b/compiler/oat_writer.h
index db84166..5113714 100644
--- a/compiler/oat_writer.h
+++ b/compiler/oat_writer.h
@@ -225,10 +225,6 @@
return oat_data_offset_;
}
- ArrayRef<const uintptr_t> GetAbsolutePatchLocations() const {
- return ArrayRef<const uintptr_t>(absolute_patch_locations_);
- }
-
~OatWriter();
void AddMethodDebugInfos(const std::vector<debug::MethodDebugInfo>& infos) {
diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc
index 7b84ef8..e34f116 100644
--- a/compiler/optimizing/code_generator_arm.cc
+++ b/compiler/optimizing/code_generator_arm.cc
@@ -636,56 +636,25 @@
DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM);
};
-// Slow path marking an object reference `ref` during a read
-// barrier. The field `obj.field` in the object `obj` holding this
-// reference does not get updated by this slow path after marking (see
-// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that).
+// Abstract base class for read barrier slow paths marking a reference
+// `ref`.
//
-// This means that after the execution of this slow path, `ref` will
-// always be up-to-date, but `obj.field` may not; i.e., after the
-// flip, `ref` will be a to-space reference, but `obj.field` will
-// probably still be a from-space reference (unless it gets updated by
-// another thread, or if another thread installed another object
-// reference (different from `ref`) in `obj.field`).
-//
-// If `entrypoint` is a valid location it is assumed to already be
-// holding the entrypoint. The case where the entrypoint is passed in
-// is for the GcRoot read barrier.
-class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM {
- public:
- ReadBarrierMarkSlowPathARM(HInstruction* instruction,
- Location ref,
- Location entrypoint = Location::NoLocation())
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class ReadBarrierMarkSlowPathBaseARM : public SlowPathCodeARM {
+ protected:
+ ReadBarrierMarkSlowPathBaseARM(HInstruction* instruction, Location ref, Location entrypoint)
: SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) {
DCHECK(kEmitCompilerReadBarrier);
}
- const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; }
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathBaseARM"; }
- void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
- LocationSummary* locations = instruction_->GetLocations();
+ // Generate assembly code calling the read barrier marking runtime
+ // entry point (ReadBarrierMarkRegX).
+ void GenerateReadBarrierMarkRuntimeCall(CodeGenerator* codegen) {
Register ref_reg = ref_.AsRegister<Register>();
- DCHECK(locations->CanCall());
- DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg;
- DCHECK(instruction_->IsInstanceFieldGet() ||
- instruction_->IsStaticFieldGet() ||
- instruction_->IsArrayGet() ||
- instruction_->IsArraySet() ||
- instruction_->IsLoadClass() ||
- instruction_->IsLoadString() ||
- instruction_->IsInstanceOf() ||
- instruction_->IsCheckCast() ||
- (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
- (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
- << "Unexpected instruction in read barrier marking slow path: "
- << instruction_->DebugName();
- // The read barrier instrumentation of object ArrayGet
- // instructions does not support the HIntermediateAddress
- // instruction.
- DCHECK(!(instruction_->IsArrayGet() &&
- instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
- __ Bind(GetEntryLabel());
// No need to save live registers; it's taken care of by the
// entrypoint. Also, there is no need to update the stack mask,
// as this runtime call will not trigger a garbage collection.
@@ -715,116 +684,331 @@
arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this);
__ blx(entrypoint_.AsRegister<Register>());
} else {
+ // Entrypoint is not already loaded, load from the thread.
int32_t entry_point_offset =
CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg);
// This runtime call does not require a stack map.
arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
}
+ }
+
+ // The location (register) of the marked object reference.
+ const Location ref_;
+
+ // The location of the entrypoint if it is already loaded.
+ const Location entrypoint_;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathBaseARM);
+};
+
+// Slow path marking an object reference `ref` during a read
+// barrier. The field `obj.field` in the object `obj` holding this
+// reference does not get updated by this slow path after marking.
+//
+// This means that after the execution of this slow path, `ref` will
+// always be up-to-date, but `obj.field` may not; i.e., after the
+// flip, `ref` will be a to-space reference, but `obj.field` will
+// probably still be a from-space reference (unless it gets updated by
+// another thread, or if another thread installed another object
+// reference (different from `ref`) in `obj.field`).
+//
+// If `entrypoint` is a valid location it is assumed to already be
+// holding the entrypoint. The case where the entrypoint is passed in
+// is when the decision to mark is based on whether the GC is marking.
+class ReadBarrierMarkSlowPathARM : public ReadBarrierMarkSlowPathBaseARM {
+ public:
+ ReadBarrierMarkSlowPathARM(HInstruction* instruction,
+ Location ref,
+ Location entrypoint = Location::NoLocation())
+ : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint) {
+ DCHECK(kEmitCompilerReadBarrier);
+ }
+
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ DCHECK(locations->CanCall());
+ if (kIsDebugBuild) {
+ Register ref_reg = ref_.AsRegister<Register>();
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg;
+ }
+ DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString())
+ << "Unexpected instruction in read barrier marking slow path: "
+ << instruction_->DebugName();
+
+ __ Bind(GetEntryLabel());
+ GenerateReadBarrierMarkRuntimeCall(codegen);
__ b(GetExitLabel());
}
private:
- // The location (register) of the marked object reference.
- const Location ref_;
-
- // The location of the entrypoint if already loaded.
- const Location entrypoint_;
-
DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM);
};
-// Slow path marking an object reference `ref` during a read barrier,
-// and if needed, atomically updating the field `obj.field` in the
-// object `obj` holding this reference after marking (contrary to
-// ReadBarrierMarkSlowPathARM above, which never tries to update
-// `obj.field`).
+// Slow path loading `obj`'s lock word, loading a reference from
+// object `*(obj + offset + (index << scale_factor))` into `ref`, and
+// marking `ref` if `obj` is gray according to the lock word (Baker
+// read barrier). The field `obj.field` in the object `obj` holding
+// this reference does not get updated by this slow path after marking
+// (see LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM
+// below for that).
//
-// This means that after the execution of this slow path, both `ref`
-// and `obj.field` will be up-to-date; i.e., after the flip, both will
-// hold the same to-space reference (unless another thread installed
-// another object reference (different from `ref`) in `obj.field`).
-class ReadBarrierMarkAndUpdateFieldSlowPathARM : public SlowPathCodeARM {
+// This means that after the execution of this slow path, `ref` will
+// always be up-to-date, but `obj.field` may not; i.e., after the
+// flip, `ref` will be a to-space reference, but `obj.field` will
+// probably still be a from-space reference (unless it gets updated by
+// another thread, or if another thread installed another object
+// reference (different from `ref`) in `obj.field`).
+//
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class LoadReferenceWithBakerReadBarrierSlowPathARM : public ReadBarrierMarkSlowPathBaseARM {
public:
- ReadBarrierMarkAndUpdateFieldSlowPathARM(HInstruction* instruction,
- Location ref,
- Register obj,
- Location field_offset,
- Register temp1,
- Register temp2)
- : SlowPathCodeARM(instruction),
- ref_(ref),
+ LoadReferenceWithBakerReadBarrierSlowPathARM(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check,
+ Register temp,
+ Location entrypoint)
+ : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint),
obj_(obj),
- field_offset_(field_offset),
- temp1_(temp1),
- temp2_(temp2) {
+ offset_(offset),
+ index_(index),
+ scale_factor_(scale_factor),
+ needs_null_check_(needs_null_check),
+ temp_(temp) {
DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
}
- const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkAndUpdateFieldSlowPathARM"; }
+ const char* GetDescription() const OVERRIDE {
+ return "LoadReferenceWithBakerReadBarrierSlowPathARM";
+ }
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
LocationSummary* locations = instruction_->GetLocations();
Register ref_reg = ref_.AsRegister<Register>();
DCHECK(locations->CanCall());
DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg;
- // This slow path is only used by the UnsafeCASObject intrinsic.
+ DCHECK_NE(ref_reg, temp_);
+ DCHECK(instruction_->IsInstanceFieldGet() ||
+ instruction_->IsStaticFieldGet() ||
+ instruction_->IsArrayGet() ||
+ instruction_->IsArraySet() ||
+ instruction_->IsInstanceOf() ||
+ instruction_->IsCheckCast() ||
+ (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
+ (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
+ << "Unexpected instruction in read barrier marking slow path: "
+ << instruction_->DebugName();
+ // The read barrier instrumentation of object ArrayGet
+ // instructions does not support the HIntermediateAddress
+ // instruction.
+ DCHECK(!(instruction_->IsArrayGet() &&
+ instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
+
+ __ Bind(GetEntryLabel());
+
+ // When using MaybeGenerateReadBarrierSlow, the read barrier call is
+ // inserted after the original load. However, in fast path based
+ // Baker's read barriers, we need to perform the load of
+ // mirror::Object::monitor_ *before* the original reference load.
+ // This load-load ordering is required by the read barrier.
+ // The fast path/slow path (for Baker's algorithm) should look like:
+ //
+ // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
+ // lfence; // Load fence or artificial data dependency to prevent load-load reordering
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
+ // if (is_gray) {
+ // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call.
+ // }
+ //
+ // Note: the original implementation in ReadBarrier::Barrier is
+ // slightly more complex as it performs additional checks that we do
+ // not do here for performance reasons.
+
+ // /* int32_t */ monitor = obj->monitor_
+ uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+ __ LoadFromOffset(kLoadWord, temp_, obj_, monitor_offset);
+ if (needs_null_check_) {
+ codegen->MaybeRecordImplicitNullCheck(instruction_);
+ }
+ // /* LockWord */ lock_word = LockWord(monitor)
+ static_assert(sizeof(LockWord) == sizeof(int32_t),
+ "art::LockWord and int32_t have different sizes.");
+
+ // Introduce a dependency on the lock_word including the rb_state,
+ // which shall prevent load-load reordering without using
+ // a memory barrier (which would be more expensive).
+ // `obj` is unchanged by this operation, but its value now depends
+ // on `temp`.
+ __ add(obj_, obj_, ShifterOperand(temp_, LSR, 32));
+
+ // The actual reference load.
+ // A possible implicit null check has already been handled above.
+ CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
+ arm_codegen->GenerateRawReferenceLoad(
+ instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false);
+
+ // Mark the object `ref` when `obj` is gray.
+ //
+ // if (rb_state == ReadBarrier::GrayState())
+ // ref = ReadBarrier::Mark(ref);
+ //
+ // Given the numeric representation, it's enough to check the low bit of the
+ // rb_state. We do that by shifting the bit out of the lock word with LSRS
+ // which can be a 16-bit instruction unlike the TST immediate.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Lsrs(temp_, temp_, LockWord::kReadBarrierStateShift + 1);
+ __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS.
+ GenerateReadBarrierMarkRuntimeCall(codegen);
+
+ __ b(GetExitLabel());
+ }
+
+ private:
+ // The register containing the object holding the marked object reference field.
+ Register obj_;
+ // The offset, index and scale factor to access the reference in `obj_`.
+ uint32_t offset_;
+ Location index_;
+ ScaleFactor scale_factor_;
+ // Is a null check required?
+ bool needs_null_check_;
+ // A temporary register used to hold the lock word of `obj_`.
+ Register temp_;
+
+ DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierSlowPathARM);
+};
+
+// Slow path loading `obj`'s lock word, loading a reference from
+// object `*(obj + offset + (index << scale_factor))` into `ref`, and
+// marking `ref` if `obj` is gray according to the lock word (Baker
+// read barrier). If needed, this slow path also atomically updates
+// the field `obj.field` in the object `obj` holding this reference
+// after marking (contrary to
+// LoadReferenceWithBakerReadBarrierSlowPathARM above, which never
+// tries to update `obj.field`).
+//
+// This means that after the execution of this slow path, both `ref`
+// and `obj.field` will be up-to-date; i.e., after the flip, both will
+// hold the same to-space reference (unless another thread installed
+// another object reference (different from `ref`) in `obj.field`).
+//
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM
+ : public ReadBarrierMarkSlowPathBaseARM {
+ public:
+ LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check,
+ Register temp1,
+ Register temp2,
+ Location entrypoint)
+ : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint),
+ obj_(obj),
+ offset_(offset),
+ index_(index),
+ scale_factor_(scale_factor),
+ needs_null_check_(needs_null_check),
+ temp1_(temp1),
+ temp2_(temp2) {
+ DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
+ }
+
+ const char* GetDescription() const OVERRIDE {
+ return "LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM";
+ }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ Register ref_reg = ref_.AsRegister<Register>();
+ DCHECK(locations->CanCall());
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg;
+ DCHECK_NE(ref_reg, temp1_);
+
+ // This slow path is only used by the UnsafeCASObject intrinsic at the moment.
DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()))
<< "Unexpected instruction in read barrier marking and field updating slow path: "
<< instruction_->DebugName();
DCHECK(instruction_->GetLocations()->Intrinsified());
DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject);
- DCHECK(field_offset_.IsRegisterPair()) << field_offset_;
+ DCHECK_EQ(offset_, 0u);
+ DCHECK_EQ(scale_factor_, ScaleFactor::TIMES_1);
+ // The location of the offset of the marked reference field within `obj_`.
+ Location field_offset = index_;
+ DCHECK(field_offset.IsRegisterPair()) << field_offset;
__ Bind(GetEntryLabel());
- // Save the old reference.
+ // /* int32_t */ monitor = obj->monitor_
+ uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+ __ LoadFromOffset(kLoadWord, temp1_, obj_, monitor_offset);
+ if (needs_null_check_) {
+ codegen->MaybeRecordImplicitNullCheck(instruction_);
+ }
+ // /* LockWord */ lock_word = LockWord(monitor)
+ static_assert(sizeof(LockWord) == sizeof(int32_t),
+ "art::LockWord and int32_t have different sizes.");
+
+ // Introduce a dependency on the lock_word including the rb_state,
+ // which shall prevent load-load reordering without using
+ // a memory barrier (which would be more expensive).
+ // `obj` is unchanged by this operation, but its value now depends
+ // on `temp1`.
+ __ add(obj_, obj_, ShifterOperand(temp1_, LSR, 32));
+
+ // The actual reference load.
+ // A possible implicit null check has already been handled above.
+ CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
+ arm_codegen->GenerateRawReferenceLoad(
+ instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false);
+
+ // Mark the object `ref` when `obj` is gray.
+ //
+ // if (rb_state == ReadBarrier::GrayState())
+ // ref = ReadBarrier::Mark(ref);
+ //
+ // Given the numeric representation, it's enough to check the low bit of the
+ // rb_state. We do that by shifting the bit out of the lock word with LSRS
+ // which can be a 16-bit instruction unlike the TST immediate.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Lsrs(temp1_, temp1_, LockWord::kReadBarrierStateShift + 1);
+ __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS.
+
+ // Save the old value of the reference before marking it.
// Note that we cannot use IP to save the old reference, as IP is
// used internally by the ReadBarrierMarkRegX entry point, and we
// need the old reference after the call to that entry point.
DCHECK_NE(temp1_, IP);
__ Mov(temp1_, ref_reg);
- // No need to save live registers; it's taken care of by the
- // entrypoint. Also, there is no need to update the stack mask,
- // as this runtime call will not trigger a garbage collection.
- CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
- DCHECK_NE(ref_reg, SP);
- DCHECK_NE(ref_reg, LR);
- DCHECK_NE(ref_reg, PC);
- // IP is used internally by the ReadBarrierMarkRegX entry point
- // as a temporary, it cannot be the entry point's input/output.
- DCHECK_NE(ref_reg, IP);
- DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg;
- // "Compact" slow path, saving two moves.
- //
- // Instead of using the standard runtime calling convention (input
- // and output in R0):
- //
- // R0 <- ref
- // R0 <- ReadBarrierMark(R0)
- // ref <- R0
- //
- // we just use rX (the register containing `ref`) as input and output
- // of a dedicated entrypoint:
- //
- // rX <- ReadBarrierMarkRegX(rX)
- //
- int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg);
- // This runtime call does not require a stack map.
- arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ GenerateReadBarrierMarkRuntimeCall(codegen);
// If the new reference is different from the old reference,
- // update the field in the holder (`*(obj_ + field_offset_)`).
+ // update the field in the holder (`*(obj_ + field_offset)`).
//
// Note that this field could also hold a different object, if
// another thread had concurrently changed it. In that case, the
// LDREX/SUBS/ITNE sequence of instructions in the compare-and-set
// (CAS) operation below would abort the CAS, leaving the field
// as-is.
- Label done;
__ cmp(temp1_, ShifterOperand(ref_reg));
- __ b(&done, EQ);
+ __ b(GetExitLabel(), EQ);
// Update the the holder's field atomically. This may fail if
// mutator updates before us, but it's OK. This is achieved
@@ -837,7 +1021,7 @@
// The UnsafeCASObject intrinsic uses a register pair as field
// offset ("long offset"), of which only the low part contains
// data.
- Register offset = field_offset_.AsRegisterPairLow<Register>();
+ Register offset = field_offset.AsRegisterPairLow<Register>();
Register expected = temp1_;
Register value = ref_reg;
Register tmp_ptr = IP; // Pointer to actual memory.
@@ -887,22 +1071,27 @@
}
}
- __ Bind(&done);
__ b(GetExitLabel());
}
private:
- // The location (register) of the marked object reference.
- const Location ref_;
// The register containing the object holding the marked object reference field.
const Register obj_;
- // The location of the offset of the marked reference field within `obj_`.
- Location field_offset_;
-
+ // The offset, index and scale factor to access the reference in `obj_`.
+ uint32_t offset_;
+ Location index_;
+ ScaleFactor scale_factor_;
+ // Is a null check required?
+ bool needs_null_check_;
+ // A temporary register used to hold the lock word of `obj_`; and
+ // also to hold the original reference value, when the reference is
+ // marked.
const Register temp1_;
+ // A temporary register used in the implementation of the CAS, to
+ // update the object's reference field.
const Register temp2_;
- DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARM);
+ DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM);
};
// Slow path generating a read barrier for a heap reference.
@@ -1375,10 +1564,332 @@
}
}
+static void GenerateVcmp(HInstruction* instruction, CodeGeneratorARM* codegen) {
+ Primitive::Type type = instruction->InputAt(0)->GetType();
+ Location lhs_loc = instruction->GetLocations()->InAt(0);
+ Location rhs_loc = instruction->GetLocations()->InAt(1);
+ if (rhs_loc.IsConstant()) {
+ // 0.0 is the only immediate that can be encoded directly in
+ // a VCMP instruction.
+ //
+ // Both the JLS (section 15.20.1) and the JVMS (section 6.5)
+ // specify that in a floating-point comparison, positive zero
+ // and negative zero are considered equal, so we can use the
+ // literal 0.0 for both cases here.
+ //
+ // Note however that some methods (Float.equal, Float.compare,
+ // Float.compareTo, Double.equal, Double.compare,
+ // Double.compareTo, Math.max, Math.min, StrictMath.max,
+ // StrictMath.min) consider 0.0 to be (strictly) greater than
+ // -0.0. So if we ever translate calls to these methods into a
+ // HCompare instruction, we must handle the -0.0 case with
+ // care here.
+ DCHECK(rhs_loc.GetConstant()->IsArithmeticZero());
+ if (type == Primitive::kPrimFloat) {
+ __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>());
+ } else {
+ DCHECK_EQ(type, Primitive::kPrimDouble);
+ __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()));
+ }
+ } else {
+ if (type == Primitive::kPrimFloat) {
+ __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>());
+ } else {
+ DCHECK_EQ(type, Primitive::kPrimDouble);
+ __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()),
+ FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>()));
+ }
+ }
+}
+
+static Condition GenerateLongTestConstant(HCondition* condition,
+ bool invert,
+ CodeGeneratorARM* codegen) {
+ DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong);
+
+ const LocationSummary* const locations = condition->GetLocations();
+ IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition();
+ Condition ret = EQ;
+ const Location left = locations->InAt(0);
+ const Location right = locations->InAt(1);
+
+ DCHECK(right.IsConstant());
+
+ const Register left_high = left.AsRegisterPairHigh<Register>();
+ const Register left_low = left.AsRegisterPairLow<Register>();
+ int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
+
+ switch (cond) {
+ case kCondEQ:
+ case kCondNE:
+ case kCondB:
+ case kCondBE:
+ case kCondA:
+ case kCondAE:
+ __ CmpConstant(left_high, High32Bits(value));
+ __ it(EQ);
+ __ cmp(left_low, ShifterOperand(Low32Bits(value)), EQ);
+ ret = ARMUnsignedCondition(cond);
+ break;
+ case kCondLE:
+ case kCondGT:
+ // Trivially true or false.
+ if (value == std::numeric_limits<int64_t>::max()) {
+ __ cmp(left_low, ShifterOperand(left_low));
+ ret = cond == kCondLE ? EQ : NE;
+ break;
+ }
+
+ if (cond == kCondLE) {
+ cond = kCondLT;
+ } else {
+ DCHECK_EQ(cond, kCondGT);
+ cond = kCondGE;
+ }
+
+ value++;
+ FALLTHROUGH_INTENDED;
+ case kCondGE:
+ case kCondLT:
+ __ CmpConstant(left_low, Low32Bits(value));
+ __ sbcs(IP, left_high, ShifterOperand(High32Bits(value)));
+ ret = ARMCondition(cond);
+ break;
+ default:
+ LOG(FATAL) << "Unreachable";
+ UNREACHABLE();
+ }
+
+ return ret;
+}
+
+static Condition GenerateLongTest(HCondition* condition,
+ bool invert,
+ CodeGeneratorARM* codegen) {
+ DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong);
+
+ const LocationSummary* const locations = condition->GetLocations();
+ IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition();
+ Condition ret = EQ;
+ Location left = locations->InAt(0);
+ Location right = locations->InAt(1);
+
+ DCHECK(right.IsRegisterPair());
+
+ switch (cond) {
+ case kCondEQ:
+ case kCondNE:
+ case kCondB:
+ case kCondBE:
+ case kCondA:
+ case kCondAE:
+ __ cmp(left.AsRegisterPairHigh<Register>(),
+ ShifterOperand(right.AsRegisterPairHigh<Register>()));
+ __ it(EQ);
+ __ cmp(left.AsRegisterPairLow<Register>(),
+ ShifterOperand(right.AsRegisterPairLow<Register>()),
+ EQ);
+ ret = ARMUnsignedCondition(cond);
+ break;
+ case kCondLE:
+ case kCondGT:
+ if (cond == kCondLE) {
+ cond = kCondGE;
+ } else {
+ DCHECK_EQ(cond, kCondGT);
+ cond = kCondLT;
+ }
+
+ std::swap(left, right);
+ FALLTHROUGH_INTENDED;
+ case kCondGE:
+ case kCondLT:
+ __ cmp(left.AsRegisterPairLow<Register>(),
+ ShifterOperand(right.AsRegisterPairLow<Register>()));
+ __ sbcs(IP,
+ left.AsRegisterPairHigh<Register>(),
+ ShifterOperand(right.AsRegisterPairHigh<Register>()));
+ ret = ARMCondition(cond);
+ break;
+ default:
+ LOG(FATAL) << "Unreachable";
+ UNREACHABLE();
+ }
+
+ return ret;
+}
+
+static Condition GenerateTest(HInstruction* instruction,
+ Location loc,
+ bool invert,
+ CodeGeneratorARM* codegen) {
+ DCHECK(!instruction->IsConstant());
+
+ Condition ret = invert ? EQ : NE;
+
+ if (IsBooleanValueOrMaterializedCondition(instruction)) {
+ __ CmpConstant(loc.AsRegister<Register>(), 0);
+ } else {
+ HCondition* const condition = instruction->AsCondition();
+ const LocationSummary* const locations = condition->GetLocations();
+ const Primitive::Type type = condition->GetLeft()->GetType();
+ const IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition();
+ const Location right = locations->InAt(1);
+
+ if (type == Primitive::kPrimLong) {
+ ret = condition->GetLocations()->InAt(1).IsConstant()
+ ? GenerateLongTestConstant(condition, invert, codegen)
+ : GenerateLongTest(condition, invert, codegen);
+ } else if (Primitive::IsFloatingPointType(type)) {
+ GenerateVcmp(condition, codegen);
+ __ vmstat();
+ ret = ARMFPCondition(cond, condition->IsGtBias());
+ } else {
+ DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type;
+
+ const Register left = locations->InAt(0).AsRegister<Register>();
+
+ if (right.IsRegister()) {
+ __ cmp(left, ShifterOperand(right.AsRegister<Register>()));
+ } else {
+ DCHECK(right.IsConstant());
+ __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant()));
+ }
+
+ ret = ARMCondition(cond);
+ }
+ }
+
+ return ret;
+}
+
+static bool CanGenerateTest(HInstruction* condition, ArmAssembler* assembler) {
+ if (!IsBooleanValueOrMaterializedCondition(condition)) {
+ const HCondition* const cond = condition->AsCondition();
+
+ if (cond->GetLeft()->GetType() == Primitive::kPrimLong) {
+ const LocationSummary* const locations = cond->GetLocations();
+ const IfCondition c = cond->GetCondition();
+
+ if (locations->InAt(1).IsConstant()) {
+ const int64_t value = locations->InAt(1).GetConstant()->AsLongConstant()->GetValue();
+ ShifterOperand so;
+
+ if (c < kCondLT || c > kCondGE) {
+ // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8,
+ // we check that the least significant half of the first input to be compared
+ // is in a low register (the other half is read outside an IT block), and
+ // the constant fits in an 8-bit unsigned integer, so that a 16-bit CMP
+ // encoding can be used.
+ if (!ArmAssembler::IsLowRegister(locations->InAt(0).AsRegisterPairLow<Register>()) ||
+ !IsUint<8>(Low32Bits(value))) {
+ return false;
+ }
+ } else if (c == kCondLE || c == kCondGT) {
+ if (value < std::numeric_limits<int64_t>::max() &&
+ !assembler->ShifterOperandCanHold(kNoRegister,
+ kNoRegister,
+ SBC,
+ High32Bits(value + 1),
+ kCcSet,
+ &so)) {
+ return false;
+ }
+ } else if (!assembler->ShifterOperandCanHold(kNoRegister,
+ kNoRegister,
+ SBC,
+ High32Bits(value),
+ kCcSet,
+ &so)) {
+ return false;
+ }
+ }
+ }
+ }
+
+ return true;
+}
+
+static bool CanEncodeConstantAs8BitImmediate(HConstant* constant) {
+ const Primitive::Type type = constant->GetType();
+ bool ret = false;
+
+ DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type;
+
+ if (type == Primitive::kPrimLong) {
+ const uint64_t value = constant->AsLongConstant()->GetValueAsUint64();
+
+ ret = IsUint<8>(Low32Bits(value)) && IsUint<8>(High32Bits(value));
+ } else {
+ ret = IsUint<8>(CodeGenerator::GetInt32ValueOf(constant));
+ }
+
+ return ret;
+}
+
+static Location Arm8BitEncodableConstantOrRegister(HInstruction* constant) {
+ DCHECK(!Primitive::IsFloatingPointType(constant->GetType()));
+
+ if (constant->IsConstant() && CanEncodeConstantAs8BitImmediate(constant->AsConstant())) {
+ return Location::ConstantLocation(constant->AsConstant());
+ }
+
+ return Location::RequiresRegister();
+}
+
+static bool CanGenerateConditionalMove(const Location& out, const Location& src) {
+ // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8,
+ // we check that we are not dealing with floating-point output (there is no
+ // 16-bit VMOV encoding).
+ if (!out.IsRegister() && !out.IsRegisterPair()) {
+ return false;
+ }
+
+ // For constants, we also check that the output is in one or two low registers,
+ // and that the constants fit in an 8-bit unsigned integer, so that a 16-bit
+ // MOV encoding can be used.
+ if (src.IsConstant()) {
+ if (!CanEncodeConstantAs8BitImmediate(src.GetConstant())) {
+ return false;
+ }
+
+ if (out.IsRegister()) {
+ if (!ArmAssembler::IsLowRegister(out.AsRegister<Register>())) {
+ return false;
+ }
+ } else {
+ DCHECK(out.IsRegisterPair());
+
+ if (!ArmAssembler::IsLowRegister(out.AsRegisterPairHigh<Register>())) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
#undef __
// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy.
#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT
+Label* CodeGeneratorARM::GetFinalLabel(HInstruction* instruction, Label* final_label) {
+ DCHECK(!instruction->IsControlFlow() && !instruction->IsSuspendCheck());
+
+ const HBasicBlock* const block = instruction->GetBlock();
+ const HLoopInformation* const info = block->GetLoopInformation();
+ HInstruction* const next = instruction->GetNext();
+
+ // Avoid a branch to a branch.
+ if (next->IsGoto() && (info == nullptr ||
+ !info->IsBackEdge(*block) ||
+ !info->HasSuspendCheck())) {
+ final_label = GetLabelOf(next->AsGoto()->GetSuccessor());
+ }
+
+ return final_label;
+}
+
void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const {
stream << Register(reg);
}
@@ -1437,8 +1948,6 @@
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- boot_image_address_patches_(std::less<uint32_t>(),
- graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_string_patches_(StringReferenceValueComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_class_patches_(TypeReferenceValueComparator(),
@@ -1905,44 +2414,6 @@
void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
}
-void InstructionCodeGeneratorARM::GenerateVcmp(HInstruction* instruction) {
- Primitive::Type type = instruction->InputAt(0)->GetType();
- Location lhs_loc = instruction->GetLocations()->InAt(0);
- Location rhs_loc = instruction->GetLocations()->InAt(1);
- if (rhs_loc.IsConstant()) {
- // 0.0 is the only immediate that can be encoded directly in
- // a VCMP instruction.
- //
- // Both the JLS (section 15.20.1) and the JVMS (section 6.5)
- // specify that in a floating-point comparison, positive zero
- // and negative zero are considered equal, so we can use the
- // literal 0.0 for both cases here.
- //
- // Note however that some methods (Float.equal, Float.compare,
- // Float.compareTo, Double.equal, Double.compare,
- // Double.compareTo, Math.max, Math.min, StrictMath.max,
- // StrictMath.min) consider 0.0 to be (strictly) greater than
- // -0.0. So if we ever translate calls to these methods into a
- // HCompare instruction, we must handle the -0.0 case with
- // care here.
- DCHECK(rhs_loc.GetConstant()->IsArithmeticZero());
- if (type == Primitive::kPrimFloat) {
- __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>());
- } else {
- DCHECK_EQ(type, Primitive::kPrimDouble);
- __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()));
- }
- } else {
- if (type == Primitive::kPrimFloat) {
- __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>());
- } else {
- DCHECK_EQ(type, Primitive::kPrimDouble);
- __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()),
- FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>()));
- }
- }
-}
-
void InstructionCodeGeneratorARM::GenerateFPJumps(HCondition* cond,
Label* true_label,
Label* false_label ATTRIBUTE_UNUSED) {
@@ -2050,7 +2521,7 @@
break;
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
- GenerateVcmp(condition);
+ GenerateVcmp(condition, codegen_);
GenerateFPJumps(condition, true_target, false_target);
break;
default:
@@ -2120,20 +2591,38 @@
return;
}
+ Label* non_fallthrough_target;
+ Condition arm_cond;
LocationSummary* locations = cond->GetLocations();
DCHECK(locations->InAt(0).IsRegister());
Register left = locations->InAt(0).AsRegister<Register>();
Location right = locations->InAt(1);
- if (right.IsRegister()) {
- __ cmp(left, ShifterOperand(right.AsRegister<Register>()));
- } else {
- DCHECK(right.IsConstant());
- __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant()));
- }
+
if (true_target == nullptr) {
- __ b(false_target, ARMCondition(condition->GetOppositeCondition()));
+ arm_cond = ARMCondition(condition->GetOppositeCondition());
+ non_fallthrough_target = false_target;
} else {
- __ b(true_target, ARMCondition(condition->GetCondition()));
+ arm_cond = ARMCondition(condition->GetCondition());
+ non_fallthrough_target = true_target;
+ }
+
+ if (right.IsConstant() && (arm_cond == NE || arm_cond == EQ) &&
+ CodeGenerator::GetInt32ValueOf(right.GetConstant()) == 0) {
+ if (arm_cond == EQ) {
+ __ CompareAndBranchIfZero(left, non_fallthrough_target);
+ } else {
+ DCHECK_EQ(arm_cond, NE);
+ __ CompareAndBranchIfNonZero(left, non_fallthrough_target);
+ }
+ } else {
+ if (right.IsRegister()) {
+ __ cmp(left, ShifterOperand(right.AsRegister<Register>()));
+ } else {
+ DCHECK(right.IsConstant());
+ __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant()));
+ }
+
+ __ b(non_fallthrough_target, arm_cond);
}
}
@@ -2193,28 +2682,140 @@
void LocationsBuilderARM::VisitSelect(HSelect* select) {
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select);
- if (Primitive::IsFloatingPointType(select->GetType())) {
+ const bool is_floating_point = Primitive::IsFloatingPointType(select->GetType());
+
+ if (is_floating_point) {
locations->SetInAt(0, Location::RequiresFpuRegister());
- locations->SetInAt(1, Location::RequiresFpuRegister());
+ locations->SetInAt(1, Location::FpuRegisterOrConstant(select->GetTrueValue()));
} else {
locations->SetInAt(0, Location::RequiresRegister());
- locations->SetInAt(1, Location::RequiresRegister());
+ locations->SetInAt(1, Arm8BitEncodableConstantOrRegister(select->GetTrueValue()));
}
+
if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) {
- locations->SetInAt(2, Location::RequiresRegister());
+ locations->SetInAt(2, Location::RegisterOrConstant(select->GetCondition()));
+ // The code generator handles overlap with the values, but not with the condition.
+ locations->SetOut(Location::SameAsFirstInput());
+ } else if (is_floating_point) {
+ locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
+ } else {
+ if (!locations->InAt(1).IsConstant()) {
+ locations->SetInAt(0, Arm8BitEncodableConstantOrRegister(select->GetFalseValue()));
+ }
+
+ locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}
- locations->SetOut(Location::SameAsFirstInput());
}
void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) {
- LocationSummary* locations = select->GetLocations();
- Label false_target;
- GenerateTestAndBranch(select,
- /* condition_input_index */ 2,
- /* true_target */ nullptr,
- &false_target);
- codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType());
- __ Bind(&false_target);
+ HInstruction* const condition = select->GetCondition();
+ const LocationSummary* const locations = select->GetLocations();
+ const Primitive::Type type = select->GetType();
+ const Location first = locations->InAt(0);
+ const Location out = locations->Out();
+ const Location second = locations->InAt(1);
+ Location src;
+
+ if (condition->IsIntConstant()) {
+ if (condition->AsIntConstant()->IsFalse()) {
+ src = first;
+ } else {
+ src = second;
+ }
+
+ codegen_->MoveLocation(out, src, type);
+ return;
+ }
+
+ if (!Primitive::IsFloatingPointType(type) &&
+ CanGenerateTest(condition, codegen_->GetAssembler())) {
+ bool invert = false;
+
+ if (out.Equals(second)) {
+ src = first;
+ invert = true;
+ } else if (out.Equals(first)) {
+ src = second;
+ } else if (second.IsConstant()) {
+ DCHECK(CanEncodeConstantAs8BitImmediate(second.GetConstant()));
+ src = second;
+ } else if (first.IsConstant()) {
+ DCHECK(CanEncodeConstantAs8BitImmediate(first.GetConstant()));
+ src = first;
+ invert = true;
+ } else {
+ src = second;
+ }
+
+ if (CanGenerateConditionalMove(out, src)) {
+ if (!out.Equals(first) && !out.Equals(second)) {
+ codegen_->MoveLocation(out, src.Equals(first) ? second : first, type);
+ }
+
+ const Condition cond = GenerateTest(condition, locations->InAt(2), invert, codegen_);
+
+ if (out.IsRegister()) {
+ ShifterOperand operand;
+
+ if (src.IsConstant()) {
+ operand = ShifterOperand(CodeGenerator::GetInt32ValueOf(src.GetConstant()));
+ } else {
+ DCHECK(src.IsRegister());
+ operand = ShifterOperand(src.AsRegister<Register>());
+ }
+
+ __ it(cond);
+ __ mov(out.AsRegister<Register>(), operand, cond);
+ } else {
+ DCHECK(out.IsRegisterPair());
+
+ ShifterOperand operand_high;
+ ShifterOperand operand_low;
+
+ if (src.IsConstant()) {
+ const int64_t value = src.GetConstant()->AsLongConstant()->GetValue();
+
+ operand_high = ShifterOperand(High32Bits(value));
+ operand_low = ShifterOperand(Low32Bits(value));
+ } else {
+ DCHECK(src.IsRegisterPair());
+ operand_high = ShifterOperand(src.AsRegisterPairHigh<Register>());
+ operand_low = ShifterOperand(src.AsRegisterPairLow<Register>());
+ }
+
+ __ it(cond);
+ __ mov(out.AsRegisterPairLow<Register>(), operand_low, cond);
+ __ it(cond);
+ __ mov(out.AsRegisterPairHigh<Register>(), operand_high, cond);
+ }
+
+ return;
+ }
+ }
+
+ Label* false_target = nullptr;
+ Label* true_target = nullptr;
+ Label select_end;
+ Label* target = codegen_->GetFinalLabel(select, &select_end);
+
+ if (out.Equals(second)) {
+ true_target = target;
+ src = first;
+ } else {
+ false_target = target;
+ src = second;
+
+ if (!out.Equals(first)) {
+ codegen_->MoveLocation(out, first, type);
+ }
+ }
+
+ GenerateTestAndBranch(select, 2, true_target, false_target);
+ codegen_->MoveLocation(out, src, type);
+
+ if (select_end.IsLinked()) {
+ __ Bind(&select_end);
+ }
}
void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) {
@@ -2293,7 +2894,7 @@
break;
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
- GenerateVcmp(cond);
+ GenerateVcmp(cond, codegen_);
GenerateFPJumps(cond, &true_label, &false_label);
break;
}
@@ -4347,7 +4948,7 @@
case Primitive::kPrimFloat:
case Primitive::kPrimDouble: {
__ LoadImmediate(out, 0);
- GenerateVcmp(compare);
+ GenerateVcmp(compare, codegen_);
__ vmstat(); // transfer FP status register to ARM APSR.
less_cond = ARMFPCondition(kCondLT, compare->IsGtBias());
break;
@@ -4703,17 +5304,29 @@
return true;
}
Opcode neg_opcode = kNoOperand;
+ uint32_t neg_value = 0;
switch (opcode) {
- case AND: neg_opcode = BIC; value = ~value; break;
- case ORR: neg_opcode = ORN; value = ~value; break;
- case ADD: neg_opcode = SUB; value = -value; break;
- case ADC: neg_opcode = SBC; value = ~value; break;
- case SUB: neg_opcode = ADD; value = -value; break;
- case SBC: neg_opcode = ADC; value = ~value; break;
+ case AND: neg_opcode = BIC; neg_value = ~value; break;
+ case ORR: neg_opcode = ORN; neg_value = ~value; break;
+ case ADD: neg_opcode = SUB; neg_value = -value; break;
+ case ADC: neg_opcode = SBC; neg_value = ~value; break;
+ case SUB: neg_opcode = ADD; neg_value = -value; break;
+ case SBC: neg_opcode = ADC; neg_value = ~value; break;
+ case MOV: neg_opcode = MVN; neg_value = ~value; break;
default:
return false;
}
- return assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, neg_opcode, value, set_cc, &so);
+
+ if (assembler->ShifterOperandCanHold(kNoRegister,
+ kNoRegister,
+ neg_opcode,
+ neg_value,
+ set_cc,
+ &so)) {
+ return true;
+ }
+
+ return opcode == AND && IsPowerOfTwo(value + 1);
}
void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction,
@@ -5615,21 +6228,59 @@
caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves);
- locations->SetInAt(0, Location::RequiresRegister());
- locations->SetInAt(1, Location::RequiresRegister());
+
+ HInstruction* index = instruction->InputAt(0);
+ HInstruction* length = instruction->InputAt(1);
+ // If both index and length are constants we can statically check the bounds. But if at least one
+ // of them is not encodable ArmEncodableConstantOrRegister will create
+ // Location::RequiresRegister() which is not desired to happen. Instead we create constant
+ // locations.
+ bool both_const = index->IsConstant() && length->IsConstant();
+ locations->SetInAt(0, both_const
+ ? Location::ConstantLocation(index->AsConstant())
+ : ArmEncodableConstantOrRegister(index, CMP));
+ locations->SetInAt(1, both_const
+ ? Location::ConstantLocation(length->AsConstant())
+ : ArmEncodableConstantOrRegister(length, CMP));
}
void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) {
LocationSummary* locations = instruction->GetLocations();
- SlowPathCodeARM* slow_path =
- new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction);
- codegen_->AddSlowPath(slow_path);
+ Location index_loc = locations->InAt(0);
+ Location length_loc = locations->InAt(1);
- Register index = locations->InAt(0).AsRegister<Register>();
- Register length = locations->InAt(1).AsRegister<Register>();
+ if (length_loc.IsConstant()) {
+ int32_t length = helpers::Int32ConstantFrom(length_loc);
+ if (index_loc.IsConstant()) {
+ // BCE will remove the bounds check if we are guaranteed to pass.
+ int32_t index = helpers::Int32ConstantFrom(index_loc);
+ if (index < 0 || index >= length) {
+ SlowPathCodeARM* slow_path =
+ new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction);
+ codegen_->AddSlowPath(slow_path);
+ __ b(slow_path->GetEntryLabel());
+ } else {
+ // Some optimization after BCE may have generated this, and we should not
+ // generate a bounds check if it is a valid range.
+ }
+ return;
+ }
- __ cmp(index, ShifterOperand(length));
- __ b(slow_path->GetEntryLabel(), HS);
+ SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction);
+ __ cmp(index_loc.AsRegister<Register>(), ShifterOperand(length));
+ codegen_->AddSlowPath(slow_path);
+ __ b(slow_path->GetEntryLabel(), HS);
+ } else {
+ SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction);
+ if (index_loc.IsConstant()) {
+ int32_t index = helpers::Int32ConstantFrom(index_loc);
+ __ cmp(length_loc.AsRegister<Register>(), ShifterOperand(index));
+ } else {
+ __ cmp(length_loc.AsRegister<Register>(), ShifterOperand(index_loc.AsRegister<Register>()));
+ }
+ codegen_->AddSlowPath(slow_path);
+ __ b(slow_path->GetEntryLabel(), LS);
+ }
}
void CodeGeneratorARM::MarkGCCard(Register temp,
@@ -6969,9 +7620,11 @@
ShifterOperand so;
if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) {
__ and_(out, first, so);
- } else {
- DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so));
+ } else if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)) {
__ bic(out, first, ShifterOperand(~value));
+ } else {
+ DCHECK(IsPowerOfTwo(value + 1));
+ __ ubfx(out, first, 0, WhichPowerOf2(value + 1));
}
}
@@ -7185,14 +7838,35 @@
DCHECK(kEmitCompilerReadBarrier);
if (kUseBakerReadBarrier) {
// Fast path implementation of art::ReadBarrier::BarrierForRoot when
- // Baker's read barrier are used:
+ // Baker's read barrier are used.
//
- // root = obj.field;
+ // Note that we do not actually check the value of
+ // `GetIsGcMarking()` to decide whether to mark the loaded GC
+ // root or not. Instead, we load into `temp` the read barrier
+ // mark entry point corresponding to register `root`. If `temp`
+ // is null, it means that `GetIsGcMarking()` is false, and vice
+ // versa.
+ //
// temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
- // if (temp != null) {
- // root = temp(root)
+ // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load.
+ // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking()
+ // // Slow path.
+ // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call.
// }
+ // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`.
+ Location temp = Location::RegisterLocation(LR);
+ SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(
+ instruction, root, /* entrypoint */ temp);
+ codegen_->AddSlowPath(slow_path);
+
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset);
+
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
__ LoadFromOffset(kLoadWord, root_reg, obj, offset);
static_assert(
@@ -7203,21 +7877,6 @@
"art::mirror::CompressedReference<mirror::Object> and int32_t "
"have different sizes.");
- // Slow path marking the GC root `root`.
- Location temp = Location::RegisterLocation(LR);
- SlowPathCodeARM* slow_path =
- new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(
- instruction,
- root,
- /*entrypoint*/ temp);
- codegen_->AddSlowPath(slow_path);
-
- // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
- const int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg());
- // Loading the entrypoint does not require a load acquire since it is only changed when
- // threads are suspended or running a checkpoint.
- __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset);
// The entrypoint is null when the GC is not marking, this prevents one load compared to
// checking GetIsGcMarking.
__ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel());
@@ -7288,51 +7947,101 @@
DCHECK(kEmitCompilerReadBarrier);
DCHECK(kUseBakerReadBarrier);
- // In slow path based read barriers, the read barrier call is
- // inserted after the original load. However, in fast path based
- // Baker's read barriers, we need to perform the load of
- // mirror::Object::monitor_ *before* the original reference load.
- // This load-load ordering is required by the read barrier.
- // The fast path/slow path (for Baker's algorithm) should look like:
+ // Query `art::Thread::Current()->GetIsGcMarking()` to decide
+ // whether we need to enter the slow path to mark the reference.
+ // Then, in the slow path, check the gray bit in the lock word of
+ // the reference's holder (`obj`) to decide whether to mark `ref` or
+ // not.
//
- // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
- // lfence; // Load fence or artificial data dependency to prevent load-load reordering
- // HeapReference<Object> ref = *src; // Original reference load.
- // bool is_gray = (rb_state == ReadBarrier::GrayState());
- // if (is_gray) {
- // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path.
+ // Note that we do not actually check the value of `GetIsGcMarking()`;
+ // instead, we load into `temp3` the read barrier mark entry point
+ // corresponding to register `ref`. If `temp3` is null, it means
+ // that `GetIsGcMarking()` is false, and vice versa.
+ //
+ // temp3 = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ // if (temp3 != nullptr) { // <=> Thread::Current()->GetIsGcMarking()
+ // // Slow path.
+ // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
+ // lfence; // Load fence or artificial data dependency to prevent load-load reordering
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
+ // if (is_gray) {
+ // ref = temp3(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call.
+ // }
+ // } else {
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
// }
- //
- // Note: the original implementation in ReadBarrier::Barrier is
- // slightly more complex as it performs additional checks that we do
- // not do here for performance reasons.
- Register ref_reg = ref.AsRegister<Register>();
Register temp_reg = temp.AsRegister<Register>();
- uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
- // /* int32_t */ monitor = obj->monitor_
- __ LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset);
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
+ // Slow path marking the object `ref` when the GC is marking. The
+ // entrypoint will already be loaded in `temp3`.
+ Location temp3 = Location::RegisterLocation(LR);
+ SlowPathCodeARM* slow_path;
+ if (always_update_field) {
+ DCHECK(temp2 != nullptr);
+ // LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM only
+ // supports address of the form `obj + field_offset`, where `obj`
+ // is a register and `field_offset` is a register pair (of which
+ // only the lower half is used). Thus `offset` and `scale_factor`
+ // above are expected to be null in this code path.
+ DCHECK_EQ(offset, 0u);
+ DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1);
+ Location field_offset = index;
+ slow_path =
+ new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM(
+ instruction,
+ ref,
+ obj,
+ offset,
+ /* index */ field_offset,
+ scale_factor,
+ needs_null_check,
+ temp_reg,
+ *temp2,
+ /* entrypoint */ temp3);
+ } else {
+ slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARM(
+ instruction,
+ ref,
+ obj,
+ offset,
+ index,
+ scale_factor,
+ needs_null_check,
+ temp_reg,
+ /* entrypoint */ temp3);
}
- // /* LockWord */ lock_word = LockWord(monitor)
- static_assert(sizeof(LockWord) == sizeof(int32_t),
- "art::LockWord and int32_t have different sizes.");
+ AddSlowPath(slow_path);
- // Introduce a dependency on the lock_word including the rb_state,
- // which shall prevent load-load reordering without using
- // a memory barrier (which would be more expensive).
- // `obj` is unchanged by this operation, but its value now depends
- // on `temp_reg`.
- __ add(obj, obj, ShifterOperand(temp_reg, LSR, 32));
+ // temp3 = Thread::Current()->pReadBarrierMarkReg ## ref.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ __ LoadFromOffset(kLoadWord, temp3.AsRegister<Register>(), TR, entry_point_offset);
+ // The entrypoint is null when the GC is not marking, this prevents one load compared to
+ // checking GetIsGcMarking.
+ __ CompareAndBranchIfNonZero(temp3.AsRegister<Register>(), slow_path->GetEntryLabel());
+ // Fast path: just load the reference.
+ GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check);
+ __ Bind(slow_path->GetExitLabel());
+}
- // The actual reference load.
+void CodeGeneratorARM::GenerateRawReferenceLoad(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check) {
+ Register ref_reg = ref.AsRegister<Register>();
+
if (index.IsValid()) {
// Load types involving an "index": ArrayGet,
// UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject
// intrinsics.
- // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor))
+ // /* HeapReference<mirror::Object> */ ref = *(obj + offset + (index << scale_factor))
if (index.IsConstant()) {
size_t computed_offset =
(index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset;
@@ -7349,41 +8058,16 @@
__ LoadFromOffset(kLoadWord, ref_reg, IP, offset);
}
} else {
- // /* HeapReference<Object> */ ref = *(obj + offset)
+ // /* HeapReference<mirror::Object> */ ref = *(obj + offset)
__ LoadFromOffset(kLoadWord, ref_reg, obj, offset);
}
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+
// Object* ref = ref_addr->AsMirrorPtr()
__ MaybeUnpoisonHeapReference(ref_reg);
-
- // Slow path marking the object `ref` when it is gray.
- SlowPathCodeARM* slow_path;
- if (always_update_field) {
- DCHECK(temp2 != nullptr);
- // ReadBarrierMarkAndUpdateFieldSlowPathARM only supports address
- // of the form `obj + field_offset`, where `obj` is a register and
- // `field_offset` is a register pair (of which only the lower half
- // is used). Thus `offset` and `scale_factor` above are expected
- // to be null in this code path.
- DCHECK_EQ(offset, 0u);
- DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1);
- slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARM(
- instruction, ref, obj, /* field_offset */ index, temp_reg, *temp2);
- } else {
- slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, ref);
- }
- AddSlowPath(slow_path);
-
- // if (rb_state == ReadBarrier::GrayState())
- // ref = ReadBarrier::Mark(ref);
- // Given the numeric representation, it's enough to check the low bit of the
- // rb_state. We do that by shifting the bit out of the lock word with LSRS
- // which can be a 16-bit instruction unlike the TST immediate.
- static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
- __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1);
- __ b(slow_path->GetEntryLabel(), CS); // Carry flag is the last bit shifted out by LSRS.
- __ Bind(slow_path->GetExitLabel());
}
void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction,
@@ -7626,9 +8310,7 @@
}
Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) {
- bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
- Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
- return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_);
}
Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file,
@@ -7679,8 +8361,7 @@
/* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() +
boot_image_type_patches_.size() +
/* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() +
- /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size() +
- boot_image_address_patches_.size();
+ /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size();
linker_patches->reserve(size);
EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_,
linker_patches);
@@ -7714,13 +8395,6 @@
target_type.dex_file,
target_type.type_index.index_));
}
- for (const auto& entry : boot_image_address_patches_) {
- DCHECK(GetCompilerOptions().GetIncludePatchInformation());
- Literal* literal = entry.second;
- DCHECK(literal->GetLabel()->IsBound());
- uint32_t literal_offset = literal->GetLabel()->Position();
- linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
- }
DCHECK_EQ(size, linker_patches->size());
}
diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h
index df2dbc7..5b15902 100644
--- a/compiler/optimizing/code_generator_arm.h
+++ b/compiler/optimizing/code_generator_arm.h
@@ -299,7 +299,6 @@
void GenerateCompareTestAndBranch(HCondition* condition,
Label* true_target,
Label* false_target);
- void GenerateVcmp(HInstruction* instruction);
void GenerateFPJumps(HCondition* cond, Label* true_label, Label* false_label);
void GenerateLongComparesAndJumps(HCondition* cond, Label* true_label, Label* false_label);
void DivRemOneOrMinusOne(HBinaryOperation* instruction);
@@ -422,6 +421,8 @@
return CommonGetLabelOf<Label>(block_labels_, block);
}
+ Label* GetFinalLabel(HInstruction* instruction, Label* final_label);
+
void Initialize() OVERRIDE {
block_labels_ = CommonInitializeLabels<Label>();
}
@@ -520,9 +521,6 @@
Location index,
Location temp,
bool needs_null_check);
- // Factored implementation used by GenerateFieldLoadWithBakerReadBarrier
- // and GenerateArrayLoadWithBakerReadBarrier.
-
// Factored implementation, used by GenerateFieldLoadWithBakerReadBarrier,
// GenerateArrayLoadWithBakerReadBarrier and some intrinsics.
//
@@ -545,6 +543,15 @@
bool always_update_field = false,
Register* temp2 = nullptr);
+ // Generate a heap reference load (with no read barrier).
+ void GenerateRawReferenceLoad(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check);
+
// Generate a read barrier for a heap reference within `instruction`
// using a slow path.
//
@@ -642,8 +649,6 @@
ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_;
// PC-relative type patch info for kBssEntry.
ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
- // Deduplication map for patchable boot image addresses.
- Uint32ToLiteralMap boot_image_address_patches_;
// Patches for string literals in JIT compiled code.
StringToLiteralMap jit_string_patches_;
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index 18c95b3..97b61ed 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -633,58 +633,23 @@
}
}
-// Slow path marking an object reference `ref` during a read
-// barrier. The field `obj.field` in the object `obj` holding this
-// reference does not get updated by this slow path after marking (see
-// ReadBarrierMarkAndUpdateFieldSlowPathARM64 below for that).
+// Abstract base class for read barrier slow paths marking a reference
+// `ref`.
//
-// This means that after the execution of this slow path, `ref` will
-// always be up-to-date, but `obj.field` may not; i.e., after the
-// flip, `ref` will be a to-space reference, but `obj.field` will
-// probably still be a from-space reference (unless it gets updated by
-// another thread, or if another thread installed another object
-// reference (different from `ref`) in `obj.field`).
-//
-// If `entrypoint` is a valid location it is assumed to already be
-// holding the entrypoint. The case where the entrypoint is passed in
-// is for the GcRoot read barrier.
-class ReadBarrierMarkSlowPathARM64 : public SlowPathCodeARM64 {
- public:
- ReadBarrierMarkSlowPathARM64(HInstruction* instruction,
- Location ref,
- Location entrypoint = Location::NoLocation())
- : SlowPathCodeARM64(instruction),
- ref_(ref),
- entrypoint_(entrypoint) {
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class ReadBarrierMarkSlowPathBaseARM64 : public SlowPathCodeARM64 {
+ protected:
+ ReadBarrierMarkSlowPathBaseARM64(HInstruction* instruction, Location ref, Location entrypoint)
+ : SlowPathCodeARM64(instruction), ref_(ref), entrypoint_(entrypoint) {
DCHECK(kEmitCompilerReadBarrier);
}
- const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM64"; }
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathBaseARM64"; }
- void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
- LocationSummary* locations = instruction_->GetLocations();
- DCHECK(locations->CanCall());
- DCHECK(ref_.IsRegister()) << ref_;
- DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_.reg())) << ref_.reg();
- DCHECK(instruction_->IsInstanceFieldGet() ||
- instruction_->IsStaticFieldGet() ||
- instruction_->IsArrayGet() ||
- instruction_->IsArraySet() ||
- instruction_->IsLoadClass() ||
- instruction_->IsLoadString() ||
- instruction_->IsInstanceOf() ||
- instruction_->IsCheckCast() ||
- (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
- (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
- << "Unexpected instruction in read barrier marking slow path: "
- << instruction_->DebugName();
- // The read barrier instrumentation of object ArrayGet
- // instructions does not support the HIntermediateAddress
- // instruction.
- DCHECK(!(instruction_->IsArrayGet() &&
- instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
-
- __ Bind(GetEntryLabel());
+ // Generate assembly code calling the read barrier marking runtime
+ // entry point (ReadBarrierMarkRegX).
+ void GenerateReadBarrierMarkRuntimeCall(CodeGenerator* codegen) {
// No need to save live registers; it's taken care of by the
// entrypoint. Also, there is no need to update the stack mask,
// as this runtime call will not trigger a garbage collection.
@@ -720,46 +685,261 @@
// This runtime call does not require a stack map.
arm64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
}
- __ B(GetExitLabel());
}
- private:
// The location (register) of the marked object reference.
const Location ref_;
// The location of the entrypoint if it is already loaded.
const Location entrypoint_;
+ private:
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathBaseARM64);
+};
+
+// Slow path marking an object reference `ref` during a read
+// barrier. The field `obj.field` in the object `obj` holding this
+// reference does not get updated by this slow path after marking.
+//
+// This means that after the execution of this slow path, `ref` will
+// always be up-to-date, but `obj.field` may not; i.e., after the
+// flip, `ref` will be a to-space reference, but `obj.field` will
+// probably still be a from-space reference (unless it gets updated by
+// another thread, or if another thread installed another object
+// reference (different from `ref`) in `obj.field`).
+//
+// If `entrypoint` is a valid location it is assumed to already be
+// holding the entrypoint. The case where the entrypoint is passed in
+// is when the decision to mark is based on whether the GC is marking.
+class ReadBarrierMarkSlowPathARM64 : public ReadBarrierMarkSlowPathBaseARM64 {
+ public:
+ ReadBarrierMarkSlowPathARM64(HInstruction* instruction,
+ Location ref,
+ Location entrypoint = Location::NoLocation())
+ : ReadBarrierMarkSlowPathBaseARM64(instruction, ref, entrypoint) {
+ DCHECK(kEmitCompilerReadBarrier);
+ }
+
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM64"; }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ DCHECK(locations->CanCall());
+ DCHECK(ref_.IsRegister()) << ref_;
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_.reg())) << ref_.reg();
+ DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString())
+ << "Unexpected instruction in read barrier marking slow path: "
+ << instruction_->DebugName();
+
+ __ Bind(GetEntryLabel());
+ GenerateReadBarrierMarkRuntimeCall(codegen);
+ __ B(GetExitLabel());
+ }
+
+ private:
DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM64);
};
-// Slow path marking an object reference `ref` during a read barrier,
-// and if needed, atomically updating the field `obj.field` in the
-// object `obj` holding this reference after marking (contrary to
-// ReadBarrierMarkSlowPathARM64 above, which never tries to update
-// `obj.field`).
+// Slow path loading `obj`'s lock word, loading a reference from
+// object `*(obj + offset + (index << scale_factor))` into `ref`, and
+// marking `ref` if `obj` is gray according to the lock word (Baker
+// read barrier). The field `obj.field` in the object `obj` holding
+// this reference does not get updated by this slow path after marking
+// (see LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64
+// below for that).
+//
+// This means that after the execution of this slow path, `ref` will
+// always be up-to-date, but `obj.field` may not; i.e., after the
+// flip, `ref` will be a to-space reference, but `obj.field` will
+// probably still be a from-space reference (unless it gets updated by
+// another thread, or if another thread installed another object
+// reference (different from `ref`) in `obj.field`).
+//
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class LoadReferenceWithBakerReadBarrierSlowPathARM64 : public ReadBarrierMarkSlowPathBaseARM64 {
+ public:
+ LoadReferenceWithBakerReadBarrierSlowPathARM64(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ size_t scale_factor,
+ bool needs_null_check,
+ bool use_load_acquire,
+ Register temp,
+ Location entrypoint)
+ : ReadBarrierMarkSlowPathBaseARM64(instruction, ref, entrypoint),
+ obj_(obj),
+ offset_(offset),
+ index_(index),
+ scale_factor_(scale_factor),
+ needs_null_check_(needs_null_check),
+ use_load_acquire_(use_load_acquire),
+ temp_(temp) {
+ DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
+ }
+
+ const char* GetDescription() const OVERRIDE {
+ return "LoadReferenceWithBakerReadBarrierSlowPathARM64";
+ }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ DCHECK(locations->CanCall());
+ DCHECK(ref_.IsRegister()) << ref_;
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_.reg())) << ref_.reg();
+ DCHECK(obj_.IsW());
+ DCHECK_NE(ref_.reg(), LocationFrom(temp_).reg());
+ DCHECK(instruction_->IsInstanceFieldGet() ||
+ instruction_->IsStaticFieldGet() ||
+ instruction_->IsArrayGet() ||
+ instruction_->IsArraySet() ||
+ instruction_->IsInstanceOf() ||
+ instruction_->IsCheckCast() ||
+ (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
+ (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
+ << "Unexpected instruction in read barrier marking slow path: "
+ << instruction_->DebugName();
+ // The read barrier instrumentation of object ArrayGet
+ // instructions does not support the HIntermediateAddress
+ // instruction.
+ DCHECK(!(instruction_->IsArrayGet() &&
+ instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
+
+ // Temporary register `temp_`, used to store the lock word, must
+ // not be IP0 nor IP1, as we may use them to emit the reference
+ // load (in the call to GenerateRawReferenceLoad below), and we
+ // need the lock word to still be in `temp_` after the reference
+ // load.
+ DCHECK_NE(LocationFrom(temp_).reg(), IP0);
+ DCHECK_NE(LocationFrom(temp_).reg(), IP1);
+
+ __ Bind(GetEntryLabel());
+
+ // When using MaybeGenerateReadBarrierSlow, the read barrier call is
+ // inserted after the original load. However, in fast path based
+ // Baker's read barriers, we need to perform the load of
+ // mirror::Object::monitor_ *before* the original reference load.
+ // This load-load ordering is required by the read barrier.
+ // The fast path/slow path (for Baker's algorithm) should look like:
+ //
+ // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
+ // lfence; // Load fence or artificial data dependency to prevent load-load reordering
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
+ // if (is_gray) {
+ // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call.
+ // }
+ //
+ // Note: the original implementation in ReadBarrier::Barrier is
+ // slightly more complex as it performs additional checks that we do
+ // not do here for performance reasons.
+
+ // /* int32_t */ monitor = obj->monitor_
+ uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+ __ Ldr(temp_, HeapOperand(obj_, monitor_offset));
+ if (needs_null_check_) {
+ codegen->MaybeRecordImplicitNullCheck(instruction_);
+ }
+ // /* LockWord */ lock_word = LockWord(monitor)
+ static_assert(sizeof(LockWord) == sizeof(int32_t),
+ "art::LockWord and int32_t have different sizes.");
+
+ // Introduce a dependency on the lock_word including rb_state,
+ // to prevent load-load reordering, and without using
+ // a memory barrier (which would be more expensive).
+ // `obj` is unchanged by this operation, but its value now depends
+ // on `temp`.
+ __ Add(obj_.X(), obj_.X(), Operand(temp_.X(), LSR, 32));
+
+ // The actual reference load.
+ // A possible implicit null check has already been handled above.
+ CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
+ arm64_codegen->GenerateRawReferenceLoad(instruction_,
+ ref_,
+ obj_,
+ offset_,
+ index_,
+ scale_factor_,
+ /* needs_null_check */ false,
+ use_load_acquire_);
+
+ // Mark the object `ref` when `obj` is gray.
+ //
+ // if (rb_state == ReadBarrier::GrayState())
+ // ref = ReadBarrier::Mark(ref);
+ //
+ // Given the numeric representation, it's enough to check the low bit of the rb_state.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Tbz(temp_, LockWord::kReadBarrierStateShift, GetExitLabel());
+ GenerateReadBarrierMarkRuntimeCall(codegen);
+
+ __ B(GetExitLabel());
+ }
+
+ private:
+ // The register containing the object holding the marked object reference field.
+ Register obj_;
+ // The offset, index and scale factor to access the reference in `obj_`.
+ uint32_t offset_;
+ Location index_;
+ size_t scale_factor_;
+ // Is a null check required?
+ bool needs_null_check_;
+ // Should this reference load use Load-Acquire semantics?
+ bool use_load_acquire_;
+ // A temporary register used to hold the lock word of `obj_`.
+ Register temp_;
+
+ DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierSlowPathARM64);
+};
+
+// Slow path loading `obj`'s lock word, loading a reference from
+// object `*(obj + offset + (index << scale_factor))` into `ref`, and
+// marking `ref` if `obj` is gray according to the lock word (Baker
+// read barrier). If needed, this slow path also atomically updates
+// the field `obj.field` in the object `obj` holding this reference
+// after marking (contrary to
+// LoadReferenceWithBakerReadBarrierSlowPathARM64 above, which never
+// tries to update `obj.field`).
//
// This means that after the execution of this slow path, both `ref`
// and `obj.field` will be up-to-date; i.e., after the flip, both will
// hold the same to-space reference (unless another thread installed
// another object reference (different from `ref`) in `obj.field`).
-class ReadBarrierMarkAndUpdateFieldSlowPathARM64 : public SlowPathCodeARM64 {
+//
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64
+ : public ReadBarrierMarkSlowPathBaseARM64 {
public:
- ReadBarrierMarkAndUpdateFieldSlowPathARM64(HInstruction* instruction,
- Location ref,
- Register obj,
- Location field_offset,
- Register temp)
- : SlowPathCodeARM64(instruction),
- ref_(ref),
+ LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ size_t scale_factor,
+ bool needs_null_check,
+ bool use_load_acquire,
+ Register temp,
+ Location entrypoint)
+ : ReadBarrierMarkSlowPathBaseARM64(instruction, ref, entrypoint),
obj_(obj),
- field_offset_(field_offset),
+ offset_(offset),
+ index_(index),
+ scale_factor_(scale_factor),
+ needs_null_check_(needs_null_check),
+ use_load_acquire_(use_load_acquire),
temp_(temp) {
DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
}
const char* GetDescription() const OVERRIDE {
- return "ReadBarrierMarkAndUpdateFieldSlowPathARM64";
+ return "LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64";
}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
@@ -768,64 +948,90 @@
DCHECK(locations->CanCall());
DCHECK(ref_.IsRegister()) << ref_;
DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_.reg())) << ref_.reg();
- // This slow path is only used by the UnsafeCASObject intrinsic.
+ DCHECK(obj_.IsW());
+ DCHECK_NE(ref_.reg(), LocationFrom(temp_).reg());
+
+ // This slow path is only used by the UnsafeCASObject intrinsic at the moment.
DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()))
<< "Unexpected instruction in read barrier marking and field updating slow path: "
<< instruction_->DebugName();
DCHECK(instruction_->GetLocations()->Intrinsified());
DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject);
- DCHECK(field_offset_.IsRegister()) << field_offset_;
+ DCHECK_EQ(offset_, 0u);
+ DCHECK_EQ(scale_factor_, 0u);
+ DCHECK_EQ(use_load_acquire_, false);
+ // The location of the offset of the marked reference field within `obj_`.
+ Location field_offset = index_;
+ DCHECK(field_offset.IsRegister()) << field_offset;
+
+ // Temporary register `temp_`, used to store the lock word, must
+ // not be IP0 nor IP1, as we may use them to emit the reference
+ // load (in the call to GenerateRawReferenceLoad below), and we
+ // need the lock word to still be in `temp_` after the reference
+ // load.
+ DCHECK_NE(LocationFrom(temp_).reg(), IP0);
+ DCHECK_NE(LocationFrom(temp_).reg(), IP1);
__ Bind(GetEntryLabel());
- // Save the old reference.
+ // /* int32_t */ monitor = obj->monitor_
+ uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+ __ Ldr(temp_, HeapOperand(obj_, monitor_offset));
+ if (needs_null_check_) {
+ codegen->MaybeRecordImplicitNullCheck(instruction_);
+ }
+ // /* LockWord */ lock_word = LockWord(monitor)
+ static_assert(sizeof(LockWord) == sizeof(int32_t),
+ "art::LockWord and int32_t have different sizes.");
+
+ // Introduce a dependency on the lock_word including rb_state,
+ // to prevent load-load reordering, and without using
+ // a memory barrier (which would be more expensive).
+ // `obj` is unchanged by this operation, but its value now depends
+ // on `temp`.
+ __ Add(obj_.X(), obj_.X(), Operand(temp_.X(), LSR, 32));
+
+ // The actual reference load.
+ // A possible implicit null check has already been handled above.
+ CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
+ arm64_codegen->GenerateRawReferenceLoad(instruction_,
+ ref_,
+ obj_,
+ offset_,
+ index_,
+ scale_factor_,
+ /* needs_null_check */ false,
+ use_load_acquire_);
+
+ // Mark the object `ref` when `obj` is gray.
+ //
+ // if (rb_state == ReadBarrier::GrayState())
+ // ref = ReadBarrier::Mark(ref);
+ //
+ // Given the numeric representation, it's enough to check the low bit of the rb_state.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Tbz(temp_, LockWord::kReadBarrierStateShift, GetExitLabel());
+
+ // Save the old value of the reference before marking it.
// Note that we cannot use IP to save the old reference, as IP is
// used internally by the ReadBarrierMarkRegX entry point, and we
// need the old reference after the call to that entry point.
DCHECK_NE(LocationFrom(temp_).reg(), IP0);
__ Mov(temp_.W(), ref_reg);
- // No need to save live registers; it's taken care of by the
- // entrypoint. Also, there is no need to update the stack mask,
- // as this runtime call will not trigger a garbage collection.
- CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
- DCHECK_NE(ref_.reg(), LR);
- DCHECK_NE(ref_.reg(), WSP);
- DCHECK_NE(ref_.reg(), WZR);
- // IP0 is used internally by the ReadBarrierMarkRegX entry point
- // as a temporary, it cannot be the entry point's input/output.
- DCHECK_NE(ref_.reg(), IP0);
- DCHECK(0 <= ref_.reg() && ref_.reg() < kNumberOfWRegisters) << ref_.reg();
- // "Compact" slow path, saving two moves.
- //
- // Instead of using the standard runtime calling convention (input
- // and output in W0):
- //
- // W0 <- ref
- // W0 <- ReadBarrierMark(W0)
- // ref <- W0
- //
- // we just use rX (the register containing `ref`) as input and output
- // of a dedicated entrypoint:
- //
- // rX <- ReadBarrierMarkRegX(rX)
- //
- int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref_.reg());
- // This runtime call does not require a stack map.
- arm64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ GenerateReadBarrierMarkRuntimeCall(codegen);
// If the new reference is different from the old reference,
- // update the field in the holder (`*(obj_ + field_offset_)`).
+ // update the field in the holder (`*(obj_ + field_offset)`).
//
// Note that this field could also hold a different object, if
// another thread had concurrently changed it. In that case, the
// LDXR/CMP/BNE sequence of instructions in the compare-and-set
// (CAS) operation below would abort the CAS, leaving the field
// as-is.
- vixl::aarch64::Label done;
__ Cmp(temp_.W(), ref_reg);
- __ B(eq, &done);
+ __ B(eq, GetExitLabel());
// Update the the holder's field atomically. This may fail if
// mutator updates before us, but it's OK. This is achieved
@@ -838,7 +1044,7 @@
// Convenience aliases.
Register base = obj_.W();
- Register offset = XRegisterFrom(field_offset_);
+ Register offset = XRegisterFrom(field_offset);
Register expected = temp_.W();
Register value = ref_reg;
Register tmp_ptr = temps.AcquireX(); // Pointer to actual memory.
@@ -882,21 +1088,26 @@
}
}
- __ Bind(&done);
__ B(GetExitLabel());
}
private:
- // The location (register) of the marked object reference.
- const Location ref_;
// The register containing the object holding the marked object reference field.
const Register obj_;
- // The location of the offset of the marked reference field within `obj_`.
- Location field_offset_;
-
+ // The offset, index and scale factor to access the reference in `obj_`.
+ uint32_t offset_;
+ Location index_;
+ size_t scale_factor_;
+ // Is a null check required?
+ bool needs_null_check_;
+ // Should this reference load use Load-Acquire semantics?
+ bool use_load_acquire_;
+ // A temporary register used to hold the lock word of `obj_`; and
+ // also to hold the original reference value, when the reference is
+ // marked.
const Register temp_;
- DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARM64);
+ DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64);
};
// Slow path generating a read barrier for a heap reference.
@@ -1200,8 +1411,6 @@
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- boot_image_address_patches_(std::less<uint32_t>(),
- graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_string_patches_(StringReferenceValueComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_class_patches_(TypeReferenceValueComparator(),
@@ -2427,6 +2636,9 @@
LocationSummary::kNoCall);
if (object_array_get_with_read_barrier && kUseBakerReadBarrier) {
locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
+ // We need a temporary register for the read barrier marking slow
+ // path in CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier.
+ locations->AddTemp(Location::RequiresRegister());
}
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
@@ -2462,7 +2674,7 @@
if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
// Object ArrayGet with Baker's read barrier case.
- Register temp = temps.AcquireW();
+ Register temp = WRegisterFrom(locations->GetTemp(0));
// Note that a potential implicit null check is handled in the
// CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier call.
codegen_->GenerateArrayLoadWithBakerReadBarrier(
@@ -3419,7 +3631,7 @@
if (Primitive::IsFloatingPointType(select->GetType())) {
locations->SetInAt(0, Location::RequiresFpuRegister());
locations->SetInAt(1, Location::RequiresFpuRegister());
- locations->SetOut(Location::RequiresFpuRegister());
+ locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
} else {
HConstant* cst_true_value = select->GetTrueValue()->AsConstant();
HConstant* cst_false_value = select->GetFalseValue()->AsConstant();
@@ -3442,7 +3654,7 @@
: Location::ConstantLocation(cst_true_value));
locations->SetInAt(0, false_value_in_register ? Location::RequiresRegister()
: Location::ConstantLocation(cst_false_value));
- locations->SetOut(Location::RequiresRegister());
+ locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}
if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) {
@@ -4328,9 +4540,7 @@
vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageAddressLiteral(
uint64_t address) {
- bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
- Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
- return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_);
}
vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateJitStringLiteral(
@@ -4398,8 +4608,7 @@
pc_relative_string_patches_.size() +
boot_image_type_patches_.size() +
pc_relative_type_patches_.size() +
- type_bss_entry_patches_.size() +
- boot_image_address_patches_.size();
+ type_bss_entry_patches_.size();
linker_patches->reserve(size);
for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) {
linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.GetLocation(),
@@ -4433,11 +4642,6 @@
target_type.dex_file,
target_type.type_index.index_));
}
- for (const auto& entry : boot_image_address_patches_) {
- DCHECK(GetCompilerOptions().GetIncludePatchInformation());
- vixl::aarch64::Literal<uint32_t>* literal = entry.second;
- linker_patches->push_back(LinkerPatch::RecordPosition(literal->GetOffset()));
- }
DCHECK_EQ(size, linker_patches->size());
}
@@ -5614,14 +5818,35 @@
DCHECK(kEmitCompilerReadBarrier);
if (kUseBakerReadBarrier) {
// Fast path implementation of art::ReadBarrier::BarrierForRoot when
- // Baker's read barrier are used:
+ // Baker's read barrier are used.
//
- // root = obj.field;
+ // Note that we do not actually check the value of
+ // `GetIsGcMarking()` to decide whether to mark the loaded GC
+ // root or not. Instead, we load into `temp` the read barrier
+ // mark entry point corresponding to register `root`. If `temp`
+ // is null, it means that `GetIsGcMarking()` is false, and vice
+ // versa.
+ //
// temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
- // if (temp != null) {
- // root = temp(root)
+ // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load.
+ // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking()
+ // // Slow path.
+ // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call.
// }
+ // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`.
+ Register temp = lr;
+ SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(
+ instruction, root, /* entrypoint */ LocationFrom(temp));
+ codegen_->AddSlowPath(slow_path);
+
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(root.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ __ Ldr(temp, MemOperand(tr, entry_point_offset));
+
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
if (fixup_label == nullptr) {
__ Ldr(root_reg, MemOperand(obj, offset));
@@ -5636,20 +5861,6 @@
"art::mirror::CompressedReference<mirror::Object> and int32_t "
"have different sizes.");
- Register temp = lr;
-
- // Slow path marking the GC root `root`. The entrypoint will alrady be loaded in temp.
- SlowPathCodeARM64* slow_path =
- new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction,
- root,
- LocationFrom(temp));
- codegen_->AddSlowPath(slow_path);
- const int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(root.reg());
- // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
- // Loading the entrypoint does not require a load acquire since it is only changed when
- // threads are suspended or running a checkpoint.
- __ Ldr(temp, MemOperand(tr, entry_point_offset));
// The entrypoint is null when the GC is not marking, this prevents one load compared to
// checking GetIsGcMarking.
__ Cbnz(temp, slow_path->GetEntryLabel());
@@ -5751,54 +5962,103 @@
// `instruction->IsArrayGet()` => `!use_load_acquire`.
DCHECK(!instruction->IsArrayGet() || !use_load_acquire);
- MacroAssembler* masm = GetVIXLAssembler();
- UseScratchRegisterScope temps(masm);
-
- // In slow path based read barriers, the read barrier call is
- // inserted after the original load. However, in fast path based
- // Baker's read barriers, we need to perform the load of
- // mirror::Object::monitor_ *before* the original reference load.
- // This load-load ordering is required by the read barrier.
- // The fast path/slow path (for Baker's algorithm) should look like:
+ // Query `art::Thread::Current()->GetIsGcMarking()` to decide
+ // whether we need to enter the slow path to mark the reference.
+ // Then, in the slow path, check the gray bit in the lock word of
+ // the reference's holder (`obj`) to decide whether to mark `ref` or
+ // not.
//
- // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
- // lfence; // Load fence or artificial data dependency to prevent load-load reordering
- // HeapReference<Object> ref = *src; // Original reference load.
- // bool is_gray = (rb_state == ReadBarrier::GrayState());
- // if (is_gray) {
- // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path.
+ // Note that we do not actually check the value of `GetIsGcMarking()`;
+ // instead, we load into `temp2` the read barrier mark entry point
+ // corresponding to register `ref`. If `temp2` is null, it means
+ // that `GetIsGcMarking()` is false, and vice versa.
+ //
+ // temp2 = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ // if (temp2 != nullptr) { // <=> Thread::Current()->GetIsGcMarking()
+ // // Slow path.
+ // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
+ // lfence; // Load fence or artificial data dependency to prevent load-load reordering
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
+ // if (is_gray) {
+ // ref = temp2(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call.
+ // }
+ // } else {
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
// }
- //
- // Note: the original implementation in ReadBarrier::Barrier is
- // slightly more complex as it performs additional checks that we do
- // not do here for performance reasons.
+ // Slow path marking the object `ref` when the GC is marking. The
+ // entrypoint will already be loaded in `temp2`.
+ Register temp2 = lr;
+ Location temp2_loc = LocationFrom(temp2);
+ SlowPathCodeARM64* slow_path;
+ if (always_update_field) {
+ // LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64
+ // only supports address of the form `obj + field_offset`, where
+ // `obj` is a register and `field_offset` is a register. Thus
+ // `offset` and `scale_factor` above are expected to be null in
+ // this code path.
+ DCHECK_EQ(offset, 0u);
+ DCHECK_EQ(scale_factor, 0u); /* "times 1" */
+ Location field_offset = index;
+ slow_path =
+ new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64(
+ instruction,
+ ref,
+ obj,
+ offset,
+ /* index */ field_offset,
+ scale_factor,
+ needs_null_check,
+ use_load_acquire,
+ temp,
+ /* entrypoint */ temp2_loc);
+ } else {
+ slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARM64(
+ instruction,
+ ref,
+ obj,
+ offset,
+ index,
+ scale_factor,
+ needs_null_check,
+ use_load_acquire,
+ temp,
+ /* entrypoint */ temp2_loc);
+ }
+ AddSlowPath(slow_path);
+
+ // temp2 = Thread::Current()->pReadBarrierMarkReg ## ref.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ __ Ldr(temp2, MemOperand(tr, entry_point_offset));
+ // The entrypoint is null when the GC is not marking, this prevents one load compared to
+ // checking GetIsGcMarking.
+ __ Cbnz(temp2, slow_path->GetEntryLabel());
+ // Fast path: just load the reference.
+ GenerateRawReferenceLoad(
+ instruction, ref, obj, offset, index, scale_factor, needs_null_check, use_load_acquire);
+ __ Bind(slow_path->GetExitLabel());
+}
+
+void CodeGeneratorARM64::GenerateRawReferenceLoad(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ size_t scale_factor,
+ bool needs_null_check,
+ bool use_load_acquire) {
+ DCHECK(obj.IsW());
Primitive::Type type = Primitive::kPrimNot;
Register ref_reg = RegisterFrom(ref, type);
- DCHECK(obj.IsW());
- uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
- {
- // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
- EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
- // /* int32_t */ monitor = obj->monitor_
- __ Ldr(temp, HeapOperand(obj, monitor_offset));
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
- }
- }
- // /* LockWord */ lock_word = LockWord(monitor)
- static_assert(sizeof(LockWord) == sizeof(int32_t),
- "art::LockWord and int32_t have different sizes.");
+ // If needed, vixl::EmissionCheckScope guards are used to ensure
+ // that no pools are emitted between the load (macro) instruction
+ // and MaybeRecordImplicitNullCheck.
- // Introduce a dependency on the lock_word including rb_state,
- // to prevent load-load reordering, and without using
- // a memory barrier (which would be more expensive).
- // `obj` is unchanged by this operation, but its value now depends
- // on `temp`.
- __ Add(obj.X(), obj.X(), Operand(temp.X(), LSR, 32));
-
- // The actual reference load.
if (index.IsValid()) {
// Load types involving an "index": ArrayGet,
// UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject
@@ -5813,59 +6073,50 @@
<< instruction->AsInvoke()->GetIntrinsic();
DCHECK_EQ(offset, 0u);
DCHECK_EQ(scale_factor, 0u);
- DCHECK_EQ(needs_null_check, 0u);
- // /* HeapReference<Object> */ ref = *(obj + index)
+ DCHECK_EQ(needs_null_check, false);
+ // /* HeapReference<mirror::Object> */ ref = *(obj + index)
MemOperand field = HeapOperand(obj, XRegisterFrom(index));
LoadAcquire(instruction, ref_reg, field, /* needs_null_check */ false);
} else {
- // ArrayGet and UnsafeGetObject intrinsics cases.
- // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor))
+ // ArrayGet and UnsafeGetObject and UnsafeCASObject intrinsics cases.
+ // /* HeapReference<mirror::Object> */ ref = *(obj + offset + (index << scale_factor))
if (index.IsConstant()) {
uint32_t computed_offset = offset + (Int64ConstantFrom(index) << scale_factor);
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
Load(type, ref_reg, HeapOperand(obj, computed_offset));
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
} else {
- Register temp3 = temps.AcquireW();
- __ Add(temp3, obj, offset);
- Load(type, ref_reg, HeapOperand(temp3, XRegisterFrom(index), LSL, scale_factor));
- temps.Release(temp3);
+ UseScratchRegisterScope temps(GetVIXLAssembler());
+ Register temp = temps.AcquireW();
+ __ Add(temp, obj, offset);
+ {
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ Load(type, ref_reg, HeapOperand(temp, XRegisterFrom(index), LSL, scale_factor));
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
}
}
} else {
- // /* HeapReference<Object> */ ref = *(obj + offset)
+ // /* HeapReference<mirror::Object> */ ref = *(obj + offset)
MemOperand field = HeapOperand(obj, offset);
if (use_load_acquire) {
- LoadAcquire(instruction, ref_reg, field, /* needs_null_check */ false);
+ // Implicit null checks are handled by CodeGeneratorARM64::LoadAcquire.
+ LoadAcquire(instruction, ref_reg, field, needs_null_check);
} else {
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
Load(type, ref_reg, field);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
}
}
// Object* ref = ref_addr->AsMirrorPtr()
GetAssembler()->MaybeUnpoisonHeapReference(ref_reg);
-
- // Slow path marking the object `ref` when it is gray.
- SlowPathCodeARM64* slow_path;
- if (always_update_field) {
- // ReadBarrierMarkAndUpdateFieldSlowPathARM64 only supports
- // address of the form `obj + field_offset`, where `obj` is a
- // register and `field_offset` is a register. Thus `offset` and
- // `scale_factor` above are expected to be null in this code path.
- DCHECK_EQ(offset, 0u);
- DCHECK_EQ(scale_factor, 0u); /* "times 1" */
- slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARM64(
- instruction, ref, obj, /* field_offset */ index, temp);
- } else {
- slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, ref);
- }
- AddSlowPath(slow_path);
-
- // if (rb_state == ReadBarrier::GrayState())
- // ref = ReadBarrier::Mark(ref);
- // Given the numeric representation, it's enough to check the low bit of the rb_state.
- static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
- __ Tbnz(temp, LockWord::kReadBarrierStateShift, slow_path->GetEntryLabel());
- __ Bind(slow_path->GetExitLabel());
}
void CodeGeneratorARM64::GenerateReadBarrierSlow(HInstruction* instruction,
diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h
index 5faf29a..7471cd5 100644
--- a/compiler/optimizing/code_generator_arm64.h
+++ b/compiler/optimizing/code_generator_arm64.h
@@ -616,8 +616,8 @@
Location index,
vixl::aarch64::Register temp,
bool needs_null_check);
- // Factored implementation used by GenerateFieldLoadWithBakerReadBarrier
- // and GenerateArrayLoadWithBakerReadBarrier.
+ // Factored implementation, used by GenerateFieldLoadWithBakerReadBarrier,
+ // GenerateArrayLoadWithBakerReadBarrier and some intrinsics.
//
// Load the object reference located at the address
// `obj + offset + (index << scale_factor)`, held by object `obj`, into
@@ -636,6 +636,16 @@
bool use_load_acquire,
bool always_update_field = false);
+ // Generate a heap reference load (with no read barrier).
+ void GenerateRawReferenceLoad(HInstruction* instruction,
+ Location ref,
+ vixl::aarch64::Register obj,
+ uint32_t offset,
+ Location index,
+ size_t scale_factor,
+ bool needs_null_check,
+ bool use_load_acquire);
+
// Generate a read barrier for a heap reference within `instruction`
// using a slow path.
//
@@ -761,8 +771,6 @@
ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_;
// PC-relative type patch info for kBssEntry.
ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
- // Deduplication map for patchable boot image addresses.
- Uint32ToLiteralMap boot_image_address_patches_;
// Patches for string literals in JIT compiled code.
StringToLiteralMap jit_string_patches_;
diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc
index 6bfbe4a..f5ada52 100644
--- a/compiler/optimizing/code_generator_arm_vixl.cc
+++ b/compiler/optimizing/code_generator_arm_vixl.cc
@@ -42,6 +42,7 @@
using helpers::DWARFReg;
using helpers::HighDRegisterFrom;
using helpers::HighRegisterFrom;
+using helpers::InputDRegisterAt;
using helpers::InputOperandAt;
using helpers::InputRegister;
using helpers::InputRegisterAt;
@@ -53,6 +54,7 @@
using helpers::LocationFrom;
using helpers::LowRegisterFrom;
using helpers::LowSRegisterFrom;
+using helpers::OperandFrom;
using helpers::OutputRegister;
using helpers::OutputSRegister;
using helpers::OutputVRegister;
@@ -657,52 +659,25 @@
DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARMVIXL);
};
-// Slow path marking an object reference `ref` during a read
-// barrier. The field `obj.field` in the object `obj` holding this
-// reference does not get updated by this slow path after marking (see
-// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that).
+// Abstract base class for read barrier slow paths marking a reference
+// `ref`.
//
-// This means that after the execution of this slow path, `ref` will
-// always be up-to-date, but `obj.field` may not; i.e., after the
-// flip, `ref` will be a to-space reference, but `obj.field` will
-// probably still be a from-space reference (unless it gets updated by
-// another thread, or if another thread installed another object
-// reference (different from `ref`) in `obj.field`).
-class ReadBarrierMarkSlowPathARMVIXL : public SlowPathCodeARMVIXL {
- public:
- ReadBarrierMarkSlowPathARMVIXL(HInstruction* instruction,
- Location ref,
- Location entrypoint = Location::NoLocation())
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class ReadBarrierMarkSlowPathBaseARMVIXL : public SlowPathCodeARMVIXL {
+ protected:
+ ReadBarrierMarkSlowPathBaseARMVIXL(HInstruction* instruction, Location ref, Location entrypoint)
: SlowPathCodeARMVIXL(instruction), ref_(ref), entrypoint_(entrypoint) {
DCHECK(kEmitCompilerReadBarrier);
}
- const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARMVIXL"; }
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathBaseARMVIXL"; }
- void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
- LocationSummary* locations = instruction_->GetLocations();
+ // Generate assembly code calling the read barrier marking runtime
+ // entry point (ReadBarrierMarkRegX).
+ void GenerateReadBarrierMarkRuntimeCall(CodeGenerator* codegen) {
vixl32::Register ref_reg = RegisterFrom(ref_);
- DCHECK(locations->CanCall());
- DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg.GetCode())) << ref_reg;
- DCHECK(instruction_->IsInstanceFieldGet() ||
- instruction_->IsStaticFieldGet() ||
- instruction_->IsArrayGet() ||
- instruction_->IsArraySet() ||
- instruction_->IsLoadClass() ||
- instruction_->IsLoadString() ||
- instruction_->IsInstanceOf() ||
- instruction_->IsCheckCast() ||
- (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
- (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
- << "Unexpected instruction in read barrier marking slow path: "
- << instruction_->DebugName();
- // The read barrier instrumentation of object ArrayGet
- // instructions does not support the HIntermediateAddress
- // instruction.
- DCHECK(!(instruction_->IsArrayGet() &&
- instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
- __ Bind(GetEntryLabel());
// No need to save live registers; it's taken care of by the
// entrypoint. Also, there is no need to update the stack mask,
// as this runtime call will not trigger a garbage collection.
@@ -732,53 +707,108 @@
arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this);
__ Blx(RegisterFrom(entrypoint_));
} else {
+ // Entrypoint is not already loaded, load from the thread.
int32_t entry_point_offset =
CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg.GetCode());
// This runtime call does not require a stack map.
arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
}
- __ B(GetExitLabel());
}
- private:
// The location (register) of the marked object reference.
const Location ref_;
// The location of the entrypoint if already loaded.
const Location entrypoint_;
- DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARMVIXL);
+ private:
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathBaseARMVIXL);
};
-// Slow path marking an object reference `ref` during a read barrier,
-// and if needed, atomically updating the field `obj.field` in the
-// object `obj` holding this reference after marking (contrary to
-// ReadBarrierMarkSlowPathARM above, which never tries to update
-// `obj.field`).
+// Slow path marking an object reference `ref` during a read
+// barrier. The field `obj.field` in the object `obj` holding this
+// reference does not get updated by this slow path after marking.
//
-// This means that after the execution of this slow path, both `ref`
-// and `obj.field` will be up-to-date; i.e., after the flip, both will
-// hold the same to-space reference (unless another thread installed
-// another object reference (different from `ref`) in `obj.field`).
-class ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL : public SlowPathCodeARMVIXL {
+// This means that after the execution of this slow path, `ref` will
+// always be up-to-date, but `obj.field` may not; i.e., after the
+// flip, `ref` will be a to-space reference, but `obj.field` will
+// probably still be a from-space reference (unless it gets updated by
+// another thread, or if another thread installed another object
+// reference (different from `ref`) in `obj.field`).
+//
+// If `entrypoint` is a valid location it is assumed to already be
+// holding the entrypoint. The case where the entrypoint is passed in
+// is when the decision to mark is based on whether the GC is marking.
+class ReadBarrierMarkSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL {
public:
- ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL(HInstruction* instruction,
- Location ref,
- vixl32::Register obj,
- Location field_offset,
- vixl32::Register temp1,
- vixl32::Register temp2)
- : SlowPathCodeARMVIXL(instruction),
- ref_(ref),
- obj_(obj),
- field_offset_(field_offset),
- temp1_(temp1),
- temp2_(temp2) {
+ ReadBarrierMarkSlowPathARMVIXL(HInstruction* instruction,
+ Location ref,
+ Location entrypoint = Location::NoLocation())
+ : ReadBarrierMarkSlowPathBaseARMVIXL(instruction, ref, entrypoint) {
DCHECK(kEmitCompilerReadBarrier);
}
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARMVIXL"; }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ DCHECK(locations->CanCall());
+ DCHECK(ref_.IsRegister()) << ref_;
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_.reg())) << ref_.reg();
+ DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString())
+ << "Unexpected instruction in read barrier marking slow path: "
+ << instruction_->DebugName();
+
+ __ Bind(GetEntryLabel());
+ GenerateReadBarrierMarkRuntimeCall(codegen);
+ __ B(GetExitLabel());
+ }
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARMVIXL);
+};
+
+// Slow path loading `obj`'s lock word, loading a reference from
+// object `*(obj + offset + (index << scale_factor))` into `ref`, and
+// marking `ref` if `obj` is gray according to the lock word (Baker
+// read barrier). The field `obj.field` in the object `obj` holding
+// this reference does not get updated by this slow path after marking
+// (see LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL
+// below for that).
+//
+// This means that after the execution of this slow path, `ref` will
+// always be up-to-date, but `obj.field` may not; i.e., after the
+// flip, `ref` will be a to-space reference, but `obj.field` will
+// probably still be a from-space reference (unless it gets updated by
+// another thread, or if another thread installed another object
+// reference (different from `ref`) in `obj.field`).
+//
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class LoadReferenceWithBakerReadBarrierSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL {
+ public:
+ LoadReferenceWithBakerReadBarrierSlowPathARMVIXL(HInstruction* instruction,
+ Location ref,
+ vixl32::Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check,
+ vixl32::Register temp,
+ Location entrypoint)
+ : ReadBarrierMarkSlowPathBaseARMVIXL(instruction, ref, entrypoint),
+ obj_(obj),
+ offset_(offset),
+ index_(index),
+ scale_factor_(scale_factor),
+ needs_null_check_(needs_null_check),
+ temp_(temp) {
+ DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
+ }
+
const char* GetDescription() const OVERRIDE {
- return "ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL";
+ return "LoadReferenceWithBakerReadBarrierSlowPathARMVIXL";
}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
@@ -786,64 +816,233 @@
vixl32::Register ref_reg = RegisterFrom(ref_);
DCHECK(locations->CanCall());
DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg.GetCode())) << ref_reg;
- // This slow path is only used by the UnsafeCASObject intrinsic.
+ DCHECK(instruction_->IsInstanceFieldGet() ||
+ instruction_->IsStaticFieldGet() ||
+ instruction_->IsArrayGet() ||
+ instruction_->IsArraySet() ||
+ instruction_->IsInstanceOf() ||
+ instruction_->IsCheckCast() ||
+ (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
+ (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
+ << "Unexpected instruction in read barrier marking slow path: "
+ << instruction_->DebugName();
+ // The read barrier instrumentation of object ArrayGet
+ // instructions does not support the HIntermediateAddress
+ // instruction.
+ DCHECK(!(instruction_->IsArrayGet() &&
+ instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
+
+ // Temporary register `temp_`, used to store the lock word, must
+ // not be IP, as we may use it to emit the reference load (in the
+ // call to GenerateRawReferenceLoad below), and we need the lock
+ // word to still be in `temp_` after the reference load.
+ DCHECK(!temp_.Is(ip));
+
+ __ Bind(GetEntryLabel());
+
+ // When using MaybeGenerateReadBarrierSlow, the read barrier call is
+ // inserted after the original load. However, in fast path based
+ // Baker's read barriers, we need to perform the load of
+ // mirror::Object::monitor_ *before* the original reference load.
+ // This load-load ordering is required by the read barrier.
+ // The fast path/slow path (for Baker's algorithm) should look like:
+ //
+ // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
+ // lfence; // Load fence or artificial data dependency to prevent load-load reordering
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
+ // if (is_gray) {
+ // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call.
+ // }
+ //
+ // Note: the original implementation in ReadBarrier::Barrier is
+ // slightly more complex as it performs additional checks that we do
+ // not do here for performance reasons.
+
+ CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
+
+ // /* int32_t */ monitor = obj->monitor_
+ uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+ arm_codegen->GetAssembler()->LoadFromOffset(kLoadWord, temp_, obj_, monitor_offset);
+ if (needs_null_check_) {
+ codegen->MaybeRecordImplicitNullCheck(instruction_);
+ }
+ // /* LockWord */ lock_word = LockWord(monitor)
+ static_assert(sizeof(LockWord) == sizeof(int32_t),
+ "art::LockWord and int32_t have different sizes.");
+
+ // Introduce a dependency on the lock_word including the rb_state,
+ // which shall prevent load-load reordering without using
+ // a memory barrier (which would be more expensive).
+ // `obj` is unchanged by this operation, but its value now depends
+ // on `temp`.
+ __ Add(obj_, obj_, Operand(temp_, ShiftType::LSR, 32));
+
+ // The actual reference load.
+ // A possible implicit null check has already been handled above.
+ arm_codegen->GenerateRawReferenceLoad(
+ instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false);
+
+ // Mark the object `ref` when `obj` is gray.
+ //
+ // if (rb_state == ReadBarrier::GrayState())
+ // ref = ReadBarrier::Mark(ref);
+ //
+ // Given the numeric representation, it's enough to check the low bit of the
+ // rb_state. We do that by shifting the bit out of the lock word with LSRS
+ // which can be a 16-bit instruction unlike the TST immediate.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Lsrs(temp_, temp_, LockWord::kReadBarrierStateShift + 1);
+ __ B(cc, GetExitLabel()); // Carry flag is the last bit shifted out by LSRS.
+ GenerateReadBarrierMarkRuntimeCall(codegen);
+
+ __ B(GetExitLabel());
+ }
+
+ private:
+ // The register containing the object holding the marked object reference field.
+ vixl32::Register obj_;
+ // The offset, index and scale factor to access the reference in `obj_`.
+ uint32_t offset_;
+ Location index_;
+ ScaleFactor scale_factor_;
+ // Is a null check required?
+ bool needs_null_check_;
+ // A temporary register used to hold the lock word of `obj_`.
+ vixl32::Register temp_;
+
+ DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierSlowPathARMVIXL);
+};
+
+// Slow path loading `obj`'s lock word, loading a reference from
+// object `*(obj + offset + (index << scale_factor))` into `ref`, and
+// marking `ref` if `obj` is gray according to the lock word (Baker
+// read barrier). If needed, this slow path also atomically updates
+// the field `obj.field` in the object `obj` holding this reference
+// after marking (contrary to
+// LoadReferenceWithBakerReadBarrierSlowPathARMVIXL above, which never
+// tries to update `obj.field`).
+//
+// This means that after the execution of this slow path, both `ref`
+// and `obj.field` will be up-to-date; i.e., after the flip, both will
+// hold the same to-space reference (unless another thread installed
+// another object reference (different from `ref`) in `obj.field`).
+//
+//
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL
+ : public ReadBarrierMarkSlowPathBaseARMVIXL {
+ public:
+ LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL(HInstruction* instruction,
+ Location ref,
+ vixl32::Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check,
+ vixl32::Register temp1,
+ vixl32::Register temp2,
+ Location entrypoint)
+ : ReadBarrierMarkSlowPathBaseARMVIXL(instruction, ref, entrypoint),
+ obj_(obj),
+ offset_(offset),
+ index_(index),
+ scale_factor_(scale_factor),
+ needs_null_check_(needs_null_check),
+ temp1_(temp1),
+ temp2_(temp2) {
+ DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
+ }
+
+ const char* GetDescription() const OVERRIDE {
+ return "LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL";
+ }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ vixl32::Register ref_reg = RegisterFrom(ref_);
+ DCHECK(locations->CanCall());
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg.GetCode())) << ref_reg;
+ DCHECK_NE(ref_.reg(), LocationFrom(temp1_).reg());
+
+ // This slow path is only used by the UnsafeCASObject intrinsic at the moment.
DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()))
<< "Unexpected instruction in read barrier marking and field updating slow path: "
<< instruction_->DebugName();
DCHECK(instruction_->GetLocations()->Intrinsified());
DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject);
- DCHECK(field_offset_.IsRegisterPair()) << field_offset_;
+ DCHECK_EQ(offset_, 0u);
+ DCHECK_EQ(scale_factor_, ScaleFactor::TIMES_1);
+ Location field_offset = index_;
+ DCHECK(field_offset.IsRegisterPair()) << field_offset;
+
+ // Temporary register `temp1_`, used to store the lock word, must
+ // not be IP, as we may use it to emit the reference load (in the
+ // call to GenerateRawReferenceLoad below), and we need the lock
+ // word to still be in `temp1_` after the reference load.
+ DCHECK(!temp1_.Is(ip));
__ Bind(GetEntryLabel());
- // Save the old reference.
+ CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
+
+ // /* int32_t */ monitor = obj->monitor_
+ uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+ arm_codegen->GetAssembler()->LoadFromOffset(kLoadWord, temp1_, obj_, monitor_offset);
+ if (needs_null_check_) {
+ codegen->MaybeRecordImplicitNullCheck(instruction_);
+ }
+ // /* LockWord */ lock_word = LockWord(monitor)
+ static_assert(sizeof(LockWord) == sizeof(int32_t),
+ "art::LockWord and int32_t have different sizes.");
+
+ // Introduce a dependency on the lock_word including the rb_state,
+ // which shall prevent load-load reordering without using
+ // a memory barrier (which would be more expensive).
+ // `obj` is unchanged by this operation, but its value now depends
+ // on `temp`.
+ __ Add(obj_, obj_, Operand(temp1_, ShiftType::LSR, 32));
+
+ // The actual reference load.
+ // A possible implicit null check has already been handled above.
+ arm_codegen->GenerateRawReferenceLoad(
+ instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false);
+
+ // Mark the object `ref` when `obj` is gray.
+ //
+ // if (rb_state == ReadBarrier::GrayState())
+ // ref = ReadBarrier::Mark(ref);
+ //
+ // Given the numeric representation, it's enough to check the low bit of the
+ // rb_state. We do that by shifting the bit out of the lock word with LSRS
+ // which can be a 16-bit instruction unlike the TST immediate.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Lsrs(temp1_, temp1_, LockWord::kReadBarrierStateShift + 1);
+ __ B(cc, GetExitLabel()); // Carry flag is the last bit shifted out by LSRS.
+
+ // Save the old value of the reference before marking it.
// Note that we cannot use IP to save the old reference, as IP is
// used internally by the ReadBarrierMarkRegX entry point, and we
// need the old reference after the call to that entry point.
DCHECK(!temp1_.Is(ip));
__ Mov(temp1_, ref_reg);
- // No need to save live registers; it's taken care of by the
- // entrypoint. Also, there is no need to update the stack mask,
- // as this runtime call will not trigger a garbage collection.
- CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
- DCHECK(!ref_reg.Is(sp));
- DCHECK(!ref_reg.Is(lr));
- DCHECK(!ref_reg.Is(pc));
- // IP is used internally by the ReadBarrierMarkRegX entry point
- // as a temporary, it cannot be the entry point's input/output.
- DCHECK(!ref_reg.Is(ip));
- DCHECK(ref_reg.IsRegister()) << ref_reg;
- // "Compact" slow path, saving two moves.
- //
- // Instead of using the standard runtime calling convention (input
- // and output in R0):
- //
- // R0 <- ref
- // R0 <- ReadBarrierMark(R0)
- // ref <- R0
- //
- // we just use rX (the register containing `ref`) as input and output
- // of a dedicated entrypoint:
- //
- // rX <- ReadBarrierMarkRegX(rX)
- //
- int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg.GetCode());
- // This runtime call does not require a stack map.
- arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ GenerateReadBarrierMarkRuntimeCall(codegen);
// If the new reference is different from the old reference,
- // update the field in the holder (`*(obj_ + field_offset_)`).
+ // update the field in the holder (`*(obj_ + field_offset)`).
//
// Note that this field could also hold a different object, if
// another thread had concurrently changed it. In that case, the
// LDREX/SUBS/ITNE sequence of instructions in the compare-and-set
// (CAS) operation below would abort the CAS, leaving the field
// as-is.
- vixl32::Label done;
__ Cmp(temp1_, ref_reg);
- __ B(eq, &done, /* far_target */ false);
+ __ B(eq, GetExitLabel());
// Update the the holder's field atomically. This may fail if
// mutator updates before us, but it's OK. This is achieved
@@ -857,7 +1056,7 @@
// The UnsafeCASObject intrinsic uses a register pair as field
// offset ("long offset"), of which only the low part contains
// data.
- vixl32::Register offset = LowRegisterFrom(field_offset_);
+ vixl32::Register offset = LowRegisterFrom(field_offset);
vixl32::Register expected = temp1_;
vixl32::Register value = ref_reg;
vixl32::Register tmp_ptr = temps.Acquire(); // Pointer to actual memory.
@@ -913,22 +1112,27 @@
}
}
- __ Bind(&done);
__ B(GetExitLabel());
}
private:
- // The location (register) of the marked object reference.
- const Location ref_;
// The register containing the object holding the marked object reference field.
const vixl32::Register obj_;
- // The location of the offset of the marked reference field within `obj_`.
- Location field_offset_;
-
+ // The offset, index and scale factor to access the reference in `obj_`.
+ uint32_t offset_;
+ Location index_;
+ ScaleFactor scale_factor_;
+ // Is a null check required?
+ bool needs_null_check_;
+ // A temporary register used to hold the lock word of `obj_`; and
+ // also to hold the original reference value, when the reference is
+ // marked.
const vixl32::Register temp1_;
+ // A temporary register used in the implementation of the CAS, to
+ // update the object's reference field.
const vixl32::Register temp2_;
- DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL);
+ DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL);
};
// Slow path generating a read barrier for a heap reference.
@@ -1450,8 +1654,317 @@
}
}
+static void GenerateVcmp(HInstruction* instruction, CodeGeneratorARMVIXL* codegen) {
+ const Location rhs_loc = instruction->GetLocations()->InAt(1);
+ if (rhs_loc.IsConstant()) {
+ // 0.0 is the only immediate that can be encoded directly in
+ // a VCMP instruction.
+ //
+ // Both the JLS (section 15.20.1) and the JVMS (section 6.5)
+ // specify that in a floating-point comparison, positive zero
+ // and negative zero are considered equal, so we can use the
+ // literal 0.0 for both cases here.
+ //
+ // Note however that some methods (Float.equal, Float.compare,
+ // Float.compareTo, Double.equal, Double.compare,
+ // Double.compareTo, Math.max, Math.min, StrictMath.max,
+ // StrictMath.min) consider 0.0 to be (strictly) greater than
+ // -0.0. So if we ever translate calls to these methods into a
+ // HCompare instruction, we must handle the -0.0 case with
+ // care here.
+ DCHECK(rhs_loc.GetConstant()->IsArithmeticZero());
+
+ const Primitive::Type type = instruction->InputAt(0)->GetType();
+
+ if (type == Primitive::kPrimFloat) {
+ __ Vcmp(F32, InputSRegisterAt(instruction, 0), 0.0);
+ } else {
+ DCHECK_EQ(type, Primitive::kPrimDouble);
+ __ Vcmp(F64, InputDRegisterAt(instruction, 0), 0.0);
+ }
+ } else {
+ __ Vcmp(InputVRegisterAt(instruction, 0), InputVRegisterAt(instruction, 1));
+ }
+}
+
+static vixl32::Condition GenerateLongTestConstant(HCondition* condition,
+ bool invert,
+ CodeGeneratorARMVIXL* codegen) {
+ DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong);
+
+ const LocationSummary* const locations = condition->GetLocations();
+ IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition();
+ vixl32::Condition ret = eq;
+ const Location left = locations->InAt(0);
+ const Location right = locations->InAt(1);
+
+ DCHECK(right.IsConstant());
+
+ const vixl32::Register left_high = HighRegisterFrom(left);
+ const vixl32::Register left_low = LowRegisterFrom(left);
+ int64_t value = Int64ConstantFrom(right);
+
+ switch (cond) {
+ case kCondEQ:
+ case kCondNE:
+ case kCondB:
+ case kCondBE:
+ case kCondA:
+ case kCondAE: {
+ __ Cmp(left_high, High32Bits(value));
+
+ ExactAssemblyScope guard(codegen->GetVIXLAssembler(),
+ 2 * vixl32::k16BitT32InstructionSizeInBytes,
+ CodeBufferCheckScope::kExactSize);
+
+ __ it(eq);
+ __ cmp(eq, left_low, Low32Bits(value));
+ ret = ARMUnsignedCondition(cond);
+ break;
+ }
+ case kCondLE:
+ case kCondGT:
+ // Trivially true or false.
+ if (value == std::numeric_limits<int64_t>::max()) {
+ __ Cmp(left_low, left_low);
+ ret = cond == kCondLE ? eq : ne;
+ break;
+ }
+
+ if (cond == kCondLE) {
+ cond = kCondLT;
+ } else {
+ DCHECK_EQ(cond, kCondGT);
+ cond = kCondGE;
+ }
+
+ value++;
+ FALLTHROUGH_INTENDED;
+ case kCondGE:
+ case kCondLT: {
+ UseScratchRegisterScope temps(codegen->GetVIXLAssembler());
+
+ __ Cmp(left_low, Low32Bits(value));
+ __ Sbcs(temps.Acquire(), left_high, High32Bits(value));
+ ret = ARMCondition(cond);
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unreachable";
+ UNREACHABLE();
+ }
+
+ return ret;
+}
+
+static vixl32::Condition GenerateLongTest(HCondition* condition,
+ bool invert,
+ CodeGeneratorARMVIXL* codegen) {
+ DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong);
+
+ const LocationSummary* const locations = condition->GetLocations();
+ IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition();
+ vixl32::Condition ret = eq;
+ Location left = locations->InAt(0);
+ Location right = locations->InAt(1);
+
+ DCHECK(right.IsRegisterPair());
+
+ switch (cond) {
+ case kCondEQ:
+ case kCondNE:
+ case kCondB:
+ case kCondBE:
+ case kCondA:
+ case kCondAE: {
+ __ Cmp(HighRegisterFrom(left), HighRegisterFrom(right));
+
+ ExactAssemblyScope guard(codegen->GetVIXLAssembler(),
+ 2 * vixl32::k16BitT32InstructionSizeInBytes,
+ CodeBufferCheckScope::kExactSize);
+
+ __ it(eq);
+ __ cmp(eq, LowRegisterFrom(left), LowRegisterFrom(right));
+ ret = ARMUnsignedCondition(cond);
+ break;
+ }
+ case kCondLE:
+ case kCondGT:
+ if (cond == kCondLE) {
+ cond = kCondGE;
+ } else {
+ DCHECK_EQ(cond, kCondGT);
+ cond = kCondLT;
+ }
+
+ std::swap(left, right);
+ FALLTHROUGH_INTENDED;
+ case kCondGE:
+ case kCondLT: {
+ UseScratchRegisterScope temps(codegen->GetVIXLAssembler());
+
+ __ Cmp(LowRegisterFrom(left), LowRegisterFrom(right));
+ __ Sbcs(temps.Acquire(), HighRegisterFrom(left), HighRegisterFrom(right));
+ ret = ARMCondition(cond);
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unreachable";
+ UNREACHABLE();
+ }
+
+ return ret;
+}
+
+static vixl32::Condition GenerateTest(HInstruction* instruction,
+ Location loc,
+ bool invert,
+ CodeGeneratorARMVIXL* codegen) {
+ DCHECK(!instruction->IsConstant());
+
+ vixl32::Condition ret = invert ? eq : ne;
+
+ if (IsBooleanValueOrMaterializedCondition(instruction)) {
+ __ Cmp(RegisterFrom(loc), 0);
+ } else {
+ HCondition* const condition = instruction->AsCondition();
+ const Primitive::Type type = condition->GetLeft()->GetType();
+ const IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition();
+
+ if (type == Primitive::kPrimLong) {
+ ret = condition->GetLocations()->InAt(1).IsConstant()
+ ? GenerateLongTestConstant(condition, invert, codegen)
+ : GenerateLongTest(condition, invert, codegen);
+ } else if (Primitive::IsFloatingPointType(type)) {
+ GenerateVcmp(condition, codegen);
+ __ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR);
+ ret = ARMFPCondition(cond, condition->IsGtBias());
+ } else {
+ DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type;
+ __ Cmp(InputRegisterAt(condition, 0), InputOperandAt(condition, 1));
+ ret = ARMCondition(cond);
+ }
+ }
+
+ return ret;
+}
+
+static bool CanGenerateTest(HInstruction* condition, ArmVIXLAssembler* assembler) {
+ if (!IsBooleanValueOrMaterializedCondition(condition)) {
+ const HCondition* const cond = condition->AsCondition();
+
+ if (cond->GetLeft()->GetType() == Primitive::kPrimLong) {
+ const LocationSummary* const locations = cond->GetLocations();
+ const IfCondition c = cond->GetCondition();
+
+ if (locations->InAt(1).IsConstant()) {
+ const int64_t value = Int64ConstantFrom(locations->InAt(1));
+
+ if (c < kCondLT || c > kCondGE) {
+ // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8,
+ // we check that the least significant half of the first input to be compared
+ // is in a low register (the other half is read outside an IT block), and
+ // the constant fits in an 8-bit unsigned integer, so that a 16-bit CMP
+ // encoding can be used.
+ if (!LowRegisterFrom(locations->InAt(0)).IsLow() || !IsUint<8>(Low32Bits(value))) {
+ return false;
+ }
+ // TODO(VIXL): The rest of the checks are there to keep the backend in sync with
+ // the previous one, but are not strictly necessary.
+ } else if (c == kCondLE || c == kCondGT) {
+ if (value < std::numeric_limits<int64_t>::max() &&
+ !assembler->ShifterOperandCanHold(SBC, High32Bits(value + 1), kCcSet)) {
+ return false;
+ }
+ } else if (!assembler->ShifterOperandCanHold(SBC, High32Bits(value), kCcSet)) {
+ return false;
+ }
+ }
+ }
+ }
+
+ return true;
+}
+
+static bool CanEncodeConstantAs8BitImmediate(HConstant* constant) {
+ const Primitive::Type type = constant->GetType();
+ bool ret = false;
+
+ DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type;
+
+ if (type == Primitive::kPrimLong) {
+ const uint64_t value = Uint64ConstantFrom(constant);
+
+ ret = IsUint<8>(Low32Bits(value)) && IsUint<8>(High32Bits(value));
+ } else {
+ ret = IsUint<8>(Int32ConstantFrom(constant));
+ }
+
+ return ret;
+}
+
+static Location Arm8BitEncodableConstantOrRegister(HInstruction* constant) {
+ DCHECK(!Primitive::IsFloatingPointType(constant->GetType()));
+
+ if (constant->IsConstant() && CanEncodeConstantAs8BitImmediate(constant->AsConstant())) {
+ return Location::ConstantLocation(constant->AsConstant());
+ }
+
+ return Location::RequiresRegister();
+}
+
+static bool CanGenerateConditionalMove(const Location& out, const Location& src) {
+ // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8,
+ // we check that we are not dealing with floating-point output (there is no
+ // 16-bit VMOV encoding).
+ if (!out.IsRegister() && !out.IsRegisterPair()) {
+ return false;
+ }
+
+ // For constants, we also check that the output is in one or two low registers,
+ // and that the constants fit in an 8-bit unsigned integer, so that a 16-bit
+ // MOV encoding can be used.
+ if (src.IsConstant()) {
+ if (!CanEncodeConstantAs8BitImmediate(src.GetConstant())) {
+ return false;
+ }
+
+ if (out.IsRegister()) {
+ if (!RegisterFrom(out).IsLow()) {
+ return false;
+ }
+ } else {
+ DCHECK(out.IsRegisterPair());
+
+ if (!HighRegisterFrom(out).IsLow()) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
#undef __
+vixl32::Label* CodeGeneratorARMVIXL::GetFinalLabel(HInstruction* instruction,
+ vixl32::Label* final_label) {
+ DCHECK(!instruction->IsControlFlow() && !instruction->IsSuspendCheck());
+
+ const HBasicBlock* const block = instruction->GetBlock();
+ const HLoopInformation* const info = block->GetLoopInformation();
+ HInstruction* const next = instruction->GetNext();
+
+ // Avoid a branch to a branch.
+ if (next->IsGoto() && (info == nullptr ||
+ !info->IsBackEdge(*block) ||
+ !info->HasSuspendCheck())) {
+ final_label = GetLabelOf(next->AsGoto()->GetSuccessor());
+ }
+
+ return final_label;
+}
+
CodeGeneratorARMVIXL::CodeGeneratorARMVIXL(HGraph* graph,
const ArmInstructionSetFeatures& isa_features,
const CompilerOptions& compiler_options,
@@ -1481,8 +1994,6 @@
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- boot_image_address_patches_(std::less<uint32_t>(),
- graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_string_patches_(StringReferenceValueComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_class_patches_(TypeReferenceValueComparator(),
@@ -1945,43 +2456,6 @@
void InstructionCodeGeneratorARMVIXL::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
}
-void InstructionCodeGeneratorARMVIXL::GenerateVcmp(HInstruction* instruction) {
- Primitive::Type type = instruction->InputAt(0)->GetType();
- Location lhs_loc = instruction->GetLocations()->InAt(0);
- Location rhs_loc = instruction->GetLocations()->InAt(1);
- if (rhs_loc.IsConstant()) {
- // 0.0 is the only immediate that can be encoded directly in
- // a VCMP instruction.
- //
- // Both the JLS (section 15.20.1) and the JVMS (section 6.5)
- // specify that in a floating-point comparison, positive zero
- // and negative zero are considered equal, so we can use the
- // literal 0.0 for both cases here.
- //
- // Note however that some methods (Float.equal, Float.compare,
- // Float.compareTo, Double.equal, Double.compare,
- // Double.compareTo, Math.max, Math.min, StrictMath.max,
- // StrictMath.min) consider 0.0 to be (strictly) greater than
- // -0.0. So if we ever translate calls to these methods into a
- // HCompare instruction, we must handle the -0.0 case with
- // care here.
- DCHECK(rhs_loc.GetConstant()->IsArithmeticZero());
- if (type == Primitive::kPrimFloat) {
- __ Vcmp(F32, InputSRegisterAt(instruction, 0), 0.0);
- } else {
- DCHECK_EQ(type, Primitive::kPrimDouble);
- __ Vcmp(F64, DRegisterFrom(lhs_loc), 0.0);
- }
- } else {
- if (type == Primitive::kPrimFloat) {
- __ Vcmp(InputSRegisterAt(instruction, 0), InputSRegisterAt(instruction, 1));
- } else {
- DCHECK_EQ(type, Primitive::kPrimDouble);
- __ Vcmp(DRegisterFrom(lhs_loc), DRegisterFrom(rhs_loc));
- }
- }
-}
-
void InstructionCodeGeneratorARMVIXL::GenerateFPJumps(HCondition* cond,
vixl32::Label* true_label,
vixl32::Label* false_label ATTRIBUTE_UNUSED) {
@@ -2090,7 +2564,7 @@
break;
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
- GenerateVcmp(condition);
+ GenerateVcmp(condition, codegen_);
GenerateFPJumps(condition, true_target, false_target);
break;
default:
@@ -2167,20 +2641,29 @@
return;
}
- LocationSummary* locations = cond->GetLocations();
- DCHECK(locations->InAt(0).IsRegister());
- vixl32::Register left = InputRegisterAt(cond, 0);
- Location right = locations->InAt(1);
- if (right.IsRegister()) {
- __ Cmp(left, InputRegisterAt(cond, 1));
- } else {
- DCHECK(right.IsConstant());
- __ Cmp(left, CodeGenerator::GetInt32ValueOf(right.GetConstant()));
- }
+ vixl32::Label* non_fallthrough_target;
+ vixl32::Condition arm_cond = vixl32::Condition::None();
+ const vixl32::Register left = InputRegisterAt(cond, 0);
+ const Operand right = InputOperandAt(cond, 1);
+
if (true_target == nullptr) {
- __ B(ARMCondition(condition->GetOppositeCondition()), false_target);
+ arm_cond = ARMCondition(condition->GetOppositeCondition());
+ non_fallthrough_target = false_target;
} else {
- __ B(ARMCondition(condition->GetCondition()), true_target);
+ arm_cond = ARMCondition(condition->GetCondition());
+ non_fallthrough_target = true_target;
+ }
+
+ if (right.IsImmediate() && right.GetImmediate() == 0 && (arm_cond.Is(ne) || arm_cond.Is(eq))) {
+ if (arm_cond.Is(eq)) {
+ __ CompareAndBranchIfZero(left, non_fallthrough_target);
+ } else {
+ DCHECK(arm_cond.Is(ne));
+ __ CompareAndBranchIfNonZero(left, non_fallthrough_target);
+ }
+ } else {
+ __ Cmp(left, right);
+ __ B(arm_cond, non_fallthrough_target);
}
}
@@ -2241,29 +2724,135 @@
void LocationsBuilderARMVIXL::VisitSelect(HSelect* select) {
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select);
- if (Primitive::IsFloatingPointType(select->GetType())) {
+ const bool is_floating_point = Primitive::IsFloatingPointType(select->GetType());
+
+ if (is_floating_point) {
locations->SetInAt(0, Location::RequiresFpuRegister());
- locations->SetInAt(1, Location::RequiresFpuRegister());
+ locations->SetInAt(1, Location::FpuRegisterOrConstant(select->GetTrueValue()));
} else {
locations->SetInAt(0, Location::RequiresRegister());
- locations->SetInAt(1, Location::RequiresRegister());
+ locations->SetInAt(1, Arm8BitEncodableConstantOrRegister(select->GetTrueValue()));
}
+
if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) {
- locations->SetInAt(2, Location::RequiresRegister());
+ locations->SetInAt(2, Location::RegisterOrConstant(select->GetCondition()));
+ // The code generator handles overlap with the values, but not with the condition.
+ locations->SetOut(Location::SameAsFirstInput());
+ } else if (is_floating_point) {
+ locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
+ } else {
+ if (!locations->InAt(1).IsConstant()) {
+ locations->SetInAt(0, Arm8BitEncodableConstantOrRegister(select->GetFalseValue()));
+ }
+
+ locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}
- locations->SetOut(Location::SameAsFirstInput());
}
void InstructionCodeGeneratorARMVIXL::VisitSelect(HSelect* select) {
- LocationSummary* locations = select->GetLocations();
- vixl32::Label false_target;
- GenerateTestAndBranch(select,
- /* condition_input_index */ 2,
- /* true_target */ nullptr,
- &false_target,
- /* far_target */ false);
- codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType());
- __ Bind(&false_target);
+ HInstruction* const condition = select->GetCondition();
+ const LocationSummary* const locations = select->GetLocations();
+ const Primitive::Type type = select->GetType();
+ const Location first = locations->InAt(0);
+ const Location out = locations->Out();
+ const Location second = locations->InAt(1);
+ Location src;
+
+ if (condition->IsIntConstant()) {
+ if (condition->AsIntConstant()->IsFalse()) {
+ src = first;
+ } else {
+ src = second;
+ }
+
+ codegen_->MoveLocation(out, src, type);
+ return;
+ }
+
+ if (!Primitive::IsFloatingPointType(type) &&
+ CanGenerateTest(condition, codegen_->GetAssembler())) {
+ bool invert = false;
+
+ if (out.Equals(second)) {
+ src = first;
+ invert = true;
+ } else if (out.Equals(first)) {
+ src = second;
+ } else if (second.IsConstant()) {
+ DCHECK(CanEncodeConstantAs8BitImmediate(second.GetConstant()));
+ src = second;
+ } else if (first.IsConstant()) {
+ DCHECK(CanEncodeConstantAs8BitImmediate(first.GetConstant()));
+ src = first;
+ invert = true;
+ } else {
+ src = second;
+ }
+
+ if (CanGenerateConditionalMove(out, src)) {
+ if (!out.Equals(first) && !out.Equals(second)) {
+ codegen_->MoveLocation(out, src.Equals(first) ? second : first, type);
+ }
+
+ const vixl32::Condition cond = GenerateTest(condition, locations->InAt(2), invert, codegen_);
+ const size_t instr_count = out.IsRegisterPair() ? 4 : 2;
+ ExactAssemblyScope guard(GetVIXLAssembler(),
+ instr_count * vixl32::k16BitT32InstructionSizeInBytes,
+ CodeBufferCheckScope::kExactSize);
+
+ if (out.IsRegister()) {
+ __ it(cond);
+ __ mov(cond, RegisterFrom(out), OperandFrom(src, type));
+ } else {
+ DCHECK(out.IsRegisterPair());
+
+ Operand operand_high(0);
+ Operand operand_low(0);
+
+ if (src.IsConstant()) {
+ const int64_t value = Int64ConstantFrom(src);
+
+ operand_high = High32Bits(value);
+ operand_low = Low32Bits(value);
+ } else {
+ DCHECK(src.IsRegisterPair());
+ operand_high = HighRegisterFrom(src);
+ operand_low = LowRegisterFrom(src);
+ }
+
+ __ it(cond);
+ __ mov(cond, LowRegisterFrom(out), operand_low);
+ __ it(cond);
+ __ mov(cond, HighRegisterFrom(out), operand_high);
+ }
+
+ return;
+ }
+ }
+
+ vixl32::Label* false_target = nullptr;
+ vixl32::Label* true_target = nullptr;
+ vixl32::Label select_end;
+ vixl32::Label* const target = codegen_->GetFinalLabel(select, &select_end);
+
+ if (out.Equals(second)) {
+ true_target = target;
+ src = first;
+ } else {
+ false_target = target;
+ src = second;
+
+ if (!out.Equals(first)) {
+ codegen_->MoveLocation(out, first, type);
+ }
+ }
+
+ GenerateTestAndBranch(select, 2, true_target, false_target, /* far_target */ false);
+ codegen_->MoveLocation(out, src, type);
+
+ if (select_end.IsReferenced()) {
+ __ Bind(&select_end);
+ }
}
void LocationsBuilderARMVIXL::VisitNativeDebugInfo(HNativeDebugInfo* info) {
@@ -2341,7 +2930,7 @@
break;
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
- GenerateVcmp(cond);
+ GenerateVcmp(cond, codegen_);
GenerateFPJumps(cond, &true_label, &false_label);
break;
}
@@ -4356,7 +4945,7 @@
case Primitive::kPrimFloat:
case Primitive::kPrimDouble: {
__ Mov(out, 0);
- GenerateVcmp(compare);
+ GenerateVcmp(compare, codegen_);
// To branch on the FP compare result we transfer FPSCR to APSR (encoded as PC in VMRS).
__ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR);
less_cond = ARMFPCondition(kCondLT, compare->IsGtBias());
@@ -4726,17 +5315,24 @@
return true;
}
Opcode neg_opcode = kNoOperand;
+ uint32_t neg_value = 0;
switch (opcode) {
- case AND: neg_opcode = BIC; value = ~value; break;
- case ORR: neg_opcode = ORN; value = ~value; break;
- case ADD: neg_opcode = SUB; value = -value; break;
- case ADC: neg_opcode = SBC; value = ~value; break;
- case SUB: neg_opcode = ADD; value = -value; break;
- case SBC: neg_opcode = ADC; value = ~value; break;
+ case AND: neg_opcode = BIC; neg_value = ~value; break;
+ case ORR: neg_opcode = ORN; neg_value = ~value; break;
+ case ADD: neg_opcode = SUB; neg_value = -value; break;
+ case ADC: neg_opcode = SBC; neg_value = ~value; break;
+ case SUB: neg_opcode = ADD; neg_value = -value; break;
+ case SBC: neg_opcode = ADC; neg_value = ~value; break;
+ case MOV: neg_opcode = MVN; neg_value = ~value; break;
default:
return false;
}
- return assembler->ShifterOperandCanHold(neg_opcode, value, set_cc);
+
+ if (assembler->ShifterOperandCanHold(neg_opcode, neg_value, set_cc)) {
+ return true;
+ }
+
+ return opcode == AND && IsPowerOfTwo(value + 1);
}
void InstructionCodeGeneratorARMVIXL::HandleFieldGet(HInstruction* instruction,
@@ -5674,20 +6270,56 @@
caller_saves.Add(LocationFrom(calling_convention.GetRegisterAt(0)));
caller_saves.Add(LocationFrom(calling_convention.GetRegisterAt(1)));
LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves);
- locations->SetInAt(0, Location::RequiresRegister());
- locations->SetInAt(1, Location::RequiresRegister());
+
+ HInstruction* index = instruction->InputAt(0);
+ HInstruction* length = instruction->InputAt(1);
+ // If both index and length are constants we can statically check the bounds. But if at least one
+ // of them is not encodable ArmEncodableConstantOrRegister will create
+ // Location::RequiresRegister() which is not desired to happen. Instead we create constant
+ // locations.
+ bool both_const = index->IsConstant() && length->IsConstant();
+ locations->SetInAt(0, both_const
+ ? Location::ConstantLocation(index->AsConstant())
+ : ArmEncodableConstantOrRegister(index, CMP));
+ locations->SetInAt(1, both_const
+ ? Location::ConstantLocation(length->AsConstant())
+ : ArmEncodableConstantOrRegister(length, CMP));
}
void InstructionCodeGeneratorARMVIXL::VisitBoundsCheck(HBoundsCheck* instruction) {
- SlowPathCodeARMVIXL* slow_path =
- new (GetGraph()->GetArena()) BoundsCheckSlowPathARMVIXL(instruction);
- codegen_->AddSlowPath(slow_path);
+ LocationSummary* locations = instruction->GetLocations();
+ Location index_loc = locations->InAt(0);
+ Location length_loc = locations->InAt(1);
- vixl32::Register index = InputRegisterAt(instruction, 0);
- vixl32::Register length = InputRegisterAt(instruction, 1);
+ if (length_loc.IsConstant()) {
+ int32_t length = Int32ConstantFrom(length_loc);
+ if (index_loc.IsConstant()) {
+ // BCE will remove the bounds check if we are guaranteed to pass.
+ int32_t index = Int32ConstantFrom(index_loc);
+ if (index < 0 || index >= length) {
+ SlowPathCodeARMVIXL* slow_path =
+ new (GetGraph()->GetArena()) BoundsCheckSlowPathARMVIXL(instruction);
+ codegen_->AddSlowPath(slow_path);
+ __ B(slow_path->GetEntryLabel());
+ } else {
+ // Some optimization after BCE may have generated this, and we should not
+ // generate a bounds check if it is a valid range.
+ }
+ return;
+ }
- __ Cmp(index, length);
- __ B(hs, slow_path->GetEntryLabel());
+ SlowPathCodeARMVIXL* slow_path =
+ new (GetGraph()->GetArena()) BoundsCheckSlowPathARMVIXL(instruction);
+ __ Cmp(RegisterFrom(index_loc), length);
+ codegen_->AddSlowPath(slow_path);
+ __ B(hs, slow_path->GetEntryLabel());
+ } else {
+ SlowPathCodeARMVIXL* slow_path =
+ new (GetGraph()->GetArena()) BoundsCheckSlowPathARMVIXL(instruction);
+ __ Cmp(RegisterFrom(length_loc), InputOperandAt(instruction, 0));
+ codegen_->AddSlowPath(slow_path);
+ __ B(ls, slow_path->GetEntryLabel());
+ }
}
void CodeGeneratorARMVIXL::MarkGCCard(vixl32::Register temp,
@@ -7041,10 +7673,12 @@
return;
}
if (GetAssembler()->ShifterOperandCanHold(AND, value)) {
- __ And(out, first, value);
+ __ And(out, first, value);
+ } else if (GetAssembler()->ShifterOperandCanHold(BIC, ~value)) {
+ __ Bic(out, first, ~value);
} else {
- DCHECK(GetAssembler()->ShifterOperandCanHold(BIC, ~value));
- __ Bic(out, first, ~value);
+ DCHECK(IsPowerOfTwo(value + 1));
+ __ Ubfx(out, first, 0, WhichPowerOf2(value + 1));
}
}
@@ -7263,14 +7897,35 @@
DCHECK(kEmitCompilerReadBarrier);
if (kUseBakerReadBarrier) {
// Fast path implementation of art::ReadBarrier::BarrierForRoot when
- // Baker's read barrier are used:
+ // Baker's read barrier are used.
//
- // root = obj.field;
+ // Note that we do not actually check the value of
+ // `GetIsGcMarking()` to decide whether to mark the loaded GC
+ // root or not. Instead, we load into `temp` the read barrier
+ // mark entry point corresponding to register `root`. If `temp`
+ // is null, it means that `GetIsGcMarking()` is false, and vice
+ // versa.
+ //
// temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
- // if (temp != null) {
- // root = temp(root)
+ // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load.
+ // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking()
+ // // Slow path.
+ // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call.
// }
+ // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`.
+ Location temp = LocationFrom(lr);
+ SlowPathCodeARMVIXL* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARMVIXL(
+ instruction, root, /* entrypoint */ temp);
+ codegen_->AddSlowPath(slow_path);
+
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp), tr, entry_point_offset);
+
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
GetAssembler()->LoadFromOffset(kLoadWord, root_reg, obj, offset);
static_assert(
@@ -7281,21 +7936,6 @@
"art::mirror::CompressedReference<mirror::Object> and int32_t "
"have different sizes.");
- // Slow path marking the GC root `root`.
- Location temp = LocationFrom(lr);
- SlowPathCodeARMVIXL* slow_path =
- new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARMVIXL(
- instruction,
- root,
- /*entrypoint*/ temp);
- codegen_->AddSlowPath(slow_path);
-
- // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
- const int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg());
- // Loading the entrypoint does not require a load acquire since it is only changed when
- // threads are suspended or running a checkpoint.
- GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp), tr, entry_point_offset);
// The entrypoint is null when the GC is not marking, this prevents one load compared to
// checking GetIsGcMarking.
__ CompareAndBranchIfNonZero(RegisterFrom(temp), slow_path->GetEntryLabel());
@@ -7366,55 +8006,114 @@
DCHECK(kEmitCompilerReadBarrier);
DCHECK(kUseBakerReadBarrier);
- // In slow path based read barriers, the read barrier call is
- // inserted after the original load. However, in fast path based
- // Baker's read barriers, we need to perform the load of
- // mirror::Object::monitor_ *before* the original reference load.
- // This load-load ordering is required by the read barrier.
- // The fast path/slow path (for Baker's algorithm) should look like:
+ // Query `art::Thread::Current()->GetIsGcMarking()` to decide
+ // whether we need to enter the slow path to mark the reference.
+ // Then, in the slow path, check the gray bit in the lock word of
+ // the reference's holder (`obj`) to decide whether to mark `ref` or
+ // not.
//
- // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
- // lfence; // Load fence or artificial data dependency to prevent load-load reordering
- // HeapReference<Object> ref = *src; // Original reference load.
- // bool is_gray = (rb_state == ReadBarrier::GrayState());
- // if (is_gray) {
- // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path.
+ // Note that we do not actually check the value of `GetIsGcMarking()`;
+ // instead, we load into `temp3` the read barrier mark entry point
+ // corresponding to register `ref`. If `temp3` is null, it means
+ // that `GetIsGcMarking()` is false, and vice versa.
+ //
+ // temp3 = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ // if (temp3 != nullptr) { // <=> Thread::Current()->GetIsGcMarking()
+ // // Slow path.
+ // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
+ // lfence; // Load fence or artificial data dependency to prevent load-load reordering
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
+ // if (is_gray) {
+ // ref = temp3(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call.
+ // }
+ // } else {
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
// }
- //
- // Note: the original implementation in ReadBarrier::Barrier is
- // slightly more complex as it performs additional checks that we do
- // not do here for performance reasons.
- vixl32::Register ref_reg = RegisterFrom(ref);
vixl32::Register temp_reg = RegisterFrom(temp);
- uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
- // /* int32_t */ monitor = obj->monitor_
- GetAssembler()->LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset);
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
+ // Slow path marking the object `ref` when the GC is marking. The
+ // entrypoint will already be loaded in `temp3`.
+ Location temp3 = LocationFrom(lr);
+ SlowPathCodeARMVIXL* slow_path;
+ if (always_update_field) {
+ DCHECK(temp2 != nullptr);
+ // LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL
+ // only supports address of the form `obj + field_offset`, where
+ // `obj` is a register and `field_offset` is a register pair (of
+ // which only the lower half is used). Thus `offset` and
+ // `scale_factor` above are expected to be null in this code path.
+ DCHECK_EQ(offset, 0u);
+ DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1);
+ Location field_offset = index;
+ slow_path =
+ new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL(
+ instruction,
+ ref,
+ obj,
+ offset,
+ /* index */ field_offset,
+ scale_factor,
+ needs_null_check,
+ temp_reg,
+ *temp2,
+ /* entrypoint */ temp3);
+ } else {
+ slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARMVIXL(
+ instruction,
+ ref,
+ obj,
+ offset,
+ index,
+ scale_factor,
+ needs_null_check,
+ temp_reg,
+ /* entrypoint */ temp3);
}
- // /* LockWord */ lock_word = LockWord(monitor)
- static_assert(sizeof(LockWord) == sizeof(int32_t),
- "art::LockWord and int32_t have different sizes.");
+ AddSlowPath(slow_path);
- // Introduce a dependency on the lock_word including the rb_state,
- // which shall prevent load-load reordering without using
- // a memory barrier (which would be more expensive).
- // `obj` is unchanged by this operation, but its value now depends
- // on `temp_reg`.
- __ Add(obj, obj, Operand(temp_reg, ShiftType::LSR, 32));
+ // temp3 = Thread::Current()->pReadBarrierMarkReg ## ref.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp3), tr, entry_point_offset);
+ // The entrypoint is null when the GC is not marking, this prevents one load compared to
+ // checking GetIsGcMarking.
+ __ CompareAndBranchIfNonZero(RegisterFrom(temp3), slow_path->GetEntryLabel());
+ // Fast path: just load the reference.
+ GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check);
+ __ Bind(slow_path->GetExitLabel());
+}
- // The actual reference load.
+void CodeGeneratorARMVIXL::GenerateRawReferenceLoad(HInstruction* instruction,
+ Location ref,
+ vixl::aarch32::Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check) {
+ Primitive::Type type = Primitive::kPrimNot;
+ vixl32::Register ref_reg = RegisterFrom(ref, type);
+
+ // If needed, vixl::EmissionCheckScope guards are used to ensure
+ // that no pools are emitted between the load (macro) instruction
+ // and MaybeRecordImplicitNullCheck.
+
if (index.IsValid()) {
// Load types involving an "index": ArrayGet,
// UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject
// intrinsics.
- // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor))
+ // /* HeapReference<mirror::Object> */ ref = *(obj + offset + (index << scale_factor))
if (index.IsConstant()) {
size_t computed_offset =
(Int32ConstantFrom(index) << scale_factor) + offset;
+ vixl::EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
} else {
// Handle the special case of the
// UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject
@@ -7424,46 +8123,27 @@
? LowRegisterFrom(index)
: RegisterFrom(index);
UseScratchRegisterScope temps(GetVIXLAssembler());
- const vixl32::Register temp3 = temps.Acquire();
- __ Add(temp3, obj, Operand(index_reg, ShiftType::LSL, scale_factor));
- GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, temp3, offset);
+ vixl32::Register temp = temps.Acquire();
+ __ Add(temp, obj, Operand(index_reg, ShiftType::LSL, scale_factor));
+ {
+ vixl::EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, temp, offset);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
}
} else {
- // /* HeapReference<Object> */ ref = *(obj + offset)
+ // /* HeapReference<mirror::Object> */ ref = *(obj + offset)
+ vixl::EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, obj, offset);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
}
// Object* ref = ref_addr->AsMirrorPtr()
GetAssembler()->MaybeUnpoisonHeapReference(ref_reg);
-
- // Slow path marking the object `ref` when it is gray.
- SlowPathCodeARMVIXL* slow_path;
- if (always_update_field) {
- DCHECK(temp2 != nullptr);
- // ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL only supports address
- // of the form `obj + field_offset`, where `obj` is a register and
- // `field_offset` is a register pair (of which only the lower half
- // is used). Thus `offset` and `scale_factor` above are expected
- // to be null in this code path.
- DCHECK_EQ(offset, 0u);
- DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1);
- slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL(
- instruction, ref, obj, /* field_offset */ index, temp_reg, *temp2);
- } else {
- slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARMVIXL(instruction, ref);
- }
- AddSlowPath(slow_path);
-
- // if (rb_state == ReadBarrier::GrayState())
- // ref = ReadBarrier::Mark(ref);
- // Given the numeric representation, it's enough to check the low bit of the
- // rb_state. We do that by shifting the bit out of the lock word with LSRS
- // which can be a 16-bit instruction unlike the TST immediate.
- static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
- __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1);
- __ B(cs, slow_path->GetEntryLabel()); // Carry flag is the last bit shifted out by LSRS.
- __ Bind(slow_path->GetExitLabel());
}
void CodeGeneratorARMVIXL::GenerateReadBarrierSlow(HInstruction* instruction,
@@ -7738,9 +8418,7 @@
}
VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateBootImageAddressLiteral(uint32_t address) {
- bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
- Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
- return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_);
}
VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateDexCacheAddressLiteral(uint32_t address) {
@@ -7800,8 +8478,7 @@
/* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() +
boot_image_type_patches_.size() +
/* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() +
- /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size() +
- boot_image_address_patches_.size();
+ /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size();
linker_patches->reserve(size);
EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_,
linker_patches);
@@ -7835,13 +8512,6 @@
target_type.dex_file,
target_type.type_index.index_));
}
- for (const auto& entry : boot_image_address_patches_) {
- DCHECK(GetCompilerOptions().GetIncludePatchInformation());
- VIXLUInt32Literal* literal = entry.second;
- DCHECK(literal->IsBound());
- uint32_t literal_offset = literal->GetLocation();
- linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
- }
DCHECK_EQ(size, linker_patches->size());
}
diff --git a/compiler/optimizing/code_generator_arm_vixl.h b/compiler/optimizing/code_generator_arm_vixl.h
index 3f52c72..781027a 100644
--- a/compiler/optimizing/code_generator_arm_vixl.h
+++ b/compiler/optimizing/code_generator_arm_vixl.h
@@ -45,6 +45,11 @@
namespace art {
namespace arm {
+// This constant is used as an approximate margin when emission of veneer and literal pools
+// must be blocked.
+static constexpr int kMaxMacroInstructionSizeInBytes =
+ 15 * vixl::aarch32::kMaxInstructionSizeInBytes;
+
static const vixl::aarch32::Register kParameterCoreRegistersVIXL[] = {
vixl::aarch32::r1,
vixl::aarch32::r2,
@@ -396,7 +401,6 @@
void GenerateCompareTestAndBranch(HCondition* condition,
vixl::aarch32::Label* true_target,
vixl::aarch32::Label* false_target);
- void GenerateVcmp(HInstruction* instruction);
void GenerateFPJumps(HCondition* cond,
vixl::aarch32::Label* true_label,
vixl::aarch32::Label* false_label);
@@ -505,6 +509,8 @@
return &(block_labels_[block->GetBlockId()]);
}
+ vixl32::Label* GetFinalLabel(HInstruction* instruction, vixl32::Label* final_label);
+
void Initialize() OVERRIDE {
block_labels_.resize(GetGraph()->GetBlocks().size());
}
@@ -625,6 +631,15 @@
bool always_update_field = false,
vixl::aarch32::Register* temp2 = nullptr);
+ // Generate a heap reference load (with no read barrier).
+ void GenerateRawReferenceLoad(HInstruction* instruction,
+ Location ref,
+ vixl::aarch32::Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check);
+
// Generate a read barrier for a heap reference within `instruction`
// using a slow path.
//
@@ -738,8 +753,6 @@
ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_;
// PC-relative type patch info for kBssEntry.
ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
- // Deduplication map for patchable boot image addresses.
- Uint32ToLiteralMap boot_image_address_patches_;
// Patches for string literals in JIT compiled code.
StringToLiteralMap jit_string_patches_;
diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc
index 56f009b..5f02a52 100644
--- a/compiler/optimizing/code_generator_mips.cc
+++ b/compiler/optimizing/code_generator_mips.cc
@@ -491,8 +491,6 @@
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- boot_image_address_patches_(std::less<uint32_t>(),
- graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_class_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
clobbered_ra_(false) {
@@ -1035,8 +1033,7 @@
pc_relative_type_patches_.size() +
type_bss_entry_patches_.size() +
boot_image_string_patches_.size() +
- boot_image_type_patches_.size() +
- boot_image_address_patches_.size();
+ boot_image_type_patches_.size();
linker_patches->reserve(size);
EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_,
linker_patches);
@@ -1070,13 +1067,6 @@
target_type.dex_file,
target_type.type_index.index_));
}
- for (const auto& entry : boot_image_address_patches_) {
- DCHECK(GetCompilerOptions().GetIncludePatchInformation());
- Literal* literal = entry.second;
- DCHECK(literal->GetLabel()->IsBound());
- uint32_t literal_offset = __ GetLabelLocation(literal->GetLabel());
- linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
- }
DCHECK_EQ(size, linker_patches->size());
}
@@ -1134,9 +1124,7 @@
}
Literal* CodeGeneratorMIPS::DeduplicateBootImageAddressLiteral(uint32_t address) {
- bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
- Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
- return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_);
}
void CodeGeneratorMIPS::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info,
diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h
index 0ccd80a..98fee24 100644
--- a/compiler/optimizing/code_generator_mips.h
+++ b/compiler/optimizing/code_generator_mips.h
@@ -535,8 +535,6 @@
ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_;
// PC-relative type patch info for kBssEntry.
ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
- // Deduplication map for patchable boot image addresses.
- Uint32ToLiteralMap boot_image_address_patches_;
// Patches for string root accesses in JIT compiled code.
ArenaDeque<JitPatchInfo> jit_string_patches_;
// Patches for class root accesses in JIT compiled code.
diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc
index f53ee2c..02c3ad6 100644
--- a/compiler/optimizing/code_generator_mips64.cc
+++ b/compiler/optimizing/code_generator_mips64.cc
@@ -439,8 +439,6 @@
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- boot_image_address_patches_(std::less<uint32_t>(),
- graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_string_patches_(StringReferenceValueComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_class_patches_(TypeReferenceValueComparator(),
@@ -946,8 +944,7 @@
pc_relative_type_patches_.size() +
type_bss_entry_patches_.size() +
boot_image_string_patches_.size() +
- boot_image_type_patches_.size() +
- boot_image_address_patches_.size();
+ boot_image_type_patches_.size();
linker_patches->reserve(size);
EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_,
linker_patches);
@@ -981,13 +978,6 @@
target_type.dex_file,
target_type.type_index.index_));
}
- for (const auto& entry : boot_image_address_patches_) {
- DCHECK(GetCompilerOptions().GetIncludePatchInformation());
- Literal* literal = entry.second;
- DCHECK(literal->GetLabel()->IsBound());
- uint32_t literal_offset = __ GetLabelLocation(literal->GetLabel());
- linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
- }
DCHECK_EQ(size, linker_patches->size());
}
@@ -1051,9 +1041,7 @@
}
Literal* CodeGeneratorMIPS64::DeduplicateBootImageAddressLiteral(uint64_t address) {
- bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
- Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
- return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_);
}
void CodeGeneratorMIPS64::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info,
diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h
index 26cc7dc..3056f7f 100644
--- a/compiler/optimizing/code_generator_mips64.h
+++ b/compiler/optimizing/code_generator_mips64.h
@@ -492,8 +492,6 @@
ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_;
// PC-relative type patch info for kBssEntry.
ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
- // Deduplication map for patchable boot image addresses.
- Uint32ToLiteralMap boot_image_address_patches_;
// Patches for string root accesses in JIT compiled code.
StringToLiteralMap jit_string_patches_;
// Patches for class root accesses in JIT compiled code.
diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc
index b779aed..0b50619 100644
--- a/compiler/optimizing/code_generator_x86.cc
+++ b/compiler/optimizing/code_generator_x86.cc
@@ -1015,7 +1015,6 @@
assembler_(graph->GetArena()),
isa_features_(isa_features),
pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
boot_image_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
@@ -4603,13 +4602,6 @@
temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86PointerSize).Int32Value()));
}
-void CodeGeneratorX86::RecordSimplePatch() {
- if (GetCompilerOptions().GetIncludePatchInformation()) {
- simple_patches_.emplace_back();
- __ Bind(&simple_patches_.back());
- }
-}
-
void CodeGeneratorX86::RecordBootStringPatch(HLoadString* load_string) {
DCHECK(GetCompilerOptions().IsBootImage());
HX86ComputeBaseMethodAddress* address = nullptr;
@@ -4682,17 +4674,12 @@
DCHECK(linker_patches->empty());
size_t size =
pc_relative_dex_cache_patches_.size() +
- simple_patches_.size() +
string_patches_.size() +
boot_image_type_patches_.size() +
type_bss_entry_patches_.size();
linker_patches->reserve(size);
EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_,
linker_patches);
- for (const Label& label : simple_patches_) {
- uint32_t literal_offset = label.Position() - kLabelPositionToLiteralOffsetAdjustment;
- linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
- }
if (!GetCompilerOptions().IsBootImage()) {
DCHECK(boot_image_type_patches_.empty());
EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(string_patches_, linker_patches);
@@ -6154,7 +6141,6 @@
reinterpret_cast<uintptr_t>(cls->GetClass().Get()));
DCHECK_NE(address, 0u);
__ movl(out, Immediate(address));
- codegen_->RecordSimplePatch();
break;
}
case HLoadClass::LoadKind::kBssEntry: {
@@ -6311,7 +6297,6 @@
reinterpret_cast<uintptr_t>(load->GetString().Get()));
DCHECK_NE(address, 0u);
__ movl(out, Immediate(address));
- codegen_->RecordSimplePatch();
return; // No dex cache slow path.
}
case HLoadString::LoadKind::kBssEntry: {
diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h
index 5360dc9..65ee383 100644
--- a/compiler/optimizing/code_generator_x86.h
+++ b/compiler/optimizing/code_generator_x86.h
@@ -412,7 +412,6 @@
// Generate a call to a virtual method.
void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE;
- void RecordSimplePatch();
void RecordBootStringPatch(HLoadString* load_string);
void RecordBootTypePatch(HLoadClass* load_class);
Label* NewTypeBssEntryPatch(HLoadClass* load_class);
@@ -633,8 +632,6 @@
// PC-relative DexCache access info.
ArenaDeque<X86PcRelativePatchInfo> pc_relative_dex_cache_patches_;
- // Patch locations for patchoat where the linker doesn't do any other work.
- ArenaDeque<Label> simple_patches_;
// String patch locations; type depends on configuration (app .bss or boot image PIC/non-PIC).
ArenaDeque<X86PcRelativePatchInfo> string_patches_;
// Type patch locations for boot image; type depends on configuration (boot image PIC/non-PIC).
diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc
index 179bf6d..644fcee 100644
--- a/compiler/optimizing/code_generator_x86_64.cc
+++ b/compiler/optimizing/code_generator_x86_64.cc
@@ -1070,13 +1070,6 @@
kX86_64PointerSize).SizeValue()));
}
-void CodeGeneratorX86_64::RecordSimplePatch() {
- if (GetCompilerOptions().GetIncludePatchInformation()) {
- simple_patches_.emplace_back();
- __ Bind(&simple_patches_.back());
- }
-}
-
void CodeGeneratorX86_64::RecordBootStringPatch(HLoadString* load_string) {
DCHECK(GetCompilerOptions().IsBootImage());
string_patches_.emplace_back(load_string->GetDexFile(), load_string->GetStringIndex().index_);
@@ -1126,17 +1119,12 @@
DCHECK(linker_patches->empty());
size_t size =
pc_relative_dex_cache_patches_.size() +
- simple_patches_.size() +
string_patches_.size() +
boot_image_type_patches_.size() +
type_bss_entry_patches_.size();
linker_patches->reserve(size);
EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_,
linker_patches);
- for (const Label& label : simple_patches_) {
- uint32_t literal_offset = label.Position() - kLabelPositionToLiteralOffsetAdjustment;
- linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
- }
if (!GetCompilerOptions().IsBootImage()) {
DCHECK(boot_image_type_patches_.empty());
EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(string_patches_, linker_patches);
@@ -1227,7 +1215,6 @@
isa_features_(isa_features),
constant_area_start_(0),
pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
boot_image_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
@@ -5545,7 +5532,6 @@
reinterpret_cast<uintptr_t>(cls->GetClass().Get()));
DCHECK_NE(address, 0u);
__ movl(out, Immediate(address)); // Zero-extended.
- codegen_->RecordSimplePatch();
break;
}
case HLoadClass::LoadKind::kBssEntry: {
@@ -5681,7 +5667,6 @@
reinterpret_cast<uintptr_t>(load->GetString().Get()));
DCHECK_NE(address, 0u);
__ movl(out, Immediate(address)); // Zero-extended.
- codegen_->RecordSimplePatch();
return; // No dex cache slow path.
}
case HLoadString::LoadKind::kBssEntry: {
diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h
index 3a83731..376c3ce 100644
--- a/compiler/optimizing/code_generator_x86_64.h
+++ b/compiler/optimizing/code_generator_x86_64.h
@@ -406,7 +406,6 @@
void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE;
void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE;
- void RecordSimplePatch();
void RecordBootStringPatch(HLoadString* load_string);
void RecordBootTypePatch(HLoadClass* load_class);
Label* NewTypeBssEntryPatch(HLoadClass* load_class);
@@ -602,8 +601,6 @@
// PC-relative DexCache access info.
ArenaDeque<PatchInfo<Label>> pc_relative_dex_cache_patches_;
- // Patch locations for patchoat where the linker doesn't do any other work.
- ArenaDeque<Label> simple_patches_;
// String patch locations; type depends on configuration (app .bss or boot image PIC).
ArenaDeque<PatchInfo<Label>> string_patches_;
// Type patch locations for boot image (always PIC).
diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc
index e3926c5..583008b 100644
--- a/compiler/optimizing/inliner.cc
+++ b/compiler/optimizing/inliner.cc
@@ -249,20 +249,25 @@
ProfilingInfo* const profiling_info_;
};
-static bool IsMonomorphic(Handle<mirror::ObjectArray<mirror::Class>> classes)
- REQUIRES_SHARED(Locks::mutator_lock_) {
- DCHECK_GE(InlineCache::kIndividualCacheSize, 2);
- return classes->Get(0) != nullptr && classes->Get(1) == nullptr;
-}
-
-static bool IsMegamorphic(Handle<mirror::ObjectArray<mirror::Class>> classes)
- REQUIRES_SHARED(Locks::mutator_lock_) {
- for (size_t i = 0; i < InlineCache::kIndividualCacheSize; ++i) {
- if (classes->Get(i) == nullptr) {
- return false;
+HInliner::InlineCacheType HInliner::GetInlineCacheType(
+ const Handle<mirror::ObjectArray<mirror::Class>>& classes)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ uint8_t number_of_types = 0;
+ for (; number_of_types < InlineCache::kIndividualCacheSize; ++number_of_types) {
+ if (classes->Get(number_of_types) == nullptr) {
+ break;
}
}
- return true;
+
+ if (number_of_types == 0) {
+ return kInlineCacheUninitialized;
+ } else if (number_of_types == 1) {
+ return kInlineCacheMonomorphic;
+ } else if (number_of_types == InlineCache::kIndividualCacheSize) {
+ return kInlineCacheMegamorphic;
+ } else {
+ return kInlineCachePolymorphic;
+ }
}
static mirror::Class* GetMonomorphicType(Handle<mirror::ObjectArray<mirror::Class>> classes)
@@ -271,18 +276,6 @@
return classes->Get(0);
}
-static bool IsUninitialized(Handle<mirror::ObjectArray<mirror::Class>> classes)
- REQUIRES_SHARED(Locks::mutator_lock_) {
- return classes->Get(0) == nullptr;
-}
-
-static bool IsPolymorphic(Handle<mirror::ObjectArray<mirror::Class>> classes)
- REQUIRES_SHARED(Locks::mutator_lock_) {
- DCHECK_GE(InlineCache::kIndividualCacheSize, 3);
- return classes->Get(1) != nullptr &&
- classes->Get(InlineCache::kIndividualCacheSize - 1) == nullptr;
-}
-
ArtMethod* HInliner::TryCHADevirtualization(ArtMethod* resolved_method) {
if (!resolved_method->HasSingleImplementation()) {
return nullptr;
@@ -353,67 +346,209 @@
}
return result;
}
-
DCHECK(!invoke_instruction->IsInvokeStaticOrDirect());
- // Check if we can use an inline cache.
- ArtMethod* caller = graph_->GetArtMethod();
- if (Runtime::Current()->UseJitCompilation()) {
- // Under JIT, we should always know the caller.
- DCHECK(caller != nullptr);
- ScopedProfilingInfoInlineUse spiis(caller, soa.Self());
- ProfilingInfo* profiling_info = spiis.GetProfilingInfo();
- if (profiling_info != nullptr) {
- StackHandleScope<1> hs(soa.Self());
- ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker();
- Handle<mirror::ObjectArray<mirror::Class>> inline_cache = hs.NewHandle(
- mirror::ObjectArray<mirror::Class>::Alloc(
- soa.Self(),
- class_linker->GetClassRoot(ClassLinker::kClassArrayClass),
- InlineCache::kIndividualCacheSize));
- if (inline_cache == nullptr) {
- // We got an OOME. Just clear the exception, and don't inline.
- DCHECK(soa.Self()->IsExceptionPending());
- soa.Self()->ClearException();
- VLOG(compiler) << "Out of memory in the compiler when trying to inline";
- return false;
+ // Try using inline caches.
+ return TryInlineFromInlineCache(caller_dex_file, invoke_instruction, resolved_method);
+}
+
+static Handle<mirror::ObjectArray<mirror::Class>> AllocateInlineCacheHolder(
+ const DexCompilationUnit& compilation_unit,
+ StackHandleScope<1>* hs)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ Thread* self = Thread::Current();
+ ClassLinker* class_linker = compilation_unit.GetClassLinker();
+ Handle<mirror::ObjectArray<mirror::Class>> inline_cache = hs->NewHandle(
+ mirror::ObjectArray<mirror::Class>::Alloc(
+ self,
+ class_linker->GetClassRoot(ClassLinker::kClassArrayClass),
+ InlineCache::kIndividualCacheSize));
+ if (inline_cache == nullptr) {
+ // We got an OOME. Just clear the exception, and don't inline.
+ DCHECK(self->IsExceptionPending());
+ self->ClearException();
+ VLOG(compiler) << "Out of memory in the compiler when trying to inline";
+ }
+ return inline_cache;
+}
+
+bool HInliner::TryInlineFromInlineCache(const DexFile& caller_dex_file,
+ HInvoke* invoke_instruction,
+ ArtMethod* resolved_method)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ StackHandleScope<1> hs(Thread::Current());
+ Handle<mirror::ObjectArray<mirror::Class>> inline_cache;
+ InlineCacheType inline_cache_type = Runtime::Current()->IsAotCompiler()
+ ? GetInlineCacheAOT(caller_dex_file, invoke_instruction, &hs, &inline_cache)
+ : GetInlineCacheJIT(invoke_instruction, &hs, &inline_cache);
+
+ switch (inline_cache_type) {
+ case kInlineCacheNoData:
+ break;
+
+ case kInlineCacheUninitialized:
+ VLOG(compiler) << "Interface or virtual call to "
+ << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex())
+ << " is not hit and not inlined";
+ return false;
+
+ case kInlineCacheMonomorphic:
+ MaybeRecordStat(kMonomorphicCall);
+ if (outermost_graph_->IsCompilingOsr()) {
+ // If we are compiling OSR, we pretend this call is polymorphic, as we may come from the
+ // interpreter and it may have seen different receiver types.
+ return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache);
} else {
- Runtime::Current()->GetJit()->GetCodeCache()->CopyInlineCacheInto(
- *profiling_info->GetInlineCache(invoke_instruction->GetDexPc()),
- inline_cache);
- if (IsUninitialized(inline_cache)) {
- VLOG(compiler) << "Interface or virtual call to "
- << caller_dex_file.PrettyMethod(method_index)
- << " is not hit and not inlined";
- return false;
- } else if (IsMonomorphic(inline_cache)) {
- MaybeRecordStat(kMonomorphicCall);
- if (outermost_graph_->IsCompilingOsr()) {
- // If we are compiling OSR, we pretend this call is polymorphic, as we may come from the
- // interpreter and it may have seen different receiver types.
- return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache);
- } else {
- return TryInlineMonomorphicCall(invoke_instruction, resolved_method, inline_cache);
- }
- } else if (IsPolymorphic(inline_cache)) {
- MaybeRecordStat(kPolymorphicCall);
- return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache);
- } else {
- DCHECK(IsMegamorphic(inline_cache));
- VLOG(compiler) << "Interface or virtual call to "
- << caller_dex_file.PrettyMethod(method_index)
- << " is megamorphic and not inlined";
- MaybeRecordStat(kMegamorphicCall);
- return false;
- }
+ return TryInlineMonomorphicCall(invoke_instruction, resolved_method, inline_cache);
}
+
+ case kInlineCachePolymorphic:
+ MaybeRecordStat(kPolymorphicCall);
+ return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache);
+
+ case kInlineCacheMegamorphic:
+ VLOG(compiler) << "Interface or virtual call to "
+ << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex())
+ << " is megamorphic and not inlined";
+ MaybeRecordStat(kMegamorphicCall);
+ return false;
+
+ case kInlineCacheMissingTypes:
+ VLOG(compiler) << "Interface or virtual call to "
+ << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex())
+ << " is missing types and not inlined";
+ return false;
+ }
+ UNREACHABLE();
+}
+
+HInliner::InlineCacheType HInliner::GetInlineCacheJIT(
+ HInvoke* invoke_instruction,
+ StackHandleScope<1>* hs,
+ /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ DCHECK(Runtime::Current()->UseJitCompilation());
+
+ ArtMethod* caller = graph_->GetArtMethod();
+ // Under JIT, we should always know the caller.
+ DCHECK(caller != nullptr);
+ ScopedProfilingInfoInlineUse spiis(caller, Thread::Current());
+ ProfilingInfo* profiling_info = spiis.GetProfilingInfo();
+
+ if (profiling_info == nullptr) {
+ return kInlineCacheNoData;
+ }
+
+ *inline_cache = AllocateInlineCacheHolder(caller_compilation_unit_, hs);
+ if (inline_cache->Get() == nullptr) {
+ // We can't extract any data if we failed to allocate;
+ return kInlineCacheNoData;
+ } else {
+ Runtime::Current()->GetJit()->GetCodeCache()->CopyInlineCacheInto(
+ *profiling_info->GetInlineCache(invoke_instruction->GetDexPc()),
+ *inline_cache);
+ return GetInlineCacheType(*inline_cache);
+ }
+}
+
+HInliner::InlineCacheType HInliner::GetInlineCacheAOT(
+ const DexFile& caller_dex_file,
+ HInvoke* invoke_instruction,
+ StackHandleScope<1>* hs,
+ /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ DCHECK(Runtime::Current()->IsAotCompiler());
+ const ProfileCompilationInfo* pci = compiler_driver_->GetProfileCompilationInfo();
+ if (pci == nullptr) {
+ return kInlineCacheNoData;
+ }
+
+ ProfileCompilationInfo::OfflineProfileMethodInfo offline_profile;
+ bool found = pci->GetMethod(caller_dex_file.GetLocation(),
+ caller_dex_file.GetLocationChecksum(),
+ caller_compilation_unit_.GetDexMethodIndex(),
+ &offline_profile);
+ if (!found) {
+ return kInlineCacheNoData; // no profile information for this invocation.
+ }
+
+ *inline_cache = AllocateInlineCacheHolder(caller_compilation_unit_, hs);
+ if (inline_cache == nullptr) {
+ // We can't extract any data if we failed to allocate;
+ return kInlineCacheNoData;
+ } else {
+ return ExtractClassesFromOfflineProfile(invoke_instruction,
+ offline_profile,
+ *inline_cache);
+ }
+}
+
+HInliner::InlineCacheType HInliner::ExtractClassesFromOfflineProfile(
+ const HInvoke* invoke_instruction,
+ const ProfileCompilationInfo::OfflineProfileMethodInfo& offline_profile,
+ /*out*/Handle<mirror::ObjectArray<mirror::Class>> inline_cache)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ const auto it = offline_profile.inline_caches.find(invoke_instruction->GetDexPc());
+ if (it == offline_profile.inline_caches.end()) {
+ return kInlineCacheUninitialized;
+ }
+
+ const ProfileCompilationInfo::DexPcData& dex_pc_data = it->second;
+
+ if (dex_pc_data.is_missing_types) {
+ return kInlineCacheMissingTypes;
+ }
+ if (dex_pc_data.is_megamorphic) {
+ return kInlineCacheMegamorphic;
+ }
+
+ DCHECK_LE(dex_pc_data.classes.size(), InlineCache::kIndividualCacheSize);
+ Thread* self = Thread::Current();
+ // We need to resolve the class relative to the containing dex file.
+ // So first, build a mapping from the index of dex file in the profile to
+ // its dex cache. This will avoid repeating the lookup when walking over
+ // the inline cache types.
+ std::vector<ObjPtr<mirror::DexCache>> dex_profile_index_to_dex_cache(
+ offline_profile.dex_references.size());
+ for (size_t i = 0; i < offline_profile.dex_references.size(); i++) {
+ bool found = false;
+ for (const DexFile* dex_file : compiler_driver_->GetDexFilesForOatFile()) {
+ if (offline_profile.dex_references[i].MatchesDex(dex_file)) {
+ dex_profile_index_to_dex_cache[i] =
+ caller_compilation_unit_.GetClassLinker()->FindDexCache(self, *dex_file);
+ found = true;
+ }
+ }
+ if (!found) {
+ VLOG(compiler) << "Could not find profiled dex file: "
+ << offline_profile.dex_references[i].dex_location;
+ return kInlineCacheMissingTypes;
}
}
- VLOG(compiler) << "Interface or virtual call to "
- << caller_dex_file.PrettyMethod(method_index)
- << " could not be statically determined";
- return false;
+ // Walk over the classes and resolve them. If we cannot find a type we return
+ // kInlineCacheMissingTypes.
+ int ic_index = 0;
+ for (const ProfileCompilationInfo::ClassReference& class_ref : dex_pc_data.classes) {
+ ObjPtr<mirror::DexCache> dex_cache =
+ dex_profile_index_to_dex_cache[class_ref.dex_profile_index];
+ DCHECK(dex_cache != nullptr);
+ ObjPtr<mirror::Class> clazz = ClassLinker::LookupResolvedType(
+ class_ref.type_index,
+ dex_cache,
+ caller_compilation_unit_.GetClassLoader().Get());
+ if (clazz != nullptr) {
+ inline_cache->Set(ic_index++, clazz);
+ } else {
+ VLOG(compiler) << "Could not resolve class from inline cache in AOT mode "
+ << caller_compilation_unit_.GetDexFile()->PrettyMethod(
+ invoke_instruction->GetDexMethodIndex()) << " : "
+ << caller_compilation_unit_
+ .GetDexFile()->StringByTypeIdx(class_ref.type_index);
+ return kInlineCacheMissingTypes;
+ }
+ }
+ return GetInlineCacheType(inline_cache);
}
HInstanceFieldGet* HInliner::BuildGetReceiverClass(ClassLinker* class_linker,
@@ -556,6 +691,13 @@
// Insert before setting the kind, as setting the kind affects the inputs.
bb_cursor->InsertInstructionAfter(load_class, receiver_class);
load_class->SetLoadKind(kind);
+ // In AOT mode, we will most likely load the class from BSS, which will involve a call
+ // to the runtime. In this case, the load instruction will need an environment so copy
+ // it from the invoke instruction.
+ if (load_class->NeedsEnvironment()) {
+ DCHECK(Runtime::Current()->IsAotCompiler());
+ load_class->CopyEnvironmentFrom(invoke_instruction->GetEnvironment());
+ }
HNotEqual* compare = new (graph_->GetArena()) HNotEqual(load_class, receiver_class);
bb_cursor->InsertInstructionAfter(compare, load_class);
@@ -746,7 +888,10 @@
ArtMethod* resolved_method,
Handle<mirror::ObjectArray<mirror::Class>> classes) {
// This optimization only works under JIT for now.
- DCHECK(Runtime::Current()->UseJitCompilation());
+ if (!Runtime::Current()->UseJitCompilation()) {
+ return false;
+ }
+
if (graph_->GetInstructionSet() == kMips64) {
// TODO: Support HClassTableGet for mips64.
return false;
@@ -1064,9 +1209,8 @@
// TODO: Needs null check.
return false;
}
- Handle<mirror::DexCache> dex_cache(handles_->NewHandle(resolved_method->GetDexCache()));
HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, data.object_arg);
- HInstanceFieldGet* iget = CreateInstanceFieldGet(dex_cache, data.field_idx, obj);
+ HInstanceFieldGet* iget = CreateInstanceFieldGet(data.field_idx, resolved_method, obj);
DCHECK_EQ(iget->GetFieldOffset().Uint32Value(), data.field_offset);
DCHECK_EQ(iget->IsVolatile() ? 1u : 0u, data.is_volatile);
invoke_instruction->GetBlock()->InsertInstructionBefore(iget, invoke_instruction);
@@ -1079,10 +1223,9 @@
// TODO: Needs null check.
return false;
}
- Handle<mirror::DexCache> dex_cache(handles_->NewHandle(resolved_method->GetDexCache()));
HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, data.object_arg);
HInstruction* value = GetInvokeInputForArgVRegIndex(invoke_instruction, data.src_arg);
- HInstanceFieldSet* iput = CreateInstanceFieldSet(dex_cache, data.field_idx, obj, value);
+ HInstanceFieldSet* iput = CreateInstanceFieldSet(data.field_idx, resolved_method, obj, value);
DCHECK_EQ(iput->GetFieldOffset().Uint32Value(), data.field_offset);
DCHECK_EQ(iput->IsVolatile() ? 1u : 0u, data.is_volatile);
invoke_instruction->GetBlock()->InsertInstructionBefore(iput, invoke_instruction);
@@ -1116,24 +1259,19 @@
[](uint16_t index) { return index != DexFile::kDexNoIndex16; }));
// Create HInstanceFieldSet for each IPUT that stores non-zero data.
- Handle<mirror::DexCache> dex_cache;
HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, /* this */ 0u);
bool needs_constructor_barrier = false;
for (size_t i = 0; i != number_of_iputs; ++i) {
HInstruction* value = GetInvokeInputForArgVRegIndex(invoke_instruction, iput_args[i]);
if (!value->IsConstant() || !value->AsConstant()->IsZeroBitPattern()) {
- if (dex_cache.GetReference() == nullptr) {
- dex_cache = handles_->NewHandle(resolved_method->GetDexCache());
- }
uint16_t field_index = iput_field_indexes[i];
- HInstanceFieldSet* iput = CreateInstanceFieldSet(dex_cache, field_index, obj, value);
+ bool is_final;
+ HInstanceFieldSet* iput =
+ CreateInstanceFieldSet(field_index, resolved_method, obj, value, &is_final);
invoke_instruction->GetBlock()->InsertInstructionBefore(iput, invoke_instruction);
// Check whether the field is final. If it is, we need to add a barrier.
- PointerSize pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet());
- ArtField* resolved_field = dex_cache->GetResolvedField(field_index, pointer_size);
- DCHECK(resolved_field != nullptr);
- if (resolved_field->IsFinal()) {
+ if (is_final) {
needs_constructor_barrier = true;
}
}
@@ -1152,12 +1290,13 @@
return true;
}
-HInstanceFieldGet* HInliner::CreateInstanceFieldGet(Handle<mirror::DexCache> dex_cache,
- uint32_t field_index,
+HInstanceFieldGet* HInliner::CreateInstanceFieldGet(uint32_t field_index,
+ ArtMethod* referrer,
HInstruction* obj)
REQUIRES_SHARED(Locks::mutator_lock_) {
- PointerSize pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet());
- ArtField* resolved_field = dex_cache->GetResolvedField(field_index, pointer_size);
+ ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
+ ArtField* resolved_field =
+ class_linker->LookupResolvedField(field_index, referrer, /* is_static */ false);
DCHECK(resolved_field != nullptr);
HInstanceFieldGet* iget = new (graph_->GetArena()) HInstanceFieldGet(
obj,
@@ -1167,12 +1306,13 @@
resolved_field->IsVolatile(),
field_index,
resolved_field->GetDeclaringClass()->GetDexClassDefIndex(),
- *dex_cache->GetDexFile(),
+ *referrer->GetDexFile(),
// Read barrier generates a runtime call in slow path and we need a valid
// dex pc for the associated stack map. 0 is bogus but valid. Bug: 26854537.
/* dex_pc */ 0);
if (iget->GetType() == Primitive::kPrimNot) {
// Use the same dex_cache that we used for field lookup as the hint_dex_cache.
+ Handle<mirror::DexCache> dex_cache = handles_->NewHandle(referrer->GetDexCache());
ReferenceTypePropagation rtp(graph_,
outer_compilation_unit_.GetClassLoader(),
dex_cache,
@@ -1183,14 +1323,21 @@
return iget;
}
-HInstanceFieldSet* HInliner::CreateInstanceFieldSet(Handle<mirror::DexCache> dex_cache,
- uint32_t field_index,
+HInstanceFieldSet* HInliner::CreateInstanceFieldSet(uint32_t field_index,
+ ArtMethod* referrer,
HInstruction* obj,
- HInstruction* value)
+ HInstruction* value,
+ bool* is_final)
REQUIRES_SHARED(Locks::mutator_lock_) {
- PointerSize pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet());
- ArtField* resolved_field = dex_cache->GetResolvedField(field_index, pointer_size);
+ ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
+ ArtField* resolved_field =
+ class_linker->LookupResolvedField(field_index, referrer, /* is_static */ false);
DCHECK(resolved_field != nullptr);
+ if (is_final != nullptr) {
+ // This information is needed only for constructors.
+ DCHECK(referrer->IsConstructor());
+ *is_final = resolved_field->IsFinal();
+ }
HInstanceFieldSet* iput = new (graph_->GetArena()) HInstanceFieldSet(
obj,
value,
@@ -1200,7 +1347,7 @@
resolved_field->IsVolatile(),
field_index,
resolved_field->GetDeclaringClass()->GetDexClassDefIndex(),
- *dex_cache->GetDexFile(),
+ *referrer->GetDexFile(),
// Read barrier generates a runtime call in slow path and we need a valid
// dex pc for the associated stack map. 0 is bogus but valid. Bug: 26854537.
/* dex_pc */ 0);
@@ -1380,6 +1527,13 @@
<< " could not be inlined because one branch always throws and"
<< " caller does not have an exit block";
return false;
+ } else if (graph_->HasIrreducibleLoops()) {
+ // TODO(ngeoffray): Support re-computing loop information to graphs with
+ // irreducible loops?
+ VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index)
+ << " could not be inlined because one branch always throws and"
+ << " caller has irreducible loops";
+ return false;
}
} else {
has_one_return = true;
diff --git a/compiler/optimizing/inliner.h b/compiler/optimizing/inliner.h
index 75d025a..a032042 100644
--- a/compiler/optimizing/inliner.h
+++ b/compiler/optimizing/inliner.h
@@ -20,6 +20,7 @@
#include "dex_file_types.h"
#include "invoke_type.h"
#include "optimization.h"
+#include "jit/profile_compilation_info.h"
namespace art {
@@ -59,6 +60,15 @@
static constexpr const char* kInlinerPassName = "inliner";
private:
+ enum InlineCacheType {
+ kInlineCacheNoData = 0,
+ kInlineCacheUninitialized = 1,
+ kInlineCacheMonomorphic = 2,
+ kInlineCachePolymorphic = 3,
+ kInlineCacheMegamorphic = 4,
+ kInlineCacheMissingTypes = 5
+ };
+
bool TryInline(HInvoke* invoke_instruction);
// Try to inline `resolved_method` in place of `invoke_instruction`. `do_rtp` is whether
@@ -97,14 +107,54 @@
REQUIRES_SHARED(Locks::mutator_lock_);
// Create a new HInstanceFieldGet.
- HInstanceFieldGet* CreateInstanceFieldGet(Handle<mirror::DexCache> dex_cache,
- uint32_t field_index,
+ HInstanceFieldGet* CreateInstanceFieldGet(uint32_t field_index,
+ ArtMethod* referrer,
HInstruction* obj);
// Create a new HInstanceFieldSet.
- HInstanceFieldSet* CreateInstanceFieldSet(Handle<mirror::DexCache> dex_cache,
- uint32_t field_index,
+ HInstanceFieldSet* CreateInstanceFieldSet(uint32_t field_index,
+ ArtMethod* referrer,
HInstruction* obj,
- HInstruction* value);
+ HInstruction* value,
+ bool* is_final = nullptr);
+
+ // Try inlining the invoke instruction using inline caches.
+ bool TryInlineFromInlineCache(
+ const DexFile& caller_dex_file,
+ HInvoke* invoke_instruction,
+ ArtMethod* resolved_method)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
+ // Try getting the inline cache from JIT code cache.
+ // Return true if the inline cache was successfully allocated and the
+ // invoke info was found in the profile info.
+ InlineCacheType GetInlineCacheJIT(
+ HInvoke* invoke_instruction,
+ StackHandleScope<1>* hs,
+ /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
+ // Try getting the inline cache from AOT offline profile.
+ // Return true if the inline cache was successfully allocated and the
+ // invoke info was found in the profile info.
+ InlineCacheType GetInlineCacheAOT(const DexFile& caller_dex_file,
+ HInvoke* invoke_instruction,
+ StackHandleScope<1>* hs,
+ /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
+ // Extract the mirror classes from the offline profile and add them to the `inline_cache`.
+ // Note that even if we have profile data for the invoke the inline_cache might contain
+ // only null entries if the types cannot be resolved.
+ InlineCacheType ExtractClassesFromOfflineProfile(
+ const HInvoke* invoke_instruction,
+ const ProfileCompilationInfo::OfflineProfileMethodInfo& offline_profile,
+ /*out*/Handle<mirror::ObjectArray<mirror::Class>> inline_cache)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
+ // Compute the inline cache type.
+ InlineCacheType GetInlineCacheType(
+ const Handle<mirror::ObjectArray<mirror::Class>>& classes)
+ REQUIRES_SHARED(Locks::mutator_lock_);
// Try to inline the target of a monomorphic call. If successful, the code
// in the graph will look like:
diff --git a/compiler/optimizing/intrinsics_arm.cc b/compiler/optimizing/intrinsics_arm.cc
index 86000e9..28095c4 100644
--- a/compiler/optimizing/intrinsics_arm.cc
+++ b/compiler/optimizing/intrinsics_arm.cc
@@ -1947,6 +1947,8 @@
}
if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
+ // TODO: Also convert this intrinsic to the IsGcMarking strategy?
+
// The base destination address is computed later, as `temp2` is
// used for intermediate computations.
diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc
index 6c3938c..934ba1b 100644
--- a/compiler/optimizing/intrinsics_arm64.cc
+++ b/compiler/optimizing/intrinsics_arm64.cc
@@ -853,7 +853,6 @@
DCHECK((type == Primitive::kPrimInt) ||
(type == Primitive::kPrimLong) ||
(type == Primitive::kPrimNot));
- MacroAssembler* masm = codegen->GetVIXLAssembler();
Location base_loc = locations->InAt(1);
Register base = WRegisterFrom(base_loc); // Object pointer.
Location offset_loc = locations->InAt(2);
@@ -863,8 +862,7 @@
if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
// UnsafeGetObject/UnsafeGetObjectVolatile with Baker's read barrier case.
- UseScratchRegisterScope temps(masm);
- Register temp = temps.AcquireW();
+ Register temp = WRegisterFrom(locations->GetTemp(0));
codegen->GenerateReferenceLoadWithBakerReadBarrier(invoke,
trg_loc,
base,
@@ -901,6 +899,9 @@
kIntrinsified);
if (can_call && kUseBakerReadBarrier) {
locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
+ // We need a temporary register for the read barrier marking slow
+ // path in CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier.
+ locations->AddTemp(Location::RequiresRegister());
}
locations->SetInAt(0, Location::NoLocation()); // Unused receiver.
locations->SetInAt(1, Location::RequiresRegister());
@@ -2381,9 +2382,14 @@
// Temporary register IP0, obtained from the VIXL scratch register
// pool, cannot be used in ReadBarrierSystemArrayCopySlowPathARM64
// (because that register is clobbered by ReadBarrierMarkRegX
- // entry points). Get an extra temporary register from the
- // register allocator.
+ // entry points). It cannot be used in calls to
+ // CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier
+ // either. For these reasons, get a third extra temporary register
+ // from the register allocator.
locations->AddTemp(Location::RequiresRegister());
+ } else {
+ // Cases other than Baker read barriers: the third temporary will
+ // be acquired from the VIXL scratch register pool.
}
}
@@ -2494,11 +2500,12 @@
// We use a block to end the scratch scope before the write barrier, thus
// freeing the temporary registers so they can be used in `MarkGCCard`.
UseScratchRegisterScope temps(masm);
- // Note: Because it is acquired from VIXL's scratch register pool,
- // `temp3` might be IP0, and thus cannot be used as `ref` argument
- // of CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier
- // calls below (see ReadBarrierMarkSlowPathARM64 for more details).
- Register temp3 = temps.AcquireW();
+ Register temp3;
+ if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
+ temp3 = WRegisterFrom(locations->GetTemp(2));
+ } else {
+ temp3 = temps.AcquireW();
+ }
if (!optimizations.GetDoesNotNeedTypeCheck()) {
// Check whether all elements of the source array are assignable to the component
@@ -2704,19 +2711,7 @@
Register src_curr_addr = temp1.X();
Register dst_curr_addr = temp2.X();
- Register src_stop_addr;
- if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
- // Temporary register IP0, obtained from the VIXL scratch
- // register pool as `temp3`, cannot be used in
- // ReadBarrierSystemArrayCopySlowPathARM64 (because that
- // register is clobbered by ReadBarrierMarkRegX entry points).
- // So another temporary register allocated by the register
- // allocator instead.
- DCHECK_EQ(LocationFrom(temp3).reg(), IP0);
- src_stop_addr = XRegisterFrom(locations->GetTemp(2));
- } else {
- src_stop_addr = temp3.X();
- }
+ Register src_stop_addr = temp3.X();
GenSystemArrayCopyAddresses(masm,
Primitive::kPrimNot,
@@ -2732,6 +2727,8 @@
const int32_t element_size = Primitive::ComponentSize(Primitive::kPrimNot);
if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
+ // TODO: Also convert this intrinsic to the IsGcMarking strategy?
+
// SystemArrayCopy implementation for Baker read barriers (see
// also CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier):
//
@@ -2758,10 +2755,11 @@
__ Cmp(src_curr_addr, src_stop_addr);
__ B(&done, eq);
- Register tmp = temps.AcquireW();
// Make sure `tmp` is not IP0, as it is clobbered by
// ReadBarrierMarkRegX entry points in
// ReadBarrierSystemArrayCopySlowPathARM64.
+ temps.Exclude(ip0);
+ Register tmp = temps.AcquireW();
DCHECK_NE(LocationFrom(tmp).reg(), IP0);
// /* int32_t */ monitor = src->monitor_
diff --git a/compiler/optimizing/intrinsics_arm_vixl.cc b/compiler/optimizing/intrinsics_arm_vixl.cc
index aa89dea..60bcf2c 100644
--- a/compiler/optimizing/intrinsics_arm_vixl.cc
+++ b/compiler/optimizing/intrinsics_arm_vixl.cc
@@ -2265,6 +2265,8 @@
}
if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
+ // TODO: Also convert this intrinsic to the IsGcMarking strategy?
+
// The base destination address is computed later, as `temp2` is
// used for intermediate computations.
diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc
index 62c8910..020e446 100644
--- a/compiler/optimizing/nodes.cc
+++ b/compiler/optimizing/nodes.cc
@@ -2179,6 +2179,9 @@
}
}
if (rerun_loop_analysis) {
+ DCHECK(!outer_graph->HasIrreducibleLoops())
+ << "Recomputing loop information in graphs with irreducible loops "
+ << "is unsupported, as it could lead to loop header changes";
outer_graph->ClearLoopInformation();
outer_graph->ClearDominanceInformation();
outer_graph->BuildDominatorTree();
diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h
index f912d7d..542b218 100644
--- a/compiler/optimizing/nodes.h
+++ b/compiler/optimizing/nodes.h
@@ -5545,8 +5545,6 @@
// Use a known boot image Class* address, embedded in the code by the codegen.
// Used for boot image classes referenced by apps in AOT- and JIT-compiled code.
- // Note: codegen needs to emit a linker patch if indicated by compiler options'
- // GetIncludePatchInformation().
kBootImageAddress,
// Load from an entry in the .bss section using a PC-relative load.
@@ -5750,8 +5748,6 @@
// Use a known boot image String* address, embedded in the code by the codegen.
// Used for boot image strings referenced by apps in AOT- and JIT-compiled code.
- // Note: codegen needs to emit a linker patch if indicated by compiler options'
- // GetIncludePatchInformation().
kBootImageAddress,
// Load from an entry in the .bss section using a PC-relative load.
diff --git a/compiler/optimizing/register_allocator_linear_scan.cc b/compiler/optimizing/register_allocator_linear_scan.cc
index 1a391ce..6354e76 100644
--- a/compiler/optimizing/register_allocator_linear_scan.cc
+++ b/compiler/optimizing/register_allocator_linear_scan.cc
@@ -629,21 +629,21 @@
if (!locations->OutputCanOverlapWithInputs() && locations->Out().IsUnallocated()) {
HInputsRef inputs = defined_by->GetInputs();
for (size_t i = 0; i < inputs.size(); ++i) {
- // Take the last interval of the input. It is the location of that interval
- // that will be used at `defined_by`.
- LiveInterval* interval = inputs[i]->GetLiveInterval()->GetLastSibling();
- // Note that interval may have not been processed yet.
- // TODO: Handle non-split intervals last in the work list.
- if (locations->InAt(i).IsValid()
- && interval->HasRegister()
- && interval->SameRegisterKind(*current)) {
- // The input must be live until the end of `defined_by`, to comply to
- // the linear scan algorithm. So we use `defined_by`'s end lifetime
- // position to check whether the input is dead or is inactive after
- // `defined_by`.
- DCHECK(interval->CoversSlow(defined_by->GetLifetimePosition()));
- size_t position = defined_by->GetLifetimePosition() + 1;
- FreeIfNotCoverAt(interval, position, free_until);
+ if (locations->InAt(i).IsValid()) {
+ // Take the last interval of the input. It is the location of that interval
+ // that will be used at `defined_by`.
+ LiveInterval* interval = inputs[i]->GetLiveInterval()->GetLastSibling();
+ // Note that interval may have not been processed yet.
+ // TODO: Handle non-split intervals last in the work list.
+ if (interval->HasRegister() && interval->SameRegisterKind(*current)) {
+ // The input must be live until the end of `defined_by`, to comply to
+ // the linear scan algorithm. So we use `defined_by`'s end lifetime
+ // position to check whether the input is dead or is inactive after
+ // `defined_by`.
+ DCHECK(interval->CoversSlow(defined_by->GetLifetimePosition()));
+ size_t position = defined_by->GetLifetimePosition() + 1;
+ FreeIfNotCoverAt(interval, position, free_until);
+ }
}
}
}
diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc
index 8f1827b..7bd38c7 100644
--- a/compiler/optimizing/sharpening.cc
+++ b/compiler/optimizing/sharpening.cc
@@ -65,9 +65,7 @@
}
static bool AOTCanEmbedMethod(ArtMethod* method, const CompilerOptions& options) {
- // Including patch information means the AOT code will be patched, which we don't
- // support in the compiler, and is anyways moving away b/33192586.
- return IsInBootImage(method) && !options.GetCompilePic() && !options.GetIncludePatchInformation();
+ return IsInBootImage(method) && !options.GetCompilePic();
}
diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc
index 7fdad95..3fa30fa 100644
--- a/dex2oat/dex2oat.cc
+++ b/dex2oat/dex2oat.cc
@@ -328,11 +328,6 @@
UsageError("");
UsageError(" --dump-timing: display a breakdown of where time was spent");
UsageError("");
- UsageError(" --include-patch-information: Include patching information so the generated code");
- UsageError(" can have its base address moved without full recompilation.");
- UsageError("");
- UsageError(" --no-include-patch-information: Do not include patching information.");
- UsageError("");
UsageError(" -g");
UsageError(" --generate-debug-info: Generate debug information for native debugging,");
UsageError(" such as stack unwinding information, ELF symbols and DWARF sections.");
@@ -1958,7 +1953,6 @@
elf_writer->WriteDynamicSection();
elf_writer->WriteDebugInfo(oat_writer->GetMethodDebugInfo());
- elf_writer->WritePatchLocations(oat_writer->GetAbsolutePatchLocations());
if (!elf_writer->End()) {
LOG(ERROR) << "Failed to write ELF file " << oat_file->GetPath();
diff --git a/dexlayout/dex_visualize.cc b/dexlayout/dex_visualize.cc
index 75d47e4..8997146 100644
--- a/dexlayout/dex_visualize.cc
+++ b/dexlayout/dex_visualize.cc
@@ -35,6 +35,12 @@
namespace art {
+std::string MultidexName(const std::string& prefix,
+ size_t dex_file_index,
+ const std::string& suffix) {
+ return prefix + ((dex_file_index > 0) ? std::to_string(dex_file_index + 1) : "") + suffix;
+}
+
struct FileSection {
public:
std::string name_;
@@ -43,8 +49,22 @@
std::function<uint32_t(const dex_ir::Collections&)> offset_fn_;
};
+static uint32_t HeaderOffset(const dex_ir::Collections& collections ATTRIBUTE_UNUSED) {
+ return 0;
+}
+
+static uint32_t HeaderSize(const dex_ir::Collections& collections ATTRIBUTE_UNUSED) {
+ // Size is in elements, so there is only one header.
+ return 1;
+}
+
static const std::vector<FileSection> kFileSections = {
{
+ "Header",
+ DexFile::kDexTypeHeaderItem,
+ &HeaderSize,
+ &HeaderOffset,
+ }, {
"StringId",
DexFile::kDexTypeStringIdItem,
&dex_ir::Collections::StringIdsSize,
@@ -127,58 +147,71 @@
}
};
+static constexpr bool kSortAscending = false;
+static constexpr bool kSortDescending = true;
+
+static std::vector<const FileSection*> GetSortedSections(
+ const dex_ir::Collections& collections,
+ bool sort_descending) {
+ std::vector<const FileSection*> sorted_sections;
+ // Build the table that will map from offset to color
+ for (const FileSection& s : kFileSections) {
+ sorted_sections.push_back(&s);
+ }
+ // Sort by offset.
+ std::sort(sorted_sections.begin(),
+ sorted_sections.end(),
+ [&](const FileSection* a, const FileSection* b) {
+ if (sort_descending) {
+ return a->offset_fn_(collections) > b->offset_fn_(collections);
+ } else {
+ return a->offset_fn_(collections) < b->offset_fn_(collections);
+ }
+ });
+ return sorted_sections;
+}
+
class Dumper {
public:
// Colors are based on the type of the section in MapList.
- Dumper(const dex_ir::Collections& collections, size_t dex_file_index) {
- // Build the table that will map from offset to color
- table_.emplace_back(DexFile::kDexTypeHeaderItem, 0u);
- for (const FileSection& s : kFileSections) {
- table_.emplace_back(s.type_, s.offset_fn_(collections));
- }
- // Sort into descending order by offset.
- std::sort(table_.begin(),
- table_.end(),
- [](const SectionColor& a, const SectionColor& b) { return a.offset_ > b.offset_; });
+ explicit Dumper(const dex_ir::Collections& collections)
+ : collections_(collections), out_file_(nullptr),
+ sorted_sections_(GetSortedSections(collections, kSortDescending)) { }
+
+ bool OpenAndPrintHeader(size_t dex_index) {
// Open the file and emit the gnuplot prologue.
- std::string dex_file_name("classes");
- std::string out_file_base_name("layout");
- if (dex_file_index > 0) {
- out_file_base_name += std::to_string(dex_file_index + 1);
- dex_file_name += std::to_string(dex_file_index + 1);
+ out_file_ = fopen(MultidexName("layout", dex_index, ".gnuplot").c_str(), "w");
+ if (out_file_ == nullptr) {
+ return false;
}
- dex_file_name += ".dex";
- std::string out_file_name(out_file_base_name + ".gnuplot");
- std::string png_file_name(out_file_base_name + ".png");
- out_file_ = fopen(out_file_name.c_str(), "w");
fprintf(out_file_, "set terminal png size 1920,1080\n");
- fprintf(out_file_, "set output \"%s\"\n", png_file_name.c_str());
- fprintf(out_file_, "set title \"%s\"\n", dex_file_name.c_str());
+ fprintf(out_file_, "set output \"%s\"\n", MultidexName("layout", dex_index, ".png").c_str());
+ fprintf(out_file_, "set title \"%s\"\n", MultidexName("classes", dex_index, ".dex").c_str());
fprintf(out_file_, "set xlabel \"Page offset into dex\"\n");
fprintf(out_file_, "set ylabel \"ClassDef index\"\n");
fprintf(out_file_, "set xtics rotate out (");
- fprintf(out_file_, "\"Header\" %d, ", 0);
bool printed_one = false;
for (const FileSection& s : kFileSections) {
- if (s.size_fn_(collections) > 0) {
+ if (s.size_fn_(collections_) > 0) {
if (printed_one) {
fprintf(out_file_, ", ");
}
- fprintf(out_file_, "\"%s\" %d", s.name_.c_str(), s.offset_fn_(collections) / kPageSize);
+ fprintf(out_file_, "\"%s\" %d", s.name_.c_str(), s.offset_fn_(collections_) / kPageSize);
printed_one = true;
}
}
fprintf(out_file_, ")\n");
fprintf(out_file_,
"plot \"-\" using 1:2:3:4:5 with vector nohead linewidth 1 lc variable notitle\n");
+ return true;
}
int GetColor(uint32_t offset) const {
// The dread linear search to find the right section for the reference.
uint16_t section = 0;
- for (uint16_t i = 0; i < table_.size(); ++i) {
- if (table_[i].offset_ < offset) {
- section = table_[i].type_;
+ for (const FileSection* file_section : sorted_sections_) {
+ if (file_section->offset_fn_(collections_) < offset) {
+ section = file_section->type_;
break;
}
}
@@ -308,13 +341,6 @@
}
private:
- struct SectionColor {
- public:
- SectionColor(uint16_t type, uint32_t offset) : type_(type), offset_(offset) { }
- uint16_t type_;
- uint32_t offset_;
- };
-
using ColorMapType = std::map<uint16_t, int>;
const ColorMapType kColorMap = {
{ DexFile::kDexTypeHeaderItem, 1 },
@@ -336,8 +362,9 @@
{ DexFile::kDexTypeAnnotationsDirectoryItem, 16 }
};
- std::vector<SectionColor> table_;
+ const dex_ir::Collections& collections_;
FILE* out_file_;
+ std::vector<const FileSection*> sorted_sections_;
DISALLOW_COPY_AND_ASSIGN(Dumper);
};
@@ -350,7 +377,11 @@
const DexFile* dex_file,
size_t dex_file_index,
ProfileCompilationInfo* profile_info) {
- std::unique_ptr<Dumper> dumper(new Dumper(header->GetCollections(), dex_file_index));
+ std::unique_ptr<Dumper> dumper(new Dumper(header->GetCollections()));
+ if (!dumper->OpenAndPrintHeader(dex_file_index)) {
+ fprintf(stderr, "Could not open output file.\n");
+ return;
+ }
const uint32_t class_defs_size = header->GetCollections().ClassDefsSize();
for (uint32_t class_index = 0; class_index < class_defs_size; class_index++) {
@@ -401,4 +432,22 @@
} // for
}
+/*
+ * Dumps the offset and size of sections within the file.
+ */
+void ShowDexSectionStatistics(dex_ir::Header* header, size_t dex_file_index) {
+ // Compute the (multidex) class file name).
+ fprintf(stdout, "%s\n", MultidexName("classes", dex_file_index, ".dex").c_str());
+ fprintf(stdout, "section offset items\n");
+ const dex_ir::Collections& collections = header->GetCollections();
+ std::vector<const FileSection*> sorted_sections(GetSortedSections(collections, kSortAscending));
+ for (const FileSection* file_section : sorted_sections) {
+ fprintf(stdout, "%-10s 0x%08x 0x%08x\n",
+ file_section->name_.c_str(),
+ file_section->offset_fn_(collections),
+ file_section->size_fn_(collections));
+ }
+ fprintf(stdout, "\n");
+}
+
} // namespace art
diff --git a/dexlayout/dex_visualize.h b/dexlayout/dex_visualize.h
index 09f8306..a1aa2cd 100644
--- a/dexlayout/dex_visualize.h
+++ b/dexlayout/dex_visualize.h
@@ -38,6 +38,8 @@
size_t dex_file_index,
ProfileCompilationInfo* profile_info);
+void ShowDexSectionStatistics(dex_ir::Header* header, size_t dex_file_index);
+
} // namespace art
#endif // ART_DEXLAYOUT_DEX_VISUALIZE_H_
diff --git a/dexlayout/dexlayout.cc b/dexlayout/dexlayout.cc
index 4aa8b82..f74fb4e 100644
--- a/dexlayout/dexlayout.cc
+++ b/dexlayout/dexlayout.cc
@@ -34,6 +34,7 @@
#include "dex_ir_builder.h"
#include "dex_file-inl.h"
+#include "dex_file_verifier.h"
#include "dex_instruction-inl.h"
#include "dex_visualize.h"
#include "dex_writer.h"
@@ -1724,6 +1725,19 @@
if (new_file != nullptr) {
UNUSED(new_file->FlushCloseOrErase());
}
+ // Verify the output dex file is ok on debug builds.
+ if (kIsDebugBuild) {
+ std::string location = "memory mapped file for " + dex_file_location;
+ std::unique_ptr<const DexFile> dex_file(DexFile::Open(mem_map_->Begin(),
+ mem_map_->Size(),
+ location,
+ header_->Checksum(),
+ /*oat_dex_file*/ nullptr,
+ /*verify*/ true,
+ /*verify_checksum*/ false,
+ &error_msg));
+ DCHECK(dex_file != nullptr) << "Failed to re-open output file:" << error_msg;
+ }
}
/*
@@ -1745,6 +1759,11 @@
return;
}
+ if (options_.show_section_statistics_) {
+ ShowDexSectionStatistics(header_, dex_file_index);
+ return;
+ }
+
// Dump dex file.
if (options_.dump_) {
DumpDexFile();
diff --git a/dexlayout/dexlayout.h b/dexlayout/dexlayout.h
index 3918706..74b5253 100644
--- a/dexlayout/dexlayout.h
+++ b/dexlayout/dexlayout.h
@@ -56,6 +56,7 @@
bool show_annotations_ = false;
bool show_file_headers_ = false;
bool show_section_headers_ = false;
+ bool show_section_statistics_ = false;
bool verbose_ = false;
bool visualize_pattern_ = false;
OutputFormat output_format_ = kOutputPlain;
diff --git a/dexlayout/dexlayout_main.cc b/dexlayout/dexlayout_main.cc
index ad599ae..3eac660 100644
--- a/dexlayout/dexlayout_main.cc
+++ b/dexlayout/dexlayout_main.cc
@@ -57,6 +57,7 @@
fprintf(stderr, " -o : output file name (defaults to stdout)\n");
fprintf(stderr, " -p : profile file name (defaults to no profile)\n");
fprintf(stderr, " -s : visualize reference pattern\n");
+ fprintf(stderr, " -t : display file section sizes\n");
fprintf(stderr, " -w : output dex directory \n");
}
@@ -75,7 +76,7 @@
// Parse all arguments.
while (1) {
- const int ic = getopt(argc, argv, "abcdefghil:mo:p:sw:");
+ const int ic = getopt(argc, argv, "abcdefghil:mo:p:stw:");
if (ic < 0) {
break; // done
}
@@ -127,6 +128,10 @@
options.visualize_pattern_ = true;
options.verbose_ = false;
break;
+ case 't': // display section statistics
+ options.show_section_statistics_ = true;
+ options.verbose_ = false;
+ break;
case 'w': // output dex files directory
options.output_dex_directory_ = optarg;
break;
diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc
index becb827..e767023 100644
--- a/oatdump/oatdump.cc
+++ b/oatdump/oatdump.cc
@@ -2210,13 +2210,13 @@
ScopedIndentation indent2(&state->vios_);
auto* resolved_fields = dex_cache->GetResolvedFields();
for (size_t i = 0, length = dex_cache->NumResolvedFields(); i < length; ++i) {
- auto* elem = mirror::DexCache::GetElementPtrSize(
- resolved_fields, i, image_pointer_size);
+ auto* elem = mirror::DexCache::GetNativePairPtrSize(
+ resolved_fields, i, image_pointer_size).object;
size_t run = 0;
for (size_t j = i + 1;
- j != length && elem == mirror::DexCache::GetElementPtrSize(resolved_fields,
- j,
- image_pointer_size);
+ j != length &&
+ elem == mirror::DexCache::GetNativePairPtrSize(
+ resolved_fields, j, image_pointer_size).object;
++j) {
++run;
}
diff --git a/patchoat/patchoat.cc b/patchoat/patchoat.cc
index 18a6670..dfaae7d 100644
--- a/patchoat/patchoat.cc
+++ b/patchoat/patchoat.cc
@@ -534,17 +534,18 @@
mirror::DexCache::SetElementPtrSize(copy_methods, j, copy, pointer_size);
}
}
- ArtField** orig_fields = orig_dex_cache->GetResolvedFields();
- ArtField** relocated_fields = RelocatedAddressOfPointer(orig_fields);
+ mirror::FieldDexCacheType* orig_fields = orig_dex_cache->GetResolvedFields();
+ mirror::FieldDexCacheType* relocated_fields = RelocatedAddressOfPointer(orig_fields);
copy_dex_cache->SetField64<false>(
mirror::DexCache::ResolvedFieldsOffset(),
static_cast<int64_t>(reinterpret_cast<uintptr_t>(relocated_fields)));
if (orig_fields != nullptr) {
- ArtField** copy_fields = RelocatedCopyOf(orig_fields);
+ mirror::FieldDexCacheType* copy_fields = RelocatedCopyOf(orig_fields);
for (size_t j = 0, num = orig_dex_cache->NumResolvedFields(); j != num; ++j) {
- ArtField* orig = mirror::DexCache::GetElementPtrSize(orig_fields, j, pointer_size);
- ArtField* copy = RelocatedAddressOfPointer(orig);
- mirror::DexCache::SetElementPtrSize(copy_fields, j, copy, pointer_size);
+ mirror::FieldDexCachePair orig =
+ mirror::DexCache::GetNativePairPtrSize(orig_fields, j, pointer_size);
+ mirror::FieldDexCachePair copy(RelocatedAddressOfPointer(orig.object), orig.index);
+ mirror::DexCache::SetNativePairPtrSize(copy_fields, j, copy, pointer_size);
}
}
mirror::MethodTypeDexCacheType* orig_method_types = orig_dex_cache->GetResolvedMethodTypes();
diff --git a/runtime/arch/arm/fault_handler_arm.cc b/runtime/arch/arm/fault_handler_arm.cc
index daa2dff..923ff4f 100644
--- a/runtime/arch/arm/fault_handler_arm.cc
+++ b/runtime/arch/arm/fault_handler_arm.cc
@@ -47,24 +47,6 @@
return instr_size;
}
-void FaultManager::HandleNestedSignal(int sig ATTRIBUTE_UNUSED, siginfo_t* info ATTRIBUTE_UNUSED,
- void* context) {
- // Note that in this handler we set up the registers and return to
- // longjmp directly rather than going through an assembly language stub. The
- // reason for this is that longjmp is (currently) in ARM mode and that would
- // require switching modes in the stub - incurring an unwanted relocation.
-
- struct ucontext *uc = reinterpret_cast<struct ucontext*>(context);
- struct sigcontext *sc = reinterpret_cast<struct sigcontext*>(&uc->uc_mcontext);
- Thread* self = Thread::Current();
- CHECK(self != nullptr); // This will cause a SIGABRT if self is null.
-
- sc->arm_r0 = reinterpret_cast<uintptr_t>(*self->GetNestedSignalState());
- sc->arm_r1 = 1;
- sc->arm_pc = reinterpret_cast<uintptr_t>(longjmp);
- VLOG(signals) << "longjmp address: " << reinterpret_cast<void*>(sc->arm_pc);
-}
-
void FaultManager::GetMethodAndReturnPcAndSp(siginfo_t* siginfo ATTRIBUTE_UNUSED, void* context,
ArtMethod** out_method,
uintptr_t* out_return_pc, uintptr_t* out_sp) {
diff --git a/runtime/arch/arm64/fault_handler_arm64.cc b/runtime/arch/arm64/fault_handler_arm64.cc
index c02be87..193af58 100644
--- a/runtime/arch/arm64/fault_handler_arm64.cc
+++ b/runtime/arch/arm64/fault_handler_arm64.cc
@@ -39,21 +39,6 @@
namespace art {
-void FaultManager::HandleNestedSignal(int sig ATTRIBUTE_UNUSED, siginfo_t* info ATTRIBUTE_UNUSED,
- void* context) {
- // To match the case used in ARM we return directly to the longjmp function
- // rather than through a trivial assembly language stub.
-
- struct ucontext *uc = reinterpret_cast<struct ucontext*>(context);
- struct sigcontext *sc = reinterpret_cast<struct sigcontext*>(&uc->uc_mcontext);
- Thread* self = Thread::Current();
- CHECK(self != nullptr); // This will cause a SIGABRT if self is null.
-
- sc->regs[0] = reinterpret_cast<uintptr_t>(*self->GetNestedSignalState());
- sc->regs[1] = 1;
- sc->pc = reinterpret_cast<uintptr_t>(longjmp);
-}
-
void FaultManager::GetMethodAndReturnPcAndSp(siginfo_t* siginfo ATTRIBUTE_UNUSED, void* context,
ArtMethod** out_method,
uintptr_t* out_return_pc, uintptr_t* out_sp) {
diff --git a/runtime/arch/mips/fault_handler_mips.cc b/runtime/arch/mips/fault_handler_mips.cc
index 1792f31..f9c19e8 100644
--- a/runtime/arch/mips/fault_handler_mips.cc
+++ b/runtime/arch/mips/fault_handler_mips.cc
@@ -35,10 +35,6 @@
namespace art {
-void FaultManager::HandleNestedSignal(int sig ATTRIBUTE_UNUSED, siginfo_t* info ATTRIBUTE_UNUSED,
- void* context ATTRIBUTE_UNUSED) {
-}
-
void FaultManager::GetMethodAndReturnPcAndSp(siginfo_t* siginfo, void* context,
ArtMethod** out_method,
uintptr_t* out_return_pc, uintptr_t* out_sp) {
diff --git a/runtime/arch/mips64/fault_handler_mips64.cc b/runtime/arch/mips64/fault_handler_mips64.cc
index 709cab5..d668d3a 100644
--- a/runtime/arch/mips64/fault_handler_mips64.cc
+++ b/runtime/arch/mips64/fault_handler_mips64.cc
@@ -35,10 +35,6 @@
namespace art {
-void FaultManager::HandleNestedSignal(int sig ATTRIBUTE_UNUSED, siginfo_t* info ATTRIBUTE_UNUSED,
- void* context ATTRIBUTE_UNUSED) {
-}
-
void FaultManager::GetMethodAndReturnPcAndSp(siginfo_t* siginfo, void* context,
ArtMethod** out_method,
uintptr_t* out_return_pc, uintptr_t* out_sp) {
diff --git a/runtime/arch/x86/fault_handler_x86.cc b/runtime/arch/x86/fault_handler_x86.cc
index a4d6bb4..f407ebf 100644
--- a/runtime/arch/x86/fault_handler_x86.cc
+++ b/runtime/arch/x86/fault_handler_x86.cc
@@ -75,12 +75,6 @@
extern "C" void art_quick_throw_stack_overflow();
extern "C" void art_quick_test_suspend();
-// Note this is different from the others (no underscore on 64 bit mac) due to
-// the way the symbol is defined in the .S file.
-// TODO: fix the symbols for 64 bit mac - there is a double underscore prefix for some
-// of them.
-extern "C" void art_nested_signal_return();
-
// Get the size of an instruction in bytes.
// Return 0 if the instruction is not handled.
static uint32_t GetInstructionSize(const uint8_t* pc) {
@@ -247,21 +241,6 @@
return pc - startpc;
}
-void FaultManager::HandleNestedSignal(int, siginfo_t*, void* context) {
- // For the Intel architectures we need to go to an assembly language
- // stub. This is because the 32 bit call to longjmp is much different
- // from the 64 bit ABI call and pushing things onto the stack inside this
- // handler was unwieldy and ugly. The use of the stub means we can keep
- // this code the same for both 32 and 64 bit.
-
- Thread* self = Thread::Current();
- CHECK(self != nullptr); // This will cause a SIGABRT if self is null.
-
- struct ucontext* uc = reinterpret_cast<struct ucontext*>(context);
- uc->CTX_JMP_BUF = reinterpret_cast<uintptr_t>(*self->GetNestedSignalState());
- uc->CTX_EIP = reinterpret_cast<uintptr_t>(art_nested_signal_return);
-}
-
void FaultManager::GetMethodAndReturnPcAndSp(siginfo_t* siginfo, void* context,
ArtMethod** out_method,
uintptr_t* out_return_pc, uintptr_t* out_sp) {
diff --git a/runtime/arch/x86/quick_entrypoints_x86.S b/runtime/arch/x86/quick_entrypoints_x86.S
index ff7ba92..5f38dc8 100644
--- a/runtime/arch/x86/quick_entrypoints_x86.S
+++ b/runtime/arch/x86/quick_entrypoints_x86.S
@@ -2137,19 +2137,6 @@
ret
END_FUNCTION art_quick_string_compareto
-// Return from a nested signal:
-// Entry:
-// eax: address of jmp_buf in TLS
-
-DEFINE_FUNCTION art_nested_signal_return
- SETUP_GOT_NOSAVE ebx // sets %ebx for call into PLT
- movl LITERAL(1), %ecx
- PUSH ecx // second arg to longjmp (1)
- PUSH eax // first arg to longjmp (jmp_buf)
- call PLT_SYMBOL(longjmp)
- UNREACHABLE
-END_FUNCTION art_nested_signal_return
-
// Create a function `name` calling the ReadBarrier::Mark routine,
// getting its argument and returning its result through register
// `reg`, saving and restoring all caller-save registers.
diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
index 8a663d1..e87b165 100644
--- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S
+++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
@@ -2100,18 +2100,6 @@
ret
END_FUNCTION art_quick_instance_of
-
-// Return from a nested signal:
-// Entry:
-// rdi: address of jmp_buf in TLS
-
-DEFINE_FUNCTION art_nested_signal_return
- // first arg to longjmp is already in correct register
- movq LITERAL(1), %rsi // second arg to longjmp (1)
- call PLT_SYMBOL(longjmp)
- UNREACHABLE
-END_FUNCTION art_nested_signal_return
-
// Create a function `name` calling the ReadBarrier::Mark routine,
// getting its argument and returning its result through register
// `reg`, saving and restoring all caller-save registers.
diff --git a/runtime/base/arena_allocator.cc b/runtime/base/arena_allocator.cc
index db43319..5aede38 100644
--- a/runtime/base/arena_allocator.cc
+++ b/runtime/base/arena_allocator.cc
@@ -15,6 +15,7 @@
*/
#include <algorithm>
+#include <cstddef>
#include <iomanip>
#include <numeric>
@@ -27,7 +28,7 @@
namespace art {
-static constexpr size_t kMemoryToolRedZoneBytes = 8;
+constexpr size_t kMemoryToolRedZoneBytes = 8;
constexpr size_t Arena::kDefaultSize;
template <bool kCount>
@@ -168,23 +169,75 @@
Arena::Arena() : bytes_allocated_(0), next_(nullptr) {
}
+class MallocArena FINAL : public Arena {
+ public:
+ explicit MallocArena(size_t size = Arena::kDefaultSize);
+ virtual ~MallocArena();
+ private:
+ static constexpr size_t RequiredOverallocation() {
+ return (alignof(std::max_align_t) < ArenaAllocator::kArenaAlignment)
+ ? ArenaAllocator::kArenaAlignment - alignof(std::max_align_t)
+ : 0u;
+ }
+
+ uint8_t* unaligned_memory_;
+};
+
MallocArena::MallocArena(size_t size) {
- memory_ = reinterpret_cast<uint8_t*>(calloc(1, size));
- CHECK(memory_ != nullptr); // Abort on OOM.
- DCHECK_ALIGNED(memory_, ArenaAllocator::kAlignment);
+ // We need to guarantee kArenaAlignment aligned allocation for the new arena.
+ // TODO: Use std::aligned_alloc() when it becomes available with C++17.
+ constexpr size_t overallocation = RequiredOverallocation();
+ unaligned_memory_ = reinterpret_cast<uint8_t*>(calloc(1, size + overallocation));
+ CHECK(unaligned_memory_ != nullptr); // Abort on OOM.
+ DCHECK_ALIGNED(unaligned_memory_, alignof(std::max_align_t));
+ if (overallocation == 0u) {
+ memory_ = unaligned_memory_;
+ } else {
+ memory_ = AlignUp(unaligned_memory_, ArenaAllocator::kArenaAlignment);
+ if (UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) {
+ size_t head = memory_ - unaligned_memory_;
+ size_t tail = overallocation - head;
+ MEMORY_TOOL_MAKE_NOACCESS(unaligned_memory_, head);
+ MEMORY_TOOL_MAKE_NOACCESS(memory_ + size, tail);
+ }
+ }
+ DCHECK_ALIGNED(memory_, ArenaAllocator::kArenaAlignment);
size_ = size;
}
MallocArena::~MallocArena() {
- free(reinterpret_cast<void*>(memory_));
+ constexpr size_t overallocation = RequiredOverallocation();
+ if (overallocation != 0u && UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) {
+ size_t head = memory_ - unaligned_memory_;
+ size_t tail = overallocation - head;
+ MEMORY_TOOL_MAKE_UNDEFINED(unaligned_memory_, head);
+ MEMORY_TOOL_MAKE_UNDEFINED(memory_ + size_, tail);
+ }
+ free(reinterpret_cast<void*>(unaligned_memory_));
}
+class MemMapArena FINAL : public Arena {
+ public:
+ MemMapArena(size_t size, bool low_4gb, const char* name);
+ virtual ~MemMapArena();
+ void Release() OVERRIDE;
+
+ private:
+ std::unique_ptr<MemMap> map_;
+};
+
MemMapArena::MemMapArena(size_t size, bool low_4gb, const char* name) {
+ // Round up to a full page as that's the smallest unit of allocation for mmap()
+ // and we want to be able to use all memory that we actually allocate.
+ size = RoundUp(size, kPageSize);
std::string error_msg;
map_.reset(MemMap::MapAnonymous(
name, nullptr, size, PROT_READ | PROT_WRITE, low_4gb, false, &error_msg));
CHECK(map_.get() != nullptr) << error_msg;
memory_ = map_->Begin();
+ static_assert(ArenaAllocator::kArenaAlignment <= kPageSize,
+ "Arena should not need stronger alignment than kPageSize.");
+ DCHECK_ALIGNED(memory_, ArenaAllocator::kArenaAlignment);
size_ = map_->Size();
}
@@ -332,20 +385,7 @@
ArenaAllocatorStats::RecordAlloc(rounded_bytes, kind);
uint8_t* ret;
if (UNLIKELY(rounded_bytes > static_cast<size_t>(end_ - ptr_))) {
- ret = AllocFromNewArena(rounded_bytes);
- uint8_t* noaccess_begin = ret + bytes;
- uint8_t* noaccess_end;
- if (ret == arena_head_->Begin()) {
- DCHECK(ptr_ - rounded_bytes == ret);
- noaccess_end = end_;
- } else {
- // We're still using the old arena but `ret` comes from a new one just after it.
- DCHECK(arena_head_->next_ != nullptr);
- DCHECK(ret == arena_head_->next_->Begin());
- DCHECK_EQ(rounded_bytes, arena_head_->next_->GetBytesAllocated());
- noaccess_end = arena_head_->next_->End();
- }
- MEMORY_TOOL_MAKE_NOACCESS(noaccess_begin, noaccess_end - noaccess_begin);
+ ret = AllocFromNewArenaWithMemoryTool(rounded_bytes);
} else {
ret = ptr_;
ptr_ += rounded_bytes;
@@ -356,6 +396,30 @@
return ret;
}
+void* ArenaAllocator::AllocWithMemoryToolAlign16(size_t bytes, ArenaAllocKind kind) {
+ // We mark all memory for a newly retrieved arena as inaccessible and then
+ // mark only the actually allocated memory as defined. That leaves red zones
+ // and padding between allocations marked as inaccessible.
+ size_t rounded_bytes = bytes + kMemoryToolRedZoneBytes;
+ DCHECK_ALIGNED(rounded_bytes, 8); // `bytes` is 16-byte aligned, red zone is 8-byte aligned.
+ uintptr_t padding =
+ ((reinterpret_cast<uintptr_t>(ptr_) + 15u) & 15u) - reinterpret_cast<uintptr_t>(ptr_);
+ ArenaAllocatorStats::RecordAlloc(rounded_bytes, kind);
+ uint8_t* ret;
+ if (UNLIKELY(padding + rounded_bytes > static_cast<size_t>(end_ - ptr_))) {
+ static_assert(kArenaAlignment >= 16, "Expecting sufficient alignment for new Arena.");
+ ret = AllocFromNewArenaWithMemoryTool(rounded_bytes);
+ } else {
+ ptr_ += padding; // Leave padding inaccessible.
+ ret = ptr_;
+ ptr_ += rounded_bytes;
+ }
+ MEMORY_TOOL_MAKE_DEFINED(ret, bytes);
+ // Check that the memory is already zeroed out.
+ DCHECK(std::all_of(ret, ret + bytes, [](uint8_t val) { return val == 0u; }));
+ return ret;
+}
+
ArenaAllocator::~ArenaAllocator() {
// Reclaim all the arenas by giving them back to the thread pool.
UpdateBytesAllocated();
@@ -386,6 +450,24 @@
return new_arena->Begin();
}
+uint8_t* ArenaAllocator::AllocFromNewArenaWithMemoryTool(size_t bytes) {
+ uint8_t* ret = AllocFromNewArena(bytes);
+ uint8_t* noaccess_begin = ret + bytes;
+ uint8_t* noaccess_end;
+ if (ret == arena_head_->Begin()) {
+ DCHECK(ptr_ - bytes == ret);
+ noaccess_end = end_;
+ } else {
+ // We're still using the old arena but `ret` comes from a new one just after it.
+ DCHECK(arena_head_->next_ != nullptr);
+ DCHECK(ret == arena_head_->next_->Begin());
+ DCHECK_EQ(bytes, arena_head_->next_->GetBytesAllocated());
+ noaccess_end = arena_head_->next_->End();
+ }
+ MEMORY_TOOL_MAKE_NOACCESS(noaccess_begin, noaccess_end - noaccess_begin);
+ return ret;
+}
+
bool ArenaAllocator::Contains(const void* ptr) const {
if (ptr >= begin_ && ptr < end_) {
return true;
@@ -398,7 +480,9 @@
return false;
}
-MemStats::MemStats(const char* name, const ArenaAllocatorStats* stats, const Arena* first_arena,
+MemStats::MemStats(const char* name,
+ const ArenaAllocatorStats* stats,
+ const Arena* first_arena,
ssize_t lost_bytes_adjustment)
: name_(name),
stats_(stats),
diff --git a/runtime/base/arena_allocator.h b/runtime/base/arena_allocator.h
index f92fbea..c39429c 100644
--- a/runtime/base/arena_allocator.h
+++ b/runtime/base/arena_allocator.h
@@ -34,7 +34,6 @@
class ArenaAllocator;
class ArenaStack;
class ScopedArenaAllocator;
-class MemMap;
class MemStats;
template <typename T>
@@ -244,22 +243,6 @@
DISALLOW_COPY_AND_ASSIGN(Arena);
};
-class MallocArena FINAL : public Arena {
- public:
- explicit MallocArena(size_t size = Arena::kDefaultSize);
- virtual ~MallocArena();
-};
-
-class MemMapArena FINAL : public Arena {
- public:
- MemMapArena(size_t size, bool low_4gb, const char* name);
- virtual ~MemMapArena();
- void Release() OVERRIDE;
-
- private:
- std::unique_ptr<MemMap> map_;
-};
-
class ArenaPool {
public:
explicit ArenaPool(bool use_malloc = true,
@@ -319,8 +302,31 @@
return ret;
}
+ // Returns zeroed memory.
+ void* AllocAlign16(size_t bytes, ArenaAllocKind kind = kArenaAllocMisc) ALWAYS_INLINE {
+ // It is an error to request 16-byte aligned allocation of unaligned size.
+ DCHECK_ALIGNED(bytes, 16);
+ if (UNLIKELY(IsRunningOnMemoryTool())) {
+ return AllocWithMemoryToolAlign16(bytes, kind);
+ }
+ uintptr_t padding =
+ ((reinterpret_cast<uintptr_t>(ptr_) + 15u) & 15u) - reinterpret_cast<uintptr_t>(ptr_);
+ ArenaAllocatorStats::RecordAlloc(bytes, kind);
+ if (UNLIKELY(padding + bytes > static_cast<size_t>(end_ - ptr_))) {
+ static_assert(kArenaAlignment >= 16, "Expecting sufficient alignment for new Arena.");
+ return AllocFromNewArena(bytes);
+ }
+ ptr_ += padding;
+ uint8_t* ret = ptr_;
+ DCHECK_ALIGNED(ret, 16);
+ ptr_ += bytes;
+ return ret;
+ }
+
// Realloc never frees the input pointer, it is the caller's job to do this if necessary.
- void* Realloc(void* ptr, size_t ptr_size, size_t new_size,
+ void* Realloc(void* ptr,
+ size_t ptr_size,
+ size_t new_size,
ArenaAllocKind kind = kArenaAllocMisc) ALWAYS_INLINE {
DCHECK_GE(new_size, ptr_size);
DCHECK_EQ(ptr == nullptr, ptr_size == 0u);
@@ -371,12 +377,17 @@
bool Contains(const void* ptr) const;
- static constexpr size_t kAlignment = 8;
+ // The alignment guaranteed for individual allocations.
+ static constexpr size_t kAlignment = 8u;
+
+ // The alignment required for the whole Arena rather than individual allocations.
+ static constexpr size_t kArenaAlignment = 16u;
private:
void* AllocWithMemoryTool(size_t bytes, ArenaAllocKind kind);
+ void* AllocWithMemoryToolAlign16(size_t bytes, ArenaAllocKind kind);
uint8_t* AllocFromNewArena(size_t bytes);
-
+ uint8_t* AllocFromNewArenaWithMemoryTool(size_t bytes);
void UpdateBytesAllocated();
@@ -396,7 +407,9 @@
class MemStats {
public:
- MemStats(const char* name, const ArenaAllocatorStats* stats, const Arena* first_arena,
+ MemStats(const char* name,
+ const ArenaAllocatorStats* stats,
+ const Arena* first_arena,
ssize_t lost_bytes_adjustment = 0);
void Dump(std::ostream& os) const;
diff --git a/runtime/base/scoped_arena_allocator.h b/runtime/base/scoped_arena_allocator.h
index 55044b3..1a0eb5e 100644
--- a/runtime/base/scoped_arena_allocator.h
+++ b/runtime/base/scoped_arena_allocator.h
@@ -39,8 +39,6 @@
kFree,
};
-static constexpr size_t kArenaAlignment = 8;
-
// Holds a list of Arenas for use by ScopedArenaAllocator stack.
// The memory is returned to the ArenaPool when the ArenaStack is destroyed.
class ArenaStack : private DebugStackRefCounter, private ArenaAllocatorMemoryTool {
@@ -67,6 +65,9 @@
return *(reinterpret_cast<ArenaFreeTag*>(ptr) - 1);
}
+ // The alignment guaranteed for individual allocations.
+ static constexpr size_t kAlignment = 8u;
+
private:
struct Peak;
struct Current;
@@ -89,8 +90,8 @@
if (UNLIKELY(IsRunningOnMemoryTool())) {
return AllocWithMemoryTool(bytes, kind);
}
- // Add kArenaAlignment for the free or used tag. Required to preserve alignment.
- size_t rounded_bytes = RoundUp(bytes + (kIsDebugBuild ? kArenaAlignment : 0u), kArenaAlignment);
+ // Add kAlignment for the free or used tag. Required to preserve alignment.
+ size_t rounded_bytes = RoundUp(bytes + (kIsDebugBuild ? kAlignment : 0u), kAlignment);
uint8_t* ptr = top_ptr_;
if (UNLIKELY(static_cast<size_t>(top_end_ - ptr) < rounded_bytes)) {
ptr = AllocateFromNextArena(rounded_bytes);
@@ -98,7 +99,7 @@
CurrentStats()->RecordAlloc(bytes, kind);
top_ptr_ = ptr + rounded_bytes;
if (kIsDebugBuild) {
- ptr += kArenaAlignment;
+ ptr += kAlignment;
ArenaTagForAllocation(ptr) = ArenaFreeTag::kUsed;
}
return ptr;
diff --git a/runtime/class_linker-inl.h b/runtime/class_linker-inl.h
index bd510ca..9ddc6cf 100644
--- a/runtime/class_linker-inl.h
+++ b/runtime/class_linker-inl.h
@@ -161,9 +161,15 @@
return resolved_method;
}
-inline ArtField* ClassLinker::GetResolvedField(uint32_t field_idx,
- ObjPtr<mirror::DexCache> dex_cache) {
- return dex_cache->GetResolvedField(field_idx, image_pointer_size_);
+inline ArtField* ClassLinker::LookupResolvedField(uint32_t field_idx,
+ ArtMethod* referrer,
+ bool is_static) {
+ ObjPtr<mirror::DexCache> dex_cache = referrer->GetDexCache();
+ ArtField* field = dex_cache->GetResolvedField(field_idx, image_pointer_size_);
+ if (field == nullptr) {
+ field = LookupResolvedField(field_idx, dex_cache, referrer->GetClassLoader(), is_static);
+ }
+ return field;
}
inline ArtField* ClassLinker::ResolveField(uint32_t field_idx,
@@ -171,7 +177,8 @@
bool is_static) {
Thread::PoisonObjectPointersIfDebug();
ObjPtr<mirror::Class> declaring_class = referrer->GetDeclaringClass();
- ArtField* resolved_field = GetResolvedField(field_idx, referrer->GetDexCache());
+ ArtField* resolved_field =
+ referrer->GetDexCache()->GetResolvedField(field_idx, image_pointer_size_);
if (UNLIKELY(resolved_field == nullptr)) {
StackHandleScope<2> hs(Thread::Current());
Handle<mirror::DexCache> dex_cache(hs.NewHandle(referrer->GetDexCache()));
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index b611aa2..ab2b395 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -1282,7 +1282,10 @@
num_types = dex_file->NumTypeIds();
}
const size_t num_methods = dex_file->NumMethodIds();
- const size_t num_fields = dex_file->NumFieldIds();
+ size_t num_fields = mirror::DexCache::kDexCacheFieldCacheSize;
+ if (dex_file->NumFieldIds() < num_fields) {
+ num_fields = dex_file->NumFieldIds();
+ }
size_t num_method_types = mirror::DexCache::kDexCacheMethodTypeCacheSize;
if (dex_file->NumProtoIds() < num_method_types) {
num_method_types = dex_file->NumProtoIds();
@@ -1326,17 +1329,22 @@
dex_cache->SetResolvedMethods(methods);
}
if (num_fields != 0u) {
- ArtField** const fields =
- reinterpret_cast<ArtField**>(raw_arrays + layout.FieldsOffset());
- for (size_t j = 0; kIsDebugBuild && j < num_fields; ++j) {
- DCHECK(fields[j] == nullptr);
+ mirror::FieldDexCacheType* const image_resolved_fields = dex_cache->GetResolvedFields();
+ mirror::FieldDexCacheType* const fields =
+ reinterpret_cast<mirror::FieldDexCacheType*>(raw_arrays + layout.FieldsOffset());
+ for (size_t j = 0; j < num_fields; ++j) {
+ DCHECK_EQ(mirror::DexCache::GetNativePairPtrSize(fields, j, image_pointer_size_).index,
+ 0u);
+ DCHECK(mirror::DexCache::GetNativePairPtrSize(fields, j, image_pointer_size_).object ==
+ nullptr);
+ mirror::DexCache::SetNativePairPtrSize(
+ fields,
+ j,
+ mirror::DexCache::GetNativePairPtrSize(image_resolved_fields,
+ j,
+ image_pointer_size_),
+ image_pointer_size_);
}
- CopyNonNull(dex_cache->GetResolvedFields(),
- num_fields,
- fields,
- [] (const ArtField* field) {
- return field == nullptr;
- });
dex_cache->SetResolvedFields(fields);
}
if (num_method_types != 0u) {
@@ -8260,6 +8268,43 @@
return resolved;
}
+ArtField* ClassLinker::LookupResolvedField(uint32_t field_idx,
+ ObjPtr<mirror::DexCache> dex_cache,
+ ObjPtr<mirror::ClassLoader> class_loader,
+ bool is_static) {
+ const DexFile& dex_file = *dex_cache->GetDexFile();
+ const DexFile::FieldId& field_id = dex_file.GetFieldId(field_idx);
+ ObjPtr<mirror::Class> klass = dex_cache->GetResolvedType(field_id.class_idx_);
+ if (klass == nullptr) {
+ klass = LookupResolvedType(dex_file, field_id.class_idx_, dex_cache, class_loader);
+ }
+ if (klass == nullptr) {
+ // The class has not been resolved yet, so the field is also unresolved.
+ return nullptr;
+ }
+ DCHECK(klass->IsResolved());
+ Thread* self = is_static ? Thread::Current() : nullptr;
+
+ // First try to find a field declared directly by `klass` by the field index.
+ ArtField* resolved_field = is_static
+ ? mirror::Class::FindStaticField(self, klass, dex_cache, field_idx)
+ : klass->FindInstanceField(dex_cache, field_idx);
+
+ if (resolved_field == nullptr) {
+ // If not found in `klass` by field index, search the class hierarchy using the name and type.
+ const char* name = dex_file.GetFieldName(field_id);
+ const char* type = dex_file.GetFieldTypeDescriptor(field_id);
+ resolved_field = is_static
+ ? mirror::Class::FindStaticField(self, klass, name, type)
+ : klass->FindInstanceField(name, type);
+ }
+
+ if (resolved_field != nullptr) {
+ dex_cache->SetResolvedField(field_idx, resolved_field, image_pointer_size_);
+ }
+ return resolved_field;
+}
+
ArtField* ClassLinker::ResolveField(const DexFile& dex_file,
uint32_t field_idx,
Handle<mirror::DexCache> dex_cache,
@@ -8320,9 +8365,8 @@
return nullptr;
}
- StringPiece name(dex_file.StringDataByIdx(field_id.name_idx_));
- StringPiece type(dex_file.StringDataByIdx(
- dex_file.GetTypeId(field_id.type_idx_).descriptor_idx_));
+ StringPiece name(dex_file.GetFieldName(field_id));
+ StringPiece type(dex_file.GetFieldTypeDescriptor(field_id));
resolved = mirror::Class::FindField(self, klass, name, type);
if (resolved != nullptr) {
dex_cache->SetResolvedField(field_idx, resolved, image_pointer_size_);
diff --git a/runtime/class_linker.h b/runtime/class_linker.h
index a5d26c7..6254acb 100644
--- a/runtime/class_linker.h
+++ b/runtime/class_linker.h
@@ -333,7 +333,7 @@
REQUIRES_SHARED(Locks::mutator_lock_)
REQUIRES(!Locks::dex_lock_, !Roles::uninterruptible_);
- ArtField* GetResolvedField(uint32_t field_idx, ObjPtr<mirror::DexCache> dex_cache)
+ ArtField* LookupResolvedField(uint32_t field_idx, ArtMethod* referrer, bool is_static)
REQUIRES_SHARED(Locks::mutator_lock_);
ArtField* ResolveField(uint32_t field_idx, ArtMethod* referrer, bool is_static)
REQUIRES_SHARED(Locks::mutator_lock_)
@@ -842,6 +842,13 @@
REQUIRES(!Locks::classlinker_classes_lock_)
REQUIRES_SHARED(Locks::mutator_lock_);
+ // Find a field by its field index.
+ ArtField* LookupResolvedField(uint32_t field_idx,
+ ObjPtr<mirror::DexCache> dex_cache,
+ ObjPtr<mirror::ClassLoader> class_loader,
+ bool is_static)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
void RegisterDexFileLocked(const DexFile& dex_file,
ObjPtr<mirror::DexCache> dex_cache,
ObjPtr<mirror::ClassLoader> class_loader)
diff --git a/runtime/common_dex_operations.h b/runtime/common_dex_operations.h
index 8776061..6693eef 100644
--- a/runtime/common_dex_operations.h
+++ b/runtime/common_dex_operations.h
@@ -36,8 +36,8 @@
void ArtInterpreterToCompiledCodeBridge(Thread* self,
ArtMethod* caller,
+ const DexFile::CodeItem* code_item,
ShadowFrame* shadow_frame,
- uint16_t arg_offset,
JValue* result);
} // namespace interpreter
@@ -56,7 +56,7 @@
interpreter::ArtInterpreterToInterpreterBridge(self, code_item, callee_frame, result);
} else {
interpreter::ArtInterpreterToCompiledCodeBridge(
- self, caller_method, callee_frame, first_dest_reg, result);
+ self, caller_method, code_item, callee_frame, result);
}
} else {
interpreter::UnstartedRuntime::Invoke(self, code_item, callee_frame, result, first_dest_reg);
diff --git a/runtime/entrypoints/quick/quick_throw_entrypoints.cc b/runtime/entrypoints/quick/quick_throw_entrypoints.cc
index c8ee99a..1520e13 100644
--- a/runtime/entrypoints/quick/quick_throw_entrypoints.cc
+++ b/runtime/entrypoints/quick/quick_throw_entrypoints.cc
@@ -62,9 +62,7 @@
extern "C" NO_RETURN void artThrowNullPointerExceptionFromSignal(uintptr_t addr, Thread* self)
REQUIRES_SHARED(Locks::mutator_lock_) {
ScopedQuickEntrypointChecks sqec(self);
- self->NoteSignalBeingHandled();
ThrowNullPointerExceptionFromDexPC(/* check_address */ true, addr);
- self->NoteSignalHandlerDone();
self->QuickDeliverException();
}
@@ -95,9 +93,7 @@
extern "C" NO_RETURN void artThrowStackOverflowFromCode(Thread* self)
REQUIRES_SHARED(Locks::mutator_lock_) {
ScopedQuickEntrypointChecks sqec(self);
- self->NoteSignalBeingHandled();
ThrowStackOverflowError(self);
- self->NoteSignalHandlerDone();
self->QuickDeliverException();
}
diff --git a/runtime/entrypoints_order_test.cc b/runtime/entrypoints_order_test.cc
index d0687ce..55a4625 100644
--- a/runtime/entrypoints_order_test.cc
+++ b/runtime/entrypoints_order_test.cc
@@ -133,9 +133,8 @@
EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, thread_local_alloc_stack_top, thread_local_alloc_stack_end,
sizeof(void*));
EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, thread_local_alloc_stack_end, held_mutexes, sizeof(void*));
- EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, held_mutexes, nested_signal_state,
+ EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, held_mutexes, flip_function,
sizeof(void*) * kLockLevelCount);
- EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, nested_signal_state, flip_function, sizeof(void*));
EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, flip_function, method_verifier, sizeof(void*));
EXPECT_OFFSET_DIFFP(Thread, tlsPtr_, method_verifier, thread_local_mark_stack, sizeof(void*));
EXPECT_OFFSET_DIFF(Thread, tlsPtr_.thread_local_mark_stack, Thread, wait_mutex_, sizeof(void*),
diff --git a/runtime/fault_handler.cc b/runtime/fault_handler.cc
index f9345b6..64128cc 100644
--- a/runtime/fault_handler.cc
+++ b/runtime/fault_handler.cc
@@ -28,47 +28,6 @@
#include "thread-inl.h"
#include "verify_object-inl.h"
-// Note on nested signal support
-// -----------------------------
-//
-// Typically a signal handler should not need to deal with signals that occur within it.
-// However, when a SIGSEGV occurs that is in generated code and is not one of the
-// handled signals (implicit checks), we call a function to try to dump the stack
-// to the log. This enhances the debugging experience but may have the side effect
-// that it may not work. If the cause of the original SIGSEGV is a corrupted stack or other
-// memory region, the stack backtrace code may run into trouble and may either crash
-// or fail with an abort (SIGABRT). In either case we don't want that (new) signal to
-// mask the original signal and thus prevent useful debug output from being presented.
-//
-// In order to handle this situation, before we call the stack tracer we do the following:
-//
-// 1. shutdown the fault manager so that we are talking to the real signal management
-// functions rather than those in sigchain.
-// 2. use pthread_sigmask to allow SIGSEGV and SIGABRT signals to be delivered to the
-// thread running the signal handler.
-// 3. set the handler for SIGSEGV and SIGABRT to a secondary signal handler.
-// 4. save the thread's state to the TLS of the current thread using 'setjmp'
-//
-// We then call the stack tracer and one of two things may happen:
-// a. it completes successfully
-// b. it crashes and a signal is raised.
-//
-// In the former case, we fall through and everything is fine. In the latter case
-// our secondary signal handler gets called in a signal context. This results in
-// a call to FaultManager::HandledNestedSignal(), an archirecture specific function
-// whose purpose is to call 'longjmp' on the jmp_buf saved in the TLS of the current
-// thread. This results in a return with a non-zero value from 'setjmp'. We detect this
-// and write something to the log to tell the user that it happened.
-//
-// Regardless of how we got there, we reach the code after the stack tracer and we
-// restore the signal states to their original values, reinstate the fault manager (thus
-// reestablishing the signal chain) and continue.
-
-// This is difficult to test with a runtime test. To invoke the nested signal code
-// on any signal, uncomment the following line and run something that throws a
-// NullPointerException.
-// #define TEST_NESTED_SIGNAL
-
namespace art {
// Static fault manger object accessed by signal handler.
FaultManager fault_manager;
@@ -83,11 +42,6 @@
fault_manager.HandleFault(sig, info, context);
}
-// Signal handler for dealing with a nested signal.
-static void art_nested_signal_handler(int sig, siginfo_t* info, void* context) {
- fault_manager.HandleNestedSignal(sig, info, context);
-}
-
FaultManager::FaultManager() : initialized_(false) {
sigaction(SIGSEGV, nullptr, &oldaction_);
}
@@ -156,122 +110,93 @@
DCHECK(self != nullptr);
DCHECK(Runtime::Current() != nullptr);
DCHECK(Runtime::Current()->IsStarted());
-
- // Now set up the nested signal handler.
-
- // TODO: add SIGSEGV back to the nested signals when we can handle running out stack gracefully.
- static const int handled_nested_signals[] = {SIGABRT};
- constexpr size_t num_handled_nested_signals = arraysize(handled_nested_signals);
-
- // Release the fault manager so that it will remove the signal chain for
- // SIGSEGV and we call the real sigaction.
- fault_manager.Release();
-
- // The action for SIGSEGV should be the default handler now.
-
- // Unblock the signals we allow so that they can be delivered in the signal handler.
- sigset_t sigset;
- sigemptyset(&sigset);
- for (int signal : handled_nested_signals) {
- sigaddset(&sigset, signal);
- }
- pthread_sigmask(SIG_UNBLOCK, &sigset, nullptr);
-
- // If we get a signal in this code we want to invoke our nested signal
- // handler.
- struct sigaction action;
- struct sigaction oldactions[num_handled_nested_signals];
- action.sa_sigaction = art_nested_signal_handler;
-
- // Explicitly mask out SIGSEGV and SIGABRT from the nested signal handler. This
- // should be the default but we definitely don't want these happening in our
- // nested signal handler.
- sigemptyset(&action.sa_mask);
- for (int signal : handled_nested_signals) {
- sigaddset(&action.sa_mask, signal);
- }
-
- action.sa_flags = SA_SIGINFO | SA_ONSTACK;
-#if !defined(__APPLE__) && !defined(__mips__)
- action.sa_restorer = nullptr;
-#endif
-
- // Catch handled signals to invoke our nested handler.
- bool success = true;
- for (size_t i = 0; i < num_handled_nested_signals; ++i) {
- success = sigaction(handled_nested_signals[i], &action, &oldactions[i]) == 0;
- if (!success) {
- PLOG(ERROR) << "Unable to set up nested signal handler";
- break;
+ for (const auto& handler : other_handlers_) {
+ if (handler->Action(sig, info, context)) {
+ return true;
}
}
-
- if (success) {
- // Save the current state and call the handlers. If anything causes a signal
- // our nested signal handler will be invoked and this will longjmp to the saved
- // state.
- if (setjmp(*self->GetNestedSignalState()) == 0) {
- for (const auto& handler : other_handlers_) {
- if (handler->Action(sig, info, context)) {
- // Restore the signal handlers, reinit the fault manager and return. Signal was
- // handled.
- for (size_t i = 0; i < num_handled_nested_signals; ++i) {
- success = sigaction(handled_nested_signals[i], &oldactions[i], nullptr) == 0;
- if (!success) {
- PLOG(ERROR) << "Unable to restore signal handler";
- }
- }
- fault_manager.Init();
- return true;
- }
- }
- } else {
- LOG(ERROR) << "Nested signal detected - original signal being reported";
- }
-
- // Restore the signal handlers.
- for (size_t i = 0; i < num_handled_nested_signals; ++i) {
- success = sigaction(handled_nested_signals[i], &oldactions[i], nullptr) == 0;
- if (!success) {
- PLOG(ERROR) << "Unable to restore signal handler";
- }
- }
- }
-
- // Now put the fault manager back in place.
- fault_manager.Init();
return false;
}
+class ScopedSignalUnblocker {
+ public:
+ explicit ScopedSignalUnblocker(const std::initializer_list<int>& signals) {
+ sigset_t new_mask;
+ sigemptyset(&new_mask);
+ for (int signal : signals) {
+ sigaddset(&new_mask, signal);
+ }
+ if (sigprocmask(SIG_UNBLOCK, &new_mask, &previous_mask_) != 0) {
+ PLOG(FATAL) << "failed to unblock signals";
+ }
+ }
+
+ ~ScopedSignalUnblocker() {
+ if (sigprocmask(SIG_SETMASK, &previous_mask_, nullptr) != 0) {
+ PLOG(FATAL) << "failed to unblock signals";
+ }
+ }
+
+ private:
+ sigset_t previous_mask_;
+};
+
+class ScopedHandlingSignalSetter {
+ public:
+ explicit ScopedHandlingSignalSetter(Thread* thread) : thread_(thread) {
+ CHECK(!thread->HandlingSignal());
+ thread_->SetHandlingSignal(true);
+ }
+
+ ~ScopedHandlingSignalSetter() {
+ CHECK(thread_->HandlingSignal());
+ thread_->SetHandlingSignal(false);
+ }
+
+ private:
+ Thread* thread_;
+};
+
void FaultManager::HandleFault(int sig, siginfo_t* info, void* context) {
// BE CAREFUL ALLOCATING HERE INCLUDING USING LOG(...)
//
// If malloc calls abort, it will be holding its lock.
// If the handler tries to call malloc, it will deadlock.
- VLOG(signals) << "Handling fault";
- if (IsInGeneratedCode(info, context, true)) {
- VLOG(signals) << "in generated code, looking for handler";
- for (const auto& handler : generated_code_handlers_) {
- VLOG(signals) << "invoking Action on handler " << handler;
- if (handler->Action(sig, info, context)) {
-#ifdef TEST_NESTED_SIGNAL
- // In test mode we want to fall through to stack trace handler
- // on every signal (in reality this will cause a crash on the first
- // signal).
- break;
-#else
- // We have handled a signal so it's time to return from the
- // signal handler to the appropriate place.
- return;
-#endif
- }
- }
- // We hit a signal we didn't handle. This might be something for which
- // we can give more information about so call all registered handlers to see
- // if it is.
- if (HandleFaultByOtherHandlers(sig, info, context)) {
+ // Use a thread local field to track whether we're recursing, and fall back.
+ // (e.g.. if one of our handlers crashed)
+ Thread* thread = Thread::Current();
+
+ if (thread != nullptr && !thread->HandlingSignal()) {
+ // Unblock some signals and set thread->handling_signal_ to true,
+ // so that we can catch crashes in our signal handler.
+ ScopedHandlingSignalSetter setter(thread);
+ ScopedSignalUnblocker unblocker { SIGABRT, SIGBUS, SIGSEGV }; // NOLINT
+
+ VLOG(signals) << "Handling fault";
+
+#ifdef TEST_NESTED_SIGNAL
+ // Simulate a crash in a handler.
+ raise(SIGSEGV);
+#endif
+
+ if (IsInGeneratedCode(info, context, true)) {
+ VLOG(signals) << "in generated code, looking for handler";
+ for (const auto& handler : generated_code_handlers_) {
+ VLOG(signals) << "invoking Action on handler " << handler;
+ if (handler->Action(sig, info, context)) {
+ // We have handled a signal so it's time to return from the
+ // signal handler to the appropriate place.
+ return;
+ }
+ }
+
+ // We hit a signal we didn't handle. This might be something for which
+ // we can give more information about so call all registered handlers to
+ // see if it is.
+ if (HandleFaultByOtherHandlers(sig, info, context)) {
return;
+ }
}
}
@@ -417,11 +342,7 @@
bool JavaStackTraceHandler::Action(int sig ATTRIBUTE_UNUSED, siginfo_t* siginfo, void* context) {
// Make sure that we are in the generated code, but we may not have a dex pc.
-#ifdef TEST_NESTED_SIGNAL
- bool in_generated_code = true;
-#else
bool in_generated_code = manager_->IsInGeneratedCode(siginfo, context, false);
-#endif
if (in_generated_code) {
LOG(ERROR) << "Dumping java stack trace for crash in generated code";
ArtMethod* method = nullptr;
@@ -432,12 +353,6 @@
manager_->GetMethodAndReturnPcAndSp(siginfo, context, &method, &return_pc, &sp);
// Inside of generated code, sp[0] is the method, so sp is the frame.
self->SetTopOfStack(reinterpret_cast<ArtMethod**>(sp));
-#ifdef TEST_NESTED_SIGNAL
- // To test the nested signal handler we raise a signal here. This will cause the
- // nested signal handler to be called and perform a longjmp back to the setjmp
- // above.
- abort();
-#endif
self->DumpJavaStack(LOG_STREAM(ERROR));
}
diff --git a/runtime/fault_handler.h b/runtime/fault_handler.h
index 56e0fb7..ce59ba7 100644
--- a/runtime/fault_handler.h
+++ b/runtime/fault_handler.h
@@ -45,7 +45,6 @@
void EnsureArtActionInFrontOfSignalChain();
void HandleFault(int sig, siginfo_t* info, void* context);
- void HandleNestedSignal(int sig, siginfo_t* info, void* context);
// Added handlers are owned by the fault handler and will be freed on Shutdown().
void AddHandler(FaultHandler* handler, bool generated_code);
diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc
index 010ef11..568f8d6 100644
--- a/runtime/gc/space/image_space.cc
+++ b/runtime/gc/space/image_space.cc
@@ -1259,17 +1259,18 @@
}
}
}
- ArtField** fields = dex_cache->GetResolvedFields();
+ mirror::FieldDexCacheType* fields = dex_cache->GetResolvedFields();
if (fields != nullptr) {
- ArtField** new_fields = fixup_adapter.ForwardObject(fields);
+ mirror::FieldDexCacheType* new_fields = fixup_adapter.ForwardObject(fields);
if (fields != new_fields) {
dex_cache->SetResolvedFields(new_fields);
}
for (size_t j = 0, num = dex_cache->NumResolvedFields(); j != num; ++j) {
- ArtField* orig = mirror::DexCache::GetElementPtrSize(new_fields, j, pointer_size);
- ArtField* copy = fixup_adapter.ForwardObject(orig);
- if (orig != copy) {
- mirror::DexCache::SetElementPtrSize(new_fields, j, copy, pointer_size);
+ mirror::FieldDexCachePair orig =
+ mirror::DexCache::GetNativePairPtrSize(new_fields, j, pointer_size);
+ mirror::FieldDexCachePair copy(fixup_adapter.ForwardObject(orig.object), orig.index);
+ if (orig.object != copy.object) {
+ mirror::DexCache::SetNativePairPtrSize(new_fields, j, copy, pointer_size);
}
}
}
diff --git a/runtime/image.cc b/runtime/image.cc
index 88f28f3..5fbb7a6 100644
--- a/runtime/image.cc
+++ b/runtime/image.cc
@@ -25,7 +25,7 @@
namespace art {
const uint8_t ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' };
-const uint8_t ImageHeader::kImageVersion[] = { '0', '4', '0', '\0' }; // Integer.valueOf intrinsic
+const uint8_t ImageHeader::kImageVersion[] = { '0', '4', '1', '\0' }; // hash-based DexCache fields
ImageHeader::ImageHeader(uint32_t image_begin,
uint32_t image_size,
diff --git a/runtime/interpreter/interpreter.cc b/runtime/interpreter/interpreter.cc
index b03ffc9..1b3d339 100644
--- a/runtime/interpreter/interpreter.cc
+++ b/runtime/interpreter/interpreter.cc
@@ -270,12 +270,7 @@
// Pop the shadow frame before calling into compiled code.
self->PopShadowFrame();
- // Calculate the offset of the first input reg. The input registers are in the high regs.
- // If there is no code item, all the registers are inputs.
- uint16_t arg_offset = (code_item == nullptr)
- ? 0
- : code_item->registers_size_ - code_item->ins_size_;
- ArtInterpreterToCompiledCodeBridge(self, nullptr, &shadow_frame, arg_offset, &result);
+ ArtInterpreterToCompiledCodeBridge(self, nullptr, code_item, &shadow_frame, &result);
// Push the shadow frame back as the caller will expect it.
self->PushShadowFrame(&shadow_frame);
diff --git a/runtime/interpreter/interpreter_common.cc b/runtime/interpreter/interpreter_common.cc
index f47db16..8978bfd 100644
--- a/runtime/interpreter/interpreter_common.cc
+++ b/runtime/interpreter/interpreter_common.cc
@@ -459,8 +459,8 @@
void ArtInterpreterToCompiledCodeBridge(Thread* self,
ArtMethod* caller,
+ const DexFile::CodeItem* code_item,
ShadowFrame* shadow_frame,
- uint16_t arg_offset,
JValue* result)
REQUIRES_SHARED(Locks::mutator_lock_) {
ArtMethod* method = shadow_frame->GetMethod();
@@ -471,25 +471,21 @@
self->PushShadowFrame(shadow_frame);
StackHandleScope<1> hs(self);
Handle<mirror::Class> h_class(hs.NewHandle(declaringClass));
- if (UNLIKELY(!Runtime::Current()->GetClassLinker()->EnsureInitialized(
- self, h_class, true, true))) {
+ if (UNLIKELY(!Runtime::Current()->GetClassLinker()->EnsureInitialized(self, h_class, true,
+ true))) {
self->PopShadowFrame();
DCHECK(self->IsExceptionPending());
return;
}
self->PopShadowFrame();
CHECK(h_class->IsInitializing());
+ // Reload from shadow frame in case the method moved, this is faster than adding a handle.
+ method = shadow_frame->GetMethod();
}
}
- // Basic checks for the arg_offset. If there's no code item, the arg_offset must be 0. Otherwise,
- // check that the arg_offset isn't greater than the number of registers. A stronger check is hard
- // to do because the method being called may have been replaced if it was a string init method,
- // and the arguments are handled differently in that case.
- if (method->GetCodeItem() == nullptr) {
- DCHECK_EQ(0u, arg_offset);
- } else {
- DCHECK_LE(arg_offset, shadow_frame->NumberOfVRegs());
- }
+ uint16_t arg_offset = (code_item == nullptr)
+ ? 0
+ : code_item->registers_size_ - code_item->ins_size_;
jit::Jit* jit = Runtime::Current()->GetJit();
if (jit != nullptr && caller != nullptr) {
jit->NotifyInterpreterToCompiledCodeTransition(self, caller);
@@ -926,23 +922,12 @@
// Number of registers for the callee's call frame.
uint16_t num_regs;
- // When transitioning to compiled code, the frame only contains input registers.
- // We can avoid accessing compiled code items here so they remain untouched during runtime,
- // saving memory since they never get paged in.
- bool use_compiler_entrypoint = Runtime::Current()->IsStarted() &&
- !ClassLinker::ShouldUseInterpreterEntrypoint(
- called_method, called_method->GetEntryPointFromQuickCompiledCode());
- if (LIKELY(code_item != nullptr && !use_compiler_entrypoint)) {
+ if (LIKELY(code_item != nullptr)) {
num_regs = code_item->registers_size_;
- } else {
- DCHECK(called_method->IsNative() || called_method->IsProxyMethod() || use_compiler_entrypoint);
- num_regs = number_of_inputs;
- }
- // Check that the number of inputs passed in agrees with the code item. If a string initializer
- // is called, it will have been replaced by an equivalent StringFactory call above, and they
- // have one fewer input (no this pointer).
- if (code_item != nullptr) {
DCHECK_EQ(string_init ? number_of_inputs - 1 : number_of_inputs, code_item->ins_size_);
+ } else {
+ DCHECK(called_method->IsNative() || called_method->IsProxyMethod());
+ num_regs = number_of_inputs;
}
// Hack for String init:
diff --git a/runtime/interpreter/interpreter_common.h b/runtime/interpreter/interpreter_common.h
index 67e072d..2589ad0 100644
--- a/runtime/interpreter/interpreter_common.h
+++ b/runtime/interpreter/interpreter_common.h
@@ -527,11 +527,10 @@
}
}
-// The arg_offset is the offset to the first input register in the frame.
void ArtInterpreterToCompiledCodeBridge(Thread* self,
ArtMethod* caller,
+ const DexFile::CodeItem* code_item,
ShadowFrame* shadow_frame,
- uint16_t arg_offset,
JValue* result);
// Set string value created from StringFactory.newStringFromXXX() into all aliases of
diff --git a/runtime/interpreter/unstarted_runtime.cc b/runtime/interpreter/unstarted_runtime.cc
index 80554c2..eb0a9d1 100644
--- a/runtime/interpreter/unstarted_runtime.cc
+++ b/runtime/interpreter/unstarted_runtime.cc
@@ -1672,6 +1672,12 @@
}
}
+void UnstartedRuntime::UnstartedSystemIdentityHashCode(
+ Thread* self ATTRIBUTE_UNUSED, ShadowFrame* shadow_frame, JValue* result, size_t arg_offset)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ mirror::Object* obj = shadow_frame->GetVRegReference(arg_offset);
+ result->SetI((obj != nullptr) ? obj->IdentityHashCode() : 0);
+}
void UnstartedRuntime::UnstartedJNIVMRuntimeNewUnpaddedArray(
Thread* self, ArtMethod* method ATTRIBUTE_UNUSED, mirror::Object* receiver ATTRIBUTE_UNUSED,
@@ -1836,13 +1842,6 @@
}
}
-void UnstartedRuntime::UnstartedJNISystemIdentityHashCode(
- Thread* self ATTRIBUTE_UNUSED, ArtMethod* method ATTRIBUTE_UNUSED,
- mirror::Object* receiver ATTRIBUTE_UNUSED, uint32_t* args, JValue* result) {
- mirror::Object* obj = reinterpret_cast<mirror::Object*>(args[0]);
- result->SetI((obj != nullptr) ? obj->IdentityHashCode() : 0);
-}
-
void UnstartedRuntime::UnstartedJNIByteOrderIsLittleEndian(
Thread* self ATTRIBUTE_UNUSED, ArtMethod* method ATTRIBUTE_UNUSED,
mirror::Object* receiver ATTRIBUTE_UNUSED, uint32_t* args ATTRIBUTE_UNUSED, JValue* result) {
diff --git a/runtime/interpreter/unstarted_runtime_list.h b/runtime/interpreter/unstarted_runtime_list.h
index e9435e4..2560a92 100644
--- a/runtime/interpreter/unstarted_runtime_list.h
+++ b/runtime/interpreter/unstarted_runtime_list.h
@@ -76,7 +76,8 @@
V(UnsafePutObjectVolatile, "void sun.misc.Unsafe.putObjectVolatile(java.lang.Object, long, java.lang.Object)") \
V(UnsafePutOrderedObject, "void sun.misc.Unsafe.putOrderedObject(java.lang.Object, long, java.lang.Object)") \
V(IntegerParseInt, "int java.lang.Integer.parseInt(java.lang.String)") \
- V(LongParseLong, "long java.lang.Long.parseLong(java.lang.String)")
+ V(LongParseLong, "long java.lang.Long.parseLong(java.lang.String)") \
+ V(SystemIdentityHashCode, "int java.lang.System.identityHashCode(java.lang.Object)")
// Methods that are native.
#define UNSTARTED_RUNTIME_JNI_LIST(V) \
@@ -98,7 +99,6 @@
V(ArrayCreateMultiArray, "java.lang.Object java.lang.reflect.Array.createMultiArray(java.lang.Class, int[])") \
V(ArrayCreateObjectArray, "java.lang.Object java.lang.reflect.Array.createObjectArray(java.lang.Class, int)") \
V(ThrowableNativeFillInStackTrace, "java.lang.Object java.lang.Throwable.nativeFillInStackTrace()") \
- V(SystemIdentityHashCode, "int java.lang.System.identityHashCode(java.lang.Object)") \
V(ByteOrderIsLittleEndian, "boolean java.nio.ByteOrder.isLittleEndian()") \
V(UnsafeCompareAndSwapInt, "boolean sun.misc.Unsafe.compareAndSwapInt(java.lang.Object, long, int, int)") \
V(UnsafeGetIntVolatile, "int sun.misc.Unsafe.getIntVolatile(java.lang.Object, long)") \
diff --git a/runtime/interpreter/unstarted_runtime_test.cc b/runtime/interpreter/unstarted_runtime_test.cc
index db222fa..56e261c 100644
--- a/runtime/interpreter/unstarted_runtime_test.cc
+++ b/runtime/interpreter/unstarted_runtime_test.cc
@@ -1367,5 +1367,26 @@
ShadowFrame::DeleteDeoptimizedFrame(shadow_frame);
}
+TEST_F(UnstartedRuntimeTest, IdentityHashCode) {
+ Thread* self = Thread::Current();
+ ScopedObjectAccess soa(self);
+ ShadowFrame* tmp = ShadowFrame::CreateDeoptimizedFrame(10, nullptr, nullptr, 0);
+
+ JValue result;
+ UnstartedSystemIdentityHashCode(self, tmp, &result, 0);
+
+ EXPECT_EQ(0, result.GetI());
+ ASSERT_FALSE(self->IsExceptionPending());
+
+ ObjPtr<mirror::String> str = mirror::String::AllocFromModifiedUtf8(self, "abd");
+ tmp->SetVRegReference(0, str.Ptr());
+ UnstartedSystemIdentityHashCode(self, tmp, &result, 0);
+ EXPECT_NE(0, result.GetI());
+ EXPECT_EQ(str->IdentityHashCode(), result.GetI());
+ ASSERT_FALSE(self->IsExceptionPending());
+
+ ShadowFrame::DeleteDeoptimizedFrame(tmp);
+}
+
} // namespace interpreter
} // namespace art
diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc
index 8b2a2b4..e7b23dc 100644
--- a/runtime/jit/jit_code_cache.cc
+++ b/runtime/jit/jit_code_cache.cc
@@ -1262,6 +1262,7 @@
for (size_t i = 0; i < info->number_of_inline_caches_; ++i) {
std::vector<ProfileMethodInfo::ProfileClassReference> profile_classes;
const InlineCache& cache = info->cache_[i];
+ ArtMethod* caller = info->GetMethod();
bool is_missing_types = false;
for (size_t k = 0; k < InlineCache::kIndividualCacheSize; k++) {
mirror::Class* cls = cache.classes_[k].Read();
@@ -1269,6 +1270,15 @@
break;
}
+ // Check if the receiver is in the boot class path or if it's in the
+ // same class loader as the caller. If not, skip it, as there is not
+ // much we can do during AOT.
+ if (!cls->IsBootStrapClassLoaded() &&
+ caller->GetClassLoader() != cls->GetClassLoader()) {
+ is_missing_types = true;
+ continue;
+ }
+
const DexFile* class_dex_file = nullptr;
dex::TypeIndex type_index;
diff --git a/runtime/linear_alloc.cc b/runtime/linear_alloc.cc
index f91b0ed..e9db9b8 100644
--- a/runtime/linear_alloc.cc
+++ b/runtime/linear_alloc.cc
@@ -33,6 +33,11 @@
return allocator_.Alloc(size);
}
+void* LinearAlloc::AllocAlign16(Thread* self, size_t size) {
+ MutexLock mu(self, lock_);
+ return allocator_.AllocAlign16(size);
+}
+
size_t LinearAlloc::GetUsedMemory() const {
MutexLock mu(Thread::Current(), lock_);
return allocator_.BytesUsed();
diff --git a/runtime/linear_alloc.h b/runtime/linear_alloc.h
index df7f17d..384b2e3 100644
--- a/runtime/linear_alloc.h
+++ b/runtime/linear_alloc.h
@@ -29,6 +29,7 @@
explicit LinearAlloc(ArenaPool* pool);
void* Alloc(Thread* self, size_t size) REQUIRES(!lock_);
+ void* AllocAlign16(Thread* self, size_t size) REQUIRES(!lock_);
// Realloc never frees the input pointer, it is the caller's job to do this if necessary.
void* Realloc(Thread* self, void* ptr, size_t old_size, size_t new_size) REQUIRES(!lock_);
diff --git a/runtime/lock_word.h b/runtime/lock_word.h
index 2f2565b..edc64f3 100644
--- a/runtime/lock_word.h
+++ b/runtime/lock_word.h
@@ -57,7 +57,8 @@
* |10|9|87654321098765432109876543210|
* |11|0| ForwardingAddress |
*
- * The rb bits store the read barrier state.
+ * The `r` bit stores the read barrier state.
+ * The `m` bit stores the mark state.
*/
class LockWord {
public:
diff --git a/runtime/mirror/class.h b/runtime/mirror/class.h
index b68eedc..c52b66a 100644
--- a/runtime/mirror/class.h
+++ b/runtime/mirror/class.h
@@ -595,7 +595,7 @@
// The size of java.lang.Class.class.
static uint32_t ClassClassSize(PointerSize pointer_size) {
// The number of vtable entries in java.lang.Class.
- uint32_t vtable_entries = Object::kVTableLength + 73;
+ uint32_t vtable_entries = Object::kVTableLength + 70;
return ComputeClassSize(true, vtable_entries, 0, 0, 4, 1, 0, pointer_size);
}
diff --git a/runtime/mirror/dex_cache-inl.h b/runtime/mirror/dex_cache-inl.h
index 29bf6a0..582ecb2 100644
--- a/runtime/mirror/dex_cache-inl.h
+++ b/runtime/mirror/dex_cache-inl.h
@@ -24,6 +24,7 @@
#include "base/casts.h"
#include "base/enums.h"
#include "base/logging.h"
+#include "dex_file.h"
#include "gc_root.h"
#include "mirror/class.h"
#include "mirror/call_site.h"
@@ -36,6 +37,15 @@
namespace art {
namespace mirror {
+template <typename T>
+inline void NativeDexCachePair<T>::Initialize(std::atomic<NativeDexCachePair<T>>* dex_cache,
+ PointerSize pointer_size) {
+ NativeDexCachePair<T> first_elem;
+ first_elem.object = nullptr;
+ first_elem.index = InvalidIndexForSlot(0);
+ DexCache::SetNativePairPtrSize(dex_cache, 0, first_elem, pointer_size);
+}
+
inline uint32_t DexCache::ClassSize(PointerSize pointer_size) {
uint32_t vtable_entries = Object::kVTableLength + 5;
return Class::ComputeClassSize(true, vtable_entries, 0, 0, 0, 0, 0, pointer_size);
@@ -164,20 +174,36 @@
}
}
+inline uint32_t DexCache::FieldSlotIndex(uint32_t field_idx) {
+ DCHECK_LT(field_idx, GetDexFile()->NumFieldIds());
+ const uint32_t slot_idx = field_idx % kDexCacheFieldCacheSize;
+ DCHECK_LT(slot_idx, NumResolvedFields());
+ return slot_idx;
+}
+
inline ArtField* DexCache::GetResolvedField(uint32_t field_idx, PointerSize ptr_size) {
DCHECK_EQ(Runtime::Current()->GetClassLinker()->GetImagePointerSize(), ptr_size);
- DCHECK_LT(field_idx, NumResolvedFields()); // NOTE: Unchecked, i.e. not throwing AIOOB.
- ArtField* field = GetElementPtrSize(GetResolvedFields(), field_idx, ptr_size);
- if (field == nullptr || field->GetDeclaringClass()->IsErroneous()) {
- return nullptr;
- }
- return field;
+ auto pair = GetNativePairPtrSize(GetResolvedFields(), FieldSlotIndex(field_idx), ptr_size);
+ return pair.GetObjectForIndex(field_idx);
}
inline void DexCache::SetResolvedField(uint32_t field_idx, ArtField* field, PointerSize ptr_size) {
DCHECK_EQ(Runtime::Current()->GetClassLinker()->GetImagePointerSize(), ptr_size);
- DCHECK_LT(field_idx, NumResolvedFields()); // NOTE: Unchecked, i.e. not throwing AIOOB.
- SetElementPtrSize(GetResolvedFields(), field_idx, field, ptr_size);
+ DCHECK(field != nullptr);
+ FieldDexCachePair pair(field, field_idx);
+ SetNativePairPtrSize(GetResolvedFields(), FieldSlotIndex(field_idx), pair, ptr_size);
+}
+
+inline void DexCache::ClearResolvedField(uint32_t field_idx, PointerSize ptr_size) {
+ DCHECK_EQ(Runtime::Current()->GetClassLinker()->GetImagePointerSize(), ptr_size);
+ uint32_t slot_idx = FieldSlotIndex(field_idx);
+ auto* resolved_fields = GetResolvedFields();
+ // This is racy but should only be called from the single-threaded ImageWriter.
+ DCHECK(Runtime::Current()->IsAotCompiler());
+ if (GetNativePairPtrSize(resolved_fields, slot_idx, ptr_size).index == field_idx) {
+ FieldDexCachePair cleared(nullptr, FieldDexCachePair::InvalidIndexForSlot(slot_idx));
+ SetNativePairPtrSize(resolved_fields, slot_idx, cleared, ptr_size);
+ }
}
inline ArtMethod* DexCache::GetResolvedMethod(uint32_t method_idx, PointerSize ptr_size) {
@@ -225,6 +251,40 @@
}
}
+template <typename T>
+NativeDexCachePair<T> DexCache::GetNativePairPtrSize(std::atomic<NativeDexCachePair<T>>* pair_array,
+ size_t idx,
+ PointerSize ptr_size) {
+ if (ptr_size == PointerSize::k64) {
+ auto* array = reinterpret_cast<std::atomic<ConversionPair64>*>(pair_array);
+ ConversionPair64 value = AtomicLoadRelaxed16B(&array[idx]);
+ return NativeDexCachePair<T>(reinterpret_cast64<T*>(value.first),
+ dchecked_integral_cast<size_t>(value.second));
+ } else {
+ auto* array = reinterpret_cast<std::atomic<ConversionPair32>*>(pair_array);
+ ConversionPair32 value = array[idx].load(std::memory_order_relaxed);
+ return NativeDexCachePair<T>(reinterpret_cast<T*>(value.first), value.second);
+ }
+}
+
+template <typename T>
+void DexCache::SetNativePairPtrSize(std::atomic<NativeDexCachePair<T>>* pair_array,
+ size_t idx,
+ NativeDexCachePair<T> pair,
+ PointerSize ptr_size) {
+ if (ptr_size == PointerSize::k64) {
+ auto* array = reinterpret_cast<std::atomic<ConversionPair64>*>(pair_array);
+ ConversionPair64 v(reinterpret_cast64<uint64_t>(pair.object), pair.index);
+ AtomicStoreRelease16B(&array[idx], v);
+ } else {
+ auto* array = reinterpret_cast<std::atomic<ConversionPair32>*>(pair_array);
+ ConversionPair32 v(
+ dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(pair.object)),
+ dchecked_integral_cast<uint32_t>(pair.index));
+ array[idx].store(v, std::memory_order_release);
+ }
+}
+
template <typename T,
ReadBarrierOption kReadBarrierOption,
typename Visitor>
diff --git a/runtime/mirror/dex_cache.cc b/runtime/mirror/dex_cache.cc
index 1b8b391..c95d92e 100644
--- a/runtime/mirror/dex_cache.cc
+++ b/runtime/mirror/dex_cache.cc
@@ -52,8 +52,12 @@
dex_file->NumTypeIds() != 0u ||
dex_file->NumMethodIds() != 0u ||
dex_file->NumFieldIds() != 0u) {
+ static_assert(ArenaAllocator::kAlignment == 8, "Expecting arena alignment of 8.");
+ DCHECK(layout.Alignment() == 8u || layout.Alignment() == 16u);
// Zero-initialized.
- raw_arrays = reinterpret_cast<uint8_t*>(linear_alloc->Alloc(self, layout.Size()));
+ raw_arrays = (layout.Alignment() == 16u)
+ ? reinterpret_cast<uint8_t*>(linear_alloc->AllocAlign16(self, layout.Size()))
+ : reinterpret_cast<uint8_t*>(linear_alloc->Alloc(self, layout.Size()));
}
mirror::StringDexCacheType* strings = (dex_file->NumStringIds() == 0u) ? nullptr :
@@ -62,17 +66,21 @@
reinterpret_cast<mirror::TypeDexCacheType*>(raw_arrays + layout.TypesOffset());
ArtMethod** methods = (dex_file->NumMethodIds() == 0u) ? nullptr :
reinterpret_cast<ArtMethod**>(raw_arrays + layout.MethodsOffset());
- ArtField** fields = (dex_file->NumFieldIds() == 0u) ? nullptr :
- reinterpret_cast<ArtField**>(raw_arrays + layout.FieldsOffset());
+ mirror::FieldDexCacheType* fields = (dex_file->NumFieldIds() == 0u) ? nullptr :
+ reinterpret_cast<mirror::FieldDexCacheType*>(raw_arrays + layout.FieldsOffset());
- size_t num_strings = mirror::DexCache::kDexCacheStringCacheSize;
+ size_t num_strings = kDexCacheStringCacheSize;
if (dex_file->NumStringIds() < num_strings) {
num_strings = dex_file->NumStringIds();
}
- size_t num_types = mirror::DexCache::kDexCacheTypeCacheSize;
+ size_t num_types = kDexCacheTypeCacheSize;
if (dex_file->NumTypeIds() < num_types) {
num_types = dex_file->NumTypeIds();
}
+ size_t num_fields = kDexCacheFieldCacheSize;
+ if (dex_file->NumFieldIds() < num_fields) {
+ num_fields = dex_file->NumFieldIds();
+ }
// Note that we allocate the method type dex caches regardless of this flag,
// and we make sure here that they're not used by the runtime. This is in the
@@ -80,17 +88,17 @@
//
// If this needs to be mitigated in a production system running this code,
// DexCache::kDexCacheMethodTypeCacheSize can be set to zero.
- mirror::MethodTypeDexCacheType* method_types = nullptr;
+ MethodTypeDexCacheType* method_types = nullptr;
size_t num_method_types = 0;
- if (dex_file->NumProtoIds() < mirror::DexCache::kDexCacheMethodTypeCacheSize) {
+ if (dex_file->NumProtoIds() < kDexCacheMethodTypeCacheSize) {
num_method_types = dex_file->NumProtoIds();
} else {
- num_method_types = mirror::DexCache::kDexCacheMethodTypeCacheSize;
+ num_method_types = kDexCacheMethodTypeCacheSize;
}
if (num_method_types > 0) {
- method_types = reinterpret_cast<mirror::MethodTypeDexCacheType*>(
+ method_types = reinterpret_cast<MethodTypeDexCacheType*>(
raw_arrays + layout.MethodTypesOffset());
}
@@ -98,13 +106,13 @@
? nullptr
: reinterpret_cast<GcRoot<mirror::CallSite>*>(raw_arrays + layout.CallSitesOffset());
- DCHECK_ALIGNED(raw_arrays, alignof(mirror::StringDexCacheType)) <<
+ DCHECK_ALIGNED(raw_arrays, alignof(StringDexCacheType)) <<
"Expected raw_arrays to align to StringDexCacheType.";
- DCHECK_ALIGNED(layout.StringsOffset(), alignof(mirror::StringDexCacheType)) <<
+ DCHECK_ALIGNED(layout.StringsOffset(), alignof(StringDexCacheType)) <<
"Expected StringsOffset() to align to StringDexCacheType.";
- DCHECK_ALIGNED(strings, alignof(mirror::StringDexCacheType)) <<
+ DCHECK_ALIGNED(strings, alignof(StringDexCacheType)) <<
"Expected strings to align to StringDexCacheType.";
- static_assert(alignof(mirror::StringDexCacheType) == 8u,
+ static_assert(alignof(StringDexCacheType) == 8u,
"Expected StringDexCacheType to have align of 8.");
if (kIsDebugBuild) {
// Sanity check to make sure all the dex cache arrays are empty. b/28992179
@@ -117,10 +125,11 @@
CHECK(types[i].load(std::memory_order_relaxed).object.IsNull());
}
for (size_t i = 0; i < dex_file->NumMethodIds(); ++i) {
- CHECK(mirror::DexCache::GetElementPtrSize(methods, i, image_pointer_size) == nullptr);
+ CHECK(GetElementPtrSize(methods, i, image_pointer_size) == nullptr);
}
- for (size_t i = 0; i < dex_file->NumFieldIds(); ++i) {
- CHECK(mirror::DexCache::GetElementPtrSize(fields, i, image_pointer_size) == nullptr);
+ for (size_t i = 0; i < num_fields; ++i) {
+ CHECK_EQ(GetNativePairPtrSize(fields, i, image_pointer_size).index, 0u);
+ CHECK(GetNativePairPtrSize(fields, i, image_pointer_size).object == nullptr);
}
for (size_t i = 0; i < num_method_types; ++i) {
CHECK_EQ(method_types[i].load(std::memory_order_relaxed).index, 0u);
@@ -136,6 +145,9 @@
if (types != nullptr) {
mirror::TypeDexCachePair::Initialize(types);
}
+ if (fields != nullptr) {
+ mirror::FieldDexCachePair::Initialize(fields, image_pointer_size);
+ }
if (method_types != nullptr) {
mirror::MethodTypeDexCachePair::Initialize(method_types);
}
@@ -148,7 +160,7 @@
methods,
dex_file->NumMethodIds(),
fields,
- dex_file->NumFieldIds(),
+ num_fields,
method_types,
num_method_types,
call_sites,
@@ -164,7 +176,7 @@
uint32_t num_resolved_types,
ArtMethod** resolved_methods,
uint32_t num_resolved_methods,
- ArtField** resolved_fields,
+ FieldDexCacheType* resolved_fields,
uint32_t num_resolved_fields,
MethodTypeDexCacheType* resolved_method_types,
uint32_t num_resolved_method_types,
@@ -218,5 +230,23 @@
SetFieldObject<false>(OFFSET_OF_OBJECT_MEMBER(DexCache, location_), location);
}
+#if !defined(__aarch64__) && !defined(__x86_64__)
+static pthread_mutex_t dex_cache_slow_atomic_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+DexCache::ConversionPair64 DexCache::AtomicLoadRelaxed16B(std::atomic<ConversionPair64>* target) {
+ pthread_mutex_lock(&dex_cache_slow_atomic_mutex);
+ DexCache::ConversionPair64 value = *reinterpret_cast<ConversionPair64*>(target);
+ pthread_mutex_unlock(&dex_cache_slow_atomic_mutex);
+ return value;
+}
+
+void DexCache::AtomicStoreRelease16B(std::atomic<ConversionPair64>* target,
+ ConversionPair64 value) {
+ pthread_mutex_lock(&dex_cache_slow_atomic_mutex);
+ *reinterpret_cast<ConversionPair64*>(target) = value;
+ pthread_mutex_unlock(&dex_cache_slow_atomic_mutex);
+}
+#endif
+
} // namespace mirror
} // namespace art
diff --git a/runtime/mirror/dex_cache.h b/runtime/mirror/dex_cache.h
index 0579198..35707ef 100644
--- a/runtime/mirror/dex_cache.h
+++ b/runtime/mirror/dex_cache.h
@@ -91,12 +91,44 @@
}
};
+template <typename T> struct PACKED(2 * __SIZEOF_POINTER__) NativeDexCachePair {
+ T* object;
+ size_t index;
+ // This is similar to DexCachePair except that we're storing a native pointer
+ // instead of a GC root. See DexCachePair for the details.
+ NativeDexCachePair(T* object, uint32_t index)
+ : object(object),
+ index(index) {}
+ NativeDexCachePair() : object(nullptr), index(0u) { }
+ NativeDexCachePair(const NativeDexCachePair<T>&) = default;
+ NativeDexCachePair& operator=(const NativeDexCachePair<T>&) = default;
+
+ static void Initialize(std::atomic<NativeDexCachePair<T>>* dex_cache, PointerSize pointer_size);
+
+ static uint32_t InvalidIndexForSlot(uint32_t slot) {
+ // Since the cache size is a power of two, 0 will always map to slot 0.
+ // Use 1 for slot 0 and 0 for all other slots.
+ return (slot == 0) ? 1u : 0u;
+ }
+
+ T* GetObjectForIndex(uint32_t idx) REQUIRES_SHARED(Locks::mutator_lock_) {
+ if (idx != index) {
+ return nullptr;
+ }
+ DCHECK(object != nullptr);
+ return object;
+ }
+};
+
using TypeDexCachePair = DexCachePair<Class>;
using TypeDexCacheType = std::atomic<TypeDexCachePair>;
using StringDexCachePair = DexCachePair<String>;
using StringDexCacheType = std::atomic<StringDexCachePair>;
+using FieldDexCachePair = NativeDexCachePair<ArtField>;
+using FieldDexCacheType = std::atomic<FieldDexCachePair>;
+
using MethodTypeDexCachePair = DexCachePair<MethodType>;
using MethodTypeDexCacheType = std::atomic<MethodTypeDexCachePair>;
@@ -116,6 +148,11 @@
static_assert(IsPowerOfTwo(kDexCacheStringCacheSize),
"String dex cache size is not a power of 2.");
+ // Size of field dex cache. Needs to be a power of 2 for entrypoint assumptions to hold.
+ static constexpr size_t kDexCacheFieldCacheSize = 1024;
+ static_assert(IsPowerOfTwo(kDexCacheFieldCacheSize),
+ "Field dex cache size is not a power of 2.");
+
// Size of method type dex cache. Needs to be a power of 2 for entrypoint assumptions
// to hold.
static constexpr size_t kDexCacheMethodTypeCacheSize = 1024;
@@ -130,6 +167,10 @@
return kDexCacheStringCacheSize;
}
+ static constexpr size_t StaticArtFieldSize() {
+ return kDexCacheFieldCacheSize;
+ }
+
static constexpr size_t StaticMethodTypeSize() {
return kDexCacheMethodTypeCacheSize;
}
@@ -255,6 +296,8 @@
// Pointer sized variant, used for patching.
ALWAYS_INLINE void SetResolvedField(uint32_t idx, ArtField* field, PointerSize ptr_size)
REQUIRES_SHARED(Locks::mutator_lock_);
+ ALWAYS_INLINE void ClearResolvedField(uint32_t idx, PointerSize ptr_size)
+ REQUIRES_SHARED(Locks::mutator_lock_);
MethodType* GetResolvedMethodType(uint32_t proto_idx) REQUIRES_SHARED(Locks::mutator_lock_);
@@ -299,11 +342,11 @@
SetFieldPtr<false>(ResolvedMethodsOffset(), resolved_methods);
}
- ArtField** GetResolvedFields() ALWAYS_INLINE REQUIRES_SHARED(Locks::mutator_lock_) {
- return GetFieldPtr<ArtField**>(ResolvedFieldsOffset());
+ FieldDexCacheType* GetResolvedFields() ALWAYS_INLINE REQUIRES_SHARED(Locks::mutator_lock_) {
+ return GetFieldPtr<FieldDexCacheType*>(ResolvedFieldsOffset());
}
- void SetResolvedFields(ArtField** resolved_fields)
+ void SetResolvedFields(FieldDexCacheType* resolved_fields)
ALWAYS_INLINE
REQUIRES_SHARED(Locks::mutator_lock_) {
SetFieldPtr<false>(ResolvedFieldsOffset(), resolved_fields);
@@ -376,6 +419,17 @@
template <typename PtrType>
static void SetElementPtrSize(PtrType* ptr_array, size_t idx, PtrType ptr, PointerSize ptr_size);
+ template <typename T>
+ static NativeDexCachePair<T> GetNativePairPtrSize(std::atomic<NativeDexCachePair<T>>* pair_array,
+ size_t idx,
+ PointerSize ptr_size);
+
+ template <typename T>
+ static void SetNativePairPtrSize(std::atomic<NativeDexCachePair<T>>* pair_array,
+ size_t idx,
+ NativeDexCachePair<T> pair,
+ PointerSize ptr_size);
+
private:
void Init(const DexFile* dex_file,
ObjPtr<String> location,
@@ -385,7 +439,7 @@
uint32_t num_resolved_types,
ArtMethod** resolved_methods,
uint32_t num_resolved_methods,
- ArtField** resolved_fields,
+ FieldDexCacheType* resolved_fields,
uint32_t num_resolved_fields,
MethodTypeDexCacheType* resolved_method_types,
uint32_t num_resolved_method_types,
@@ -394,8 +448,22 @@
PointerSize pointer_size)
REQUIRES_SHARED(Locks::mutator_lock_);
+ // std::pair<> is not trivially copyable and as such it is unsuitable for atomic operations,
+ // so we use a custom pair class for loading and storing the NativeDexCachePair<>.
+ template <typename IntType>
+ struct PACKED(2 * sizeof(IntType)) ConversionPair {
+ ConversionPair(IntType f, IntType s) : first(f), second(s) { }
+ ConversionPair(const ConversionPair&) = default;
+ ConversionPair& operator=(const ConversionPair&) = default;
+ IntType first;
+ IntType second;
+ };
+ using ConversionPair32 = ConversionPair<uint32_t>;
+ using ConversionPair64 = ConversionPair<uint64_t>;
+
uint32_t StringSlotIndex(dex::StringIndex string_idx) REQUIRES_SHARED(Locks::mutator_lock_);
uint32_t TypeSlotIndex(dex::TypeIndex type_idx) REQUIRES_SHARED(Locks::mutator_lock_);
+ uint32_t FieldSlotIndex(uint32_t field_idx) REQUIRES_SHARED(Locks::mutator_lock_);
uint32_t MethodTypeSlotIndex(uint32_t proto_idx) REQUIRES_SHARED(Locks::mutator_lock_);
// Visit instance fields of the dex cache as well as its associated arrays.
@@ -406,12 +474,55 @@
void VisitReferences(ObjPtr<Class> klass, const Visitor& visitor)
REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_);
+ // Due to lack of 16-byte atomics support, we use hand-crafted routines.
+#if defined(__aarch64__)
+ // 16-byte atomics are supported on aarch64.
+ ALWAYS_INLINE static ConversionPair64 AtomicLoadRelaxed16B(
+ std::atomic<ConversionPair64>* target) {
+ return target->load(std::memory_order_relaxed);
+ }
+
+ ALWAYS_INLINE static void AtomicStoreRelease16B(
+ std::atomic<ConversionPair64>* target, ConversionPair64 value) {
+ target->store(value, std::memory_order_release);
+ }
+#elif defined(__x86_64__)
+ ALWAYS_INLINE static ConversionPair64 AtomicLoadRelaxed16B(
+ std::atomic<ConversionPair64>* target) {
+ uint64_t first, second;
+ __asm__ __volatile__(
+ "lock cmpxchg16b (%2)"
+ : "=&a"(first), "=&d"(second)
+ : "r"(target), "a"(0), "d"(0), "b"(0), "c"(0)
+ : "cc");
+ return ConversionPair64(first, second);
+ }
+
+ ALWAYS_INLINE static void AtomicStoreRelease16B(
+ std::atomic<ConversionPair64>* target, ConversionPair64 value) {
+ uint64_t first, second;
+ __asm__ __volatile__ (
+ "movq (%2), %%rax\n\t"
+ "movq 8(%2), %%rdx\n\t"
+ "1:\n\t"
+ "lock cmpxchg16b (%2)\n\t"
+ "jnz 1b"
+ : "=&a"(first), "=&d"(second)
+ : "r"(target), "b"(value.first), "c"(value.second)
+ : "cc");
+ }
+#else
+ static ConversionPair64 AtomicLoadRelaxed16B(std::atomic<ConversionPair64>* target);
+ static void AtomicStoreRelease16B(std::atomic<ConversionPair64>* target, ConversionPair64 value);
+#endif
+
HeapReference<Object> dex_;
HeapReference<String> location_;
uint64_t dex_file_; // const DexFile*
uint64_t resolved_call_sites_; // GcRoot<CallSite>* array with num_resolved_call_sites_
// elements.
- uint64_t resolved_fields_; // ArtField*, array with num_resolved_fields_ elements.
+ uint64_t resolved_fields_; // std::atomic<FieldDexCachePair>*, array with
+ // num_resolved_fields_ elements.
uint64_t resolved_method_types_; // std::atomic<MethodTypeDexCachePair>* array with
// num_resolved_method_types_ elements.
uint64_t resolved_methods_; // ArtMethod*, array with num_resolved_methods_ elements.
diff --git a/runtime/mirror/dex_cache_test.cc b/runtime/mirror/dex_cache_test.cc
index ef0aaaa..71a47f6 100644
--- a/runtime/mirror/dex_cache_test.cc
+++ b/runtime/mirror/dex_cache_test.cc
@@ -54,7 +54,8 @@
EXPECT_TRUE(dex_cache->StaticTypeSize() == dex_cache->NumResolvedTypes()
|| java_lang_dex_file_->NumTypeIds() == dex_cache->NumResolvedTypes());
EXPECT_EQ(java_lang_dex_file_->NumMethodIds(), dex_cache->NumResolvedMethods());
- EXPECT_EQ(java_lang_dex_file_->NumFieldIds(), dex_cache->NumResolvedFields());
+ EXPECT_TRUE(dex_cache->StaticArtFieldSize() == dex_cache->NumResolvedFields()
+ || java_lang_dex_file_->NumFieldIds() == dex_cache->NumResolvedFields());
EXPECT_TRUE(dex_cache->StaticMethodTypeSize() == dex_cache->NumResolvedMethodTypes()
|| java_lang_dex_file_->NumProtoIds() == dex_cache->NumResolvedMethodTypes());
}
diff --git a/runtime/mirror/field.cc b/runtime/mirror/field.cc
index f6b6489..54034c2 100644
--- a/runtime/mirror/field.cc
+++ b/runtime/mirror/field.cc
@@ -68,8 +68,16 @@
}
}
mirror::DexCache* const dex_cache = declaring_class->GetDexCache();
- ArtField* const art_field = dex_cache->GetResolvedField(GetDexFieldIndex(), kRuntimePointerSize);
- CHECK(art_field != nullptr);
+ ArtField* art_field = dex_cache->GetResolvedField(GetDexFieldIndex(), kRuntimePointerSize);
+ if (UNLIKELY(art_field == nullptr)) {
+ if (IsStatic()) {
+ art_field = declaring_class->FindDeclaredStaticField(dex_cache, GetDexFieldIndex());
+ } else {
+ art_field = declaring_class->FindInstanceField(dex_cache, GetDexFieldIndex());
+ }
+ CHECK(art_field != nullptr);
+ dex_cache->SetResolvedField(GetDexFieldIndex(), art_field, kRuntimePointerSize);
+ }
CHECK_EQ(declaring_class, art_field->GetDeclaringClass());
return art_field;
}
diff --git a/runtime/native/dalvik_system_VMRuntime.cc b/runtime/native/dalvik_system_VMRuntime.cc
index 9b707f8..d81c13d 100644
--- a/runtime/native/dalvik_system_VMRuntime.cc
+++ b/runtime/native/dalvik_system_VMRuntime.cc
@@ -444,6 +444,7 @@
if (!kPreloadDexCachesCollectStats) {
return;
}
+ // TODO: Update for hash-based DexCache arrays.
ClassLinker* const class_linker = Runtime::Current()->GetClassLinker();
Thread* const self = Thread::Current();
for (const DexFile* dex_file : class_linker->GetBootClassPath()) {
@@ -463,7 +464,7 @@
}
}
for (size_t j = 0; j < dex_cache->NumResolvedFields(); j++) {
- ArtField* field = class_linker->GetResolvedField(j, dex_cache);
+ ArtField* field = dex_cache->GetResolvedField(j, class_linker->GetImagePointerSize());
if (field != nullptr) {
filled->num_fields++;
}
diff --git a/runtime/oat.h b/runtime/oat.h
index 1544121..df43107 100644
--- a/runtime/oat.h
+++ b/runtime/oat.h
@@ -32,7 +32,7 @@
class PACKED(4) OatHeader {
public:
static constexpr uint8_t kOatMagic[] = { 'o', 'a', 't', '\n' };
- static constexpr uint8_t kOatVersion[] = { '1', '1', '4', '\0' }; // hash-based DexCache types.
+ static constexpr uint8_t kOatVersion[] = { '1', '1', '5', '\0' }; // hash-based DexCache fields
static constexpr const char* kImageLocationKey = "image-location";
static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline";
diff --git a/runtime/openjdkjvmti/ti_redefine.cc b/runtime/openjdkjvmti/ti_redefine.cc
index a173a4ab..9c1d6ef 100644
--- a/runtime/openjdkjvmti/ti_redefine.cc
+++ b/runtime/openjdkjvmti/ti_redefine.cc
@@ -178,7 +178,7 @@
art::ClassLinker* cl = runtime->GetClassLinker();
auto ptr_size = cl->GetImagePointerSize();
const size_t method_size = art::ArtMethod::Size(ptr_size);
- auto* method_storage = allocator_->Alloc(GetThread(), method_size);
+ auto* method_storage = allocator_->Alloc(art::Thread::Current(), method_size);
CHECK(method_storage != nullptr) << "Unable to allocate storage for obsolete version of '"
<< old_method->PrettyMethod() << "'";
new_obsolete_method = new (method_storage) art::ArtMethod();
diff --git a/runtime/quick/inline_method_analyser.cc b/runtime/quick/inline_method_analyser.cc
index b009b47..3347070 100644
--- a/runtime/quick/inline_method_analyser.cc
+++ b/runtime/quick/inline_method_analyser.cc
@@ -215,9 +215,8 @@
REQUIRES_SHARED(Locks::mutator_lock_) {
DCHECK(IsInstructionIPut(new_iput->Opcode()));
uint32_t field_index = new_iput->VRegC_22c();
- PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize();
- mirror::DexCache* dex_cache = method->GetDexCache();
- ArtField* field = dex_cache->GetResolvedField(field_index, pointer_size);
+ ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
+ ArtField* field = class_linker->LookupResolvedField(field_index, method, /* is_static */ false);
if (UNLIKELY(field == nullptr)) {
return false;
}
@@ -227,7 +226,9 @@
if (iputs[old_pos].field_index == DexFile::kDexNoIndex16) {
break;
}
- ArtField* f = dex_cache->GetResolvedField(iputs[old_pos].field_index, pointer_size);
+ ArtField* f = class_linker->LookupResolvedField(iputs[old_pos].field_index,
+ method,
+ /* is_static */ false);
DCHECK(f != nullptr);
if (f == field) {
auto back_it = std::copy(iputs + old_pos + 1, iputs + arraysize(iputs), iputs + old_pos);
@@ -732,9 +733,9 @@
if (method == nullptr) {
return false;
}
- mirror::DexCache* dex_cache = method->GetDexCache();
- PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize();
- ArtField* field = dex_cache->GetResolvedField(field_idx, pointer_size);
+ ObjPtr<mirror::DexCache> dex_cache = method->GetDexCache();
+ ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
+ ArtField* field = class_linker->LookupResolvedField(field_idx, method, /* is_static */ false);
if (field == nullptr || field->IsStatic()) {
return false;
}
diff --git a/runtime/thread.cc b/runtime/thread.cc
index ff66cc1..30a4046 100644
--- a/runtime/thread.cc
+++ b/runtime/thread.cc
@@ -1934,7 +1934,6 @@
wait_cond_ = new ConditionVariable("a thread wait condition variable", *wait_mutex_);
tlsPtr_.instrumentation_stack = new std::deque<instrumentation::InstrumentationStackFrame>;
tlsPtr_.name = new std::string(kThreadNameDuringStartup);
- tlsPtr_.nested_signal_state = static_cast<jmp_buf*>(malloc(sizeof(jmp_buf)));
static_assert((sizeof(Thread) % 4) == 0U,
"art::Thread has a size which is not a multiple of 4.");
@@ -2118,7 +2117,6 @@
delete tlsPtr_.instrumentation_stack;
delete tlsPtr_.name;
delete tlsPtr_.deps_or_stack_trace_sample.stack_trace_sample;
- free(tlsPtr_.nested_signal_state);
Runtime::Current()->GetHeap()->AssertThreadLocalBuffersAreRevoked(this);
diff --git a/runtime/thread.h b/runtime/thread.h
index d5fd9e9..de0b892 100644
--- a/runtime/thread.h
+++ b/runtime/thread.h
@@ -1115,21 +1115,12 @@
return tlsPtr_.mterp_alt_ibase;
}
- // Notify that a signal is being handled. This is to protect us from doing recursive
- // NPE handling after a SIGSEGV.
- void NoteSignalBeingHandled() {
- if (tls32_.handling_signal_) {
- LOG(FATAL) << "Detected signal while processing a signal";
- }
- tls32_.handling_signal_ = true;
+ bool HandlingSignal() const {
+ return tls32_.handling_signal_;
}
- void NoteSignalHandlerDone() {
- tls32_.handling_signal_ = false;
- }
-
- jmp_buf* GetNestedSignalState() {
- return tlsPtr_.nested_signal_state;
+ void SetHandlingSignal(bool handling_signal) {
+ tls32_.handling_signal_ = handling_signal;
}
bool IsTransitioningToRunnable() const {
@@ -1460,7 +1451,7 @@
thread_local_start(nullptr), thread_local_pos(nullptr), thread_local_end(nullptr),
thread_local_objects(0), mterp_current_ibase(nullptr), mterp_default_ibase(nullptr),
mterp_alt_ibase(nullptr), thread_local_alloc_stack_top(nullptr),
- thread_local_alloc_stack_end(nullptr), nested_signal_state(nullptr),
+ thread_local_alloc_stack_end(nullptr),
flip_function(nullptr), method_verifier(nullptr), thread_local_mark_stack(nullptr) {
std::fill(held_mutexes, held_mutexes + kLockLevelCount, nullptr);
}
@@ -1606,9 +1597,6 @@
// Support for Mutex lock hierarchy bug detection.
BaseMutex* held_mutexes[kLockLevelCount];
- // Recorded thread state for nested signals.
- jmp_buf* nested_signal_state;
-
// The function used for thread flip.
Closure* flip_function;
diff --git a/runtime/type_lookup_table.h b/runtime/type_lookup_table.h
index 3f6f76f..fd68deb 100644
--- a/runtime/type_lookup_table.h
+++ b/runtime/type_lookup_table.h
@@ -148,7 +148,7 @@
return mask_;
}
- // Attempt to set an entry on it's hash' slot. If there is alrady something there, return false.
+ // Attempt to set an entry on its hash's slot. If there is already something there, return false.
// Otherwise return true.
bool SetOnInitialPos(const Entry& entry, uint32_t hash);
diff --git a/runtime/utils/dex_cache_arrays_layout-inl.h b/runtime/utils/dex_cache_arrays_layout-inl.h
index f9a1405..95904af 100644
--- a/runtime/utils/dex_cache_arrays_layout-inl.h
+++ b/runtime/utils/dex_cache_arrays_layout-inl.h
@@ -51,7 +51,11 @@
: DexCacheArraysLayout(pointer_size, dex_file->GetHeader(), dex_file->NumCallSiteIds()) {
}
-constexpr size_t DexCacheArraysLayout::Alignment() {
+inline size_t DexCacheArraysLayout::Alignment() const {
+ return Alignment(pointer_size_);
+}
+
+inline constexpr size_t DexCacheArraysLayout::Alignment(PointerSize pointer_size) {
// mirror::Type/String/MethodTypeDexCacheType alignment is 8,
// i.e. higher than or equal to the pointer alignment.
static_assert(alignof(mirror::TypeDexCacheType) == 8,
@@ -60,8 +64,8 @@
"Expecting alignof(StringDexCacheType) == 8");
static_assert(alignof(mirror::MethodTypeDexCacheType) == 8,
"Expecting alignof(MethodTypeDexCacheType) == 8");
- // This is the same as alignof(MethodTypeDexCacheType).
- return alignof(mirror::StringDexCacheType);
+ // This is the same as alignof(FieldDexCacheType) for the given pointer size.
+ return 2u * static_cast<size_t>(pointer_size);
}
template <typename T>
@@ -100,8 +104,8 @@
}
inline size_t DexCacheArraysLayout::StringOffset(uint32_t string_idx) const {
- return strings_offset_ + ElementOffset(PointerSize::k64,
- string_idx % mirror::DexCache::kDexCacheStringCacheSize);
+ uint32_t string_hash = string_idx % mirror::DexCache::kDexCacheStringCacheSize;
+ return strings_offset_ + ElementOffset(PointerSize::k64, string_hash);
}
inline size_t DexCacheArraysLayout::StringsSize(size_t num_elements) const {
@@ -119,15 +123,20 @@
}
inline size_t DexCacheArraysLayout::FieldOffset(uint32_t field_idx) const {
- return fields_offset_ + ElementOffset(pointer_size_, field_idx);
+ uint32_t field_hash = field_idx % mirror::DexCache::kDexCacheFieldCacheSize;
+ return fields_offset_ + 2u * static_cast<size_t>(pointer_size_) * field_hash;
}
inline size_t DexCacheArraysLayout::FieldsSize(size_t num_elements) const {
- return ArraySize(pointer_size_, num_elements);
+ size_t cache_size = mirror::DexCache::kDexCacheFieldCacheSize;
+ if (num_elements < cache_size) {
+ cache_size = num_elements;
+ }
+ return 2u * static_cast<size_t>(pointer_size_) * num_elements;
}
inline size_t DexCacheArraysLayout::FieldsAlignment() const {
- return static_cast<size_t>(pointer_size_);
+ return 2u * static_cast<size_t>(pointer_size_);
}
inline size_t DexCacheArraysLayout::MethodTypesSize(size_t num_elements) const {
diff --git a/runtime/utils/dex_cache_arrays_layout.h b/runtime/utils/dex_cache_arrays_layout.h
index ed677ed..377a374 100644
--- a/runtime/utils/dex_cache_arrays_layout.h
+++ b/runtime/utils/dex_cache_arrays_layout.h
@@ -57,7 +57,9 @@
return size_;
}
- static constexpr size_t Alignment();
+ size_t Alignment() const;
+
+ static constexpr size_t Alignment(PointerSize pointer_size);
size_t TypesOffset() const {
return types_offset_;
@@ -125,8 +127,6 @@
const size_t call_sites_offset_;
const size_t size_;
- static size_t Alignment(PointerSize pointer_size);
-
static size_t ElementOffset(PointerSize element_size, uint32_t idx);
static size_t ArraySize(PointerSize element_size, uint32_t num_elements);
diff --git a/test/538-checker-embed-constants/src/Main.java b/test/538-checker-embed-constants/src/Main.java
index 4f34ec9..94aad9d 100644
--- a/test/538-checker-embed-constants/src/Main.java
+++ b/test/538-checker-embed-constants/src/Main.java
@@ -37,13 +37,20 @@
}
/// CHECK-START-ARM: int Main.and511(int) disassembly (after)
- /// CHECK: mov {{r\d+}}, #511
- /// CHECK: and{{(\.w)?}} {{r\d+}}, {{r\d+}}, {{r\d+}}
+ /// CHECK: ubfx {{r\d+}}, {{r\d+}}, #0, #9
public static int and511(int arg) {
return arg & 511;
}
+ /// CHECK-START-ARM: int Main.andF00D(int) disassembly (after)
+ /// CHECK: mov {{r\d+}}, #61453
+ /// CHECK: and{{(\.w)?}} {{r\d+}}, {{r\d+}}, {{r\d+}}
+
+ public static int andF00D(int arg) {
+ return arg & 0xF00D;
+ }
+
/// CHECK-START-ARM: int Main.andNot15(int) disassembly (after)
/// CHECK-NOT: mvn {{r\d+}}, #15
/// CHECK: bic {{r\d+}}, {{r\d+}}, #0xf
@@ -114,19 +121,31 @@
}
/// CHECK-START-ARM: long Main.and511(long) disassembly (after)
- /// CHECK: mov {{r\d+}}, #511
+ /// CHECK: ubfx {{r\d+}}, {{r\d+}}, #0, #9
/// CHECK-NEXT: mov{{s?}} {{r\d+}}, #0
/// CHECK-NOT: and{{(\.w)?}}
/// CHECK-NOT: bic{{(\.w)?}}
- /// CHECK: and{{(\.w)?}} {{r\d+}}, {{r\d+}}, {{r\d+}}
- /// CHECK-NEXT: and{{(\.w)?}} {{r\d+}}, {{r\d+}}, {{r\d+}}
- /// CHECK-NOT: and{{(\.w)?}}
- /// CHECK-NOT: bic{{(\.w)?}}
public static long and511(long arg) {
return arg & 511L;
}
+ /// CHECK-START-ARM: long Main.andF00D(long) disassembly (after)
+ /// CHECK: mov {{r\d+}}, #61453
+ /// CHECK-NEXT: mov{{s?}} {{r\d+}}, #0
+ /// CHECK-NOT: and{{(\.w)?}}
+ /// CHECK-NOT: bic{{(\.w)?}}
+ /// CHECK-NOT: ubfx
+ /// CHECK: and{{(\.w)?}} {{r\d+}}, {{r\d+}}, {{r\d+}}
+ /// CHECK-NEXT: and{{(\.w)?}} {{r\d+}}, {{r\d+}}, {{r\d+}}
+ /// CHECK-NOT: and{{(\.w)?}}
+ /// CHECK-NOT: bic{{(\.w)?}}
+ /// CHECK-NOT: ubfx
+
+ public static long andF00D(long arg) {
+ return arg & 0xF00DL;
+ }
+
/// CHECK-START-ARM: long Main.andNot15(long) disassembly (after)
/// CHECK-NOT: mvn {{r\d+}}, #15
/// CHECK-NOT: and{{(\.w)?}}
@@ -631,6 +650,7 @@
int arg = 0x87654321;
assertIntEquals(and255(arg), 0x21);
assertIntEquals(and511(arg), 0x121);
+ assertIntEquals(andF00D(arg), 0x4001);
assertIntEquals(andNot15(arg), 0x87654320);
assertIntEquals(or255(arg), 0x876543ff);
assertIntEquals(or511(arg), 0x876543ff);
@@ -642,6 +662,7 @@
long longArg = 0x1234567887654321L;
assertLongEquals(and255(longArg), 0x21L);
assertLongEquals(and511(longArg), 0x121L);
+ assertLongEquals(andF00D(longArg), 0x4001L);
assertLongEquals(andNot15(longArg), 0x1234567887654320L);
assertLongEquals(and0xfffffff00000000f(longArg), 0x1234567000000001L);
assertLongEquals(or255(longArg), 0x12345678876543ffL);
diff --git a/test/570-checker-select/src/Main.java b/test/570-checker-select/src/Main.java
index e0a76ca..3ac6f89 100644
--- a/test/570-checker-select/src/Main.java
+++ b/test/570-checker-select/src/Main.java
@@ -371,6 +371,49 @@
return a > b ? x : y;
}
+ /// CHECK-START-ARM: long Main.$noinline$LongEqNonmatCond_LongVarVar(long, long, long, long) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp {{r\d+}}, {{r\d+}}
+ /// CHECK-NEXT: it eq
+ /// CHECK-NEXT: cmpeq {{r\d+}}, {{r\d+}}
+ /// CHECK-NEXT: it eq
+
+ public static long $noinline$LongEqNonmatCond_LongVarVar(long a, long b, long x, long y) {
+ return a == b ? x : y;
+ }
+
+ /// CHECK-START-ARM: long Main.$noinline$LongNonmatCondCst_LongVarVar(long, long, long) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: mov ip, #52720
+ /// CHECK-NEXT: movt ip, #35243
+ /// CHECK-NEXT: cmp {{r\d+}}, ip
+ /// CHECK-NEXT: sbcs ip, {{r\d+}}, #{{\d+}}
+ /// CHECK-NEXT: it ge
+
+ public static long $noinline$LongNonmatCondCst_LongVarVar(long a, long x, long y) {
+ return a > 0x89ABCDEFL ? x : y;
+ }
+
+ /// CHECK-START-ARM: long Main.$noinline$LongNonmatCondCst_LongVarVar2(long, long, long) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: mov ip, #{{\d+}}
+ /// CHECK-NEXT: movt ip, #{{\d+}}
+ /// CHECK-NEXT: cmp {{r\d+}}, ip
+
+ public static long $noinline$LongNonmatCondCst_LongVarVar2(long a, long x, long y) {
+ return a > 0x0123456789ABCDEFL ? x : y;
+ }
+
+ /// CHECK-START-ARM: long Main.$noinline$LongNonmatCondCst_LongVarVar3(long, long, long) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp {{r\d+}}, {{r\d+}}
+ /// CHECK-NOT: sbcs
+ /// CHECK-NOT: cmp
+
+ public static long $noinline$LongNonmatCondCst_LongVarVar3(long a, long x, long y) {
+ return a > 0x7FFFFFFFFFFFFFFFL ? x : y;
+ }
+
/// CHECK-START: long Main.LongMatCond_LongVarVar(long, long, long, long) register (after)
/// CHECK: <<Cond:z\d+>> LessThanOrEqual [{{j\d+}},{{j\d+}}]
/// CHECK: <<Sel1:j\d+>> Select [{{j\d+}},{{j\d+}},<<Cond>>]
@@ -612,6 +655,39 @@
assertEqual(5, IntMatCond_IntVarVar(3, 2, 5, 7));
assertEqual(8, IntMatCond_IntVarVar(2, 3, 5, 7));
+ assertEqual(0xAAAAAAAA55555555L,
+ LongNonmatCond_LongVarVar(3L, 2L, 0xAAAAAAAA55555555L, 0x8888888877777777L));
+ assertEqual(0x8888888877777777L,
+ LongNonmatCond_LongVarVar(2L, 2L, 0xAAAAAAAA55555555L, 0x8888888877777777L));
+ assertEqual(0x8888888877777777L,
+ LongNonmatCond_LongVarVar(2L, 3L, 0xAAAAAAAA55555555L, 0x8888888877777777L));
+ assertEqual(0xAAAAAAAA55555555L, LongNonmatCond_LongVarVar(0x0000000100000000L,
+ 0x00000000FFFFFFFFL,
+ 0xAAAAAAAA55555555L,
+ 0x8888888877777777L));
+ assertEqual(0x8888888877777777L, LongNonmatCond_LongVarVar(0x00000000FFFFFFFFL,
+ 0x0000000100000000L,
+ 0xAAAAAAAA55555555L,
+ 0x8888888877777777L));
+
+ assertEqual(0x8888888877777777L, $noinline$LongEqNonmatCond_LongVarVar(2L,
+ 3L,
+ 0xAAAAAAAA55555555L,
+ 0x8888888877777777L));
+ assertEqual(0xAAAAAAAA55555555L, $noinline$LongEqNonmatCond_LongVarVar(2L,
+ 2L,
+ 0xAAAAAAAA55555555L,
+ 0x8888888877777777L));
+ assertEqual(0x8888888877777777L, $noinline$LongEqNonmatCond_LongVarVar(0x10000000000L,
+ 0L,
+ 0xAAAAAAAA55555555L,
+ 0x8888888877777777L));
+
+ assertEqual(5L, $noinline$LongNonmatCondCst_LongVarVar2(0x7FFFFFFFFFFFFFFFL, 5L, 7L));
+ assertEqual(7L, $noinline$LongNonmatCondCst_LongVarVar2(2L, 5L, 7L));
+
+ assertEqual(7L, $noinline$LongNonmatCondCst_LongVarVar3(2L, 5L, 7L));
+
assertEqual(5, FloatLtNonmatCond_IntVarVar(3, 2, 5, 7));
assertEqual(7, FloatLtNonmatCond_IntVarVar(2, 3, 5, 7));
assertEqual(7, FloatLtNonmatCond_IntVarVar(Float.NaN, 2, 5, 7));
diff --git a/test/626-checker-arm64-scratch-register/src/Main.java b/test/626-checker-arm64-scratch-register/src/Main.java
index 6dd4374..1394917 100644
--- a/test/626-checker-arm64-scratch-register/src/Main.java
+++ b/test/626-checker-arm64-scratch-register/src/Main.java
@@ -70,7 +70,7 @@
/// CHECK: end_block
/// CHECK: begin_block
/// CHECK: name "<<ElseBlock>>"
- /// CHECK: ParallelMove moves:[#100->d17,32(sp)->d1,36(sp)->d2,d17->d3,d3->d4,d4->d5,d5->d6,d6->d7,d7->d18,d18->d19,d19->d20,d20->d21,d21->d22,d22->d23,d23->d10,d10->d11,d11->d12,24(sp)->d13,28(sp)->d14,d14->16(sp),d12->20(sp),d13->24(sp),d1->28(sp),d2->32(sp),16(sp)->36(sp),20(sp)->40(sp)]
+ /// CHECK: ParallelMove moves:[40(sp)->d0,24(sp)->32(sp),28(sp)->36(sp),d0->d3,d3->d4,d2->d5,d4->d6,d5->d7,d6->d18,d7->d19,d18->d20,d19->d21,d20->d22,d21->d23,d22->d10,d23->d11,16(sp)->24(sp),20(sp)->28(sp),d10->d14,d11->d12,d12->d13,d13->d1,d14->d2,32(sp)->16(sp),36(sp)->20(sp)]
/// CHECK: end_block
/// CHECK-START-ARM64: void Main.test() disassembly (after)
@@ -85,7 +85,7 @@
/// CHECK: end_block
/// CHECK: begin_block
/// CHECK: name "<<ElseBlock>>"
- /// CHECK: ParallelMove moves:[invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid]
+ /// CHECK: ParallelMove moves:[invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid]
/// CHECK: fmov d31, d2
/// CHECK: ldr s2, [sp, #36]
/// CHECK: ldr w16, [sp, #16]
@@ -111,11 +111,10 @@
/// CHECK: fmov d6, d5
/// CHECK: fmov d5, d4
/// CHECK: fmov d4, d3
- /// CHECK: fmov d3, d17
- /// CHECK: fmov d17, d13
+ /// CHECK: fmov d3, d13
/// CHECK: ldr s13, [sp, #24]
- /// CHECK: str s17, [sp, #24]
- /// CHECK: ldr s17, pc+{{\d+}} (addr {{0x[0-9a-f]+}}) (100)
+ /// CHECK: str s3, [sp, #24]
+ /// CHECK: ldr s3, pc+{{\d+}} (addr {{0x[0-9a-f]+}}) (100)
/// CHECK: end_block
public void test() {
diff --git a/test/638-checker-inline-caches/expected.txt b/test/638-checker-inline-caches/expected.txt
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/638-checker-inline-caches/expected.txt
diff --git a/test/638-checker-inline-caches/info.txt b/test/638-checker-inline-caches/info.txt
new file mode 100644
index 0000000..1fac628
--- /dev/null
+++ b/test/638-checker-inline-caches/info.txt
@@ -0,0 +1 @@
+Verify the use of inline caches in AOT mode.
diff --git a/test/638-checker-inline-caches/multidex.jpp b/test/638-checker-inline-caches/multidex.jpp
new file mode 100644
index 0000000..69a2cc1
--- /dev/null
+++ b/test/638-checker-inline-caches/multidex.jpp
@@ -0,0 +1,12 @@
+Main:
+ @@com.android.jack.annotations.ForceInMainDex
+ class Main
+Super:
+ @@com.android.jack.annotations.ForceInMainDex
+ class Super
+SubA:
+ @@com.android.jack.annotations.ForceInMainDex
+ class SubA
+SubB
+ @@com.android.jack.annotations.ForceInMainDex
+ class SubB
diff --git a/test/638-checker-inline-caches/profile b/test/638-checker-inline-caches/profile
new file mode 100644
index 0000000..1ca6d7b
--- /dev/null
+++ b/test/638-checker-inline-caches/profile
@@ -0,0 +1,6 @@
+LMain;->inlineMonomorphicSubA(LSuper;)I+LSubA;
+LMain;->inlinePolymophicSubASubB(LSuper;)I+LSubA;,LSubB;
+LMain;->inlinePolymophicCrossDexSubASubC(LSuper;)I+LSubA;,LSubC;
+LMain;->inlineMegamorphic(LSuper;)I+LSubA;,LSubB;,LSubC;,LSubD;,LSubE;
+LMain;->inlineMissingTypes(LSuper;)I+missing_types
+LMain;->noInlineCache(LSuper;)I
diff --git a/test/638-checker-inline-caches/run b/test/638-checker-inline-caches/run
new file mode 100644
index 0000000..146e180
--- /dev/null
+++ b/test/638-checker-inline-caches/run
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+exec ${RUN} $@ --profile -Xcompiler-option --compiler-filter=speed-profile
diff --git a/test/638-checker-inline-caches/src-multidex/SubC.java b/test/638-checker-inline-caches/src-multidex/SubC.java
new file mode 100644
index 0000000..f7e3c08
--- /dev/null
+++ b/test/638-checker-inline-caches/src-multidex/SubC.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class SubC extends Super {
+ public int getValue() { return 24; }
+}
diff --git a/test/638-checker-inline-caches/src/Main.java b/test/638-checker-inline-caches/src/Main.java
new file mode 100644
index 0000000..2cee47e
--- /dev/null
+++ b/test/638-checker-inline-caches/src/Main.java
@@ -0,0 +1,192 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+class SubA extends Super {
+ int getValue() { return 42; }
+}
+
+class SubB extends Super {
+ int getValue() { return 38; }
+}
+
+class SubD extends Super {
+ int getValue() { return 10; }
+}
+
+class SubE extends Super {
+ int getValue() { return -4; }
+}
+
+public class Main {
+
+ /// CHECK-START: int Main.inlineMonomorphicSubA(Super) inliner (before)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.inlineMonomorphicSubA(Super) inliner (after)
+ /// CHECK-NOT: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.inlineMonomorphicSubA(Super) inliner (after)
+ /// CHECK: <<SubARet:i\d+>> IntConstant 42
+ /// CHECK: <<ObjClass:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_
+ /// CHECK: <<InlineClass:l\d+>> LoadClass class_name:SubA
+ /// CHECK: <<Test:z\d+>> NotEqual [<<InlineClass>>,<<ObjClass>>]
+ /// CHECK: Deoptimize [<<Test>>]
+ /// CHECK: Return [<<SubARet>>]
+ public static int inlineMonomorphicSubA(Super a) {
+ return a.getValue();
+ }
+
+ /// CHECK-START: int Main.inlinePolymophicSubASubB(Super) inliner (before)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.inlinePolymophicSubASubB(Super) inliner (after)
+ /// CHECK-NOT: InvokeVirtual method_name:Super.getValue
+
+ // Note that the order in which the types are added to the inline cache in the profile matters.
+
+ /// CHECK-START: int Main.inlinePolymophicSubASubB(Super) inliner (after)
+ /// CHECK-DAG: <<SubARet:i\d+>> IntConstant 42
+ /// CHECK-DAG: <<SubBRet:i\d+>> IntConstant 38
+ /// CHECK: <<ObjClassSubA:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_
+ /// CHECK: <<InlineClassSubA:l\d+>> LoadClass class_name:SubA
+ /// CHECK: <<TestSubA:z\d+>> NotEqual [<<InlineClassSubA>>,<<ObjClassSubA>>]
+ /// CHECK: If [<<TestSubA>>]
+
+ /// CHECK: <<ObjClassSubB:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_
+ /// CHECK: <<InlineClassSubB:l\d+>> LoadClass class_name:SubB
+ /// CHECK: <<TestSubB:z\d+>> NotEqual [<<InlineClassSubB>>,<<ObjClassSubB>>]
+ /// CHECK: Deoptimize [<<TestSubB>>]
+
+ /// CHECK: <<Ret:i\d+>> Phi [<<SubARet>>,<<SubBRet>>]
+ /// CHECK: Return [<<Ret>>]
+ public static int inlinePolymophicSubASubB(Super a) {
+ return a.getValue();
+ }
+
+ /// CHECK-START: int Main.inlinePolymophicCrossDexSubASubC(Super) inliner (before)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.inlinePolymophicCrossDexSubASubC(Super) inliner (after)
+ /// CHECK-NOT: InvokeVirtual method_name:Super.getValue
+
+ // Note that the order in which the types are added to the inline cache in the profile matters.
+
+ /// CHECK-START: int Main.inlinePolymophicCrossDexSubASubC(Super) inliner (after)
+ /// CHECK-DAG: <<SubARet:i\d+>> IntConstant 42
+ /// CHECK-DAG: <<SubCRet:i\d+>> IntConstant 24
+ /// CHECK: <<ObjClassSubA:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_
+ /// CHECK: <<InlineClassSubA:l\d+>> LoadClass class_name:SubA
+ /// CHECK: <<TestSubA:z\d+>> NotEqual [<<InlineClassSubA>>,<<ObjClassSubA>>]
+ /// CHECK: If [<<TestSubA>>]
+
+ /// CHECK: <<ObjClassSubC:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_
+ /// CHECK: <<InlineClassSubC:l\d+>> LoadClass class_name:SubC
+ /// CHECK: <<TestSubC:z\d+>> NotEqual [<<InlineClassSubC>>,<<ObjClassSubC>>]
+ /// CHECK: Deoptimize [<<TestSubC>>]
+
+ /// CHECK: <<Ret:i\d+>> Phi [<<SubARet>>,<<SubCRet>>]
+ /// CHECK: Return [<<Ret>>]
+ public static int inlinePolymophicCrossDexSubASubC(Super a) {
+ return a.getValue();
+ }
+
+ /// CHECK-START: int Main.inlineMegamorphic(Super) inliner (before)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.inlineMegamorphic(Super) inliner (after)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+ public static int inlineMegamorphic(Super a) {
+ return a.getValue();
+ }
+
+ /// CHECK-START: int Main.inlineMissingTypes(Super) inliner (before)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.inlineMissingTypes(Super) inliner (after)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+ public static int inlineMissingTypes(Super a) {
+ return a.getValue();
+ }
+
+ /// CHECK-START: int Main.noInlineCache(Super) inliner (before)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.noInlineCache(Super) inliner (after)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+ public static int noInlineCache(Super a) {
+ return a.getValue();
+ }
+
+ public static void testInlineMonomorphic() {
+ if (inlineMonomorphicSubA(new SubA()) != 42) {
+ throw new Error("Expected 42");
+ }
+
+ // Call with a different type than the one from the inline cache.
+ if (inlineMonomorphicSubA(new SubB()) != 38) {
+ throw new Error("Expected 38");
+ }
+ }
+
+ public static void testInlinePolymorhic() {
+ if (inlinePolymophicSubASubB(new SubA()) != 42) {
+ throw new Error("Expected 42");
+ }
+
+ if (inlinePolymophicSubASubB(new SubB()) != 38) {
+ throw new Error("Expected 38");
+ }
+
+ // Call with a different type than the one from the inline cache.
+ if (inlinePolymophicSubASubB(new SubC()) != 24) {
+ throw new Error("Expected 25");
+ }
+
+ if (inlinePolymophicCrossDexSubASubC(new SubA()) != 42) {
+ throw new Error("Expected 42");
+ }
+
+ if (inlinePolymophicCrossDexSubASubC(new SubC()) != 24) {
+ throw new Error("Expected 24");
+ }
+
+ // Call with a different type than the one from the inline cache.
+ if (inlinePolymophicCrossDexSubASubC(new SubB()) != 38) {
+ throw new Error("Expected 38");
+ }
+ }
+
+ public static void testInlineMegamorphic() {
+ if (inlineMegamorphic(new SubA()) != 42) {
+ throw new Error("Expected 42");
+ }
+ }
+
+
+ public static void testNoInlineCache() {
+ if (noInlineCache(new SubA()) != 42) {
+ throw new Error("Expected 42");
+ }
+ }
+
+ public static void main(String[] args) {
+ testInlineMonomorphic();
+ testInlinePolymorhic();
+ testInlineMegamorphic();
+ testNoInlineCache();
+ }
+
+}
diff --git a/test/638-checker-inline-caches/src/Super.java b/test/638-checker-inline-caches/src/Super.java
new file mode 100644
index 0000000..30cdf30
--- /dev/null
+++ b/test/638-checker-inline-caches/src/Super.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public abstract class Super {
+ abstract int getValue();
+}
diff --git a/test/641-irreducible-inline/expected.txt b/test/641-irreducible-inline/expected.txt
new file mode 100644
index 0000000..d81cc07
--- /dev/null
+++ b/test/641-irreducible-inline/expected.txt
@@ -0,0 +1 @@
+42
diff --git a/test/641-irreducible-inline/info.txt b/test/641-irreducible-inline/info.txt
new file mode 100644
index 0000000..ec6d0d2
--- /dev/null
+++ b/test/641-irreducible-inline/info.txt
@@ -0,0 +1,2 @@
+Regression test for optimizing in the presence of
+inlining a method that throws in an irreducible loop
diff --git a/test/641-irreducible-inline/smali/IrreducibleLoop.smali b/test/641-irreducible-inline/smali/IrreducibleLoop.smali
new file mode 100644
index 0000000..3e6c1f1
--- /dev/null
+++ b/test/641-irreducible-inline/smali/IrreducibleLoop.smali
@@ -0,0 +1,54 @@
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+.class public LIrreducibleLoop;
+
+.super Ljava/lang/Object;
+
+.method public static simpleLoop(I)I
+ .registers 3
+ const/16 v0, 42
+ if-eqz p0, :loop_entry
+ goto :other_loop_pre_entry
+
+ # The then part: beginning of the irreducible loop.
+ :loop_entry
+ if-nez p0, :exit
+ invoke-static {v0},LIrreducibleLoop;->foo(I)V
+ :other_loop_entry
+ goto :loop_entry
+
+ # The else part.
+ :other_loop_pre_entry
+ if-eqz p0, :other_loop_entry
+ invoke-static {v0},LIrreducibleLoop;->foo(I)V
+ goto :other_loop_entry
+
+ :exit
+ return v0
+.end method
+
+.method public static foo(I)V
+ .registers 3
+ const/16 v0, 0
+ sget-boolean v1,LIrreducibleLoop;->doThrow:Z
+ if-eqz v1, :exit
+ # Inlining a method that throws requires re-computing loop information
+ # which is unsupported when the caller has an irreducible loop.
+ throw v0
+ :exit
+ return-void
+.end method
+
+.field public static doThrow:Z
diff --git a/test/641-irreducible-inline/src/Main.java b/test/641-irreducible-inline/src/Main.java
new file mode 100644
index 0000000..53244f7
--- /dev/null
+++ b/test/641-irreducible-inline/src/Main.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.reflect.Method;
+
+public class Main {
+ // Workaround for b/18051191.
+ class InnerClass {}
+
+ public static void main(String[] args) throws Exception {
+ Class<?> c = Class.forName("IrreducibleLoop");
+ Method m = c.getMethod("simpleLoop", int.class);
+ Object[] arguments = { 42 };
+ System.out.println(m.invoke(null, arguments));
+ }
+}
diff --git a/test/924-threads/src/Main.java b/test/924-threads/src/Main.java
index f18d70e..716f59e 100644
--- a/test/924-threads/src/Main.java
+++ b/test/924-threads/src/Main.java
@@ -135,8 +135,12 @@
synchronized(cdl3_2) {
cdl3_1.countDown();
cdl3_2.await();
- Thread.yield();
- Thread.sleep(100);
+ // While the latch improves the chances to make good progress, scheduling might still be
+ // messy. Wait till we get the right Java-side Thread state.
+ do {
+ Thread.yield();
+ } while (t.getState() != Thread.State.BLOCKED);
+ Thread.sleep(10);
printThreadState(t);
}
diff --git a/test/951-threaded-obsolete/expected.txt b/test/951-threaded-obsolete/expected.txt
new file mode 100644
index 0000000..83efda1
--- /dev/null
+++ b/test/951-threaded-obsolete/expected.txt
@@ -0,0 +1,9 @@
+hello
+Not doing anything here
+goodbye
+hello
+transforming calling function
+goodbye
+Hello - Transformed
+Not doing anything here
+Goodbye - Transformed
diff --git a/test/951-threaded-obsolete/info.txt b/test/951-threaded-obsolete/info.txt
new file mode 100644
index 0000000..e7ef4a2
--- /dev/null
+++ b/test/951-threaded-obsolete/info.txt
@@ -0,0 +1,4 @@
+Tests basic obsolete method support
+
+This test ensures that obsolete methods will work even if the obsolete method is
+on a different thread then where the redefinition was triggered.
diff --git a/test/951-threaded-obsolete/run b/test/951-threaded-obsolete/run
new file mode 100755
index 0000000..c6e62ae
--- /dev/null
+++ b/test/951-threaded-obsolete/run
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Copyright 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+./default-run "$@" --jvmti
diff --git a/test/951-threaded-obsolete/src/Main.java b/test/951-threaded-obsolete/src/Main.java
new file mode 100644
index 0000000..98e7236
--- /dev/null
+++ b/test/951-threaded-obsolete/src/Main.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Base64;
+import java.util.concurrent.Semaphore;
+
+public class Main {
+ // class Transform {
+ // public void sayHi(Runnable r) {
+ // System.out.println("Hello - Transformed");
+ // r.run();
+ // System.out.println("Goodbye - Transformed");
+ // }
+ // }
+ private static final byte[] CLASS_BYTES = Base64.getDecoder().decode(
+ "yv66vgAAADQAJAoACAARCQASABMIABQKABUAFgsAFwAYCAAZBwAaBwAbAQAGPGluaXQ+AQADKClW" +
+ "AQAEQ29kZQEAD0xpbmVOdW1iZXJUYWJsZQEABXNheUhpAQAXKExqYXZhL2xhbmcvUnVubmFibGU7" +
+ "KVYBAApTb3VyY2VGaWxlAQAOVHJhbnNmb3JtLmphdmEMAAkACgcAHAwAHQAeAQATSGVsbG8gLSBU" +
+ "cmFuc2Zvcm1lZAcAHwwAIAAhBwAiDAAjAAoBABVHb29kYnllIC0gVHJhbnNmb3JtZWQBAAlUcmFu" +
+ "c2Zvcm0BABBqYXZhL2xhbmcvT2JqZWN0AQAQamF2YS9sYW5nL1N5c3RlbQEAA291dAEAFUxqYXZh" +
+ "L2lvL1ByaW50U3RyZWFtOwEAE2phdmEvaW8vUHJpbnRTdHJlYW0BAAdwcmludGxuAQAVKExqYXZh" +
+ "L2xhbmcvU3RyaW5nOylWAQASamF2YS9sYW5nL1J1bm5hYmxlAQADcnVuACAABwAIAAAAAAACAAAA" +
+ "CQAKAAEACwAAAB0AAQABAAAABSq3AAGxAAAAAQAMAAAABgABAAAAAQABAA0ADgABAAsAAAA7AAIA" +
+ "AgAAABeyAAISA7YABCu5AAUBALIAAhIGtgAEsQAAAAEADAAAABIABAAAAAMACAAEAA4ABQAWAAYA" +
+ "AQAPAAAAAgAQ");
+ private static final byte[] DEX_BYTES = Base64.getDecoder().decode(
+ "ZGV4CjAzNQAYeAMMXgYWxoeSHAS9EWKCCtVRSAGpqZVQAwAAcAAAAHhWNBIAAAAAAAAAALACAAAR" +
+ "AAAAcAAAAAcAAAC0AAAAAwAAANAAAAABAAAA9AAAAAUAAAD8AAAAAQAAACQBAAAMAgAARAEAAKIB" +
+ "AACqAQAAwQEAANYBAADjAQAA+gEAAA4CAAAkAgAAOAIAAEwCAABcAgAAXwIAAGMCAAB3AgAAfAIA" +
+ "AIUCAACKAgAAAwAAAAQAAAAFAAAABgAAAAcAAAAIAAAACgAAAAoAAAAGAAAAAAAAAAsAAAAGAAAA" +
+ "lAEAAAsAAAAGAAAAnAEAAAUAAQANAAAAAAAAAAAAAAAAAAEAEAAAAAEAAgAOAAAAAgAAAAAAAAAD" +
+ "AAAADwAAAAAAAAAAAAAAAgAAAAAAAAAJAAAAAAAAAJ8CAAAAAAAAAQABAAEAAACRAgAABAAAAHAQ" +
+ "AwAAAA4ABAACAAIAAACWAgAAFAAAAGIAAAAbAQIAAABuIAIAEAByEAQAAwBiAAAAGwEBAAAAbiAC" +
+ "ABAADgABAAAAAwAAAAEAAAAEAAY8aW5pdD4AFUdvb2RieWUgLSBUcmFuc2Zvcm1lZAATSGVsbG8g" +
+ "LSBUcmFuc2Zvcm1lZAALTFRyYW5zZm9ybTsAFUxqYXZhL2lvL1ByaW50U3RyZWFtOwASTGphdmEv" +
+ "bGFuZy9PYmplY3Q7ABRMamF2YS9sYW5nL1J1bm5hYmxlOwASTGphdmEvbGFuZy9TdHJpbmc7ABJM" +
+ "amF2YS9sYW5nL1N5c3RlbTsADlRyYW5zZm9ybS5qYXZhAAFWAAJWTAASZW1pdHRlcjogamFjay00" +
+ "LjEzAANvdXQAB3ByaW50bG4AA3J1bgAFc2F5SGkAAQAHDgADAQAHDoc8hwAAAAEBAICABMQCAQHc" +
+ "AgAAAA0AAAAAAAAAAQAAAAAAAAABAAAAEQAAAHAAAAACAAAABwAAALQAAAADAAAAAwAAANAAAAAE" +
+ "AAAAAQAAAPQAAAAFAAAABQAAAPwAAAAGAAAAAQAAACQBAAABIAAAAgAAAEQBAAABEAAAAgAAAJQB" +
+ "AAACIAAAEQAAAKIBAAADIAAAAgAAAJECAAAAIAAAAQAAAJ8CAAAAEAAAAQAAALACAAA=");
+
+ public static void main(String[] args) {
+ // Semaphores to let each thread know where the other is. We could use barriers but semaphores
+ // mean we don't need to have the worker thread be waiting around.
+ final Semaphore sem_redefine_start = new Semaphore(0);
+ final Semaphore sem_redefine_end = new Semaphore(0);
+ // Create a thread to do the actual redefinition. We will just communicate through an
+ // atomic-integer.
+ new Thread(() -> {
+ try {
+ // Wait for the other thread to ask for redefinition.
+ sem_redefine_start.acquire();
+ // Do the redefinition.
+ doCommonClassRedefinition(Transform.class, CLASS_BYTES, DEX_BYTES);
+ // Allow the other thread to wake up if it is waiting.
+ sem_redefine_end.release();
+ } catch (InterruptedException e) {
+ throw new Error("unable to do redefinition", e);
+ }
+ }).start();
+
+ Transform t = new Transform();
+ t.sayHi(() -> { System.out.println("Not doing anything here"); });
+ t.sayHi(() -> {
+ try {
+ System.out.println("transforming calling function");
+ // Wake up the waiting thread.
+ sem_redefine_start.release();
+ // Wait for the other thread to finish with redefinition.
+ sem_redefine_end.acquire();
+ } catch (InterruptedException e) {
+ throw new Error("unable to do redefinition", e);
+ }
+ });
+ t.sayHi(() -> { System.out.println("Not doing anything here"); });
+ }
+
+ // Transforms the class
+ private static native void doCommonClassRedefinition(Class<?> target,
+ byte[] classfile,
+ byte[] dexfile);
+}
diff --git a/test/951-threaded-obsolete/src/Transform.java b/test/951-threaded-obsolete/src/Transform.java
new file mode 100644
index 0000000..8cda6cd
--- /dev/null
+++ b/test/951-threaded-obsolete/src/Transform.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+class Transform {
+ public void sayHi(Runnable r) {
+ // Use lower 'h' to make sure the string will have a different string id
+ // than the transformation (the transformation code is the same except
+ // the actual printed String, which was making the test inacurately passing
+ // in JIT mode when loading the string from the dex cache, as the string ids
+ // of the two different strings were the same).
+ // We know the string ids will be different because lexicographically:
+ // "Hello" < "LTransform;" < "hello".
+ System.out.println("hello");
+ r.run();
+ System.out.println("goodbye");
+ }
+}
diff --git a/test/etc/run-test-jar b/test/etc/run-test-jar
index 1473149..e4e571c 100755
--- a/test/etc/run-test-jar
+++ b/test/etc/run-test-jar
@@ -63,6 +63,7 @@
TEST_IS_NDEBUG="n"
APP_IMAGE="y"
VDEX_FILTER=""
+PROFILE="n"
# if "y", run 'sync' before dalvikvm to make sure all files from
# build step (e.g. dex2oat) were finished writing.
@@ -269,6 +270,9 @@
elif [ "x$1" = "x--sync" ]; then
SYNC_BEFORE_RUN="y"
shift
+ elif [ "x$1" = "x--profile" ]; then
+ PROFILE="y"
+ shift
elif expr "x$1" : "x--" >/dev/null 2>&1; then
echo "unknown $0 option: $1" 1>&2
exit 1
@@ -441,8 +445,8 @@
JNI_OPTS="-Xjnigreflimit:512 -Xcheck:jni"
if [ "$RELOCATE" = "y" ]; then
- COMPILE_FLAGS="${COMPILE_FLAGS} --include-patch-information --runtime-arg -Xnorelocate"
- FLAGS="${FLAGS} -Xrelocate -Xcompiler-option --include-patch-information"
+ COMPILE_FLAGS="${COMPILE_FLAGS} --runtime-arg -Xnorelocate"
+ FLAGS="${FLAGS} -Xrelocate"
if [ "$HOST" = "y" ]; then
# Run test sets a fairly draconian ulimit that we will likely blow right over
# since we are relocating. Get the total size of the /system/framework directory
@@ -511,12 +515,23 @@
exit 1
fi
+profman_cmdline="true"
dex2oat_cmdline="true"
vdex_cmdline="true"
mkdir_locations="${DEX_LOCATION}/dalvik-cache/$ISA"
strip_cmdline="true"
sync_cmdline="true"
+if [ "$PROFILE" = "y" ]; then
+ profman_cmdline="${ANDROID_ROOT}/bin/profman \
+ --apk=$DEX_LOCATION/$TEST_NAME.jar \
+ --dex-location=$DEX_LOCATION/$TEST_NAME.jar \
+ --create-profile-from=$DEX_LOCATION/profile \
+ --reference-profile-file=$DEX_LOCATION/$TEST_NAME.prof"
+ COMPILE_FLAGS="${COMPILE_FLAGS} --profile-file=$DEX_LOCATION/$TEST_NAME.prof"
+ FLAGS="${FLAGS} -Xcompiler-option --profile-file=$DEX_LOCATION/$TEST_NAME.prof"
+fi
+
if [ "$PREBUILD" = "y" ]; then
mkdir_locations="${mkdir_locations} ${DEX_LOCATION}/oat/$ISA"
@@ -594,6 +609,7 @@
dex2oat_cmdline=$(echo $dex2oat_cmdline)
dalvikvm_cmdline=$(echo $dalvikvm_cmdline)
vdex_cmdline=$(echo $vdex_cmdline)
+profman_cmdline=$(echo $profman_cmdline)
if [ "$HOST" = "n" ]; then
adb root > /dev/null
@@ -603,11 +619,18 @@
adb shell mkdir -p $DEX_LOCATION
adb push $TEST_NAME.jar $DEX_LOCATION
adb push $TEST_NAME-ex.jar $DEX_LOCATION
+ if [ "$PROFILE" = "y" ]; then
+ adb push profile $DEX_LOCATION
+ fi
else
adb shell rm -r $DEX_LOCATION >/dev/null 2>&1
adb shell mkdir -p $DEX_LOCATION >/dev/null 2>&1
adb push $TEST_NAME.jar $DEX_LOCATION >/dev/null 2>&1
adb push $TEST_NAME-ex.jar $DEX_LOCATION >/dev/null 2>&1
+ if [ "$PROFILE" = "y" ]; then
+ adb push profile $DEX_LOCATION >/dev/null 2>&1
+ fi
+
fi
LD_LIBRARY_PATH=/data/$TEST_DIRECTORY/art/$ISA
@@ -634,6 +657,7 @@
mkdir -p ${mkdir_locations} && \
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH && \
export PATH=$ANDROID_ROOT/bin:$PATH && \
+ $profman_cmdline && \
$dex2oat_cmdline && \
$vdex_cmdline && \
$strip_cmdline && \
@@ -710,13 +734,14 @@
fi
if [ "$DEV_MODE" = "y" ]; then
- echo "mkdir -p ${mkdir_locations} && $dex2oat_cmdline && $vdex_cmdline && $strip_cmdline && $sync_cmdline && $cmdline"
+ echo "mkdir -p ${mkdir_locations} && $profman_cmdline && $dex2oat_cmdline && $vdex_cmdline && $strip_cmdline && $sync_cmdline && $cmdline"
fi
cd $ANDROID_BUILD_TOP
rm -rf ${DEX_LOCATION}/dalvik-cache/
mkdir -p ${mkdir_locations} || exit 1
+ $profman_cmdline || { echo "Profman failed." >&2 ; exit 2; }
$dex2oat_cmdline || { echo "Dex2oat failed." >&2 ; exit 2; }
$vdex_cmdline || { echo "Dex2oat failed." >&2 ; exit 2; }
$strip_cmdline || { echo "Strip failed." >&2 ; exit 3; }
